niceeval 0.6.1 → 0.6.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agents/types.d.ts +67 -5
- package/dist/context/types.d.ts +32 -12
- package/dist/i18n/en.d.ts +54 -0
- package/dist/i18n/zh-CN.d.ts +55 -1
- package/dist/o11y/types.d.ts +16 -2
- package/dist/report/aggregate.d.ts +5 -3
- package/dist/report/aggregate.js +32 -5
- package/dist/report/built-ins/experiment-comparison.d.ts +39 -1
- package/dist/report/built-ins/experiment-comparison.js +116 -10
- package/dist/report/built-ins/index.d.ts +1 -0
- package/dist/report/built-ins/index.js +1 -1
- package/dist/report/components.d.ts +8 -2
- package/dist/report/components.js +3 -3
- package/dist/report/compute.d.ts +11 -18
- package/dist/report/compute.js +54 -34
- package/dist/report/flag.d.ts +16 -1
- package/dist/report/flag.js +19 -1
- package/dist/report/format.d.ts +16 -8
- package/dist/report/format.js +27 -12
- package/dist/report/index.d.ts +4 -3
- package/dist/report/index.js +5 -4
- package/dist/report/locale.d.ts +11 -2
- package/dist/report/locale.js +23 -5
- package/dist/report/metrics.d.ts +13 -1
- package/dist/report/metrics.js +65 -14
- package/dist/report/primitives.d.ts +6 -0
- package/dist/report/react/AttemptList.d.ts +2 -2
- package/dist/report/react/AttemptList.js +5 -6
- package/dist/report/react/EvalList.d.ts +1 -1
- package/dist/report/react/EvalList.js +0 -0
- package/dist/report/react/ExperimentComparison.d.ts +8 -0
- package/dist/report/react/ExperimentComparison.js +11 -0
- package/dist/report/react/ExperimentList.d.ts +2 -1
- package/dist/report/react/ExperimentList.js +8 -10
- package/dist/report/react/MetricScatter.js +5 -11
- package/dist/report/react/chart-math.d.ts +23 -6
- package/dist/report/react/chart-math.js +71 -19
- package/dist/report/react/fixtures.d.ts +3 -3
- package/dist/report/react/fixtures.js +21 -14
- package/dist/report/report.d.ts +5 -1
- package/dist/report/report.js +6 -2
- package/dist/report/text/faces.d.ts +1 -1
- package/dist/report/text/faces.js +42 -41
- package/dist/report/text/table.js +36 -5
- package/dist/report/types.d.ts +39 -21
- package/dist/results/types.d.ts +11 -0
- package/dist/runner/feedback/sink.d.ts +110 -0
- package/dist/runner/types.d.ts +513 -22
- package/dist/sandbox/docker.d.ts +23 -2
- package/dist/sandbox/e2b.d.ts +15 -1
- package/dist/sandbox/errors.d.ts +30 -3
- package/dist/sandbox/io-retry.d.ts +17 -0
- package/dist/sandbox/registry.d.ts +2 -0
- package/dist/sandbox/resolve.d.ts +18 -5
- package/dist/sandbox/retry.d.ts +11 -1
- package/dist/sandbox/types.d.ts +39 -5
- package/dist/sandbox/vercel.d.ts +7 -1
- package/dist/scoring/coverage.d.ts +30 -0
- package/dist/scoring/display.d.ts +21 -0
- package/dist/scoring/display.js +120 -0
- package/dist/scoring/types.d.ts +103 -20
- package/dist/shared/aggregate.d.ts +1 -0
- package/dist/shared/aggregate.js +3 -3
- package/dist/shared/types.d.ts +28 -0
- package/dist/tty-line.d.ts +0 -4
- package/dist/util.d.ts +23 -0
- package/docs-site/zh/concepts/adapter.mdx +22 -4
- package/docs-site/zh/concepts/experiment.mdx +1 -1
- package/docs-site/zh/concepts/overview.mdx +6 -6
- package/docs-site/zh/guides/agent-feedback-loop.mdx +28 -26
- package/docs-site/zh/guides/authoring.mdx +33 -0
- package/docs-site/zh/guides/ci-integration.mdx +23 -12
- package/docs-site/zh/guides/connect-your-agent.mdx +29 -3
- package/docs-site/zh/guides/custom-reports.mdx +29 -34
- package/docs-site/zh/guides/dataset-fanout.mdx +25 -3
- package/docs-site/zh/guides/debug-sandbox.mdx +57 -0
- package/docs-site/zh/guides/debugging.mdx +210 -0
- package/docs-site/zh/guides/experiments.mdx +10 -3
- package/docs-site/zh/guides/official-adapters.mdx +26 -2
- package/docs-site/zh/guides/publish-report.mdx +30 -16
- package/docs-site/zh/guides/report-components.mdx +42 -30
- package/docs-site/zh/guides/reporters.mdx +2 -2
- package/docs-site/zh/guides/results-data.mdx +17 -9
- package/docs-site/zh/guides/runner.mdx +17 -7
- package/docs-site/zh/guides/sandbox-agent.mdx +56 -7
- package/docs-site/zh/guides/sandbox-providers.mdx +257 -9
- package/docs-site/zh/guides/scoring-guide.mdx +4 -4
- package/docs-site/zh/guides/viewing-results.mdx +79 -36
- package/docs-site/zh/guides/write-experiment.mdx +5 -3
- package/docs-site/zh/guides/write-send.mdx +17 -1
- package/docs-site/zh/index.mdx +1 -1
- package/docs-site/zh/reference/builtin-agents.mdx +27 -0
- package/docs-site/zh/reference/capabilities.mdx +2 -2
- package/docs-site/zh/reference/cli.mdx +33 -7
- package/docs-site/zh/reference/define-agent.mdx +57 -4
- package/docs-site/zh/reference/define-config.mdx +1 -1
- package/docs-site/zh/reference/define-eval.mdx +42 -9
- package/docs-site/zh/reference/expect.mdx +26 -1
- package/package.json +5 -1
- package/src/agents/ai-sdk-otel.test.ts +1 -0
- package/src/agents/ai-sdk.test.ts +3 -0
- package/src/agents/ai-sdk.ts +3 -0
- package/src/agents/bub-install-spec.test.ts +34 -0
- package/src/agents/bub-install-spec.ts +32 -0
- package/src/agents/bub.ts +31 -32
- package/src/agents/claude-code.test.ts +130 -9
- package/src/agents/claude-code.ts +76 -4
- package/src/agents/codex.test.ts +189 -40
- package/src/agents/codex.ts +155 -14
- package/src/agents/coding-cli-versions.test.ts +15 -0
- package/src/agents/coding-cli-versions.ts +3 -0
- package/src/agents/index.ts +11 -0
- package/src/agents/langgraph.test.ts +204 -0
- package/src/agents/langgraph.ts +495 -0
- package/src/agents/marketplace.ts +85 -0
- package/src/agents/native-config.test.ts +179 -0
- package/src/agents/native-config.ts +267 -0
- package/src/agents/openai-compat.test.ts +1 -0
- package/src/agents/openclaw.test.ts +31 -0
- package/src/agents/openclaw.ts +171 -0
- package/src/agents/plugin-config.test.ts +1 -0
- package/src/agents/sdk-streams.test.ts +79 -0
- package/src/agents/sdk-streams.ts +55 -10
- package/src/agents/skills.test.ts +1 -0
- package/src/agents/streaming.test.ts +3 -9
- package/src/agents/types.ts +68 -5
- package/src/agents/ui-message-stream.test.ts +3 -0
- package/src/cli.ts +411 -108
- package/src/context/context.test.ts +51 -12
- package/src/context/context.ts +161 -29
- package/src/context/session.test.ts +1 -0
- package/src/context/session.ts +114 -6
- package/src/context/types.ts +30 -12
- package/src/define.test.ts +13 -8
- package/src/define.ts +25 -4
- package/src/expect/index.ts +53 -23
- package/src/i18n/en.ts +64 -2
- package/src/i18n/zh-CN.ts +65 -3
- package/src/o11y/cost.test.ts +1 -0
- package/src/o11y/execution-tree.test.ts +1 -20
- package/src/o11y/otlp/mappers/claude-code.test.ts +1 -0
- package/src/o11y/otlp/parse.test.ts +1 -0
- package/src/o11y/otlp/turn-otel.test.ts +1 -0
- package/src/o11y/parsers/bub.test.ts +1 -0
- package/src/o11y/parsers/claude-code.test.ts +1 -34
- package/src/o11y/parsers/openclaw.test.ts +154 -0
- package/src/o11y/parsers/openclaw.ts +310 -0
- package/src/o11y/prices.json +746 -311
- package/src/o11y/tool-names.test.ts +1 -0
- package/src/o11y/types.ts +16 -2
- package/src/report/aggregate.ts +34 -5
- package/src/report/built-in-user-parity.test.tsx +110 -153
- package/src/report/built-ins/experiment-comparison.tsx +173 -13
- package/src/report/built-ins/index.ts +6 -1
- package/src/report/components.tsx +9 -3
- package/src/report/compute.ts +70 -40
- package/src/report/dual-render.test.tsx +194 -67
- package/src/report/flag.ts +30 -2
- package/src/report/format.ts +35 -11
- package/src/report/index.ts +22 -4
- package/src/report/locale.ts +25 -5
- package/src/report/metrics.ts +67 -14
- package/src/report/primitives.tsx +6 -0
- package/src/report/react/AttemptList.tsx +6 -31
- package/src/report/react/EvalList.tsx +0 -0
- package/src/report/react/ExperimentComparison.tsx +68 -0
- package/src/report/react/ExperimentList.tsx +15 -9
- package/src/report/react/MetricScatter.tsx +12 -14
- package/src/report/react/chart-math.test.ts +85 -0
- package/src/report/react/chart-math.ts +101 -22
- package/src/report/react/enhance.js +33 -1
- package/src/report/react/fixtures.ts +24 -17
- package/src/report/react/render.test.tsx +9 -64
- package/src/report/react/styles.css +73 -2
- package/src/report/report.test.ts +306 -98
- package/src/report/report.ts +6 -2
- package/src/report/text/faces.ts +47 -43
- package/src/report/text/table.ts +42 -5
- package/src/report/types.ts +41 -21
- package/src/results/annotated-source.test.ts +62 -9
- package/src/results/annotated-source.ts +64 -6
- package/src/results/attempt-evidence.test.ts +9 -7
- package/src/results/attempt-evidence.ts +15 -8
- package/src/results/attempt-source.ts +6 -3
- package/src/results/copy.ts +145 -55
- package/src/results/host-equivalence.test.ts +8 -6
- package/src/results/index.ts +2 -0
- package/src/results/locator.test.ts +1 -22
- package/src/results/open.ts +7 -1
- package/src/results/publish.ts +149 -0
- package/src/results/results.test.ts +85 -51
- package/src/results/truncate.ts +90 -0
- package/src/results/types.ts +7 -0
- package/src/results/writer.ts +31 -13
- package/src/runner/attempt.test.ts +138 -7
- package/src/runner/attempt.ts +603 -104
- package/src/runner/discover.test.ts +47 -0
- package/src/runner/discover.ts +36 -2
- package/src/runner/eval-source.test.ts +1 -27
- package/src/runner/feedback/agent.test.ts +504 -0
- package/src/runner/feedback/agent.ts +409 -0
- package/src/runner/feedback/ci.test.ts +562 -0
- package/src/runner/feedback/ci.ts +401 -0
- package/src/runner/feedback/coordinator.test.ts +317 -0
- package/src/runner/feedback/coordinator.ts +397 -0
- package/src/runner/feedback/failure.ts +40 -0
- package/src/runner/feedback/human.test.ts +616 -0
- package/src/runner/feedback/human.ts +535 -0
- package/src/runner/feedback/index.ts +66 -0
- package/src/runner/feedback/io.ts +78 -0
- package/src/runner/feedback/profile.test.ts +50 -0
- package/src/runner/feedback/profile.ts +58 -0
- package/src/runner/feedback/reducer.test.ts +395 -0
- package/src/runner/feedback/reducer.ts +260 -0
- package/src/runner/feedback/renderer.ts +82 -0
- package/src/runner/feedback/sink.ts +203 -0
- package/src/runner/feedback/testing.ts +106 -0
- package/src/runner/ledger.test.ts +230 -0
- package/src/runner/ledger.ts +329 -0
- package/src/runner/report.test.ts +128 -3
- package/src/runner/report.ts +33 -9
- package/src/runner/reporters/artifacts.ts +8 -2
- package/src/runner/reporters/braintrust.test.ts +8 -7
- package/src/runner/reporters/braintrust.ts +9 -2
- package/src/runner/reporters/index.ts +2 -2
- package/src/runner/reporters/json.test.ts +162 -0
- package/src/runner/reporters/json.ts +35 -8
- package/src/runner/reporters/shared.ts +1 -5
- package/src/runner/run.test.ts +760 -3
- package/src/runner/run.ts +242 -36
- package/src/runner/sandbox-prep.ts +3 -42
- package/src/runner/timing.ts +158 -0
- package/src/runner/types.ts +518 -22
- package/src/sandbox/checkpoint.test.ts +55 -0
- package/src/sandbox/checkpoint.ts +29 -8
- package/src/sandbox/cli-commands.ts +407 -0
- package/src/sandbox/docker.ts +115 -16
- package/src/sandbox/e2b-agent-template.test.ts +56 -0
- package/src/sandbox/e2b-agent-template.ts +94 -0
- package/src/sandbox/e2b.ts +74 -9
- package/src/sandbox/errors.ts +111 -4
- package/src/sandbox/index.ts +2 -0
- package/src/sandbox/io-retry.test.ts +58 -0
- package/src/sandbox/io-retry.ts +45 -0
- package/src/sandbox/keep-registry.test.ts +86 -0
- package/src/sandbox/keep-registry.ts +142 -0
- package/src/sandbox/keep.ts +178 -0
- package/src/sandbox/paths.test.ts +1 -0
- package/src/sandbox/paths.ts +19 -8
- package/src/sandbox/registry.ts +20 -3
- package/src/sandbox/resolve.ts +76 -11
- package/src/sandbox/retry.test.ts +70 -0
- package/src/sandbox/retry.ts +46 -4
- package/src/sandbox/types.ts +44 -6
- package/src/sandbox/vercel.ts +43 -20
- package/src/scoring/collector.ts +60 -17
- package/src/scoring/coverage.ts +95 -0
- package/src/scoring/diff.ts +81 -0
- package/src/scoring/display.test.ts +121 -0
- package/src/scoring/display.ts +133 -0
- package/src/scoring/evidence.test.ts +189 -0
- package/src/scoring/judge.test.ts +142 -0
- package/src/scoring/judge.ts +15 -18
- package/src/scoring/scoped.ts +217 -50
- package/src/scoring/types.ts +117 -20
- package/src/scoring/verdict.ts +16 -4
- package/src/shared/aggregate.ts +3 -2
- package/src/shared/types.ts +31 -0
- package/src/show/compose.ts +2 -2
- package/src/show/index.ts +21 -1
- package/src/show/render.ts +619 -104
- package/src/show/show.test.ts +235 -19
- package/src/tty-line.ts +8 -26
- package/src/util.test.ts +1 -0
- package/src/util.ts +41 -0
- package/src/view/app/components/AttemptModal.tsx +153 -2
- package/src/view/app/components/CodeView.tsx +32 -11
- package/src/view/app/components/CopyControls.tsx +2 -2
- package/src/view/app/i18n.ts +6 -0
- package/src/view/app/lib/attempt-route.test.ts +1 -0
- package/src/view/app/lib/verdict.ts +7 -9
- package/src/view/artifact-serving.test.ts +2 -1
- package/src/view/client-dist/app.css +1 -1
- package/src/view/client-dist/app.js +17 -17
- package/src/view/data.test.ts +1 -0
- package/src/view/data.ts +11 -1
- package/src/view/index.ts +11 -0
- package/src/view/server.ts +2 -0
- package/src/view/styles.css +3 -0
- package/src/view/view-report.test.ts +6 -5
- package/src/runner/reporters/console.ts +0 -70
- package/src/runner/reporters/live.test.ts +0 -56
- package/src/runner/reporters/live.ts +0 -247
- package/src/runner/reporters/quiet.test.ts +0 -66
- package/src/runner/reporters/quiet.ts +0 -49
- package/src/runner/reporters/table.ts +0 -277
package/src/show/show.test.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
// cases: docs/engineering/unit-tests/reports/cases.md
|
|
1
2
|
// niceeval show 终端宿主的测试(行为规范:docs-site/zh/guides/viewing-results.mdx;
|
|
2
3
|
// 组合语义:docs/feature/reports/architecture.md「Selection 是计算入口」)。覆盖:
|
|
3
4
|
// - 榜单合成口径:每 experiment × eval 取最新判定,局部重跑从更早快照补齐,头部标注合成自几个快照;
|
|
@@ -17,7 +18,7 @@ import { tmpdir } from "node:os";
|
|
|
17
18
|
import { join } from "node:path";
|
|
18
19
|
import { createHash } from "node:crypto";
|
|
19
20
|
import { openResults } from "../results/index.ts";
|
|
20
|
-
import { RESULTS_FORMAT, RESULTS_SCHEMA_VERSION, type EvalResult, type Verdict } from "../types.ts";
|
|
21
|
+
import { RESULTS_FORMAT, RESULTS_SCHEMA_VERSION, type EvalResult, type TimingNode, type TraceSpan, type Verdict } from "../types.ts";
|
|
21
22
|
import { selectCurrentResults } from "../results/select.ts";
|
|
22
23
|
import { evalHistory } from "./compose.ts";
|
|
23
24
|
import { runShow, type ShowFlags } from "./index.ts";
|
|
@@ -52,7 +53,7 @@ type AttemptFixture = Pick<EvalResult, "id" | "verdict"> &
|
|
|
52
53
|
Partial<
|
|
53
54
|
Pick<
|
|
54
55
|
EvalResult,
|
|
55
|
-
"attempt" | "durationMs" | "assertions" | "estimatedCostUSD" | "usage" | "error" | "startedAt" | "artifactBase" | "hasEvents" | "hasTrace"
|
|
56
|
+
"attempt" | "durationMs" | "assertions" | "estimatedCostUSD" | "usage" | "error" | "diagnostics" | "startedAt" | "artifactBase" | "hasEvents" | "hasTrace" | "phases"
|
|
56
57
|
>
|
|
57
58
|
>;
|
|
58
59
|
|
|
@@ -135,7 +136,7 @@ async function seedComposedRoot(): Promise<string> {
|
|
|
135
136
|
name: 'fileChanged("src/components/Button.tsx")',
|
|
136
137
|
severity: "gate",
|
|
137
138
|
score: 0,
|
|
138
|
-
|
|
139
|
+
outcome: "failed" as const,
|
|
139
140
|
detail: "file was not modified",
|
|
140
141
|
},
|
|
141
142
|
],
|
|
@@ -151,12 +152,12 @@ describe("默认报告:跨快照合成的现刻水位(ExperimentComparison text
|
|
|
151
152
|
const root = await seedComposedRoot();
|
|
152
153
|
const { out, code } = await show(root, []);
|
|
153
154
|
expect(code).toBe(0);
|
|
154
|
-
expect(out).toContain("No data to plot Cost ×
|
|
155
|
+
expect(out).toContain("No data to plot Cost × End-to-end pass rate");
|
|
155
156
|
expect(out).toContain("1 point missing data");
|
|
156
157
|
expect(out).not.toContain("needed to compare");
|
|
157
158
|
expect(out).not.toContain("COMPARISON");
|
|
158
|
-
expect(out).toMatch(
|
|
159
|
-
expect(out).
|
|
159
|
+
expect(out).toMatch(/\bbub\s+default\s+bub\s+1s\s+50%/);
|
|
160
|
+
expect(out).toMatch(/1 passed[\s\S]*?\/ 1\s+failed/);
|
|
160
161
|
expect(out).toMatch(/✗ failed\s+fixtures\/button[\s\S]*└─ @1[0-9a-z]{7}[\s\S]*fileChanged/);
|
|
161
162
|
expect(out).toMatch(/✓ passed\s+weather\/brooklyn[\s\S]*└─ @1[0-9a-z]{7}/);
|
|
162
163
|
expect(out).not.toMatch(/\[[EXD⏱,]+\]/);
|
|
@@ -168,9 +169,9 @@ describe("默认报告:跨快照合成的现刻水位(ExperimentComparison text
|
|
|
168
169
|
try {
|
|
169
170
|
const { out, code } = await show(root, []);
|
|
170
171
|
expect(code).toBe(0);
|
|
171
|
-
expect(out).toContain("预估成本 ×
|
|
172
|
+
expect(out).toContain("预估成本 × 端到端成功率 没有可绘制的数据");
|
|
172
173
|
expect(out).toContain("1 通过 / 1 失败");
|
|
173
|
-
expect(out).
|
|
174
|
+
expect(out).toMatch(/\bbub\s+默认\s+bub\s+1s\s+50%/);
|
|
174
175
|
} finally {
|
|
175
176
|
process.env.NICEEVAL_LANG = "en";
|
|
176
177
|
}
|
|
@@ -200,8 +201,8 @@ describe("默认报告:跨快照合成的现刻水位(ExperimentComparison text
|
|
|
200
201
|
expect(code).toBe(0);
|
|
201
202
|
expect(out).toContain("better → upper right");
|
|
202
203
|
expect(out).toContain("A compare/a B compare/b");
|
|
203
|
-
expect(out).toMatch(
|
|
204
|
-
expect(out).toMatch(
|
|
204
|
+
expect(out).toMatch(/\bb\s+large\s+claude\s+1s\s+100%/);
|
|
205
|
+
expect(out).toMatch(/\ba\s+mini\s+codex\s+1s\s+50%/);
|
|
205
206
|
expect(out).toMatch(/✓ passed\s+q1[\s\S]*└─ @1[0-9a-z]{7}/);
|
|
206
207
|
expect(out).toMatch(/✗ failed\s+q2[\s\S]*└─ @1[0-9a-z]{7}/);
|
|
207
208
|
expect(out).not.toContain("needed to compare");
|
|
@@ -280,15 +281,15 @@ describe("单 eval 详情", () => {
|
|
|
280
281
|
name: 'calledTool("get_weather")',
|
|
281
282
|
severity: "gate",
|
|
282
283
|
score: 0,
|
|
283
|
-
|
|
284
|
+
outcome: "failed" as const,
|
|
284
285
|
detail: "tool was never called",
|
|
285
286
|
},
|
|
286
|
-
{ name: "succeeded()", severity: "gate", score: 1,
|
|
287
|
+
{ name: "succeeded()", severity: "gate", score: 1, outcome: "passed" as const },
|
|
287
288
|
{
|
|
288
289
|
name: 'judge("回答基于实时数据")',
|
|
289
290
|
severity: "soft",
|
|
290
291
|
score: 0.2,
|
|
291
|
-
|
|
292
|
+
outcome: "failed" as const,
|
|
292
293
|
detail: "reply invents a temperature without any tool call",
|
|
293
294
|
},
|
|
294
295
|
],
|
|
@@ -341,7 +342,7 @@ describe("--history 时间轴", () => {
|
|
|
341
342
|
res("weather/brooklyn", "failed", {
|
|
342
343
|
attempt: 1,
|
|
343
344
|
estimatedCostUSD: 0.04,
|
|
344
|
-
assertions: [{ name: 'calledTool("get_weather")', severity: "gate", score: 0,
|
|
345
|
+
assertions: [{ name: 'calledTool("get_weather")', severity: "gate", score: 0, outcome: "failed" as const }],
|
|
345
346
|
}),
|
|
346
347
|
],
|
|
347
348
|
);
|
|
@@ -549,7 +550,7 @@ describe("--eval", () => {
|
|
|
549
550
|
root,
|
|
550
551
|
"2026-07-08T10-00-00-000Z",
|
|
551
552
|
{ experimentId: "compare/bub", startedAt: "2026-07-08T10:00:00.000Z" },
|
|
552
|
-
[res("weather/brooklyn", "failed", { assertions: [{ name: "succeeded()", severity: "gate", score: 0,
|
|
553
|
+
[res("weather/brooklyn", "failed", { assertions: [{ name: "succeeded()", severity: "gate", score: 0, outcome: "failed" as const }] })],
|
|
553
554
|
);
|
|
554
555
|
const { out, code } = await show(root, ["weather/brooklyn"], { eval: true });
|
|
555
556
|
expect(code).toBe(0);
|
|
@@ -569,11 +570,11 @@ describe("--eval", () => {
|
|
|
569
570
|
name: 'calledTool("get_weather")',
|
|
570
571
|
severity: "gate",
|
|
571
572
|
score: 0,
|
|
572
|
-
|
|
573
|
+
outcome: "failed" as const,
|
|
573
574
|
detail: "tool was never called",
|
|
574
575
|
loc: { file: "evals/weather/brooklyn.eval.ts", line: 2 },
|
|
575
576
|
},
|
|
576
|
-
{ name: "unlocated()", severity: "soft", score: 0.5,
|
|
577
|
+
{ name: "unlocated()", severity: "soft", score: 0.5, outcome: "failed" as const, detail: "no loc on this one" },
|
|
577
578
|
],
|
|
578
579
|
}),
|
|
579
580
|
],
|
|
@@ -599,7 +600,7 @@ describe("--eval", () => {
|
|
|
599
600
|
it("源码行标注保留 group 与值断言的 expected/received", async () => {
|
|
600
601
|
const root = await makeRoot();
|
|
601
602
|
const dir = await writeSnapshot(root, "2026-07-08T10-00-00-000Z", { experimentId: "compare/bub", startedAt: "2026-07-08T10:00:00.000Z" }, [
|
|
602
|
-
res("manager", "failed", { assertions: [{ name: "equals(4)", severity: "gate", score: 0,
|
|
603
|
+
res("manager", "failed", { assertions: [{ name: "equals(4)", severity: "gate", score: 0, outcome: "failed" as const, expected: "4", received: "1", groupPath: ["Issue 15193"], loc: { file: "evals/manager.eval.ts", line: 1 } }] }),
|
|
603
604
|
]);
|
|
604
605
|
const attemptDir = join(dir, "manager/a0");
|
|
605
606
|
const content = "t.check(actual, equals(expected));\n";
|
|
@@ -618,7 +619,7 @@ describe("show @<locator>", () => {
|
|
|
618
619
|
it("默认页直接解释失败断言的 group、期望值、实际值和源码位置", async () => {
|
|
619
620
|
const root = await makeRoot();
|
|
620
621
|
await writeSnapshot(root, "2026-07-08T10-00-00-000Z", { experimentId: "compare/bub", startedAt: "2026-07-08T10:00:00.000Z" }, [
|
|
621
|
-
res("manager", "failed", { assertions: [{ name: "equals(4)", severity: "gate", score: 0,
|
|
622
|
+
res("manager", "failed", { assertions: [{ name: "equals(4)", severity: "gate", score: 0, outcome: "failed" as const, expected: "4", received: "1", groupPath: ["Issue 15193"], loc: { file: "evals/manager.eval.ts", line: 40, column: 11 } }] }),
|
|
622
623
|
]);
|
|
623
624
|
const results = await openResults(root);
|
|
624
625
|
const locator = results.experiments[0]!.latest.evals[0]!.attempts[0]!.locator!;
|
|
@@ -649,6 +650,41 @@ describe("show @<locator>", () => {
|
|
|
649
650
|
expect(out).not.toContain("available:");
|
|
650
651
|
});
|
|
651
652
|
|
|
653
|
+
it("errored attempt 首页展开结构化 error(phase/code/message/cause)与 diagnostics", async () => {
|
|
654
|
+
const root = await makeRoot();
|
|
655
|
+
await writeSnapshot(root, "2026-07-08T10-00-00-000Z", { experimentId: "compare/claude-e2b", startedAt: "2026-07-08T10:00:00.000Z" }, [
|
|
656
|
+
res("agent-029", "errored", {
|
|
657
|
+
error: {
|
|
658
|
+
code: "sandbox-rate-limit",
|
|
659
|
+
message: "E2B sandbox allocation failed after 5 attempts",
|
|
660
|
+
phase: "sandbox.create",
|
|
661
|
+
cause: { name: "RateLimitError", message: "too many concurrent sandboxes" },
|
|
662
|
+
},
|
|
663
|
+
diagnostics: [
|
|
664
|
+
{ code: "teardown-failed", level: "warning", message: "container stop timed out", phase: "sandbox.teardown" },
|
|
665
|
+
],
|
|
666
|
+
}),
|
|
667
|
+
]);
|
|
668
|
+
const results = await openResults(root);
|
|
669
|
+
const locator = results.experiments[0]!.latest.evals[0]!.attempts[0]!.locator!;
|
|
670
|
+
|
|
671
|
+
const { out, code } = await show(root, [locator]);
|
|
672
|
+
expect(code).toBe(0);
|
|
673
|
+
expect(out).toContain(`${locator} · agent-029 · compare/claude-e2b · errored`);
|
|
674
|
+
// 结构化 error 块:phase 直接展示闭集点分名,code/message/cause 各一行(见 docs/feature/reports/show.md)
|
|
675
|
+
expect(out).toContain("error:");
|
|
676
|
+
expect(out).toContain("phase: sandbox.create");
|
|
677
|
+
expect(out).toContain("code: sandbox-rate-limit");
|
|
678
|
+
expect(out).toContain("message: E2B sandbox allocation failed after 5 attempts");
|
|
679
|
+
expect(out).toContain("cause: RateLimitError · too many concurrent sandboxes");
|
|
680
|
+
// attempt 级 diagnostics 块(与 verdict 独立)
|
|
681
|
+
expect(out).toContain("diagnostics:");
|
|
682
|
+
expect(out).toContain("warning · sandbox.teardown · teardown-failed");
|
|
683
|
+
expect(out).toContain("container stop timed out");
|
|
684
|
+
// errored 且无断言:不打印空的 assertions 汇总行
|
|
685
|
+
expect(out).not.toContain("assertions:");
|
|
686
|
+
});
|
|
687
|
+
|
|
652
688
|
it("available 只逐行列出实际存在的证据命令,不打印能力字母或合并式 next", async () => {
|
|
653
689
|
const root = await makeRoot();
|
|
654
690
|
const dir = await writeSnapshot(root, "2026-07-08T10-00-00-000Z", { experimentId: "compare/bub", startedAt: "2026-07-08T10:00:00.000Z" }, [
|
|
@@ -688,6 +724,186 @@ describe("show @<locator>", () => {
|
|
|
688
724
|
expect(out).not.toContain("available:");
|
|
689
725
|
});
|
|
690
726
|
|
|
727
|
+
it("单独 --timing 进入完整时间树,不回落到 attempt 首页", async () => {
|
|
728
|
+
const root = await makeRoot();
|
|
729
|
+
await writeSnapshot(root, "2026-07-08T10-00-00-000Z", { experimentId: "compare/bub", startedAt: "2026-07-08T10:00:00.000Z" }, [
|
|
730
|
+
res("weather/brooklyn", "passed", {
|
|
731
|
+
durationMs: 50_000,
|
|
732
|
+
phases: [
|
|
733
|
+
{ name: "eval.run", durationMs: 42_000 },
|
|
734
|
+
{ name: "sandbox.stop", durationMs: 800 },
|
|
735
|
+
],
|
|
736
|
+
}),
|
|
737
|
+
]);
|
|
738
|
+
const results = await openResults(root);
|
|
739
|
+
const locator = results.experiments[0]!.latest.evals[0]!.attempts[0]!.locator!;
|
|
740
|
+
|
|
741
|
+
const { out, code } = await show(root, [locator], { timing: true });
|
|
742
|
+
expect(code).toBe(0);
|
|
743
|
+
expect(out).toContain("total 50.0s");
|
|
744
|
+
expect(out).toContain("eval.run 42.0s");
|
|
745
|
+
expect(out).toContain("teardown (not counted in total):");
|
|
746
|
+
expect(out).toContain("sandbox.stop 800ms");
|
|
747
|
+
expect(out).not.toContain("assertions:");
|
|
748
|
+
expect(out).not.toContain("available:");
|
|
749
|
+
});
|
|
750
|
+
|
|
751
|
+
it("--timing 在 80 个 detail node 内与 full 等价,81 个开始原位省略", async () => {
|
|
752
|
+
const root = await makeRoot();
|
|
753
|
+
const children: TimingNode[] = Array.from({ length: 81 }, (_, index) => ({
|
|
754
|
+
id: `command-${String(index).padStart(3, "0")}`,
|
|
755
|
+
kind: "command",
|
|
756
|
+
label: "tool",
|
|
757
|
+
startOffsetMs: index * 10,
|
|
758
|
+
durationMs: 81 - index,
|
|
759
|
+
command: { display: `tool --item=${index}` },
|
|
760
|
+
}));
|
|
761
|
+
await writeSnapshot(root, "2026-07-08T10-00-00-000Z", { experimentId: "compare/bub", startedAt: "2026-07-08T10:00:00.000Z" }, [
|
|
762
|
+
res("weather/brooklyn", "passed", {
|
|
763
|
+
durationMs: 50_000,
|
|
764
|
+
phases: [
|
|
765
|
+
{ name: "sandbox.create", durationMs: 1_000 },
|
|
766
|
+
{ name: "eval.run", durationMs: 42_000, children },
|
|
767
|
+
{ name: "sandbox.stop", durationMs: 800 },
|
|
768
|
+
],
|
|
769
|
+
}),
|
|
770
|
+
]);
|
|
771
|
+
const results = await openResults(root);
|
|
772
|
+
const locator = results.experiments[0]!.latest.evals[0]!.attempts[0]!.locator!;
|
|
773
|
+
|
|
774
|
+
const bounded = await show(root, [locator], { timing: "summary" });
|
|
775
|
+
const full = await show(root, [locator], { timing: "full" });
|
|
776
|
+
expect(bounded.out).toContain("sandbox.create");
|
|
777
|
+
expect(bounded.out).toContain("eval.run");
|
|
778
|
+
expect(bounded.out).toContain("sandbox.stop");
|
|
779
|
+
expect(bounded.out).toMatch(/… 1 nodes omitted/);
|
|
780
|
+
expect(bounded.out).toContain(`niceeval show ${locator} --timing=full`);
|
|
781
|
+
expect(full.out).not.toContain("nodes omitted");
|
|
782
|
+
expect(full.out).toContain("tool --item=0");
|
|
783
|
+
expect(full.out).toContain("tool --item=80");
|
|
784
|
+
|
|
785
|
+
// 去掉第 81 个后,默认投影不再省略,detail 行与 full 完全一致。
|
|
786
|
+
children.pop();
|
|
787
|
+
const root80 = await makeRoot();
|
|
788
|
+
await writeSnapshot(root80, "2026-07-08T10-00-00-000Z", { experimentId: "compare/bub", startedAt: "2026-07-08T10:00:00.000Z" }, [
|
|
789
|
+
res("weather/brooklyn", "passed", {
|
|
790
|
+
durationMs: 50_000,
|
|
791
|
+
phases: [{ name: "eval.run", durationMs: 42_000, children }],
|
|
792
|
+
}),
|
|
793
|
+
]);
|
|
794
|
+
const results80 = await openResults(root80);
|
|
795
|
+
const locator80 = results80.experiments[0]!.latest.evals[0]!.attempts[0]!.locator!;
|
|
796
|
+
const bounded80 = await show(root80, [locator80], { timing: "summary" });
|
|
797
|
+
const full80 = await show(root80, [locator80], { timing: "full" });
|
|
798
|
+
expect(bounded80.out).toBe(full80.out);
|
|
799
|
+
});
|
|
800
|
+
|
|
801
|
+
it("默认投影保留失败路径/慢点/首尾,omission 报失败数且不虚构 children 合计", async () => {
|
|
802
|
+
const root = await makeRoot();
|
|
803
|
+
const children: TimingNode[] = Array.from({ length: 120 }, (_, index) => ({
|
|
804
|
+
id: `node-${String(index).padStart(3, "0")}`,
|
|
805
|
+
kind: "provider",
|
|
806
|
+
label: `provider-step-${index}`,
|
|
807
|
+
startOffsetMs: index * 100,
|
|
808
|
+
durationMs: index === 60 ? 99_000 : index + 1,
|
|
809
|
+
...(index === 55 ? { failed: true as const } : {}),
|
|
810
|
+
}));
|
|
811
|
+
await writeSnapshot(root, "2026-07-08T10-00-00-000Z", { experimentId: "compare/bub", startedAt: "2026-07-08T10:00:00.000Z" }, [
|
|
812
|
+
res("weather/brooklyn", "errored", {
|
|
813
|
+
durationMs: 120_000,
|
|
814
|
+
error: { code: "provider-failed", message: "boom", phase: "sandbox.create" },
|
|
815
|
+
phases: [{ name: "sandbox.create", durationMs: 120_000, failed: true, children }],
|
|
816
|
+
}),
|
|
817
|
+
]);
|
|
818
|
+
const results = await openResults(root);
|
|
819
|
+
const locator = results.experiments[0]!.latest.evals[0]!.attempts[0]!.locator!;
|
|
820
|
+
const { out } = await show(root, [locator], { timing: "summary" });
|
|
821
|
+
|
|
822
|
+
expect(out).toContain("provider-step-0");
|
|
823
|
+
expect(out).toContain("provider-step-55");
|
|
824
|
+
expect(out).toContain("provider-step-60");
|
|
825
|
+
expect(out).toContain("provider-step-119");
|
|
826
|
+
expect(out).not.toContain("combined");
|
|
827
|
+
});
|
|
828
|
+
|
|
829
|
+
it("operation 使用 producer label;full 按 parentSpanId 展开唯一关联 OTel 树", async () => {
|
|
830
|
+
const root = await makeRoot();
|
|
831
|
+
const dir = await writeSnapshot(root, "2026-07-08T10-00-00-000Z", { experimentId: "compare/bub", startedAt: "2026-07-08T10:00:00.000Z" }, [
|
|
832
|
+
res("weather/brooklyn", "passed", {
|
|
833
|
+
durationMs: 50_000,
|
|
834
|
+
hasTrace: true,
|
|
835
|
+
phases: [{
|
|
836
|
+
name: "workspace.diff",
|
|
837
|
+
durationMs: 5_000,
|
|
838
|
+
children: [{
|
|
839
|
+
id: "operation-1",
|
|
840
|
+
kind: "operation",
|
|
841
|
+
label: "export workspace diff · 1 window · 3,302 files",
|
|
842
|
+
startOffsetMs: 100,
|
|
843
|
+
durationMs: 4_000,
|
|
844
|
+
children: [{
|
|
845
|
+
id: "turn-1",
|
|
846
|
+
kind: "turn",
|
|
847
|
+
label: "s1/t1",
|
|
848
|
+
startOffsetMs: 200,
|
|
849
|
+
durationMs: 3_000,
|
|
850
|
+
traceId: "trace-1",
|
|
851
|
+
}],
|
|
852
|
+
}],
|
|
853
|
+
}],
|
|
854
|
+
}),
|
|
855
|
+
]);
|
|
856
|
+
const trace: TraceSpan[] = [
|
|
857
|
+
{ traceId: "trace-1", spanId: "child", parentSpanId: "root", name: "chat", kind: "model", startMs: 1_100, endMs: 2_000 },
|
|
858
|
+
{ traceId: "trace-1", spanId: "root", name: "codex.exec", kind: "agent", startMs: 1_000, endMs: 3_000 },
|
|
859
|
+
];
|
|
860
|
+
await writeFile(join(dir, "weather/brooklyn/a0/trace.json"), JSON.stringify(trace), "utf-8");
|
|
861
|
+
const results = await openResults(root);
|
|
862
|
+
const locator = results.experiments[0]!.latest.evals[0]!.attempts[0]!.locator!;
|
|
863
|
+
const { out } = await show(root, [locator], { timing: "full" });
|
|
864
|
+
|
|
865
|
+
expect(out).toContain("operation · export workspace diff · 1 window · 3,302 files");
|
|
866
|
+
expect(out).toMatch(/agent · codex\.exec[\s\S]*model · chat/);
|
|
867
|
+
expect(out).not.toContain("git show ×");
|
|
868
|
+
});
|
|
869
|
+
|
|
870
|
+
it("attempt 首页 timing 只列大头,但保留变慢的 telemetry 阶段", async () => {
|
|
871
|
+
const root = await makeRoot();
|
|
872
|
+
const dir = await writeSnapshot(root, "2026-07-08T10-00-00-000Z", { experimentId: "compare/bub", startedAt: "2026-07-08T10:00:00.000Z" }, [
|
|
873
|
+
res("weather/brooklyn", "passed", {
|
|
874
|
+
durationMs: 100_000,
|
|
875
|
+
hasEvents: true,
|
|
876
|
+
phases: [
|
|
877
|
+
{ name: "sandbox.queue", durationMs: 0 },
|
|
878
|
+
{ name: "sandbox.create", durationMs: 3_000 },
|
|
879
|
+
{ name: "workspace.baseline", durationMs: 310 },
|
|
880
|
+
{ name: "telemetry.configure", durationMs: 358 },
|
|
881
|
+
{ name: "eval.run", durationMs: 45_000 },
|
|
882
|
+
{ name: "telemetry.collect", durationMs: 35_300 },
|
|
883
|
+
{ name: "sandbox.stop", durationMs: 8_500 },
|
|
884
|
+
],
|
|
885
|
+
}),
|
|
886
|
+
]);
|
|
887
|
+
await writeFile(
|
|
888
|
+
join(dir, "weather/brooklyn/a0/events.json"),
|
|
889
|
+
JSON.stringify([{ type: "message", role: "assistant", text: "done" }]),
|
|
890
|
+
"utf-8",
|
|
891
|
+
);
|
|
892
|
+
const results = await openResults(root);
|
|
893
|
+
const locator = results.experiments[0]!.latest.evals[0]!.attempts[0]!.locator!;
|
|
894
|
+
|
|
895
|
+
const { out } = await show(root, [locator]);
|
|
896
|
+
const timingLine = out.split("\n").find((line) => line.startsWith("timing:"));
|
|
897
|
+
expect(out).toContain("1 AI message");
|
|
898
|
+
expect(timingLine).toContain("sandbox.queue 0ms");
|
|
899
|
+
expect(timingLine).toContain("sandbox.create 3.0s");
|
|
900
|
+
expect(timingLine).toContain("eval.run 45.0s");
|
|
901
|
+
expect(timingLine).toContain("telemetry.collect 35.3s");
|
|
902
|
+
expect(timingLine).toContain("teardown +8.5s");
|
|
903
|
+
expect(timingLine).not.toContain("workspace.baseline");
|
|
904
|
+
expect(timingLine).not.toContain("telemetry.configure");
|
|
905
|
+
});
|
|
906
|
+
|
|
691
907
|
it("语法不对的 locator 报「not a valid attempt locator」,退出码 1,不崩", async () => {
|
|
692
908
|
const root = await makeRoot();
|
|
693
909
|
await writeSnapshot(root, "2026-07-08T10-00-00-000Z", { experimentId: "compare/bub", startedAt: "2026-07-08T10:00:00.000Z" }, [
|
package/src/tty-line.ts
CHANGED
|
@@ -1,32 +1,14 @@
|
|
|
1
|
-
//
|
|
1
|
+
// 独立诊断行的 bootstrap 终端出口:只服务 run 启动前的 argv/config 错误,以及崩溃兜底/
|
|
2
|
+
// 尚无活跃 coordinator 时的兜底(见 src/runner/feedback/sink.ts 的 no-active-coordinator 分支)。
|
|
2
3
|
//
|
|
3
|
-
//
|
|
4
|
-
//
|
|
5
|
-
//
|
|
6
|
-
//
|
|
7
|
-
//
|
|
8
|
-
//
|
|
9
|
-
// memory/live-raw-stderr-write-desyncs-redraw.md)。
|
|
10
|
-
//
|
|
11
|
-
// 所以任何不经过 Reporter.progress() / onEvalComplete() 的独立诊断行,写之前都要先广播
|
|
12
|
-
// beforeExternalTerminalWrite(),live.ts 借此机会把已画内容清掉、drawnLines 归零,下一次
|
|
13
|
-
// draw() 就会把这行诊断消息之下当成全新起点重画,不会累积。
|
|
14
|
-
|
|
15
|
-
const listeners = new Set<() => void>();
|
|
16
|
-
|
|
17
|
-
/** live.ts 用来订阅"即将有一行独立诊断消息落地"。返回取消订阅函数。 */
|
|
18
|
-
export function onBeforeExternalTerminalWrite(fn: () => void): () => void {
|
|
19
|
-
listeners.add(fn);
|
|
20
|
-
return () => listeners.delete(fn);
|
|
21
|
-
}
|
|
22
|
-
|
|
23
|
-
/** 任何要绕开 Reporter 直接往终端打一行独立诊断消息的地方,写之前都先调用这个。 */
|
|
24
|
-
export function beforeExternalTerminalWrite(): void {
|
|
25
|
-
for (const fn of listeners) fn();
|
|
26
|
-
}
|
|
4
|
+
// run 激活之后的全部反馈(dashboard、durable event、heartbeat)由
|
|
5
|
+
// src/runner/feedback/coordinator.ts 统一排序;它通过 sink.ts 的模块级 coordinator 栈直接接管
|
|
6
|
+
// reportXxx() 调用的目的地(显式路由,不是广播-订阅),所以这里不再需要"写之前先广播、订阅方
|
|
7
|
+
// 自己清屏"的机制——那套机制是为已删除的 live.ts 的 ANSI 回跳重画设计的(见
|
|
8
|
+
// memory/live-raw-stderr-write-desyncs-redraw.md),新 coordinator 的 clear → append → redraw
|
|
9
|
+
// 顺序由 sink.ts 的显式路由保证,不依赖本文件。
|
|
27
10
|
|
|
28
11
|
/** process.stderr.write 的替代:文本需自带换行(沿用现有 i18n 字符串的约定)。 */
|
|
29
12
|
export function writeStderrLine(text: string): void {
|
|
30
|
-
beforeExternalTerminalWrite();
|
|
31
13
|
process.stderr.write(text);
|
|
32
14
|
}
|
package/src/util.test.ts
CHANGED
package/src/util.ts
CHANGED
|
@@ -32,6 +32,47 @@ export function formatThrown(e: unknown): string {
|
|
|
32
32
|
return String(e);
|
|
33
33
|
}
|
|
34
34
|
|
|
35
|
+
/**
|
|
36
|
+
* 截到第一个换行为止。`formatThrown()` 优先带完整 `.stack`(含本地绝对文件路径的多行调用栈)
|
|
37
|
+
* 给需要按 file:line 定位问题的落盘产物(`EvalResult.error`、`niceeval show`);但机器消费的
|
|
38
|
+
* 单行 envelope(`FailureNotice.reason`、reporter 失败诊断的 `message`……)只要「一层可行动摘要」
|
|
39
|
+
* ——`Error.stack` 的第一行恒为 `name: message`,不含栈帧,直接满足这个要求。完整栈仍然原样
|
|
40
|
+
* 留在调用方各自的落盘字段里,这个函数只负责第二次、更短的那份表达,不是唯一出口。
|
|
41
|
+
*/
|
|
42
|
+
export function firstLine(text: string): string {
|
|
43
|
+
const idx = text.indexOf("\n");
|
|
44
|
+
return idx === -1 ? text : text.slice(0, idx);
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* 把 catch 到的 e 拆成 `AttemptError` 的三段:`message`(一层原因,Error.message,不带 name 前缀、
|
|
49
|
+
* 不拼 stack)、`stack`(完整多行栈,供 `niceeval show` 展开)、`cause`(沿 `e.cause` 取一层结构化
|
|
50
|
+
* 摘要 `{ name?, code?, message }`)。Node 的 `Error.stack` 不展开 cause 链,所以 cause 只能从
|
|
51
|
+
* `e.cause` 单独取。非 Error 值只给 message。
|
|
52
|
+
*/
|
|
53
|
+
export function describeError(e: unknown): {
|
|
54
|
+
message: string;
|
|
55
|
+
stack?: string;
|
|
56
|
+
cause?: { name?: string; code?: string; message: string };
|
|
57
|
+
} {
|
|
58
|
+
if (!(e instanceof Error)) return { message: String(e) };
|
|
59
|
+
const out: { message: string; stack?: string; cause?: { name?: string; code?: string; message: string } } = {
|
|
60
|
+
message: e.message || e.name,
|
|
61
|
+
};
|
|
62
|
+
if (e.stack) out.stack = e.stack;
|
|
63
|
+
const raw = (e as { cause?: unknown }).cause;
|
|
64
|
+
if (raw instanceof Error) {
|
|
65
|
+
const c: { name?: string; code?: string; message: string } = { message: raw.message || raw.name };
|
|
66
|
+
if (raw.name) c.name = raw.name;
|
|
67
|
+
const code = (raw as { code?: unknown }).code;
|
|
68
|
+
if (typeof code === "string") c.code = code;
|
|
69
|
+
out.cause = c;
|
|
70
|
+
} else if (typeof raw === "string" && raw.trim() !== "") {
|
|
71
|
+
out.cause = { message: raw };
|
|
72
|
+
}
|
|
73
|
+
return out;
|
|
74
|
+
}
|
|
75
|
+
|
|
35
76
|
/** 零填充到 4 位(数据集扇出的 id:sql/0000)。 */
|
|
36
77
|
export function pad4(n: number): string {
|
|
37
78
|
return String(n).padStart(4, "0");
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { useEffect, useState } from "react";
|
|
1
|
+
import { Fragment, useEffect, useState } from "react";
|
|
2
2
|
import type { ArtifactLoadState, T } from "../shared.ts";
|
|
3
3
|
import type { ViewResult } from "../types.ts";
|
|
4
4
|
import { artifactUrl } from "../lib/artifact-url.ts";
|
|
@@ -54,7 +54,16 @@ export function AttemptModal({ result, onClose, t }: { result: ViewResult; onClo
|
|
|
54
54
|
</DialogClose>
|
|
55
55
|
</div>
|
|
56
56
|
<div className="flex-1 overflow-y-auto px-[18px] pb-[18px] pt-[14px]">
|
|
57
|
-
{result.error ?
|
|
57
|
+
{result.error ? (
|
|
58
|
+
<div className="modal-error">
|
|
59
|
+
<ErrorDetailBlock error={result.error} />
|
|
60
|
+
</div>
|
|
61
|
+
) : null}
|
|
62
|
+
{result.diagnostics && result.diagnostics.length > 0 ? (
|
|
63
|
+
<AttemptDiagnostics diagnostics={result.diagnostics} />
|
|
64
|
+
) : null}
|
|
65
|
+
{result.phases && result.phases.length > 0 ? <PhaseTimingBlock phases={result.phases} /> : null}
|
|
66
|
+
<UsageDiffLine result={result} />
|
|
58
67
|
{data.status === "loading" ? <div className="conv-loading">{t("trace.loading")}</div> : null}
|
|
59
68
|
{hasCode ? (
|
|
60
69
|
<CodeView sources={data.sources ?? []} events={data.events || []} assertions={allAssertions} t={t} />
|
|
@@ -82,3 +91,145 @@ export function AttemptModal({ result, onClose, t }: { result: ViewResult; onClo
|
|
|
82
91
|
</Dialog>
|
|
83
92
|
);
|
|
84
93
|
}
|
|
94
|
+
|
|
95
|
+
/** errored attempt 的结构化 error 明细(见 docs/feature/reports/view.md「结构化错误」)。结构化
|
|
96
|
+
* `AttemptError`:operation / code / message + 可选 cause / stack。字段标签是低层技术标识,与终端
|
|
97
|
+
* `niceeval show` 一样保持英文,不进 view 的 i18n 词典。 */
|
|
98
|
+
function ErrorDetailBlock({ error }: { error: NonNullable<ViewResult["error"]> }) {
|
|
99
|
+
const rows: [string, string][] = [
|
|
100
|
+
["phase", error.phase],
|
|
101
|
+
["code", error.code],
|
|
102
|
+
["message", error.message],
|
|
103
|
+
];
|
|
104
|
+
if (error.cause) rows.push(["cause", error.cause.name ? `${error.cause.name} · ${error.cause.message}` : error.cause.message]);
|
|
105
|
+
const stack = error.stack?.replace(/\n+$/, "") ?? "";
|
|
106
|
+
return (
|
|
107
|
+
<div>
|
|
108
|
+
<dl className="grid grid-cols-[max-content_1fr] gap-x-3 gap-y-1 text-xs">
|
|
109
|
+
{rows.map(([k, v]) => (
|
|
110
|
+
<Fragment key={k}>
|
|
111
|
+
<dt className="text-muted">{k}</dt>
|
|
112
|
+
<dd className="min-w-0 break-words">{v}</dd>
|
|
113
|
+
</Fragment>
|
|
114
|
+
))}
|
|
115
|
+
</dl>
|
|
116
|
+
{stack ? (
|
|
117
|
+
<pre className="mt-2 overflow-x-auto whitespace-pre-wrap text-[11px] text-muted">{stack}</pre>
|
|
118
|
+
) : null}
|
|
119
|
+
</div>
|
|
120
|
+
);
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
/** attempt 级诊断,按 lifecycle 阶段分组(teardown/cleanup 等,与 verdict 独立;
|
|
124
|
+
* 见 docs/feature/reports/view.md「Attempt 详情」)。 */
|
|
125
|
+
function AttemptDiagnostics({ diagnostics }: { diagnostics: NonNullable<ViewResult["diagnostics"]> }) {
|
|
126
|
+
const groups = new Map<string, typeof diagnostics>();
|
|
127
|
+
for (const d of diagnostics) {
|
|
128
|
+
const list = groups.get(d.phase) ?? [];
|
|
129
|
+
groups.set(d.phase, [...list, d]);
|
|
130
|
+
}
|
|
131
|
+
return (
|
|
132
|
+
<div className="mt-3">
|
|
133
|
+
<div className="text-xs font-[640] text-text">diagnostics</div>
|
|
134
|
+
{[...groups.entries()].map(([phase, list]) => (
|
|
135
|
+
<div key={phase} className="mt-1">
|
|
136
|
+
<div className="text-[11px] text-muted">{phase}</div>
|
|
137
|
+
<ul className="mt-0.5 space-y-1 pl-3">
|
|
138
|
+
{list.map((d, i) => (
|
|
139
|
+
<li key={i} className="text-xs">
|
|
140
|
+
<span className="text-muted">
|
|
141
|
+
{d.level} · {d.code}
|
|
142
|
+
</span>
|
|
143
|
+
<div className="min-w-0 break-words">
|
|
144
|
+
{d.message}
|
|
145
|
+
{d.count && d.count > 1 ? ` (${d.count} occurrences)` : ""}
|
|
146
|
+
</div>
|
|
147
|
+
</li>
|
|
148
|
+
))}
|
|
149
|
+
</ul>
|
|
150
|
+
</div>
|
|
151
|
+
))}
|
|
152
|
+
</div>
|
|
153
|
+
);
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
const CLOSING_PHASES = new Set(["eval.teardown", "agent.teardown", "sandbox.teardown", "sandbox.suspend", "sandbox.stop"]);
|
|
157
|
+
|
|
158
|
+
function fmtMs(ms: number): string {
|
|
159
|
+
if (ms >= 60_000) return `${Math.floor(ms / 60_000)}m ${Math.round((ms % 60_000) / 1000)}s`;
|
|
160
|
+
if (ms >= 1000) return `${(ms / 1000).toFixed(1)}s`;
|
|
161
|
+
return `${ms}ms`;
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
/** 统一时间树(见 docs/feature/reports/view.md「Attempt 详情」):`result.json.phases` 的主链
|
|
165
|
+
* 分解与收尾段;每个 phase 可展开 runner 直接观察到的 hook / 命令 / turn(嵌套 children)。 */
|
|
166
|
+
function PhaseTimingBlock({ phases }: { phases: NonNullable<ViewResult["phases"]> }) {
|
|
167
|
+
const main = phases.filter((p) => !CLOSING_PHASES.has(p.name));
|
|
168
|
+
const closing = phases.filter((p) => CLOSING_PHASES.has(p.name));
|
|
169
|
+
return (
|
|
170
|
+
<div className="mt-3">
|
|
171
|
+
<div className="text-xs font-[640] text-text">timing</div>
|
|
172
|
+
<ul className="mt-1 space-y-0.5">
|
|
173
|
+
{main.map((p, i) => (
|
|
174
|
+
<TimingRow key={i} name={p.name} durationMs={p.durationMs} failed={p.failed} children_={p.children} />
|
|
175
|
+
))}
|
|
176
|
+
</ul>
|
|
177
|
+
{closing.length > 0 ? (
|
|
178
|
+
<div className="mt-1">
|
|
179
|
+
<div className="text-[11px] text-muted">teardown (not counted in total)</div>
|
|
180
|
+
<ul className="mt-0.5 space-y-0.5">
|
|
181
|
+
{closing.map((p, i) => (
|
|
182
|
+
<TimingRow key={i} name={p.name} durationMs={p.durationMs} failed={p.failed} children_={p.children} />
|
|
183
|
+
))}
|
|
184
|
+
</ul>
|
|
185
|
+
</div>
|
|
186
|
+
) : null}
|
|
187
|
+
</div>
|
|
188
|
+
);
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
type TimingChild = NonNullable<NonNullable<ViewResult["phases"]>[number]["children"]>[number];
|
|
192
|
+
|
|
193
|
+
function TimingRow({
|
|
194
|
+
name,
|
|
195
|
+
durationMs,
|
|
196
|
+
failed,
|
|
197
|
+
children_,
|
|
198
|
+
}: {
|
|
199
|
+
name: string;
|
|
200
|
+
durationMs: number;
|
|
201
|
+
failed?: true;
|
|
202
|
+
children_?: TimingChild[];
|
|
203
|
+
}) {
|
|
204
|
+
const kids = children_ ?? [];
|
|
205
|
+
const label = (node: TimingChild): string =>
|
|
206
|
+
node.kind === "command" && node.command ? `shell · ${node.command.display}` : node.kind === "turn" ? `turn ${node.label}` : node.label;
|
|
207
|
+
const renderChild = (node: TimingChild, depth: number): React.ReactNode => (
|
|
208
|
+
<li key={node.id} className="text-[11px]" style={{ paddingLeft: `${depth * 12}px` }}>
|
|
209
|
+
<span className="text-muted">{label(node)}</span> {fmtMs(node.durationMs)}
|
|
210
|
+
{node.failed ? <span className="text-bad"> ✗</span> : null}
|
|
211
|
+
{(node.children ?? []).map((c) => renderChild(c, depth + 1))}
|
|
212
|
+
</li>
|
|
213
|
+
);
|
|
214
|
+
return (
|
|
215
|
+
<li className="text-xs">
|
|
216
|
+
<span>{name}</span> <span className="text-muted">{fmtMs(durationMs)}</span>
|
|
217
|
+
{failed ? <span className="text-bad"> ✗</span> : null}
|
|
218
|
+
{kids.length > 0 ? <ul className="mt-0.5 space-y-0.5 pl-3">{kids.map((c) => renderChild(c, 0))}</ul> : null}
|
|
219
|
+
</li>
|
|
220
|
+
);
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
/** usage 与 diff 入口行(见 docs/feature/reports/view.md「Attempt 详情」)。 */
|
|
224
|
+
function UsageDiffLine({ result }: { result: ViewResult }) {
|
|
225
|
+
const parts: string[] = [];
|
|
226
|
+
if (result.usage) {
|
|
227
|
+
const tok = result.usage.inputTokens + result.usage.outputTokens;
|
|
228
|
+
parts.push(`usage: ${tok.toLocaleString()} tok${result.usage.costUSD !== undefined ? ` · $${result.usage.costUSD.toFixed(4)}` : ""}`);
|
|
229
|
+
}
|
|
230
|
+
if (result.sandbox) {
|
|
231
|
+
parts.push(`sandbox: ${result.sandbox.provider} · ${result.sandbox.sandboxId}${result.sandbox.kept ? " · kept" : ""}`);
|
|
232
|
+
}
|
|
233
|
+
if (parts.length === 0) return null;
|
|
234
|
+
return <div className="mt-2 text-xs text-muted">{parts.join(" ")}</div>;
|
|
235
|
+
}
|