niceeval 0.5.3 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/INDEX.md +55 -0
- package/dist/agents/types.d.ts +317 -0
- package/dist/context/types.d.ts +335 -0
- package/dist/i18n/core.d.ts +11 -0
- package/dist/i18n/en.d.ts +181 -0
- package/dist/i18n/index.d.ts +5 -0
- package/dist/i18n/zh-CN.d.ts +183 -0
- package/dist/o11y/execution-tree.d.ts +103 -0
- package/dist/o11y/otlp/parse.d.ts +3 -0
- package/dist/o11y/otlp/receiver.d.ts +21 -0
- package/dist/o11y/otlp/select.d.ts +22 -0
- package/dist/o11y/otlp/turn-otel.d.ts +43 -0
- package/dist/o11y/types.d.ts +190 -0
- package/dist/report/aggregate.d.ts +57 -0
- package/dist/report/aggregate.js +212 -0
- package/dist/report/built-ins/cost-pass-rate-comparison.d.ts +1 -0
- package/dist/report/built-ins/cost-pass-rate-comparison.js +17 -0
- package/dist/report/built-ins/index.d.ts +1 -0
- package/dist/report/built-ins/index.js +2 -0
- package/dist/report/components.d.ts +178 -0
- package/dist/report/components.js +132 -0
- package/dist/report/compute.d.ts +106 -0
- package/dist/report/compute.js +618 -0
- package/dist/report/flag.d.ts +6 -0
- package/dist/report/flag.js +8 -0
- package/dist/report/format.d.ts +32 -0
- package/dist/report/format.js +126 -0
- package/dist/report/index.d.ts +19 -0
- package/dist/report/index.js +30 -0
- package/dist/report/load.d.ts +12 -0
- package/dist/report/load.js +56 -0
- package/dist/report/locale.d.ts +83 -0
- package/dist/report/locale.js +160 -0
- package/dist/report/metrics.d.ts +21 -0
- package/dist/report/metrics.js +115 -0
- package/dist/report/primitives.d.ts +64 -0
- package/dist/report/primitives.js +113 -0
- package/dist/report/react/AttemptList.d.ts +23 -0
- package/dist/report/react/AttemptList.js +20 -0
- package/dist/report/react/DeltaTable.d.ts +8 -0
- package/dist/report/react/DeltaTable.js +24 -0
- package/dist/report/react/EvalList.d.ts +10 -0
- package/dist/report/react/EvalList.js +0 -0
- package/dist/report/react/ExperimentList.d.ts +10 -0
- package/dist/report/react/ExperimentList.js +15 -0
- package/dist/report/react/GroupSummary.d.ts +8 -0
- package/dist/report/react/GroupSummary.js +8 -0
- package/dist/report/react/MetricBars.d.ts +10 -0
- package/dist/report/react/MetricBars.js +48 -0
- package/dist/report/react/MetricLine.d.ts +9 -0
- package/dist/report/react/MetricLine.js +68 -0
- package/dist/report/react/MetricMatrix.d.ts +10 -0
- package/dist/report/react/MetricMatrix.js +23 -0
- package/dist/report/react/MetricScatter.d.ts +9 -0
- package/dist/report/react/MetricScatter.js +100 -0
- package/dist/report/react/MetricTable.d.ts +11 -0
- package/dist/report/react/MetricTable.js +26 -0
- package/dist/report/react/RunOverview.d.ts +8 -0
- package/dist/report/react/RunOverview.js +12 -0
- package/dist/report/react/Scoreboard.d.ts +8 -0
- package/dist/report/react/Scoreboard.js +26 -0
- package/dist/report/react/cell.d.ts +9 -0
- package/dist/report/react/cell.js +9 -0
- package/dist/report/react/chart-math.d.ts +14 -0
- package/dist/report/react/chart-math.js +68 -0
- package/dist/report/react/colors.d.ts +14 -0
- package/dist/report/react/colors.js +45 -0
- package/dist/report/react/fixtures.d.ts +19 -0
- package/dist/report/react/fixtures.js +393 -0
- package/dist/report/react/format.d.ts +3 -0
- package/dist/report/react/format.js +7 -0
- package/dist/report/react/index.d.ts +6 -0
- package/dist/report/react/index.js +14 -0
- package/dist/report/report.d.ts +24 -0
- package/dist/report/report.js +49 -0
- package/dist/report/text/faces.d.ts +29 -0
- package/dist/report/text/faces.js +503 -0
- package/dist/report/text/layout.d.ts +75 -0
- package/dist/report/text/layout.js +173 -0
- package/dist/report/text/plot.d.ts +37 -0
- package/dist/report/text/plot.js +81 -0
- package/dist/report/text/table.d.ts +8 -0
- package/dist/report/text/table.js +86 -0
- package/dist/report/tree.d.ts +99 -0
- package/dist/report/tree.js +228 -0
- package/dist/report/types.d.ts +420 -0
- package/dist/report/types.js +4 -0
- package/dist/report/web.d.ts +15 -0
- package/dist/report/web.js +37 -0
- package/dist/results/annotated-source.d.ts +61 -0
- package/dist/results/attempt-evidence.d.ts +69 -0
- package/dist/results/attempt-source.d.ts +15 -0
- package/dist/results/locator.d.ts +75 -0
- package/dist/results/locator.js +0 -0
- package/dist/results/select.d.ts +45 -0
- package/dist/results/select.js +230 -0
- package/dist/results/source-hash.d.ts +9 -0
- package/dist/results/types.d.ts +204 -0
- package/dist/runner/eval-source.d.ts +22 -0
- package/dist/runner/fingerprint.d.ts +23 -0
- package/dist/runner/types.d.ts +332 -0
- package/dist/sandbox/docker-stream.d.ts +24 -0
- package/dist/sandbox/docker.d.ts +84 -0
- package/dist/sandbox/e2b.d.ts +31 -0
- package/dist/sandbox/errors.d.ts +4 -0
- package/dist/sandbox/local-files.d.ts +6 -0
- package/dist/sandbox/paths.d.ts +4 -0
- package/dist/sandbox/registry.d.ts +13 -0
- package/dist/sandbox/resolve.d.ts +40 -0
- package/dist/sandbox/retry.d.ts +10 -0
- package/dist/sandbox/shell.d.ts +11 -0
- package/dist/sandbox/source-files.d.ts +13 -0
- package/dist/sandbox/types.d.ts +203 -0
- package/dist/sandbox/vercel.d.ts +37 -0
- package/dist/scoring/types.d.ts +70 -0
- package/dist/shared/aggregate.d.ts +29 -0
- package/dist/shared/aggregate.js +55 -0
- package/dist/shared/types.d.ts +27 -0
- package/dist/shared/verdict.d.ts +27 -0
- package/dist/shared/verdict.js +34 -0
- package/dist/tty-line.d.ts +6 -0
- package/dist/types.d.ts +7 -0
- package/dist/util.d.ts +22 -0
- package/docs-site/zh/concepts/adapter.mdx +9 -4
- package/docs-site/zh/concepts/assert.mdx +8 -8
- package/docs-site/zh/concepts/evals.mdx +5 -5
- package/docs-site/zh/concepts/experiment.mdx +3 -1
- package/docs-site/zh/concepts/judge.mdx +9 -9
- package/docs-site/zh/concepts/overview.mdx +2 -2
- package/docs-site/zh/concepts/tier.mdx +6 -6
- package/docs-site/zh/example/claude-code-codex-plugin.mdx +9 -5
- package/docs-site/zh/example/claude-code-codex-skill.mdx +12 -7
- package/docs-site/zh/guides/agent-feedback-loop.mdx +235 -78
- package/docs-site/zh/guides/connect-otel.mdx +3 -3
- package/docs-site/zh/guides/connect-your-agent.mdx +2 -2
- package/docs-site/zh/guides/custom-reports.mdx +196 -36
- package/docs-site/zh/guides/experiments.mdx +7 -0
- package/docs-site/zh/guides/fixtures.mdx +1 -1
- package/docs-site/zh/guides/official-adapters.mdx +31 -9
- package/docs-site/zh/guides/publish-report.mdx +4 -4
- package/docs-site/zh/guides/report-components.mdx +194 -55
- package/docs-site/zh/guides/reporters.mdx +2 -2
- package/docs-site/zh/guides/results-data.mdx +35 -34
- package/docs-site/zh/guides/runner.mdx +2 -2
- package/docs-site/zh/guides/sandbox-agent.mdx +10 -5
- package/docs-site/zh/guides/sandbox-providers.mdx +102 -0
- package/docs-site/zh/guides/viewing-results.mdx +199 -79
- package/docs-site/zh/guides/write-experiment.mdx +23 -1
- package/docs-site/zh/guides/write-send.mdx +2 -2
- package/docs-site/zh/index.mdx +4 -3
- package/docs-site/zh/introduction.mdx +2 -1
- package/docs-site/zh/quickstart.mdx +5 -3
- package/docs-site/zh/reference/builtin-agents.mdx +38 -13
- package/docs-site/zh/reference/capabilities.mdx +1 -1
- package/docs-site/zh/reference/cli.mdx +36 -13
- package/docs-site/zh/reference/define-agent.mdx +24 -12
- package/docs-site/zh/reference/define-config.mdx +1 -1
- package/docs-site/zh/reference/define-eval.mdx +3 -3
- package/docs-site/zh/reference/events.mdx +7 -1
- package/docs-site/zh/reference/expect.mdx +3 -3
- package/package.json +10 -6
- package/src/agents/bub.ts +134 -43
- package/src/agents/claude-code.test.ts +152 -0
- package/src/agents/claude-code.ts +127 -13
- package/src/agents/codex.test.ts +192 -0
- package/src/agents/codex.ts +151 -12
- package/src/agents/index.ts +11 -3
- package/src/agents/manifest.ts +41 -0
- package/src/agents/plugin-config.test.ts +94 -0
- package/src/agents/shared.ts +33 -2
- package/src/agents/skills.test.ts +214 -0
- package/src/agents/skills.ts +304 -0
- package/src/agents/streaming.ts +5 -5
- package/src/agents/types.ts +62 -2
- package/src/cli.ts +74 -47
- package/src/context/context.test.ts +20 -2
- package/src/context/context.ts +3 -0
- package/src/context/control-flow.ts +1 -1
- package/src/context/session.test.ts +92 -1
- package/src/context/session.ts +26 -1
- package/src/context/types.ts +4 -4
- package/src/define.test.ts +82 -0
- package/src/define.ts +72 -11
- package/src/expect/index.ts +5 -5
- package/src/i18n/en.ts +36 -15
- package/src/i18n/zh-CN.ts +39 -21
- package/src/index.ts +1 -1
- package/src/o11y/execution-tree.test.ts +452 -0
- package/src/o11y/execution-tree.ts +367 -0
- package/src/o11y/otlp/receiver.ts +1 -1
- package/src/o11y/otlp/select.ts +7 -3
- package/src/o11y/parsers/claude-code.test.ts +175 -0
- package/src/o11y/parsers/claude-code.ts +32 -0
- package/src/o11y/types.ts +6 -1
- package/src/report/aggregate.ts +26 -20
- package/src/report/built-in-user-parity.test.tsx +643 -0
- package/src/report/built-ins/cost-pass-rate-comparison.tsx +23 -0
- package/src/report/built-ins/index.ts +2 -0
- package/src/report/components.tsx +223 -50
- package/src/report/compute.ts +358 -53
- package/src/report/dual-render.test.tsx +1125 -0
- package/src/report/flag.ts +1 -1
- package/src/report/format.ts +50 -2
- package/src/report/index.ts +48 -14
- package/src/report/load.ts +1 -1
- package/src/report/locale.ts +206 -0
- package/src/report/metrics.ts +39 -15
- package/src/report/primitives.tsx +100 -2
- package/src/report/react/AttemptList.tsx +115 -0
- package/src/report/react/DeltaTable.tsx +9 -6
- package/src/report/react/EvalList.tsx +0 -0
- package/src/report/react/ExperimentList.tsx +108 -0
- package/src/report/react/GroupSummary.tsx +66 -0
- package/src/report/react/MetricBars.tsx +24 -13
- package/src/report/react/MetricLine.tsx +21 -21
- package/src/report/react/MetricMatrix.tsx +8 -4
- package/src/report/react/MetricScatter.tsx +136 -96
- package/src/report/react/MetricTable.tsx +115 -15
- package/src/report/react/RunOverview.tsx +48 -26
- package/src/report/react/Scoreboard.tsx +15 -7
- package/src/report/react/cell.tsx +18 -9
- package/src/{view/app/lib/chart.ts → report/react/chart-math.ts} +2 -6
- package/src/report/react/colors.ts +24 -15
- package/src/report/react/enhance.js +129 -0
- package/src/report/react/fixtures.ts +163 -34
- package/src/report/react/format.ts +1 -1
- package/src/report/react/index.tsx +22 -6
- package/src/report/react/render.test.tsx +164 -31
- package/src/report/react/styles.css +405 -213
- package/src/report/report.test.ts +403 -94
- package/src/report/report.ts +29 -11
- package/src/report/text/faces.ts +303 -95
- package/src/report/text/layout.ts +76 -11
- package/src/report/text/table.ts +98 -0
- package/src/report/tree.ts +103 -20
- package/src/report/types.ts +196 -25
- package/src/report/web.ts +46 -19
- package/src/results/annotated-source.test.ts +146 -0
- package/src/results/annotated-source.ts +128 -0
- package/src/results/attempt-evidence.test.ts +158 -0
- package/src/results/attempt-evidence.ts +158 -0
- package/src/results/attempt-source.ts +48 -0
- package/src/results/copy.ts +154 -139
- package/src/results/format.ts +54 -47
- package/src/results/host-equivalence.test.ts +527 -0
- package/src/results/index.ts +56 -14
- package/src/results/locator.test.ts +198 -0
- package/src/results/locator.ts +0 -0
- package/src/results/open.ts +392 -175
- package/src/results/results.test.ts +943 -494
- package/src/results/select.ts +129 -23
- package/src/results/skipped-notice.ts +0 -0
- package/src/results/source-hash.ts +28 -0
- package/src/results/types.ts +98 -62
- package/src/results/writer.ts +304 -176
- package/src/runner/attempt.test.ts +170 -0
- package/src/runner/attempt.ts +97 -15
- package/src/runner/discover.test.ts +65 -0
- package/src/runner/discover.ts +6 -2
- package/src/runner/eval-source.test.ts +121 -0
- package/src/runner/eval-source.ts +45 -0
- package/src/runner/fingerprint.ts +54 -1
- package/src/runner/remote-sandbox.ts +1 -1
- package/src/runner/report.test.ts +6 -8
- package/src/runner/report.ts +5 -4
- package/src/runner/reporters/artifacts.ts +27 -41
- package/src/runner/reporters/braintrust.test.ts +3 -3
- package/src/runner/reporters/braintrust.ts +1 -1
- package/src/runner/reporters/console.ts +6 -6
- package/src/runner/reporters/json.ts +4 -4
- package/src/runner/reporters/live.test.ts +56 -0
- package/src/runner/reporters/live.ts +36 -10
- package/src/runner/reporters/quiet.test.ts +66 -0
- package/src/runner/reporters/quiet.ts +49 -0
- package/src/runner/reporters/shared.ts +5 -5
- package/src/runner/reporters/table.ts +17 -17
- package/src/runner/run.ts +61 -102
- package/src/runner/types.ts +43 -32
- package/src/sandbox/checkpoint.ts +1 -1
- package/src/sandbox/docker.ts +15 -2
- package/src/sandbox/e2b.ts +11 -5
- package/src/sandbox/errors.ts +13 -0
- package/src/sandbox/index.ts +3 -3
- package/src/sandbox/local-files.ts +1 -1
- package/src/sandbox/registry.ts +6 -5
- package/src/sandbox/resolve.ts +38 -23
- package/src/sandbox/retry.test.ts +68 -0
- package/src/sandbox/retry.ts +44 -0
- package/src/sandbox/types.ts +69 -28
- package/src/sandbox/vercel.ts +17 -3
- package/src/scoring/collector.ts +3 -3
- package/src/scoring/judge.ts +2 -2
- package/src/scoring/scoped.ts +13 -1
- package/src/scoring/types.ts +7 -7
- package/src/scoring/verdict.ts +4 -4
- package/src/shared/aggregate.ts +23 -3
- package/src/shared/types.ts +2 -2
- package/src/shared/{outcome.ts → verdict.ts} +15 -15
- package/src/show/compose.ts +12 -132
- package/src/show/index.ts +125 -94
- package/src/show/render.ts +660 -160
- package/src/show/show.test.ts +472 -253
- package/src/source-loc.ts +1 -1
- package/src/tty-line.ts +32 -0
- package/src/types.ts +2 -2
- package/src/util.test.ts +21 -1
- package/src/util.ts +5 -1
- package/src/view/app/App.tsx +33 -198
- package/src/view/app/components/AttemptModal.tsx +6 -6
- package/src/view/app/components/CodeView.tsx +2 -2
- package/src/view/app/components/CopyControls.tsx +32 -36
- package/src/view/app/components/SkippedRunsBanner.tsx +140 -0
- package/src/view/app/components/ui/badge.tsx +1 -1
- package/src/view/app/components/ui/dialog.tsx +2 -2
- package/src/view/app/i18n.ts +40 -31
- package/src/view/app/lib/artifact-url.ts +1 -1
- package/src/view/app/lib/attempt-route.test.ts +37 -47
- package/src/view/app/lib/attempt-route.ts +26 -26
- package/src/view/app/lib/rows.ts +9 -112
- package/src/view/app/lib/verdict.ts +25 -0
- package/src/view/app/main.tsx +9 -12
- package/src/view/app/pages/RunsPage.tsx +4 -4
- package/src/view/app/pages/TracesPage.tsx +3 -3
- package/src/view/app/shared.ts +0 -1
- package/src/view/app/types.ts +6 -51
- package/src/view/artifact-serving.test.ts +136 -0
- package/src/view/client-dist/app.css +1 -1
- package/src/view/client-dist/app.js +21 -21
- package/src/view/data.test.ts +214 -143
- package/src/view/data.ts +166 -120
- package/src/view/index.ts +23 -11
- package/src/view/server.ts +52 -17
- package/src/view/shared/types.ts +22 -31
- package/src/view/styles.css +117 -327
- package/src/view/view-report.test.ts +146 -98
- package/docs-site/zh/guides/sandbox-backends.mdx +0 -75
- package/src/report/default-report.tsx +0 -222
- package/src/report/dual-face.test.tsx +0 -527
- package/src/report/react/CaseList.tsx +0 -70
- package/src/view/app/components/CostScoreChart.tsx +0 -147
- package/src/view/app/components/ExperimentTable.tsx +0 -284
- package/src/view/app/components/GroupSelector.tsx +0 -61
- package/src/view/app/components/primitives.tsx +0 -43
- package/src/view/app/lib/outcome.ts +0 -82
package/src/source-loc.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
// 源码位置回溯:在 `t.send` / 断言落地时(collector.record / SessionManager.send)抓一次
|
|
2
2
|
// 调用栈,挑出**第一帧不属于 niceeval 自身**的位置——也就是用户 eval 里那一行。view 据此把
|
|
3
|
-
// 运行结果(回复 / 分数 /
|
|
3
|
+
// 运行结果(回复 / 分数 / 判定)叠回真实源码行,渲染成 github-diff 式代码视图。
|
|
4
4
|
//
|
|
5
5
|
// 为什么靠栈而不是改 API:这样 `t` 的表面一个字不用动,作者照常写 `t.judge.autoevals...`,
|
|
6
6
|
// 位置在底层免费拿到。tsx 直接跑 .ts,栈里就是真实 .ts 路径+行号(ESM 下是 file:// URL)。
|
package/src/tty-line.ts
ADDED
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
// 独立诊断行的统一终端出口。
|
|
2
|
+
//
|
|
3
|
+
// live 表格(src/runner/reporters/live.ts)重画靠 \x1B[{drawnLines}A 回跳到上一帧起点,
|
|
4
|
+
// 这个回跳量的前提是"stderr/stdout 这块屏幕在两帧之间只有它自己在写"。sandbox teardown
|
|
5
|
+
// 失败、budget 不可执行、reporter 抛错这类跟 attempt 进度无关的独立诊断行,如果在 live
|
|
6
|
+
// 表格激活期间直接 process.stderr.write / console.log,就会插进两次 draw() 之间——下一帧
|
|
7
|
+
// 按旧的 drawnLines 回跳,已经够不到表格真正的起点,清行重画变成往下多写一份,且这个偏移
|
|
8
|
+
// 被记进新的 drawnLines,此后每帧都在错位的基础上继续错位,越滚越多(见
|
|
9
|
+
// memory/live-raw-stderr-write-desyncs-redraw.md)。
|
|
10
|
+
//
|
|
11
|
+
// 所以任何不经过 Reporter.progress() / onEvalComplete() 的独立诊断行,写之前都要先广播
|
|
12
|
+
// beforeExternalTerminalWrite(),live.ts 借此机会把已画内容清掉、drawnLines 归零,下一次
|
|
13
|
+
// draw() 就会把这行诊断消息之下当成全新起点重画,不会累积。
|
|
14
|
+
|
|
15
|
+
const listeners = new Set<() => void>();
|
|
16
|
+
|
|
17
|
+
/** live.ts 用来订阅"即将有一行独立诊断消息落地"。返回取消订阅函数。 */
|
|
18
|
+
export function onBeforeExternalTerminalWrite(fn: () => void): () => void {
|
|
19
|
+
listeners.add(fn);
|
|
20
|
+
return () => listeners.delete(fn);
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
/** 任何要绕开 Reporter 直接往终端打一行独立诊断消息的地方,写之前都先调用这个。 */
|
|
24
|
+
export function beforeExternalTerminalWrite(): void {
|
|
25
|
+
for (const fn of listeners) fn();
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/** process.stderr.write 的替代:文本需自带换行(沿用现有 i18n 字符串的约定)。 */
|
|
29
|
+
export function writeStderrLine(text: string): void {
|
|
30
|
+
beforeExternalTerminalWrite();
|
|
31
|
+
process.stderr.write(text);
|
|
32
|
+
}
|
package/src/types.ts
CHANGED
|
@@ -4,9 +4,9 @@
|
|
|
4
4
|
// 模块代码统一 `import type { … } from "../types.ts"`,不必记住每个类型的家:
|
|
5
5
|
// · shared/types.ts 跨域原子(JsonValue / Severity / SourceLoc / Cleanup / LocalizedText)
|
|
6
6
|
// · o11y/types.ts 标准事件流 / DerivedFacts / TraceSpan / Usage / O11ySummary
|
|
7
|
-
// · sandbox/types.ts Sandbox 接口 /
|
|
7
|
+
// · sandbox/types.ts Sandbox 接口 / provider spec / 命令与文件 IO
|
|
8
8
|
// · agents/types.ts Agent / Adapter 契约 / 会话 / tracing 导出
|
|
9
|
-
// · scoring/types.ts 断言(
|
|
9
|
+
// · scoring/types.ts 断言(值 / 记录 / 结果)/ ScoringContext / JudgeConfig
|
|
10
10
|
// · context/types.ts TestContext(t)与子句柄(turn / session / sandbox 视图)
|
|
11
11
|
// · runner/types.ts EvalResult / RunSummary / Reporter / eval・experiment・config 定义 / 调度编排
|
|
12
12
|
//
|
package/src/util.test.ts
CHANGED
|
@@ -1,5 +1,25 @@
|
|
|
1
1
|
import { describe, expect, it } from "vitest";
|
|
2
|
-
import { formatThrown, upsertManagedBlock } from "./util.ts";
|
|
2
|
+
import { brief, formatThrown, upsertManagedBlock } from "./util.ts";
|
|
3
|
+
|
|
4
|
+
describe("brief", () => {
|
|
5
|
+
// JSON.stringify(undefined) 返回值 undefined(不是字符串 "undefined"),不特殊处理会让
|
|
6
|
+
// 下游 `s.length` 抛 TypeError——真实复现见 previewCheckedValue 断言 undefined 字段值时
|
|
7
|
+
// (2026-07-13 codex native plugin e2e,resolvedVersion 缺失时 equals() 的 gate 预览崩溃)。
|
|
8
|
+
it("does not throw on undefined and returns a readable placeholder", () => {
|
|
9
|
+
expect(() => brief(undefined)).not.toThrow();
|
|
10
|
+
expect(brief(undefined)).toBe("undefined");
|
|
11
|
+
});
|
|
12
|
+
|
|
13
|
+
it("still handles ordinary values (string passthrough, object JSON-stringified)", () => {
|
|
14
|
+
expect(brief("hello")).toBe("hello");
|
|
15
|
+
expect(brief({ a: 1 })).toBe('{"a":1}');
|
|
16
|
+
});
|
|
17
|
+
|
|
18
|
+
it("truncates long output with an ellipsis", () => {
|
|
19
|
+
const long = "x".repeat(300);
|
|
20
|
+
expect(brief(long, 10)).toBe(`${"x".repeat(10)}…`);
|
|
21
|
+
});
|
|
22
|
+
});
|
|
3
23
|
|
|
4
24
|
describe("upsertManagedBlock", () => {
|
|
5
25
|
const begin = "<!-- BEGIN:x -->";
|
package/src/util.ts
CHANGED
|
@@ -57,7 +57,11 @@ export function upsertManagedBlock(source: string, begin: string, end: string, c
|
|
|
57
57
|
export function brief(value: unknown, max = 200): string {
|
|
58
58
|
let s: string;
|
|
59
59
|
try {
|
|
60
|
-
|
|
60
|
+
// `JSON.stringify` 对 undefined / function / symbol 返回**值** undefined,不是字符串
|
|
61
|
+
// "undefined"——不兜底会让 s 变成非字符串,下面 `s.length` 直接抛
|
|
62
|
+
// `Cannot read properties of undefined (reading 'length')`(2026-07-13 native plugin
|
|
63
|
+
// e2e 断言 `equals(undefined 的字段)` 时真实复现,不是假设场景)。
|
|
64
|
+
s = typeof value === "string" ? value : (JSON.stringify(value) ?? String(value));
|
|
61
65
|
} catch {
|
|
62
66
|
s = String(value);
|
|
63
67
|
}
|
package/src/view/app/App.tsx
CHANGED
|
@@ -1,29 +1,20 @@
|
|
|
1
1
|
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
|
|
2
2
|
import { detectLocale, makeTranslator, persistLocale, setDocumentLocale } from "./i18n.ts";
|
|
3
3
|
import type { MessageKey } from "./i18n.ts";
|
|
4
|
-
import type { Locale, LocalizedText,
|
|
5
|
-
import {
|
|
6
|
-
import {
|
|
7
|
-
import {
|
|
8
|
-
import { Metric } from "./components/primitives.tsx";
|
|
9
|
-
import { GroupSelector } from "./components/GroupSelector.tsx";
|
|
10
|
-
import { CostScoreChart } from "./components/CostScoreChart.tsx";
|
|
11
|
-
import { ExperimentTable } from "./components/ExperimentTable.tsx";
|
|
4
|
+
import type { Locale, LocalizedText, ReportSlotHtml, Tab, ViewData, ViewResult } from "./types.ts";
|
|
5
|
+
import { flattenAttempts, resultFromUrl } from "./lib/rows.ts";
|
|
6
|
+
import { parseAttemptHash, resolveAttemptLocator, unresolvedAttemptWarning } from "./lib/attempt-route.ts";
|
|
7
|
+
import { formatDateTime } from "./lib/format.ts";
|
|
12
8
|
import { CopyFixPrompt } from "./components/CopyControls.tsx";
|
|
9
|
+
import { SkippedRunsBanner } from "./components/SkippedRunsBanner.tsx";
|
|
13
10
|
import { AttemptModal } from "./components/AttemptModal.tsx";
|
|
14
11
|
import { Tabs, TabsContent, TabsList, TabsTrigger } from "./components/ui/tabs.tsx";
|
|
15
12
|
import { RunsView } from "./pages/RunsPage.tsx";
|
|
16
13
|
import { TracesView } from "./pages/TracesPage.tsx";
|
|
17
14
|
|
|
18
|
-
//
|
|
19
|
-
// 证据室(Runs / Traces / AttemptModal)是 view
|
|
15
|
+
// 首页 = 报告槽:恒在恒默认,裸跑填充内置默认报告,--report 整槽替换(两种填充走同一个 tab)。
|
|
16
|
+
// 证据室(Runs / Traces / AttemptModal)是 view 本体,任何填充下原样保留。
|
|
20
17
|
export const navItems: { id: Tab; label: MessageKey }[] = [
|
|
21
|
-
{ id: "experiments", label: "nav.experiments" },
|
|
22
|
-
{ id: "runs", label: "nav.runs" },
|
|
23
|
-
{ id: "traces", label: "nav.traces" },
|
|
24
|
-
];
|
|
25
|
-
|
|
26
|
-
export const reportNavItems: { id: Tab; label: MessageKey }[] = [
|
|
27
18
|
{ id: "report", label: "nav.report" },
|
|
28
19
|
{ id: "runs", label: "nav.runs" },
|
|
29
20
|
{ id: "traces", label: "nav.traces" },
|
|
@@ -36,34 +27,25 @@ export function localizedText(text: LocalizedText | undefined, locale: Locale):
|
|
|
36
27
|
return text[locale] ?? text.en ?? Object.values(text)[0];
|
|
37
28
|
}
|
|
38
29
|
|
|
39
|
-
/** 初始 URL → 直接打开的 attempt:先认 #/attempt
|
|
30
|
+
/** 初始 URL → 直接打开的 attempt:先认 #/attempt/@<locator> 深链,回退旧版 ?modal= 参数。 */
|
|
40
31
|
function modalResultFromLocation(snapshots: ViewData["snapshots"]): ViewResult | null {
|
|
41
|
-
const
|
|
42
|
-
if (
|
|
43
|
-
const found =
|
|
32
|
+
const locator = parseAttemptHash(location.hash);
|
|
33
|
+
if (locator) {
|
|
34
|
+
const found = resolveAttemptLocator(snapshots, locator);
|
|
44
35
|
if (found) return found;
|
|
45
|
-
// 定位不到(
|
|
36
|
+
// 定位不到(locator 不在、快照未加载、旧格式数据):不开空 modal,页面照常渲染。
|
|
46
37
|
console.warn(unresolvedAttemptWarning(location.hash));
|
|
47
38
|
return null;
|
|
48
39
|
}
|
|
49
40
|
return resultFromUrl(snapshots);
|
|
50
41
|
}
|
|
51
42
|
|
|
52
|
-
export function App({ data, reportHtml }: { data: ViewData; reportHtml
|
|
43
|
+
export function App({ data, reportHtml }: { data: ViewData; reportHtml: ReportSlotHtml }) {
|
|
53
44
|
const snapshots = data.snapshots ?? [];
|
|
54
|
-
const hasReport = reportHtml !== undefined;
|
|
55
|
-
const rows = useMemo(() => buildRows(data), [data]);
|
|
56
45
|
const attempts = useMemo(() => flattenAttempts(snapshots), [snapshots]);
|
|
57
46
|
const [locale, setLocale] = useState<Locale>(() => detectLocale());
|
|
58
47
|
const t = useMemo(() => makeTranslator(locale), [locale]);
|
|
59
|
-
const [tab, setTab] = useState<Tab>(
|
|
60
|
-
const [sort, setSort] = useState<SortState>({ key: "passRate", dir: -1 });
|
|
61
|
-
const [query, setQuery] = useState("");
|
|
62
|
-
const [openRows, setOpenRows] = useState<Set<string>>(() => new Set());
|
|
63
|
-
const [selectedGroup, setSelectedGroup] = useState(() => {
|
|
64
|
-
const groups = [...new Set(rows.map((r) => r.group).filter(Boolean))].sort();
|
|
65
|
-
return groups[0] ?? null;
|
|
66
|
-
});
|
|
48
|
+
const [tab, setTab] = useState<Tab>("report");
|
|
67
49
|
const [modalResult, setModalResult] = useState<ViewResult | null>(() => modalResultFromLocation(snapshots));
|
|
68
50
|
// 当前 modal 的 hash 历史条目前面是否还有本页条目(本页 push 的 / 前进键回到的):
|
|
69
51
|
// true → UI 关闭走 history.back(),前进键还能重新打开;false(深链直接落地)→ 原地抹 hash,
|
|
@@ -75,25 +57,6 @@ export function App({ data, reportHtml }: { data: ViewData; reportHtml?: string
|
|
|
75
57
|
persistLocale(locale);
|
|
76
58
|
}, [locale]);
|
|
77
59
|
|
|
78
|
-
const openModal = useCallback((result: ViewResult) => {
|
|
79
|
-
setModalResult(result);
|
|
80
|
-
const ref = result.attemptRef;
|
|
81
|
-
// 旧格式烘焙的数据没有 attemptRef:modal 照常打开,只是这条 attempt 产不出可分享链接。
|
|
82
|
-
if (!ref) return;
|
|
83
|
-
try {
|
|
84
|
-
// pushState/replaceState 不触发 hashchange,不会和下面的监听器重复开合。
|
|
85
|
-
if (parseAttemptHash(location.hash)) {
|
|
86
|
-
// 已经在某条 attempt 深链上(防御浏览器导航竞态):替换当前条目,不叠历史。
|
|
87
|
-
history.replaceState(null, "", formatAttemptHash(ref));
|
|
88
|
-
} else {
|
|
89
|
-
history.pushState(null, "", formatAttemptHash(ref));
|
|
90
|
-
modalOwnsHistory.current = true;
|
|
91
|
-
}
|
|
92
|
-
} catch {
|
|
93
|
-
// file:// 等受限环境可能拒绝写 history;URL 同步失败不影响打开 modal。
|
|
94
|
-
}
|
|
95
|
-
}, []);
|
|
96
|
-
|
|
97
60
|
const closeModal = useCallback(() => {
|
|
98
61
|
setModalResult(null);
|
|
99
62
|
if (modalOwnsHistory.current) {
|
|
@@ -109,16 +72,16 @@ export function App({ data, reportHtml }: { data: ViewData; reportHtml?: string
|
|
|
109
72
|
}
|
|
110
73
|
}, []);
|
|
111
74
|
|
|
112
|
-
// 浏览器前进/后退、手改 hash、页内 attempt
|
|
75
|
+
// 浏览器前进/后退、手改 hash、页内 attempt 链接(含报告槽里的深链)统一从 hashchange 开合 modal。
|
|
113
76
|
useEffect(() => {
|
|
114
77
|
const onHashChange = () => {
|
|
115
|
-
const
|
|
116
|
-
if (!
|
|
78
|
+
const locator = parseAttemptHash(location.hash);
|
|
79
|
+
if (!locator) {
|
|
117
80
|
modalOwnsHistory.current = false;
|
|
118
81
|
setModalResult(null);
|
|
119
82
|
return;
|
|
120
83
|
}
|
|
121
|
-
const found =
|
|
84
|
+
const found = resolveAttemptLocator(snapshots, locator);
|
|
122
85
|
if (!found) {
|
|
123
86
|
console.warn(unresolvedAttemptWarning(location.hash));
|
|
124
87
|
setModalResult(null);
|
|
@@ -132,48 +95,6 @@ export function App({ data, reportHtml }: { data: ViewData; reportHtml?: string
|
|
|
132
95
|
return () => window.removeEventListener("hashchange", onHashChange);
|
|
133
96
|
}, [snapshots]);
|
|
134
97
|
|
|
135
|
-
const groupMap = useMemo<Map<string, ViewRow[]>>(() => buildGroupMap(rows), [rows]);
|
|
136
|
-
const pool = selectedGroup ? groupMap.get(selectedGroup) ?? [] : rows;
|
|
137
|
-
const filtered = useMemo(() => {
|
|
138
|
-
const q = query.trim().toLowerCase();
|
|
139
|
-
return pool
|
|
140
|
-
.filter((row: ViewRow) => {
|
|
141
|
-
if (!q) return true;
|
|
142
|
-
return [
|
|
143
|
-
row.label,
|
|
144
|
-
row.group || "",
|
|
145
|
-
row.experimentId || "",
|
|
146
|
-
row.agent,
|
|
147
|
-
row.model || "",
|
|
148
|
-
...(row.results ?? []).map((r: ViewResult) => r.id),
|
|
149
|
-
]
|
|
150
|
-
.join(" ")
|
|
151
|
-
.toLowerCase()
|
|
152
|
-
.includes(q);
|
|
153
|
-
})
|
|
154
|
-
.sort((a: ViewRow, b: ViewRow) => compareRows(a, b, sort.key) * sort.dir);
|
|
155
|
-
}, [pool, query, sort]);
|
|
156
|
-
|
|
157
|
-
const setSortKey = (key: SortKey) => {
|
|
158
|
-
setSort((prev) =>
|
|
159
|
-
prev.key === key ? { key, dir: prev.dir === 1 ? -1 : 1 } : { key, dir: key === "experiment" || key === "agent" ? 1 : -1 },
|
|
160
|
-
);
|
|
161
|
-
};
|
|
162
|
-
|
|
163
|
-
const toggleRow = (key: string) => {
|
|
164
|
-
setOpenRows((prev) => {
|
|
165
|
-
const next = new Set(prev);
|
|
166
|
-
if (next.has(key)) next.delete(key);
|
|
167
|
-
else next.add(key);
|
|
168
|
-
return next;
|
|
169
|
-
});
|
|
170
|
-
};
|
|
171
|
-
|
|
172
|
-
// hero 的官方数字:整体通过率是 MetricTable.data 常量维度的单行格子,其余来自 RunOverview.data。
|
|
173
|
-
const overallPassRate = data.overall?.rows[0]?.cells[CELL_KEYS.passRate];
|
|
174
|
-
const totals = data.overview?.totals;
|
|
175
|
-
const warnings = data.overview?.warnings ?? [];
|
|
176
|
-
|
|
177
98
|
return (
|
|
178
99
|
<Tabs value={tab} onValueChange={(v) => setTab(v as Tab)}>
|
|
179
100
|
<header className="topbar">
|
|
@@ -182,7 +103,7 @@ export function App({ data, reportHtml }: { data: ViewData; reportHtml?: string
|
|
|
182
103
|
<span>NiceEval</span>
|
|
183
104
|
</a>
|
|
184
105
|
<TabsList aria-label={t("nav.label")}>
|
|
185
|
-
{
|
|
106
|
+
{navItems.map((item) => (
|
|
186
107
|
<TabsTrigger key={item.id} value={item.id}>
|
|
187
108
|
{t(item.label)}
|
|
188
109
|
</TabsTrigger>
|
|
@@ -211,117 +132,31 @@ export function App({ data, reportHtml }: { data: ViewData; reportHtml?: string
|
|
|
211
132
|
<b>{t("hero.lastRun")}</b> {data.lastRunAt ? formatDateTime(data.lastRunAt, locale) : t("hero.noRuns")}
|
|
212
133
|
</span>
|
|
213
134
|
{data.composedRuns > 0 ? (
|
|
214
|
-
//
|
|
135
|
+
// 报告槽是跨 run 合成的现刻水位,hero 如实标注合成来源(几个 run)。
|
|
215
136
|
<span>{t("hero.composedFrom", { count: data.composedRuns })}</span>
|
|
216
137
|
) : null}
|
|
217
138
|
</div>
|
|
218
139
|
</section>
|
|
219
140
|
|
|
220
|
-
{
|
|
221
|
-
摆不摆(<DefaultReport /> / <RunOverview /> 都能把它们摆回来)。 */}
|
|
222
|
-
{!hasReport && (
|
|
223
|
-
<section className="summary" aria-label="Run summary">
|
|
224
|
-
<Metric label={t("metric.passRate")} value={overallPassRate?.display ?? "—"} />
|
|
225
|
-
<Metric label={t("metric.evalResults")} value={String(totals?.evals ?? 0)} />
|
|
226
|
-
<Metric label={t("metric.duration")} value={formatDuration(totals?.durationMs ?? 0)} />
|
|
227
|
-
<Metric label={t("metric.cost")} value={formatCost(totals?.costUSD ?? undefined)} />
|
|
228
|
-
</section>
|
|
229
|
-
)}
|
|
230
|
-
|
|
231
|
-
{!hasReport && warnings.length > 0 && (
|
|
232
|
-
// 选集警告(partial-coverage / stale-snapshot / synthetic-experiment-id):
|
|
233
|
-
// message 是挑选器渲染好的英文句子,原样打;data-kind 供样式与测试定位。
|
|
234
|
-
<section className="incompatible-banner selection-warnings" role="alert">
|
|
235
|
-
<b>{t("banner.warningsTitle")}</b>
|
|
236
|
-
<ul>
|
|
237
|
-
{warnings.map((w, i) => (
|
|
238
|
-
<li key={`${w.kind}-${i}`} data-kind={w.kind}>
|
|
239
|
-
<span className="ib-meta">{w.message}</span>
|
|
240
|
-
</li>
|
|
241
|
-
))}
|
|
242
|
-
</ul>
|
|
243
|
-
</section>
|
|
244
|
-
)}
|
|
141
|
+
<SkippedRunsBanner skippedRuns={data.skippedRuns ?? []} t={t} />
|
|
245
142
|
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
<ul>
|
|
250
|
-
{data.skippedRuns!.map((run) => (
|
|
251
|
-
<li key={run.dir} data-reason={run.reason}>
|
|
252
|
-
<span className="ib-dir">{run.dir}</span>
|
|
253
|
-
<span className="ib-meta">
|
|
254
|
-
{run.reason === "malformed"
|
|
255
|
-
? t("banner.skipped.malformed", { detail: run.detail ?? "?" })
|
|
256
|
-
: run.reason === "incomplete"
|
|
257
|
-
? t("banner.skipped.incomplete")
|
|
258
|
-
: run.producerName && run.producerName !== "niceeval"
|
|
259
|
-
? // 第三方 harness 写的落盘:如实报名字和版本,不拼 npx 命令。
|
|
260
|
-
t("banner.skipped.incompatibleForeign", {
|
|
261
|
-
name: run.producerName,
|
|
262
|
-
version: run.producerVersion ?? "?",
|
|
263
|
-
schemaVersion: run.schemaVersion,
|
|
264
|
-
})
|
|
265
|
-
: t("banner.skipped.incompatible", {
|
|
266
|
-
producer: run.producerVersion ?? "?",
|
|
267
|
-
schemaVersion: run.schemaVersion,
|
|
268
|
-
})}
|
|
269
|
-
</span>
|
|
270
|
-
{run.command ? <code>{run.command}</code> : null}
|
|
271
|
-
</li>
|
|
272
|
-
))}
|
|
273
|
-
</ul>
|
|
274
|
-
</section>
|
|
275
|
-
)}
|
|
276
|
-
|
|
277
|
-
{hasReport && (
|
|
278
|
-
<TabsContent value="report" id="tab-report">
|
|
279
|
-
{/* 报告槽:server 侧渲染好的静态 HTML(含 <Style> 产物),这里只摆放。
|
|
280
|
-
attempt 深链是普通 <a href="#/attempt/…">,经 hashchange 打开证据室弹窗。 */}
|
|
281
|
-
<div className="report-slot" dangerouslySetInnerHTML={{ __html: reportHtml }} />
|
|
282
|
-
</TabsContent>
|
|
283
|
-
)}
|
|
284
|
-
|
|
285
|
-
{!hasReport && (
|
|
286
|
-
<TabsContent value="experiments" id="tab-experiments">
|
|
287
|
-
<div className="section-head">
|
|
288
|
-
<h2>{t("section.experiments")}</h2>
|
|
289
|
-
</div>
|
|
290
|
-
<GroupSelector groupMap={groupMap} selectedGroup={selectedGroup} onSelect={setSelectedGroup} t={t} />
|
|
143
|
+
<TabsContent value="report" id="tab-report">
|
|
144
|
+
{/* 壳区:报告槽上方靠右的批量修复 prompt 按钮。失败清单从 viewData.snapshots
|
|
145
|
+
现算(latest 口径),默认报告与 --report 两种填充下都在。 */}
|
|
291
146
|
<div className="section-sub-head">
|
|
292
|
-
<span className="group-detail-label"
|
|
147
|
+
<span className="group-detail-label" />
|
|
293
148
|
<div className="controls">
|
|
294
|
-
<
|
|
295
|
-
className="search"
|
|
296
|
-
type="search"
|
|
297
|
-
placeholder={t("search.experiments")}
|
|
298
|
-
autoComplete="off"
|
|
299
|
-
value={query}
|
|
300
|
-
onChange={(e) => setQuery(e.target.value)}
|
|
301
|
-
/>
|
|
302
|
-
<CopyFixPrompt rows={filtered} t={t} />
|
|
149
|
+
<CopyFixPrompt snapshots={snapshots} t={t} />
|
|
303
150
|
</div>
|
|
304
151
|
</div>
|
|
305
|
-
{
|
|
306
|
-
|
|
307
|
-
<
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
openRows={openRows}
|
|
313
|
-
toggleRow={toggleRow}
|
|
314
|
-
openModal={openModal}
|
|
315
|
-
t={t}
|
|
316
|
-
/>
|
|
317
|
-
</>
|
|
318
|
-
) : (
|
|
319
|
-
<div className="empty">
|
|
320
|
-
{t("empty.summary")}
|
|
321
|
-
</div>
|
|
322
|
-
)}
|
|
152
|
+
{/* 报告槽:server 侧渲染好的静态 HTML(含 <Style> 产物),按当前界面语言摆放
|
|
153
|
+
对应语言的块;Selection 警告由报告槽内的 RunOverview 呈现,壳不设第二条通道。
|
|
154
|
+
attempt 深链是普通 <a href="#/attempt/…">,经 hashchange 打开证据室弹窗。 */}
|
|
155
|
+
<div
|
|
156
|
+
className="report-slot"
|
|
157
|
+
dangerouslySetInnerHTML={{ __html: reportHtml[locale] || reportHtml.en }}
|
|
158
|
+
/>
|
|
323
159
|
</TabsContent>
|
|
324
|
-
)}
|
|
325
160
|
|
|
326
161
|
<TabsContent value="runs">
|
|
327
162
|
<RunsView attempts={attempts} t={t} />
|
|
@@ -3,7 +3,7 @@ import type { ArtifactLoadState, T } from "../shared.ts";
|
|
|
3
3
|
import type { ViewResult } from "../types.ts";
|
|
4
4
|
import { artifactUrl } from "../lib/artifact-url.ts";
|
|
5
5
|
import { asEvents, asSources } from "../lib/guards.ts";
|
|
6
|
-
import {
|
|
6
|
+
import { verdictClass, verdictLabel } from "../lib/verdict.ts";
|
|
7
7
|
import { CodeView, NoSourceBody } from "./CodeView.tsx";
|
|
8
8
|
import { CopyAttemptPrompt } from "./CopyControls.tsx";
|
|
9
9
|
import { LazyArtifact } from "./LazyArtifact.tsx";
|
|
@@ -15,13 +15,13 @@ export function AttemptModal({ result, onClose, t }: { result: ViewResult; onClo
|
|
|
15
15
|
const base = result.artifactBase;
|
|
16
16
|
const [data, setData] = useState<ArtifactLoadState>({ sources: null, events: null, status: "loading" });
|
|
17
17
|
|
|
18
|
-
// Esc / 焦点陷阱 / 背景滚动锁 / 点遮罩关闭 都交给 Radix Dialog
|
|
18
|
+
// Esc / 焦点陷阱 / 背景滚动锁 / 点遮罩关闭 都交给 Radix Dialog;这里只保留 artifact 拉取。
|
|
19
19
|
useEffect(() => {
|
|
20
20
|
if (!base) { setData({ sources: null, events: null, status: "none" }); return; }
|
|
21
21
|
let alive = true;
|
|
22
22
|
const grab = (name: string, has?: boolean): Promise<unknown> =>
|
|
23
23
|
has
|
|
24
|
-
? fetch(artifactUrl(`${base}/${name}`))
|
|
24
|
+
? fetch( artifactUrl(`${base}/${name}`))
|
|
25
25
|
.then((r) => (r.ok ? r.json() : null))
|
|
26
26
|
.catch(() => null)
|
|
27
27
|
: Promise.resolve(null);
|
|
@@ -31,7 +31,7 @@ export function AttemptModal({ result, onClose, t }: { result: ViewResult; onClo
|
|
|
31
31
|
return () => { alive = false; };
|
|
32
32
|
}, [base, result.hasSources, result.hasEvents]);
|
|
33
33
|
|
|
34
|
-
const
|
|
34
|
+
const verdict = result.verdict;
|
|
35
35
|
const hasCode = Boolean(data.sources?.length);
|
|
36
36
|
|
|
37
37
|
return (
|
|
@@ -39,7 +39,7 @@ export function AttemptModal({ result, onClose, t }: { result: ViewResult; onClo
|
|
|
39
39
|
<DialogContent aria-describedby={undefined}>
|
|
40
40
|
<div className="flex min-w-0 shrink-0 items-center justify-between gap-3 border-b border-line px-[18px] pb-[11px] pt-[13px]">
|
|
41
41
|
<div className="flex min-w-0 flex-col gap-[3px]">
|
|
42
|
-
<Badge tone={
|
|
42
|
+
<Badge tone={verdictClass(verdict)}>{verdictLabel(verdict, t)}</Badge>
|
|
43
43
|
<DialogTitle asChild>
|
|
44
44
|
<span className="truncate text-sm font-[640] text-text">{result.id}</span>
|
|
45
45
|
</DialogTitle>
|
|
@@ -59,7 +59,7 @@ export function AttemptModal({ result, onClose, t }: { result: ViewResult; onClo
|
|
|
59
59
|
{hasCode ? (
|
|
60
60
|
<CodeView sources={data.sources ?? []} events={data.events || []} assertions={allAssertions} t={t} />
|
|
61
61
|
) : data.status !== "loading" ? (
|
|
62
|
-
// hasSources 为真却取不到 →
|
|
62
|
+
// hasSources 为真却取不到 → 源码捕获过,是 artifact 文件在当前托管里缺失;和「从未捕获」分开提示。
|
|
63
63
|
<NoSourceBody
|
|
64
64
|
assertions={allAssertions}
|
|
65
65
|
events={data.events || []}
|
|
@@ -6,7 +6,7 @@ import { highlightTs, indexAsserts, indexTurns, locKey } from "../lib/transcript
|
|
|
6
6
|
import { formatScore } from "../lib/format.ts";
|
|
7
7
|
import { InputBlock, ToolBlock, Transcript } from "./Transcript.tsx";
|
|
8
8
|
|
|
9
|
-
/** soft 断言没过阈值不影响
|
|
9
|
+
/** soft 断言没过阈值不影响 verdict,颜色上跟 gate 失败(红)区分开,用 warn(黄)。 */
|
|
10
10
|
function assertTone(a: Assertion): "good" | "warn" | "bad" {
|
|
11
11
|
if (a.passed) return "good";
|
|
12
12
|
return a.severity === "soft" ? "warn" : "bad";
|
|
@@ -118,7 +118,7 @@ export function CodeLine({
|
|
|
118
118
|
}) {
|
|
119
119
|
const hasReply = !!turn;
|
|
120
120
|
const hasAsserts = !!(asserts && asserts.length);
|
|
121
|
-
// 只有 gate 断言没过才算这一行真的"fail";只剩 soft 断言没过阈值时是"warn"(不影响
|
|
121
|
+
// 只有 gate 断言没过才算这一行真的"fail";只剩 soft 断言没过阈值时是"warn"(不影响 verdict)。
|
|
122
122
|
const status = hasAsserts
|
|
123
123
|
? asserts?.every((a: Assertion) => a.passed)
|
|
124
124
|
? "pass"
|
|
@@ -1,62 +1,47 @@
|
|
|
1
1
|
import React, { useState } from "react";
|
|
2
2
|
import { Check, Copy } from "lucide-react";
|
|
3
3
|
import type { T } from "../shared.ts";
|
|
4
|
-
import type { ViewResult,
|
|
5
|
-
import {
|
|
6
|
-
|
|
7
|
-
export function CopyReason({ text, t }: { text: string; t: T }) {
|
|
8
|
-
const [copied, setCopied] = useState(false);
|
|
9
|
-
const copy = async (event: React.MouseEvent<HTMLButtonElement>) => {
|
|
10
|
-
event.stopPropagation();
|
|
11
|
-
try {
|
|
12
|
-
await copyText(text);
|
|
13
|
-
setCopied(true);
|
|
14
|
-
setTimeout(() => setCopied(false), 1200);
|
|
15
|
-
} catch {
|
|
16
|
-
setCopied(false);
|
|
17
|
-
}
|
|
18
|
-
};
|
|
19
|
-
return (
|
|
20
|
-
<button className={`copy-reason${copied ? " is-copied" : ""}`} onClick={copy} aria-label={t("action.copyReason")} title={t("action.copyReason")}>
|
|
21
|
-
{copied ? <Check aria-hidden="true" /> : <Copy aria-hidden="true" />}
|
|
22
|
-
</button>
|
|
23
|
-
);
|
|
24
|
-
}
|
|
4
|
+
import type { ViewResult, ViewSnapshot } from "../types.ts";
|
|
5
|
+
import { snapshotLabel } from "../lib/rows.ts";
|
|
6
|
+
import { failingAssertions, reasonFor } from "../lib/verdict.ts";
|
|
25
7
|
|
|
26
8
|
/** 修复 prompt 的一条失败条目;路径均相对 view 输入根(默认 `.niceeval/`)。 */
|
|
27
9
|
export interface FixPromptEntry {
|
|
28
10
|
experiment: string;
|
|
29
11
|
evalId: string;
|
|
30
|
-
|
|
12
|
+
verdict: string;
|
|
31
13
|
reason: string;
|
|
32
14
|
artifactBase?: string;
|
|
33
|
-
|
|
15
|
+
resultPath?: string;
|
|
34
16
|
}
|
|
35
17
|
|
|
36
18
|
export function toFixPromptEntry(r: ViewResult, experimentLabel: string): FixPromptEntry {
|
|
37
19
|
return {
|
|
38
20
|
experiment: r.experimentId ?? experimentLabel,
|
|
39
21
|
evalId: r.id,
|
|
40
|
-
|
|
22
|
+
verdict: r.verdict,
|
|
41
23
|
reason: reasonFor(r, failingAssertions(r)),
|
|
42
24
|
artifactBase: r.artifactBase,
|
|
43
|
-
|
|
25
|
+
// result.json 与其它 artifact(events.json 等)同目录落盘,resultPath 因此复用同一个
|
|
26
|
+
// artifactBase(不再靠 attemptRef 的 {snapshot, attempt} 两段拼——locator 是不透明的,
|
|
27
|
+
// 浏览器端解不出磁盘路径)。
|
|
28
|
+
resultPath: r.artifactBase ? `${r.artifactBase}/result.json` : undefined,
|
|
44
29
|
};
|
|
45
30
|
}
|
|
46
31
|
|
|
47
32
|
/**
|
|
48
33
|
* 学 Next.js 16.3 的「Copy prompt」:把失败打包成一段可直接粘给 coding agent 的修复
|
|
49
|
-
* prompt——失败清单 +
|
|
34
|
+
* prompt——失败清单 + artifact 路径 + 先读随包文档 / 判断缺陷在哪一侧 / 重跑验证的步骤。
|
|
50
35
|
* prompt 面向 agent,固定英文;按钮文案走界面 i18n。
|
|
51
36
|
*/
|
|
52
37
|
export function buildFixPrompt(entries: FixPromptEntry[]): string {
|
|
53
38
|
const failures = entries
|
|
54
39
|
.map((e, i) =>
|
|
55
40
|
[
|
|
56
|
-
`${i + 1}. eval "${e.evalId}" [experiment ${e.experiment}] — ${e.
|
|
41
|
+
`${i + 1}. eval "${e.evalId}" [experiment ${e.experiment}] — ${e.verdict}`,
|
|
57
42
|
e.reason ? ` reason: ${e.reason}` : null,
|
|
58
43
|
e.artifactBase ? ` artifacts: ${e.artifactBase}/` : null,
|
|
59
|
-
e.
|
|
44
|
+
e.resultPath ? ` result: ${e.resultPath}` : null,
|
|
60
45
|
]
|
|
61
46
|
.filter(Boolean)
|
|
62
47
|
.join("\n"),
|
|
@@ -74,18 +59,29 @@ export function buildFixPrompt(entries: FixPromptEntry[]): string {
|
|
|
74
59
|
"2. The paths above are relative to the results directory (default `.niceeval/`). For each failure, read `events.json` in its artifacts directory — the full agent transcript including tool calls — plus `trace.json` (execution trace) and `diff.json` (workspace diff) when present, to see what actually happened.",
|
|
75
60
|
"3. Decide which side the defect is on: the program under test, or the eval itself (over-tight assertion, wrong fixture, missing setup). Fix that side; do not weaken assertions just to turn the run green.",
|
|
76
61
|
`4. Re-run: \`npx niceeval exp ${experiments || "<experiment>"} <eval-id-prefix>\`. Already-passing evals are skipped by the fingerprint cache; pass \`--force\` to re-run everything.`,
|
|
77
|
-
"5.
|
|
62
|
+
"5. Run `npx niceeval show` and confirm these failures are gone.",
|
|
78
63
|
].join("\n");
|
|
79
64
|
}
|
|
80
65
|
|
|
81
|
-
|
|
66
|
+
/**
|
|
67
|
+
* 报告槽同款口径的失败清单:每个 experiment 最新一次快照(latest 标记;快照明细已在
|
|
68
|
+
* server 侧跨快照去重)里的 failed / errored attempt,从 viewData.snapshots 现算——
|
|
69
|
+
* 默认报告与 --report 两种填充下按钮都在,不依赖任何统计产物。
|
|
70
|
+
*/
|
|
71
|
+
export function fixPromptEntries(snapshots: ViewSnapshot[]): FixPromptEntry[] {
|
|
72
|
+
return snapshots
|
|
73
|
+
.filter((s) => s.latest)
|
|
74
|
+
.flatMap((snapshot) =>
|
|
75
|
+
snapshot.results
|
|
76
|
+
.filter((r: ViewResult) => r.verdict === "failed" || r.verdict === "errored")
|
|
77
|
+
.map((r: ViewResult) => toFixPromptEntry(r, snapshotLabel(snapshot))),
|
|
78
|
+
);
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
export function CopyFixPrompt({ snapshots, t }: { snapshots: ViewSnapshot[]; t: T }) {
|
|
82
82
|
const [copied, setCopied] = useState(false);
|
|
83
83
|
|
|
84
|
-
const entries =
|
|
85
|
-
(row.results ?? [])
|
|
86
|
-
.filter((r: ViewResult) => r.outcome === "failed" || r.outcome === "errored")
|
|
87
|
-
.map((r: ViewResult) => toFixPromptEntry(r, row.label)),
|
|
88
|
-
);
|
|
84
|
+
const entries = fixPromptEntries(snapshots);
|
|
89
85
|
|
|
90
86
|
if (!entries.length) return null;
|
|
91
87
|
|
|
@@ -111,7 +107,7 @@ export function CopyFixPrompt({ rows, t }: { rows: ViewRow[]; t: T }) {
|
|
|
111
107
|
/** attempt 弹窗里的单条版:只打包当前 attempt 的失败,供逐条转交 agent。 */
|
|
112
108
|
export function CopyAttemptPrompt({ result, t }: { result: ViewResult; t: T }) {
|
|
113
109
|
const [copied, setCopied] = useState(false);
|
|
114
|
-
if (result.
|
|
110
|
+
if (result.verdict !== "failed" && result.verdict !== "errored") return null;
|
|
115
111
|
|
|
116
112
|
const copy = async (event: React.MouseEvent<HTMLButtonElement>) => {
|
|
117
113
|
event.stopPropagation();
|