niceeval 0.6.2 → 0.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/INDEX.md +23 -23
- package/README.zh.md +6 -6
- package/dist/agents/types.d.ts +2 -2
- package/dist/i18n/zh-CN.d.ts +3 -3
- package/dist/report/aggregate.d.ts +32 -26
- package/dist/report/aggregate.js +157 -76
- package/dist/report/built-in/index.d.ts +2 -0
- package/dist/report/built-in/index.js +8 -0
- package/dist/report/components.d.ts +91 -164
- package/dist/report/components.js +377 -114
- package/dist/report/compute.d.ts +86 -73
- package/dist/report/compute.js +592 -432
- package/dist/report/flag.d.ts +28 -17
- package/dist/report/flag.js +86 -16
- package/dist/report/format.d.ts +11 -11
- package/dist/report/format.js +17 -15
- package/dist/report/index.d.ts +16 -17
- package/dist/report/index.js +20 -22
- package/dist/report/load.js +3 -2
- package/dist/report/locale.d.ts +49 -34
- package/dist/report/locale.js +106 -58
- package/dist/report/metrics.d.ts +10 -3
- package/dist/report/metrics.js +46 -12
- package/dist/report/primitives.d.ts +42 -15
- package/dist/report/primitives.js +135 -26
- package/dist/report/react/AttemptList.d.ts +10 -8
- package/dist/report/react/AttemptList.js +18 -10
- package/dist/report/react/DeltaTable.js +19 -18
- package/dist/report/react/EvalList.d.ts +3 -3
- package/dist/report/react/EvalList.js +0 -0
- package/dist/report/react/ExperimentComparison.d.ts +4 -2
- package/dist/report/react/ExperimentComparison.js +5 -4
- package/dist/report/react/ExperimentList.d.ts +3 -3
- package/dist/report/react/ExperimentList.js +16 -15
- package/dist/report/react/MetricBars.js +5 -4
- package/dist/report/react/MetricLine.js +12 -5
- package/dist/report/react/MetricMatrix.js +1 -1
- package/dist/report/react/MetricScatter.js +54 -17
- package/dist/report/react/MetricTable.js +2 -12
- package/dist/report/react/ScopeSummary.d.ts +10 -0
- package/dist/report/react/ScopeSummary.js +17 -0
- package/dist/report/react/Scoreboard.js +6 -6
- package/dist/report/react/cell.js +2 -2
- package/dist/report/react/fixtures.d.ts +5 -9
- package/dist/report/react/fixtures.js +105 -149
- package/dist/report/react/index.d.ts +15 -5
- package/dist/report/react/index.js +18 -7
- package/dist/report/report.d.ts +137 -20
- package/dist/report/report.js +261 -34
- package/dist/report/text/faces.d.ts +17 -19
- package/dist/report/text/faces.js +225 -157
- package/dist/report/text/plot.js +1 -1
- package/dist/report/text/table.js +2 -2
- package/dist/report/tree.d.ts +90 -40
- package/dist/report/tree.js +252 -94
- package/dist/report/types.d.ts +245 -300
- package/dist/report/types.js +4 -3
- package/dist/report/web.d.ts +21 -5
- package/dist/report/web.js +42 -16
- package/dist/results/select.d.ts +38 -16
- package/dist/results/select.js +73 -25
- package/dist/results/types.d.ts +38 -14
- package/dist/shared/aggregate.d.ts +3 -2
- package/dist/shared/aggregate.js +5 -4
- package/docs-site/zh/README.md +44 -0
- package/docs-site/zh/examples/ai-agent-application.mdx +63 -0
- package/docs-site/zh/examples/coding-agent-extensions.mdx +57 -0
- package/docs-site/zh/examples/index.mdx +50 -0
- package/docs-site/zh/{concepts → explanation}/adapter.mdx +11 -11
- package/docs-site/zh/{concepts → explanation}/assert.mdx +7 -7
- package/docs-site/zh/{concepts → explanation}/drive.mdx +8 -8
- package/docs-site/zh/{concepts → explanation}/evals.mdx +4 -4
- package/docs-site/zh/{concepts → explanation}/experiment.mdx +8 -8
- package/docs-site/zh/{concepts → explanation}/hitl.mdx +8 -8
- package/docs-site/zh/{concepts → explanation}/judge.mdx +5 -5
- package/docs-site/zh/{concepts → explanation}/overview.mdx +5 -5
- package/docs-site/zh/{guides → explanation}/runner.mdx +1 -1
- package/docs-site/zh/{concepts → explanation}/tier.mdx +6 -6
- package/docs-site/zh/{guides → how-to}/agent-feedback-loop.mdx +7 -7
- package/docs-site/zh/{guides → how-to}/authoring.mdx +2 -2
- package/docs-site/zh/{guides → how-to}/connect-otel.mdx +6 -6
- package/docs-site/zh/{guides → how-to}/connect-your-agent.mdx +18 -18
- package/docs-site/zh/{guides → how-to}/custom-reports.mdx +6 -6
- package/docs-site/zh/{guides → how-to}/experiments.mdx +3 -3
- package/docs-site/zh/{guides → how-to}/publish-report.mdx +2 -2
- package/docs-site/zh/{guides → how-to}/sandbox-agent.mdx +2 -2
- package/docs-site/zh/{guides → how-to}/sandbox-providers.mdx +1 -1
- package/docs-site/zh/{guides → how-to}/viewing-results.mdx +6 -6
- package/docs-site/zh/{guides → how-to}/write-experiment.mdx +3 -3
- package/docs-site/zh/{guides → how-to}/write-send.mdx +13 -13
- package/docs-site/zh/index.mdx +23 -25
- package/docs-site/zh/introduction.mdx +8 -8
- package/docs-site/zh/reference/builtin-agents.mdx +5 -5
- package/docs-site/zh/reference/capabilities.mdx +6 -6
- package/docs-site/zh/reference/cli.mdx +9 -7
- package/docs-site/zh/reference/define-agent.mdx +1 -1
- package/docs-site/zh/reference/events.mdx +3 -3
- package/docs-site/zh/{guides → reference}/official-adapters.mdx +7 -7
- package/docs-site/zh/{guides → reference}/report-components.mdx +5 -5
- package/docs-site/zh/{guides → reference}/results-data.mdx +5 -5
- package/docs-site/zh/{guides → troubleshooting}/debug-sandbox.mdx +2 -2
- package/docs-site/zh/{guides → troubleshooting}/debugging.mdx +4 -2
- package/docs-site/zh/{quickstart.mdx → tutorials/quickstart.mdx} +5 -17
- package/package.json +6 -2
- package/src/agents/index.ts +2 -2
- package/src/agents/openai-compat.ts +1 -1
- package/src/agents/streaming.ts +2 -2
- package/src/agents/types.ts +3 -3
- package/src/cli.ts +42 -23
- package/src/context/context.ts +1 -1
- package/src/context/session.test.ts +1 -1
- package/src/context/session.ts +1 -1
- package/src/i18n/en.ts +18 -16
- package/src/i18n/zh-CN.ts +16 -15
- package/src/report/aggregate.ts +175 -87
- package/src/report/built-in/index.tsx +9 -0
- package/src/report/components.tsx +625 -285
- package/src/report/compute.ts +717 -515
- package/src/report/dual-render.test.tsx +738 -1148
- package/src/report/flag.ts +97 -33
- package/src/report/format.ts +18 -22
- package/src/report/index.ts +113 -58
- package/src/report/load.ts +3 -2
- package/src/report/locale.ts +120 -69
- package/src/report/metrics.ts +42 -12
- package/src/report/primitives.tsx +190 -45
- package/src/report/react/AttemptList.tsx +32 -20
- package/src/report/react/DeltaTable.tsx +63 -45
- package/src/report/react/EvalList.tsx +0 -0
- package/src/report/react/ExperimentComparison.tsx +12 -7
- package/src/report/react/ExperimentList.tsx +38 -26
- package/src/report/react/MetricBars.tsx +5 -4
- package/src/report/react/MetricLine.tsx +13 -8
- package/src/report/react/MetricMatrix.tsx +2 -2
- package/src/report/react/MetricScatter.tsx +74 -20
- package/src/report/react/MetricTable.tsx +4 -76
- package/src/report/react/ScopeSummary.tsx +86 -0
- package/src/report/react/Scoreboard.tsx +28 -10
- package/src/report/react/cell.tsx +2 -2
- package/src/report/react/enhance.js +57 -5
- package/src/report/react/fixtures.ts +109 -156
- package/src/report/react/index.tsx +24 -39
- package/src/report/react/render.test.tsx +139 -104
- package/src/report/react/styles.css +181 -91
- package/src/report/report.test.ts +761 -1031
- package/src/report/report.ts +425 -47
- package/src/report/text/faces.ts +257 -164
- package/src/report/text/plot.ts +1 -1
- package/src/report/text/table.ts +2 -2
- package/src/report/tree.ts +362 -104
- package/src/report/types.ts +257 -287
- package/src/report/web.ts +63 -20
- package/src/results/attempt-evidence.test.ts +4 -4
- package/src/results/attempt-evidence.ts +5 -5
- package/src/results/copy.ts +6 -6
- package/src/results/host-equivalence.test.ts +26 -14
- package/src/results/index.ts +10 -4
- package/src/results/open.ts +8 -4
- package/src/results/results.test.ts +4 -3
- package/src/results/select.ts +104 -34
- package/src/results/types.ts +36 -14
- package/src/runner/feedback/human.test.ts +1 -1
- package/src/runner/run.ts +1 -1
- package/src/sandbox/cli-commands.ts +2 -2
- package/src/scoring/judge.test.ts +1 -1
- package/src/shared/aggregate.ts +5 -4
- package/src/show/compose.ts +50 -67
- package/src/show/index.ts +107 -56
- package/src/show/render.ts +43 -27
- package/src/show/report-host.test.ts +188 -0
- package/src/show/report-host.ts +375 -0
- package/src/show/show.test.ts +86 -36
- package/src/view/app/App.test.tsx +69 -0
- package/src/view/app/App.tsx +144 -48
- package/src/view/app/components/AttemptModal.tsx +324 -63
- package/src/view/app/components/CodeView.tsx +10 -4
- package/src/view/app/i18n.ts +31 -17
- package/src/view/app/lib/artifact-url.ts +14 -3
- package/src/view/app/main.tsx +13 -8
- package/src/view/app/pages/{RunsPage.tsx → AttemptsPage.tsx} +6 -6
- package/src/view/app/types.ts +4 -1
- package/src/view/artifact-serving.test.ts +21 -1
- package/src/view/client-dist/app.css +1 -1
- package/src/view/client-dist/app.js +6 -6
- package/src/view/data.test.ts +9 -3
- package/src/view/data.ts +145 -49
- package/src/view/index.ts +48 -44
- package/src/view/server.ts +35 -15
- package/src/view/shared/types.ts +34 -5
- package/src/view/styles.css +224 -0
- package/src/view/view-report.test.ts +161 -57
- package/dist/report/built-ins/experiment-comparison.d.ts +0 -39
- package/dist/report/built-ins/experiment-comparison.js +0 -119
- package/dist/report/built-ins/index.d.ts +0 -2
- package/dist/report/built-ins/index.js +0 -2
- package/dist/report/react/GroupSummary.d.ts +0 -8
- package/dist/report/react/GroupSummary.js +0 -8
- package/dist/report/react/RunOverview.d.ts +0 -8
- package/dist/report/react/RunOverview.js +0 -12
- package/docs-site/zh/example/ai-agent-application.mdx +0 -152
- package/docs-site/zh/example/claude-code-codex-plugin.mdx +0 -167
- package/docs-site/zh/example/claude-code-codex-skill.mdx +0 -152
- package/docs-site/zh/example/showcase.mdx +0 -39
- package/src/report/built-in-user-parity.test.tsx +0 -597
- package/src/report/built-ins/experiment-comparison.tsx +0 -179
- package/src/report/built-ins/index.ts +0 -7
- package/src/report/react/GroupSummary.tsx +0 -66
- package/src/report/react/RunOverview.tsx +0 -109
- /package/docs-site/zh/{example/tier1-ai-sdk-v7.mdx → examples/integrations/ai-sdk-v7.mdx} +0 -0
- /package/docs-site/zh/{example/tier1-claude-sdk.mdx → examples/integrations/claude-sdk.mdx} +0 -0
- /package/docs-site/zh/{example/tier1-codex-sdk.mdx → examples/integrations/codex-sdk.mdx} +0 -0
- /package/docs-site/zh/{example/tier1-langgraph.mdx → examples/integrations/langgraph.mdx} +0 -0
- /package/docs-site/zh/{example/tier1-pi-sdk.mdx → examples/integrations/pi-sdk.mdx} +0 -0
- /package/docs-site/zh/{guides → how-to}/ci-integration.mdx +0 -0
- /package/docs-site/zh/{guides → how-to}/dataset-fanout.mdx +0 -0
- /package/docs-site/zh/{guides → how-to}/fixtures.mdx +0 -0
- /package/docs-site/zh/{guides → how-to}/reporters.mdx +0 -0
- /package/docs-site/zh/{guides → how-to}/scoring-guide.mdx +0 -0
|
@@ -0,0 +1,375 @@
|
|
|
1
|
+
// 报告装载的宿主联系面:show 与 view 共用(view 从这里 import;两个宿主对同一份
|
|
2
|
+
// defineReport 产物必须得到同一个规范化结果,见 docs/feature/reports/library/shell.md)。
|
|
3
|
+
//
|
|
4
|
+
// 契约(docs/feature/reports/library/shell.md「行为约束」/ architecture.md「外壳与页:装载规范化」):
|
|
5
|
+
// - `--report` 文件默认导出恒为 defineReport 产物;裸宿主装载 `niceeval/report/built-in` 的默认导出,
|
|
6
|
+
// 走同一条「装载 → resolve → validate → render」管线。
|
|
7
|
+
// - 装载规范化唯一产物是「外壳 + 非空页列表」:`defineReport(树)` ≡ `{ content: 树 }` ≡
|
|
8
|
+
// `pages: [{ id: "report", title: 内置页名, content: 树 }]`。
|
|
9
|
+
// - 标题回退单点:def.title → Scope 中唯一且相同(LocalizedText 深相等)的非空快照 name →
|
|
10
|
+
// 内置文案「Eval 运行结果 / Eval Results」。落点是首页 hero、浏览器标题与 show 页索引标题行;
|
|
11
|
+
// 页头品牌位恒为 NiceEval 字标,不归 title。
|
|
12
|
+
// - LocalizedText 回退:当前 locale → en → 按 locale 键字典序的第一个非空值。
|
|
13
|
+
//
|
|
14
|
+
// ⚠ 集成状态:src/report/** 正被并行重写(plan/reports-redesign-implementation.md)。这里把宿主
|
|
15
|
+
// 需要的新报告 runtime API 收敛成唯一联系面,并对「新 dist 尚未产出」提供旧 API 桥接
|
|
16
|
+
// (旧 ReportDefinition 是 build 函数形态,恒为单页)。集成阶段的对接点(搜索 INTEGRATION):
|
|
17
|
+
// 1. `dist/report/built-in/index.js` 默认导出(内建报告入口;恒为新格式,无桥接——
|
|
18
|
+
// 它随本包源码一起构建,不存在旧产物)。
|
|
19
|
+
// 2. `normalizeReport(definition)`:装载规范化(外壳 + 非空页列表);旧桥接对 build 形态自行包单页。
|
|
20
|
+
// 3. `renderReportTreeToText(tree, ctx, options)` / `renderReportTreeToStaticHtml(tree, ctx, options)`:
|
|
21
|
+
// 逐页渲染入口(ctx = { scope, results, report });旧桥接调 renderReportToText /
|
|
22
|
+
// renderReportToStaticHtml(definition 整体)。
|
|
23
|
+
|
|
24
|
+
import type { Results } from "../results/index.ts";
|
|
25
|
+
import type { LocalizedText } from "../types.ts";
|
|
26
|
+
|
|
27
|
+
/** 页:宿主寻址单位(`--page <id>` / `#/page/<id>`)。content 是报告树(旧桥接下是整个旧 definition)。 */
|
|
28
|
+
export interface HostReportPage {
|
|
29
|
+
id: string;
|
|
30
|
+
title: LocalizedText;
|
|
31
|
+
content: unknown;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
export interface HostReportLink {
|
|
35
|
+
label: LocalizedText;
|
|
36
|
+
href: string;
|
|
37
|
+
/**
|
|
38
|
+
* 可选内联 SVG 字标(shell.md「字段穷尽」):web 面渲染在 label 前,静态导出原样内联;
|
|
39
|
+
* 不收组件(外壳声明经 viewData 序列化边界);show 不消费。
|
|
40
|
+
*/
|
|
41
|
+
icon?: { svg: string };
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/** `{src}` 与 `{inline}` 两种形态不可同时出现(shell.md「字段穷尽」)。 */
|
|
45
|
+
export type HostReportAsset = { src: string; inline?: never } | { inline: string; src?: never };
|
|
46
|
+
|
|
47
|
+
/** 装载规范化产物:外壳 + 非空页列表。show 只消费 title / pages;其余是 web 面属性。 */
|
|
48
|
+
export interface HostReport {
|
|
49
|
+
title?: LocalizedText;
|
|
50
|
+
links: HostReportLink[];
|
|
51
|
+
footer?: LocalizedText;
|
|
52
|
+
scripts: HostReportAsset[];
|
|
53
|
+
styles: HostReportAsset[];
|
|
54
|
+
pages: HostReportPage[];
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/** 单页缩写展开出的唯一页使用内置页名(shell.md:「报告 / Report」)。 */
|
|
58
|
+
export const BUILT_IN_PAGE_TITLE: LocalizedText = { en: "Report", "zh-CN": "报告" };
|
|
59
|
+
export const SINGLE_PAGE_ID = "report";
|
|
60
|
+
|
|
61
|
+
/** 标题回退链的终点:内置文案「Eval 运行结果 / Eval Results」(shell.md「行为约束」)。 */
|
|
62
|
+
export const BUILT_IN_REPORT_TITLE: LocalizedText = { en: "Eval Results", "zh-CN": "Eval 运行结果" };
|
|
63
|
+
|
|
64
|
+
/** 规范化报告声明的只读注入(组合组件 ctx.report;ReportMeta,见 library/layout.md)。 */
|
|
65
|
+
export interface HostReportMeta {
|
|
66
|
+
title: LocalizedText;
|
|
67
|
+
links: HostReportLink[];
|
|
68
|
+
footer?: LocalizedText;
|
|
69
|
+
pages: { id: string; title: LocalizedText }[];
|
|
70
|
+
pageId: string;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/** 可预期的装载用户错误(与 ReportLoadError 同待遇:打一句直说问题与下一步,不抛堆栈)。 */
|
|
74
|
+
export class HostReportError extends Error {}
|
|
75
|
+
|
|
76
|
+
// ───────────────────────── LocalizedText ─────────────────────────
|
|
77
|
+
|
|
78
|
+
/**
|
|
79
|
+
* LocalizedText 的确定回退(shell.md「行为约束」):当前 locale → en → 按 locale 键字典序的
|
|
80
|
+
* 第一个非空值。对象没有任何非空值时返回 undefined(装载期应当已报错,渲染兜底不再抛)。
|
|
81
|
+
*/
|
|
82
|
+
export function localizeText(text: LocalizedText | undefined, locale: string): string | undefined {
|
|
83
|
+
if (text === undefined) return undefined;
|
|
84
|
+
if (typeof text === "string") return text || undefined;
|
|
85
|
+
const exact = text[locale];
|
|
86
|
+
if (exact) return exact;
|
|
87
|
+
if (text.en) return text.en;
|
|
88
|
+
for (const key of Object.keys(text).sort()) {
|
|
89
|
+
if (text[key]) return text[key];
|
|
90
|
+
}
|
|
91
|
+
return undefined;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
/** LocalizedText 深相等:按字段值比较,对象键顺序不影响结果(shell.md「标题回退必须确定」)。 */
|
|
95
|
+
export function localizedTextEquals(a: LocalizedText, b: LocalizedText): boolean {
|
|
96
|
+
if (typeof a === "string" || typeof b === "string") return a === b;
|
|
97
|
+
const keysA = Object.keys(a).sort();
|
|
98
|
+
const keysB = Object.keys(b).sort();
|
|
99
|
+
if (keysA.length !== keysB.length) return false;
|
|
100
|
+
return keysA.every((key, i) => key === keysB[i] && a[key] === b[key]);
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
/** LocalizedText 是否有任何非空值(空串 / 全空对象都算空)。 */
|
|
104
|
+
function hasText(text: LocalizedText | undefined): text is LocalizedText {
|
|
105
|
+
if (text === undefined) return false;
|
|
106
|
+
if (typeof text === "string") return text.length > 0;
|
|
107
|
+
return Object.values(text).some((v) => typeof v === "string" && v.length > 0);
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
/**
|
|
111
|
+
* 标题回退单点(show 页索引标题与 view hero / 浏览器标题共用):def.title → Scope 中唯一且相同
|
|
112
|
+
* (深相等)的非空快照 name → 内置文案「Eval 运行结果 / Eval Results」。多个不同 name 时
|
|
113
|
+
* 不随机挑一个,直接落到内置文案。
|
|
114
|
+
*/
|
|
115
|
+
export function resolveReportTitle(
|
|
116
|
+
defTitle: LocalizedText | undefined,
|
|
117
|
+
snapshots: readonly { name?: LocalizedText }[],
|
|
118
|
+
): LocalizedText {
|
|
119
|
+
if (hasText(defTitle)) return defTitle;
|
|
120
|
+
let unique: LocalizedText | undefined;
|
|
121
|
+
for (const snapshot of snapshots) {
|
|
122
|
+
if (!hasText(snapshot.name)) continue;
|
|
123
|
+
if (unique === undefined) {
|
|
124
|
+
unique = snapshot.name;
|
|
125
|
+
} else if (!localizedTextEquals(unique, snapshot.name)) {
|
|
126
|
+
return BUILT_IN_REPORT_TITLE;
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
return unique ?? BUILT_IN_REPORT_TITLE;
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
/** 组合组件 ctx.report 的形状(走完回退链的 title;scripts / styles 不进)。 */
|
|
133
|
+
export function reportMetaFor(
|
|
134
|
+
report: HostReport,
|
|
135
|
+
pageId: string,
|
|
136
|
+
snapshots: readonly { name?: LocalizedText }[],
|
|
137
|
+
): HostReportMeta {
|
|
138
|
+
return {
|
|
139
|
+
title: resolveReportTitle(report.title, snapshots),
|
|
140
|
+
links: report.links,
|
|
141
|
+
...(report.footer !== undefined ? { footer: report.footer } : {}),
|
|
142
|
+
pages: report.pages.map((p) => ({ id: p.id, title: p.title })),
|
|
143
|
+
pageId,
|
|
144
|
+
};
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
// ───────────────────────── 装载与规范化 ─────────────────────────
|
|
148
|
+
|
|
149
|
+
/** 旧(集成前)ReportDefinition:build 函数形态,恒为单页。集成阶段删除。 */
|
|
150
|
+
interface LegacyReportDefinition {
|
|
151
|
+
build: (ctx: unknown) => unknown;
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
function isLegacyDefinition(value: unknown): value is LegacyReportDefinition {
|
|
155
|
+
return (
|
|
156
|
+
(typeof value === "object" || typeof value === "function") &&
|
|
157
|
+
value !== null &&
|
|
158
|
+
typeof (value as Partial<LegacyReportDefinition>).build === "function"
|
|
159
|
+
);
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
const PAGE_ID_PATTERN = /^[a-z0-9-]+$/;
|
|
163
|
+
|
|
164
|
+
/**
|
|
165
|
+
* 装载期校验 `ReportLink.icon`(shell.md「字段穷尽」):唯一合法形状是 `{ svg: string }`。
|
|
166
|
+
* 无类型 JS 传组件 / ReactNode / 裸字符串都在这里被完整用户反馈拒绝——外壳声明要过
|
|
167
|
+
* viewData 的序列化边界,ReactNode 过不去,可序列化是外壳契约的一部分。
|
|
168
|
+
*/
|
|
169
|
+
function assertLinkIcon(link: unknown, sourceLabel: string): void {
|
|
170
|
+
const icon = (link as { icon?: unknown })?.icon;
|
|
171
|
+
if (icon === undefined) return;
|
|
172
|
+
const svg = (icon as { svg?: unknown })?.svg;
|
|
173
|
+
if (typeof icon !== "object" || icon === null || typeof svg !== "string" || svg.length === 0) {
|
|
174
|
+
throw new HostReportError(
|
|
175
|
+
`${sourceLabel}: a link "icon" must be { svg: string } — an inline SVG string rendered before the label. ` +
|
|
176
|
+
"Components and React nodes are not accepted: the shell declaration crosses a serialization boundary. " +
|
|
177
|
+
'Write e.g. icon: { svg: "<svg …>…</svg>" }.',
|
|
178
|
+
);
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
/**
|
|
183
|
+
* 装载规范化:把 defineReport 产物折成「外壳 + 非空页列表」。三种入参写法
|
|
184
|
+
* (树 / `content:` / `pages:`)展开成同一形状;page id 的唯一性与字符纪律在这里校验
|
|
185
|
+
* (shell.md「校验分两期」的装载期部分)。
|
|
186
|
+
*/
|
|
187
|
+
export function normalizeHostReport(definition: unknown, sourceLabel: string): HostReport {
|
|
188
|
+
// INTEGRATION(2):新 defineReport 产物(kind === "report")。规范化后的外壳与页列表
|
|
189
|
+
// 若由新报告 runtime 导出的 normalizeReport 承担,这里改为直接调它;当前按 docs 的
|
|
190
|
+
// 字段穷尽读取产物自带的声明。
|
|
191
|
+
if (typeof definition === "object" && definition !== null && (definition as { kind?: unknown }).kind === "report") {
|
|
192
|
+
const def = definition as {
|
|
193
|
+
title?: LocalizedText;
|
|
194
|
+
links?: HostReportLink[];
|
|
195
|
+
footer?: LocalizedText;
|
|
196
|
+
scripts?: HostReportAsset[];
|
|
197
|
+
styles?: HostReportAsset[];
|
|
198
|
+
content?: unknown;
|
|
199
|
+
pages?: { id: string; title: LocalizedText; content: unknown }[];
|
|
200
|
+
};
|
|
201
|
+
const hasContent = def.content !== undefined;
|
|
202
|
+
const hasPages = def.pages !== undefined;
|
|
203
|
+
if (hasContent === hasPages) {
|
|
204
|
+
// content / pages 同缺或同给:装载期完整用户反馈,下一步是 content: <ExperimentComparison />。
|
|
205
|
+
throw new HostReportError(
|
|
206
|
+
`${sourceLabel}: a report declares exactly one of "content" or "pages" — ` +
|
|
207
|
+
(hasContent ? "it declares both. " : "it declares neither. ") +
|
|
208
|
+
`To render the built-in report content, write: content: <ExperimentComparison />.`,
|
|
209
|
+
);
|
|
210
|
+
}
|
|
211
|
+
const pages: HostReportPage[] = hasPages
|
|
212
|
+
? def.pages!.map((p) => ({ id: p.id, title: p.title, content: p.content }))
|
|
213
|
+
: [{ id: SINGLE_PAGE_ID, title: BUILT_IN_PAGE_TITLE, content: def.content }];
|
|
214
|
+
if (pages.length === 0) {
|
|
215
|
+
throw new HostReportError(`${sourceLabel}: "pages" must be a non-empty list of report pages.`);
|
|
216
|
+
}
|
|
217
|
+
const seen = new Set<string>();
|
|
218
|
+
for (const page of pages) {
|
|
219
|
+
if (!PAGE_ID_PATTERN.test(page.id)) {
|
|
220
|
+
throw new HostReportError(
|
|
221
|
+
`${sourceLabel}: page id "${page.id}" is invalid. Page ids use lowercase letters, digits and hyphens.`,
|
|
222
|
+
);
|
|
223
|
+
}
|
|
224
|
+
if (seen.has(page.id)) {
|
|
225
|
+
throw new HostReportError(`${sourceLabel}: duplicate page id "${page.id}". Page ids must be unique per file.`);
|
|
226
|
+
}
|
|
227
|
+
seen.add(page.id);
|
|
228
|
+
}
|
|
229
|
+
for (const link of def.links ?? []) assertLinkIcon(link, sourceLabel);
|
|
230
|
+
return {
|
|
231
|
+
...(def.title !== undefined ? { title: def.title } : {}),
|
|
232
|
+
links: def.links ?? [],
|
|
233
|
+
...(def.footer !== undefined ? { footer: def.footer } : {}),
|
|
234
|
+
scripts: def.scripts ?? [],
|
|
235
|
+
styles: def.styles ?? [],
|
|
236
|
+
pages,
|
|
237
|
+
};
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
// INTEGRATION(2) 桥接:旧 build 函数形态(集成前的 dist);恒为单页,无外壳字段。
|
|
241
|
+
if (isLegacyDefinition(definition)) {
|
|
242
|
+
return {
|
|
243
|
+
links: [],
|
|
244
|
+
scripts: [],
|
|
245
|
+
styles: [],
|
|
246
|
+
pages: [{ id: SINGLE_PAGE_ID, title: BUILT_IN_PAGE_TITLE, content: definition }],
|
|
247
|
+
};
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
throw new HostReportError(
|
|
251
|
+
`${sourceLabel} does not default-export a report. Export default defineReport(<ExperimentComparison />) (or defineReport({ title, content })) from "niceeval/report".`,
|
|
252
|
+
);
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
/**
|
|
256
|
+
* 装载宿主报告:`--report <file>` 走 loadReportFile;缺省(裸 show / 裸 view)装载内建报告的
|
|
257
|
+
* 默认导出。两条路都进 normalizeHostReport,同一条装载管线。
|
|
258
|
+
*/
|
|
259
|
+
export async function loadHostReport(
|
|
260
|
+
cwd: string,
|
|
261
|
+
reportPath: string | undefined,
|
|
262
|
+
options?: { freshImport?: boolean },
|
|
263
|
+
): Promise<HostReport> {
|
|
264
|
+
if (reportPath !== undefined) {
|
|
265
|
+
const { loadReportFile } = await import("../../dist/report/load.js");
|
|
266
|
+
const definition = await loadReportFile(cwd, reportPath, options);
|
|
267
|
+
return normalizeHostReport(definition, reportPath);
|
|
268
|
+
}
|
|
269
|
+
return normalizeHostReport(await loadBuiltInDefinition(), "the built-in report");
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
/** 内建报告即 `niceeval/report/built-in` 的默认导出(dist 预编译产物,`prepare` 时构建)。 */
|
|
273
|
+
async function loadBuiltInDefinition(): Promise<unknown> {
|
|
274
|
+
const mod = await import("../../dist/report/built-in/index.js");
|
|
275
|
+
return mod.default;
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
// ───────────────────────── 逐页渲染 ─────────────────────────
|
|
279
|
+
|
|
280
|
+
export interface HostRenderContext {
|
|
281
|
+
/** 宿主按官方现刻水位挑好的 Scope(集成前类型名仍是 Selection;两者同物)。 */
|
|
282
|
+
scope: unknown;
|
|
283
|
+
results: Results;
|
|
284
|
+
report: HostReportMeta;
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
export interface HostTextRenderOptions {
|
|
288
|
+
width?: number;
|
|
289
|
+
locale?: string;
|
|
290
|
+
/**
|
|
291
|
+
* 索引命令的完整上下文(docs/feature/reports/show/reports.md:「索引命令携带完整上下文」):
|
|
292
|
+
* 组索引 / 页索引输出的每一条可复制命令都保留当前 --results / --report / --page 与位置参数。
|
|
293
|
+
* INTEGRATION(3):由新 text 面消费;旧桥接的 renderReportToText 不认识该字段,原样忽略。
|
|
294
|
+
*/
|
|
295
|
+
commandContext?: HostCommandContext;
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
export interface HostCommandContext {
|
|
299
|
+
patterns: string[];
|
|
300
|
+
results?: string;
|
|
301
|
+
experiment?: string;
|
|
302
|
+
report?: string;
|
|
303
|
+
page?: string;
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
/** 按上下文拼一条可复制的 show 命令(页索引 / 组索引共用的携带规则)。 */
|
|
307
|
+
export function showCommand(ctx: HostCommandContext, extra: string[] = []): string {
|
|
308
|
+
const parts = ["niceeval show", ...ctx.patterns];
|
|
309
|
+
if (ctx.experiment !== undefined) parts.push(`--experiment ${ctx.experiment}`);
|
|
310
|
+
if (ctx.results !== undefined) parts.push(`--results ${ctx.results}`);
|
|
311
|
+
if (ctx.report !== undefined) parts.push(`--report ${ctx.report}`);
|
|
312
|
+
if (ctx.page !== undefined) parts.push(`--page ${ctx.page}`);
|
|
313
|
+
parts.push(...extra);
|
|
314
|
+
return parts.join(" ");
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
/**
|
|
318
|
+
* 渲染一页的 text 面。INTEGRATION(3):新 runtime 的入口是
|
|
319
|
+
* `renderReportTreeToText(tree, { scope, results, report }, options)`;旧桥接下 content 是
|
|
320
|
+
* 整个旧 definition,调 renderReportToText(definition, { selection, results }, options)。
|
|
321
|
+
*/
|
|
322
|
+
export async function renderHostPageText(
|
|
323
|
+
page: HostReportPage,
|
|
324
|
+
ctx: HostRenderContext,
|
|
325
|
+
options: HostTextRenderOptions,
|
|
326
|
+
): Promise<string> {
|
|
327
|
+
const mod = (await import("../../dist/report/report.js")) as Record<string, unknown>;
|
|
328
|
+
const renderTree = mod.renderReportTreeToText as
|
|
329
|
+
| ((tree: unknown, ctx: HostRenderContext, options: HostTextRenderOptions) => Promise<string>)
|
|
330
|
+
| undefined;
|
|
331
|
+
if (typeof renderTree === "function" && !isLegacyDefinition(page.content)) {
|
|
332
|
+
return renderTree(page.content, ctx, options);
|
|
333
|
+
}
|
|
334
|
+
if (isLegacyDefinition(page.content)) {
|
|
335
|
+
const renderLegacy = mod.renderReportToText as (
|
|
336
|
+
definition: unknown,
|
|
337
|
+
ctx: { selection: unknown; results: Results },
|
|
338
|
+
options: { width?: number; locale?: string },
|
|
339
|
+
) => Promise<string>;
|
|
340
|
+
return renderLegacy(
|
|
341
|
+
page.content,
|
|
342
|
+
{ selection: ctx.scope, results: ctx.results },
|
|
343
|
+
{ ...(options.width !== undefined ? { width: options.width } : {}), ...(options.locale !== undefined ? { locale: options.locale } : {}) },
|
|
344
|
+
);
|
|
345
|
+
}
|
|
346
|
+
throw new HostReportError(
|
|
347
|
+
"The installed report runtime cannot render this report page yet (renderReportTreeToText is missing). Rebuild dist/report with `pnpm run build:report`.",
|
|
348
|
+
);
|
|
349
|
+
}
|
|
350
|
+
|
|
351
|
+
/** 渲染一页的 web 面(静态 HTML)。INTEGRATION(3) 同上,web 侧入口在 dist/report/web.js。 */
|
|
352
|
+
export async function renderHostPageHtml(
|
|
353
|
+
page: HostReportPage,
|
|
354
|
+
ctx: HostRenderContext,
|
|
355
|
+
options: { locale: string },
|
|
356
|
+
): Promise<string> {
|
|
357
|
+
const mod = (await import("../../dist/report/web.js")) as Record<string, unknown>;
|
|
358
|
+
const renderTree = mod.renderReportTreeToStaticHtml as
|
|
359
|
+
| ((tree: unknown, ctx: HostRenderContext, options: { locale: string }) => Promise<string>)
|
|
360
|
+
| undefined;
|
|
361
|
+
if (typeof renderTree === "function" && !isLegacyDefinition(page.content)) {
|
|
362
|
+
return renderTree(page.content, ctx, options);
|
|
363
|
+
}
|
|
364
|
+
if (isLegacyDefinition(page.content)) {
|
|
365
|
+
const renderLegacy = mod.renderReportToStaticHtml as (
|
|
366
|
+
definition: unknown,
|
|
367
|
+
ctx: { selection: unknown; results: Results },
|
|
368
|
+
options: { locale: string },
|
|
369
|
+
) => Promise<string>;
|
|
370
|
+
return renderLegacy(page.content, { selection: ctx.scope, results: ctx.results }, options);
|
|
371
|
+
}
|
|
372
|
+
throw new HostReportError(
|
|
373
|
+
"The installed report runtime cannot render this report page yet (renderReportTreeToStaticHtml is missing). Rebuild dist/report with `pnpm run build:report`.",
|
|
374
|
+
);
|
|
375
|
+
}
|
package/src/show/show.test.ts
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
// cases: docs/engineering/unit-tests/reports/cases.md
|
|
2
|
-
// niceeval show 终端宿主的测试(行为规范:docs-site/zh/
|
|
2
|
+
// niceeval show 终端宿主的测试(行为规范:docs-site/zh/how-to/viewing-results.mdx;
|
|
3
3
|
// 组合语义:docs/feature/reports/architecture.md「Selection 是计算入口」)。覆盖:
|
|
4
4
|
// - 榜单合成口径:每 experiment × eval 取最新判定,局部重跑从更早快照补齐,头部标注合成自几个快照;
|
|
5
5
|
// - 前缀过滤收窄 Selection,覆盖警告分母 = 已知并集 ∩ 范围;
|
|
6
6
|
// - --history 时间轴只列真实执行,resume 携带的复印件不占行;
|
|
7
7
|
// - --report 装载(合法 / 非法默认导出 / 文件缺失)、位置前缀收窄注入 Selection、attemptCommand 下钻;
|
|
8
8
|
// - 互斥:--history 与 --report;
|
|
9
|
-
// - 单 eval 详情、`@<locator>` 精确定位与 --
|
|
9
|
+
// - 单 eval 详情、`@<locator>` 精确定位与 --source / --execution / --diff 证据切面的输出形态。
|
|
10
10
|
//
|
|
11
11
|
// fixture 直接写新布局(<expDir>/<snapDir>/snapshot.json + <evalId>/a<n>/result.json),
|
|
12
12
|
// 依据是 docs/feature/results/architecture.md 的稳定磁盘契约,不经 writer 运行时 API(避免与并行重写的
|
|
@@ -20,7 +20,7 @@ import { createHash } from "node:crypto";
|
|
|
20
20
|
import { openResults } from "../results/index.ts";
|
|
21
21
|
import { RESULTS_FORMAT, RESULTS_SCHEMA_VERSION, type EvalResult, type TimingNode, type TraceSpan, type Verdict } from "../types.ts";
|
|
22
22
|
import { selectCurrentResults } from "../results/select.ts";
|
|
23
|
-
import {
|
|
23
|
+
import { attemptHistory } from "./compose.ts";
|
|
24
24
|
import { runShow, type ShowFlags } from "./index.ts";
|
|
25
25
|
import { stringWidth } from "../report/text/layout.ts";
|
|
26
26
|
|
|
@@ -113,7 +113,7 @@ interface Captured {
|
|
|
113
113
|
async function show(root: string, patterns: string[], flags: ShowFlags = {}, width = 100): Promise<Captured> {
|
|
114
114
|
let out = "";
|
|
115
115
|
let err = "";
|
|
116
|
-
const code = await runShow(root, patterns, {
|
|
116
|
+
const code = await runShow(root, patterns, { results: root, ...flags }, {
|
|
117
117
|
out: (s) => (out += s),
|
|
118
118
|
err: (s) => (err += s),
|
|
119
119
|
width,
|
|
@@ -169,7 +169,7 @@ describe("默认报告:跨快照合成的现刻水位(ExperimentComparison text
|
|
|
169
169
|
try {
|
|
170
170
|
const { out, code } = await show(root, []);
|
|
171
171
|
expect(code).toBe(0);
|
|
172
|
-
expect(out).toContain("
|
|
172
|
+
expect(out).toContain("成本 × 端到端成功率 没有可绘制的数据");
|
|
173
173
|
expect(out).toContain("1 通过 / 1 失败");
|
|
174
174
|
expect(out).toMatch(/\bbub\s+默认\s+bub\s+1s\s+50%/);
|
|
175
175
|
} finally {
|
|
@@ -199,6 +199,7 @@ describe("默认报告:跨快照合成的现刻水位(ExperimentComparison text
|
|
|
199
199
|
]);
|
|
200
200
|
const { out, code } = await show(root, []);
|
|
201
201
|
expect(code).toBe(0);
|
|
202
|
+
// 成本轴(better: lower)反向,「更好」恒指向右上;两轴都声明 better → 提示在场
|
|
202
203
|
expect(out).toContain("better → upper right");
|
|
203
204
|
expect(out).toContain("A compare/a B compare/b");
|
|
204
205
|
expect(out).toMatch(/\bb\s+large\s+claude\s+1s\s+100%/);
|
|
@@ -312,7 +313,7 @@ describe("单 eval 详情", () => {
|
|
|
312
313
|
// 每 experiment 一行都带紧凑索引(locator + 失败原因),详情块再补一条精确的 attempt locator
|
|
313
314
|
expect(out).toMatch(/✗ failed\s+2 attempts.*@1[0-9a-z]{7}.*gate calledTool/);
|
|
314
315
|
expect(out).toMatch(/attempt locator: @1[0-9a-z]{7}/);
|
|
315
|
-
expect(out).toMatch(/next: niceeval show @1[0-9a-z]{7} \[--
|
|
316
|
+
expect(out).toMatch(/next: niceeval show @1[0-9a-z]{7} \[--source\|--execution\|--diff\]/);
|
|
316
317
|
});
|
|
317
318
|
});
|
|
318
319
|
|
|
@@ -349,61 +350,87 @@ describe("--history 时间轴", () => {
|
|
|
349
350
|
return root;
|
|
350
351
|
}
|
|
351
352
|
|
|
352
|
-
it("
|
|
353
|
+
it("逐 attempt 分节:复印件按身份键去重不占行,startedAt 升序,行带摘要 / 成本 / locator", async () => {
|
|
353
354
|
const root = await seedHistoryRoot();
|
|
354
355
|
const results = await openResults(root);
|
|
355
356
|
const exp = results.experiments.find((e) => e.id === "compare/bub")!;
|
|
356
|
-
const rows =
|
|
357
|
-
// 快照2
|
|
357
|
+
const rows = attemptHistory(exp, "weather/brooklyn");
|
|
358
|
+
// 快照2 里复印件被识别(与快照1 的真实执行同身份键),历次 attempt = 快照1 的 passed +
|
|
359
|
+
// 快照2 的 failed(新 attempt);startedAt 升序,旧的在前。
|
|
358
360
|
expect(rows).toHaveLength(2);
|
|
359
|
-
expect(rows[0]).toMatchObject({ verdict: "
|
|
360
|
-
expect(rows[0].
|
|
361
|
-
expect(rows[1]).toMatchObject({ verdict: "
|
|
361
|
+
expect(rows[0]).toMatchObject({ verdict: "passed", costUSD: 0.03 });
|
|
362
|
+
expect(rows[0].summary).toBeUndefined();
|
|
363
|
+
expect(rows[1]).toMatchObject({ verdict: "failed", costUSD: 0.04 });
|
|
364
|
+
expect(rows[1].summary).toContain('calledTool("get_weather")');
|
|
365
|
+
expect(rows[1].locator).toMatch(/^@/);
|
|
362
366
|
|
|
363
367
|
const { out, code } = await show(root, ["weather/brooklyn"], { history: true });
|
|
364
368
|
expect(code).toBe(0);
|
|
365
|
-
|
|
366
|
-
expect(out).toContain("
|
|
367
|
-
|
|
368
|
-
// 复印件那份 passed 判定只出现一行(快照1),不在快照2 再占一行
|
|
369
|
+
// 分节头:experimentId + evalId 一起出现,计数按 attempt。
|
|
370
|
+
expect(out).toContain("weather/brooklyn · compare/bub · 2 attempts · passed 1/2");
|
|
371
|
+
// 复印件那份 passed 判定只出现一行,不在快照2 再占一行;升序:passed 行在 failed 行之前。
|
|
369
372
|
expect(out.match(/✓ passed/g)).toHaveLength(1);
|
|
373
|
+
expect(out.indexOf("✓ passed")).toBeLessThan(out.indexOf("✗ failed"));
|
|
374
|
+
// 每行携带可复制的 locator。
|
|
375
|
+
expect(out).toMatch(/@\w+/);
|
|
370
376
|
});
|
|
371
377
|
|
|
372
|
-
it("裸 --history
|
|
378
|
+
it("裸 --history:对 Scope 中每个 experimentId + evalId 分节,不再输出快照级序列", async () => {
|
|
373
379
|
const root = await seedHistoryRoot();
|
|
374
380
|
const { out, code } = await show(root, [], { history: true });
|
|
375
381
|
expect(code).toBe(0);
|
|
376
|
-
expect(out).toContain("compare/bub · 2
|
|
377
|
-
//
|
|
378
|
-
expect(out.
|
|
382
|
+
expect(out).toContain("weather/brooklyn · compare/bub · 2 attempts · passed 1/2");
|
|
383
|
+
// 不是「每个快照一行」的通过率序列。
|
|
384
|
+
expect(out).not.toContain("1/1 passed");
|
|
385
|
+
});
|
|
386
|
+
|
|
387
|
+
it("--history 与 --page 组合是用法矛盾:直说", async () => {
|
|
388
|
+
const root = await seedHistoryRoot();
|
|
389
|
+
const { err, code } = await show(root, [], { history: true, page: "report" });
|
|
390
|
+
expect(code).toBe(1);
|
|
391
|
+
expect(err).toContain("--page");
|
|
379
392
|
});
|
|
380
393
|
});
|
|
381
394
|
|
|
382
395
|
// ───────────────────────── --report 装载与组合语义 ─────────────────────────
|
|
383
396
|
|
|
384
397
|
describe("--report 装载", () => {
|
|
385
|
-
/**
|
|
398
|
+
/**
|
|
399
|
+
* 不经 niceeval 包也能造出合法报告:判别、组合组件与双面组件都锚在 Symbol.for 上
|
|
400
|
+
* (docs/feature/reports/library/shell.md「defineReport 产物」、layout.md「自定义组件」)。
|
|
401
|
+
* Custom 是双面组件,读 props 直接渲染;Wrapper 是组合组件,用 ComposeContext 的 scope
|
|
402
|
+
* 算出 evals / locator 再装配进 Custom —— 对应 defineReport(<Wrapper />) 的展开形态。
|
|
403
|
+
*/
|
|
386
404
|
async function writeReportFile(dir: string): Promise<string> {
|
|
387
405
|
const path = join(dir, "report.mjs");
|
|
388
406
|
await writeFile(
|
|
389
407
|
path,
|
|
390
408
|
[
|
|
391
409
|
'const FACES = Symbol.for("niceeval.report.faces");',
|
|
410
|
+
'const COMPOSE = Symbol.for("niceeval.report.compose");',
|
|
411
|
+
'const DEFINITION = Symbol.for("niceeval.report.definition");',
|
|
392
412
|
"const Custom = () => null;",
|
|
393
413
|
"Custom[FACES] = {",
|
|
394
414
|
" web: () => null,",
|
|
395
415
|
" text: (props, ctx) => `CUSTOM ${props.evals} · drill ${ctx.attemptCommand(props.locator)}`,",
|
|
396
416
|
"};",
|
|
397
|
-
"
|
|
398
|
-
|
|
399
|
-
"
|
|
400
|
-
"
|
|
401
|
-
"
|
|
402
|
-
"
|
|
403
|
-
"
|
|
404
|
-
"
|
|
405
|
-
"
|
|
417
|
+
"const Wrapper = () => null;",
|
|
418
|
+
"Wrapper[COMPOSE] = (props, ctx) => ({",
|
|
419
|
+
" type: Custom,",
|
|
420
|
+
" props: {",
|
|
421
|
+
" evals: ctx.scope.snapshots.flatMap((s) => s.evals.map((e) => e.id)).sort().join(\",\"),",
|
|
422
|
+
" locator: ctx.scope.snapshots[0].evals[0].attempts[0].locator,",
|
|
423
|
+
" },",
|
|
424
|
+
"});",
|
|
425
|
+
"const definition = {",
|
|
426
|
+
' kind: "report",',
|
|
427
|
+
" links: [],",
|
|
428
|
+
" scripts: [],",
|
|
429
|
+
" styles: [],",
|
|
430
|
+
' pages: [{ id: "report", title: "Report", content: { type: Wrapper, props: {} } }],',
|
|
406
431
|
"};",
|
|
432
|
+
"Object.defineProperty(definition, DEFINITION, { value: true });",
|
|
433
|
+
"export default definition;",
|
|
407
434
|
"",
|
|
408
435
|
].join("\n"),
|
|
409
436
|
"utf-8",
|
|
@@ -473,6 +500,29 @@ describe("--report 装载", () => {
|
|
|
473
500
|
expect(out).not.toContain("CUSTOM");
|
|
474
501
|
expect(out).toContain("no diff recorded"); // fixture 没有 diff artifact:如实说缺
|
|
475
502
|
});
|
|
503
|
+
|
|
504
|
+
it("--page 命中单页定义的唯一页 id `report`:与直接渲染等价", async () => {
|
|
505
|
+
const root = await seedComposedRoot();
|
|
506
|
+
const report = await writeReportFile(root);
|
|
507
|
+
const bare = await show(root, [], { report });
|
|
508
|
+
const paged = await show(root, [], { report, page: "report" });
|
|
509
|
+
expect(paged.code).toBe(0);
|
|
510
|
+
expect(paged.out).toBe(bare.out);
|
|
511
|
+
});
|
|
512
|
+
|
|
513
|
+
it("--page 未命中:按用法错误非零退出并列出可用页 id(内建报告同样成立)", async () => {
|
|
514
|
+
const root = await seedComposedRoot();
|
|
515
|
+
const report = await writeReportFile(root);
|
|
516
|
+
const miss = await show(root, [], { report, page: "typo" });
|
|
517
|
+
expect(miss.code).toBe(1);
|
|
518
|
+
expect(miss.err).toContain(`page "typo" not found in ${report}`);
|
|
519
|
+
expect(miss.err).toContain("Available pages: report");
|
|
520
|
+
|
|
521
|
+
const builtin = await show(root, [], { page: "typo" });
|
|
522
|
+
expect(builtin.code).toBe(1);
|
|
523
|
+
expect(builtin.err).toContain('page "typo" not found in the built-in report');
|
|
524
|
+
expect(builtin.err).toContain("Available pages: report");
|
|
525
|
+
});
|
|
476
526
|
});
|
|
477
527
|
|
|
478
528
|
// ───────────────────────── 证据切面:--execution ─────────────────────────
|
|
@@ -541,9 +591,9 @@ describe("--execution", () => {
|
|
|
541
591
|
});
|
|
542
592
|
});
|
|
543
593
|
|
|
544
|
-
// ───────────────────────── 证据切面:--
|
|
594
|
+
// ───────────────────────── 证据切面:--source ─────────────────────────
|
|
545
595
|
|
|
546
|
-
describe("--
|
|
596
|
+
describe("--source", () => {
|
|
547
597
|
it("evalSource === null 时如实说源码未捕获", async () => {
|
|
548
598
|
const root = await makeRoot();
|
|
549
599
|
await writeSnapshot(
|
|
@@ -552,7 +602,7 @@ describe("--eval", () => {
|
|
|
552
602
|
{ experimentId: "compare/bub", startedAt: "2026-07-08T10:00:00.000Z" },
|
|
553
603
|
[res("weather/brooklyn", "failed", { assertions: [{ name: "succeeded()", severity: "gate", score: 0, outcome: "failed" as const }] })],
|
|
554
604
|
);
|
|
555
|
-
const { out, code } = await show(root, ["weather/brooklyn"], {
|
|
605
|
+
const { out, code } = await show(root, ["weather/brooklyn"], { source: true });
|
|
556
606
|
expect(code).toBe(0);
|
|
557
607
|
expect(out).toContain("eval source unavailable for this attempt");
|
|
558
608
|
});
|
|
@@ -586,7 +636,7 @@ describe("--eval", () => {
|
|
|
586
636
|
await mkdir(join(dir, "sources"), { recursive: true });
|
|
587
637
|
await writeFile(join(dir, "sources", `${sha}.json`), JSON.stringify({ content }), "utf-8");
|
|
588
638
|
|
|
589
|
-
const { out, code } = await show(root, ["weather/brooklyn"], {
|
|
639
|
+
const { out, code } = await show(root, ["weather/brooklyn"], { source: true });
|
|
590
640
|
expect(code).toBe(0);
|
|
591
641
|
expect(out).toContain(`eval source: evals/weather/brooklyn.eval.ts · sha256:${sha.slice(0, 8)}`);
|
|
592
642
|
// 源码行的缩进是语义的一部分,渲染必须原样保留(不按词重排折叠掉这两个空格)。
|
|
@@ -608,7 +658,7 @@ describe("--eval", () => {
|
|
|
608
658
|
await writeFile(join(attemptDir, "sources.json"), JSON.stringify([{ path: "evals/manager.eval.ts", sha256: sha }]), "utf-8");
|
|
609
659
|
await mkdir(join(dir, "sources"), { recursive: true });
|
|
610
660
|
await writeFile(join(dir, "sources", `${sha}.json`), JSON.stringify({ content }), "utf-8");
|
|
611
|
-
const { out } = await show(root, ["manager"], {
|
|
661
|
+
const { out } = await show(root, ["manager"], { source: true });
|
|
612
662
|
expect(out).toContain("gate · Issue 15193 · equals(4) · expected 4 · received 1");
|
|
613
663
|
});
|
|
614
664
|
});
|
|
@@ -702,7 +752,7 @@ describe("show @<locator>", () => {
|
|
|
702
752
|
|
|
703
753
|
const { out, code } = await show(root, [locator]);
|
|
704
754
|
expect(code).toBe(0);
|
|
705
|
-
expect(out).toContain(`available:\n niceeval show ${locator} --
|
|
755
|
+
expect(out).toContain(`available:\n niceeval show ${locator} --source\n niceeval show ${locator} --execution`);
|
|
706
756
|
expect(out).not.toContain(`niceeval show ${locator} --diff`);
|
|
707
757
|
expect(out).not.toContain("evidence:");
|
|
708
758
|
expect(out).not.toContain("next:");
|