niceeval 0.4.5 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/docs-site/zh/concepts/adapter.mdx +6 -6
- package/docs-site/zh/concepts/experiment.mdx +7 -7
- package/docs-site/zh/concepts/overview.mdx +1 -1
- package/docs-site/zh/concepts/tier.mdx +6 -6
- package/docs-site/zh/guides/agent-feedback-loop.mdx +63 -20
- package/docs-site/zh/guides/connect-your-agent.mdx +7 -7
- package/docs-site/zh/guides/custom-reports.mdx +243 -0
- package/docs-site/zh/guides/experiments.mdx +3 -3
- package/docs-site/zh/guides/report-components.mdx +252 -0
- package/docs-site/zh/guides/reporters.mdx +1 -1
- package/docs-site/zh/guides/results-data.mdx +224 -0
- package/docs-site/zh/guides/runner.mdx +1 -1
- package/docs-site/zh/guides/sandbox-agent.mdx +2 -2
- package/docs-site/zh/guides/viewing-results.mdx +147 -77
- package/docs-site/zh/guides/write-experiment.mdx +9 -9
- package/docs-site/zh/guides/write-send.mdx +6 -6
- package/docs-site/zh/index.mdx +1 -1
- package/docs-site/zh/introduction.mdx +1 -1
- package/docs-site/zh/reference/cli.mdx +10 -2
- package/docs-site/zh/reference/define-agent.mdx +5 -5
- package/docs-site/zh/reference/define-config.mdx +1 -1
- package/docs-site/zh/reference/define-eval.mdx +3 -3
- package/package.json +3 -2
- package/src/agents/ai-sdk.test.ts +1 -1
- package/src/agents/ai-sdk.ts +2 -2
- package/src/agents/bub.ts +14 -4
- package/src/agents/claude-code.ts +1 -1
- package/src/agents/codex.ts +2 -2
- package/src/agents/streaming.test.ts +1 -1
- package/src/agents/types.ts +4 -4
- package/src/agents/ui-message-stream.test.ts +1 -1
- package/src/cli.ts +104 -14
- package/src/context/context.test.ts +1 -1
- package/src/context/context.ts +3 -3
- package/src/context/session.ts +2 -2
- package/src/context/types.ts +2 -2
- package/src/i18n/en.ts +29 -7
- package/src/i18n/zh-CN.ts +24 -2
- package/src/report/aggregate.ts +79 -50
- package/src/report/components.tsx +190 -0
- package/src/report/compute.ts +149 -84
- package/src/report/default-report.tsx +222 -0
- package/src/report/dual-face.test.tsx +527 -0
- package/src/report/format.ts +29 -0
- package/src/report/index.ts +79 -24
- package/src/report/load.ts +67 -0
- package/src/report/metrics.ts +2 -1
- package/src/report/param.ts +18 -0
- package/src/report/primitives.tsx +117 -0
- package/src/report/react/CaseList.tsx +2 -2
- package/src/report/react/DeltaTable.tsx +2 -2
- package/src/report/react/MetricBars.tsx +109 -0
- package/src/report/react/MetricLine.tsx +200 -0
- package/src/report/react/MetricMatrix.tsx +1 -1
- package/src/report/react/MetricScatter.tsx +1 -1
- package/src/report/react/MetricTable.tsx +1 -1
- package/src/report/react/RunOverview.tsx +4 -4
- package/src/report/react/Scoreboard.tsx +3 -3
- package/src/report/react/cell.tsx +1 -1
- package/src/report/react/fixtures.ts +98 -36
- package/src/report/react/format.ts +3 -28
- package/src/report/react/index.tsx +41 -24
- package/src/report/react/render.test.tsx +8 -8
- package/src/report/react/styles.css +77 -0
- package/src/report/report.test.ts +248 -128
- package/src/report/report.ts +69 -0
- package/src/report/text/faces.ts +356 -0
- package/src/report/text/layout.ts +125 -0
- package/src/report/text/plot.ts +127 -0
- package/src/report/tree.ts +220 -0
- package/src/report/types.ts +75 -27
- package/src/report/web.ts +40 -0
- package/src/results/copy.ts +95 -41
- package/src/results/format.ts +12 -11
- package/src/results/index.ts +34 -17
- package/src/results/open.ts +201 -81
- package/src/results/results.test.ts +595 -191
- package/src/results/select.ts +107 -55
- package/src/results/types.ts +137 -45
- package/src/results/writer.ts +258 -0
- package/src/runner/attempt.ts +3 -3
- package/src/runner/fingerprint.ts +1 -1
- package/src/runner/reporters/artifacts.ts +38 -78
- package/src/runner/reporters/braintrust.test.ts +2 -2
- package/src/runner/reporters/braintrust.ts +3 -3
- package/src/runner/run.ts +1 -1
- package/src/runner/types.ts +18 -12
- package/src/show/compose.ts +215 -0
- package/src/show/index.ts +263 -0
- package/src/show/render.ts +456 -0
- package/src/show/show.test.ts +505 -0
- package/src/view/app/App.tsx +85 -24
- package/src/view/app/components/CostScoreChart.tsx +29 -10
- package/src/view/app/components/ExperimentTable.tsx +27 -9
- package/src/view/app/components/GroupSelector.tsx +1 -1
- package/src/view/app/i18n.ts +18 -3
- package/src/view/app/lib/attempt-route.test.ts +9 -9
- package/src/view/app/lib/attempt-route.ts +5 -5
- package/src/view/app/lib/outcome.ts +1 -3
- package/src/view/app/lib/rows.ts +95 -15
- package/src/view/app/main.tsx +18 -6
- package/src/view/app/pages/RunsPage.tsx +4 -7
- package/src/view/app/pages/TracesPage.tsx +3 -8
- package/src/view/app/types.ts +50 -6
- package/src/view/client-dist/app.css +1 -1
- package/src/view/client-dist/app.js +20 -20
- package/src/view/data.test.ts +254 -0
- package/src/view/data.ts +336 -0
- package/src/view/index.ts +64 -10
- package/src/view/server.ts +31 -9
- package/src/view/shared/types.ts +47 -41
- package/src/view/styles.css +5 -0
- package/src/view/template.html +1 -0
- package/src/view/view-report.test.ts +308 -0
- package/docs-site/.mintignore +0 -13
- package/docs-site/AGENTS.md +0 -46
- package/docs-site/concepts/adapter.mdx +0 -287
- package/docs-site/concepts/assert.mdx +0 -243
- package/docs-site/concepts/drive.mdx +0 -118
- package/docs-site/concepts/evals.mdx +0 -108
- package/docs-site/concepts/experiment.mdx +0 -37
- package/docs-site/concepts/hitl.mdx +0 -83
- package/docs-site/concepts/judge.mdx +0 -129
- package/docs-site/concepts/overview.mdx +0 -98
- package/docs-site/concepts/tier.mdx +0 -42
- package/docs-site/docs-ref/00-index.md +0 -23
- package/docs-site/docs-ref/01-page-types.md +0 -43
- package/docs-site/docs-ref/03-style-rules.md +0 -45
- package/docs-site/docs-ref/04-code-examples.md +0 -37
- package/docs-site/docs-ref/05-dx-failure-paths.md +0 -45
- package/docs-site/docs-ref/06-checklists.md +0 -53
- package/docs-site/docs.json +0 -256
- package/docs-site/example/ai-agent-application.mdx +0 -148
- package/docs-site/example/claude-code-codex-plugin.mdx +0 -162
- package/docs-site/example/claude-code-codex-skill.mdx +0 -147
- package/docs-site/example/showcase.mdx +0 -39
- package/docs-site/example/tier1-ai-sdk-v7.mdx +0 -136
- package/docs-site/example/tier1-claude-sdk.mdx +0 -119
- package/docs-site/example/tier1-codex-sdk.mdx +0 -111
- package/docs-site/example/tier1-langgraph.mdx +0 -119
- package/docs-site/example/tier1-pi-sdk.mdx +0 -119
- package/docs-site/favicon.svg +0 -5
- package/docs-site/github-diff.css +0 -135
- package/docs-site/github-diff.js +0 -11
- package/docs-site/guides/authoring.mdx +0 -129
- package/docs-site/guides/ci-integration.mdx +0 -97
- package/docs-site/guides/connect-otel.mdx +0 -209
- package/docs-site/guides/connect-your-agent.mdx +0 -221
- package/docs-site/guides/dataset-fanout.mdx +0 -98
- package/docs-site/guides/experiments.mdx +0 -71
- package/docs-site/guides/fixtures.mdx +0 -147
- package/docs-site/guides/reporters.mdx +0 -113
- package/docs-site/guides/runner.mdx +0 -79
- package/docs-site/guides/sandbox-agent.mdx +0 -138
- package/docs-site/guides/sandbox-backends.mdx +0 -64
- package/docs-site/guides/scoring-guide.mdx +0 -92
- package/docs-site/guides/viewing-results.mdx +0 -195
- package/docs-site/guides/write-experiment.mdx +0 -81
- package/docs-site/guides/write-send.mdx +0 -347
- package/docs-site/index.mdx +0 -181
- package/docs-site/introduction.mdx +0 -141
- package/docs-site/quickstart.mdx +0 -136
- package/docs-site/reference/builtin-agents.mdx +0 -161
- package/docs-site/reference/capabilities.mdx +0 -76
- package/docs-site/reference/cli.mdx +0 -120
- package/docs-site/reference/define-agent.mdx +0 -168
- package/docs-site/reference/define-config.mdx +0 -41
- package/docs-site/reference/define-eval.mdx +0 -160
- package/docs-site/reference/events.mdx +0 -131
- package/docs-site/reference/expect.mdx +0 -112
- package/docs-site/tracker.js +0 -8
- package/src/report/react/data.ts +0 -17
- package/src/view/aggregate.ts +0 -103
- package/src/view/loader.test.ts +0 -61
- package/src/view/loader.ts +0 -257
package/src/view/index.ts
CHANGED
|
@@ -1,29 +1,83 @@
|
|
|
1
1
|
// 本地结果查看器入口:只做编排与对外导出。
|
|
2
|
-
//
|
|
3
|
-
// server
|
|
2
|
+
// 读取(openResults)与统计(官方计算函数)在 data.ts,HTTP 与 HTML 烘焙在 server.ts,
|
|
3
|
+
// server/前端共用的数据契约在 shared/types.ts。
|
|
4
4
|
|
|
5
5
|
import { copyFile, mkdir, writeFile } from "node:fs/promises";
|
|
6
|
-
import { existsSync } from "node:fs";
|
|
6
|
+
import { existsSync, statSync } from "node:fs";
|
|
7
7
|
import { join, resolve } from "node:path";
|
|
8
|
-
import {
|
|
8
|
+
import { ViewInputError, loadViewScan, type ViewScan } from "./data.ts";
|
|
9
9
|
import { renderHtml, type ViewOptions } from "./server.ts";
|
|
10
10
|
|
|
11
11
|
export { startViewServer, type ViewOptions, type ViewServer } from "./server.ts";
|
|
12
12
|
export {
|
|
13
13
|
IncompatibleResultsError,
|
|
14
|
+
ViewInputError,
|
|
14
15
|
incompatibleHint,
|
|
15
16
|
incompatibleViewCommand,
|
|
16
17
|
loadLatestResultsPerEval,
|
|
17
|
-
|
|
18
|
+
loadViewScan,
|
|
18
19
|
type IncompatibleRun,
|
|
19
|
-
type
|
|
20
|
-
type
|
|
21
|
-
} from "./
|
|
20
|
+
type ViewScan,
|
|
21
|
+
type ViewScanOptions,
|
|
22
|
+
} from "./data.ts";
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* view 的位置参数语义(与 CLI 模型一致:位置参数选「看哪些 eval」):
|
|
26
|
+
* - 恰好一个位置参数且指向存在的文件 → 单文件模式(`niceeval view <summary.json>`),
|
|
27
|
+
* 不与 --run 或其它位置参数混用;
|
|
28
|
+
* - 指向存在的目录 → 报错直说:目录经 `--run <dir>` 递入,位置参数留给 eval id 前缀;
|
|
29
|
+
* - 其余 → eval id 前缀,收窄报告槽选集(经 show 同一套选集合成)。
|
|
30
|
+
*/
|
|
31
|
+
export function resolveViewInput(
|
|
32
|
+
cwd: string,
|
|
33
|
+
positionals: string[],
|
|
34
|
+
run?: string,
|
|
35
|
+
): { input?: string; patterns: string[] } {
|
|
36
|
+
if (run !== undefined) {
|
|
37
|
+
const dir = resolve(cwd, run);
|
|
38
|
+
if (!existsSync(dir)) {
|
|
39
|
+
throw new ViewInputError(`Results directory not found: ${dir}`);
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
const kindOf = (p: string): "file" | "dir" | "pattern" => {
|
|
43
|
+
try {
|
|
44
|
+
const stat = statSync(resolve(cwd, p));
|
|
45
|
+
return stat.isFile() ? "file" : stat.isDirectory() ? "dir" : "pattern";
|
|
46
|
+
} catch {
|
|
47
|
+
return "pattern";
|
|
48
|
+
}
|
|
49
|
+
};
|
|
50
|
+
for (const p of positionals) {
|
|
51
|
+
const kind = kindOf(p);
|
|
52
|
+
if (kind === "dir") {
|
|
53
|
+
throw new ViewInputError(
|
|
54
|
+
`"${p}" is a directory. Pass results directories with --run (niceeval view --run ${p}); positional arguments select evals by id prefix.`,
|
|
55
|
+
);
|
|
56
|
+
}
|
|
57
|
+
if (kind !== "file") continue;
|
|
58
|
+
if (positionals.length > 1) {
|
|
59
|
+
throw new ViewInputError(
|
|
60
|
+
`"${p}" is a summary file, but more positional arguments were given. Single-file mode takes exactly one path: niceeval view ${p}`,
|
|
61
|
+
);
|
|
62
|
+
}
|
|
63
|
+
if (run !== undefined) {
|
|
64
|
+
throw new ViewInputError(
|
|
65
|
+
`"${p}" is a summary file and cannot be combined with --run. Pass either the file (niceeval view ${p}) or the results dir (niceeval view --run <dir>).`,
|
|
66
|
+
);
|
|
67
|
+
}
|
|
68
|
+
return { input: resolve(cwd, p), patterns: [] };
|
|
69
|
+
}
|
|
70
|
+
return {
|
|
71
|
+
...(run !== undefined ? { input: resolve(cwd, run) } : {}),
|
|
72
|
+
patterns: positionals,
|
|
73
|
+
};
|
|
74
|
+
}
|
|
22
75
|
|
|
23
76
|
/**
|
|
24
77
|
* 导出静态报告(--out):只有目录式一种形态。写 <dir>/index.html,并把前端会 fetch 的工件
|
|
25
78
|
* (sources.json / events.json / trace.json)复制到 <dir>/artifact/<base>/——与本地
|
|
26
79
|
* server 的 /artifact/<rel> 路由同一布局,整个目录扔给任何静态托管即是完整体验。
|
|
80
|
+
* 带 --report(opts.scan.report)时,报告页即首页的报告槽,证据室同站。
|
|
27
81
|
* 单文件(*.html)导出已移除:代码/transcript/trace 视图依赖工件文件,单文件注定残缺,
|
|
28
82
|
* 存在本身就在诱导用户导出一份看不了证据的报告(docs/view.md「静态导出」)。
|
|
29
83
|
*/
|
|
@@ -34,7 +88,7 @@ export async function buildView(opts: ViewOptions = {}): Promise<string> {
|
|
|
34
88
|
`--out expects a directory, got "${opts.out}". Single-file HTML export was removed: code, transcript and trace views need artifact files next to the page. Export a directory instead (e.g. --out site) and serve it with any static host.`,
|
|
35
89
|
);
|
|
36
90
|
}
|
|
37
|
-
const scan = await
|
|
91
|
+
const scan = await loadViewScan(opts.input, opts.scan);
|
|
38
92
|
await mkdir(out, { recursive: true });
|
|
39
93
|
await writeFile(join(out, "index.html"), await renderHtml(scan), "utf-8");
|
|
40
94
|
await copyFetchedArtifacts(scan, join(out, "artifact"));
|
|
@@ -45,7 +99,7 @@ export async function buildView(opts: ViewOptions = {}): Promise<string> {
|
|
|
45
99
|
// 且 diff 可达上百 MB,带进静态导出只会拖垮部署体积。
|
|
46
100
|
const FETCHED_ARTIFACTS = ["sources.json", "events.json", "trace.json"];
|
|
47
101
|
|
|
48
|
-
async function copyFetchedArtifacts(scan:
|
|
102
|
+
async function copyFetchedArtifacts(scan: ViewScan, artifactRoot: string): Promise<void> {
|
|
49
103
|
for (const [base, srcDir] of scan.artifactDirs) {
|
|
50
104
|
const destDir = join(artifactRoot, base);
|
|
51
105
|
// 输入本身已经是导出布局(比如对着上次导出的目录重新生成 index.html)时不自拷。
|
package/src/view/server.ts
CHANGED
|
@@ -1,17 +1,18 @@
|
|
|
1
1
|
// HTTP server 与静态资源:起本地 web、按需吐工件、把 viewData 烘焙进单个 HTML。
|
|
2
|
-
//
|
|
2
|
+
// 数据读取与统计在 data.ts(openResults + 官方计算函数);这里只管「怎么送到浏览器」。
|
|
3
3
|
|
|
4
4
|
import { createServer, type Server } from "node:http";
|
|
5
5
|
import { readFile } from "node:fs/promises";
|
|
6
6
|
import { resolve } from "node:path";
|
|
7
|
-
import {
|
|
8
|
-
import { buildViewData } from "./aggregate.ts";
|
|
7
|
+
import { loadViewScan, viewRoot, type ViewScan, type ViewScanOptions } from "./data.ts";
|
|
9
8
|
import { formatThrown } from "../util.ts";
|
|
10
9
|
|
|
11
10
|
export interface ViewOptions {
|
|
12
11
|
input?: string;
|
|
13
12
|
out?: string;
|
|
14
13
|
port?: number;
|
|
14
|
+
/** 报告槽的组合语义(位置前缀 / --experiment / --report),透传给 loadViewScan。 */
|
|
15
|
+
scan?: ViewScanOptions;
|
|
15
16
|
}
|
|
16
17
|
|
|
17
18
|
export interface ViewServer {
|
|
@@ -23,11 +24,15 @@ const TEMPLATE_PLACEHOLDERS = {
|
|
|
23
24
|
styles: "<!-- __NICEEVAL_STYLES__ -->",
|
|
24
25
|
appCode: "__NICEEVAL_APP_CODE__",
|
|
25
26
|
viewData: "__NICEEVAL_VIEW_DATA_JSON__",
|
|
27
|
+
reportSlot: "<!-- __NICEEVAL_REPORT_SLOT__ -->",
|
|
26
28
|
} as const;
|
|
27
29
|
|
|
28
30
|
export async function startViewServer(opts: ViewOptions = {}): Promise<ViewServer> {
|
|
29
31
|
const input = opts.input;
|
|
30
32
|
const root = viewRoot(input);
|
|
33
|
+
// 数据装载先跑一遍:单文件模式指向读不了的报告、--report 装载失败、
|
|
34
|
+
// 前缀匹配不到,都要在起 server 前就失败并给出提示。
|
|
35
|
+
await loadViewScan(input, opts.scan);
|
|
31
36
|
const server = createServer(async (req, res) => {
|
|
32
37
|
try {
|
|
33
38
|
const url = new URL(req.url ?? "/", "http://127.0.0.1");
|
|
@@ -57,7 +62,9 @@ export async function startViewServer(opts: ViewOptions = {}): Promise<ViewServe
|
|
|
57
62
|
"content-type": "text/html; charset=utf-8",
|
|
58
63
|
"cache-control": "no-store",
|
|
59
64
|
});
|
|
60
|
-
|
|
65
|
+
// 每次请求现读现算,永远是盘上最新数据;--report 的报告文件变更同样在
|
|
66
|
+
// 下次请求整页重算(装载走 mtime cache-busting,见 report/load.ts)。
|
|
67
|
+
res.end(await renderHtml(await loadViewScan(input, opts.scan)));
|
|
61
68
|
} catch (e) {
|
|
62
69
|
res.writeHead(500, { "content-type": "text/plain; charset=utf-8" });
|
|
63
70
|
res.end(formatThrown(e));
|
|
@@ -74,16 +81,31 @@ export async function startViewServer(opts: ViewOptions = {}): Promise<ViewServe
|
|
|
74
81
|
};
|
|
75
82
|
}
|
|
76
83
|
|
|
77
|
-
/**
|
|
78
|
-
|
|
84
|
+
/**
|
|
85
|
+
* 把 viewData(只含原始值与相对路径,不含宿主机绝对路径)和前端产物烘焙进单个 HTML。
|
|
86
|
+
* --report 在场时,报告 HTML 作为 <template id="niceeval-report"> 静态块烘在
|
|
87
|
+
* __NICEEVAL_VIEW_DATA__ 旁(零客户端 JS、不 hydrate,自定义组件的 <Style> 产物
|
|
88
|
+
* 已内联其中),并附官方组件样式(report/react/styles.css);前端只把这块摆进
|
|
89
|
+
* 报告槽位置,不解析。
|
|
90
|
+
*/
|
|
91
|
+
export async function renderHtml(scan: ViewScan): Promise<string> {
|
|
79
92
|
const template = await readViewAsset("template.html");
|
|
80
93
|
const styles = await readViewAsset("client-dist/app.css");
|
|
81
94
|
const app = await readViewAsset("client-dist/app.js");
|
|
82
|
-
const
|
|
95
|
+
const reportStyles =
|
|
96
|
+
scan.reportHtml !== undefined
|
|
97
|
+
? await readFile(new URL("../report/react/styles.css", import.meta.url), "utf-8")
|
|
98
|
+
: undefined;
|
|
83
99
|
|
|
84
100
|
return template
|
|
85
|
-
.replace(
|
|
86
|
-
|
|
101
|
+
.replace(
|
|
102
|
+
TEMPLATE_PLACEHOLDERS.styles,
|
|
103
|
+
() => `<style>\n${styles}\n</style>${reportStyles !== undefined ? `\n<style>\n${reportStyles}\n</style>` : ""}`,
|
|
104
|
+
)
|
|
105
|
+
.replace(TEMPLATE_PLACEHOLDERS.reportSlot, () =>
|
|
106
|
+
scan.reportHtml !== undefined ? `<template id="niceeval-report">${scan.reportHtml}</template>` : "",
|
|
107
|
+
)
|
|
108
|
+
.replace(TEMPLATE_PLACEHOLDERS.viewData, () => JSON.stringify(scan.viewData).replace(/</g, "\\u003c"))
|
|
87
109
|
.replace(TEMPLATE_PLACEHOLDERS.appCode, () => JSON.stringify(app).replace(/</g, "\\u003c"));
|
|
88
110
|
}
|
|
89
111
|
|
package/src/view/shared/types.ts
CHANGED
|
@@ -1,56 +1,59 @@
|
|
|
1
|
-
// server(
|
|
1
|
+
// server(data.ts)与前端(app/)共用的 view 数据形状。
|
|
2
2
|
// viewData 会被序列化进静态 HTML,两边必须对同一份声明编程;只允许 type import。
|
|
3
|
+
//
|
|
4
|
+
// 统计层收编后(docs/view.md「用 Reports 积木重建 view」迁移顺序 2),烘进 HTML 的不再是
|
|
5
|
+
// 私有 rows,而是官方数据契约:OverviewData / TableData(niceeval/report 的计算函数产物)
|
|
6
|
+
// + 快照元信息(证据室数据)+ skipped / warnings。前端只做渲染与展示态交互,不再聚合。
|
|
3
7
|
|
|
4
|
-
import type { EvalResult, LocalizedText
|
|
8
|
+
import type { EvalResult, LocalizedText } from "../../types.ts";
|
|
9
|
+
import type { OverviewData, TableData } from "../../report/types.ts";
|
|
10
|
+
|
|
11
|
+
// 官方契约的类型再导出(全部 type-only,前端打包时被擦除)。
|
|
12
|
+
export type { MetricCell, MetricColumn, OverviewData, SelectionWarning, TableData } from "../../report/types.ts";
|
|
5
13
|
|
|
6
14
|
/**
|
|
7
|
-
* attempt 的深链身份:run
|
|
8
|
-
*
|
|
9
|
-
*
|
|
15
|
+
* attempt 的深链身份:run 目录名 + 该 attempt 在 summary.results 里的下标。
|
|
16
|
+
* `#/attempt/<run>/<result>` 路由的参数——与 niceeval/results 的 AttemptRef、Reports 的
|
|
17
|
+
* MetricCell.refs 同一身份契约,报告页(前门)与 view(证据室)靠它指向同一个 attempt。
|
|
10
18
|
*/
|
|
11
19
|
export interface AttemptRef {
|
|
12
20
|
run: string;
|
|
13
21
|
result: number;
|
|
14
22
|
}
|
|
15
23
|
|
|
16
|
-
/** view 侧的 attempt 结果 = 瘦身后的 EvalResult +
|
|
24
|
+
/** view 侧的 attempt 结果 = 瘦身后的 EvalResult + loader 注入的深链身份与工件基址。 */
|
|
17
25
|
export type ViewEvalResult = EvalResult & { attemptRef?: AttemptRef };
|
|
18
26
|
|
|
19
|
-
/**
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
27
|
+
/**
|
|
28
|
+
* 快照 = 单次跑的实验(experiment × run 切片),与 niceeval/results 的 Snapshot 同口径。
|
|
29
|
+
* 携带 attempt 明细供证据室(钻取 / AttemptModal / Runs / Traces)渲染;
|
|
30
|
+
* 榜单统计不从这里算,吃 ViewData.table / overview 的官方产物。
|
|
31
|
+
*/
|
|
32
|
+
export interface ViewSnapshot {
|
|
33
|
+
/** 落盘缺 experimentId 时是 "<agent>/<model>" 合成键(synthetic: true 同源)。 */
|
|
34
|
+
experimentId: string;
|
|
35
|
+
synthetic?: boolean;
|
|
26
36
|
agent: string;
|
|
27
37
|
model?: string;
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
errored: number;
|
|
35
|
-
skipped: number;
|
|
36
|
-
scored?: number;
|
|
37
|
-
passRate: number;
|
|
38
|
-
avgDurationMs: number;
|
|
39
|
-
usage: Usage;
|
|
40
|
-
estimatedCostUSD?: number;
|
|
41
|
-
/** 该实验组里最新一次 run 的 startedAt(ISO);详情展示「运行时间」。 */
|
|
42
|
-
lastRunAt?: string;
|
|
38
|
+
startedAt: string;
|
|
39
|
+
/** 物理 run 目录名(= AttemptRef.run)。 */
|
|
40
|
+
run: string;
|
|
41
|
+
/** 是否该实验最新一次快照 —— 榜单(results.latest() 口径)的成员。 */
|
|
42
|
+
latest: boolean;
|
|
43
|
+
/** 该快照的 attempt 明细(跨快照去重后的幸存条目;attemptRef / artifactBase 已注入)。 */
|
|
43
44
|
results: ViewEvalResult[];
|
|
44
45
|
}
|
|
45
46
|
|
|
46
|
-
/** 目录扫描里被跳过的 run
|
|
47
|
+
/** 目录扫描里被跳过的 run 在页面顶部的提示条目;三种原因与 niceeval/results 的 skipped 一致。 */
|
|
47
48
|
export interface SkippedRunNotice {
|
|
48
|
-
/** run
|
|
49
|
+
/** run 目录,相对 cwd。 */
|
|
49
50
|
dir: string;
|
|
50
|
-
reason: "incompatible-version" | "malformed";
|
|
51
|
+
reason: "incompatible-version" | "malformed" | "incomplete";
|
|
51
52
|
schemaVersion?: number;
|
|
53
|
+
/** 完整 producer:只有 name === "niceeval" 才配得出 npx 命令,第三方 harness 如实报名字。 */
|
|
54
|
+
producerName?: string;
|
|
52
55
|
producerVersion?: string;
|
|
53
|
-
/** incompatible-version
|
|
56
|
+
/** incompatible-version 且 producer 是 niceeval:服务端拼好的查看命令。 */
|
|
54
57
|
command?: string;
|
|
55
58
|
/** malformed:一句诊断(invalid JSON / results 不是数组 …)。 */
|
|
56
59
|
detail?: string;
|
|
@@ -58,20 +61,23 @@ export interface SkippedRunNotice {
|
|
|
58
61
|
|
|
59
62
|
/**
|
|
60
63
|
* 烘焙进 HTML 的页面数据。时间/比率/成本一律传原始值(ISO 字符串、number),
|
|
61
|
-
* 格式化统一由前端按当前界面 locale
|
|
64
|
+
* 格式化统一由前端按当前界面 locale 做;官方 MetricCell 自带 display,前端直接渲染。
|
|
62
65
|
*/
|
|
63
66
|
export interface ViewData {
|
|
64
|
-
rows?: ViewRow[];
|
|
65
67
|
/** 项目名(来自 config.name);hero 标题,可按 locale 多语言。 */
|
|
66
68
|
name?: LocalizedText;
|
|
67
69
|
/** 最近一次 run 的 startedAt(ISO);没有历史 run 时缺省。 */
|
|
68
70
|
lastRunAt?: string;
|
|
69
|
-
/**
|
|
70
|
-
|
|
71
|
-
/**
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
/**
|
|
71
|
+
/** 榜单合成自几个物理 run(latest 选集覆盖的 run 数);表头「合成来源」标注。 */
|
|
72
|
+
composedRuns: number;
|
|
73
|
+
/** 官方 KPI(RunOverview.data 产物):totals + 选中快照元信息 + 选集 warnings 随行。 */
|
|
74
|
+
overview: OverviewData;
|
|
75
|
+
/** 官方榜单(MetricTable.data,rows: "experiment"):每个实验最新一次快照的口径。 */
|
|
76
|
+
table: TableData;
|
|
77
|
+
/** 整体单行(MetricTable.data,常量维度):hero 的官方通过率,同一台聚合引擎。 */
|
|
78
|
+
overall: TableData;
|
|
79
|
+
/** 全部历史快照(跨快照按身份键去重后);Experiments 钻取吃 latest,Runs / Traces 吃全部。 */
|
|
80
|
+
snapshots: ViewSnapshot[];
|
|
81
|
+
/** 读不了的落盘(三种原因);前端顶部横幅展示,不静默。 */
|
|
76
82
|
skippedRuns?: SkippedRunNotice[];
|
|
77
83
|
}
|
package/src/view/styles.css
CHANGED
|
@@ -146,6 +146,11 @@ h1 {
|
|
|
146
146
|
font-size: 14px;
|
|
147
147
|
}
|
|
148
148
|
.meta b { color: var(--text); font-weight: 570; }
|
|
149
|
+
/* --report 报告槽:server 侧渲染好的静态报告 HTML(nre-* 类的样式随包 styles.css 注入),
|
|
150
|
+
这里只给槽位留呼吸空间,不覆盖报告自己的样式。 */
|
|
151
|
+
.report-slot {
|
|
152
|
+
margin: 0 auto 36px;
|
|
153
|
+
}
|
|
149
154
|
.summary {
|
|
150
155
|
display: grid;
|
|
151
156
|
grid-template-columns: repeat(4, minmax(0, 1fr));
|
package/src/view/template.html
CHANGED
|
@@ -0,0 +1,308 @@
|
|
|
1
|
+
// niceeval view 的 --report 报告槽与宿主组合语义(docs/reports.md「宿主输入的组合语义」
|
|
2
|
+
// 与裁决记录 6;公开行为准绳 docs-site/zh/guides/viewing-results.mdx / custom-reports.mdx)。
|
|
3
|
+
// 覆盖:
|
|
4
|
+
// - 组合语义与 show 对齐:位置前缀收窄报告槽选集、--experiment 过滤、匹配不到直说;
|
|
5
|
+
// - 单文件模式共存:存在的文件路径 → 单文件模式,目录报错直说走 --run,其余按 eval 前缀;
|
|
6
|
+
// - 报告槽整槽替换:报告 HTML(含 <Style> 产物与证据室深链)烘进静态块,证据室数据原样保留;
|
|
7
|
+
// - --out 静态导出:index.html 含报告块与官方样式,报告 HTML 零 <script>;
|
|
8
|
+
// - dev server 装载语义:报告文件变更 → 下次装载整页重算(mtime cache-busting)。
|
|
9
|
+
|
|
10
|
+
import { mkdir, mkdtemp, readFile, rm, utimes, writeFile } from "node:fs/promises";
|
|
11
|
+
import { existsSync } from "node:fs";
|
|
12
|
+
import { tmpdir } from "node:os";
|
|
13
|
+
import { join, resolve } from "node:path";
|
|
14
|
+
import { afterEach, describe, expect, it } from "vitest";
|
|
15
|
+
import { ReportLoadError } from "../report/load.ts";
|
|
16
|
+
import { ViewInputError, loadViewScan } from "./data.ts";
|
|
17
|
+
import { buildView, resolveViewInput } from "./index.ts";
|
|
18
|
+
import { runShow } from "../show/index.ts";
|
|
19
|
+
import { RESULTS_FORMAT, RESULTS_SCHEMA_VERSION, type EvalResult, type RunSummary } from "../types.ts";
|
|
20
|
+
|
|
21
|
+
const EXAM_REPORT = resolve(__dirname, "../../test/fixtures/report/exam-report.tsx");
|
|
22
|
+
|
|
23
|
+
const roots: string[] = [];
|
|
24
|
+
async function makeRoot(): Promise<string> {
|
|
25
|
+
const root = await mkdtemp(join(tmpdir(), "niceeval-viewreport-"));
|
|
26
|
+
roots.push(root);
|
|
27
|
+
return root;
|
|
28
|
+
}
|
|
29
|
+
afterEach(async () => {
|
|
30
|
+
await Promise.all(roots.splice(0).map((r) => rm(r, { recursive: true, force: true })));
|
|
31
|
+
});
|
|
32
|
+
|
|
33
|
+
function res(over: Partial<EvalResult> & Pick<EvalResult, "id">): EvalResult {
|
|
34
|
+
return { agent: "bub", outcome: "passed", attempt: 0, durationMs: 1000, assertions: [], ...over };
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
function summaryOf(results: EvalResult[], over: Partial<RunSummary> = {}): RunSummary {
|
|
38
|
+
const count = (o: EvalResult["outcome"]) => results.filter((r) => r.outcome === o).length;
|
|
39
|
+
return {
|
|
40
|
+
format: RESULTS_FORMAT,
|
|
41
|
+
schemaVersion: RESULTS_SCHEMA_VERSION,
|
|
42
|
+
producer: { name: "niceeval", version: "0.4.6" },
|
|
43
|
+
agent: results[0]?.agent ?? "bub",
|
|
44
|
+
startedAt: "2026-07-08T10:00:00.000Z",
|
|
45
|
+
completedAt: "2026-07-08T10:10:00.000Z",
|
|
46
|
+
passed: count("passed"),
|
|
47
|
+
failed: count("failed"),
|
|
48
|
+
skipped: count("skipped"),
|
|
49
|
+
errored: count("errored"),
|
|
50
|
+
durationMs: 60_000,
|
|
51
|
+
results,
|
|
52
|
+
...over,
|
|
53
|
+
};
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
async function writeRun(root: string, dirName: string, summary: RunSummary): Promise<string> {
|
|
57
|
+
const dir = join(root, dirName);
|
|
58
|
+
await mkdir(dir, { recursive: true });
|
|
59
|
+
await writeFile(join(dir, "summary.json"), JSON.stringify(summary, null, 2), "utf-8");
|
|
60
|
+
return dir;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/** 两个实验、weather 通过 + button 失败:报告槽/证据室/深链断言都要用到失败案例。 */
|
|
64
|
+
async function seedRoot(): Promise<string> {
|
|
65
|
+
const root = await makeRoot();
|
|
66
|
+
await writeRun(
|
|
67
|
+
root,
|
|
68
|
+
"2026-07-08T10-00-00-000Z",
|
|
69
|
+
summaryOf(
|
|
70
|
+
[
|
|
71
|
+
res({ id: "weather/brooklyn", experimentId: "compare/bub", startedAt: "2026-07-08T10:00:01.000Z" }),
|
|
72
|
+
res({
|
|
73
|
+
id: "fixtures/button",
|
|
74
|
+
experimentId: "compare/bub",
|
|
75
|
+
outcome: "failed",
|
|
76
|
+
startedAt: "2026-07-08T10:00:02.000Z",
|
|
77
|
+
assertions: [
|
|
78
|
+
{ name: 'fileChanged("Button.tsx")', severity: "gate", score: 0, passed: false },
|
|
79
|
+
],
|
|
80
|
+
}),
|
|
81
|
+
],
|
|
82
|
+
{ startedAt: "2026-07-08T10:00:00.000Z" },
|
|
83
|
+
),
|
|
84
|
+
);
|
|
85
|
+
await writeRun(
|
|
86
|
+
root,
|
|
87
|
+
"2026-07-09T10-00-00-000Z",
|
|
88
|
+
summaryOf(
|
|
89
|
+
[res({ id: "weather/brooklyn", experimentId: "compare/codex", agent: "codex", startedAt: "2026-07-09T10:00:01.000Z" })],
|
|
90
|
+
{ agent: "codex", startedAt: "2026-07-09T10:00:00.000Z" },
|
|
91
|
+
),
|
|
92
|
+
);
|
|
93
|
+
return root;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
// ───────────────────────── 位置参数语义(单文件模式共存) ─────────────────────────
|
|
97
|
+
|
|
98
|
+
describe("resolveViewInput · 位置参数语义", () => {
|
|
99
|
+
it("存在的文件路径 → 单文件模式;其余位置参数按 eval id 前缀", async () => {
|
|
100
|
+
const root = await seedRoot();
|
|
101
|
+
const file = join(root, "2026-07-08T10-00-00-000Z", "summary.json");
|
|
102
|
+
expect(resolveViewInput(root, [file])).toEqual({ input: file, patterns: [] });
|
|
103
|
+
expect(resolveViewInput(root, ["weather", "fixtures/button"])).toEqual({
|
|
104
|
+
patterns: ["weather", "fixtures/button"],
|
|
105
|
+
});
|
|
106
|
+
expect(resolveViewInput(root, [])).toEqual({ patterns: [] });
|
|
107
|
+
});
|
|
108
|
+
|
|
109
|
+
it("目录位置参数:报错直说走 --run(位置参数留给 eval id 前缀)", async () => {
|
|
110
|
+
const root = await seedRoot();
|
|
111
|
+
expect(() => resolveViewInput(root, ["2026-07-08T10-00-00-000Z"])).toThrow(ViewInputError);
|
|
112
|
+
expect(() => resolveViewInput(root, ["2026-07-08T10-00-00-000Z"])).toThrow(/--run/);
|
|
113
|
+
});
|
|
114
|
+
|
|
115
|
+
it("单文件模式不与其它位置参数或 --run 混用,歧义报错直说", async () => {
|
|
116
|
+
const root = await seedRoot();
|
|
117
|
+
const file = join(root, "2026-07-08T10-00-00-000Z", "summary.json");
|
|
118
|
+
expect(() => resolveViewInput(root, [file, "weather"])).toThrow(/exactly one path/);
|
|
119
|
+
expect(() => resolveViewInput(root, [file], root)).toThrow(/--run/);
|
|
120
|
+
});
|
|
121
|
+
|
|
122
|
+
it("--run 指向不存在的目录:直说", async () => {
|
|
123
|
+
const root = await seedRoot();
|
|
124
|
+
expect(() => resolveViewInput(root, [], join(root, "nope"))).toThrow(/Results directory not found/);
|
|
125
|
+
});
|
|
126
|
+
|
|
127
|
+
it("--run 换结果根,位置参数仍是前缀", async () => {
|
|
128
|
+
const root = await seedRoot();
|
|
129
|
+
expect(resolveViewInput("/elsewhere", ["weather"], root)).toEqual({ input: root, patterns: ["weather"] });
|
|
130
|
+
});
|
|
131
|
+
});
|
|
132
|
+
|
|
133
|
+
// ───────────────────────── 组合语义(与 show 对齐) ─────────────────────────
|
|
134
|
+
|
|
135
|
+
describe("loadViewScan · 组合语义", () => {
|
|
136
|
+
it("位置前缀收窄报告槽选集(榜单/overview);证据室快照不收窄,深链恒可达", async () => {
|
|
137
|
+
const root = await seedRoot();
|
|
138
|
+
const { viewData } = await loadViewScan(root, { patterns: ["weather"] });
|
|
139
|
+
// 报告槽:两实验都只剩 weather/brooklyn 一题,全过。
|
|
140
|
+
expect(viewData.overview.totals.evals).toBe(1);
|
|
141
|
+
expect(viewData.overview.totals.attempts).toBe(2); // 2 个实验各 1 attempt
|
|
142
|
+
for (const row of viewData.table.rows) {
|
|
143
|
+
expect(row.cells["pass-rate"]!.value).toBe(1);
|
|
144
|
+
}
|
|
145
|
+
// 证据室:快照明细仍含 fixtures/button(attempt 深链在收窄下也能解析)。
|
|
146
|
+
const allIds = viewData.snapshots.flatMap((s) => s.results.map((r) => r.id));
|
|
147
|
+
expect(allIds).toContain("fixtures/button");
|
|
148
|
+
});
|
|
149
|
+
|
|
150
|
+
it("--experiment 过滤:选集只留该实验", async () => {
|
|
151
|
+
const root = await seedRoot();
|
|
152
|
+
const { viewData } = await loadViewScan(root, { experiment: "compare/codex" });
|
|
153
|
+
expect(viewData.table.rows.map((r) => r.key)).toEqual(["compare/codex"]);
|
|
154
|
+
});
|
|
155
|
+
|
|
156
|
+
it("前缀/实验匹配不到:直说,不渲染空页面", async () => {
|
|
157
|
+
const root = await seedRoot();
|
|
158
|
+
await expect(loadViewScan(root, { patterns: ["nosuch"] })).rejects.toBeInstanceOf(ViewInputError);
|
|
159
|
+
await expect(loadViewScan(root, { patterns: ["nosuch"] })).rejects.toThrow(/weather\/brooklyn/);
|
|
160
|
+
await expect(loadViewScan(root, { experiment: "nosuch" })).rejects.toBeInstanceOf(ViewInputError);
|
|
161
|
+
});
|
|
162
|
+
|
|
163
|
+
it("全部缺省:默认行为不变(results.latest() 口径,不走合成)", async () => {
|
|
164
|
+
const root = await seedRoot();
|
|
165
|
+
const { viewData, reportHtml } = await loadViewScan(root);
|
|
166
|
+
expect(reportHtml).toBeUndefined();
|
|
167
|
+
expect(viewData.table.rows.map((r) => r.key).sort()).toEqual(["compare/bub", "compare/codex"]);
|
|
168
|
+
});
|
|
169
|
+
});
|
|
170
|
+
|
|
171
|
+
// ───────────────────────── 报告槽整槽替换 ─────────────────────────
|
|
172
|
+
|
|
173
|
+
describe("loadViewScan · --report 报告槽", () => {
|
|
174
|
+
it("报告树渲染为静态 HTML:官方水位 + 自定义摆法 + <Style> 产物 + 证据室深链,零 <script>", async () => {
|
|
175
|
+
const root = await seedRoot();
|
|
176
|
+
const scan = await loadViewScan(root, { report: { path: EXAM_REPORT, cwd: root } });
|
|
177
|
+
const html = scan.reportHtml!;
|
|
178
|
+
expect(html).toContain("考试成绩单"); // 自定义 Section
|
|
179
|
+
expect(html).toContain("nre-"); // 官方组件的稳定类名
|
|
180
|
+
expect(html).toContain("<style>.exam-note { color: #4a7; }</style>"); // <Style> 随树带走
|
|
181
|
+
expect(html).toContain("#/attempt/2026-07-08T10-00-00-000Z/"); // 失败案例深链进证据室
|
|
182
|
+
expect(html).not.toContain("<script"); // 报告槽产物零客户端 JS,不 hydrate
|
|
183
|
+
// 证据室数据契约(__NICEEVAL_VIEW_DATA__)原样保留:快照、attemptRef、skipped 不动。
|
|
184
|
+
expect(scan.viewData.snapshots.length).toBeGreaterThan(0);
|
|
185
|
+
expect(scan.viewData.snapshots.flatMap((s) => s.results).every((r) => r.attemptRef)).toBe(true);
|
|
186
|
+
expect(JSON.stringify(scan.viewData)).not.toContain("考试成绩单"); // 报告块不进 viewData
|
|
187
|
+
});
|
|
188
|
+
|
|
189
|
+
it("位置前缀对 --report 生效:收窄注入选集,报告只见范围内的 eval", async () => {
|
|
190
|
+
const root = await seedRoot();
|
|
191
|
+
const scan = await loadViewScan(root, {
|
|
192
|
+
patterns: ["weather"],
|
|
193
|
+
report: { path: EXAM_REPORT, cwd: root },
|
|
194
|
+
});
|
|
195
|
+
// 范围外的失败(fixtures/button)不再出现在报告里;范围内的实验行都在。
|
|
196
|
+
expect(scan.reportHtml).not.toContain("fixtures/button");
|
|
197
|
+
expect(scan.reportHtml).toContain("compare/bub");
|
|
198
|
+
expect(scan.reportHtml).toContain("compare/codex");
|
|
199
|
+
});
|
|
200
|
+
|
|
201
|
+
it("show --report 与 view --report 吃同一个报告文件,判决口径一致", async () => {
|
|
202
|
+
const root = await seedRoot();
|
|
203
|
+
let text = "";
|
|
204
|
+
const code = await runShow(root, [], { run: root, report: EXAM_REPORT }, {
|
|
205
|
+
out: (s) => (text += s),
|
|
206
|
+
err: () => {},
|
|
207
|
+
width: 120,
|
|
208
|
+
});
|
|
209
|
+
expect(code).toBe(0);
|
|
210
|
+
const scan = await loadViewScan(root, { report: { path: EXAM_REPORT, cwd: root } });
|
|
211
|
+
// 同一棵树的两个面:同一份失败清单与同一个自定义 Section。
|
|
212
|
+
for (const needle of ["考试成绩单", "compare/bub", "compare/codex", "fixtures/button"]) {
|
|
213
|
+
expect(text).toContain(needle);
|
|
214
|
+
expect(scan.reportHtml).toContain(needle);
|
|
215
|
+
}
|
|
216
|
+
});
|
|
217
|
+
|
|
218
|
+
it("报告文件缺失 / 默认导出不是 defineReport 产物:ReportLoadError 直说", async () => {
|
|
219
|
+
const root = await seedRoot();
|
|
220
|
+
await expect(
|
|
221
|
+
loadViewScan(root, { report: { path: join(root, "missing.tsx"), cwd: root } }),
|
|
222
|
+
).rejects.toBeInstanceOf(ReportLoadError);
|
|
223
|
+
const bad = join(root, "bad.mjs");
|
|
224
|
+
await writeFile(bad, "export default {};\n", "utf-8");
|
|
225
|
+
await expect(loadViewScan(root, { report: { path: bad, cwd: root } })).rejects.toThrow(
|
|
226
|
+
/does not default-export a report/,
|
|
227
|
+
);
|
|
228
|
+
});
|
|
229
|
+
});
|
|
230
|
+
|
|
231
|
+
// ───────────────────────── dev server 装载语义:整页重算 ─────────────────────────
|
|
232
|
+
|
|
233
|
+
describe("loadViewScan · 报告文件变更整页重算", () => {
|
|
234
|
+
/** 不经包入口也合法的最小报告(与 show.test.ts 同一姿势):写 tmp .mjs 才能改内容重载。 */
|
|
235
|
+
function reportSource(marker: string): string {
|
|
236
|
+
return [
|
|
237
|
+
'const FACES = Symbol.for("niceeval.report.faces");',
|
|
238
|
+
"const Block = (props) => Block[FACES].web(props);",
|
|
239
|
+
"Block[FACES] = {",
|
|
240
|
+
` web: () => "${marker}",`,
|
|
241
|
+
` text: () => "${marker}",`,
|
|
242
|
+
"};",
|
|
243
|
+
"export default {",
|
|
244
|
+
' [Symbol.for("niceeval.report.definition")]: true,',
|
|
245
|
+
" build: () => ({",
|
|
246
|
+
' $$typeof: Symbol.for("react.transitional.element"),',
|
|
247
|
+
" type: Block,",
|
|
248
|
+
" props: {},",
|
|
249
|
+
" key: null,",
|
|
250
|
+
" }),",
|
|
251
|
+
"};",
|
|
252
|
+
"",
|
|
253
|
+
].join("\n");
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
it("重写报告文件后,下一次装载渲染新内容(mtime cache-busting)", async () => {
|
|
257
|
+
const root = await seedRoot();
|
|
258
|
+
const path = join(root, "report.mjs");
|
|
259
|
+
await writeFile(path, reportSource("FIRST_RENDER"), "utf-8");
|
|
260
|
+
const first = await loadViewScan(root, { report: { path, cwd: root } });
|
|
261
|
+
expect(first.reportHtml).toContain("FIRST_RENDER");
|
|
262
|
+
|
|
263
|
+
await writeFile(path, reportSource("SECOND_RENDER"), "utf-8");
|
|
264
|
+
// mtime 精度兜底:显式把 mtime 拨到未来,确保与首次装载可区分。
|
|
265
|
+
const future = new Date(Date.now() + 5000);
|
|
266
|
+
await utimes(path, future, future);
|
|
267
|
+
const second = await loadViewScan(root, { report: { path, cwd: root } });
|
|
268
|
+
expect(second.reportHtml).toContain("SECOND_RENDER");
|
|
269
|
+
expect(second.reportHtml).not.toContain("FIRST_RENDER");
|
|
270
|
+
});
|
|
271
|
+
});
|
|
272
|
+
|
|
273
|
+
// ───────────────────────── --out 静态导出 ─────────────────────────
|
|
274
|
+
|
|
275
|
+
describe("buildView · --out 与 --report", () => {
|
|
276
|
+
it("报告页为首页报告槽,证据室同站:index.html 含报告块与官方样式,工件照常复制", async () => {
|
|
277
|
+
const root = await seedRoot();
|
|
278
|
+
// 给 weather attempt 一份 events 工件,验证证据室工件照常进导出。
|
|
279
|
+
const artifactDir = join(root, "2026-07-09T10-00-00-000Z", "artifacts", "weather");
|
|
280
|
+
await mkdir(artifactDir, { recursive: true });
|
|
281
|
+
await writeFile(join(artifactDir, "events.json"), "[]", "utf-8");
|
|
282
|
+
const summaryPath = join(root, "2026-07-09T10-00-00-000Z", "summary.json");
|
|
283
|
+
const summary = JSON.parse(await readFile(summaryPath, "utf-8")) as RunSummary;
|
|
284
|
+
summary.results[0]!.artifactsDir = "artifacts/weather";
|
|
285
|
+
await writeFile(summaryPath, JSON.stringify(summary), "utf-8");
|
|
286
|
+
|
|
287
|
+
const out = join(root, "site");
|
|
288
|
+
await buildView({ input: root, out, scan: { report: { path: EXAM_REPORT, cwd: root } } });
|
|
289
|
+
|
|
290
|
+
const html = await readFile(join(out, "index.html"), "utf-8");
|
|
291
|
+
expect(html).toContain('<template id="niceeval-report">');
|
|
292
|
+
expect(html).toContain("考试成绩单");
|
|
293
|
+
expect(html).toContain("nre-"); // 官方组件样式(report/react/styles.css)随页注入
|
|
294
|
+
// 报告块本体零 <script>:静态块起于 <template>,内部只有标记与 <style>。
|
|
295
|
+
const block = html.split('<template id="niceeval-report">')[1]!.split("</template>")[0]!;
|
|
296
|
+
expect(block).not.toContain("<script");
|
|
297
|
+
// 证据室同站:工件按 /artifact/<base>/ 布局复制。
|
|
298
|
+
expect(existsSync(join(out, "artifact", "2026-07-09T10-00-00-000Z/artifacts/weather", "events.json"))).toBe(true);
|
|
299
|
+
});
|
|
300
|
+
|
|
301
|
+
it("默认导出(无 --report)不含报告块,行为不变", async () => {
|
|
302
|
+
const root = await seedRoot();
|
|
303
|
+
const out = join(root, "site");
|
|
304
|
+
await buildView({ input: root, out });
|
|
305
|
+
const html = await readFile(join(out, "index.html"), "utf-8");
|
|
306
|
+
expect(html).not.toContain('<template id="niceeval-report">');
|
|
307
|
+
});
|
|
308
|
+
});
|