niceeval 0.1.1 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -8
- package/README.zh.md +13 -13
- package/docs/README.md +119 -0
- package/docs/adapters/README.md +60 -0
- package/docs/adapters/authoring.md +179 -0
- package/docs/adapters/coding-agent-skills-plugins.md +324 -0
- package/docs/adapters/collection.md +128 -0
- package/docs/adapters/contract.md +263 -0
- package/docs/adapters/reference/agent-eval.md +215 -0
- package/docs/adapters/reference/agent-loop-apis.md +69 -0
- package/docs/adapters/reference/claude-code-otel-telemetry.md +100 -0
- package/docs/adapters/reference/eve-protocol.md +127 -0
- package/docs/adapters/reference/otel-genai.md +107 -0
- package/docs/adapters/reference/otel-instrumentation.md +65 -0
- package/docs/adapters/targets.md +99 -0
- package/docs/architecture.md +140 -0
- package/docs/assertions.md +388 -0
- package/docs/capabilities-by-construction.md +48 -0
- package/docs/cli.md +171 -0
- package/docs/concepts.md +106 -0
- package/docs/e2e-ci.md +295 -0
- package/docs/eval-authoring.md +319 -0
- package/docs/experiments.md +154 -0
- package/docs/getting-started.md +224 -0
- package/docs/multi-agent.md +145 -0
- package/docs/observability.md +333 -0
- package/docs/origin-integration.md +195 -0
- package/docs/references.md +35 -0
- package/docs/results-format.md +228 -0
- package/docs/runner.md +104 -0
- package/docs/sandbox.md +276 -0
- package/docs/scoring.md +119 -0
- package/docs/source-map.md +106 -0
- package/docs/tier-sync.md +177 -0
- package/docs/view.md +157 -0
- package/docs/vision.md +88 -0
- package/package.json +35 -5
- package/src/agents/ai-sdk-otel.ts +0 -0
- package/src/agents/ai-sdk.test.ts +377 -0
- package/src/agents/ai-sdk.ts +577 -0
- package/src/agents/bub.ts +30 -37
- package/src/agents/claude-code.ts +7 -4
- package/src/agents/codex.ts +15 -10
- package/src/agents/index.ts +43 -1
- package/src/agents/sdk-streams.test.ts +145 -0
- package/src/agents/sdk-streams.ts +457 -0
- package/src/agents/shared.ts +77 -39
- package/src/agents/streaming.test.ts +152 -0
- package/src/agents/streaming.ts +195 -0
- package/src/agents/types.ts +218 -0
- package/src/agents/ui-message-stream.test.ts +249 -0
- package/src/agents/ui-message-stream.ts +372 -0
- package/src/cli.ts +124 -77
- package/src/context/context.test.ts +203 -7
- package/src/context/context.ts +158 -49
- package/src/context/session.test.ts +96 -0
- package/src/context/session.ts +119 -9
- package/src/context/types.ts +205 -0
- package/src/define.ts +4 -14
- package/src/expect/index.ts +8 -71
- package/src/i18n/core.ts +28 -0
- package/src/i18n/en.ts +47 -5
- package/src/i18n/index.ts +5 -17
- package/src/i18n/zh-CN.ts +47 -5
- package/src/index.ts +12 -0
- package/src/loaders/index.ts +8 -39
- package/src/o11y/otlp/canonical.ts +7 -6
- package/src/o11y/otlp/mappers/codex.ts +10 -8
- package/src/o11y/otlp/mappers/index.ts +9 -21
- package/src/o11y/otlp/receiver.ts +25 -7
- package/src/o11y/otlp/sandbox-receiver.ts +57 -21
- package/src/o11y/otlp/turn-otel.test.ts +110 -0
- package/src/o11y/otlp/turn-otel.ts +125 -0
- package/src/o11y/parsers/bub.ts +13 -11
- package/src/o11y/parsers/claude-code.ts +27 -51
- package/src/o11y/parsers/codex.ts +29 -46
- package/src/o11y/parsers/index.ts +5 -14
- package/src/o11y/tool-names.test.ts +61 -0
- package/src/o11y/tool-names.ts +74 -0
- package/src/o11y/types.ts +135 -0
- package/src/runner/attempt.ts +456 -0
- package/src/runner/fingerprint.ts +59 -0
- package/src/runner/remote-sandbox.ts +53 -0
- package/src/runner/report.ts +53 -0
- package/src/runner/reporters/artifacts.ts +25 -4
- package/src/runner/reporters/console.ts +2 -8
- package/src/runner/reporters/live.ts +2 -8
- package/src/runner/reporters/shared.ts +15 -0
- package/src/runner/reporters/table.ts +10 -62
- package/src/runner/run.ts +114 -619
- package/src/runner/types.ts +237 -0
- package/src/sandbox/checkpoint.ts +15 -13
- package/src/sandbox/docker-stream.ts +87 -0
- package/src/sandbox/docker.ts +42 -120
- package/src/sandbox/e2b.ts +24 -75
- package/src/sandbox/local-files.ts +33 -0
- package/src/sandbox/paths.test.ts +85 -0
- package/src/sandbox/paths.ts +45 -0
- package/src/sandbox/resolve.ts +10 -34
- package/src/sandbox/shell.ts +17 -0
- package/src/sandbox/source-files.ts +42 -2
- package/src/sandbox/types.ts +158 -0
- package/src/sandbox/vercel.ts +55 -71
- package/src/scoring/collector.ts +3 -2
- package/src/scoring/judge.ts +72 -146
- package/src/scoring/match.ts +79 -0
- package/src/scoring/types.ts +76 -0
- package/src/shared/aggregate.ts +48 -0
- package/src/shared/format.ts +20 -0
- package/src/shared/outcome.ts +48 -0
- package/src/shared/types.ts +37 -0
- package/src/types.ts +20 -911
- package/src/view/aggregate.ts +103 -0
- package/src/view/app/App.tsx +26 -5
- package/src/view/app/components/AttemptModal.tsx +12 -3
- package/src/view/app/components/CodeView.tsx +9 -5
- package/src/view/app/components/CopyControls.tsx +4 -4
- package/src/view/app/components/ExperimentTable.tsx +5 -5
- package/src/view/app/i18n.ts +31 -7
- package/src/view/app/lib/format.ts +17 -20
- package/src/view/app/lib/outcome.ts +4 -16
- package/src/view/app/main.tsx +3 -5
- package/src/view/app/pages/RunsPage.tsx +2 -2
- package/src/view/app/pages/TracesPage.tsx +2 -2
- package/src/view/app/shared.ts +2 -1
- package/src/view/app/types.ts +6 -43
- package/src/view/client-dist/app.css +1 -1
- package/src/view/client-dist/app.js +18 -18
- package/src/view/index.ts +24 -401
- package/src/view/loader.ts +234 -0
- package/src/view/server.ts +136 -0
- package/src/view/shared/types.ts +64 -0
- package/src/view/styles.css +60 -9
package/src/view/index.ts
CHANGED
|
@@ -1,406 +1,29 @@
|
|
|
1
|
-
//
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
import { mkdir,
|
|
6
|
-
import { dirname,
|
|
7
|
-
import
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
}
|
|
24
|
-
|
|
25
|
-
interface LeaderboardRow {
|
|
26
|
-
key: string;
|
|
27
|
-
experimentId?: string;
|
|
28
|
-
experiment?: EvalResult["experiment"];
|
|
29
|
-
group?: string;
|
|
30
|
-
label: string;
|
|
31
|
-
agent: string;
|
|
32
|
-
model?: string;
|
|
33
|
-
runs: number;
|
|
34
|
-
evals: number;
|
|
35
|
-
passed: number;
|
|
36
|
-
failed: number;
|
|
37
|
-
errored: number;
|
|
38
|
-
skipped: number;
|
|
39
|
-
passRate: number;
|
|
40
|
-
avgDurationMs: number;
|
|
41
|
-
usage: Usage;
|
|
42
|
-
estimatedCostUSD?: number;
|
|
43
|
-
/** 该实验组里最新一次 run 的 startedAt(ISO);详情展示「运行时间」。 */
|
|
44
|
-
lastRunAt?: string;
|
|
45
|
-
results: EvalResult[];
|
|
46
|
-
}
|
|
47
|
-
|
|
48
|
-
const OUTCOME_ORDER: Record<EvalResult["outcome"], number> = {
|
|
49
|
-
errored: 0,
|
|
50
|
-
failed: 0,
|
|
51
|
-
skipped: 1,
|
|
52
|
-
passed: 2,
|
|
53
|
-
};
|
|
54
|
-
|
|
55
|
-
const TEMPLATE_PLACEHOLDERS = {
|
|
56
|
-
styles: "<!-- __NICEEVAL_STYLES__ -->",
|
|
57
|
-
appCode: "__NICEEVAL_APP_CODE__",
|
|
58
|
-
viewData: "__NICEEVAL_VIEW_DATA_JSON__",
|
|
59
|
-
} as const;
|
|
60
|
-
|
|
61
|
-
/** 读最近一次运行的所有 EvalResult,供 --resume 跳过已通过的 eval。 */
|
|
62
|
-
export async function loadMostRecentResults(root = ".niceeval"): Promise<EvalResult[]> {
|
|
63
|
-
const summaries = await loadSummaries(root);
|
|
64
|
-
// loadSummaries 已按 startedAt 降序,第一个是最新的
|
|
65
|
-
return summaries[0]?.summary.results ?? [];
|
|
66
|
-
}
|
|
67
|
-
|
|
1
|
+
// 本地结果查看器入口:只做编排与对外导出。
|
|
2
|
+
// 读取/版本判定在 loader.ts,聚合在 aggregate.ts,HTTP 与 HTML 烘焙在 server.ts,
|
|
3
|
+
// server/前端共用的折叠口径、格式化与数据形状在 shared/。
|
|
4
|
+
|
|
5
|
+
import { mkdir, writeFile } from "node:fs/promises";
|
|
6
|
+
import { dirname, resolve } from "node:path";
|
|
7
|
+
import { loadSummaries } from "./loader.ts";
|
|
8
|
+
import { renderHtml, type ViewOptions } from "./server.ts";
|
|
9
|
+
|
|
10
|
+
export { startViewServer, type ViewOptions, type ViewServer } from "./server.ts";
|
|
11
|
+
export {
|
|
12
|
+
IncompatibleResultsError,
|
|
13
|
+
incompatibleHint,
|
|
14
|
+
incompatibleViewCommand,
|
|
15
|
+
loadMostRecentResults,
|
|
16
|
+
loadSummaries,
|
|
17
|
+
type IncompatibleRun,
|
|
18
|
+
type ScanResult,
|
|
19
|
+
type SkippedRun,
|
|
20
|
+
} from "./loader.ts";
|
|
21
|
+
|
|
22
|
+
/** 导出静态 HTML 报告(--out):一次性把当前结果烘焙进单个可分享文件。 */
|
|
68
23
|
export async function buildView(opts: ViewOptions = {}): Promise<string> {
|
|
69
|
-
const
|
|
24
|
+
const scan = await loadSummaries(opts.input);
|
|
70
25
|
const out = resolve(opts.out ?? ".niceeval/report.html");
|
|
71
26
|
await mkdir(dirname(out), { recursive: true });
|
|
72
|
-
await writeFile(out, await renderHtml(
|
|
27
|
+
await writeFile(out, await renderHtml(scan), "utf-8");
|
|
73
28
|
return out;
|
|
74
29
|
}
|
|
75
|
-
|
|
76
|
-
export async function startViewServer(opts: ViewOptions = {}): Promise<ViewServer> {
|
|
77
|
-
const input = opts.input;
|
|
78
|
-
const root = viewRoot(input);
|
|
79
|
-
const server = createServer(async (req, res) => {
|
|
80
|
-
try {
|
|
81
|
-
const url = new URL(req.url ?? "/", "http://127.0.0.1");
|
|
82
|
-
if (url.pathname === "/healthz") {
|
|
83
|
-
res.writeHead(200, { "content-type": "text/plain; charset=utf-8" });
|
|
84
|
-
res.end("ok");
|
|
85
|
-
return;
|
|
86
|
-
}
|
|
87
|
-
// 按需提供拆分工件(trace.json / events.json / …),前端展开时 fetch。
|
|
88
|
-
if (url.pathname === "/artifact") {
|
|
89
|
-
await serveArtifact(root, url.searchParams.get("p") ?? "", res);
|
|
90
|
-
return;
|
|
91
|
-
}
|
|
92
|
-
if (url.pathname !== "/") {
|
|
93
|
-
res.writeHead(404, { "content-type": "text/plain; charset=utf-8" });
|
|
94
|
-
res.end("not found");
|
|
95
|
-
return;
|
|
96
|
-
}
|
|
97
|
-
res.writeHead(200, {
|
|
98
|
-
"content-type": "text/html; charset=utf-8",
|
|
99
|
-
"cache-control": "no-store",
|
|
100
|
-
});
|
|
101
|
-
res.end(await renderHtml(await loadSummaries(input)));
|
|
102
|
-
} catch (e) {
|
|
103
|
-
res.writeHead(500, { "content-type": "text/plain; charset=utf-8" });
|
|
104
|
-
res.end(e instanceof Error ? e.stack ?? e.message : String(e));
|
|
105
|
-
}
|
|
106
|
-
});
|
|
107
|
-
|
|
108
|
-
const port = await listen(server, opts.port ?? 0);
|
|
109
|
-
return {
|
|
110
|
-
url: `http://127.0.0.1:${port}/`,
|
|
111
|
-
close: () =>
|
|
112
|
-
new Promise((resolveClose, reject) => {
|
|
113
|
-
server.close((err) => (err ? reject(err) : resolveClose()));
|
|
114
|
-
}),
|
|
115
|
-
};
|
|
116
|
-
}
|
|
117
|
-
|
|
118
|
-
/** 安全地把 root 下的工件文件吐回去(限定 .json,且解析后必须仍在 root 内)。 */
|
|
119
|
-
async function serveArtifact(
|
|
120
|
-
root: string,
|
|
121
|
-
rel: string,
|
|
122
|
-
res: import("node:http").ServerResponse,
|
|
123
|
-
): Promise<void> {
|
|
124
|
-
const abs = resolve(root, rel);
|
|
125
|
-
const within = abs === root || abs.startsWith(root + "/");
|
|
126
|
-
if (!within || !rel.endsWith(".json")) {
|
|
127
|
-
res.writeHead(400, { "content-type": "text/plain; charset=utf-8" });
|
|
128
|
-
res.end("bad artifact path");
|
|
129
|
-
return;
|
|
130
|
-
}
|
|
131
|
-
try {
|
|
132
|
-
const body = await readFile(abs, "utf-8");
|
|
133
|
-
res.writeHead(200, { "content-type": "application/json; charset=utf-8", "cache-control": "no-store" });
|
|
134
|
-
res.end(body);
|
|
135
|
-
} catch {
|
|
136
|
-
res.writeHead(404, { "content-type": "text/plain; charset=utf-8" });
|
|
137
|
-
res.end("artifact not found");
|
|
138
|
-
}
|
|
139
|
-
}
|
|
140
|
-
|
|
141
|
-
async function renderHtml(loaded: LoadedSummary[]): Promise<string> {
|
|
142
|
-
const latest = loaded[0]?.summary;
|
|
143
|
-
const rows = aggregateRows(loaded);
|
|
144
|
-
const totals = summarizeAll(loaded);
|
|
145
|
-
const template = await readViewAsset("template.html");
|
|
146
|
-
const styles = await readViewAsset("client-dist/app.css");
|
|
147
|
-
const app = await readViewAsset("client-dist/app.js");
|
|
148
|
-
const viewData = {
|
|
149
|
-
rows,
|
|
150
|
-
name: latest?.name,
|
|
151
|
-
lastRun: latest ? formatDate(latest.startedAt) : "No runs yet",
|
|
152
|
-
passRate: formatPercent(totals.passRate),
|
|
153
|
-
resultCount: String(totals.results),
|
|
154
|
-
duration: formatDuration(totals.durationMs),
|
|
155
|
-
cost: formatCost(totals.cost),
|
|
156
|
-
};
|
|
157
|
-
|
|
158
|
-
return template
|
|
159
|
-
.replace(TEMPLATE_PLACEHOLDERS.styles, () => `<style>\n${styles}\n</style>`)
|
|
160
|
-
.replace(TEMPLATE_PLACEHOLDERS.viewData, () => JSON.stringify(viewData).replace(/</g, "\\u003c"))
|
|
161
|
-
.replace(TEMPLATE_PLACEHOLDERS.appCode, () => JSON.stringify(app).replace(/</g, "\\u003c"));
|
|
162
|
-
}
|
|
163
|
-
|
|
164
|
-
async function readViewAsset(name: string): Promise<string> {
|
|
165
|
-
return readFile(new URL(name, import.meta.url), "utf-8");
|
|
166
|
-
}
|
|
167
|
-
|
|
168
|
-
async function listen(server: Server, preferredPort: number): Promise<number> {
|
|
169
|
-
const tryListen = (port: number): Promise<number> =>
|
|
170
|
-
new Promise((resolveListen, reject) => {
|
|
171
|
-
const onError = (err: NodeJS.ErrnoException) => {
|
|
172
|
-
server.off("listening", onListening);
|
|
173
|
-
reject(err);
|
|
174
|
-
};
|
|
175
|
-
const onListening = () => {
|
|
176
|
-
server.off("error", onError);
|
|
177
|
-
const address = server.address();
|
|
178
|
-
resolveListen(typeof address === "object" && address ? address.port : port);
|
|
179
|
-
};
|
|
180
|
-
server.once("error", onError);
|
|
181
|
-
server.once("listening", onListening);
|
|
182
|
-
server.listen(port, "127.0.0.1");
|
|
183
|
-
});
|
|
184
|
-
|
|
185
|
-
if (preferredPort === 0) return tryListen(0);
|
|
186
|
-
for (let port = preferredPort; port < preferredPort + 20; port++) {
|
|
187
|
-
try {
|
|
188
|
-
return await tryListen(port);
|
|
189
|
-
} catch (e) {
|
|
190
|
-
if ((e as NodeJS.ErrnoException).code !== "EADDRINUSE") throw e;
|
|
191
|
-
}
|
|
192
|
-
}
|
|
193
|
-
throw new Error(`No available port near ${preferredPort}`);
|
|
194
|
-
}
|
|
195
|
-
|
|
196
|
-
/** 服务/解析工件的根目录:输入是目录就用它,是文件就用其所在目录。 */
|
|
197
|
-
function viewRoot(input?: string): string {
|
|
198
|
-
const t = resolve(input ?? ".niceeval");
|
|
199
|
-
try {
|
|
200
|
-
return statSync(t).isFile() ? dirname(t) : t;
|
|
201
|
-
} catch {
|
|
202
|
-
return t;
|
|
203
|
-
}
|
|
204
|
-
}
|
|
205
|
-
|
|
206
|
-
async function loadSummaries(input?: string): Promise<LoadedSummary[]> {
|
|
207
|
-
const target = resolve(input ?? ".niceeval");
|
|
208
|
-
if (!existsSync(target)) return [];
|
|
209
|
-
const root = viewRoot(input);
|
|
210
|
-
const s = await stat(target);
|
|
211
|
-
if (s.isFile()) {
|
|
212
|
-
const summary = await readSummary(target);
|
|
213
|
-
attachArtifactBase(summary, target, root);
|
|
214
|
-
return [{ path: target, summary }];
|
|
215
|
-
}
|
|
216
|
-
|
|
217
|
-
const candidates = await findSummaryFiles(target);
|
|
218
|
-
const loaded: LoadedSummary[] = [];
|
|
219
|
-
for (const path of candidates) {
|
|
220
|
-
try {
|
|
221
|
-
const summary = await readSummary(path);
|
|
222
|
-
attachArtifactBase(summary, path, root);
|
|
223
|
-
loaded.push({ path, summary });
|
|
224
|
-
} catch {
|
|
225
|
-
// Ignore unrelated JSON files under .niceeval.
|
|
226
|
-
}
|
|
227
|
-
}
|
|
228
|
-
loaded.sort((a, b) => b.summary.startedAt.localeCompare(a.summary.startedAt));
|
|
229
|
-
return loaded;
|
|
230
|
-
}
|
|
231
|
-
|
|
232
|
-
/** 给每条 result 拼出相对 view 根的工件目录(前端据此 fetch trace.json 等)。 */
|
|
233
|
-
function attachArtifactBase(summary: RunSummary, summaryPath: string, root: string): void {
|
|
234
|
-
const runDir = dirname(summaryPath);
|
|
235
|
-
for (const r of summary.results) {
|
|
236
|
-
if (!r.artifactsDir) continue;
|
|
237
|
-
const abs = join(runDir, r.artifactsDir);
|
|
238
|
-
r.artifactBase = relative(root, abs).split(/[\\/]/).join("/");
|
|
239
|
-
r.artifactAbsBase = abs;
|
|
240
|
-
}
|
|
241
|
-
}
|
|
242
|
-
|
|
243
|
-
async function findSummaryFiles(dir: string): Promise<string[]> {
|
|
244
|
-
const entries = await readdir(dir, { withFileTypes: true });
|
|
245
|
-
const direct = entries.filter((e) => e.isFile() && e.name === "summary.json").map((e) => join(dir, e.name));
|
|
246
|
-
const nested = await Promise.all(entries.filter((e) => e.isDirectory()).map((e) => findSummaryFiles(join(dir, e.name))));
|
|
247
|
-
return [...direct, ...nested.flat()];
|
|
248
|
-
}
|
|
249
|
-
|
|
250
|
-
async function readSummary(path: string): Promise<RunSummary> {
|
|
251
|
-
const data = JSON.parse(await readFile(path, "utf-8")) as RunSummary;
|
|
252
|
-
if (!Array.isArray(data.results) || typeof data.startedAt !== "string") {
|
|
253
|
-
throw new Error(`${path} is not a niceeval summary`);
|
|
254
|
-
}
|
|
255
|
-
return data;
|
|
256
|
-
}
|
|
257
|
-
|
|
258
|
-
function aggregateRows(loaded: LoadedSummary[]): LeaderboardRow[] {
|
|
259
|
-
const groups = new Map<string, EvalResult[]>();
|
|
260
|
-
const lastRunAt = new Map<string, string>();
|
|
261
|
-
for (const item of loaded) {
|
|
262
|
-
for (const result of item.summary.results) {
|
|
263
|
-
const key = result.experimentId ? `exp|||${result.experimentId}` : `legacy|||${result.agent}|||${result.model ?? ""}`;
|
|
264
|
-
groups.set(key, [...(groups.get(key) ?? []), result]);
|
|
265
|
-
const prev = lastRunAt.get(key);
|
|
266
|
-
if (!prev || item.summary.startedAt > prev) lastRunAt.set(key, item.summary.startedAt);
|
|
267
|
-
}
|
|
268
|
-
}
|
|
269
|
-
|
|
270
|
-
return Array.from(groups.entries()).map(([key, results]) => {
|
|
271
|
-
const first = results[0]!;
|
|
272
|
-
const experimentId = first.experimentId;
|
|
273
|
-
const cost = sumMaybe(results.map((r) => r.estimatedCostUSD));
|
|
274
|
-
// 一行 = 一个实验,results 内按 eval id 折叠计票(passed/failed/通过率都是 eval 级)。
|
|
275
|
-
const stats = evalLevelStats(results, (r) => r.id);
|
|
276
|
-
return {
|
|
277
|
-
key,
|
|
278
|
-
experimentId,
|
|
279
|
-
experiment: first.experiment,
|
|
280
|
-
group: experimentGroup(experimentId),
|
|
281
|
-
label: displayExperimentName(experimentId) ?? fallbackExperimentLabel(first),
|
|
282
|
-
agent: first.agent,
|
|
283
|
-
model: first.model,
|
|
284
|
-
lastRunAt: lastRunAt.get(key),
|
|
285
|
-
runs: results.length, // 总 attempt 数(详情里作次要信息)
|
|
286
|
-
evals: stats.evals, // 去重后的 eval 数(成功率分母的口径)
|
|
287
|
-
passed: stats.passed,
|
|
288
|
-
failed: stats.failed,
|
|
289
|
-
errored: stats.errored,
|
|
290
|
-
skipped: stats.skipped,
|
|
291
|
-
passRate: stats.passRate,
|
|
292
|
-
avgDurationMs: avg(results.map((r) => r.durationMs)),
|
|
293
|
-
usage: sumUsage(results.map((r) => r.usage)),
|
|
294
|
-
estimatedCostUSD: cost,
|
|
295
|
-
results: results
|
|
296
|
-
.slice()
|
|
297
|
-
.sort((a, b) => OUTCOME_ORDER[a.outcome] - OUTCOME_ORDER[b.outcome] || a.id.localeCompare(b.id)),
|
|
298
|
-
};
|
|
299
|
-
});
|
|
300
|
-
}
|
|
301
|
-
|
|
302
|
-
function resultOutcome(result: EvalResult): EvalResult["outcome"] {
|
|
303
|
-
return result.outcome;
|
|
304
|
-
}
|
|
305
|
-
|
|
306
|
-
/**
|
|
307
|
-
* 把同一个 eval 的多轮 attempt 折叠成单一判决:任一轮通过 → 该 eval 通过(对齐 earlyExit
|
|
308
|
-
* 「先过一次即停」语义),否则按 failed > errored > skipped 取最严重的一个。
|
|
309
|
-
*/
|
|
310
|
-
function foldEvalOutcome(results: EvalResult[]): EvalResult["outcome"] {
|
|
311
|
-
const outcomes = results.map(resultOutcome);
|
|
312
|
-
if (outcomes.some((o) => o === "passed")) return "passed";
|
|
313
|
-
if (outcomes.some((o) => o === "failed")) return "failed";
|
|
314
|
-
if (outcomes.some((o) => o === "errored")) return "errored";
|
|
315
|
-
return "skipped";
|
|
316
|
-
}
|
|
317
|
-
|
|
318
|
-
/**
|
|
319
|
-
* 通过率与 passed/failed 一律按 eval 计票,不按 attempt:每个 eval 不管跑几轮都只占一票,先把它
|
|
320
|
-
* 的多轮折叠成单一判决再计数。否则 runs>1 时同一 eval 的 N 次 attempt 各算一票 —— 尤其 earlyExit
|
|
321
|
-
* 开时通过的 eval 只留 1 次、失败的 eval 跑满 N 次,失败 eval 被重复计入分母,把通过率拉低
|
|
322
|
-
* (见 docs/runner.md、docs/scoring.md)。keyOf 决定「一个 eval」的粒度:单实验按 eval id,
|
|
323
|
-
* 跨实验组按 experimentId|eval id。
|
|
324
|
-
*/
|
|
325
|
-
function evalLevelStats(results: EvalResult[], keyOf: (r: EvalResult) => string) {
|
|
326
|
-
const byEval = new Map<string, EvalResult[]>();
|
|
327
|
-
for (const r of results) byEval.set(keyOf(r), [...(byEval.get(keyOf(r)) ?? []), r]);
|
|
328
|
-
const counts = { passed: 0, failed: 0, errored: 0, skipped: 0 };
|
|
329
|
-
for (const group of byEval.values()) counts[foldEvalOutcome(group)] += 1;
|
|
330
|
-
const ran = counts.passed + counts.failed + counts.errored; // skipped 不进分母
|
|
331
|
-
return { evals: byEval.size, ...counts, passRate: ran ? counts.passed / ran : 0 };
|
|
332
|
-
}
|
|
333
|
-
|
|
334
|
-
function summarizeAll(loaded: LoadedSummary[]) {
|
|
335
|
-
const results = loaded.flatMap((s) => s.summary.results);
|
|
336
|
-
// 顶部总览同样按 eval 计票:每个(实验, eval)只算一份,跨实验/跨 run 不被 runs 灌票。
|
|
337
|
-
const groupKey = (r: EvalResult) => (r.experimentId ? `exp|||${r.experimentId}` : `legacy|||${r.agent}|||${r.model ?? ""}`);
|
|
338
|
-
const stats = evalLevelStats(results, (r) => `${groupKey(r)}|||${r.id}`);
|
|
339
|
-
return {
|
|
340
|
-
results: stats.evals,
|
|
341
|
-
passRate: stats.passRate,
|
|
342
|
-
durationMs: loaded.reduce((sum, s) => sum + (s.summary.durationMs ?? 0), 0),
|
|
343
|
-
cost: sumMaybe(loaded.map((s) => s.summary.estimatedCostUSD)),
|
|
344
|
-
};
|
|
345
|
-
}
|
|
346
|
-
|
|
347
|
-
function sumUsage(items: Array<Usage | undefined>): Usage {
|
|
348
|
-
return {
|
|
349
|
-
inputTokens: items.reduce((n, u) => n + (u?.inputTokens ?? 0), 0),
|
|
350
|
-
outputTokens: items.reduce((n, u) => n + (u?.outputTokens ?? 0), 0),
|
|
351
|
-
cacheReadTokens: items.reduce((n, u) => n + (u?.cacheReadTokens ?? 0), 0),
|
|
352
|
-
cacheWriteTokens: items.reduce((n, u) => n + (u?.cacheWriteTokens ?? 0), 0),
|
|
353
|
-
requests: items.reduce((n, u) => n + (u?.requests ?? 0), 0),
|
|
354
|
-
};
|
|
355
|
-
}
|
|
356
|
-
|
|
357
|
-
function sumMaybe(items: Array<number | undefined>): number | undefined {
|
|
358
|
-
const known = items.filter((n): n is number => n !== undefined);
|
|
359
|
-
return known.length ? known.reduce((sum, n) => sum + n, 0) : undefined;
|
|
360
|
-
}
|
|
361
|
-
|
|
362
|
-
function avg(items: number[]): number {
|
|
363
|
-
return items.length ? items.reduce((sum, n) => sum + n, 0) / items.length : 0;
|
|
364
|
-
}
|
|
365
|
-
|
|
366
|
-
function displayExperimentName(id: string | undefined): string | undefined {
|
|
367
|
-
if (!id) return undefined;
|
|
368
|
-
return id.split("/").filter(Boolean).at(-1) ?? id;
|
|
369
|
-
}
|
|
370
|
-
|
|
371
|
-
function experimentGroup(id: string | undefined): string | undefined {
|
|
372
|
-
if (!id || !id.includes("/")) return undefined;
|
|
373
|
-
return id.split("/").slice(0, -1).join("/");
|
|
374
|
-
}
|
|
375
|
-
|
|
376
|
-
function fallbackExperimentLabel(result: EvalResult): string {
|
|
377
|
-
if (result.experiment?.id) return displayExperimentName(result.experiment.id) ?? result.experiment.id;
|
|
378
|
-
if (result.model) return `${result.agent}/${result.model}`;
|
|
379
|
-
return result.agent || "ad hoc run";
|
|
380
|
-
}
|
|
381
|
-
|
|
382
|
-
function formatDate(iso: string): string {
|
|
383
|
-
return new Intl.DateTimeFormat("en", {
|
|
384
|
-
year: "numeric",
|
|
385
|
-
month: "short",
|
|
386
|
-
day: "numeric",
|
|
387
|
-
hour: "2-digit",
|
|
388
|
-
minute: "2-digit",
|
|
389
|
-
}).format(new Date(iso));
|
|
390
|
-
}
|
|
391
|
-
|
|
392
|
-
function formatPercent(v: number): string {
|
|
393
|
-
return `${Math.round(v * 100)}%`;
|
|
394
|
-
}
|
|
395
|
-
|
|
396
|
-
function formatDuration(ms: number): string {
|
|
397
|
-
if (!Number.isFinite(ms) || ms <= 0) return "0ms";
|
|
398
|
-
if (ms >= 60_000) return `${(ms / 60_000).toFixed(1)}m`;
|
|
399
|
-
if (ms >= 1000) return `${(ms / 1000).toFixed(2)}s`;
|
|
400
|
-
return `${Math.round(ms)}ms`;
|
|
401
|
-
}
|
|
402
|
-
|
|
403
|
-
function formatCost(n: number | undefined): string {
|
|
404
|
-
if (n === undefined || n <= 0) return "$0";
|
|
405
|
-
return `$${n.toFixed(n < 1 ? 3 : 2)}`;
|
|
406
|
-
}
|
|
@@ -0,0 +1,234 @@
|
|
|
1
|
+
// 结果 loader:发现并读取 summary.json,做版本判定与 normalize,不做任何聚合/渲染。
|
|
2
|
+
// 设计见 docs/view.md「结果版本机制」:先 normalize 再渲染;目录扫描不让单个坏 run 拖垮整页,
|
|
3
|
+
// 但「像报告却读不了/版本不同」的 run 必须记入 skipped,不许无声消失。
|
|
4
|
+
|
|
5
|
+
import { existsSync, statSync } from "node:fs";
|
|
6
|
+
import { readFile, readdir, stat } from "node:fs/promises";
|
|
7
|
+
import { dirname, join, relative, resolve } from "node:path";
|
|
8
|
+
import { RESULTS_FORMAT, RESULTS_SCHEMA_VERSION, type EvalResult, type RunSummary } from "../types.ts";
|
|
9
|
+
import { t } from "../i18n/index.ts";
|
|
10
|
+
|
|
11
|
+
export interface LoadedSummary {
|
|
12
|
+
path: string;
|
|
13
|
+
summary: RunSummary;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
/** schemaVersion 与当前 CLI 不同、按设计直接不兼容的 run;只占位提示,不解析内容。 */
|
|
17
|
+
export interface IncompatibleRun {
|
|
18
|
+
/** run 目录(summary.json 所在目录),相对 cwd;直接可拼进 npx 命令。 */
|
|
19
|
+
dir: string;
|
|
20
|
+
schemaVersion: number;
|
|
21
|
+
producerVersion?: string;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
/** 目录扫描里被跳过、但必须让用户知道的 run(版本不同 / 像报告却损坏)。 */
|
|
25
|
+
export interface SkippedRun {
|
|
26
|
+
/** summary.json 路径,相对 cwd。 */
|
|
27
|
+
path: string;
|
|
28
|
+
/** run 目录,相对 cwd。 */
|
|
29
|
+
dir: string;
|
|
30
|
+
reason: "incompatible-version" | "malformed";
|
|
31
|
+
schemaVersion?: number;
|
|
32
|
+
producerVersion?: string;
|
|
33
|
+
/** incompatible-version:拼好的 npx 查看命令。 */
|
|
34
|
+
command?: string;
|
|
35
|
+
/** malformed:一句诊断。 */
|
|
36
|
+
detail?: string;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
export interface ScanResult {
|
|
40
|
+
loaded: LoadedSummary[];
|
|
41
|
+
skipped: SkippedRun[];
|
|
42
|
+
/**
|
|
43
|
+
* artifactBase(相对 view 根)→ 宿主机绝对目录。只在 server 端内存里保留;
|
|
44
|
+
* 绝对路径不进 viewData,避免序列化进可分享的静态 HTML(信息泄漏且浏览器端用不到)。
|
|
45
|
+
*/
|
|
46
|
+
artifactDirs: Map<string, string>;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/** 用能读这份报告的 niceeval 版本查看的命令。 */
|
|
50
|
+
export function incompatibleViewCommand(run: IncompatibleRun): string {
|
|
51
|
+
return `npx niceeval@${run.producerVersion ?? "<version>"} view ${run.dir}`;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/** 版本不匹配的完整提示文案;CLI 单文件模式和目录扫描占位共用。 */
|
|
55
|
+
export function incompatibleHint(run: IncompatibleRun): string {
|
|
56
|
+
return t("cli.view.incompatible", {
|
|
57
|
+
dir: run.dir,
|
|
58
|
+
producer: run.producerVersion ?? "?",
|
|
59
|
+
schemaVersion: run.schemaVersion,
|
|
60
|
+
supported: RESULTS_SCHEMA_VERSION,
|
|
61
|
+
command: incompatibleViewCommand(run),
|
|
62
|
+
});
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/** 单文件模式读到版本不同的 summary 时抛出;CLI 捕获后打印提示退出,不当成普通错误堆栈。 */
|
|
66
|
+
export class IncompatibleResultsError extends Error {
|
|
67
|
+
constructor(readonly run: IncompatibleRun) {
|
|
68
|
+
super(incompatibleHint(run));
|
|
69
|
+
this.name = "IncompatibleResultsError";
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/** 像 niceeval 报告但读不了(JSON 损坏 / 必需字段坏);目录扫描记入 skipped,单文件模式直接失败。 */
|
|
74
|
+
export class MalformedResultsError extends Error {
|
|
75
|
+
constructor(readonly path: string, readonly detail: string) {
|
|
76
|
+
super(`${path}: ${detail}. The report may be corrupted; re-run the eval or delete this run directory.`);
|
|
77
|
+
this.name = "MalformedResultsError";
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/** 不是 niceeval 报告的无关 JSON;目录扫描静默忽略,单文件模式直接失败。 */
|
|
82
|
+
export class NotAReportError extends Error {
|
|
83
|
+
constructor(path: string) {
|
|
84
|
+
super(`${path} is not a niceeval summary`);
|
|
85
|
+
this.name = "NotAReportError";
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
/** 读最近一次运行的所有 EvalResult,供 --resume 跳过已通过的 eval。 */
|
|
90
|
+
export async function loadMostRecentResults(root = ".niceeval"): Promise<EvalResult[]> {
|
|
91
|
+
const { loaded } = await loadSummaries(root);
|
|
92
|
+
// loadSummaries 已按 startedAt 降序,第一个是最新的
|
|
93
|
+
return loaded[0]?.summary.results ?? [];
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/** 服务/解析工件的根目录:输入是目录就用它,是文件就用其所在目录。 */
|
|
97
|
+
export function viewRoot(input?: string): string {
|
|
98
|
+
const target = resolve(input ?? ".niceeval");
|
|
99
|
+
try {
|
|
100
|
+
return statSync(target).isFile() ? dirname(target) : target;
|
|
101
|
+
} catch {
|
|
102
|
+
return target;
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
export async function loadSummaries(input?: string): Promise<ScanResult> {
|
|
107
|
+
const target = resolve(input ?? ".niceeval");
|
|
108
|
+
const artifactDirs = new Map<string, string>();
|
|
109
|
+
if (!existsSync(target)) return { loaded: [], skipped: [], artifactDirs };
|
|
110
|
+
const root = viewRoot(input);
|
|
111
|
+
const s = await stat(target);
|
|
112
|
+
if (s.isFile()) {
|
|
113
|
+
// 单文件模式:这是用户明确指定的目标,任何读不了(版本/损坏/无关)都直接抛,由 CLI 打印提示退出。
|
|
114
|
+
const summary = normalizeSummary(await parseSummary(target), target);
|
|
115
|
+
return { loaded: [{ path: target, summary: withArtifactBases(summary, target, root, artifactDirs) }], skipped: [], artifactDirs };
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
const candidates = await findSummaryFiles(target);
|
|
119
|
+
const loaded: LoadedSummary[] = [];
|
|
120
|
+
const skipped: SkippedRun[] = [];
|
|
121
|
+
for (const path of candidates) {
|
|
122
|
+
try {
|
|
123
|
+
const summary = normalizeSummary(await parseSummary(path), path);
|
|
124
|
+
loaded.push({ path, summary: withArtifactBases(summary, path, root, artifactDirs) });
|
|
125
|
+
} catch (e) {
|
|
126
|
+
const entry = classifySkip(e, path);
|
|
127
|
+
// 无关 JSON(NotAReportError)可以静默忽略;像报告却读不了的必须记下来。
|
|
128
|
+
if (entry) skipped.push(entry);
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
loaded.sort((a, b) => b.summary.startedAt.localeCompare(a.summary.startedAt));
|
|
132
|
+
// run 目录名是时间戳,降序 ≈ 最新在前。
|
|
133
|
+
skipped.sort((a, b) => b.dir.localeCompare(a.dir));
|
|
134
|
+
return { loaded, skipped, artifactDirs };
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
/** 只做 IO:读文件 + JSON.parse。解析失败按「可能损坏的报告」处理(文件名就叫 summary.json)。 */
|
|
138
|
+
export async function parseSummary(path: string): Promise<unknown> {
|
|
139
|
+
let text: string;
|
|
140
|
+
try {
|
|
141
|
+
text = await readFile(path, "utf-8");
|
|
142
|
+
} catch (e) {
|
|
143
|
+
throw new MalformedResultsError(path, `cannot read file (${e instanceof Error ? e.message : String(e)})`);
|
|
144
|
+
}
|
|
145
|
+
try {
|
|
146
|
+
return JSON.parse(text);
|
|
147
|
+
} catch {
|
|
148
|
+
throw new MalformedResultsError(path, "invalid JSON");
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
/**
|
|
153
|
+
* 版本判定与形状校验,把磁盘 JSON 收成 view 内部统一模型:
|
|
154
|
+
* - 带 format 信封:format 不是 niceeval.results → 无关 JSON;schemaVersion 与当前不同 → 不兼容,
|
|
155
|
+
* 不解析、不迁移、不降级渲染(缺 schemaVersion 的早期信封按 1 处理)。
|
|
156
|
+
* - 无信封:有 results[] + startedAt 按 legacy v0 照读(未知字段忽略);连报告痕迹都没有 → 无关 JSON。
|
|
157
|
+
*/
|
|
158
|
+
export function normalizeSummary(raw: unknown, path: string): RunSummary {
|
|
159
|
+
if (typeof raw !== "object" || raw === null || Array.isArray(raw)) {
|
|
160
|
+
throw new MalformedResultsError(path, "not a JSON object");
|
|
161
|
+
}
|
|
162
|
+
const data = raw as Partial<RunSummary>;
|
|
163
|
+
if (data.format !== undefined && data.format !== RESULTS_FORMAT) throw new NotAReportError(path);
|
|
164
|
+
if (data.format === RESULTS_FORMAT) {
|
|
165
|
+
const version = data.schemaVersion ?? 1;
|
|
166
|
+
if (typeof version !== "number") throw new MalformedResultsError(path, "schemaVersion is not a number");
|
|
167
|
+
if (version !== RESULTS_SCHEMA_VERSION) {
|
|
168
|
+
throw new IncompatibleResultsError({
|
|
169
|
+
dir: relative(process.cwd(), dirname(path)) || ".",
|
|
170
|
+
schemaVersion: version,
|
|
171
|
+
producerVersion: data.producer?.version,
|
|
172
|
+
});
|
|
173
|
+
}
|
|
174
|
+
} else if (!("results" in data) && !("startedAt" in data)) {
|
|
175
|
+
throw new NotAReportError(path);
|
|
176
|
+
}
|
|
177
|
+
if (!Array.isArray(data.results) || typeof data.startedAt !== "string") {
|
|
178
|
+
throw new MalformedResultsError(path, "missing results[] or startedAt");
|
|
179
|
+
}
|
|
180
|
+
return data as RunSummary;
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
/** 把 loader 抛出的错误归类成 skipped 条目;无关 JSON 返回 undefined(不记)。 */
|
|
184
|
+
function classifySkip(e: unknown, path: string): SkippedRun | undefined {
|
|
185
|
+
const relPath = relative(process.cwd(), path) || path;
|
|
186
|
+
const dir = relative(process.cwd(), dirname(path)) || ".";
|
|
187
|
+
if (e instanceof IncompatibleResultsError) {
|
|
188
|
+
return {
|
|
189
|
+
path: relPath,
|
|
190
|
+
dir: e.run.dir,
|
|
191
|
+
reason: "incompatible-version",
|
|
192
|
+
schemaVersion: e.run.schemaVersion,
|
|
193
|
+
producerVersion: e.run.producerVersion,
|
|
194
|
+
command: incompatibleViewCommand(e.run),
|
|
195
|
+
};
|
|
196
|
+
}
|
|
197
|
+
if (e instanceof MalformedResultsError) {
|
|
198
|
+
return { path: relPath, dir, reason: "malformed", detail: e.detail };
|
|
199
|
+
}
|
|
200
|
+
if (e instanceof NotAReportError) return undefined;
|
|
201
|
+
// 未预期的错误也按 malformed 记:比静默吞掉更容易暴露 loader 自身的 bug。
|
|
202
|
+
return { path: relPath, dir, reason: "malformed", detail: e instanceof Error ? e.message : String(e) };
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
/**
|
|
206
|
+
* 给每条 result 拼出相对 view 根的工件目录(前端据此 fetch trace.json 等)。
|
|
207
|
+
* 返回新对象,不 mutate 读入的 summary;宿主机绝对路径只写进 artifactDirs(server 端内存),
|
|
208
|
+
* 不挂到 result 上,避免随 viewData 进静态 HTML。
|
|
209
|
+
*/
|
|
210
|
+
function withArtifactBases(
|
|
211
|
+
summary: RunSummary,
|
|
212
|
+
summaryPath: string,
|
|
213
|
+
root: string,
|
|
214
|
+
artifactDirs: Map<string, string>,
|
|
215
|
+
): RunSummary {
|
|
216
|
+
const runDir = dirname(summaryPath);
|
|
217
|
+
return {
|
|
218
|
+
...summary,
|
|
219
|
+
results: summary.results.map((r) => {
|
|
220
|
+
if (!r.artifactsDir) return r;
|
|
221
|
+
const abs = join(runDir, r.artifactsDir);
|
|
222
|
+
const base = relative(root, abs).split(/[\\/]/).join("/");
|
|
223
|
+
artifactDirs.set(base, abs);
|
|
224
|
+
return { ...r, artifactBase: base };
|
|
225
|
+
}),
|
|
226
|
+
};
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
async function findSummaryFiles(dir: string): Promise<string[]> {
|
|
230
|
+
const entries = await readdir(dir, { withFileTypes: true });
|
|
231
|
+
const direct = entries.filter((e) => e.isFile() && e.name === "summary.json").map((e) => join(dir, e.name));
|
|
232
|
+
const nested = await Promise.all(entries.filter((e) => e.isDirectory()).map((e) => findSummaryFiles(join(dir, e.name))));
|
|
233
|
+
return [...direct, ...nested.flat()];
|
|
234
|
+
}
|