niceeval 0.4.6 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/docs-site/zh/concepts/adapter.mdx +6 -6
- package/docs-site/zh/concepts/experiment.mdx +7 -7
- package/docs-site/zh/concepts/overview.mdx +1 -1
- package/docs-site/zh/concepts/tier.mdx +6 -6
- package/docs-site/zh/guides/agent-feedback-loop.mdx +64 -21
- package/docs-site/zh/guides/connect-your-agent.mdx +7 -7
- package/docs-site/zh/guides/custom-reports.mdx +243 -0
- package/docs-site/zh/guides/experiments.mdx +3 -3
- package/docs-site/zh/guides/report-components.mdx +252 -0
- package/docs-site/zh/guides/reporters.mdx +1 -1
- package/docs-site/zh/guides/results-data.mdx +224 -0
- package/docs-site/zh/guides/runner.mdx +1 -1
- package/docs-site/zh/guides/sandbox-agent.mdx +2 -2
- package/docs-site/zh/guides/viewing-results.mdx +147 -77
- package/docs-site/zh/guides/write-experiment.mdx +9 -9
- package/docs-site/zh/guides/write-send.mdx +6 -6
- package/docs-site/zh/index.mdx +1 -1
- package/docs-site/zh/introduction.mdx +1 -1
- package/docs-site/zh/reference/cli.mdx +11 -3
- package/docs-site/zh/reference/define-agent.mdx +5 -5
- package/docs-site/zh/reference/define-config.mdx +1 -1
- package/docs-site/zh/reference/define-eval.mdx +3 -3
- package/package.json +3 -2
- package/src/agents/ai-sdk.test.ts +1 -1
- package/src/agents/ai-sdk.ts +2 -2
- package/src/agents/claude-code.ts +1 -1
- package/src/agents/codex.ts +2 -2
- package/src/agents/streaming.test.ts +1 -1
- package/src/agents/types.ts +4 -4
- package/src/agents/ui-message-stream.test.ts +1 -1
- package/src/cli.ts +120 -18
- package/src/context/context.test.ts +1 -1
- package/src/context/context.ts +3 -3
- package/src/context/session.ts +2 -2
- package/src/context/types.ts +2 -2
- package/src/i18n/en.ts +29 -7
- package/src/i18n/zh-CN.ts +24 -2
- package/src/report/aggregate.ts +79 -50
- package/src/report/components.tsx +190 -0
- package/src/report/compute.ts +149 -84
- package/src/report/default-report.tsx +222 -0
- package/src/report/dual-face.test.tsx +527 -0
- package/src/report/format.ts +29 -0
- package/src/report/index.ts +79 -24
- package/src/report/load.ts +67 -0
- package/src/report/metrics.ts +2 -1
- package/src/report/param.ts +18 -0
- package/src/report/primitives.tsx +117 -0
- package/src/report/react/CaseList.tsx +2 -2
- package/src/report/react/DeltaTable.tsx +2 -2
- package/src/report/react/MetricBars.tsx +109 -0
- package/src/report/react/MetricLine.tsx +200 -0
- package/src/report/react/MetricMatrix.tsx +1 -1
- package/src/report/react/MetricScatter.tsx +1 -1
- package/src/report/react/MetricTable.tsx +1 -1
- package/src/report/react/RunOverview.tsx +4 -4
- package/src/report/react/Scoreboard.tsx +3 -3
- package/src/report/react/cell.tsx +1 -1
- package/src/report/react/fixtures.ts +98 -36
- package/src/report/react/format.ts +3 -28
- package/src/report/react/index.tsx +41 -24
- package/src/report/react/render.test.tsx +8 -8
- package/src/report/react/styles.css +77 -0
- package/src/report/report.test.ts +248 -128
- package/src/report/report.ts +69 -0
- package/src/report/text/faces.ts +356 -0
- package/src/report/text/layout.ts +125 -0
- package/src/report/text/plot.ts +127 -0
- package/src/report/tree.ts +220 -0
- package/src/report/types.ts +75 -27
- package/src/report/web.ts +40 -0
- package/src/results/copy.ts +95 -41
- package/src/results/format.ts +12 -11
- package/src/results/index.ts +34 -17
- package/src/results/open.ts +201 -81
- package/src/results/results.test.ts +595 -191
- package/src/results/select.ts +107 -55
- package/src/results/types.ts +137 -45
- package/src/results/writer.ts +258 -0
- package/src/runner/attempt.ts +3 -3
- package/src/runner/fingerprint.ts +1 -1
- package/src/runner/reporters/artifacts.ts +38 -78
- package/src/runner/reporters/braintrust.test.ts +2 -2
- package/src/runner/reporters/braintrust.ts +3 -3
- package/src/runner/run.ts +1 -1
- package/src/runner/types.ts +18 -12
- package/src/show/compose.ts +215 -0
- package/src/show/index.ts +263 -0
- package/src/show/render.ts +456 -0
- package/src/show/show.test.ts +505 -0
- package/src/view/app/App.tsx +85 -24
- package/src/view/app/components/CostScoreChart.tsx +29 -10
- package/src/view/app/components/ExperimentTable.tsx +27 -9
- package/src/view/app/components/GroupSelector.tsx +1 -1
- package/src/view/app/i18n.ts +18 -3
- package/src/view/app/lib/attempt-route.test.ts +9 -9
- package/src/view/app/lib/attempt-route.ts +5 -5
- package/src/view/app/lib/outcome.ts +1 -3
- package/src/view/app/lib/rows.ts +95 -15
- package/src/view/app/main.tsx +18 -6
- package/src/view/app/pages/RunsPage.tsx +4 -7
- package/src/view/app/pages/TracesPage.tsx +3 -8
- package/src/view/app/types.ts +50 -6
- package/src/view/client-dist/app.css +1 -1
- package/src/view/client-dist/app.js +20 -20
- package/src/view/data.test.ts +254 -0
- package/src/view/data.ts +336 -0
- package/src/view/index.ts +64 -10
- package/src/view/server.ts +31 -9
- package/src/view/shared/types.ts +47 -41
- package/src/view/styles.css +5 -0
- package/src/view/template.html +1 -0
- package/src/view/view-report.test.ts +308 -0
- package/docs-site/.mintignore +0 -13
- package/docs-site/AGENTS.md +0 -46
- package/docs-site/concepts/adapter.mdx +0 -287
- package/docs-site/concepts/assert.mdx +0 -243
- package/docs-site/concepts/drive.mdx +0 -118
- package/docs-site/concepts/evals.mdx +0 -108
- package/docs-site/concepts/experiment.mdx +0 -37
- package/docs-site/concepts/hitl.mdx +0 -83
- package/docs-site/concepts/judge.mdx +0 -129
- package/docs-site/concepts/overview.mdx +0 -98
- package/docs-site/concepts/tier.mdx +0 -42
- package/docs-site/docs-ref/00-index.md +0 -23
- package/docs-site/docs-ref/01-page-types.md +0 -43
- package/docs-site/docs-ref/03-style-rules.md +0 -45
- package/docs-site/docs-ref/04-code-examples.md +0 -37
- package/docs-site/docs-ref/05-dx-failure-paths.md +0 -45
- package/docs-site/docs-ref/06-checklists.md +0 -53
- package/docs-site/docs.json +0 -256
- package/docs-site/example/ai-agent-application.mdx +0 -148
- package/docs-site/example/claude-code-codex-plugin.mdx +0 -162
- package/docs-site/example/claude-code-codex-skill.mdx +0 -147
- package/docs-site/example/showcase.mdx +0 -39
- package/docs-site/example/tier1-ai-sdk-v7.mdx +0 -136
- package/docs-site/example/tier1-claude-sdk.mdx +0 -119
- package/docs-site/example/tier1-codex-sdk.mdx +0 -111
- package/docs-site/example/tier1-langgraph.mdx +0 -119
- package/docs-site/example/tier1-pi-sdk.mdx +0 -119
- package/docs-site/favicon.svg +0 -5
- package/docs-site/github-diff.css +0 -135
- package/docs-site/github-diff.js +0 -11
- package/docs-site/guides/authoring.mdx +0 -129
- package/docs-site/guides/ci-integration.mdx +0 -97
- package/docs-site/guides/connect-otel.mdx +0 -209
- package/docs-site/guides/connect-your-agent.mdx +0 -221
- package/docs-site/guides/dataset-fanout.mdx +0 -98
- package/docs-site/guides/experiments.mdx +0 -71
- package/docs-site/guides/fixtures.mdx +0 -147
- package/docs-site/guides/reporters.mdx +0 -113
- package/docs-site/guides/runner.mdx +0 -79
- package/docs-site/guides/sandbox-agent.mdx +0 -138
- package/docs-site/guides/sandbox-backends.mdx +0 -64
- package/docs-site/guides/scoring-guide.mdx +0 -92
- package/docs-site/guides/viewing-results.mdx +0 -195
- package/docs-site/guides/write-experiment.mdx +0 -81
- package/docs-site/guides/write-send.mdx +0 -347
- package/docs-site/index.mdx +0 -181
- package/docs-site/introduction.mdx +0 -141
- package/docs-site/quickstart.mdx +0 -136
- package/docs-site/reference/builtin-agents.mdx +0 -161
- package/docs-site/reference/capabilities.mdx +0 -76
- package/docs-site/reference/cli.mdx +0 -120
- package/docs-site/reference/define-agent.mdx +0 -168
- package/docs-site/reference/define-config.mdx +0 -41
- package/docs-site/reference/define-eval.mdx +0 -160
- package/docs-site/reference/events.mdx +0 -131
- package/docs-site/reference/expect.mdx +0 -112
- package/docs-site/tracker.js +0 -8
- package/src/report/react/data.ts +0 -17
- package/src/view/aggregate.ts +0 -103
- package/src/view/loader.test.ts +0 -61
- package/src/view/loader.ts +0 -257
package/src/cli.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
// niceeval CLI 入口。执行 eval 必须以 experiment 为单位;位置参数只在 exp 后筛 eval id 前缀。
|
|
2
2
|
// niceeval exp [组|配置] [pattern] 跑实验
|
|
3
|
+
// niceeval show [pattern] 终端读结果:榜单 / 单 eval / 证据切面 / 时间轴 / --report
|
|
3
4
|
// niceeval list 只列出发现到的 eval
|
|
4
5
|
// niceeval clean 删除 .niceeval/ 历史运行工件
|
|
5
6
|
|
|
@@ -19,15 +20,27 @@ import { Console as ConsoleReporter } from "./runner/reporters/console.ts";
|
|
|
19
20
|
import { JUnit } from "./runner/reporters/json.ts";
|
|
20
21
|
import { Live as LiveReporter, type LiveRow } from "./runner/reporters/live.ts";
|
|
21
22
|
import { Artifacts as ArtifactsReporter } from "./runner/reporters/artifacts.ts";
|
|
22
|
-
import {
|
|
23
|
+
import {
|
|
24
|
+
buildView,
|
|
25
|
+
startViewServer,
|
|
26
|
+
loadLatestResultsPerEval,
|
|
27
|
+
resolveViewInput,
|
|
28
|
+
IncompatibleResultsError,
|
|
29
|
+
ViewInputError,
|
|
30
|
+
} from "./view/index.ts";
|
|
31
|
+
import { ReportLoadError } from "./report/load.ts";
|
|
32
|
+
import { runShow } from "./show/index.ts";
|
|
23
33
|
import { t } from "./i18n/index.ts";
|
|
24
34
|
import { formatThrown, upsertManagedBlock } from "./util.ts";
|
|
25
35
|
import type { Config, DiscoveredExperiment, Reporter } from "./types.ts";
|
|
26
36
|
|
|
27
|
-
/**
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
37
|
+
/**
|
|
38
|
+
* view 的可预期用户错误:版本不同的报告(npx 提示)、位置参数/组合语义错误、
|
|
39
|
+
* --report 装载失败。打一句直说问题与下一步后退出,不抛堆栈。
|
|
40
|
+
*/
|
|
41
|
+
function exitOnViewUserError(e: unknown): never {
|
|
42
|
+
if (e instanceof IncompatibleResultsError || e instanceof ViewInputError || e instanceof ReportLoadError) {
|
|
43
|
+
process.stderr.write(e.message.endsWith("\n") ? e.message : `${e.message}\n`);
|
|
31
44
|
process.exit(1);
|
|
32
45
|
}
|
|
33
46
|
throw e;
|
|
@@ -51,6 +64,17 @@ interface Flags {
|
|
|
51
64
|
out?: string;
|
|
52
65
|
port?: number;
|
|
53
66
|
help: boolean;
|
|
67
|
+
// ── show 专属(位置参数仍是 eval id 前缀;这些 flag 选「怎么看」)──
|
|
68
|
+
transcript: boolean;
|
|
69
|
+
trace: boolean;
|
|
70
|
+
diff: boolean;
|
|
71
|
+
/** --diff=<路径>(必须 = 连写;空格形式会把路径当 eval id 前缀,按文档如此)。 */
|
|
72
|
+
diffPath?: string;
|
|
73
|
+
history: boolean;
|
|
74
|
+
experiment?: string;
|
|
75
|
+
attempt?: number;
|
|
76
|
+
run?: string;
|
|
77
|
+
report?: string;
|
|
54
78
|
}
|
|
55
79
|
|
|
56
80
|
// 表驱动的 flag 定义(node:util parseArgs)。--no-x 显式声明,不依赖 allowNegative(需 Node 20.14+,
|
|
@@ -72,6 +96,17 @@ const FLAG_OPTIONS = {
|
|
|
72
96
|
// 免得照旧文档写的脚本直接崩;真正实现前不接任何行为。
|
|
73
97
|
watch: { type: "boolean" },
|
|
74
98
|
json: { type: "boolean" },
|
|
99
|
+
// show 的证据切面 / 时间轴 / 报告装载(docs-site/zh/guides/viewing-results.mdx)。
|
|
100
|
+
transcript: { type: "boolean" },
|
|
101
|
+
trace: { type: "boolean" },
|
|
102
|
+
// --diff 是布尔;--diff=<路径> 在 parseArgs 前预扫成 diffPath(路径必须 = 连写,
|
|
103
|
+
// 空格形式的下一个 token 仍是位置参数 = eval id 前缀,与文档一致)。
|
|
104
|
+
diff: { type: "boolean" },
|
|
105
|
+
history: { type: "boolean" },
|
|
106
|
+
experiment: { type: "string" },
|
|
107
|
+
attempt: { type: "string" },
|
|
108
|
+
run: { type: "string" },
|
|
109
|
+
report: { type: "string" },
|
|
75
110
|
dry: { type: "boolean" },
|
|
76
111
|
quiet: { type: "boolean" },
|
|
77
112
|
force: { type: "boolean" },
|
|
@@ -96,6 +131,17 @@ function numberFlag(name: string, raw: string | undefined): number | undefined {
|
|
|
96
131
|
function parseArgs(argv: string[]): { command: string; positionals: string[]; flags: Flags } {
|
|
97
132
|
if (argv[0] === "--") argv = argv.slice(1);
|
|
98
133
|
|
|
134
|
+
// --diff=<路径> 预扫:diff 本体是布尔(裸 --diff = 文件级摘要),路径只接受 = 连写。
|
|
135
|
+
let diffPath: string | undefined;
|
|
136
|
+
argv = argv.map((arg) => {
|
|
137
|
+
if (arg.startsWith("--diff=")) {
|
|
138
|
+
const path = arg.slice("--diff=".length);
|
|
139
|
+
if (path) diffPath = path;
|
|
140
|
+
return "--diff";
|
|
141
|
+
}
|
|
142
|
+
return arg;
|
|
143
|
+
});
|
|
144
|
+
|
|
99
145
|
let values: Record<string, string | boolean | undefined>;
|
|
100
146
|
let rawPositionals: string[];
|
|
101
147
|
try {
|
|
@@ -113,7 +159,7 @@ function parseArgs(argv: string[]): { command: string; positionals: string[]; fl
|
|
|
113
159
|
}
|
|
114
160
|
|
|
115
161
|
// 第一个位置参数若是已知命令,则为命令;其余是 eval id 前缀 / view 输入。
|
|
116
|
-
const commands = new Set(["exp", "list", "view", "clean", "init", "watch", "run"]);
|
|
162
|
+
const commands = new Set(["exp", "show", "list", "view", "clean", "init", "watch", "run"]);
|
|
117
163
|
let command = "run";
|
|
118
164
|
let positionals = rawPositionals;
|
|
119
165
|
if (rawPositionals[0] && commands.has(rawPositionals[0])) {
|
|
@@ -139,6 +185,15 @@ function parseArgs(argv: string[]): { command: string; positionals: string[]; fl
|
|
|
139
185
|
earlyExit: values["no-early-exit"] === true ? false : values["early-exit"] === true ? true : undefined,
|
|
140
186
|
open: values["no-open"] === true ? false : values.open === true ? true : undefined,
|
|
141
187
|
help: values.help === true,
|
|
188
|
+
transcript: values.transcript === true,
|
|
189
|
+
trace: values.trace === true,
|
|
190
|
+
diff: values.diff === true && diffPath === undefined,
|
|
191
|
+
diffPath,
|
|
192
|
+
history: values.history === true,
|
|
193
|
+
experiment: values.experiment as string | undefined,
|
|
194
|
+
attempt: numberFlag("attempt", values.attempt as string | undefined),
|
|
195
|
+
run: values.run as string | undefined,
|
|
196
|
+
report: values.report as string | undefined,
|
|
142
197
|
};
|
|
143
198
|
return { command, positionals, flags };
|
|
144
199
|
}
|
|
@@ -184,19 +239,35 @@ async function loadConfig(cwd: string): Promise<Config> {
|
|
|
184
239
|
return mod.default;
|
|
185
240
|
}
|
|
186
241
|
|
|
187
|
-
// AGENTS.md 托管区块:告诉在这个项目里干活的 coding agent「niceeval 不在你的训练数据里,
|
|
188
|
-
//
|
|
242
|
+
// AGENTS.md/CLAUDE.md 托管区块:告诉在这个项目里干活的 coding agent「niceeval 不在你的训练数据里,
|
|
243
|
+
// 先读随包文档,跑完读结构化结果」。随包只发中文准绳版文档(英文站是手工同步、可能滞后,
|
|
244
|
+
// 不进包,见 package.json 的 files);init 时写入/刷新;标记之外的用户内容永不触碰。
|
|
189
245
|
const AGENT_RULES_BEGIN = "<!-- BEGIN:niceeval-agent-rules -->";
|
|
190
246
|
const AGENT_RULES_END = "<!-- END:niceeval-agent-rules -->";
|
|
191
247
|
const AGENT_RULES_CONTENT = [
|
|
192
248
|
"# niceeval is NOT in your training data",
|
|
193
249
|
"",
|
|
194
250
|
"Its APIs and conventions may differ from anything you have seen. Read the relevant",
|
|
195
|
-
"guide in `node_modules/niceeval/docs-site/`
|
|
196
|
-
"
|
|
197
|
-
"
|
|
251
|
+
"guide in `node_modules/niceeval/docs-site/zh/` before writing any eval, experiment,",
|
|
252
|
+
"adapter, or niceeval config. The bundled docs are Chinese-only — that is the single",
|
|
253
|
+
"authoritative, always-current version; read it regardless of your working language.",
|
|
254
|
+
"After a run, drill into failures with `niceeval show <eval id>` (add `--transcript` /",
|
|
255
|
+
"`--trace` / `--diff` for evidence); the `summary.json` path the CLI prints and the",
|
|
256
|
+
"artifact files it references are the structured source of truth.",
|
|
198
257
|
].join("\n");
|
|
199
258
|
|
|
259
|
+
// 优先复用已有的 AGENTS.md;项目只有 CLAUDE.md(没有 AGENTS.md)时改写 CLAUDE.md 本身,
|
|
260
|
+
// 不再另建一份重复文件;两者都没有则新建 AGENTS.md。CLAUDE.md 是指向 AGENTS.md 的符号链接时,
|
|
261
|
+
// existsSync 会 follow 链接——目标存在则直接算作「AGENTS.md 已存在」,写入的还是同一份内容,
|
|
262
|
+
// 不会产生分裂;目标不存在(悬空链接)则落回新建 AGENTS.md,写入后链接自然生效。
|
|
263
|
+
function resolveAgentDocPath(cwd: string): string {
|
|
264
|
+
const agentsPath = join(cwd, "AGENTS.md");
|
|
265
|
+
if (existsSync(agentsPath)) return agentsPath;
|
|
266
|
+
const claudePath = join(cwd, "CLAUDE.md");
|
|
267
|
+
if (existsSync(claudePath)) return claudePath;
|
|
268
|
+
return agentsPath;
|
|
269
|
+
}
|
|
270
|
+
|
|
200
271
|
async function initProject(cwd: string): Promise<void> {
|
|
201
272
|
await mkdir(join(cwd, "evals"), { recursive: true });
|
|
202
273
|
const configPath = join(cwd, "niceeval.config.ts");
|
|
@@ -214,10 +285,10 @@ async function initProject(cwd: string): Promise<void> {
|
|
|
214
285
|
"utf-8",
|
|
215
286
|
);
|
|
216
287
|
}
|
|
217
|
-
const
|
|
218
|
-
const existing = existsSync(
|
|
288
|
+
const agentDocPath = resolveAgentDocPath(cwd);
|
|
289
|
+
const existing = existsSync(agentDocPath) ? await readFile(agentDocPath, "utf-8") : "";
|
|
219
290
|
const next = upsertManagedBlock(existing, AGENT_RULES_BEGIN, AGENT_RULES_END, AGENT_RULES_CONTENT);
|
|
220
|
-
if (next !== existing) await writeFile(
|
|
291
|
+
if (next !== existing) await writeFile(agentDocPath, next, "utf-8");
|
|
221
292
|
}
|
|
222
293
|
|
|
223
294
|
async function openBrowser(url: string): Promise<boolean> {
|
|
@@ -265,12 +336,27 @@ async function main(): Promise<void> {
|
|
|
265
336
|
}
|
|
266
337
|
|
|
267
338
|
if (command === "view") {
|
|
339
|
+
// 位置参数 = eval id 前缀(收窄报告槽选集);存在的文件路径 = 单文件模式;
|
|
340
|
+
// 结果目录经 --run 递入;--report 整槽替换报告槽(与 show --report 吃同一个文件)。
|
|
341
|
+
let viewInput: { input?: string; patterns: string[] };
|
|
342
|
+
try {
|
|
343
|
+
viewInput = resolveViewInput(cwd, positionals, flags.run);
|
|
344
|
+
} catch (e) {
|
|
345
|
+
exitOnViewUserError(e);
|
|
346
|
+
}
|
|
347
|
+
const scan = {
|
|
348
|
+
patterns: viewInput.patterns,
|
|
349
|
+
...(flags.experiment !== undefined ? { experiment: flags.experiment } : {}),
|
|
350
|
+
...(flags.report !== undefined ? { report: { path: flags.report, cwd } } : {}),
|
|
351
|
+
};
|
|
268
352
|
if (flags.out) {
|
|
269
|
-
const out = await buildView({ input:
|
|
353
|
+
const out = await buildView({ input: viewInput.input, out: flags.out, scan }).catch(exitOnViewUserError);
|
|
270
354
|
process.stdout.write(t("cli.view.exportedDir", { out }));
|
|
271
355
|
process.exit(0);
|
|
272
356
|
}
|
|
273
|
-
const server = await startViewServer({ input:
|
|
357
|
+
const server = await startViewServer({ input: viewInput.input, port: flags.port, scan }).catch(
|
|
358
|
+
exitOnViewUserError,
|
|
359
|
+
);
|
|
274
360
|
process.stdout.write(t("cli.view.url", { url: server.url }));
|
|
275
361
|
if (flags.open !== false) {
|
|
276
362
|
const opened = await openBrowser(server.url);
|
|
@@ -280,6 +366,22 @@ async function main(): Promise<void> {
|
|
|
280
366
|
await new Promise(() => {});
|
|
281
367
|
}
|
|
282
368
|
|
|
369
|
+
if (command === "show") {
|
|
370
|
+
// show 不依赖 niceeval.config.ts:读的是 .niceeval/(或 --run 指定目录)的落盘结果。
|
|
371
|
+
const code = await runShow(cwd, positionals, {
|
|
372
|
+
transcript: flags.transcript,
|
|
373
|
+
trace: flags.trace,
|
|
374
|
+
diff: flags.diff,
|
|
375
|
+
diffPath: flags.diffPath,
|
|
376
|
+
history: flags.history,
|
|
377
|
+
experiment: flags.experiment,
|
|
378
|
+
attempt: flags.attempt,
|
|
379
|
+
run: flags.run,
|
|
380
|
+
report: flags.report,
|
|
381
|
+
});
|
|
382
|
+
process.exit(code);
|
|
383
|
+
}
|
|
384
|
+
|
|
283
385
|
if (command === "clean") {
|
|
284
386
|
await rm(join(cwd, ".niceeval"), { recursive: true, force: true });
|
|
285
387
|
process.stdout.write(t("cli.clean.done"));
|
|
@@ -333,7 +435,7 @@ async function main(): Promise<void> {
|
|
|
333
435
|
agent: exp.agent,
|
|
334
436
|
model: exp.model,
|
|
335
437
|
reasoningEffort: exp.reasoningEffort,
|
|
336
|
-
|
|
438
|
+
params: exp.params ?? {},
|
|
337
439
|
runs: flags.runs ?? envNumber("NICEEVAL_RUNS") ?? exp.runs ?? 1,
|
|
338
440
|
earlyExit: flags.earlyExit ?? exp.earlyExit ?? true,
|
|
339
441
|
sandbox: exp.sandbox ?? config.sandbox,
|
|
@@ -349,7 +451,7 @@ async function main(): Promise<void> {
|
|
|
349
451
|
}
|
|
350
452
|
} else {
|
|
351
453
|
// 裸 run / `niceeval <eval>` 不再执行。运行配置必须来自 experiments/,
|
|
352
|
-
// 这样 agent/model/
|
|
454
|
+
// 这样 agent/model/params/runs/budget 与结果聚合都有可签入的身份。
|
|
353
455
|
const experiments = await discoverExperiments(cwd);
|
|
354
456
|
const asExp = experiments.filter((e) =>
|
|
355
457
|
positionals.some((p) => e.group === p || e.id === p || e.id.startsWith(p + "/")),
|
package/src/context/context.ts
CHANGED
|
@@ -61,7 +61,7 @@ export interface ContextDeps {
|
|
|
61
61
|
sandbox: Sandbox;
|
|
62
62
|
model?: string;
|
|
63
63
|
reasoningEffort?: string;
|
|
64
|
-
|
|
64
|
+
params: Record<string, unknown>;
|
|
65
65
|
signal: AbortSignal;
|
|
66
66
|
log(msg: string): void;
|
|
67
67
|
judge: JudgeConfig | undefined;
|
|
@@ -92,7 +92,7 @@ export function createEvalContext(deps: ContextDeps): { context: TestContext; st
|
|
|
92
92
|
sandbox: deps.sandbox,
|
|
93
93
|
model: deps.model,
|
|
94
94
|
reasoningEffort: deps.reasoningEffort,
|
|
95
|
-
|
|
95
|
+
params: deps.params,
|
|
96
96
|
signal: deps.signal,
|
|
97
97
|
log: deps.log,
|
|
98
98
|
telemetry: deps.telemetry,
|
|
@@ -284,7 +284,7 @@ export function createEvalContext(deps: ContextDeps): { context: TestContext; st
|
|
|
284
284
|
signal: deps.signal,
|
|
285
285
|
model: deps.model,
|
|
286
286
|
reasoningEffort: deps.reasoningEffort,
|
|
287
|
-
|
|
287
|
+
params: deps.params,
|
|
288
288
|
log: deps.log,
|
|
289
289
|
skip: (reason: string) => {
|
|
290
290
|
if (reason.trim().length === 0) throw new Error(t("context.skipEmpty"));
|
package/src/context/session.ts
CHANGED
|
@@ -88,7 +88,7 @@ export interface SessionDeps {
|
|
|
88
88
|
sandbox: Sandbox;
|
|
89
89
|
model?: string;
|
|
90
90
|
reasoningEffort?: string;
|
|
91
|
-
|
|
91
|
+
params: Record<string, unknown>;
|
|
92
92
|
signal: AbortSignal;
|
|
93
93
|
log(msg: string): void;
|
|
94
94
|
/** tracing agent 的 OTLP 端点(经 send ctx 透给 adapter,用于注入导出 env)。 */
|
|
@@ -138,7 +138,7 @@ export class SessionManager {
|
|
|
138
138
|
signal: this.deps.signal,
|
|
139
139
|
model: this.deps.model,
|
|
140
140
|
reasoningEffort: this.deps.reasoningEffort,
|
|
141
|
-
|
|
141
|
+
params: this.deps.params,
|
|
142
142
|
sandbox: this.deps.sandbox,
|
|
143
143
|
session,
|
|
144
144
|
telemetry: this.deps.telemetry,
|
package/src/context/types.ts
CHANGED
|
@@ -274,8 +274,8 @@ export interface TestContext {
|
|
|
274
274
|
readonly model?: string;
|
|
275
275
|
/** 本次 attempt 的推理努力程度(如 "low"/"medium"/"high",取值由 adapter/模型决定)。 */
|
|
276
276
|
readonly reasoningEffort?: string;
|
|
277
|
-
/** 本次 attempt 生效的
|
|
278
|
-
readonly
|
|
277
|
+
/** 本次 attempt 生效的 params(experiment.params 与 CLI flag 合并后的只读视图)。 */
|
|
278
|
+
readonly params: Readonly<Record<string, unknown>>;
|
|
279
279
|
/** 打一行调试日志;有 live 进度回调时走该回调,否则落到 stderr,不出现在最终结果里。 */
|
|
280
280
|
log(msg: string): void;
|
|
281
281
|
/** 立即中止本 eval 并标记为 skipped(outcome / EvalResult.skipReason),reason 不能为空。 */
|
package/src/i18n/en.ts
CHANGED
|
@@ -20,7 +20,7 @@ export const en = {
|
|
|
20
20
|
"Ways to fix:\n" +
|
|
21
21
|
" - [init] Run `npx niceeval init` to scaffold niceeval.config.ts and evals/\n" +
|
|
22
22
|
" - [cd] Run from the project root that contains niceeval.config.ts\n" +
|
|
23
|
-
" Docs: node_modules/niceeval/docs-site/quickstart.mdx",
|
|
23
|
+
" Docs: node_modules/niceeval/docs-site/zh/quickstart.mdx",
|
|
24
24
|
"cli.config.noDefault": "niceeval.config.ts must default export defineConfig(...).",
|
|
25
25
|
"cli.dry.header": "\n[dry] {{evals}} evals × {{configs}} run configs:\n",
|
|
26
26
|
"cli.dry.noMatches": "(no matches)",
|
|
@@ -31,7 +31,7 @@ export const en = {
|
|
|
31
31
|
"budget for {{budgetKey}}: several attempts completed without any cost data (agent reports no usage and the model is not in the price table) — the budget cannot be enforced for this agent; continuing without the guard.\n",
|
|
32
32
|
"judge.modelMissing":
|
|
33
33
|
"No judge model configured. Set it in defineConfig({ judge: { model: \"...\" } }), the eval's judge config, or the NICEEVAL_JUDGE_MODEL environment variable (there is no built-in default model).\n" +
|
|
34
|
-
" Docs: node_modules/niceeval/docs-site/guides/scoring-guide.mdx",
|
|
34
|
+
" Docs: node_modules/niceeval/docs-site/zh/guides/scoring-guide.mdx",
|
|
35
35
|
"loaders.yamlMissing":
|
|
36
36
|
"loadYaml(\"{{path}}\") needs a YAML parser: run `pnpm add yaml` first (or switch to loadJson with a JSON dataset).",
|
|
37
37
|
"cli.flag.parseError": "{{message}}\nRun `niceeval --help` for usage.\n",
|
|
@@ -40,8 +40,18 @@ export const en = {
|
|
|
40
40
|
"niceeval — agent-native evals\n\n" +
|
|
41
41
|
"Usage:\n" +
|
|
42
42
|
" niceeval exp [group|experiment] [eval-id-prefix…] run experiments\n" +
|
|
43
|
+
" niceeval show [eval-id-prefix…] read results in the terminal\n" +
|
|
44
|
+
" bare: current verdicts per experiment (composed across runs);\n" +
|
|
45
|
+
" a single eval id: attempts + assertion details\n" +
|
|
46
|
+
" --transcript / --trace / --diff[=file] evidence slices of one eval\n" +
|
|
47
|
+
" --history cross-run timeline (mutually exclusive with --report)\n" +
|
|
48
|
+
" --run <dir> pin a results dir --experiment <id> one experiment\n" +
|
|
49
|
+
" --attempt <n> pick an attempt --report <file> custom report\n" +
|
|
43
50
|
" niceeval list list discovered evals\n" +
|
|
44
|
-
" niceeval view [summary.json
|
|
51
|
+
" niceeval view [eval-id-prefix…|summary.json] [--out dir] [--port n] [--no-open]\n" +
|
|
52
|
+
" report slot + evidence rooms; --report <file> swaps in your report\n" +
|
|
53
|
+
" (same file as show); --run <dir> pins a results dir;\n" +
|
|
54
|
+
" --experiment <id> one experiment\n" +
|
|
45
55
|
" --out <dir> exports a static site: index.html plus the viewer\n" +
|
|
46
56
|
" artifacts, ready for any static host\n" +
|
|
47
57
|
" niceeval clean delete .niceeval/ artifacts\n" +
|
|
@@ -53,6 +63,17 @@ export const en = {
|
|
|
53
63
|
"Positional args only select which evals to run (id prefixes); which agent and\n" +
|
|
54
64
|
"how to run come from experiments/ + flags. Env overrides (flag > env > config):\n" +
|
|
55
65
|
" NICEEVAL_RUNS NICEEVAL_MAX_CONCURRENCY NICEEVAL_TIMEOUT NICEEVAL_BUDGET\n",
|
|
66
|
+
"cli.show.noResults": "No results found under {{root}}. Run `niceeval exp` first, then `niceeval show`.\n",
|
|
67
|
+
"cli.show.runDirMissing": "Results directory not found: {{dir}}\n",
|
|
68
|
+
"cli.show.noEvalMatch": "No results matched: {{patterns}}. Evals with results: {{evals}}\n",
|
|
69
|
+
"cli.show.noExperimentMatch": "No experiment matched --experiment {{arg}}. Experiments with results: {{experiments}}\n",
|
|
70
|
+
"cli.show.historyReportConflict":
|
|
71
|
+
"`--history` and `--report` are mutually exclusive: --history is the built-in trend view. For a custom trend, compose exp.snapshots inside your report file instead.\n",
|
|
72
|
+
"cli.show.evidenceNeedsEval":
|
|
73
|
+
"--transcript / --trace / --diff show one eval's evidence, but the selection matched {{matched}} evals. Narrow to a single eval id first: niceeval show <eval id> --transcript\n",
|
|
74
|
+
"cli.show.attemptNeedsEval":
|
|
75
|
+
"--attempt picks one attempt of a single eval; pass a full eval id (and --experiment when several experiments ran it).\n",
|
|
76
|
+
"cli.show.attemptNotFound": "Attempt {{attempt}} not found for {{evalId}}. Available attempts: {{available}}\n",
|
|
56
77
|
"cli.eval.noMatch": "No eval matched: {{patterns}}.\n",
|
|
57
78
|
"cli.eval.noMatchHintExperiment": "Hint: \"{{pattern}}\" is an experiment{{kind}}; you probably meant: niceeval exp {{pattern}}\n",
|
|
58
79
|
"cli.eval.noMatchKnown": "Discovered {{count}} evals: {{evals}}\n",
|
|
@@ -61,7 +82,7 @@ export const en = {
|
|
|
61
82
|
"cli.experimentGroup": " group",
|
|
62
83
|
"cli.fallbackCleanupTimeout": "\ngraceful cleanup timed out; force-cleaning sandboxes...\n",
|
|
63
84
|
"cli.forceCleanupExit": "\nForce-cleaning sandboxes and exiting...\n",
|
|
64
|
-
"cli.init.done": "Ready: evals/, niceeval.config.ts, and the niceeval agent-rules block in AGENTS.md (points coding agents at node_modules/niceeval/docs-site).\n",
|
|
85
|
+
"cli.init.done": "Ready: evals/, niceeval.config.ts, and the niceeval agent-rules block in AGENTS.md (points coding agents at node_modules/niceeval/docs-site/zh).\n",
|
|
65
86
|
"cli.interruptCleanup": "\nInterrupted; cleaning up sandbox containers... (press again to force cleanup and exit)\n",
|
|
66
87
|
"cli.list.header": "Discovered {{count}} evals:\n",
|
|
67
88
|
"cli.noAgent": "No agent specified (use --agent <name>).\n",
|
|
@@ -70,17 +91,18 @@ export const en = {
|
|
|
70
91
|
"cli.resultsPath": "Structured results: {{path}} (each result's artifactsDir holds that attempt's events.json / trace.json / diff.json)\n",
|
|
71
92
|
"cli.run.experimentRequired":
|
|
72
93
|
"Run evals through an experiment: use `niceeval exp [group|config] [eval id prefix]`.\n" +
|
|
73
|
-
" Docs: node_modules/niceeval/docs-site/guides/write-experiment.mdx\n",
|
|
94
|
+
" Docs: node_modules/niceeval/docs-site/zh/guides/write-experiment.mdx\n",
|
|
74
95
|
"cli.run.experimentRequiredHint": "Hint: \"{{pattern}}\" is an experiment{{kind}}; you probably meant: niceeval exp {{pattern}}\n",
|
|
75
96
|
"cli.run.experimentRequiredKnown": "Discovered experiments: {{experiments}}\n",
|
|
76
97
|
"cli.sandboxFlagRemoved": "`--sandbox` is not a CLI flag. Set `sandbox` in the experiment (or `niceeval.config.ts` as a project-wide fallback) to dockerSandbox() / vercelSandbox() / e2bSandbox() (import from \"niceeval/sandbox\").\n",
|
|
77
98
|
"cli.unimplemented": "Command \"{{command}}\" is not implemented yet (MVP).\n",
|
|
78
99
|
"cli.view.exportedDir": "Exported static report site: {{out}} (serve the whole directory with any static host; opening index.html via file:// cannot fetch artifacts)\n",
|
|
79
100
|
"cli.view.incompatible": "{{dir}}: written by niceeval {{producer}} (schemaVersion {{schemaVersion}}); this CLI reads schemaVersion {{supported}}.\nRun `{{command}}` to view it.\n",
|
|
101
|
+
"cli.view.incompatibleForeign": "{{dir}}: written by {{name}} {{version}} (schemaVersion {{schemaVersion}}); this CLI reads schemaVersion {{supported}}.\nOpen this report with the tool that produced it.\n",
|
|
80
102
|
"cli.view.url": "niceeval view: {{url}}\n",
|
|
81
103
|
"context.capabilityMissing":
|
|
82
104
|
"Agent \"{{agent}}\" is not sandbox-backed (built with defineSandboxAgent), so t.{{method}} is unavailable. Use an agent built with defineSandboxAgent, or drop this assertion.\n" +
|
|
83
|
-
" Docs: node_modules/niceeval/docs-site/guides/sandbox-agent.mdx",
|
|
105
|
+
" Docs: node_modules/niceeval/docs-site/zh/guides/sandbox-agent.mdx",
|
|
84
106
|
"context.skipEmpty": "skip() requires a non-empty reason.",
|
|
85
107
|
"context.turnFailed": "This send returned failed (turn status = failed): {{message}}",
|
|
86
108
|
"context.turnFailedDefault": "This send returned failed (turn status = failed)",
|
|
@@ -172,7 +194,7 @@ export const en = {
|
|
|
172
194
|
"sandbox.backendNotImplemented": "{{backend}} sandbox backend is not implemented; use docker, vercel, or e2b",
|
|
173
195
|
"sandbox.missingSpec":
|
|
174
196
|
"sandbox agent needs a sandbox, but none was given. niceeval no longer picks a default — set `sandbox` in defineExperiment()/defineConfig() to dockerSandbox() / vercelSandbox() / e2bSandbox() (import from \"niceeval/sandbox\").\n" +
|
|
175
|
-
" Docs: node_modules/niceeval/docs-site/guides/sandbox-backends.mdx",
|
|
197
|
+
" Docs: node_modules/niceeval/docs-site/zh/guides/sandbox-backends.mdx",
|
|
176
198
|
"sandbox.dependencyMissing.docker": "Docker sandbox requires 'dockerode'. Install it with: pnpm add dockerode @types/dockerode",
|
|
177
199
|
"sandbox.dependencyMissing.e2b": "E2B sandbox requires 'e2b'. Install it with: pnpm add e2b",
|
|
178
200
|
"sandbox.dependencyMissing.vercel": "Vercel sandbox requires '@vercel/sandbox'. Install it with: pnpm add @vercel/sandbox",
|
package/src/i18n/zh-CN.ts
CHANGED
|
@@ -38,8 +38,17 @@ export const zhCN = {
|
|
|
38
38
|
"niceeval — agent-native evals\n\n" +
|
|
39
39
|
"用法:\n" +
|
|
40
40
|
" niceeval exp [组|实验] [eval-id 前缀…] 跑实验\n" +
|
|
41
|
+
" niceeval show [eval-id 前缀…] 终端读结果\n" +
|
|
42
|
+
" 裸跑:每个 experiment 的现刻判决(跨 run 合成)\n" +
|
|
43
|
+
" 单个 eval id:attempt 与断言明细\n" +
|
|
44
|
+
" --transcript / --trace / --diff[=文件] 单 eval 的证据切面\n" +
|
|
45
|
+
" --history 跨 run 时间轴(与 --report 互斥)\n" +
|
|
46
|
+
" --run <目录> 钉死结果目录 --experiment <id> 只看该实验\n" +
|
|
47
|
+
" --attempt <n> 指定 attempt --report <文件> 自定义报告\n" +
|
|
41
48
|
" niceeval list 列出发现到的 eval\n" +
|
|
42
|
-
" niceeval view [summary.json
|
|
49
|
+
" niceeval view [eval-id 前缀…|summary.json] [--out 目录] [--port n] [--no-open]\n" +
|
|
50
|
+
" 报告槽 + 证据室;--report <文件> 整槽换成自定义报告(与 show 同一文件)\n" +
|
|
51
|
+
" --run <目录> 钉死结果目录 --experiment <id> 只看该实验\n" +
|
|
43
52
|
" --out <目录> 静态导出:index.html 连同查看器工件,可直接静态托管\n" +
|
|
44
53
|
" niceeval clean 删除 .niceeval/ 历史工件\n" +
|
|
45
54
|
" niceeval init 脚手架 config + evals/\n\n" +
|
|
@@ -50,6 +59,18 @@ export const zhCN = {
|
|
|
50
59
|
"位置参数只选「跑哪些 eval」(id 前缀);对着哪个 agent、怎么跑来自 experiments/ 与\n" +
|
|
51
60
|
"标志。环境变量覆盖(标志 > 环境变量 > config):\n" +
|
|
52
61
|
" NICEEVAL_RUNS NICEEVAL_MAX_CONCURRENCY NICEEVAL_TIMEOUT NICEEVAL_BUDGET\n",
|
|
62
|
+
// show 的错误文案保持英文(错误文案英文的仓库约定);noResults 是提示,翻译。
|
|
63
|
+
"cli.show.noResults": "{{root}} 下没有结果。先 `niceeval exp` 跑一轮,再 `niceeval show`。\n",
|
|
64
|
+
"cli.show.runDirMissing": "Results directory not found: {{dir}}\n",
|
|
65
|
+
"cli.show.noEvalMatch": "No results matched: {{patterns}}. Evals with results: {{evals}}\n",
|
|
66
|
+
"cli.show.noExperimentMatch": "No experiment matched --experiment {{arg}}. Experiments with results: {{experiments}}\n",
|
|
67
|
+
"cli.show.historyReportConflict":
|
|
68
|
+
"`--history` and `--report` are mutually exclusive: --history is the built-in trend view. For a custom trend, compose exp.snapshots inside your report file instead.\n",
|
|
69
|
+
"cli.show.evidenceNeedsEval":
|
|
70
|
+
"--transcript / --trace / --diff show one eval's evidence, but the selection matched {{matched}} evals. Narrow to a single eval id first: niceeval show <eval id> --transcript\n",
|
|
71
|
+
"cli.show.attemptNeedsEval":
|
|
72
|
+
"--attempt picks one attempt of a single eval; pass a full eval id (and --experiment when several experiments ran it).\n",
|
|
73
|
+
"cli.show.attemptNotFound": "Attempt {{attempt}} not found for {{evalId}}. Available attempts: {{available}}\n",
|
|
53
74
|
"cli.eval.noMatch": "没有匹配的 eval:{{patterns}}。\n",
|
|
54
75
|
"cli.eval.noMatchHintExperiment": "提示:\"{{pattern}}\" 是实验{{kind}},你大概想跑:niceeval exp {{pattern}}\n",
|
|
55
76
|
"cli.eval.noMatchKnown": "已发现 {{count}} 个 eval:{{evals}}\n",
|
|
@@ -58,7 +79,7 @@ export const zhCN = {
|
|
|
58
79
|
"cli.experimentGroup": "组",
|
|
59
80
|
"cli.fallbackCleanupTimeout": "\ngraceful 清理超时,强制清理沙箱…\n",
|
|
60
81
|
"cli.forceCleanupExit": "\n强制清理沙箱并退出…\n",
|
|
61
|
-
"cli.init.done": "已就绪:evals/、niceeval.config.ts,以及 AGENTS.md 里的 niceeval agent 指引区块(指向 node_modules/niceeval/docs-site)。\n",
|
|
82
|
+
"cli.init.done": "已就绪:evals/、niceeval.config.ts,以及 AGENTS.md 里的 niceeval agent 指引区块(指向 node_modules/niceeval/docs-site/zh)。\n",
|
|
62
83
|
"cli.interruptCleanup": "\n收到中断,正在清理沙箱容器…(再按一次强制清理并退出)\n",
|
|
63
84
|
"cli.list.header": "发现 {{count}} 个 eval:\n",
|
|
64
85
|
"cli.noAgent": "未指定 agent(用 --agent <name>)。\n",
|
|
@@ -74,6 +95,7 @@ export const zhCN = {
|
|
|
74
95
|
"cli.unimplemented": "命令 \"{{command}}\" 暂未实现(MVP)。\n",
|
|
75
96
|
"cli.view.exportedDir": "已导出静态查看站:{{out}}(整个目录可直接静态托管;本地打开 {{out}}/index.html 需经 http 服务,file:// 下工件 fetch 不可用)\n",
|
|
76
97
|
"cli.view.incompatible": "{{dir}}: 由 niceeval {{producer}} 写入(schemaVersion {{schemaVersion}}),当前 CLI 只读 schemaVersion {{supported}}。\n运行 `{{command}}` 查看这份报告。\n",
|
|
98
|
+
"cli.view.incompatibleForeign": "{{dir}}: 由 {{name}} {{version}} 写入(schemaVersion {{schemaVersion}}),当前 CLI 只读 schemaVersion {{supported}}。\n请用写出它的那个工具查看这份报告。\n",
|
|
77
99
|
"cli.view.url": "niceeval view: {{url}}\n",
|
|
78
100
|
"context.capabilityMissing":
|
|
79
101
|
"agent \"{{agent}}\" 不是沙箱型(defineSandboxAgent 构造),t.{{method}} 这类断言只有沙箱型 agent 可用。换用 defineSandboxAgent 构造的 agent,或去掉这条断言。\n" +
|