niceeval 0.6.2 → 0.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/INDEX.md +23 -23
- package/README.zh.md +6 -6
- package/dist/agents/types.d.ts +2 -2
- package/dist/i18n/zh-CN.d.ts +3 -3
- package/dist/report/aggregate.d.ts +32 -26
- package/dist/report/aggregate.js +157 -76
- package/dist/report/built-in/index.d.ts +2 -0
- package/dist/report/built-in/index.js +8 -0
- package/dist/report/components.d.ts +91 -164
- package/dist/report/components.js +377 -114
- package/dist/report/compute.d.ts +86 -73
- package/dist/report/compute.js +592 -432
- package/dist/report/flag.d.ts +28 -17
- package/dist/report/flag.js +86 -16
- package/dist/report/format.d.ts +11 -11
- package/dist/report/format.js +17 -15
- package/dist/report/index.d.ts +16 -17
- package/dist/report/index.js +20 -22
- package/dist/report/load.js +3 -2
- package/dist/report/locale.d.ts +49 -34
- package/dist/report/locale.js +106 -58
- package/dist/report/metrics.d.ts +10 -3
- package/dist/report/metrics.js +46 -12
- package/dist/report/primitives.d.ts +42 -15
- package/dist/report/primitives.js +135 -26
- package/dist/report/react/AttemptList.d.ts +10 -8
- package/dist/report/react/AttemptList.js +18 -10
- package/dist/report/react/DeltaTable.js +19 -18
- package/dist/report/react/EvalList.d.ts +3 -3
- package/dist/report/react/EvalList.js +0 -0
- package/dist/report/react/ExperimentComparison.d.ts +4 -2
- package/dist/report/react/ExperimentComparison.js +5 -4
- package/dist/report/react/ExperimentList.d.ts +3 -3
- package/dist/report/react/ExperimentList.js +16 -15
- package/dist/report/react/MetricBars.js +5 -4
- package/dist/report/react/MetricLine.js +12 -5
- package/dist/report/react/MetricMatrix.js +1 -1
- package/dist/report/react/MetricScatter.js +54 -17
- package/dist/report/react/MetricTable.js +2 -12
- package/dist/report/react/ScopeSummary.d.ts +10 -0
- package/dist/report/react/ScopeSummary.js +17 -0
- package/dist/report/react/Scoreboard.js +6 -6
- package/dist/report/react/cell.js +2 -2
- package/dist/report/react/fixtures.d.ts +5 -9
- package/dist/report/react/fixtures.js +105 -149
- package/dist/report/react/index.d.ts +15 -5
- package/dist/report/react/index.js +18 -7
- package/dist/report/report.d.ts +137 -20
- package/dist/report/report.js +261 -34
- package/dist/report/text/faces.d.ts +17 -19
- package/dist/report/text/faces.js +225 -157
- package/dist/report/text/plot.js +1 -1
- package/dist/report/text/table.js +2 -2
- package/dist/report/tree.d.ts +90 -40
- package/dist/report/tree.js +252 -94
- package/dist/report/types.d.ts +245 -300
- package/dist/report/types.js +4 -3
- package/dist/report/web.d.ts +21 -5
- package/dist/report/web.js +42 -16
- package/dist/results/select.d.ts +38 -16
- package/dist/results/select.js +73 -25
- package/dist/results/types.d.ts +38 -14
- package/dist/shared/aggregate.d.ts +3 -2
- package/dist/shared/aggregate.js +5 -4
- package/docs-site/zh/README.md +44 -0
- package/docs-site/zh/examples/ai-agent-application.mdx +63 -0
- package/docs-site/zh/examples/coding-agent-extensions.mdx +57 -0
- package/docs-site/zh/examples/index.mdx +50 -0
- package/docs-site/zh/{concepts → explanation}/adapter.mdx +11 -11
- package/docs-site/zh/{concepts → explanation}/assert.mdx +7 -7
- package/docs-site/zh/{concepts → explanation}/drive.mdx +8 -8
- package/docs-site/zh/{concepts → explanation}/evals.mdx +4 -4
- package/docs-site/zh/{concepts → explanation}/experiment.mdx +8 -8
- package/docs-site/zh/{concepts → explanation}/hitl.mdx +8 -8
- package/docs-site/zh/{concepts → explanation}/judge.mdx +5 -5
- package/docs-site/zh/{concepts → explanation}/overview.mdx +5 -5
- package/docs-site/zh/{guides → explanation}/runner.mdx +1 -1
- package/docs-site/zh/{concepts → explanation}/tier.mdx +6 -6
- package/docs-site/zh/{guides → how-to}/agent-feedback-loop.mdx +7 -7
- package/docs-site/zh/{guides → how-to}/authoring.mdx +2 -2
- package/docs-site/zh/{guides → how-to}/connect-otel.mdx +6 -6
- package/docs-site/zh/{guides → how-to}/connect-your-agent.mdx +18 -18
- package/docs-site/zh/{guides → how-to}/custom-reports.mdx +6 -6
- package/docs-site/zh/{guides → how-to}/experiments.mdx +3 -3
- package/docs-site/zh/{guides → how-to}/publish-report.mdx +2 -2
- package/docs-site/zh/{guides → how-to}/sandbox-agent.mdx +2 -2
- package/docs-site/zh/{guides → how-to}/sandbox-providers.mdx +1 -1
- package/docs-site/zh/{guides → how-to}/viewing-results.mdx +6 -6
- package/docs-site/zh/{guides → how-to}/write-experiment.mdx +3 -3
- package/docs-site/zh/{guides → how-to}/write-send.mdx +13 -13
- package/docs-site/zh/index.mdx +23 -25
- package/docs-site/zh/introduction.mdx +8 -8
- package/docs-site/zh/reference/builtin-agents.mdx +5 -5
- package/docs-site/zh/reference/capabilities.mdx +6 -6
- package/docs-site/zh/reference/cli.mdx +9 -7
- package/docs-site/zh/reference/define-agent.mdx +1 -1
- package/docs-site/zh/reference/events.mdx +3 -3
- package/docs-site/zh/{guides → reference}/official-adapters.mdx +7 -7
- package/docs-site/zh/{guides → reference}/report-components.mdx +5 -5
- package/docs-site/zh/{guides → reference}/results-data.mdx +5 -5
- package/docs-site/zh/{guides → troubleshooting}/debug-sandbox.mdx +2 -2
- package/docs-site/zh/{guides → troubleshooting}/debugging.mdx +4 -2
- package/docs-site/zh/{quickstart.mdx → tutorials/quickstart.mdx} +5 -17
- package/package.json +6 -2
- package/src/agents/index.ts +2 -2
- package/src/agents/openai-compat.ts +1 -1
- package/src/agents/streaming.ts +2 -2
- package/src/agents/types.ts +3 -3
- package/src/cli.ts +42 -23
- package/src/context/context.ts +1 -1
- package/src/context/session.test.ts +1 -1
- package/src/context/session.ts +1 -1
- package/src/i18n/en.ts +18 -16
- package/src/i18n/zh-CN.ts +16 -15
- package/src/report/aggregate.ts +175 -87
- package/src/report/built-in/index.tsx +9 -0
- package/src/report/components.tsx +625 -285
- package/src/report/compute.ts +717 -515
- package/src/report/dual-render.test.tsx +738 -1148
- package/src/report/flag.ts +97 -33
- package/src/report/format.ts +18 -22
- package/src/report/index.ts +113 -58
- package/src/report/load.ts +3 -2
- package/src/report/locale.ts +120 -69
- package/src/report/metrics.ts +42 -12
- package/src/report/primitives.tsx +190 -45
- package/src/report/react/AttemptList.tsx +32 -20
- package/src/report/react/DeltaTable.tsx +63 -45
- package/src/report/react/EvalList.tsx +0 -0
- package/src/report/react/ExperimentComparison.tsx +12 -7
- package/src/report/react/ExperimentList.tsx +38 -26
- package/src/report/react/MetricBars.tsx +5 -4
- package/src/report/react/MetricLine.tsx +13 -8
- package/src/report/react/MetricMatrix.tsx +2 -2
- package/src/report/react/MetricScatter.tsx +74 -20
- package/src/report/react/MetricTable.tsx +4 -76
- package/src/report/react/ScopeSummary.tsx +86 -0
- package/src/report/react/Scoreboard.tsx +28 -10
- package/src/report/react/cell.tsx +2 -2
- package/src/report/react/enhance.js +57 -5
- package/src/report/react/fixtures.ts +109 -156
- package/src/report/react/index.tsx +24 -39
- package/src/report/react/render.test.tsx +139 -104
- package/src/report/react/styles.css +181 -91
- package/src/report/report.test.ts +761 -1031
- package/src/report/report.ts +425 -47
- package/src/report/text/faces.ts +257 -164
- package/src/report/text/plot.ts +1 -1
- package/src/report/text/table.ts +2 -2
- package/src/report/tree.ts +362 -104
- package/src/report/types.ts +257 -287
- package/src/report/web.ts +63 -20
- package/src/results/attempt-evidence.test.ts +4 -4
- package/src/results/attempt-evidence.ts +5 -5
- package/src/results/copy.ts +6 -6
- package/src/results/host-equivalence.test.ts +26 -14
- package/src/results/index.ts +10 -4
- package/src/results/open.ts +8 -4
- package/src/results/results.test.ts +4 -3
- package/src/results/select.ts +104 -34
- package/src/results/types.ts +36 -14
- package/src/runner/feedback/human.test.ts +1 -1
- package/src/runner/run.ts +1 -1
- package/src/sandbox/cli-commands.ts +2 -2
- package/src/scoring/judge.test.ts +1 -1
- package/src/shared/aggregate.ts +5 -4
- package/src/show/compose.ts +50 -67
- package/src/show/index.ts +107 -56
- package/src/show/render.ts +43 -27
- package/src/show/report-host.test.ts +188 -0
- package/src/show/report-host.ts +375 -0
- package/src/show/show.test.ts +86 -36
- package/src/view/app/App.test.tsx +69 -0
- package/src/view/app/App.tsx +144 -48
- package/src/view/app/components/AttemptModal.tsx +324 -63
- package/src/view/app/components/CodeView.tsx +10 -4
- package/src/view/app/i18n.ts +31 -17
- package/src/view/app/lib/artifact-url.ts +14 -3
- package/src/view/app/main.tsx +13 -8
- package/src/view/app/pages/{RunsPage.tsx → AttemptsPage.tsx} +6 -6
- package/src/view/app/types.ts +4 -1
- package/src/view/artifact-serving.test.ts +21 -1
- package/src/view/client-dist/app.css +1 -1
- package/src/view/client-dist/app.js +6 -6
- package/src/view/data.test.ts +9 -3
- package/src/view/data.ts +145 -49
- package/src/view/index.ts +48 -44
- package/src/view/server.ts +35 -15
- package/src/view/shared/types.ts +34 -5
- package/src/view/styles.css +224 -0
- package/src/view/view-report.test.ts +161 -57
- package/dist/report/built-ins/experiment-comparison.d.ts +0 -39
- package/dist/report/built-ins/experiment-comparison.js +0 -119
- package/dist/report/built-ins/index.d.ts +0 -2
- package/dist/report/built-ins/index.js +0 -2
- package/dist/report/react/GroupSummary.d.ts +0 -8
- package/dist/report/react/GroupSummary.js +0 -8
- package/dist/report/react/RunOverview.d.ts +0 -8
- package/dist/report/react/RunOverview.js +0 -12
- package/docs-site/zh/example/ai-agent-application.mdx +0 -152
- package/docs-site/zh/example/claude-code-codex-plugin.mdx +0 -167
- package/docs-site/zh/example/claude-code-codex-skill.mdx +0 -152
- package/docs-site/zh/example/showcase.mdx +0 -39
- package/src/report/built-in-user-parity.test.tsx +0 -597
- package/src/report/built-ins/experiment-comparison.tsx +0 -179
- package/src/report/built-ins/index.ts +0 -7
- package/src/report/react/GroupSummary.tsx +0 -66
- package/src/report/react/RunOverview.tsx +0 -109
- /package/docs-site/zh/{example/tier1-ai-sdk-v7.mdx → examples/integrations/ai-sdk-v7.mdx} +0 -0
- /package/docs-site/zh/{example/tier1-claude-sdk.mdx → examples/integrations/claude-sdk.mdx} +0 -0
- /package/docs-site/zh/{example/tier1-codex-sdk.mdx → examples/integrations/codex-sdk.mdx} +0 -0
- /package/docs-site/zh/{example/tier1-langgraph.mdx → examples/integrations/langgraph.mdx} +0 -0
- /package/docs-site/zh/{example/tier1-pi-sdk.mdx → examples/integrations/pi-sdk.mdx} +0 -0
- /package/docs-site/zh/{guides → how-to}/ci-integration.mdx +0 -0
- /package/docs-site/zh/{guides → how-to}/dataset-fanout.mdx +0 -0
- /package/docs-site/zh/{guides → how-to}/fixtures.mdx +0 -0
- /package/docs-site/zh/{guides → how-to}/reporters.mdx +0 -0
- /package/docs-site/zh/{guides → how-to}/scoring-guide.mdx +0 -0
package/src/cli.ts
CHANGED
|
@@ -97,7 +97,7 @@ interface Flags {
|
|
|
97
97
|
help: boolean;
|
|
98
98
|
version: boolean;
|
|
99
99
|
// ── show 专属(位置参数仍是 eval id 前缀 / `@<locator>`;这些 flag 选「怎么看」)──
|
|
100
|
-
|
|
100
|
+
source: boolean;
|
|
101
101
|
execution: boolean;
|
|
102
102
|
diff: boolean;
|
|
103
103
|
/** --diff=<路径>(必须 = 连写;空格形式会把路径当 eval id 前缀,按文档如此)。 */
|
|
@@ -111,8 +111,10 @@ interface Flags {
|
|
|
111
111
|
leaveRunning: boolean;
|
|
112
112
|
history: boolean;
|
|
113
113
|
experiment?: string;
|
|
114
|
-
|
|
114
|
+
results?: string;
|
|
115
|
+
snapshot?: string;
|
|
115
116
|
report?: string;
|
|
117
|
+
page?: string;
|
|
116
118
|
}
|
|
117
119
|
|
|
118
120
|
// 表驱动的 flag 定义(node:util parseArgs)。--no-x 显式声明,不依赖 allowNegative(需 Node 20.14+,
|
|
@@ -156,11 +158,11 @@ const FLAG_OPTIONS = {
|
|
|
156
158
|
port: { type: "string" },
|
|
157
159
|
/** `view --out` 专用:对非发布根(快照没有 publish:{redaction:"applied"} 标记)导出时的显式确认——静态站会原样携带未消毒的证据文件。 */
|
|
158
160
|
"allow-sensitive-artifacts": { type: "boolean" },
|
|
159
|
-
// show 的证据切面 / 时间轴 / 报告装载(docs-site/zh/
|
|
161
|
+
// show 的证据切面 / 时间轴 / 报告装载(docs-site/zh/how-to/viewing-results.mdx)。
|
|
160
162
|
// 证据切面只认 `@<locator>`(或收窄到单个 eval 的前缀)选出的那一个 attempt——不再有
|
|
161
163
|
// 数字 `--attempt`,选哪个 attempt 由 locator 精确指名,不是「先选 eval 再挑第几次」。
|
|
162
164
|
/** `show` 命令专用:该 attempt 运行时保存的 Eval 源码,gate/soft 断言标回源码行(证据切面)。 */
|
|
163
|
-
|
|
165
|
+
source: { type: "boolean" },
|
|
164
166
|
/** `show` 命令专用:该 attempt 的标准执行事件流(消息、thinking、Skill load、工具调用/结果);有 OTel 时同一节点补时间(证据切面)。 */
|
|
165
167
|
execution: { type: "boolean" },
|
|
166
168
|
/** `show` 命令专用:整个 Attempt 的统一时间树;裸 `--timing` 给有界诊断投影,`--timing=full` 逐节点展开全部 runner/已关联 OTel 节点。 */
|
|
@@ -169,14 +171,18 @@ const FLAG_OPTIONS = {
|
|
|
169
171
|
// 空格形式的下一个 token 仍是位置参数 = eval id 前缀,与文档一致)。
|
|
170
172
|
/** `show` 命令专用:sandbox 里的文件改动摘要;`--diff=<文件路径>` 看单个文件的完整改动(路径必须 `=` 连写)。 */
|
|
171
173
|
diff: { type: "boolean" },
|
|
172
|
-
/** `show`
|
|
174
|
+
/** `show` 命令专用:执行时间轴——对匹配的每个 experiment × eval 分节,逐 attempt 列时间 / verdict / 摘要 / 耗时 / 成本 / locator;与 `--report` 互斥。 */
|
|
173
175
|
history: { type: "boolean" },
|
|
174
176
|
/** `show` / `view` 命令专用:按路径段前缀收窄 experiment;组名会选中组内全部配置。 */
|
|
175
177
|
experiment: { type: "string" },
|
|
176
|
-
/** `show` / `view`
|
|
177
|
-
|
|
178
|
+
/** `show` / `view` / `sandbox enter|list|stop` 共用:结果根目录(`.niceeval` 之外的另一个根,如 `copySnapshots` 产出的发布根)。 */
|
|
179
|
+
results: { type: "string" },
|
|
180
|
+
/** `view` 命令专用:只打开这一份快照文件(`snapshot.json`);文件不可读时命令失败(扫描模式只跳过)。 */
|
|
181
|
+
snapshot: { type: "string" },
|
|
178
182
|
/** `show` / `view` 命令专用:用文件默认导出的 `defineReport(...)` 替换两者共用的默认报告。 */
|
|
179
183
|
report: { type: "string" },
|
|
184
|
+
/** `show` / `view` 命令专用:选择多页报告的页(页 id 见 `show --report` 的页索引);`view` 里定初始页。 */
|
|
185
|
+
page: { type: "string" },
|
|
180
186
|
/** 只打印本次会匹配到的 eval × 运行配置,不实际执行(按下面 `--output` 选中的 profile 给出预览)。 */
|
|
181
187
|
dry: { type: "boolean" },
|
|
182
188
|
/** 反馈 profile:`auto`(默认)按环境自动选择,`human` / `agent` / `ci` 强制指定;只改变终端展示,不改变选择、调度、判定、artifact 或退出码。`auto` 依次判定:stderr 是 TTY → human;否则 `CI`(或其它常见 CI 平台环境变量)存在 → ci;否则 → agent。 */
|
|
@@ -294,7 +300,7 @@ function parseArgs(argv: string[]): { command: string; positionals: string[]; fl
|
|
|
294
300
|
open: values["no-open"] === true ? false : values.open === true ? true : undefined,
|
|
295
301
|
help: values.help === true,
|
|
296
302
|
version: values.version === true,
|
|
297
|
-
|
|
303
|
+
source: values.source === true,
|
|
298
304
|
execution: values.execution === true,
|
|
299
305
|
diff: values.diff === true && diffPath === undefined,
|
|
300
306
|
diffPath,
|
|
@@ -307,31 +313,35 @@ function parseArgs(argv: string[]): { command: string; positionals: string[]; fl
|
|
|
307
313
|
leaveRunning: values["leave-running"] === true,
|
|
308
314
|
history: values.history === true,
|
|
309
315
|
experiment: values.experiment as string | undefined,
|
|
310
|
-
|
|
316
|
+
results: values.results as string | undefined,
|
|
317
|
+
snapshot: values.snapshot as string | undefined,
|
|
311
318
|
report: values.report as string | undefined,
|
|
319
|
+
page: values.page as string | undefined,
|
|
312
320
|
};
|
|
313
321
|
return { command, positionals, flags };
|
|
314
322
|
}
|
|
315
323
|
|
|
316
324
|
/**
|
|
317
325
|
* exp 只接受两类输入:位置参数选「跑哪些 eval」+ 调度/输出/机器出口 flag 选「对着哪个 agent、
|
|
318
|
-
* 怎么跑」。show / view 专属的证据切面(`--
|
|
319
|
-
*
|
|
320
|
-
* 不能被 exp 静默忽略(见 docs/feature/experiments/
|
|
321
|
-
* flag 及其归属命令(用于报错),没有误用返回 undefined。
|
|
326
|
+
* 怎么跑」。show / view 专属的证据切面(`--source`/`--execution`/`--diff`)、时间轴(`--history`)、
|
|
327
|
+
* Scope 收窄(`--experiment`/`--results`)、报告装载(`--report`/`--page`)、查看器
|
|
328
|
+
* (`--snapshot`/`--out`/`--port`/`--open`)不能被 exp 静默忽略(见 docs/feature/experiments/
|
|
329
|
+
* cli.md「用法错误」)。返回第一个被误用的 flag 及其归属命令(用于报错),没有误用返回 undefined。
|
|
322
330
|
*/
|
|
323
331
|
function firstViewerOnlyFlag(flags: Flags): { flag: string; command: string } | undefined {
|
|
324
332
|
const SHOW = "show";
|
|
325
333
|
const BOTH = "show / view";
|
|
326
334
|
const VIEW = "view";
|
|
327
|
-
if (flags.
|
|
335
|
+
if (flags.source) return { flag: "--source", command: SHOW };
|
|
328
336
|
if (flags.execution) return { flag: "--execution", command: SHOW };
|
|
329
337
|
if (flags.timing !== undefined) return { flag: "--timing", command: SHOW };
|
|
330
338
|
if (flags.diff || flags.diffPath !== undefined) return { flag: "--diff", command: SHOW };
|
|
331
339
|
if (flags.history) return { flag: "--history", command: SHOW };
|
|
332
340
|
if (flags.experiment !== undefined) return { flag: "--experiment", command: BOTH };
|
|
333
|
-
if (flags.
|
|
341
|
+
if (flags.results !== undefined) return { flag: "--results", command: BOTH };
|
|
334
342
|
if (flags.report !== undefined) return { flag: "--report", command: BOTH };
|
|
343
|
+
if (flags.page !== undefined) return { flag: "--page", command: BOTH };
|
|
344
|
+
if (flags.snapshot !== undefined) return { flag: "--snapshot", command: VIEW };
|
|
335
345
|
if (flags.out !== undefined) return { flag: "--out", command: VIEW };
|
|
336
346
|
if (flags.port !== undefined) return { flag: "--port", command: VIEW };
|
|
337
347
|
if (flags.open !== undefined) return { flag: "--open", command: VIEW };
|
|
@@ -393,7 +403,7 @@ const AGENT_RULES_CONTENT = [
|
|
|
393
403
|
"the bundled Chinese docs are the authoritative version matching this installation.",
|
|
394
404
|
"After a run, drill into failures with `niceeval show` — pick an `@<locator>` from the",
|
|
395
405
|
"compact index it prints, then `niceeval show @<locator>` for a compact overview, or add",
|
|
396
|
-
"`--
|
|
406
|
+
"`--source` / `--execution` / `--diff` for evidence; the snapshot directories the CLI prints",
|
|
397
407
|
"are the structured source of truth: `snapshot.json` holds the run's metadata and each",
|
|
398
408
|
"`<evalId>/a<attempt>/result.json` holds that attempt's verdict and assertions, next to",
|
|
399
409
|
"its artifact files (`events.json` / `trace.json` / `diff.json`).",
|
|
@@ -560,11 +570,17 @@ async function main(): Promise<void> {
|
|
|
560
570
|
}
|
|
561
571
|
|
|
562
572
|
if (command === "view") {
|
|
563
|
-
//
|
|
564
|
-
//
|
|
573
|
+
// 位置参数只有一种含义:eval id 前缀(收窄报告槽 Scope)。结果根经 --results 递入,
|
|
574
|
+
// 单开一份快照经 --snapshot 递入;--report 整槽替换报告槽(与 show --report 吃同一个文件),
|
|
575
|
+
// --page 定初始页。文件与目录都不进位置参数(docs/feature/reports/view.md「打开与收窄」)。
|
|
576
|
+
// --out 与位置参数 / --experiment 的互斥在 buildView 内校验(单点,报错文案含 copySnapshots
|
|
577
|
+
// + filter 的下一步),经 exitOnViewUserError 按用法错误退出。
|
|
565
578
|
let viewInput: { input?: string; patterns: string[] };
|
|
566
579
|
try {
|
|
567
|
-
viewInput = resolveViewInput(cwd, positionals,
|
|
580
|
+
viewInput = resolveViewInput(cwd, positionals, {
|
|
581
|
+
...(flags.results !== undefined ? { results: flags.results } : {}),
|
|
582
|
+
...(flags.snapshot !== undefined ? { snapshot: flags.snapshot } : {}),
|
|
583
|
+
});
|
|
568
584
|
} catch (e) {
|
|
569
585
|
exitOnViewUserError(e);
|
|
570
586
|
}
|
|
@@ -572,6 +588,7 @@ async function main(): Promise<void> {
|
|
|
572
588
|
patterns: viewInput.patterns,
|
|
573
589
|
...(flags.experiment !== undefined ? { experiment: flags.experiment } : {}),
|
|
574
590
|
...(flags.report !== undefined ? { report: { path: flags.report, cwd } } : {}),
|
|
591
|
+
...(flags.page !== undefined ? { page: flags.page } : {}),
|
|
575
592
|
};
|
|
576
593
|
if (flags.out) {
|
|
577
594
|
const out = await buildView({ input: viewInput.input, out: flags.out, allowSensitiveArtifacts: flags.allowSensitiveArtifacts, scan }).catch(exitOnViewUserError);
|
|
@@ -599,23 +616,25 @@ async function main(): Promise<void> {
|
|
|
599
616
|
window: flags.window,
|
|
600
617
|
path: flags.sandboxPath,
|
|
601
618
|
leaveRunning: flags.leaveRunning,
|
|
602
|
-
|
|
619
|
+
// CLI flag 是 --results(结果根);sandbox 命令组的内部选项名保持 run,值语义相同。
|
|
620
|
+
run: flags.results,
|
|
603
621
|
});
|
|
604
622
|
process.exit(code);
|
|
605
623
|
}
|
|
606
624
|
|
|
607
625
|
if (command === "show") {
|
|
608
|
-
// show 不依赖 niceeval.config.ts:读的是 .niceeval/(或 --
|
|
626
|
+
// show 不依赖 niceeval.config.ts:读的是 .niceeval/(或 --results 指定的结果根)的落盘结果。
|
|
609
627
|
const code = await runShow(cwd, positionals, {
|
|
610
|
-
|
|
628
|
+
source: flags.source,
|
|
611
629
|
execution: flags.execution,
|
|
612
630
|
timing: flags.timing,
|
|
613
631
|
diff: flags.diff,
|
|
614
632
|
diffPath: flags.diffPath,
|
|
615
633
|
history: flags.history,
|
|
616
634
|
experiment: flags.experiment,
|
|
617
|
-
|
|
635
|
+
results: flags.results,
|
|
618
636
|
report: flags.report,
|
|
637
|
+
page: flags.page,
|
|
619
638
|
});
|
|
620
639
|
process.exit(code);
|
|
621
640
|
}
|
package/src/context/context.ts
CHANGED
|
@@ -91,7 +91,7 @@ export interface ContextDeps {
|
|
|
91
91
|
* 这是唯一仍需要构造证据之外强制检查的能力——`t.sandbox.file`/`t.sandbox.fileChanged()` 等直接读沙箱
|
|
92
92
|
* 文件系统,没有沙箱就没有东西可读,不报错会静默返回空结果。其余能力(多轮对话、
|
|
93
93
|
* 工具断言……)都不再问卷式声明,由「做没做到」的构造证据决定,见
|
|
94
|
-
* docs-site/zh/
|
|
94
|
+
* docs-site/zh/explanation/adapter.mdx「能力从哪来」一节。
|
|
95
95
|
*/
|
|
96
96
|
function capabilityGuard(agentName: string, cap: string, method: string): () => never {
|
|
97
97
|
return () => {
|
|
@@ -5,7 +5,7 @@ import { createAgentSession, SessionManager } from "./session.ts";
|
|
|
5
5
|
import type { Agent, Sandbox, StreamEvent, Turn, TurnInput } from "../types.ts";
|
|
6
6
|
|
|
7
7
|
// createAgentSession() 是 ctx.session 的实现——一条会话线的存取器(见
|
|
8
|
-
// docs-site/zh/
|
|
8
|
+
// docs-site/zh/explanation/adapter.mdx 的 AgentSession 契约)。这里直接测存取器本身;
|
|
9
9
|
// 端到端的「同一条线同一个 ctx.session」由 SessionManager / RunSession 保证。
|
|
10
10
|
|
|
11
11
|
function fakeSandbox(): Sandbox {
|
package/src/context/session.ts
CHANGED
|
@@ -8,7 +8,7 @@ import { captureLoc } from "../source-loc.ts";
|
|
|
8
8
|
import { t } from "../i18n/index.ts";
|
|
9
9
|
|
|
10
10
|
/**
|
|
11
|
-
* 一条会话线的存取器实现(见 docs-site/zh/
|
|
11
|
+
* 一条会话线的存取器实现(见 docs-site/zh/explanation/adapter.mdx 的 AgentSession 契约)。
|
|
12
12
|
* 私有槽都关在闭包里——`state` 只归用户,框架内部数据不往里塞。
|
|
13
13
|
*/
|
|
14
14
|
export function createAgentSession(): AgentSession {
|
package/src/i18n/en.ts
CHANGED
|
@@ -48,7 +48,7 @@ export const en = {
|
|
|
48
48
|
"Ways to fix:\n" +
|
|
49
49
|
" - [init] Run `npx niceeval init` to scaffold niceeval.config.ts and evals/\n" +
|
|
50
50
|
" - [cd] Run from the project root that contains niceeval.config.ts\n" +
|
|
51
|
-
" Docs: node_modules/niceeval/docs-site/zh/quickstart.mdx",
|
|
51
|
+
" Docs: node_modules/niceeval/docs-site/zh/tutorials/quickstart.mdx",
|
|
52
52
|
"cli.config.noDefault": "niceeval.config.ts must default export defineConfig(...).",
|
|
53
53
|
"cli.dry.header": "\n[dry] {{evals}} evals × {{configs}} run configs:\n",
|
|
54
54
|
"cli.dry.noMatches": "(no matches)",
|
|
@@ -60,7 +60,7 @@ export const en = {
|
|
|
60
60
|
"budget for {{budgetKey}}: several attempts completed without any cost data (agent reports no usage and the model is not in the price table) — the budget cannot be enforced for this agent; continuing without the guard.\n",
|
|
61
61
|
"judge.modelMissing":
|
|
62
62
|
"No judge model configured. Set it in defineConfig({ judge: { model: \"...\" } }), the eval's judge config, or the NICEEVAL_JUDGE_MODEL environment variable (there is no built-in default model).\n" +
|
|
63
|
-
" Docs: node_modules/niceeval/docs-site/zh/
|
|
63
|
+
" Docs: node_modules/niceeval/docs-site/zh/how-to/scoring-guide.mdx",
|
|
64
64
|
"loaders.yamlMissing":
|
|
65
65
|
"loadYaml(\"{{path}}\") needs a YAML parser: run `pnpm add yaml` first (or switch to loadJson with a JSON dataset).",
|
|
66
66
|
"cli.flag.parseError": "{{message}}\nRun `niceeval --help` for usage.\n",
|
|
@@ -75,20 +75,22 @@ export const en = {
|
|
|
75
75
|
" a single eval id: attempts + assertion details\n" +
|
|
76
76
|
" @<locator> exactly one attempt: no flag -> compact overview;\n" +
|
|
77
77
|
" with a flag -> that evidence slice\n" +
|
|
78
|
-
" --
|
|
78
|
+
" --source the Eval source captured when this attempt ran,\n" +
|
|
79
79
|
" assertions mapped back to source lines\n" +
|
|
80
80
|
" --execution this attempt's execution event stream (messages/thinking/\n" +
|
|
81
81
|
" Skill loads/tool calls); OTel adds timing to the same node when present\n" +
|
|
82
82
|
" --timing unified timing tree for the attempt (phases + hooks/commands/turns + per-turn OTel)\n" +
|
|
83
83
|
" --diff[=file] sandbox workspace file-change summary; =file expands one file\n" +
|
|
84
|
-
" --history
|
|
85
|
-
" --
|
|
86
|
-
" --report <file> custom report\n" +
|
|
84
|
+
" --history per experiment × eval execution timeline (mutually exclusive with --report)\n" +
|
|
85
|
+
" --results <dir> pin a results root --experiment <id> one experiment\n" +
|
|
86
|
+
" --report <file> custom report --page <id> pick a page (multi-page\n" +
|
|
87
|
+
" reports print a page index with copyable commands)\n" +
|
|
87
88
|
" niceeval list list discovered evals\n" +
|
|
88
|
-
" niceeval view [eval-id-prefix
|
|
89
|
-
" report
|
|
90
|
-
" (same file as show); --
|
|
91
|
-
" --
|
|
89
|
+
" niceeval view [eval-id-prefix…] [--out dir] [--port n] [--no-open]\n" +
|
|
90
|
+
" report pages + evidence rooms; --report <file> swaps in your report\n" +
|
|
91
|
+
" (same file as show); --page <id> picks the initial page;\n" +
|
|
92
|
+
" --results <dir> pins a results root; --snapshot <file> opens exactly\n" +
|
|
93
|
+
" one snapshot; --experiment <id> one experiment\n" +
|
|
92
94
|
" --out <dir> exports a static site: index.html plus the viewer\n" +
|
|
93
95
|
" artifacts, ready for any static host\n" +
|
|
94
96
|
" niceeval sandbox list|enter|history|diff|stop inspect & destroy sandboxes kept by --keep-sandbox\n" +
|
|
@@ -107,9 +109,9 @@ export const en = {
|
|
|
107
109
|
"cli.show.noEvalMatch": "No results matched: {{patterns}}. Evals with results: {{evals}}\n",
|
|
108
110
|
"cli.show.noExperimentMatch": "No experiment matched --experiment {{arg}}. Experiments with results: {{experiments}}\n",
|
|
109
111
|
"cli.show.historyReportConflict":
|
|
110
|
-
"`--history` and `--report` are mutually exclusive: --history is the
|
|
112
|
+
"`--history` and `--report` are mutually exclusive: both take over the main output. --history is the host's per-attempt execution timeline; for snapshot-level trends, compose exp.snapshots inside your report file instead.\n",
|
|
111
113
|
"cli.show.evidenceNeedsEval":
|
|
112
|
-
"--
|
|
114
|
+
"--source / --execution / --diff show one attempt's evidence, but the selection matched {{matched}} evals. Pick an attempt locator from the index below:\n{{index}}\n",
|
|
113
115
|
"cli.show.locatorMalformed": "{{message}}\n",
|
|
114
116
|
"cli.show.locatorNotFound": "{{message}}\n",
|
|
115
117
|
"cli.eval.noMatch": "No eval matched: {{patterns}}.\n",
|
|
@@ -132,7 +134,7 @@ export const en = {
|
|
|
132
134
|
"cli.resultsPath": "Structured results: {{path}} (snapshot.json + per-attempt result.json / events.json / trace.json / diff.json)\n",
|
|
133
135
|
"cli.run.experimentRequired":
|
|
134
136
|
"Run evals through an experiment: use `niceeval exp [group|config] [eval id prefix]`.\n" +
|
|
135
|
-
" Docs: node_modules/niceeval/docs-site/zh/
|
|
137
|
+
" Docs: node_modules/niceeval/docs-site/zh/how-to/write-experiment.mdx\n",
|
|
136
138
|
"cli.run.experimentRequiredHint": "Hint: \"{{pattern}}\" is an experiment{{kind}}; you probably meant: niceeval exp {{pattern}}\n",
|
|
137
139
|
"cli.run.experimentRequiredKnown": "Discovered experiments: {{experiments}}\n",
|
|
138
140
|
"cli.unimplemented": "Command \"{{command}}\" is not implemented yet (MVP).\n",
|
|
@@ -143,7 +145,7 @@ export const en = {
|
|
|
143
145
|
"cli.view.url": "niceeval view: {{url}}\n",
|
|
144
146
|
"context.capabilityMissing":
|
|
145
147
|
"Agent \"{{agent}}\" is not sandbox-backed (built with defineSandboxAgent), so t.{{method}} is unavailable. Use an agent built with defineSandboxAgent, or drop this assertion.\n" +
|
|
146
|
-
" Docs: node_modules/niceeval/docs-site/zh/
|
|
148
|
+
" Docs: node_modules/niceeval/docs-site/zh/how-to/sandbox-agent.mdx",
|
|
147
149
|
"context.skipEmpty": "skip() requires a non-empty reason.",
|
|
148
150
|
"context.turnFailed": "This send returned failed (turn status = failed): {{message}}",
|
|
149
151
|
"context.turnFailedDefault": "This send returned failed (turn status = failed)",
|
|
@@ -167,7 +169,7 @@ export const en = {
|
|
|
167
169
|
"feedback.human.compare": "Compare: niceeval view {{group}}",
|
|
168
170
|
"feedback.human.counts": "{{total}} total · {{reused}} reused · {{running}} running · {{queued}} queued · {{completed}} completed",
|
|
169
171
|
"feedback.human.diffHint": "Diff: niceeval show {{locator}} --diff",
|
|
170
|
-
"feedback.human.evalHint": "Eval: niceeval show {{locator}} --
|
|
172
|
+
"feedback.human.evalHint": "Eval: niceeval show {{locator}} --source",
|
|
171
173
|
"feedback.human.failuresHeader": "FAILURES",
|
|
172
174
|
"feedback.human.heartbeat": "{{elapsed}} elapsed · {{counts}}",
|
|
173
175
|
"feedback.human.inspect": "Inspect: niceeval show {{locator}}",
|
|
@@ -274,7 +276,7 @@ export const en = {
|
|
|
274
276
|
"sandbox.providerNotImplemented": "{{provider}} sandbox provider is not implemented; use docker, vercel, or e2b",
|
|
275
277
|
"sandbox.missingSpec":
|
|
276
278
|
"sandbox agent needs a sandbox, but none was given. niceeval no longer picks a default — set `sandbox` in defineExperiment()/defineConfig() to dockerSandbox() / vercelSandbox() / e2bSandbox() (import from \"niceeval/sandbox\").\n" +
|
|
277
|
-
" Docs: node_modules/niceeval/docs-site/zh/
|
|
279
|
+
" Docs: node_modules/niceeval/docs-site/zh/how-to/sandbox-providers.mdx",
|
|
278
280
|
"sandbox.dependencyMissing.docker": "Docker sandbox requires 'dockerode'. Install it with: pnpm add dockerode @types/dockerode",
|
|
279
281
|
"sandbox.dependencyMissing.e2b": "E2B sandbox requires 'e2b'. Install it with: pnpm add e2b",
|
|
280
282
|
"sandbox.dependencyMissing.vercel": "Vercel sandbox requires '@vercel/sandbox'. Install it with: pnpm add @vercel/sandbox",
|
package/src/i18n/zh-CN.ts
CHANGED
|
@@ -46,7 +46,7 @@ export const zhCN = {
|
|
|
46
46
|
"修法:\n" +
|
|
47
47
|
" - [init] 运行 `npx niceeval init` 生成 niceeval.config.ts 和 evals/\n" +
|
|
48
48
|
" - [cd] 切到包含 niceeval.config.ts 的项目根再运行\n" +
|
|
49
|
-
" 文档:node_modules/niceeval/docs-site/zh/quickstart.mdx",
|
|
49
|
+
" 文档:node_modules/niceeval/docs-site/zh/tutorials/quickstart.mdx",
|
|
50
50
|
"cli.config.noDefault": "niceeval.config.ts 需要 default export(defineConfig(...))。",
|
|
51
51
|
"cli.dry.header": "\n[dry] {{evals}} 个 eval × {{configs}} 个运行配置:\n",
|
|
52
52
|
"cli.dry.noMatches": "(无匹配)",
|
|
@@ -58,7 +58,7 @@ export const zhCN = {
|
|
|
58
58
|
"{{budgetKey}} 的 budget:连续多个 attempt 完成后都拿不到成本数据(agent 不上报用量且模型不在价格表)——该 agent 的 budget 无法执行,取消护栏继续跑。\n",
|
|
59
59
|
"judge.modelMissing":
|
|
60
60
|
"judge 未配置模型:在 defineConfig({ judge: { model: \"...\" } })、eval 的 judge 配置或环境变量 NICEEVAL_JUDGE_MODEL 里指定裁判模型(没有内置默认模型)。\n" +
|
|
61
|
-
" 文档:node_modules/niceeval/docs-site/zh/
|
|
61
|
+
" 文档:node_modules/niceeval/docs-site/zh/how-to/scoring-guide.mdx",
|
|
62
62
|
"loaders.yamlMissing":
|
|
63
63
|
"loadYaml(\"{{path}}\") 需要 YAML 解析器:请先 `pnpm add yaml`(或改用 loadJson + JSON 数据集)。",
|
|
64
64
|
"cli.flag.parseError": "{{message}}\n运行 `niceeval --help` 查看用法。\n",
|
|
@@ -72,18 +72,19 @@ export const zhCN = {
|
|
|
72
72
|
" (locator + 失败原因)\n" +
|
|
73
73
|
" 单个 eval id:attempt 与断言明细\n" +
|
|
74
74
|
" @<locator> 精确一个 attempt:无 flag → 紧凑全景;带 flag → 对应证据切面\n" +
|
|
75
|
-
" --
|
|
75
|
+
" --source 该 attempt 运行时保存的 Eval 源码,断言标回源码行\n" +
|
|
76
76
|
" --execution 该 attempt 的执行事件流(消息/thinking/Skill/工具调用),\n" +
|
|
77
77
|
" 有 OTel 时同一节点补时间\n" +
|
|
78
78
|
" --timing 整个 attempt 的统一时间树(阶段 + hook/命令/turn + 轮内 OTel)\n" +
|
|
79
79
|
" --diff[=文件] agent 归因的文件改动摘要;=文件 按窗口展开单个文件\n" +
|
|
80
|
-
" --history
|
|
81
|
-
" --
|
|
82
|
-
" --report <文件>
|
|
80
|
+
" --history 逐 experiment × eval 的执行时间轴(与 --report 互斥)\n" +
|
|
81
|
+
" --results <目录> 钉死结果根 --experiment <id> 只看该实验\n" +
|
|
82
|
+
" --report <文件> 自定义报告 --page <id> 选页(多页报告先输出页索引)\n" +
|
|
83
83
|
" niceeval list 列出发现到的 eval\n" +
|
|
84
|
-
" niceeval view [eval-id
|
|
85
|
-
"
|
|
86
|
-
" --
|
|
84
|
+
" niceeval view [eval-id 前缀…] [--out 目录] [--port n] [--no-open]\n" +
|
|
85
|
+
" 报告页 + 证据室;--report <文件> 整槽换成自定义报告(与 show 同一文件)\n" +
|
|
86
|
+
" --page <id> 定初始页 --results <目录> 钉死结果根\n" +
|
|
87
|
+
" --snapshot <文件> 只打开这一份快照 --experiment <id> 只看该实验\n" +
|
|
87
88
|
" --out <目录> 静态导出:index.html 连同查看器 artifact,可直接静态托管\n" +
|
|
88
89
|
" niceeval sandbox list|enter|history|diff|stop 查看与销毁 --keep-sandbox 留下的现场\n" +
|
|
89
90
|
" niceeval clean 删除 .niceeval/ 历史 artifact\n" +
|
|
@@ -102,9 +103,9 @@ export const zhCN = {
|
|
|
102
103
|
"cli.show.noEvalMatch": "No results matched: {{patterns}}. Evals with results: {{evals}}\n",
|
|
103
104
|
"cli.show.noExperimentMatch": "No experiment matched --experiment {{arg}}. Experiments with results: {{experiments}}\n",
|
|
104
105
|
"cli.show.historyReportConflict":
|
|
105
|
-
"`--history` and `--report` are mutually exclusive: --history is the
|
|
106
|
+
"`--history` and `--report` are mutually exclusive: both take over the main output. --history is the host's per-attempt execution timeline; for snapshot-level trends, compose exp.snapshots inside your report file instead.\n",
|
|
106
107
|
"cli.show.evidenceNeedsEval":
|
|
107
|
-
"--
|
|
108
|
+
"--source / --execution / --diff show one attempt's evidence, but the selection matched {{matched}} evals. Pick an attempt locator from the index below:\n{{index}}\n",
|
|
108
109
|
"cli.show.locatorMalformed": "{{message}}\n",
|
|
109
110
|
"cli.show.locatorNotFound": "{{message}}\n",
|
|
110
111
|
"cli.eval.noMatch": "没有匹配的 eval:{{patterns}}。\n",
|
|
@@ -127,7 +128,7 @@ export const zhCN = {
|
|
|
127
128
|
"cli.resultsPath": "结构化结果:{{path}}(snapshot.json + 每 attempt 的 result.json / events.json / trace.json / diff.json)\n",
|
|
128
129
|
"cli.run.experimentRequired":
|
|
129
130
|
"运行 eval 必须通过 experiment:用 `niceeval exp [实验组|配置] [eval id 前缀]`。\n" +
|
|
130
|
-
" 文档:node_modules/niceeval/docs-site/zh/
|
|
131
|
+
" 文档:node_modules/niceeval/docs-site/zh/how-to/write-experiment.mdx\n",
|
|
131
132
|
"cli.run.experimentRequiredHint": "提示:\"{{pattern}}\" 是实验{{kind}},你大概想跑:niceeval exp {{pattern}}\n",
|
|
132
133
|
"cli.run.experimentRequiredKnown": "已发现实验:{{experiments}}\n",
|
|
133
134
|
"cli.unimplemented": "命令 \"{{command}}\" 暂未实现(MVP)。\n",
|
|
@@ -138,7 +139,7 @@ export const zhCN = {
|
|
|
138
139
|
"cli.view.url": "niceeval view: {{url}}\n",
|
|
139
140
|
"context.capabilityMissing":
|
|
140
141
|
"agent \"{{agent}}\" 不是沙箱型(defineSandboxAgent 构造),t.{{method}} 这类断言只有沙箱型 agent 可用。换用 defineSandboxAgent 构造的 agent,或去掉这条断言。\n" +
|
|
141
|
-
" 文档:node_modules/niceeval/docs-site/zh/
|
|
142
|
+
" 文档:node_modules/niceeval/docs-site/zh/how-to/sandbox-agent.mdx",
|
|
142
143
|
"context.skipEmpty": "skip() 需要一个非空理由。",
|
|
143
144
|
"context.turnFailed": "本轮 send 返回 failed(turn status = failed):{{message}}",
|
|
144
145
|
"context.turnFailedDefault": "本轮 send 返回 failed(turn status = failed)",
|
|
@@ -162,7 +163,7 @@ export const zhCN = {
|
|
|
162
163
|
"feedback.human.compare": "Compare: niceeval view {{group}}",
|
|
163
164
|
"feedback.human.counts": "共 {{total}} · 复用 {{reused}} · 运行中 {{running}} · 排队 {{queued}} · 已完成 {{completed}}",
|
|
164
165
|
"feedback.human.diffHint": "Diff: niceeval show {{locator}} --diff",
|
|
165
|
-
"feedback.human.evalHint": "Eval: niceeval show {{locator}} --
|
|
166
|
+
"feedback.human.evalHint": "Eval: niceeval show {{locator}} --source",
|
|
166
167
|
"feedback.human.failuresHeader": "FAILURES",
|
|
167
168
|
"feedback.human.heartbeat": "已运行 {{elapsed}} · {{counts}}",
|
|
168
169
|
"feedback.human.inspect": "Inspect: niceeval show {{locator}}",
|
|
@@ -269,7 +270,7 @@ export const zhCN = {
|
|
|
269
270
|
"sandbox.providerNotImplemented": "{{provider}} sandbox provider not implemented; use docker, vercel, or e2b",
|
|
270
271
|
"sandbox.missingSpec":
|
|
271
272
|
"沙箱型 agent 需要一个 sandbox,但没有提供。niceeval 不再自动选默认 provider——请在 defineExperiment()/defineConfig() 里把 sandbox 设成 dockerSandbox() / vercelSandbox() / e2bSandbox()(从 \"niceeval/sandbox\" 导入)。\n" +
|
|
272
|
-
" 文档:node_modules/niceeval/docs-site/zh/
|
|
273
|
+
" 文档:node_modules/niceeval/docs-site/zh/how-to/sandbox-providers.mdx",
|
|
273
274
|
"sandbox.dependencyMissing.docker": "Docker sandbox requires 'dockerode'. Install it with: pnpm add dockerode @types/dockerode",
|
|
274
275
|
"sandbox.dependencyMissing.e2b": "E2B sandbox requires 'e2b'. Install it with: pnpm add e2b",
|
|
275
276
|
"sandbox.dependencyMissing.vercel": "Vercel sandbox requires '@vercel/sandbox'. Install it with: pnpm add @vercel/sandbox",
|