niceeval 0.13.0 → 0.13.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.cjs +10 -1
- package/dist/cli.cjs.map +1 -1
- package/dist/report/execution/machine.cjs +1 -1
- package/dist/report/execution/machine.cjs.map +1 -1
- package/dist/report/host/execute.cjs +14 -4
- package/dist/report/host/execute.cjs.map +1 -1
- package/dist/report/host/execute.d.cts +5 -0
- package/dist/report/host/execute.d.mts +5 -0
- package/dist/report/host/execute.d.ts +5 -0
- package/dist/report/host/from-record.cjs +6 -1
- package/dist/report/host/from-record.cjs.map +1 -1
- package/dist/report/host/from-record.d.cts +5 -0
- package/dist/report/host/from-record.d.mts +5 -0
- package/dist/report/host/from-record.d.ts +5 -0
- package/dist/report/runtime/resolved-page.cjs +16 -6
- package/dist/report/runtime/resolved-page.cjs.map +1 -1
- package/dist/report/runtime/resolved-page.d.cts +3 -0
- package/dist/report/runtime/resolved-page.d.mts +3 -0
- package/dist/report/runtime/resolved-page.d.ts +3 -0
- package/dist/report/runtime/text.cjs +1 -0
- package/dist/report/runtime/text.cjs.map +1 -1
- package/dist/report/runtime/text.d.cts +4 -0
- package/dist/report/runtime/text.d.mts +4 -0
- package/dist/report/runtime/text.d.ts +4 -0
- package/docs-site/zh/explanation/runner.mdx +2 -2
- package/docs-site/zh/index.mdx +2 -2
- package/docs-site/zh/reference/cli.mdx +14 -14
- package/docs-site/zh/reference/events.mdx +1 -1
- package/docs-site/zh/troubleshooting/debugging.mdx +2 -2
- package/docs-site/zh/tutorials/agent-feedback-loop.mdx +2 -2
- package/docs-site/zh/tutorials/ci-integration.mdx +1 -1
- package/docs-site/zh/tutorials/connect-your-agent.mdx +2 -2
- package/docs-site/zh/tutorials/custom-reports.mdx +4 -4
- package/docs-site/zh/tutorials/deploy-report-site.mdx +3 -3
- package/docs-site/zh/tutorials/experiments.mdx +2 -2
- package/docs-site/zh/tutorials/handle-execution-failures.mdx +1 -1
- package/docs-site/zh/tutorials/install-custom-sandbox-agent.mdx +1 -1
- package/docs-site/zh/tutorials/publish-report.mdx +4 -4
- package/docs-site/zh/tutorials/reporters.mdx +1 -1
- package/docs-site/zh/tutorials/viewing-results.mdx +9 -9
- package/package.json +1 -1
|
@@ -2,6 +2,7 @@ import { Effect } from "effect";
|
|
|
2
2
|
import type * as Scope from "effect/Scope";
|
|
3
3
|
import { type JsonValue, type Sample, type SampleIdentity } from "../../analysis/contracts.cjs";
|
|
4
4
|
import type { LocalizedText } from "../../shared/types.cjs";
|
|
5
|
+
import type { PanelMode } from "../model/panel.cjs";
|
|
5
6
|
/** A closed page has no route back to an author callback or a live Sample. */
|
|
6
7
|
export declare const RESOLVED_PAGE_FORMAT = "niceeval.report.resolved-page/v1";
|
|
7
8
|
export interface ResolvedPageTarget {
|
|
@@ -13,6 +14,7 @@ export interface ResolvedPageTarget {
|
|
|
13
14
|
export interface ResolvedPageTextProjection {
|
|
14
15
|
readonly locale: string;
|
|
15
16
|
readonly width: number;
|
|
17
|
+
readonly panelMode: PanelMode;
|
|
16
18
|
readonly text: string;
|
|
17
19
|
}
|
|
18
20
|
export interface ResolvedPageWebProjection {
|
|
@@ -124,6 +126,7 @@ export declare function resolvePage<Rendered, Error, Requirements>(input: Resolv
|
|
|
124
126
|
export declare function resolvedPageTextProjection(page: ResolvedPage, input: {
|
|
125
127
|
readonly locale: string;
|
|
126
128
|
readonly width: number;
|
|
129
|
+
readonly panelMode: PanelMode;
|
|
127
130
|
}): ResolvedPageTextProjection | undefined;
|
|
128
131
|
/** Finds an already-built browser projection. It never invokes a renderer. */
|
|
129
132
|
export declare function resolvedPageWebProjection(page: ResolvedPage, input: {
|
|
@@ -2,6 +2,7 @@ import { Effect } from "effect";
|
|
|
2
2
|
import type * as Scope from "effect/Scope";
|
|
3
3
|
import { type JsonValue, type Sample, type SampleIdentity } from "../../analysis/contracts.mjs";
|
|
4
4
|
import type { LocalizedText } from "../../shared/types.mjs";
|
|
5
|
+
import type { PanelMode } from "../model/panel.mjs";
|
|
5
6
|
/** A closed page has no route back to an author callback or a live Sample. */
|
|
6
7
|
export declare const RESOLVED_PAGE_FORMAT = "niceeval.report.resolved-page/v1";
|
|
7
8
|
export interface ResolvedPageTarget {
|
|
@@ -13,6 +14,7 @@ export interface ResolvedPageTarget {
|
|
|
13
14
|
export interface ResolvedPageTextProjection {
|
|
14
15
|
readonly locale: string;
|
|
15
16
|
readonly width: number;
|
|
17
|
+
readonly panelMode: PanelMode;
|
|
16
18
|
readonly text: string;
|
|
17
19
|
}
|
|
18
20
|
export interface ResolvedPageWebProjection {
|
|
@@ -124,6 +126,7 @@ export declare function resolvePage<Rendered, Error, Requirements>(input: Resolv
|
|
|
124
126
|
export declare function resolvedPageTextProjection(page: ResolvedPage, input: {
|
|
125
127
|
readonly locale: string;
|
|
126
128
|
readonly width: number;
|
|
129
|
+
readonly panelMode: PanelMode;
|
|
127
130
|
}): ResolvedPageTextProjection | undefined;
|
|
128
131
|
/** Finds an already-built browser projection. It never invokes a renderer. */
|
|
129
132
|
export declare function resolvedPageWebProjection(page: ResolvedPage, input: {
|
|
@@ -2,6 +2,7 @@ import { Effect } from "effect";
|
|
|
2
2
|
import type * as Scope from "effect/Scope";
|
|
3
3
|
import { type JsonValue, type Sample, type SampleIdentity } from "../../analysis/contracts.js";
|
|
4
4
|
import type { LocalizedText } from "../../shared/types.js";
|
|
5
|
+
import type { PanelMode } from "../model/panel.js";
|
|
5
6
|
/** A closed page has no route back to an author callback or a live Sample. */
|
|
6
7
|
export declare const RESOLVED_PAGE_FORMAT = "niceeval.report.resolved-page/v1";
|
|
7
8
|
export interface ResolvedPageTarget {
|
|
@@ -13,6 +14,7 @@ export interface ResolvedPageTarget {
|
|
|
13
14
|
export interface ResolvedPageTextProjection {
|
|
14
15
|
readonly locale: string;
|
|
15
16
|
readonly width: number;
|
|
17
|
+
readonly panelMode: PanelMode;
|
|
16
18
|
readonly text: string;
|
|
17
19
|
}
|
|
18
20
|
export interface ResolvedPageWebProjection {
|
|
@@ -124,6 +126,7 @@ export declare function resolvePage<Rendered, Error, Requirements>(input: Resolv
|
|
|
124
126
|
export declare function resolvedPageTextProjection(page: ResolvedPage, input: {
|
|
125
127
|
readonly locale: string;
|
|
126
128
|
readonly width: number;
|
|
129
|
+
readonly panelMode: PanelMode;
|
|
127
130
|
}): ResolvedPageTextProjection | undefined;
|
|
128
131
|
/** Finds an already-built browser projection. It never invokes a renderer. */
|
|
129
132
|
export declare function resolvedPageWebProjection(page: ResolvedPage, input: {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"text.js","sourceRoot":"","sources":["text.ts"],"names":[],"mappings":";;
|
|
1
|
+
{"version":3,"file":"text.js","sourceRoot":"","sources":["text.ts"],"names":[],"mappings":";;AAsBA,wDAKC;AAED,4CAQC;AArCD,yDAAoF;AAkBpF;;;GAGG;AACH,SAAgB,sBAAsB,CAAC,IAAkB,EAAE,OAAsC;IAC7F,MAAM,UAAU,GAAG,IAAA,6CAA0B,EAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC7D,IAAI,UAAU,KAAK,SAAS;QACxB,OAAO,UAAU,CAAC,IAAI,CAAC;IAC3B,MAAM,qBAAqB,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;AAC/C,CAAC;AACD,2EAA2E;AAC3E,SAAgB,gBAAgB,CAAC,IAAkB,EAAE,OAAsC;IAIvF,MAAM,UAAU,GAAG,IAAA,6CAA0B,EAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC7D,IAAI,UAAU,KAAK,SAAS;QACxB,OAAO,MAAM,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,UAAmB,EAAE,IAAI,EAAE,UAAU,CAAC,IAAI,EAAE,CAAC,CAAC;IAChF,OAAO,qBAAqB,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;AAChD,CAAC;AACD,SAAS,qBAAqB,CAAC,IAAkB,EAAE,OAAsC;IACrF,OAAO,MAAM,CAAC,MAAM,CAAC;QACjB,IAAI,EAAE,gCAAyC;QAC/C,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM;QAC1B,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK;QACxB,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,SAAS,EAAE,OAAO,CAAC,SAAS;KAC/B,CAAC,CAAC;AACP,CAAC","sourcesContent":["import { resolvedPageTextProjection, type ResolvedPage, } from \"./resolved-page.ts\";\nimport type { PanelMode } from \"../model/panel.ts\";\nexport interface RenderResolvedPageTextOptions {\n /** Explicit: a machine or terminal caller must never inherit process locale. */\n readonly locale: string;\n /** Explicit: closed terminal output is only reusable at the requested width. */\n readonly width: number;\n /** Explicit: callers select a pre-closed terminal or plain-text projection. */\n readonly panelMode: PanelMode;\n}\nexport interface ResolvedPageTextProjectionMissing {\n readonly code: \"report-text-projection-missing\";\n readonly pageId: string;\n readonly route: string;\n readonly locale: string;\n readonly width: number;\n readonly panelMode: PanelMode;\n}\n/**\n * Reads one pre-closed terminal projection. This function deliberately has no\n * Sample, callback, renderer, or Effect dependency.\n */\nexport function renderResolvedPageText(page: ResolvedPage, options: RenderResolvedPageTextOptions): string {\n const projection = resolvedPageTextProjection(page, options);\n if (projection !== undefined)\n return projection.text;\n throw textProjectionMissing(page, options);\n}\n/** A total variant for Host error mapping without exception inspection. */\nexport function resolvedPageText(page: ResolvedPage, options: RenderResolvedPageTextOptions): {\n readonly state: \"rendered\";\n readonly text: string;\n} | ResolvedPageTextProjectionMissing {\n const projection = resolvedPageTextProjection(page, options);\n if (projection !== undefined)\n return Object.freeze({ state: \"rendered\" as const, text: projection.text });\n return textProjectionMissing(page, options);\n}\nfunction textProjectionMissing(page: ResolvedPage, options: RenderResolvedPageTextOptions): ResolvedPageTextProjectionMissing {\n return Object.freeze({\n code: \"report-text-projection-missing\" as const,\n pageId: page.target.pageId,\n route: page.target.route,\n locale: options.locale,\n width: options.width,\n panelMode: options.panelMode,\n });\n}\n"]}
|
|
@@ -1,9 +1,12 @@
|
|
|
1
1
|
import { type ResolvedPage } from "./resolved-page.cjs";
|
|
2
|
+
import type { PanelMode } from "../model/panel.cjs";
|
|
2
3
|
export interface RenderResolvedPageTextOptions {
|
|
3
4
|
/** Explicit: a machine or terminal caller must never inherit process locale. */
|
|
4
5
|
readonly locale: string;
|
|
5
6
|
/** Explicit: closed terminal output is only reusable at the requested width. */
|
|
6
7
|
readonly width: number;
|
|
8
|
+
/** Explicit: callers select a pre-closed terminal or plain-text projection. */
|
|
9
|
+
readonly panelMode: PanelMode;
|
|
7
10
|
}
|
|
8
11
|
export interface ResolvedPageTextProjectionMissing {
|
|
9
12
|
readonly code: "report-text-projection-missing";
|
|
@@ -11,6 +14,7 @@ export interface ResolvedPageTextProjectionMissing {
|
|
|
11
14
|
readonly route: string;
|
|
12
15
|
readonly locale: string;
|
|
13
16
|
readonly width: number;
|
|
17
|
+
readonly panelMode: PanelMode;
|
|
14
18
|
}
|
|
15
19
|
/**
|
|
16
20
|
* Reads one pre-closed terminal projection. This function deliberately has no
|
|
@@ -1,9 +1,12 @@
|
|
|
1
1
|
import { type ResolvedPage } from "./resolved-page.mjs";
|
|
2
|
+
import type { PanelMode } from "../model/panel.mjs";
|
|
2
3
|
export interface RenderResolvedPageTextOptions {
|
|
3
4
|
/** Explicit: a machine or terminal caller must never inherit process locale. */
|
|
4
5
|
readonly locale: string;
|
|
5
6
|
/** Explicit: closed terminal output is only reusable at the requested width. */
|
|
6
7
|
readonly width: number;
|
|
8
|
+
/** Explicit: callers select a pre-closed terminal or plain-text projection. */
|
|
9
|
+
readonly panelMode: PanelMode;
|
|
7
10
|
}
|
|
8
11
|
export interface ResolvedPageTextProjectionMissing {
|
|
9
12
|
readonly code: "report-text-projection-missing";
|
|
@@ -11,6 +14,7 @@ export interface ResolvedPageTextProjectionMissing {
|
|
|
11
14
|
readonly route: string;
|
|
12
15
|
readonly locale: string;
|
|
13
16
|
readonly width: number;
|
|
17
|
+
readonly panelMode: PanelMode;
|
|
14
18
|
}
|
|
15
19
|
/**
|
|
16
20
|
* Reads one pre-closed terminal projection. This function deliberately has no
|
|
@@ -1,9 +1,12 @@
|
|
|
1
1
|
import { type ResolvedPage } from "./resolved-page.js";
|
|
2
|
+
import type { PanelMode } from "../model/panel.js";
|
|
2
3
|
export interface RenderResolvedPageTextOptions {
|
|
3
4
|
/** Explicit: a machine or terminal caller must never inherit process locale. */
|
|
4
5
|
readonly locale: string;
|
|
5
6
|
/** Explicit: closed terminal output is only reusable at the requested width. */
|
|
6
7
|
readonly width: number;
|
|
8
|
+
/** Explicit: callers select a pre-closed terminal or plain-text projection. */
|
|
9
|
+
readonly panelMode: PanelMode;
|
|
7
10
|
}
|
|
8
11
|
export interface ResolvedPageTextProjectionMissing {
|
|
9
12
|
readonly code: "report-text-projection-missing";
|
|
@@ -11,6 +14,7 @@ export interface ResolvedPageTextProjectionMissing {
|
|
|
11
14
|
readonly route: string;
|
|
12
15
|
readonly locale: string;
|
|
13
16
|
readonly width: number;
|
|
17
|
+
readonly panelMode: PanelMode;
|
|
14
18
|
}
|
|
15
19
|
/**
|
|
16
20
|
* Reads one pre-closed terminal projection. This function deliberately has no
|
|
@@ -66,8 +66,8 @@ Reporter 只接入当前 Invocation。它可以把当前进程已知的结果交
|
|
|
66
66
|
Runner 结束后,用明确 Run 选择查看结果:
|
|
67
67
|
|
|
68
68
|
```sh
|
|
69
|
-
|
|
70
|
-
|
|
69
|
+
pnpm exec niceeval show --run 01J9ZK3M6P4T7V9X2C5N8QW0RY
|
|
70
|
+
pnpm exec niceeval view --run 01J9ZK3M6P4T7V9X2C5N8QW0RY --no-open
|
|
71
71
|
```
|
|
72
72
|
|
|
73
73
|
不带 locator 或 `--run` 的 `show` 和 `view` 读取身份仍匹配当前项目的全部结果;locator 与 `--run` 读取指定历史事实。NiceEval 形成 `Sample`,再按选中 Page 或完整站点所需的 Measure 与领域视图取得闭合值。
|
package/docs-site/zh/index.mdx
CHANGED
|
@@ -107,8 +107,8 @@ export default defineExperiment({
|
|
|
107
107
|
|
|
108
108
|
```sh
|
|
109
109
|
npx niceeval exp local weather-tool
|
|
110
|
-
|
|
111
|
-
|
|
110
|
+
pnpm exec niceeval show # 终端读取结果,失败项可继续下钻
|
|
111
|
+
pnpm exec niceeval view # 网页交互浏览
|
|
112
112
|
```
|
|
113
113
|
|
|
114
114
|
如果被测对象是需要隔离工作区的 coding agent(Codex、Claude Code 插件/Skill),评估用例里改用 `t.sandbox.uploadDirectory()` 铺工作区。检查文件变化用 `t.sandbox.fileChanged()` 等归因断言。`t.sandbox.runCommand()` 跑测试。参考独立仓库 [coding-agent-skill](https://github.com/CorrectRoadH/coding-agent-skill)。
|
|
@@ -24,10 +24,10 @@ description: "NiceEval CLI 参考:exp、debug、show、view、init、list 和
|
|
|
24
24
|
<Card title="npx niceeval clean" icon="trash">
|
|
25
25
|
删除明确 writer 留下的 Record 临时目录,不删除正式 Run 或 Attempt。
|
|
26
26
|
</Card>
|
|
27
|
-
<Card title="
|
|
27
|
+
<Card title="pnpm exec niceeval show [--run]" icon="terminal">
|
|
28
28
|
在终端执行报告。不带 locator 或 `--run` 的命令读取当前项目结果;精确 Attempt 定位符或 Run ID 用于审计历史。
|
|
29
29
|
</Card>
|
|
30
|
-
<Card title="
|
|
30
|
+
<Card title="pnpm exec niceeval view" icon="eye">
|
|
31
31
|
构建完整报告站并在浏览器中托管。它与 `show` 共享选择规则,但会枚举全部页面。
|
|
32
32
|
</Card>
|
|
33
33
|
</CardGroup>
|
|
@@ -205,7 +205,7 @@ npx niceeval exp compare-models weather-tool
|
|
|
205
205
|
|
|
206
206
|
运行命名 experiment,用矩阵比较 agents、models 或 flags。第二个参数开始是评估用例 ID 前缀过滤。
|
|
207
207
|
|
|
208
|
-
`show` 与 `view` 是顶层命令,应写成 `
|
|
208
|
+
`show` 与 `view` 是顶层命令,应写成 `pnpm exec niceeval show`、`pnpm exec niceeval view`。误写成 `pnpm exec niceeval exp show` / `exp view` 且没有同名 experiment 时,CLI 会在“不存在的实验”错误后提示正确命令。仓库确实存在同名 experiment 时仍按合法 id 执行。
|
|
209
209
|
|
|
210
210
|
## 两种输出:人读文本与 `--json`
|
|
211
211
|
|
|
@@ -227,10 +227,10 @@ npx niceeval exp compare --junit ./artifacts/niceeval-junit.xml
|
|
|
227
227
|
## `view`
|
|
228
228
|
|
|
229
229
|
```bash
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
230
|
+
pnpm exec niceeval view
|
|
231
|
+
pnpm exec niceeval view --run 01H...
|
|
232
|
+
pnpm exec niceeval view @01H...
|
|
233
|
+
pnpm exec niceeval view --host 192.168.0.199
|
|
234
234
|
```
|
|
235
235
|
|
|
236
236
|
打开本地结果查看器。它和 `show` 共用默认 Report 与选择规则。不带 locator 或 `--run` 的命令读取所有身份仍匹配当前项目的结果;精确 `@<AttemptLocator>` 显示该 Attempt 的默认概览;`--run` 读取指定历史 Run。NiceEval 随后形成固定 Sample、枚举全部 Page 与参数实例,并构建完整站点。
|
|
@@ -246,13 +246,13 @@ npx niceeval view --host 192.168.0.199
|
|
|
246
246
|
## `show`
|
|
247
247
|
|
|
248
248
|
```bash
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
249
|
+
pnpm exec niceeval show
|
|
250
|
+
pnpm exec niceeval show --run 01H...
|
|
251
|
+
pnpm exec niceeval show --run 01H... --run 01J...
|
|
252
|
+
pnpm exec niceeval show @01H...
|
|
253
|
+
pnpm exec niceeval show --run 01H... --page /
|
|
254
|
+
pnpm exec niceeval show --run 01H... --report ./reports/summary.tsx --page /overview
|
|
255
|
+
pnpm exec niceeval show --run 01H... --report ./reports/summary.tsx --json
|
|
256
256
|
```
|
|
257
257
|
|
|
258
258
|
`show` 是终端报告入口,适合人直接阅读,也适合 coding agent 在上下文窗口里逐级下钻。它不接受模糊的位置选择。省略 `--page` 时选择默认 Page;提供 `--page <route>` 时只执行这个精确 route。
|
|
@@ -267,7 +267,7 @@ events.push({
|
|
|
267
267
|
});
|
|
268
268
|
```
|
|
269
269
|
|
|
270
|
-
`commandProjection()` 保留 Adapter 已确认的 original tokens,并调用同一份 `normalizeLogicalCommand()` 生成 `logical-command/v1` 投影。`pnpm exec niceeval show`
|
|
270
|
+
`commandProjection()` 保留 Adapter 已确认的 original tokens,并调用同一份 `normalizeLogicalCommand()` 生成 `logical-command/v1` 投影。`pnpm exec niceeval show` 能由 `commandMatch("niceeval", { argsStart: ["show"] })` 精确匹配。
|
|
271
271
|
|
|
272
272
|
只有原生协议已经给出 argv,或协议 grammar 能无歧义地产生单一 invocation,才能把 original 标为 available。协议只给 shell source、内容已截断或脱敏时使用 `opaqueCommandProjection(reason)`;能确认不是 command 时使用 `notCommandProjection()`。无法确认 command / not-command 时降低 actions coverage,不能从 tool name、input 或 shell 文本猜测。
|
|
273
273
|
|
|
@@ -7,8 +7,8 @@ description: "根据可见错误、Sample 状态和 rebuild 反馈,定位查
|
|
|
7
7
|
先用明确 Run 读取问题,不要从目录名或时间猜测目标。
|
|
8
8
|
|
|
9
9
|
```sh
|
|
10
|
-
|
|
11
|
-
|
|
10
|
+
pnpm exec niceeval show --run <run-id>
|
|
11
|
+
pnpm exec niceeval view --run <run-id> --no-open
|
|
12
12
|
```
|
|
13
13
|
|
|
14
14
|
这两条命令从同一范围形成 Sample,并执行同一份 Report。先确定选择是否正确,再检查页面中的度量和问题。
|
|
@@ -49,8 +49,8 @@ Agent 应把最后一条 receipt 当作本次调用的交接信息。它不是
|
|
|
49
49
|
运行结束后,先读取 Run 的 Report:
|
|
50
50
|
|
|
51
51
|
```sh
|
|
52
|
-
|
|
53
|
-
|
|
52
|
+
pnpm exec niceeval show --run 01J9ZK3M6P4T7V9X2C5N8QW0RY
|
|
53
|
+
pnpm exec niceeval view --run 01J9ZK3M6P4T7V9X2C5N8QW0RY --no-open
|
|
54
54
|
```
|
|
55
55
|
|
|
56
56
|
`show` 与 `view` 选择 Sample,再取得当前目标需要的闭合分析值。`show` 只执行一个 Page;`view` 构建完整站点。Agent 应区分以下状态:
|
|
@@ -50,7 +50,7 @@ with:
|
|
|
50
50
|
成功建立 Invocation 后,NDJSON 的最后一行包含 receipt。后续脚本可以读取它的 `runIds`,用明确 Run 选择查看或导出报告:
|
|
51
51
|
|
|
52
52
|
```sh
|
|
53
|
-
|
|
53
|
+
pnpm exec niceeval view --run 01J9ZK3M6P4T7V9X2C5N8QW0RY \
|
|
54
54
|
--out ./report-site \
|
|
55
55
|
--no-open
|
|
56
56
|
```
|
|
@@ -101,10 +101,10 @@ export default defineEval({
|
|
|
101
101
|
```bash
|
|
102
102
|
npx niceeval exp my-agent # 跑这个 experiment 下的全部 eval
|
|
103
103
|
npx niceeval exp my-agent refund # 只跑 ID 以 refund 开头的
|
|
104
|
-
|
|
104
|
+
pnpm exec niceeval view --experiment my-agent # 本地查看器里看结果
|
|
105
105
|
```
|
|
106
106
|
|
|
107
|
-
**验证运行结果**:终端会显示动态 dashboard,完成和排队数量在原位更新。失败、错误和 warning 会保留在输出中。运行结束后会打印摘要和 receipt;用其中的 Run ID 也可以执行 `
|
|
107
|
+
**验证运行结果**:终端会显示动态 dashboard,完成和排队数量在原位更新。失败、错误和 warning 会保留在输出中。运行结束后会打印摘要和 receipt;用其中的 Run ID 也可以执行 `pnpm exec niceeval view --run <runId>`,查看每条评估用例逐轮的输入、事件和评分明细。
|
|
108
108
|
|
|
109
109
|
没跑通时,按报错的位置分三类排查:
|
|
110
110
|
|
|
@@ -173,10 +173,10 @@ export default defineConfig({
|
|
|
173
173
|
## 运行、读取和发布
|
|
174
174
|
|
|
175
175
|
```sh
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
176
|
+
pnpm exec niceeval show --run <run-id> --report ./reports/quality.tsx
|
|
177
|
+
pnpm exec niceeval show --run <run-id> --report ./reports/quality.tsx --json
|
|
178
|
+
pnpm exec niceeval view --run <run-id> --report ./reports/quality.tsx --no-open
|
|
179
|
+
pnpm exec niceeval view --run <run-id> --report ./reports/quality.tsx --out ./quality-site
|
|
180
180
|
```
|
|
181
181
|
|
|
182
182
|
前三条命令从相同选择形成 Sample,但范围不同:`show` 和 `show --json` 只执行一个 Page,并只捕获该页的 projection closure;`view` 与 `view --out` 枚举全部 Page 后形成完整站点及 `_niceeval/data/projections.json`。该文件的 bytes 进入 revision identity。view 的成功 rebuild 原子替换当前站点;失败时保留上一份可用站点。`--out` 必须指向不存在的目录。
|
|
@@ -7,7 +7,7 @@ description: "将 view --out 生成的完整自包含报告目录部署到任意
|
|
|
7
7
|
先生成完整报告站,再部署输出目录。托管平台不读取源 Record,也不需要安装或运行 NiceEval。
|
|
8
8
|
|
|
9
9
|
```sh
|
|
10
|
-
|
|
10
|
+
pnpm exec niceeval view --experiment checkout \
|
|
11
11
|
--out ./report-site \
|
|
12
12
|
--no-open
|
|
13
13
|
```
|
|
@@ -15,7 +15,7 @@ npx niceeval view --experiment checkout \
|
|
|
15
15
|
要部署一次明确运行,请从 receipt 读取 Run ID:
|
|
16
16
|
|
|
17
17
|
```sh
|
|
18
|
-
|
|
18
|
+
pnpm exec niceeval view --run <run-id> --out ./report-site --no-open
|
|
19
19
|
```
|
|
20
20
|
|
|
21
21
|
## CI 流程
|
|
@@ -30,7 +30,7 @@ GitHub Actions 的构建任务示例:
|
|
|
30
30
|
|
|
31
31
|
```yaml
|
|
32
32
|
- name: 生成报告站
|
|
33
|
-
run:
|
|
33
|
+
run: pnpm exec niceeval view --experiment checkout --out ./report-site --no-open
|
|
34
34
|
|
|
35
35
|
- name: 上传静态站点目录
|
|
36
36
|
uses: actions/upload-pages-artifact@v3
|
|
@@ -75,8 +75,8 @@ api-validation claude-code pass@3 = 1/3 (33%) mean 41s
|
|
|
75
75
|
除了 pass rate,还应该看平均耗时、token、成本和失败类型。
|
|
76
76
|
|
|
77
77
|
```bash
|
|
78
|
-
|
|
79
|
-
|
|
78
|
+
pnpm exec niceeval show --experiment models
|
|
79
|
+
pnpm exec niceeval view --experiment models
|
|
80
80
|
```
|
|
81
81
|
|
|
82
82
|
每个 experiment 用自己的 `evals` 选择评估用例。函数形式会遍历所有已发现的评估用例:
|
|
@@ -108,7 +108,7 @@ npx niceeval exp compare evals/coding
|
|
|
108
108
|
需要查看某个 Attempt 的原始错误和重试摘要时,使用终端给出的定位符:
|
|
109
109
|
|
|
110
110
|
```shell
|
|
111
|
-
|
|
111
|
+
pnpm exec niceeval show @<attempt-locator>
|
|
112
112
|
```
|
|
113
113
|
|
|
114
114
|
自定义 Adapter 如果有专用的受理前拒绝信号,可以提供 `classifySendFailure`。该字段的类型和边界见 [`defineSandboxAgent` 参考](/zh/reference/define-agent#definesandboxagent)。
|
|
@@ -87,7 +87,7 @@ Sandbox 不需要 Node.js 或 npm。`createNpmCliInstaller` 在宿主机上下
|
|
|
87
87
|
安装或复检失败时,Attempt 会在 `agent.ensure` 阶段变成 `errored`。终端会给出 Attempt 定位符。
|
|
88
88
|
|
|
89
89
|
```shell
|
|
90
|
-
|
|
90
|
+
pnpm exec niceeval show @<attempt-locator>
|
|
91
91
|
```
|
|
92
92
|
|
|
93
93
|
先检查错误里的包名、版本和目标平台。修正 `identity`、npm 包或安装步骤后,重新运行原命令。
|
|
@@ -7,7 +7,7 @@ description: "用 view --out 导出可离线打开、与源 Record 脱离且包
|
|
|
7
7
|
`niceeval view --out <目录>` 从已发布结果形成固定 Sample,再构建完整报告站。它不是把当前浏览器页面保存为文件:`pages` 中显式声明的所有普通 Page、参数 Page 实例、资源和下载都必须先关闭并通过校验;Host 不补建详情页。
|
|
8
8
|
|
|
9
9
|
```sh
|
|
10
|
-
|
|
10
|
+
pnpm exec niceeval view --experiment checkout \
|
|
11
11
|
--out ./report-site \
|
|
12
12
|
--no-open
|
|
13
13
|
```
|
|
@@ -15,7 +15,7 @@ npx niceeval view --experiment checkout \
|
|
|
15
15
|
需要固定到本次运行时,从 receipt 的 `runIds` 取得 Run ID:
|
|
16
16
|
|
|
17
17
|
```sh
|
|
18
|
-
|
|
18
|
+
pnpm exec niceeval view --run <run-id> \
|
|
19
19
|
--out ./report-site \
|
|
20
20
|
--no-open
|
|
21
21
|
```
|
|
@@ -47,14 +47,14 @@ Analysis 的 `partial`、`empty`、`unsupported` 与 `failed` 是可显示的数
|
|
|
47
47
|
|
|
48
48
|
```sh
|
|
49
49
|
# 成功时写入全新目录
|
|
50
|
-
|
|
50
|
+
pnpm exec niceeval view --run <run-id> --out ./report-site-2026-08-15
|
|
51
51
|
```
|
|
52
52
|
|
|
53
53
|
## 在 CI 中生成
|
|
54
54
|
|
|
55
55
|
```sh
|
|
56
56
|
npx niceeval exp ci --json
|
|
57
|
-
|
|
57
|
+
pnpm exec niceeval view --experiment ci --out ./report-site --no-open
|
|
58
58
|
```
|
|
59
59
|
|
|
60
60
|
若 CI 必须发布刚完成的那次运行,读取第一条命令最后一行 receipt 的 `runIds`,并使用 `view --run <run-id>`。不要让托管平台读取源 Record;它只应部署生成目录。
|
|
@@ -47,7 +47,7 @@ npx niceeval exp checkout --json
|
|
|
47
47
|
`--json` 输出当前进程的 NDJSON。最后一条 receipt 包含 `invocationId`、`runIds`、开始时间、结束时间和 completion。外部自动化可以保留 receipt,再根据 `runIds` 选择要查看的 Report。
|
|
48
48
|
|
|
49
49
|
```sh
|
|
50
|
-
|
|
50
|
+
pnpm exec niceeval show --run 01J9ZK3M6P4T7V9X2C5N8QW0RY
|
|
51
51
|
```
|
|
52
52
|
|
|
53
53
|
不要把 progress 或 diagnostic 行当作长期结果格式。它们服务当前 Invocation,完整业务事实属于停稳 Record。
|
|
@@ -7,8 +7,8 @@ description: "用 show 或 view 从已发布的 Record 选择固定 Sample,并
|
|
|
7
7
|
运行结束后,从下面两条命令开始:
|
|
8
8
|
|
|
9
9
|
```sh
|
|
10
|
-
|
|
11
|
-
|
|
10
|
+
pnpm exec niceeval show
|
|
11
|
+
pnpm exec niceeval view
|
|
12
12
|
```
|
|
13
13
|
|
|
14
14
|
两条命令用相同选择规则形成固定 `Sample`,但执行范围不同。`show` 只读取一个目标 Page;`view` 会先构建完整站点,再把它托管在浏览器中。
|
|
@@ -18,7 +18,7 @@ npx niceeval view
|
|
|
18
18
|
选择一个明确 Run:
|
|
19
19
|
|
|
20
20
|
```sh
|
|
21
|
-
|
|
21
|
+
pnpm exec niceeval show --run 7b8d2ea4-b840-4870-9840-f85a436a5527
|
|
22
22
|
```
|
|
23
23
|
|
|
24
24
|
普通运行可从 `exp --json` 最后一条 receipt 的 `runIds` 读取 Run ID:
|
|
@@ -43,10 +43,10 @@ Sample 记录每个位置的状态:
|
|
|
43
43
|
## 在终端读取一个 Page
|
|
44
44
|
|
|
45
45
|
```sh
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
46
|
+
pnpm exec niceeval show --run <run-id>
|
|
47
|
+
pnpm exec niceeval show --run <run-id> --page /
|
|
48
|
+
pnpm exec niceeval show @<attempt-locator>
|
|
49
|
+
pnpm exec niceeval show --run <run-id> --report ./reports/quality.tsx --json
|
|
50
50
|
```
|
|
51
51
|
|
|
52
52
|
省略 `--page` 时,`show` 使用 Report 的默认可导航页面。提供 `--page` 时,它只 decode、load、render 和关闭这一个精确 route。参数 Page 不会因此调用 `enumerate()`;其它 Page、Analysis 查询和作者 callback 也不会执行。
|
|
@@ -69,8 +69,8 @@ JSON 不包含其它 Page、HTML、React tree、站点 identity 或 Record 读
|
|
|
69
69
|
## 在浏览器读取完整站点
|
|
70
70
|
|
|
71
71
|
```sh
|
|
72
|
-
|
|
73
|
-
|
|
72
|
+
pnpm exec niceeval view --experiment checkout --no-open
|
|
73
|
+
pnpm exec niceeval view --run <run-id> --page / --no-open
|
|
74
74
|
```
|
|
75
75
|
|
|
76
76
|
view 在启动 server 前枚举所有显式声明的普通 Page 与参数 Page 的全部实例,并验证路径、链接、资源、下载、`_niceeval/data/projections.json` 和预算。`pages` 是唯一页面集合,Host 不会补建详情 route。`--page` 只决定初始打开的已存在 route,不能缩小构建范围。全站 projections 文件的 bytes 属于 revision identity。
|