niceeval 0.7.1 → 0.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/report/components.d.ts +4 -5
- package/dist/report/components.js +7 -7
- package/dist/report/compute.d.ts +4 -4
- package/dist/report/compute.js +8 -12
- package/dist/report/index.d.ts +2 -2
- package/dist/report/report.d.ts +23 -1
- package/dist/report/report.js +82 -5
- package/dist/report/types.d.ts +0 -10
- package/dist/results/types.d.ts +0 -11
- package/docs-site/zh/how-to/custom-reports.mdx +3 -3
- package/docs-site/zh/how-to/publish-report.mdx +43 -9
- package/docs-site/zh/how-to/viewing-results.mdx +4 -4
- package/docs-site/zh/reference/cli.mdx +2 -3
- package/docs-site/zh/reference/report-components.mdx +2 -2
- package/docs-site/zh/reference/results-data.mdx +2 -4
- package/docs-site/zh/troubleshooting/debugging.mdx +2 -2
- package/package.json +7 -6
- package/src/cli.ts +1 -5
- package/src/context/context.ts +11 -5
- package/src/report/components.tsx +13 -15
- package/src/report/compute.ts +8 -20
- package/src/report/index.ts +1 -1
- package/src/report/report.test.ts +2 -20
- package/src/report/report.ts +128 -6
- package/src/report/shell-head.test.ts +102 -0
- package/src/report/types.ts +0 -11
- package/src/results/copy.ts +15 -78
- package/src/results/publish.ts +4 -146
- package/src/results/results.test.ts +8 -8
- package/src/results/types.ts +0 -7
- package/src/show/report-host.ts +13 -0
- package/src/view/app/components/CodeView.test.tsx +142 -0
- package/src/view/app/components/CodeView.tsx +15 -1
- package/src/view/app/components/Transcript.tsx +28 -1
- package/src/view/app/i18n.ts +6 -0
- package/src/view/app/lib/guards.test.ts +108 -0
- package/src/view/app/lib/guards.ts +13 -3
- package/src/view/app/lib/transcript-data.tsx +14 -0
- package/src/view/app/types.ts +17 -1
- package/src/view/artifact-serving.test.ts +1 -1
- package/src/view/client-dist/app.css +1 -1
- package/src/view/client-dist/app.js +20 -20
- package/src/view/data.ts +54 -14
- package/src/view/index.ts +18 -58
- package/src/view/server.ts +49 -144
- package/src/view/site-head.test.ts +177 -0
- package/src/view/site-parity.test.ts +117 -0
- package/src/view/site.ts +209 -0
- package/src/view/styles.css +10 -0
- package/src/view/view-report.test.ts +6 -6
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
// cases: docs/engineering/unit-tests/reports/cases.md
|
|
2
|
+
// 覆盖登记行:外壳 head 通道的白名单/宿主单例/attrs/children/scheme 分流装载校验,
|
|
3
|
+
// 与 scripts {src} 外链拒绝。
|
|
4
|
+
// defineReport 是装载校验的第一期(shell.md「校验分两期」),全部用例不落盘、不进渲染。
|
|
5
|
+
|
|
6
|
+
import { describe, expect, it } from "vitest";
|
|
7
|
+
|
|
8
|
+
import type { Scope } from "../results/index.ts";
|
|
9
|
+
import { buildReportMeta, defineReport } from "./report.ts";
|
|
10
|
+
|
|
11
|
+
const emptyScope = { snapshots: [] } as unknown as Scope;
|
|
12
|
+
|
|
13
|
+
describe("defineReport head 通道(装载校验)", () => {
|
|
14
|
+
it("tag 白名单是 meta/link/script/style,白名单外装载报错;title 指引到 title 字段", () => {
|
|
15
|
+
expect(() => defineReport({ content: null, head: [{ tag: "base", attrs: {} } as never] })).toThrow(
|
|
16
|
+
/not allowed/,
|
|
17
|
+
);
|
|
18
|
+
expect(() => defineReport({ content: null, head: [{ tag: "title", attrs: {} } as never] })).toThrow(
|
|
19
|
+
/"title" field/,
|
|
20
|
+
);
|
|
21
|
+
expect(() => defineReport({ content: null, head: [{ tag: "iframe", attrs: {} } as never] })).toThrow(
|
|
22
|
+
/meta.*link.*script.*style/,
|
|
23
|
+
);
|
|
24
|
+
});
|
|
25
|
+
|
|
26
|
+
it("宿主自有单例:meta charset 与 meta name=viewport 装载报错", () => {
|
|
27
|
+
expect(() => defineReport({ content: null, head: [{ tag: "meta", attrs: { charset: "utf-8" } }] })).toThrow(
|
|
28
|
+
/owned by the host shell/,
|
|
29
|
+
);
|
|
30
|
+
expect(() =>
|
|
31
|
+
defineReport({ content: null, head: [{ tag: "meta", attrs: { name: "Viewport", content: "x" } }] }),
|
|
32
|
+
).toThrow(/owned by the host shell/);
|
|
33
|
+
});
|
|
34
|
+
|
|
35
|
+
it("children:meta/link 不收;script children 含 </script> 装载报错(该上下文无法转义)", () => {
|
|
36
|
+
expect(() =>
|
|
37
|
+
defineReport({ content: null, head: [{ tag: "link", attrs: { rel: "icon" }, children: "x" } as never] }),
|
|
38
|
+
).toThrow(/void element/);
|
|
39
|
+
expect(() =>
|
|
40
|
+
defineReport({
|
|
41
|
+
content: null,
|
|
42
|
+
head: [{ tag: "script", children: 'document.write("</script>")' }],
|
|
43
|
+
}),
|
|
44
|
+
).toThrow(/cannot be escaped/);
|
|
45
|
+
expect(() =>
|
|
46
|
+
defineReport({ content: null, head: [{ tag: "style", children: "</StYlE>" }] }),
|
|
47
|
+
).toThrow(/cannot be escaped/);
|
|
48
|
+
});
|
|
49
|
+
|
|
50
|
+
it("attrs:值只收 string 或 true(裸布尔属性);非法属性名装载报错", () => {
|
|
51
|
+
expect(() =>
|
|
52
|
+
defineReport({ content: null, head: [{ tag: "script", attrs: { async: 1 } as never }] }),
|
|
53
|
+
).toThrow(/string or true/);
|
|
54
|
+
expect(() =>
|
|
55
|
+
defineReport({ content: null, head: [{ tag: "meta", attrs: { 'bad name"': "x" } }] }),
|
|
56
|
+
).toThrow(/attribute name/);
|
|
57
|
+
expect(() =>
|
|
58
|
+
defineReport({
|
|
59
|
+
content: null,
|
|
60
|
+
head: [{ tag: "script", attrs: { async: true, "data-project": "p1", src: "https://cdn.example/x.js" } }],
|
|
61
|
+
}),
|
|
62
|
+
).not.toThrow();
|
|
63
|
+
});
|
|
64
|
+
|
|
65
|
+
it("src/href 按 scheme 分流:http(s) 外链与本地相对路径合法;protocol-relative 与其它 scheme 装载报错", () => {
|
|
66
|
+
expect(() =>
|
|
67
|
+
defineReport({ content: null, head: [{ tag: "link", attrs: { rel: "icon", href: "./favicon.svg" } }] }),
|
|
68
|
+
).not.toThrow();
|
|
69
|
+
expect(() =>
|
|
70
|
+
defineReport({ content: null, head: [{ tag: "script", attrs: { src: "//cdn.example/x.js" } }] }),
|
|
71
|
+
).toThrow(/protocol-relative/);
|
|
72
|
+
expect(() =>
|
|
73
|
+
defineReport({ content: null, head: [{ tag: "script", attrs: { src: "data:text/javascript,1" } }] }),
|
|
74
|
+
).toThrow(/scheme other than http/);
|
|
75
|
+
expect(() =>
|
|
76
|
+
defineReport({ content: null, head: [{ tag: "link", attrs: { rel: "icon", href: "../up.svg" } }] }),
|
|
77
|
+
).toThrow(/".." segments/);
|
|
78
|
+
expect(() =>
|
|
79
|
+
defineReport({ content: null, head: [{ tag: "link", attrs: { rel: "icon", href: "/abs.svg" } }] }),
|
|
80
|
+
).toThrow(/absolute paths/);
|
|
81
|
+
});
|
|
82
|
+
|
|
83
|
+
it("scripts/styles 的 {src} 只收本地路径:外链装载报错并给出 head 写法", () => {
|
|
84
|
+
expect(() => defineReport({ content: null, scripts: [{ src: "https://cdn.example/x.js" }] })).toThrow(
|
|
85
|
+
/Declare third-party external tags in "head"/,
|
|
86
|
+
);
|
|
87
|
+
expect(() => defineReport({ content: null, styles: [{ src: "//fonts.example/a.css" }] })).toThrow(
|
|
88
|
+
/Declare third-party external tags in "head"/,
|
|
89
|
+
);
|
|
90
|
+
});
|
|
91
|
+
|
|
92
|
+
it("规范化:省略 head 恒为空数组,声明原样进产物;head 是注入资产,不进 ctx.report", () => {
|
|
93
|
+
expect(defineReport({ content: null }).head).toEqual([]);
|
|
94
|
+
const head = [
|
|
95
|
+
{ tag: "script" as const, attrs: { async: true as const, src: "https://cdn.example/x.js" } },
|
|
96
|
+
{ tag: "script" as const, children: "window.x = 1;" },
|
|
97
|
+
];
|
|
98
|
+
const definition = defineReport({ content: null, head });
|
|
99
|
+
expect(definition.head).toEqual(head);
|
|
100
|
+
expect(buildReportMeta(definition, emptyScope, "report")).not.toHaveProperty("head");
|
|
101
|
+
});
|
|
102
|
+
});
|
package/src/report/types.ts
CHANGED
|
@@ -394,14 +394,3 @@ export interface ExperimentListItem {
|
|
|
394
394
|
lastRunAt: string;
|
|
395
395
|
evalRows: ExperimentListEvalRow[];
|
|
396
396
|
}
|
|
397
|
-
|
|
398
|
-
/** 三个实体列表共用的计算选项。 */
|
|
399
|
-
export interface EntityListDataOptions {
|
|
400
|
-
/**
|
|
401
|
-
* 展示层遮蔽:只改写这次组件数据中的自由文本——条目本身与任何嵌套 attempt 条目的
|
|
402
|
-
* `failureSummary`;身份与分类字段(experimentId、evalId、locator、数值指标)不经它。
|
|
403
|
-
* 只作用于这次计算产出的组件数据,不改盘上或任何导出目录里的 artifact;
|
|
404
|
-
* 发布 artifact 的脱敏用 copySnapshots({ redact })。
|
|
405
|
-
*/
|
|
406
|
-
redact?: (text: string) => string;
|
|
407
|
-
}
|
package/src/results/copy.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// copySnapshots:把选中快照按格式感知地复制到另一个目录(定稿见 docs/feature/results/library.md「复制与瘦身」)。
|
|
2
2
|
//
|
|
3
3
|
// 发布场景的原语:只带指定 artifact、只带选中快照的全部 attempt,布局知识不外泄。
|
|
4
|
-
// artifact 复制忠实于源(
|
|
4
|
+
// artifact 复制忠实于源(原字节,不重新序列化、不改写);snapshot.json / result.json
|
|
5
5
|
// 按选中条目重建,版本元数据保留。产物是一个标准结果根目录(同布局),openResults /
|
|
6
6
|
// `niceeval view` 直接能读。唯一随行补记的是挑选时的覆盖事实:每个复制出的快照带上
|
|
7
7
|
// knownEvalIds(复制时刻该实验已知的 eval 并集),发布目录上重新 openResults().latest(),
|
|
@@ -9,21 +9,13 @@
|
|
|
9
9
|
|
|
10
10
|
import { mkdir, readdir, readFile, stat, writeFile } from "node:fs/promises";
|
|
11
11
|
import { basename, dirname, join, resolve } from "node:path";
|
|
12
|
-
import type { EvalResult
|
|
12
|
+
import type { EvalResult } from "../types.ts";
|
|
13
13
|
import { RESULTS_FORMAT } from "../types.ts";
|
|
14
14
|
import { RESULT_FILE, SNAPSHOT_FILE, artifactFileOf, experimentDirOf } from "./format.ts";
|
|
15
15
|
import { experimentOfSnapshot } from "./open.ts";
|
|
16
16
|
import { isNewerSnapshot } from "./select.ts";
|
|
17
17
|
import { hashEvalSource, normalizeEvalSource } from "./source-hash.ts";
|
|
18
|
-
import {
|
|
19
|
-
PUBLISH_FILE_MAX_BYTES,
|
|
20
|
-
redactEvents,
|
|
21
|
-
redactExperimentInfo,
|
|
22
|
-
redactJsonValue,
|
|
23
|
-
redactResultRecord,
|
|
24
|
-
redactSpans,
|
|
25
|
-
type Redactor,
|
|
26
|
-
} from "./publish.ts";
|
|
18
|
+
import { PUBLISH_FILE_MAX_BYTES } from "./publish.ts";
|
|
27
19
|
import type { ArtifactKind, AttemptHandle, Scope, Snapshot, SnapshotMeta } from "./types.ts";
|
|
28
20
|
|
|
29
21
|
/** 缺省携带的 artifact:events / trace / o11y / agentSetup / sources;diff 不截断、可达百 MB,缺省不带。 */
|
|
@@ -33,12 +25,6 @@ const VALID_ARTIFACTS: ArtifactKind[] = ["events", "trace", "o11y", "agentSetup"
|
|
|
33
25
|
export interface CopySnapshotsOptions {
|
|
34
26
|
/** 要带上的 artifact 种类;缺省带 events / trace / o11y / agentSetup / sources,不带 diff。 */
|
|
35
27
|
artifacts?: ArtifactKind[];
|
|
36
|
-
/**
|
|
37
|
-
* 发布消毒(必填,没有隐式默认):传函数逐值消毒自由文本字段,或显式传 `false` 声明
|
|
38
|
-
* 「这批数据可以原文发布」——「不消毒」必须是写在代码里的选择,不是忘了传参数的副作用。
|
|
39
|
-
* 结构字段(格式、判定、身份、路径、哈希)永不经过 redactor。
|
|
40
|
-
*/
|
|
41
|
-
redact: Redactor | false;
|
|
42
28
|
}
|
|
43
29
|
|
|
44
30
|
export interface CopySnapshotsResult {
|
|
@@ -56,7 +42,7 @@ export interface CopySnapshotsResult {
|
|
|
56
42
|
export async function copySnapshots(
|
|
57
43
|
scope: Scope | readonly Snapshot[],
|
|
58
44
|
destDir: string,
|
|
59
|
-
opts: CopySnapshotsOptions,
|
|
45
|
+
opts: CopySnapshotsOptions = {},
|
|
60
46
|
): Promise<CopySnapshotsResult> {
|
|
61
47
|
const selected = Array.isArray(scope) ? (scope as readonly Snapshot[]) : (scope as Scope).snapshots;
|
|
62
48
|
if (selected.length === 0) {
|
|
@@ -64,12 +50,6 @@ export async function copySnapshots(
|
|
|
64
50
|
"copySnapshots got no snapshots to copy. Check the experiments filter, or pass snapshots from openResults().latest().",
|
|
65
51
|
);
|
|
66
52
|
}
|
|
67
|
-
if (opts?.redact === undefined) {
|
|
68
|
-
throw new Error(
|
|
69
|
-
'copySnapshots requires an explicit "redact" option: pass a (text) => string sanitizer, or the literal false to declare this data safe to publish verbatim.',
|
|
70
|
-
);
|
|
71
|
-
}
|
|
72
|
-
const redactor: Redactor | undefined = opts.redact === false ? undefined : opts.redact;
|
|
73
53
|
const kinds = opts.artifacts ?? [...DEFAULT_PUBLISH_ARTIFACTS];
|
|
74
54
|
for (const kind of kinds) {
|
|
75
55
|
if (!VALID_ARTIFACTS.includes(kind)) {
|
|
@@ -97,11 +77,11 @@ export async function copySnapshots(
|
|
|
97
77
|
if (isNewerSnapshot(snapshot, existing)) byExperiment.set(snapshot.experimentId, snapshot);
|
|
98
78
|
}
|
|
99
79
|
|
|
100
|
-
//
|
|
101
|
-
//
|
|
80
|
+
// 发布前整文件预检:先规划并序列化全部目标文件,任一文件超过 PUBLISH_FILE_MAX_BYTES
|
|
81
|
+
// 就整体失败,不留半成品目标目录。
|
|
102
82
|
const planned: PlannedFile[] = [];
|
|
103
83
|
for (const snapshot of byExperiment.values()) {
|
|
104
|
-
planned.push(...(await planOneSnapshot(snapshot, [...selected], dest, kinds
|
|
84
|
+
planned.push(...(await planOneSnapshot(snapshot, [...selected], dest, kinds)));
|
|
105
85
|
}
|
|
106
86
|
const oversized = planned.filter((f) => f.bytes.byteLength > PUBLISH_FILE_MAX_BYTES);
|
|
107
87
|
if (oversized.length > 0) {
|
|
@@ -135,7 +115,6 @@ async function planOneSnapshot(
|
|
|
135
115
|
selected: Snapshot[],
|
|
136
116
|
destRoot: string,
|
|
137
117
|
kinds: ArtifactKind[],
|
|
138
|
-
redactor: Redactor | undefined,
|
|
139
118
|
): Promise<PlannedFile[]> {
|
|
140
119
|
const destSnapDir = join(destRoot, experimentDirOf(snapshot.experimentId), basename(snapshot.dir));
|
|
141
120
|
const planned: PlannedFile[] = [];
|
|
@@ -144,31 +123,22 @@ async function planOneSnapshot(
|
|
|
144
123
|
// 复制到目的地也只应该有一份——这个 Set 记录本快照已经规划过的 hash,整快照的 attempt 共享。
|
|
145
124
|
const plannedSourceHashes = new Set<string>();
|
|
146
125
|
for (const attempt of snapshot.attempts) {
|
|
147
|
-
planned.push(...(await planOneAttempt(attempt, destSnapDir, kinds, plannedSourceHashes
|
|
126
|
+
planned.push(...(await planOneAttempt(attempt, destSnapDir, kinds, plannedSourceHashes)));
|
|
148
127
|
}
|
|
149
128
|
|
|
150
129
|
const knownEvalIds = experimentOfSnapshot(snapshot)?.evalIds ?? fallbackUnion(selected, snapshot.experimentId);
|
|
151
|
-
const experiment =
|
|
152
|
-
snapshot.experiment !== undefined && redactor
|
|
153
|
-
? (redactExperimentInfo(snapshot.experiment as unknown as Record<string, unknown>, redactor) as unknown as SnapshotMeta["experiment"])
|
|
154
|
-
: snapshot.experiment;
|
|
155
130
|
const meta: SnapshotMeta = {
|
|
156
131
|
format: RESULTS_FORMAT,
|
|
157
132
|
schemaVersion: snapshot.schemaVersion,
|
|
158
133
|
producer: snapshot.producer,
|
|
159
134
|
experimentId: snapshot.experimentId,
|
|
160
|
-
...(experiment !== undefined ? { experiment } : {}),
|
|
135
|
+
...(snapshot.experiment !== undefined ? { experiment: snapshot.experiment } : {}),
|
|
161
136
|
agent: snapshot.agent,
|
|
162
137
|
...(snapshot.model !== undefined ? { model: snapshot.model } : {}),
|
|
163
138
|
startedAt: snapshot.startedAt,
|
|
164
139
|
...(snapshot.completedAt !== undefined ? { completedAt: snapshot.completedAt } : {}),
|
|
165
140
|
...(knownEvalIds.length ? { knownEvalIds } : {}),
|
|
166
|
-
...(snapshot.name !== undefined
|
|
167
|
-
? { name: redactor && typeof snapshot.name === "string" ? redactor(snapshot.name) : snapshot.name }
|
|
168
|
-
: {}),
|
|
169
|
-
// 发布根标记只声明流程,不证明结果:applied = 消毒函数对全部自由文本字段跑过;
|
|
170
|
-
// none = 作者显式的原文发布声明。view --out 的防呆据此分级。
|
|
171
|
-
publish: { redaction: redactor ? "applied" : "none" },
|
|
141
|
+
...(snapshot.name !== undefined ? { name: snapshot.name } : {}),
|
|
172
142
|
};
|
|
173
143
|
planned.push({ path: join(destSnapDir, SNAPSHOT_FILE), bytes: Buffer.from(JSON.stringify(meta, null, 2), "utf-8") });
|
|
174
144
|
return planned;
|
|
@@ -179,21 +149,18 @@ async function planOneAttempt(
|
|
|
179
149
|
destSnapDir: string,
|
|
180
150
|
kinds: ArtifactKind[],
|
|
181
151
|
plannedSourceHashes: Set<string>,
|
|
182
|
-
redactor: Redactor | undefined,
|
|
183
152
|
): Promise<PlannedFile[]> {
|
|
184
153
|
const destAttemptDir = join(destSnapDir, attempt.ref.attempt);
|
|
185
154
|
const planned: PlannedFile[] = [];
|
|
186
155
|
|
|
187
156
|
// sources 是唯一「两层」的 artifact(attempt 级引用 + 快照级去重仓库),不能像其它四类那样
|
|
188
157
|
// 单文件原字节完事——原字节只是引用,不带内容。走读取面已经会解引用+回退的 attempt.sources()
|
|
189
|
-
//
|
|
158
|
+
// 拿到完整内容,按内容哈希重新去重落盘——发布根里引用与内容永远一致,携带条目也被归拢进本快照。
|
|
190
159
|
const genericKinds = kinds.filter((k) => k !== "sources");
|
|
191
160
|
const files = await findArtifactFiles(attempt, genericKinds);
|
|
192
161
|
const copied = new Set(files.map((f) => f.kind));
|
|
193
162
|
for (const { kind, source } of files) {
|
|
194
|
-
|
|
195
|
-
const bytes = redactor ? redactArtifactBytes(kind, raw, redactor) : raw;
|
|
196
|
-
planned.push({ path: join(destAttemptDir, artifactFileOf(kind)), bytes, source });
|
|
163
|
+
planned.push({ path: join(destAttemptDir, artifactFileOf(kind)), bytes: await readFile(source), source });
|
|
197
164
|
}
|
|
198
165
|
|
|
199
166
|
if (kinds.includes("sources")) {
|
|
@@ -203,13 +170,12 @@ async function planOneAttempt(
|
|
|
203
170
|
const destStoreDir = join(destSnapDir, "sources");
|
|
204
171
|
const refs: { path: string; sha256: string }[] = [];
|
|
205
172
|
for (const src of sources) {
|
|
206
|
-
const
|
|
207
|
-
const sha256 = hashEvalSource(normalizeEvalSource(content));
|
|
173
|
+
const sha256 = hashEvalSource(normalizeEvalSource(src.content));
|
|
208
174
|
refs.push({ path: src.path, sha256 });
|
|
209
175
|
if (!plannedSourceHashes.has(sha256)) {
|
|
210
176
|
planned.push({
|
|
211
177
|
path: join(destStoreDir, `${sha256}.json`),
|
|
212
|
-
bytes: Buffer.from(JSON.stringify({ content }), "utf-8"),
|
|
178
|
+
bytes: Buffer.from(JSON.stringify({ content: src.content }), "utf-8"),
|
|
213
179
|
});
|
|
214
180
|
plannedSourceHashes.add(sha256);
|
|
215
181
|
}
|
|
@@ -221,40 +187,11 @@ async function planOneAttempt(
|
|
|
221
187
|
}
|
|
222
188
|
}
|
|
223
189
|
|
|
224
|
-
|
|
225
|
-
if (redactor) record = redactResultRecord(record, redactor);
|
|
190
|
+
const record = slimForCopy(attempt.result, copied);
|
|
226
191
|
planned.push({ path: join(destAttemptDir, RESULT_FILE), bytes: Buffer.from(JSON.stringify(record, null, 2), "utf-8") });
|
|
227
192
|
return planned;
|
|
228
193
|
}
|
|
229
194
|
|
|
230
|
-
/** 单个 artifact 文件的消毒:按种类解析 JSON、走各自的自由文本标注,重新序列化。 */
|
|
231
|
-
function redactArtifactBytes(kind: ArtifactKind, raw: Buffer, redactor: Redactor): Buffer {
|
|
232
|
-
try {
|
|
233
|
-
const parsed = JSON.parse(raw.toString("utf-8")) as unknown;
|
|
234
|
-
let next: unknown;
|
|
235
|
-
switch (kind) {
|
|
236
|
-
case "events":
|
|
237
|
-
next = redactEvents(parsed as StreamEvent[], redactor);
|
|
238
|
-
break;
|
|
239
|
-
case "trace":
|
|
240
|
-
next = redactSpans(parsed as TraceSpan[], redactor);
|
|
241
|
-
break;
|
|
242
|
-
// diff 的 before/after/内容与 o11y 的命令 / 错误 / URL 都是自由文本;路径键为结构字段。
|
|
243
|
-
case "diff":
|
|
244
|
-
case "o11y":
|
|
245
|
-
case "agentSetup":
|
|
246
|
-
next = redactJsonValue(parsed, redactor);
|
|
247
|
-
break;
|
|
248
|
-
default:
|
|
249
|
-
next = parsed;
|
|
250
|
-
}
|
|
251
|
-
return Buffer.from(JSON.stringify(next), "utf-8");
|
|
252
|
-
} catch {
|
|
253
|
-
// 解析不了的按原字节透传(malformed 数据不该在发布路径上被静默改写)。
|
|
254
|
-
return raw;
|
|
255
|
-
}
|
|
256
|
-
}
|
|
257
|
-
|
|
258
195
|
/** 目标目录非空即报错:盘上不该出现「我没写的东西被动过」的惊讶;发布脚本要幂等就自己先清目录。 */
|
|
259
196
|
async function assertEmptyDestination(dest: string): Promise<void> {
|
|
260
197
|
let entries: string[];
|
package/src/results/publish.ts
CHANGED
|
@@ -1,149 +1,7 @@
|
|
|
1
|
-
//
|
|
2
|
-
//
|
|
3
|
-
//
|
|
4
|
-
//
|
|
5
|
-
// redact 逐值消毒,范围由 schema 的自由文本标注决定:redactor 只对自由文本字段调用——
|
|
6
|
-
// 格式、判定、身份、路径、哈希这类结构字段永不经过它,发布根不会因 redact 变得不可读或
|
|
7
|
-
// 引用断裂。自由文本清单在下面的 FREE_TEXT 标注单点维护(与 AttemptList.redact 同一口径)。
|
|
8
|
-
|
|
9
|
-
import type { EvalResult, StreamEvent, TraceSpan } from "../types.ts";
|
|
1
|
+
// 发布预算(见 docs/feature/results/library.md「复制与瘦身:copySnapshots」)。
|
|
2
|
+
// 结果数据分两类:.niceeval/ 是本地事实根,不是默认可提交目录;任何要离开本机的拷贝是
|
|
3
|
+
// 发布拷贝,经 copySnapshots 这一条管线产出。管线只做选择、归拢与整文件大小预检,
|
|
4
|
+
// 不改写内容——保密边界由格式在采集侧划定(env 值与命令 stdout/stderr 不进结果文件)。
|
|
10
5
|
|
|
11
6
|
/** 发布前整文件预检的单文件上限(50 MiB,为 GitHub 100 MB 硬限保留余量);不是可调旋钮。 */
|
|
12
7
|
export const PUBLISH_FILE_MAX_BYTES = 50 * 1024 * 1024;
|
|
13
|
-
|
|
14
|
-
export type Redactor = (text: string) => string;
|
|
15
|
-
|
|
16
|
-
/**
|
|
17
|
-
* 结构字段键名(取值是格式、判定、身份、路径、哈希——redactor 永不触碰):
|
|
18
|
-
* 事件的 type/callId/status/role/tool/skill/requestId/optionId、span 的 ids/kind、
|
|
19
|
-
* 断言的 name/severity/outcome、错误的 code/phase、locator/artifactBase/fingerprint、
|
|
20
|
-
* 源码路径与 sha256、provider 名、时间戳等。新增字符串字段先判断体裁再决定进不进这张表。
|
|
21
|
-
*/
|
|
22
|
-
const STRUCTURAL_KEYS = new Set([
|
|
23
|
-
"type",
|
|
24
|
-
"callId",
|
|
25
|
-
"status",
|
|
26
|
-
"role",
|
|
27
|
-
"tool",
|
|
28
|
-
"requestId",
|
|
29
|
-
"optionId",
|
|
30
|
-
"id",
|
|
31
|
-
"traceId",
|
|
32
|
-
"spanId",
|
|
33
|
-
"parentSpanId",
|
|
34
|
-
"kind",
|
|
35
|
-
"format",
|
|
36
|
-
"verdict",
|
|
37
|
-
"severity",
|
|
38
|
-
"outcome",
|
|
39
|
-
"artifactBase",
|
|
40
|
-
"locator",
|
|
41
|
-
"fingerprint",
|
|
42
|
-
"sha256",
|
|
43
|
-
"path",
|
|
44
|
-
"file",
|
|
45
|
-
"provider",
|
|
46
|
-
"sandboxId",
|
|
47
|
-
"code",
|
|
48
|
-
"phase",
|
|
49
|
-
"level",
|
|
50
|
-
"agent",
|
|
51
|
-
"model",
|
|
52
|
-
"experimentId",
|
|
53
|
-
"startedAt",
|
|
54
|
-
"completedAt",
|
|
55
|
-
"schemaVersion",
|
|
56
|
-
"evalFilterFingerprint",
|
|
57
|
-
"reasoningEffort",
|
|
58
|
-
"skill",
|
|
59
|
-
"window",
|
|
60
|
-
"net",
|
|
61
|
-
"loc",
|
|
62
|
-
"dedupeKey",
|
|
63
|
-
]);
|
|
64
|
-
|
|
65
|
-
/** 深度遍历 JSON 值:字符串按「键名是否结构字段」决定过不过 redactor;结构键下整棵子树跳过。 */
|
|
66
|
-
export function redactJsonValue(value: unknown, redact: Redactor, key?: string): unknown {
|
|
67
|
-
if (typeof value === "string") {
|
|
68
|
-
if (key !== undefined && STRUCTURAL_KEYS.has(key)) return value;
|
|
69
|
-
return redact(value);
|
|
70
|
-
}
|
|
71
|
-
if (Array.isArray(value)) return value.map((v) => redactJsonValue(v, redact, key));
|
|
72
|
-
if (value !== null && typeof value === "object") {
|
|
73
|
-
const out: Record<string, unknown> = {};
|
|
74
|
-
for (const [k, v] of Object.entries(value)) {
|
|
75
|
-
if (STRUCTURAL_KEYS.has(k) && typeof v === "string") {
|
|
76
|
-
out[k] = v;
|
|
77
|
-
continue;
|
|
78
|
-
}
|
|
79
|
-
out[k] = redactJsonValue(v, redact, k);
|
|
80
|
-
}
|
|
81
|
-
return out;
|
|
82
|
-
}
|
|
83
|
-
return value;
|
|
84
|
-
}
|
|
85
|
-
|
|
86
|
-
/** events.json 的发布消毒:消息与工具入出参是自由文本,type/callId/status 等结构字段不动。 */
|
|
87
|
-
export function redactEvents(events: StreamEvent[], redact: Redactor): StreamEvent[] {
|
|
88
|
-
return redactJsonValue(events, redact) as StreamEvent[];
|
|
89
|
-
}
|
|
90
|
-
|
|
91
|
-
/** trace.json 的发布消毒:属性值与可携带动态内容的 span name 过 redactor,ids/kind 不动。 */
|
|
92
|
-
export function redactSpans(spans: TraceSpan[], redact: Redactor): TraceSpan[] {
|
|
93
|
-
return spans.map((span) => ({
|
|
94
|
-
...span,
|
|
95
|
-
name: redact(span.name),
|
|
96
|
-
...(span.attributes !== undefined
|
|
97
|
-
? { attributes: redactJsonValue(span.attributes, redact) as TraceSpan["attributes"] }
|
|
98
|
-
: {}),
|
|
99
|
-
}));
|
|
100
|
-
}
|
|
101
|
-
|
|
102
|
-
/** result.json 的发布消毒:断言 detail/evidence/expected/received、error/diagnostic 的
|
|
103
|
-
* message/cause/stack、skipReason、description 是自由文本;判定与身份字段不动。 */
|
|
104
|
-
export function redactResultRecord(record: Record<string, unknown>, redact: Redactor): Record<string, unknown> {
|
|
105
|
-
const r = record as Partial<EvalResult> & Record<string, unknown>;
|
|
106
|
-
const out: Record<string, unknown> = { ...record };
|
|
107
|
-
if (typeof r.description === "string") out.description = redact(r.description);
|
|
108
|
-
if (typeof r.skipReason === "string") out.skipReason = redact(r.skipReason);
|
|
109
|
-
if (Array.isArray(r.assertions)) {
|
|
110
|
-
out.assertions = r.assertions.map((a) => ({
|
|
111
|
-
...a,
|
|
112
|
-
...(a.detail !== undefined ? { detail: redact(a.detail) } : {}),
|
|
113
|
-
...(a.outcome !== "unavailable" && a.evidence !== undefined ? { evidence: redact(a.evidence) } : {}),
|
|
114
|
-
...(a.outcome !== "unavailable" && a.expected !== undefined ? { expected: redact(a.expected) } : {}),
|
|
115
|
-
...(a.outcome !== "unavailable" && a.received !== undefined ? { received: redact(a.received) } : {}),
|
|
116
|
-
}));
|
|
117
|
-
}
|
|
118
|
-
if (r.error !== undefined) {
|
|
119
|
-
out.error = {
|
|
120
|
-
...r.error,
|
|
121
|
-
message: redact(r.error.message),
|
|
122
|
-
...(r.error.stack !== undefined ? { stack: redact(r.error.stack) } : {}),
|
|
123
|
-
...(r.error.cause !== undefined ? { cause: { ...r.error.cause, message: redact(r.error.cause.message) } } : {}),
|
|
124
|
-
};
|
|
125
|
-
}
|
|
126
|
-
if (Array.isArray(r.diagnostics)) {
|
|
127
|
-
out.diagnostics = r.diagnostics.map((d) => ({
|
|
128
|
-
...d,
|
|
129
|
-
message: redact(d.message),
|
|
130
|
-
...(d.data !== undefined ? { data: redactJsonValue(d.data, redact) as typeof d.data } : {}),
|
|
131
|
-
}));
|
|
132
|
-
}
|
|
133
|
-
if (r.experiment !== undefined) {
|
|
134
|
-
out.experiment = redactExperimentInfo(r.experiment as unknown as Record<string, unknown>, redact);
|
|
135
|
-
}
|
|
136
|
-
return out;
|
|
137
|
-
}
|
|
138
|
-
|
|
139
|
-
/** ExperimentRunInfo 的发布消毒:description、flags 与 sandbox params 的字符串值。 */
|
|
140
|
-
export function redactExperimentInfo(info: Record<string, unknown>, redact: Redactor): Record<string, unknown> {
|
|
141
|
-
const out: Record<string, unknown> = { ...info };
|
|
142
|
-
if (typeof info.description === "string") out.description = redact(info.description);
|
|
143
|
-
if (info.flags !== undefined) out.flags = redactJsonValue(info.flags, redact);
|
|
144
|
-
const sandbox = info.sandbox as { provider: string; params?: Record<string, unknown> } | undefined;
|
|
145
|
-
if (sandbox?.params !== undefined) {
|
|
146
|
-
out.sandbox = { ...sandbox, params: redactJsonValue(sandbox.params, redact) };
|
|
147
|
-
}
|
|
148
|
-
return out;
|
|
149
|
-
}
|
|
@@ -552,7 +552,7 @@ describe("createResultsWriter", () => {
|
|
|
552
552
|
expect(await q2.agentSetup()).toBeNull();
|
|
553
553
|
|
|
554
554
|
const dest = join(await makeRoot(), "published");
|
|
555
|
-
await copySnapshots(results.latest(), dest, {
|
|
555
|
+
await copySnapshots(results.latest(), dest, { artifacts: ["agentSetup"] });
|
|
556
556
|
const copied = join(dest, "skill-ab_claude-effect", basename(snap.dir), "q1", "a1", "agent-setup.json");
|
|
557
557
|
expect(JSON.parse(await readFile(copied, "utf-8"))).toEqual(manifest);
|
|
558
558
|
});
|
|
@@ -737,7 +737,7 @@ describe("copySnapshots", () => {
|
|
|
737
737
|
|
|
738
738
|
const results = await openResults(root);
|
|
739
739
|
const dest = join(await makeRoot(), "site/data/run");
|
|
740
|
-
const copied = await copySnapshots(results.latest(), dest, {
|
|
740
|
+
const copied = await copySnapshots(results.latest(), dest, { artifacts: ["events"] });
|
|
741
741
|
|
|
742
742
|
expect(copied.warnings).toHaveLength(0);
|
|
743
743
|
expect(copied.dir).toBe(dest);
|
|
@@ -775,15 +775,15 @@ describe("copySnapshots", () => {
|
|
|
775
775
|
|
|
776
776
|
const occupied = await makeRoot();
|
|
777
777
|
await writeFile(join(occupied, "existing.txt"), "x", "utf-8");
|
|
778
|
-
await expect(copySnapshots(results.latest(), occupied
|
|
778
|
+
await expect(copySnapshots(results.latest(), occupied)).rejects.toThrow(/not empty/);
|
|
779
779
|
|
|
780
|
-
await expect(copySnapshots(results.latest(), join(await makeRoot(), "out"), {
|
|
780
|
+
await expect(copySnapshots(results.latest(), join(await makeRoot(), "out"), { artifacts: ["evnets" as never] })).rejects.toThrow(/Unknown artifact kind/);
|
|
781
781
|
|
|
782
|
-
await expect(copySnapshots([], join(await makeRoot(), "out")
|
|
782
|
+
await expect(copySnapshots([], join(await makeRoot(), "out"))).rejects.toThrow(/no snapshots/);
|
|
783
783
|
|
|
784
784
|
// 手工传入同一 experiment 的两个快照(未走 latest 去重):只带最新,记 warning。
|
|
785
785
|
const dest2 = join(await makeRoot(), "run2");
|
|
786
|
-
const collided = await copySnapshots(results.experiments[0].snapshots, dest2
|
|
786
|
+
const collided = await copySnapshots(results.experiments[0].snapshots, dest2);
|
|
787
787
|
expect(collided.warnings).toHaveLength(1);
|
|
788
788
|
expect(collided.warnings[0]).toMatch(/multiple snapshots selected/);
|
|
789
789
|
const destDirs = await readdir(join(dest2, "e"));
|
|
@@ -1005,7 +1005,7 @@ describe("AttemptLocator · 落盘 / 读取 / 携带 / 撞车", () => {
|
|
|
1005
1005
|
const locator1 = a1!.locator!;
|
|
1006
1006
|
|
|
1007
1007
|
const dest = join(await makeRoot(), "published");
|
|
1008
|
-
await copySnapshots(results.latest(), dest, {
|
|
1008
|
+
await copySnapshots(results.latest(), dest, { artifacts: [] });
|
|
1009
1009
|
|
|
1010
1010
|
const destResults = await openResults(dest);
|
|
1011
1011
|
expect(resolveLocator(destResults, locator0).result.attempt).toBe(0);
|
|
@@ -1192,7 +1192,7 @@ describe("sources · 快照级去重仓库", () => {
|
|
|
1192
1192
|
|
|
1193
1193
|
const results = await openResults(root);
|
|
1194
1194
|
const dest = join(await makeRoot(), "published");
|
|
1195
|
-
await copySnapshots(results.latest(), dest, {
|
|
1195
|
+
await copySnapshots(results.latest(), dest, { artifacts: ["sources"] });
|
|
1196
1196
|
|
|
1197
1197
|
const destSnapDir = join(dest, "e", basename(snap.dir));
|
|
1198
1198
|
const destStoreFiles = await readdir(join(destSnapDir, "sources"));
|
package/src/results/types.ts
CHANGED
|
@@ -44,11 +44,6 @@ export interface SnapshotMeta {
|
|
|
44
44
|
completedAt?: string;
|
|
45
45
|
/** 写入时刻该实验已知的 eval 并集 —— 残缺检测的分母随数据走(copySnapshots 自动补记,writer 可声明)。 */
|
|
46
46
|
knownEvalIds?: string[];
|
|
47
|
-
/**
|
|
48
|
-
* 发布拷贝的自描述标记:copySnapshots 补记,消毒函数 → "applied"、redact: false → "none";
|
|
49
|
-
* 本地事实根没有此字段。只声明流程,不证明无秘密;view --out 据此分级防呆。
|
|
50
|
-
*/
|
|
51
|
-
publish?: { redaction: "applied" | "none" };
|
|
52
47
|
/** 项目名(来自 config.name),透传给 `niceeval view` 顶部 hero 显示。 */
|
|
53
48
|
name?: LocalizedText;
|
|
54
49
|
}
|
|
@@ -130,8 +125,6 @@ export interface Snapshot {
|
|
|
130
125
|
dir: string;
|
|
131
126
|
/** 写入时刻该实验已知的 eval 并集(可选);copySnapshots 自动补记,writer.snapshot() 也可声明。 */
|
|
132
127
|
knownEvalIds?: string[];
|
|
133
|
-
/** 发布拷贝的自描述标记(见 SnapshotMeta.publish);本地事实根没有此字段。 */
|
|
134
|
-
publish?: { redaction: "applied" | "none" };
|
|
135
128
|
}
|
|
136
129
|
|
|
137
130
|
/** 一个实验的全部历史:同一 experiment id 的历次快照归在一起。 */
|
package/src/show/report-host.ts
CHANGED
|
@@ -44,11 +44,21 @@ export interface HostReportLink {
|
|
|
44
44
|
/** `{src}` 与 `{inline}` 两种形态不可同时出现(shell.md「字段穷尽」)。 */
|
|
45
45
|
export type HostReportAsset = { src: string; inline?: never } | { inline: string; src?: never };
|
|
46
46
|
|
|
47
|
+
/**
|
|
48
|
+
* 结构化 head 标签(shell.md「字段穷尽」):白名单闭集(meta/link/script/style),
|
|
49
|
+
* attrs 值 true 渲染裸布尔属性、字符串转义后渲染 `key="value"`;script/style 的
|
|
50
|
+
* children 原样落进标签。形状与闭合序列在 defineReport 装载期已校验。
|
|
51
|
+
*/
|
|
52
|
+
export type HostHeadTag =
|
|
53
|
+
| { tag: "meta" | "link"; attrs: Record<string, string | true>; children?: never }
|
|
54
|
+
| { tag: "script" | "style"; attrs?: Record<string, string | true>; children?: string };
|
|
55
|
+
|
|
47
56
|
/** 装载规范化产物:外壳 + 非空页列表。show 只消费 title / pages;其余是 web 面属性。 */
|
|
48
57
|
export interface HostReport {
|
|
49
58
|
title?: LocalizedText;
|
|
50
59
|
links: HostReportLink[];
|
|
51
60
|
footer?: LocalizedText;
|
|
61
|
+
head: HostHeadTag[];
|
|
52
62
|
scripts: HostReportAsset[];
|
|
53
63
|
styles: HostReportAsset[];
|
|
54
64
|
pages: HostReportPage[];
|
|
@@ -193,6 +203,7 @@ export function normalizeHostReport(definition: unknown, sourceLabel: string): H
|
|
|
193
203
|
title?: LocalizedText;
|
|
194
204
|
links?: HostReportLink[];
|
|
195
205
|
footer?: LocalizedText;
|
|
206
|
+
head?: HostHeadTag[];
|
|
196
207
|
scripts?: HostReportAsset[];
|
|
197
208
|
styles?: HostReportAsset[];
|
|
198
209
|
content?: unknown;
|
|
@@ -231,6 +242,7 @@ export function normalizeHostReport(definition: unknown, sourceLabel: string): H
|
|
|
231
242
|
...(def.title !== undefined ? { title: def.title } : {}),
|
|
232
243
|
links: def.links ?? [],
|
|
233
244
|
...(def.footer !== undefined ? { footer: def.footer } : {}),
|
|
245
|
+
head: def.head ?? [],
|
|
234
246
|
scripts: def.scripts ?? [],
|
|
235
247
|
styles: def.styles ?? [],
|
|
236
248
|
pages,
|
|
@@ -241,6 +253,7 @@ export function normalizeHostReport(definition: unknown, sourceLabel: string): H
|
|
|
241
253
|
if (isLegacyDefinition(definition)) {
|
|
242
254
|
return {
|
|
243
255
|
links: [],
|
|
256
|
+
head: [],
|
|
244
257
|
scripts: [],
|
|
245
258
|
styles: [],
|
|
246
259
|
pages: [{ id: SINGLE_PAGE_ID, title: BUILT_IN_PAGE_TITLE, content: definition }],
|