niceeval 0.8.0 → 0.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/report/components.d.ts +4 -5
- package/dist/report/components.js +7 -7
- package/dist/report/compute.d.ts +4 -4
- package/dist/report/compute.js +8 -12
- package/dist/report/index.d.ts +1 -1
- package/dist/report/types.d.ts +0 -10
- package/dist/results/types.d.ts +0 -11
- package/docs-site/zh/how-to/publish-report.mdx +1 -4
- package/docs-site/zh/reference/cli.mdx +0 -1
- package/docs-site/zh/reference/report-components.mdx +2 -2
- package/docs-site/zh/reference/results-data.mdx +2 -4
- package/package.json +1 -1
- package/src/cli.ts +1 -5
- package/src/context/context.ts +11 -5
- package/src/report/components.tsx +13 -15
- package/src/report/compute.ts +8 -20
- package/src/report/index.ts +0 -1
- package/src/report/report.test.ts +2 -20
- package/src/report/types.ts +0 -11
- package/src/results/copy.ts +15 -78
- package/src/results/publish.ts +4 -146
- package/src/results/results.test.ts +8 -8
- package/src/results/types.ts +0 -7
- package/src/view/artifact-serving.test.ts +1 -1
- package/src/view/data.ts +0 -11
- package/src/view/index.ts +1 -12
- package/src/view/server.ts +0 -2
- package/src/view/site-parity.test.ts +1 -1
- package/src/view/site.ts +1 -1
- package/src/view/view-report.test.ts +5 -5
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { type ReportComponent } from "./tree.ts";
|
|
2
2
|
import type { ReportLocale } from "./locale.ts";
|
|
3
3
|
import type { AttemptLocator } from "../results/locator.ts";
|
|
4
|
-
import type { AttemptListItem, DeltaData,
|
|
4
|
+
import type { AttemptListItem, DeltaData, EvalListItem, ExperimentComparisonData, ExperimentListItem, LineData, MatrixData, ReportInput, ScatterData, ScopeSummaryData, ScoreboardData, TableData } from "./types.ts";
|
|
5
5
|
import { type DeltaTableOptions, type MetricLineOptions, type MetricMatrixOptions, type MetricScatterOptions, type MetricTableOptions, type ScoreboardOptions } from "./compute.ts";
|
|
6
6
|
type Never<T> = {
|
|
7
7
|
[K in keyof T]?: never;
|
|
@@ -38,7 +38,7 @@ export declare const ExperimentComparison: ReportComponent<ExperimentComparisonP
|
|
|
38
38
|
interface EntityListChrome extends ChromeProps {
|
|
39
39
|
attemptHref?: (locator: AttemptLocator) => string;
|
|
40
40
|
}
|
|
41
|
-
export type ExperimentListProps = DataProps<readonly ExperimentListItem[],
|
|
41
|
+
export type ExperimentListProps = DataProps<readonly ExperimentListItem[], Record<never, never>, EntityListChrome & {
|
|
42
42
|
/** web 面在比较表前显示实验过滤框;text 面忽略。 */
|
|
43
43
|
filter?: boolean;
|
|
44
44
|
/**
|
|
@@ -49,10 +49,10 @@ export type ExperimentListProps = DataProps<readonly ExperimentListItem[], Entit
|
|
|
49
49
|
}>;
|
|
50
50
|
/** 实验列表:每项一个 experiment,固定八列比较表 + 展开到 Eval / Attempt。 */
|
|
51
51
|
export declare const ExperimentList: ReportComponent<ExperimentListProps>;
|
|
52
|
-
export type EvalListProps = DataProps<readonly EvalListItem[],
|
|
52
|
+
export type EvalListProps = DataProps<readonly EvalListItem[], Record<never, never>, EntityListChrome>;
|
|
53
53
|
/** Eval 列表:每项一个 experimentId + evalId,展开到这道题的 Attempt。 */
|
|
54
54
|
export declare const EvalList: ReportComponent<EvalListProps>;
|
|
55
|
-
export type AttemptListProps = DataProps<readonly AttemptListItem[],
|
|
55
|
+
export type AttemptListProps = DataProps<readonly AttemptListItem[], Record<never, never>, EntityListChrome & {
|
|
56
56
|
/** 过滤 / 截断前的总数;省略时等于 data 长度。 */
|
|
57
57
|
total?: number;
|
|
58
58
|
}>;
|
|
@@ -63,7 +63,6 @@ export interface FailureListProps {
|
|
|
63
63
|
limit?: number;
|
|
64
64
|
/** 默认宿主注入的 Scope。 */
|
|
65
65
|
input?: ReportInput;
|
|
66
|
-
redact?: (text: string) => string;
|
|
67
66
|
attemptHref?: (locator: AttemptLocator) => string;
|
|
68
67
|
locale?: ReportLocale;
|
|
69
68
|
className?: string;
|
|
@@ -261,8 +261,8 @@ export const ExperimentList = makeDataComponent({
|
|
|
261
261
|
name: "ExperimentList",
|
|
262
262
|
dataFnName: "experimentListData",
|
|
263
263
|
shapeName: "ExperimentListItem[]",
|
|
264
|
-
dataFn: (input
|
|
265
|
-
specKeys: [
|
|
264
|
+
dataFn: (input) => experimentListData(input),
|
|
265
|
+
specKeys: [],
|
|
266
266
|
validate: validateExperimentListData,
|
|
267
267
|
web: (props, ctx) => (_jsx(ExperimentListWeb, { data: props.data, filter: props.filter, relativeTo: props.relativeTo, locale: props.locale ?? ctx.locale, attemptHref: hrefOf(props, ctx) ?? ctx.attemptHref, className: props.className })),
|
|
268
268
|
text: (props, ctx) => experimentListText(props.data, ctx, props.relativeTo),
|
|
@@ -272,8 +272,8 @@ export const EvalList = makeDataComponent({
|
|
|
272
272
|
name: "EvalList",
|
|
273
273
|
dataFnName: "evalListData",
|
|
274
274
|
shapeName: "EvalListItem[]",
|
|
275
|
-
dataFn: (input
|
|
276
|
-
specKeys: [
|
|
275
|
+
dataFn: (input) => evalListData(input),
|
|
276
|
+
specKeys: [],
|
|
277
277
|
validate: validateEvalListData,
|
|
278
278
|
web: (props, ctx) => (_jsx(EvalListWeb, { data: props.data, locale: props.locale ?? ctx.locale, attemptHref: hrefOf(props, ctx) ?? ctx.attemptHref, className: props.className })),
|
|
279
279
|
text: (props, ctx) => evalListText(props.data, ctx),
|
|
@@ -283,8 +283,8 @@ export const AttemptList = makeDataComponent({
|
|
|
283
283
|
name: "AttemptList",
|
|
284
284
|
dataFnName: "attemptListData",
|
|
285
285
|
shapeName: "AttemptListItem[]",
|
|
286
|
-
dataFn: (input
|
|
287
|
-
specKeys: [
|
|
286
|
+
dataFn: (input) => attemptListData(input),
|
|
287
|
+
specKeys: [],
|
|
288
288
|
validate: validateAttemptListData,
|
|
289
289
|
web: (props, ctx) => (_jsx(AttemptListWeb, { data: props.data, total: props.total, locale: props.locale ?? ctx.locale, attemptHref: hrefOf(props, ctx) ?? ctx.attemptHref, className: props.className })),
|
|
290
290
|
text: (props, ctx) => attemptListText(props.data, props.total, ctx),
|
|
@@ -297,7 +297,7 @@ export const AttemptList = makeDataComponent({
|
|
|
297
297
|
*/
|
|
298
298
|
export const FailureList = defineComponent(async (props, ctx) => {
|
|
299
299
|
const input = props.input ?? ctx.scope;
|
|
300
|
-
const all = await attemptListData(input
|
|
300
|
+
const all = await attemptListData(input);
|
|
301
301
|
// attempt 开始时间不在列表条目里(它不是列表展示字段);从同一 input 的读取面按 locator 对回。
|
|
302
302
|
const startedAtByLocator = new Map();
|
|
303
303
|
for (const item of collectItems(resolveInput(input).snapshots)) {
|
package/dist/report/compute.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { AttemptListItem, DeltaData, DeltaPair, DimensionInput,
|
|
1
|
+
import type { AttemptListItem, DeltaData, DeltaPair, DimensionInput, EvalListItem, ExperimentComparisonData, ExperimentListItem, FlagPairs, LineData, MatrixData, Metric, NumericAxis, ReportInput, ScatterData, ScopeSummaryData, ScoreboardData, TableData } from "./types.ts";
|
|
2
2
|
import type { JsonValue } from "../types.ts";
|
|
3
3
|
export interface MetricTableOptions {
|
|
4
4
|
/** 行维度(内置 / 自定义 / flag() / runConfig())。 */
|
|
@@ -23,9 +23,9 @@ export interface MetricMatrixOptions {
|
|
|
23
23
|
}
|
|
24
24
|
export declare function metricMatrixData(input: ReportInput, options: MetricMatrixOptions): Promise<MatrixData>;
|
|
25
25
|
/** `attemptListData(input)`:每个 Attempt 一项,顺序取自 Scope 展平顺序(不重排)。 */
|
|
26
|
-
export declare function attemptListData(input: ReportInput
|
|
26
|
+
export declare function attemptListData(input: ReportInput): Promise<AttemptListItem[]>;
|
|
27
27
|
/** `evalListData(input)`:每个 `experimentId + evalId` 一项,按 evalId 再按 experimentId 升序。 */
|
|
28
|
-
export declare function evalListData(input: ReportInput
|
|
28
|
+
export declare function evalListData(input: ReportInput): Promise<EvalListItem[]>;
|
|
29
29
|
/**
|
|
30
30
|
* `experimentListData(input)`:每个 experiment 一项,展开到每道 Eval;初始按端到端成功率
|
|
31
31
|
* 从高到低(缺数据沉底,同分按 id)。一行只有一套 agent / model / flags 是输入约束:
|
|
@@ -33,7 +33,7 @@ export declare function evalListData(input: ReportInput, options?: EntityListDat
|
|
|
33
33
|
* Snapshot[] 时若同一 experiment 混入不一致的可比性配置,按完整用户反馈失败并指引——
|
|
34
34
|
* 看跨配置演化用 snapshot 维度或 MetricLine,不把两套配置拼成一行冒充单一配置。
|
|
35
35
|
*/
|
|
36
|
-
export declare function experimentListData(input: ReportInput
|
|
36
|
+
export declare function experimentListData(input: ReportInput): Promise<ExperimentListItem[]>;
|
|
37
37
|
/**
|
|
38
38
|
* `scopeSummaryData(input)`:范围摘要——快照时间窗、experiment / eval / attempt 数、
|
|
39
39
|
* 两级判定计票、端到端成功率与总成本(docs/feature/reports/library/summaries.md)。
|
package/dist/report/compute.js
CHANGED
|
@@ -117,9 +117,8 @@ function failureSummaryOf(result) {
|
|
|
117
117
|
}
|
|
118
118
|
return { summary: null, more: 0 };
|
|
119
119
|
}
|
|
120
|
-
const identityRedact = (text) => text;
|
|
121
120
|
/** AttemptList / ExperimentList / EvalList 共用的叶子构造:一个 Item → 一个 AttemptListItem。 */
|
|
122
|
-
async function attemptListItemOf(item
|
|
121
|
+
async function attemptListItemOf(item) {
|
|
123
122
|
const result = item.attempt.result;
|
|
124
123
|
const { summary, more } = failureSummaryOf(result);
|
|
125
124
|
return {
|
|
@@ -128,7 +127,7 @@ async function attemptListItemOf(item, redact) {
|
|
|
128
127
|
attempt: result.attempt,
|
|
129
128
|
agent: result.agent,
|
|
130
129
|
verdict: result.verdict,
|
|
131
|
-
failureSummary: summary
|
|
130
|
+
failureSummary: summary,
|
|
132
131
|
moreFailures: more,
|
|
133
132
|
examScore: await computeCell(examScore, [item]),
|
|
134
133
|
durationMs: result.durationMs,
|
|
@@ -137,16 +136,14 @@ async function attemptListItemOf(item, redact) {
|
|
|
137
136
|
};
|
|
138
137
|
}
|
|
139
138
|
/** `attemptListData(input)`:每个 Attempt 一项,顺序取自 Scope 展平顺序(不重排)。 */
|
|
140
|
-
export async function attemptListData(input
|
|
139
|
+
export async function attemptListData(input) {
|
|
141
140
|
const { snapshots } = resolveInput(input);
|
|
142
|
-
const redact = options?.redact ?? identityRedact;
|
|
143
141
|
const items = collectItems(snapshots);
|
|
144
|
-
return Promise.all(items.map((item) => attemptListItemOf(item
|
|
142
|
+
return Promise.all(items.map((item) => attemptListItemOf(item)));
|
|
145
143
|
}
|
|
146
144
|
/** `evalListData(input)`:每个 `experimentId + evalId` 一项,按 evalId 再按 experimentId 升序。 */
|
|
147
|
-
export async function evalListData(input
|
|
145
|
+
export async function evalListData(input) {
|
|
148
146
|
const { snapshots } = resolveInput(input);
|
|
149
|
-
const redact = options?.redact ?? identityRedact;
|
|
150
147
|
const items = collectItems(snapshots);
|
|
151
148
|
const groups = new Map();
|
|
152
149
|
for (const item of items) {
|
|
@@ -161,7 +158,7 @@ export async function evalListData(input, options) {
|
|
|
161
158
|
for (const group of groups.values()) {
|
|
162
159
|
const sorted = [...group].sort((a, b) => a.attempt.result.attempt - b.attempt.result.attempt);
|
|
163
160
|
const verdict = foldEvalVerdict(sorted.map((item) => item.attempt.result));
|
|
164
|
-
const attempts = await Promise.all(sorted.map((item) => attemptListItemOf(item
|
|
161
|
+
const attempts = await Promise.all(sorted.map((item) => attemptListItemOf(item)));
|
|
165
162
|
out.push({
|
|
166
163
|
experimentId: experimentIdOf(sorted[0]),
|
|
167
164
|
evalId: evalIdOf(sorted[0]),
|
|
@@ -182,9 +179,8 @@ export async function evalListData(input, options) {
|
|
|
182
179
|
* Snapshot[] 时若同一 experiment 混入不一致的可比性配置,按完整用户反馈失败并指引——
|
|
183
180
|
* 看跨配置演化用 snapshot 维度或 MetricLine,不把两套配置拼成一行冒充单一配置。
|
|
184
181
|
*/
|
|
185
|
-
export async function experimentListData(input
|
|
182
|
+
export async function experimentListData(input) {
|
|
186
183
|
const { snapshots } = resolveInput(input);
|
|
187
|
-
const redact = options?.redact ?? identityRedact;
|
|
188
184
|
// 可比性配置单义检查:同一 experiment 的输入快照必须共享一套可比性配置。
|
|
189
185
|
const configByExperiment = new Map();
|
|
190
186
|
for (const snapshot of snapshots) {
|
|
@@ -211,7 +207,7 @@ export async function experimentListData(input, options) {
|
|
|
211
207
|
for (const [evalId, evalItems] of evalGroups) {
|
|
212
208
|
const sorted = [...evalItems].sort((a, b) => a.attempt.result.attempt - b.attempt.result.attempt);
|
|
213
209
|
const verdict = foldEvalVerdict(sorted.map((item) => item.attempt.result));
|
|
214
|
-
const attempts = await Promise.all(sorted.map((item) => attemptListItemOf(item
|
|
210
|
+
const attempts = await Promise.all(sorted.map((item) => attemptListItemOf(item)));
|
|
215
211
|
evalRows.push({
|
|
216
212
|
evalId,
|
|
217
213
|
verdict,
|
package/dist/report/index.d.ts
CHANGED
|
@@ -14,6 +14,6 @@ export { AttemptList, DeltaTable, EvalList, ExperimentComparison, ExperimentList
|
|
|
14
14
|
export type { AttemptListProps, DataProps, DeltaTableProps, EvalListProps, ExperimentComparisonProps, ExperimentListProps, FailureListProps, MetricBarsProps, MetricLineProps, MetricMatrixProps, MetricScatterProps, MetricTableProps, ScoreboardProps, ScopeSummaryProps, } from "./components.tsx";
|
|
15
15
|
export { attemptListData, deltaTableData, evalListData, experimentComparisonData, experimentListData, metricLineData, metricMatrixData, metricScatterData, metricTableData, pairsByFlag, scopeSummaryData, scoreboardData, } from "./compute.ts";
|
|
16
16
|
export type { DeltaTableOptions, MetricLineOptions, MetricMatrixOptions, MetricScatterOptions, MetricTableOptions, ScoreboardOptions, } from "./compute.ts";
|
|
17
|
-
export type { Aggregator, AttemptListItem, AttemptLocator, BuiltInDimension, CustomDimension, DeltaData, DeltaPair, DimensionInput, DimensionOptions, DimensionRef,
|
|
17
|
+
export type { Aggregator, AttemptListItem, AttemptLocator, BuiltInDimension, CustomDimension, DeltaData, DeltaPair, DimensionInput, DimensionOptions, DimensionRef, EvalListItem, ExperimentComparisonData, ExperimentComparisonGroupData, ExperimentListEvalRow, ExperimentListItem, FlagPairs, LineData, MatrixData, Metric, MetricAggregate, MetricCell, MetricColumn, NumericAxis, NumericAxisOptions, NumericRunConfigAxisOptions, ReportInput, RunConfigKey, ScatterData, ScopeSummaryData, ScopeWarning, ScoreboardData, TableData, VerdictTally, } from "./types.ts";
|
|
18
18
|
export type { AttemptHandle, Results, Scope, Snapshot } from "../results/types.ts";
|
|
19
19
|
export { experimentGroupOf } from "../shared/aggregate.ts";
|
package/dist/report/types.d.ts
CHANGED
|
@@ -359,13 +359,3 @@ export interface ExperimentListItem {
|
|
|
359
359
|
lastRunAt: string;
|
|
360
360
|
evalRows: ExperimentListEvalRow[];
|
|
361
361
|
}
|
|
362
|
-
/** 三个实体列表共用的计算选项。 */
|
|
363
|
-
export interface EntityListDataOptions {
|
|
364
|
-
/**
|
|
365
|
-
* 展示层遮蔽:只改写这次组件数据中的自由文本——条目本身与任何嵌套 attempt 条目的
|
|
366
|
-
* `failureSummary`;身份与分类字段(experimentId、evalId、locator、数值指标)不经它。
|
|
367
|
-
* 只作用于这次计算产出的组件数据,不改盘上或任何导出目录里的 artifact;
|
|
368
|
-
* 发布 artifact 的脱敏用 copySnapshots({ redact })。
|
|
369
|
-
*/
|
|
370
|
-
redact?: (text: string) => string;
|
|
371
|
-
}
|
package/dist/results/types.d.ts
CHANGED
|
@@ -33,13 +33,6 @@ export interface SnapshotMeta {
|
|
|
33
33
|
completedAt?: string;
|
|
34
34
|
/** 写入时刻该实验已知的 eval 并集 —— 残缺检测的分母随数据走(copySnapshots 自动补记,writer 可声明)。 */
|
|
35
35
|
knownEvalIds?: string[];
|
|
36
|
-
/**
|
|
37
|
-
* 发布拷贝的自描述标记:copySnapshots 补记,消毒函数 → "applied"、redact: false → "none";
|
|
38
|
-
* 本地事实根没有此字段。只声明流程,不证明无秘密;view --out 据此分级防呆。
|
|
39
|
-
*/
|
|
40
|
-
publish?: {
|
|
41
|
-
redaction: "applied" | "none";
|
|
42
|
-
};
|
|
43
36
|
/** 项目名(来自 config.name),透传给 `niceeval view` 顶部 hero 显示。 */
|
|
44
37
|
name?: LocalizedText;
|
|
45
38
|
}
|
|
@@ -117,10 +110,6 @@ export interface Snapshot {
|
|
|
117
110
|
dir: string;
|
|
118
111
|
/** 写入时刻该实验已知的 eval 并集(可选);copySnapshots 自动补记,writer.snapshot() 也可声明。 */
|
|
119
112
|
knownEvalIds?: string[];
|
|
120
|
-
/** 发布拷贝的自描述标记(见 SnapshotMeta.publish);本地事实根没有此字段。 */
|
|
121
|
-
publish?: {
|
|
122
|
-
redaction: "applied" | "none";
|
|
123
|
-
};
|
|
124
113
|
}
|
|
125
114
|
/** 一个实验的全部历史:同一 experiment id 的历次快照归在一起。 */
|
|
126
115
|
export interface Experiment {
|
|
@@ -20,12 +20,9 @@ const results = await openResults(".niceeval");
|
|
|
20
20
|
await rm(output, { recursive: true, force: true });
|
|
21
21
|
await copySnapshots(results.latest(), output, {
|
|
22
22
|
artifacts: ["sources", "events", "trace", "o11y", "agentSetup"],
|
|
23
|
-
redact: (text) => text.replaceAll(/sk-[A-Za-z0-9]+/g, "[redacted]"),
|
|
24
23
|
});
|
|
25
24
|
```
|
|
26
25
|
|
|
27
|
-
`redact` 是必填项:给一个函数改写待发布文件里的自由文本,或者确认这批数据可以原文公开、显式传 `redact: false`。要发布的站点谁都能翻到 prompt 和工具输出,这个选择必须写在脚本里。
|
|
28
|
-
|
|
29
26
|
运行 `npx tsx scripts/publish-results.ts`,然后提交 `report-data/`。`copySnapshots` 在创建目录前检查所有待发布文件;任何文件超过 50 MiB 时整体失败并列出路径、大小和处理建议,不会留下半份目录。`diff` 缺省不发布;需要 diff 时显式加进 `artifacts`,它也受同一个预算约束。历史版本留下的超大 events / trace 不会被悄悄改写,预检会要求你排除这类证据或用当前版本重跑。
|
|
30
27
|
|
|
31
28
|
## 构建命令就是导出命令
|
|
@@ -139,4 +136,4 @@ jobs:
|
|
|
139
136
|
|
|
140
137
|
## 发布的是选中的证据
|
|
141
138
|
|
|
142
|
-
整站导出会带上发布结果根里选中的 transcript、源码快照和 trace。新写入的超大 events / trace 字符串可能带结构化截断标记。`copySnapshots`
|
|
139
|
+
整站导出会带上发布结果根里选中的 transcript、源码快照和 trace。新写入的超大 events / trace 字符串可能带结构化截断标记。`copySnapshots` 只做选择和整文件大小预检,不改写任何内容。发布的站点谁都能翻到 prompt 和工具输出,发布到公网前确认结果内容适合公开——NiceEval 在记录时就不把环境变量值和命令输出写进结果文件,transcript 里是 Agent 自己的输入输出和你的 eval 任务本身。
|
|
@@ -88,7 +88,6 @@ npx niceeval exp compare-models weather
|
|
|
88
88
|
| `--json` | string | 额外写一份 JSON 结果(`RunSummary` 原样序列化)到指定路径,供 CI 或下游脚本消费。 |
|
|
89
89
|
| `--out` | string | `view` 命令专用:把结果查看器静态导出到指定目录。 |
|
|
90
90
|
| `--port` | number | `view` 命令专用:指定本地服务器监听端口。 |
|
|
91
|
-
| `--allow-sensitive-artifacts` | boolean | `view --out` 专用:对非发布根(快照没有 publish:{redaction:"applied"} 标记)导出时的显式确认——静态站会原样携带未消毒的证据文件。 |
|
|
92
91
|
| `--source` | boolean | `show` 命令专用:该 attempt 运行时保存的 Eval 源码,gate/soft 断言标回源码行(证据切面)。 |
|
|
93
92
|
| `--execution` | boolean | `show` 命令专用:该 attempt 的标准执行事件流(消息、thinking、Skill load、工具调用/结果);有 OTel 时同一节点补时间(证据切面)。 |
|
|
94
93
|
| `--timing` | boolean | `show` 命令专用:整个 Attempt 的统一时间树;裸 `--timing` 给有界诊断投影,`--timing=full` 逐节点展开全部 runner/已关联 OTel 节点。 |
|
|
@@ -211,7 +211,7 @@ inspect: niceeval show @<id> [--eval|--execution|--diff]
|
|
|
211
211
|
每项固定代表一个 Attempt,显示 experiment、Eval、Attempt 序号、判定、耗时、成本、失败断言、结构化 error 的一层摘要、Judge 评语和证据链接。diagnostics、cause 和 stack 留给 locator 下钻详情,避免比较列表被基础设施日志撑开。它既能列失败证据,也能列通过样本,不把 verdict 过滤写死在组件名里。
|
|
212
212
|
|
|
213
213
|
```tsx
|
|
214
|
-
const attempts = await AttemptList.data(selection
|
|
214
|
+
const attempts = await AttemptList.data(selection);
|
|
215
215
|
|
|
216
216
|
<AttemptList
|
|
217
217
|
items={attempts.filter((item) => item.verdict === "failed" || item.verdict === "errored")}
|
|
@@ -234,7 +234,7 @@ inspect: niceeval show @<id> [--eval|--execution|--diff]
|
|
|
234
234
|
(3 more not shown · showing 20 of 23)
|
|
235
235
|
```
|
|
236
236
|
|
|
237
|
-
|
|
237
|
+
要展示哪些 Attempt,过滤返回的 `AttemptListItem[]`。`limit` 也由报告作者在数组上用 `.slice(0, 20)` 表达,截断时把原始数量交给组件的 `total`,组件据此显示“还有 n 项未展示”,不静默截断。
|
|
238
238
|
|
|
239
239
|
## 指标表(`MetricTable`)
|
|
240
240
|
|
|
@@ -235,9 +235,7 @@ import { openResults, copySnapshots } from "niceeval/results";
|
|
|
235
235
|
const results = await openResults(".niceeval");
|
|
236
236
|
await copySnapshots(results.latest(), "site-data/run", {
|
|
237
237
|
artifacts: ["sources", "events", "trace", "o11y"], // diff 不截断,缺省也不带;
|
|
238
|
-
|
|
239
|
-
}); // redact 必填:函数消毒,或 false 显式声明原文发布;
|
|
240
|
-
// 每个待发布文件还会经过 50 MiB 预检;
|
|
238
|
+
}); // 每个待发布文件还会经过 50 MiB 预检;
|
|
241
239
|
// o11y 只有几 KB,报告用到 turns 这类
|
|
242
240
|
// 读 o11y 的指标就把它带上,不然渲染成「—」
|
|
243
241
|
```
|
|
@@ -246,7 +244,7 @@ await copySnapshots(results.latest(), "site-data/run", {
|
|
|
246
244
|
|
|
247
245
|
复制开始前,NiceEval 会规划全部目标文件并检查序列化后的大小。任一文件超过固定的 50 MiB,整次复制在创建目标目录前失败,错误会列出路径、实际大小和处理建议。你可以从 `artifacts` 排除那类证据;如果是旧版本留下的超大 events / trace,用当前版本重跑后再发布。这个检查既覆盖没有逐值截断的源码 / diff,也覆盖单值都正常但累计过大的 JSON,避免直到 `git push` 才撞上 Git host 的单文件限制。
|
|
248
246
|
|
|
249
|
-
|
|
247
|
+
大小预检只决定整次复制成功或失败,不会从一个超大文件中间删内容。复制忠实于源:artifact 按原字节复制,不重新序列化、不改写。唯一随行补记的是挑选时的**覆盖事实**:`partial-coverage` 警告的分母是实验的历史并集,而发布目录没有历史——所以每个复制出的快照带上 `knownEvalIds`(复制时刻该实验已知的 eval 并集),reader 端把它并进 `exp.evalIds` 的计算(取本地历史与快照携带值的并集)。发布目录上重新 `openResults().latest()`,残缺警告被同一套机制重新算出来,不靠发布者转述。复制出的目录就是标准结果目录,`niceeval view --results <目录>` 直接能看;要让报告站随 push 自动更新,workflow 见[通过 CI 发布报告](/zh/how-to/publish-report)。
|
|
250
248
|
|
|
251
249
|
## 分层速览
|
|
252
250
|
|
package/package.json
CHANGED
package/src/cli.ts
CHANGED
|
@@ -105,7 +105,6 @@ interface Flags {
|
|
|
105
105
|
timing?: "summary" | "full";
|
|
106
106
|
keepSandbox?: "failed" | "all";
|
|
107
107
|
all: boolean;
|
|
108
|
-
allowSensitiveArtifacts: boolean;
|
|
109
108
|
window?: string;
|
|
110
109
|
sandboxPath?: string;
|
|
111
110
|
leaveRunning: boolean;
|
|
@@ -156,8 +155,6 @@ const FLAG_OPTIONS = {
|
|
|
156
155
|
out: { type: "string" },
|
|
157
156
|
/** `view` 命令专用:指定本地服务器监听端口。 */
|
|
158
157
|
port: { type: "string" },
|
|
159
|
-
/** `view --out` 专用:对非发布根(快照没有 publish:{redaction:"applied"} 标记)导出时的显式确认——静态站会原样携带未消毒的证据文件。 */
|
|
160
|
-
"allow-sensitive-artifacts": { type: "boolean" },
|
|
161
158
|
// show 的证据切面 / 时间轴 / 报告装载(docs-site/zh/how-to/viewing-results.mdx)。
|
|
162
159
|
// 证据切面只认 `@<locator>`(或收窄到单个 eval 的前缀)选出的那一个 attempt——不再有
|
|
163
160
|
// 数字 `--attempt`,选哪个 attempt 由 locator 精确指名,不是「先选 eval 再挑第几次」。
|
|
@@ -307,7 +304,6 @@ function parseArgs(argv: string[]): { command: string; positionals: string[]; fl
|
|
|
307
304
|
timing: values.timing === true ? (timingMode ?? "summary") : undefined,
|
|
308
305
|
keepSandbox: values["keep-sandbox"] === true ? (keepSandboxTier ?? "failed") : undefined,
|
|
309
306
|
all: values.all === true,
|
|
310
|
-
allowSensitiveArtifacts: values["allow-sensitive-artifacts"] === true,
|
|
311
307
|
window: values.window as string | undefined,
|
|
312
308
|
sandboxPath: values.path as string | undefined,
|
|
313
309
|
leaveRunning: values["leave-running"] === true,
|
|
@@ -591,7 +587,7 @@ async function main(): Promise<void> {
|
|
|
591
587
|
...(flags.page !== undefined ? { page: flags.page } : {}),
|
|
592
588
|
};
|
|
593
589
|
if (flags.out) {
|
|
594
|
-
const out = await buildView({ input: viewInput.input, out: flags.out,
|
|
590
|
+
const out = await buildView({ input: viewInput.input, out: flags.out, scan }).catch(exitOnViewUserError);
|
|
595
591
|
process.stdout.write(t("cli.view.exportedDir", { out }));
|
|
596
592
|
process.exit(0);
|
|
597
593
|
}
|
package/src/context/context.ts
CHANGED
|
@@ -165,7 +165,7 @@ export function createEvalContext(deps: ContextDeps): { context: TestContext; st
|
|
|
165
165
|
return brief(value, 4000);
|
|
166
166
|
}
|
|
167
167
|
|
|
168
|
-
/**
|
|
168
|
+
/** evidence 不区分 pass/fail(与 judge 同口径):被检查值自带命令摘要(CommandResult.command)时就是「命令行本身」。 */
|
|
169
169
|
function checkedValueEvidence(value: unknown): string | undefined {
|
|
170
170
|
const command = asCommandResult(value)?.command;
|
|
171
171
|
return typeof command === "string" && command.length > 0 ? command : undefined;
|
|
@@ -378,12 +378,15 @@ export function createEvalContext(deps: ContextDeps): { context: TestContext; st
|
|
|
378
378
|
evaluate: async (sc) => {
|
|
379
379
|
const resolved = await resolveValue(value, sc);
|
|
380
380
|
const score = await assertion.score(resolved);
|
|
381
|
-
|
|
381
|
+
const evidence = checkedValueEvidence(resolved);
|
|
382
|
+
if (computePassed(spec.severity, spec.threshold, score)) {
|
|
383
|
+
return evidence !== undefined ? { score, evidence } : score;
|
|
384
|
+
}
|
|
382
385
|
return {
|
|
383
386
|
score,
|
|
384
387
|
expected: assertion.expected,
|
|
385
388
|
received: previewCheckedValue(resolved),
|
|
386
|
-
...(
|
|
389
|
+
...(evidence !== undefined ? { evidence } : {}),
|
|
387
390
|
};
|
|
388
391
|
},
|
|
389
392
|
};
|
|
@@ -395,18 +398,21 @@ export function createEvalContext(deps: ContextDeps): { context: TestContext; st
|
|
|
395
398
|
const score = await assertion.score(v);
|
|
396
399
|
// require 恒为硬门槛(不过即中止 eval),判定口径与 finalize 同一份 computePassed。
|
|
397
400
|
const passed = computePassed("gate", assertion.threshold, score);
|
|
401
|
+
const evidence = checkedValueEvidence(v);
|
|
398
402
|
collector.record({
|
|
399
403
|
name: assertion.name,
|
|
400
404
|
severity: "gate",
|
|
401
405
|
threshold: assertion.threshold,
|
|
402
406
|
evaluate: () =>
|
|
403
407
|
passed
|
|
404
|
-
?
|
|
408
|
+
? evidence !== undefined
|
|
409
|
+
? { score, evidence }
|
|
410
|
+
: score
|
|
405
411
|
: {
|
|
406
412
|
score,
|
|
407
413
|
expected: assertion.expected,
|
|
408
414
|
received: previewCheckedValue(v),
|
|
409
|
-
...(
|
|
415
|
+
...(evidence !== undefined ? { evidence } : {}),
|
|
410
416
|
},
|
|
411
417
|
});
|
|
412
418
|
if (!passed) throw new EvalRequirementFailed(assertion.name);
|
|
@@ -26,7 +26,6 @@ import type { AttemptLocator } from "../results/locator.ts";
|
|
|
26
26
|
import type {
|
|
27
27
|
AttemptListItem,
|
|
28
28
|
DeltaData,
|
|
29
|
-
EntityListDataOptions,
|
|
30
29
|
EvalListItem,
|
|
31
30
|
ExperimentComparisonData,
|
|
32
31
|
ExperimentListItem,
|
|
@@ -400,7 +399,7 @@ interface EntityListChrome extends ChromeProps {
|
|
|
400
399
|
|
|
401
400
|
export type ExperimentListProps = DataProps<
|
|
402
401
|
readonly ExperimentListItem[],
|
|
403
|
-
|
|
402
|
+
Record<never, never>,
|
|
404
403
|
EntityListChrome & {
|
|
405
404
|
/** web 面在比较表前显示实验过滤框;text 面忽略。 */
|
|
406
405
|
filter?: boolean;
|
|
@@ -415,14 +414,14 @@ export type ExperimentListProps = DataProps<
|
|
|
415
414
|
/** 实验列表:每项一个 experiment,固定八列比较表 + 展开到 Eval / Attempt。 */
|
|
416
415
|
export const ExperimentList = makeDataComponent<
|
|
417
416
|
readonly ExperimentListItem[],
|
|
418
|
-
|
|
417
|
+
Record<never, never>,
|
|
419
418
|
EntityListChrome & { filter?: boolean; relativeTo?: string }
|
|
420
419
|
>({
|
|
421
420
|
name: "ExperimentList",
|
|
422
421
|
dataFnName: "experimentListData",
|
|
423
422
|
shapeName: "ExperimentListItem[]",
|
|
424
|
-
dataFn: (input
|
|
425
|
-
specKeys: [
|
|
423
|
+
dataFn: (input) => experimentListData(input),
|
|
424
|
+
specKeys: [],
|
|
426
425
|
validate: validateExperimentListData,
|
|
427
426
|
web: (props, ctx) => (
|
|
428
427
|
<ExperimentListWeb
|
|
@@ -437,15 +436,15 @@ export const ExperimentList = makeDataComponent<
|
|
|
437
436
|
text: (props, ctx) => experimentListText(props.data, ctx, props.relativeTo),
|
|
438
437
|
}) as unknown as ReportComponent<ExperimentListProps>;
|
|
439
438
|
|
|
440
|
-
export type EvalListProps = DataProps<readonly EvalListItem[],
|
|
439
|
+
export type EvalListProps = DataProps<readonly EvalListItem[], Record<never, never>, EntityListChrome>;
|
|
441
440
|
|
|
442
441
|
/** Eval 列表:每项一个 experimentId + evalId,展开到这道题的 Attempt。 */
|
|
443
|
-
export const EvalList = makeDataComponent<readonly EvalListItem[],
|
|
442
|
+
export const EvalList = makeDataComponent<readonly EvalListItem[], Record<never, never>, EntityListChrome>({
|
|
444
443
|
name: "EvalList",
|
|
445
444
|
dataFnName: "evalListData",
|
|
446
445
|
shapeName: "EvalListItem[]",
|
|
447
|
-
dataFn: (input
|
|
448
|
-
specKeys: [
|
|
446
|
+
dataFn: (input) => evalListData(input),
|
|
447
|
+
specKeys: [],
|
|
449
448
|
validate: validateEvalListData,
|
|
450
449
|
web: (props, ctx) => (
|
|
451
450
|
<EvalListWeb
|
|
@@ -460,7 +459,7 @@ export const EvalList = makeDataComponent<readonly EvalListItem[], EntityListDat
|
|
|
460
459
|
|
|
461
460
|
export type AttemptListProps = DataProps<
|
|
462
461
|
readonly AttemptListItem[],
|
|
463
|
-
|
|
462
|
+
Record<never, never>,
|
|
464
463
|
EntityListChrome & {
|
|
465
464
|
/** 过滤 / 截断前的总数;省略时等于 data 长度。 */
|
|
466
465
|
total?: number;
|
|
@@ -470,14 +469,14 @@ export type AttemptListProps = DataProps<
|
|
|
470
469
|
/** Attempt 列表:实体列表的叶子层,每项一次 attempt 的判定、单行摘要与 locator。 */
|
|
471
470
|
export const AttemptList = makeDataComponent<
|
|
472
471
|
readonly AttemptListItem[],
|
|
473
|
-
|
|
472
|
+
Record<never, never>,
|
|
474
473
|
EntityListChrome & { total?: number }
|
|
475
474
|
>({
|
|
476
475
|
name: "AttemptList",
|
|
477
476
|
dataFnName: "attemptListData",
|
|
478
477
|
shapeName: "AttemptListItem[]",
|
|
479
|
-
dataFn: (input
|
|
480
|
-
specKeys: [
|
|
478
|
+
dataFn: (input) => attemptListData(input),
|
|
479
|
+
specKeys: [],
|
|
481
480
|
validate: validateAttemptListData,
|
|
482
481
|
web: (props, ctx) => (
|
|
483
482
|
<AttemptListWeb
|
|
@@ -498,7 +497,6 @@ export interface FailureListProps {
|
|
|
498
497
|
limit?: number;
|
|
499
498
|
/** 默认宿主注入的 Scope。 */
|
|
500
499
|
input?: ReportInput;
|
|
501
|
-
redact?: (text: string) => string;
|
|
502
500
|
attemptHref?: (locator: AttemptLocator) => string;
|
|
503
501
|
locale?: ReportLocale;
|
|
504
502
|
className?: string;
|
|
@@ -512,7 +510,7 @@ export interface FailureListProps {
|
|
|
512
510
|
*/
|
|
513
511
|
export const FailureList = defineComponent<FailureListProps>(async (props, ctx) => {
|
|
514
512
|
const input = props.input ?? ctx.scope;
|
|
515
|
-
const all = await attemptListData(input
|
|
513
|
+
const all = await attemptListData(input);
|
|
516
514
|
// attempt 开始时间不在列表条目里(它不是列表展示字段);从同一 input 的读取面按 locator 对回。
|
|
517
515
|
const startedAtByLocator = new Map<string, string>();
|
|
518
516
|
for (const item of collectItems(resolveInput(input).snapshots)) {
|
package/src/report/compute.ts
CHANGED
|
@@ -17,7 +17,6 @@ import type {
|
|
|
17
17
|
DeltaData,
|
|
18
18
|
DeltaPair,
|
|
19
19
|
DimensionInput,
|
|
20
|
-
EntityListDataOptions,
|
|
21
20
|
EvalListItem,
|
|
22
21
|
ExperimentComparisonData,
|
|
23
22
|
ExperimentComparisonGroupData,
|
|
@@ -199,10 +198,8 @@ function failureSummaryOf(result: EvalResult): { summary: string | null; more: n
|
|
|
199
198
|
return { summary: null, more: 0 };
|
|
200
199
|
}
|
|
201
200
|
|
|
202
|
-
const identityRedact = (text: string): string => text;
|
|
203
|
-
|
|
204
201
|
/** AttemptList / ExperimentList / EvalList 共用的叶子构造:一个 Item → 一个 AttemptListItem。 */
|
|
205
|
-
async function attemptListItemOf(item: Item
|
|
202
|
+
async function attemptListItemOf(item: Item): Promise<AttemptListItem> {
|
|
206
203
|
const result = item.attempt.result;
|
|
207
204
|
const { summary, more } = failureSummaryOf(result);
|
|
208
205
|
return {
|
|
@@ -211,7 +208,7 @@ async function attemptListItemOf(item: Item, redact: (text: string) => string):
|
|
|
211
208
|
attempt: result.attempt,
|
|
212
209
|
agent: result.agent,
|
|
213
210
|
verdict: result.verdict,
|
|
214
|
-
failureSummary: summary
|
|
211
|
+
failureSummary: summary,
|
|
215
212
|
moreFailures: more,
|
|
216
213
|
examScore: await computeCell(examScore, [item]),
|
|
217
214
|
durationMs: result.durationMs,
|
|
@@ -221,20 +218,15 @@ async function attemptListItemOf(item: Item, redact: (text: string) => string):
|
|
|
221
218
|
}
|
|
222
219
|
|
|
223
220
|
/** `attemptListData(input)`:每个 Attempt 一项,顺序取自 Scope 展平顺序(不重排)。 */
|
|
224
|
-
export async function attemptListData(
|
|
225
|
-
input: ReportInput,
|
|
226
|
-
options?: EntityListDataOptions,
|
|
227
|
-
): Promise<AttemptListItem[]> {
|
|
221
|
+
export async function attemptListData(input: ReportInput): Promise<AttemptListItem[]> {
|
|
228
222
|
const { snapshots } = resolveInput(input);
|
|
229
|
-
const redact = options?.redact ?? identityRedact;
|
|
230
223
|
const items = collectItems(snapshots);
|
|
231
|
-
return Promise.all(items.map((item) => attemptListItemOf(item
|
|
224
|
+
return Promise.all(items.map((item) => attemptListItemOf(item)));
|
|
232
225
|
}
|
|
233
226
|
|
|
234
227
|
/** `evalListData(input)`:每个 `experimentId + evalId` 一项,按 evalId 再按 experimentId 升序。 */
|
|
235
|
-
export async function evalListData(input: ReportInput
|
|
228
|
+
export async function evalListData(input: ReportInput): Promise<EvalListItem[]> {
|
|
236
229
|
const { snapshots } = resolveInput(input);
|
|
237
|
-
const redact = options?.redact ?? identityRedact;
|
|
238
230
|
const items = collectItems(snapshots);
|
|
239
231
|
const groups = new Map<string, Item[]>();
|
|
240
232
|
for (const item of items) {
|
|
@@ -247,7 +239,7 @@ export async function evalListData(input: ReportInput, options?: EntityListDataO
|
|
|
247
239
|
for (const group of groups.values()) {
|
|
248
240
|
const sorted = [...group].sort((a, b) => a.attempt.result.attempt - b.attempt.result.attempt);
|
|
249
241
|
const verdict = foldEvalVerdict(sorted.map((item) => item.attempt.result));
|
|
250
|
-
const attempts = await Promise.all(sorted.map((item) => attemptListItemOf(item
|
|
242
|
+
const attempts = await Promise.all(sorted.map((item) => attemptListItemOf(item)));
|
|
251
243
|
out.push({
|
|
252
244
|
experimentId: experimentIdOf(sorted[0]!),
|
|
253
245
|
evalId: evalIdOf(sorted[0]!),
|
|
@@ -269,12 +261,8 @@ export async function evalListData(input: ReportInput, options?: EntityListDataO
|
|
|
269
261
|
* Snapshot[] 时若同一 experiment 混入不一致的可比性配置,按完整用户反馈失败并指引——
|
|
270
262
|
* 看跨配置演化用 snapshot 维度或 MetricLine,不把两套配置拼成一行冒充单一配置。
|
|
271
263
|
*/
|
|
272
|
-
export async function experimentListData(
|
|
273
|
-
input: ReportInput,
|
|
274
|
-
options?: EntityListDataOptions,
|
|
275
|
-
): Promise<ExperimentListItem[]> {
|
|
264
|
+
export async function experimentListData(input: ReportInput): Promise<ExperimentListItem[]> {
|
|
276
265
|
const { snapshots } = resolveInput(input);
|
|
277
|
-
const redact = options?.redact ?? identityRedact;
|
|
278
266
|
|
|
279
267
|
// 可比性配置单义检查:同一 experiment 的输入快照必须共享一套可比性配置。
|
|
280
268
|
const configByExperiment = new Map<string, { snapshot: Snapshot; config: unknown }>();
|
|
@@ -304,7 +292,7 @@ export async function experimentListData(
|
|
|
304
292
|
for (const [evalId, evalItems] of evalGroups) {
|
|
305
293
|
const sorted = [...evalItems].sort((a, b) => a.attempt.result.attempt - b.attempt.result.attempt);
|
|
306
294
|
const verdict = foldEvalVerdict(sorted.map((item) => item.attempt.result));
|
|
307
|
-
const attempts = await Promise.all(sorted.map((item) => attemptListItemOf(item
|
|
295
|
+
const attempts = await Promise.all(sorted.map((item) => attemptListItemOf(item)));
|
|
308
296
|
evalRows.push({
|
|
309
297
|
evalId,
|
|
310
298
|
verdict,
|
package/src/report/index.ts
CHANGED
|
@@ -3,8 +3,8 @@
|
|
|
3
3
|
// niceeval/results 的读取契约手工构造)。覆盖登记行:两级聚合 vs 平铺、errored=0 口径、
|
|
4
4
|
// skipped=null、null≠0、Scoreboard 固定分母(notRun/unscorable 分开)、权重最长前缀、
|
|
5
5
|
// 身份键去重、现刻水位、自定义指标 where/aggregate、evalGroup 完整父路径、verdict 权威、
|
|
6
|
-
// MetricCell 诚实、缺 artifact 指标、repeatedFailedCommands、实体列表 failureSummary
|
|
7
|
-
//
|
|
6
|
+
// MetricCell 诚实、缺 artifact 指标、repeatedFailedCommands、实体列表 failureSummary、
|
|
7
|
+
// scopeSummaryData 两级计票、experimentComparisonData 分区、pairsByFlag、
|
|
8
8
|
// MetricLine 点身份、空数组反馈、metricTableData sort。
|
|
9
9
|
|
|
10
10
|
import { describe, expect, it } from "vitest";
|
|
@@ -595,24 +595,6 @@ describe("实体列表 data", () => {
|
|
|
595
595
|
expect(byEval.get("list/errored")!.costUSD).toBeNull();
|
|
596
596
|
});
|
|
597
597
|
|
|
598
|
-
it("redact 只改写 failureSummary(含嵌套 attempt 条目);身份字段、locator 与数值指标原样", async () => {
|
|
599
|
-
const redact = (text: string) => text.replaceAll("41", "[redacted]");
|
|
600
|
-
const attempts = await attemptListData([listSnap()], { redact });
|
|
601
|
-
const failedItem = attempts.find((item) => item.evalId === "list/failed")!;
|
|
602
|
-
expect(failedItem.failureSummary).toContain("[redacted]");
|
|
603
|
-
expect(failedItem.evalId).toBe("list/failed");
|
|
604
|
-
expect(failedItem.experimentId).toBe("exp/list");
|
|
605
|
-
expect(failedItem.costUSD).toBe(0.1);
|
|
606
|
-
|
|
607
|
-
const evals = await evalListData([listSnap()], { redact });
|
|
608
|
-
const nested = evals.find((item) => item.evalId === "list/failed")!.attempts[0]!;
|
|
609
|
-
expect(nested.failureSummary).toContain("[redacted]");
|
|
610
|
-
|
|
611
|
-
const experiments = await experimentListData([listSnap()], { redact });
|
|
612
|
-
const nestedInExp = experiments[0]!.evalRows.find((row) => row.evalId === "list/failed")!.attempts[0]!;
|
|
613
|
-
expect(nestedInExp.failureSummary).toContain("[redacted]");
|
|
614
|
-
});
|
|
615
|
-
|
|
616
598
|
it("experimentListData:evalVerdicts / endToEndPassRate / costUSD / durationMs / tokens 齐全,默认按端到端成功率降序", async () => {
|
|
617
599
|
const winner = snap({ experimentId: "exp/win", results: [res("a", "passed"), res("b", "passed")] });
|
|
618
600
|
const loser = snap({ experimentId: "exp/lose", results: [res("a", "failed"), res("b", "passed")] });
|
package/src/report/types.ts
CHANGED
|
@@ -394,14 +394,3 @@ export interface ExperimentListItem {
|
|
|
394
394
|
lastRunAt: string;
|
|
395
395
|
evalRows: ExperimentListEvalRow[];
|
|
396
396
|
}
|
|
397
|
-
|
|
398
|
-
/** 三个实体列表共用的计算选项。 */
|
|
399
|
-
export interface EntityListDataOptions {
|
|
400
|
-
/**
|
|
401
|
-
* 展示层遮蔽:只改写这次组件数据中的自由文本——条目本身与任何嵌套 attempt 条目的
|
|
402
|
-
* `failureSummary`;身份与分类字段(experimentId、evalId、locator、数值指标)不经它。
|
|
403
|
-
* 只作用于这次计算产出的组件数据,不改盘上或任何导出目录里的 artifact;
|
|
404
|
-
* 发布 artifact 的脱敏用 copySnapshots({ redact })。
|
|
405
|
-
*/
|
|
406
|
-
redact?: (text: string) => string;
|
|
407
|
-
}
|
package/src/results/copy.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// copySnapshots:把选中快照按格式感知地复制到另一个目录(定稿见 docs/feature/results/library.md「复制与瘦身」)。
|
|
2
2
|
//
|
|
3
3
|
// 发布场景的原语:只带指定 artifact、只带选中快照的全部 attempt,布局知识不外泄。
|
|
4
|
-
// artifact 复制忠实于源(
|
|
4
|
+
// artifact 复制忠实于源(原字节,不重新序列化、不改写);snapshot.json / result.json
|
|
5
5
|
// 按选中条目重建,版本元数据保留。产物是一个标准结果根目录(同布局),openResults /
|
|
6
6
|
// `niceeval view` 直接能读。唯一随行补记的是挑选时的覆盖事实:每个复制出的快照带上
|
|
7
7
|
// knownEvalIds(复制时刻该实验已知的 eval 并集),发布目录上重新 openResults().latest(),
|
|
@@ -9,21 +9,13 @@
|
|
|
9
9
|
|
|
10
10
|
import { mkdir, readdir, readFile, stat, writeFile } from "node:fs/promises";
|
|
11
11
|
import { basename, dirname, join, resolve } from "node:path";
|
|
12
|
-
import type { EvalResult
|
|
12
|
+
import type { EvalResult } from "../types.ts";
|
|
13
13
|
import { RESULTS_FORMAT } from "../types.ts";
|
|
14
14
|
import { RESULT_FILE, SNAPSHOT_FILE, artifactFileOf, experimentDirOf } from "./format.ts";
|
|
15
15
|
import { experimentOfSnapshot } from "./open.ts";
|
|
16
16
|
import { isNewerSnapshot } from "./select.ts";
|
|
17
17
|
import { hashEvalSource, normalizeEvalSource } from "./source-hash.ts";
|
|
18
|
-
import {
|
|
19
|
-
PUBLISH_FILE_MAX_BYTES,
|
|
20
|
-
redactEvents,
|
|
21
|
-
redactExperimentInfo,
|
|
22
|
-
redactJsonValue,
|
|
23
|
-
redactResultRecord,
|
|
24
|
-
redactSpans,
|
|
25
|
-
type Redactor,
|
|
26
|
-
} from "./publish.ts";
|
|
18
|
+
import { PUBLISH_FILE_MAX_BYTES } from "./publish.ts";
|
|
27
19
|
import type { ArtifactKind, AttemptHandle, Scope, Snapshot, SnapshotMeta } from "./types.ts";
|
|
28
20
|
|
|
29
21
|
/** 缺省携带的 artifact:events / trace / o11y / agentSetup / sources;diff 不截断、可达百 MB,缺省不带。 */
|
|
@@ -33,12 +25,6 @@ const VALID_ARTIFACTS: ArtifactKind[] = ["events", "trace", "o11y", "agentSetup"
|
|
|
33
25
|
export interface CopySnapshotsOptions {
|
|
34
26
|
/** 要带上的 artifact 种类;缺省带 events / trace / o11y / agentSetup / sources,不带 diff。 */
|
|
35
27
|
artifacts?: ArtifactKind[];
|
|
36
|
-
/**
|
|
37
|
-
* 发布消毒(必填,没有隐式默认):传函数逐值消毒自由文本字段,或显式传 `false` 声明
|
|
38
|
-
* 「这批数据可以原文发布」——「不消毒」必须是写在代码里的选择,不是忘了传参数的副作用。
|
|
39
|
-
* 结构字段(格式、判定、身份、路径、哈希)永不经过 redactor。
|
|
40
|
-
*/
|
|
41
|
-
redact: Redactor | false;
|
|
42
28
|
}
|
|
43
29
|
|
|
44
30
|
export interface CopySnapshotsResult {
|
|
@@ -56,7 +42,7 @@ export interface CopySnapshotsResult {
|
|
|
56
42
|
export async function copySnapshots(
|
|
57
43
|
scope: Scope | readonly Snapshot[],
|
|
58
44
|
destDir: string,
|
|
59
|
-
opts: CopySnapshotsOptions,
|
|
45
|
+
opts: CopySnapshotsOptions = {},
|
|
60
46
|
): Promise<CopySnapshotsResult> {
|
|
61
47
|
const selected = Array.isArray(scope) ? (scope as readonly Snapshot[]) : (scope as Scope).snapshots;
|
|
62
48
|
if (selected.length === 0) {
|
|
@@ -64,12 +50,6 @@ export async function copySnapshots(
|
|
|
64
50
|
"copySnapshots got no snapshots to copy. Check the experiments filter, or pass snapshots from openResults().latest().",
|
|
65
51
|
);
|
|
66
52
|
}
|
|
67
|
-
if (opts?.redact === undefined) {
|
|
68
|
-
throw new Error(
|
|
69
|
-
'copySnapshots requires an explicit "redact" option: pass a (text) => string sanitizer, or the literal false to declare this data safe to publish verbatim.',
|
|
70
|
-
);
|
|
71
|
-
}
|
|
72
|
-
const redactor: Redactor | undefined = opts.redact === false ? undefined : opts.redact;
|
|
73
53
|
const kinds = opts.artifacts ?? [...DEFAULT_PUBLISH_ARTIFACTS];
|
|
74
54
|
for (const kind of kinds) {
|
|
75
55
|
if (!VALID_ARTIFACTS.includes(kind)) {
|
|
@@ -97,11 +77,11 @@ export async function copySnapshots(
|
|
|
97
77
|
if (isNewerSnapshot(snapshot, existing)) byExperiment.set(snapshot.experimentId, snapshot);
|
|
98
78
|
}
|
|
99
79
|
|
|
100
|
-
//
|
|
101
|
-
//
|
|
80
|
+
// 发布前整文件预检:先规划并序列化全部目标文件,任一文件超过 PUBLISH_FILE_MAX_BYTES
|
|
81
|
+
// 就整体失败,不留半成品目标目录。
|
|
102
82
|
const planned: PlannedFile[] = [];
|
|
103
83
|
for (const snapshot of byExperiment.values()) {
|
|
104
|
-
planned.push(...(await planOneSnapshot(snapshot, [...selected], dest, kinds
|
|
84
|
+
planned.push(...(await planOneSnapshot(snapshot, [...selected], dest, kinds)));
|
|
105
85
|
}
|
|
106
86
|
const oversized = planned.filter((f) => f.bytes.byteLength > PUBLISH_FILE_MAX_BYTES);
|
|
107
87
|
if (oversized.length > 0) {
|
|
@@ -135,7 +115,6 @@ async function planOneSnapshot(
|
|
|
135
115
|
selected: Snapshot[],
|
|
136
116
|
destRoot: string,
|
|
137
117
|
kinds: ArtifactKind[],
|
|
138
|
-
redactor: Redactor | undefined,
|
|
139
118
|
): Promise<PlannedFile[]> {
|
|
140
119
|
const destSnapDir = join(destRoot, experimentDirOf(snapshot.experimentId), basename(snapshot.dir));
|
|
141
120
|
const planned: PlannedFile[] = [];
|
|
@@ -144,31 +123,22 @@ async function planOneSnapshot(
|
|
|
144
123
|
// 复制到目的地也只应该有一份——这个 Set 记录本快照已经规划过的 hash,整快照的 attempt 共享。
|
|
145
124
|
const plannedSourceHashes = new Set<string>();
|
|
146
125
|
for (const attempt of snapshot.attempts) {
|
|
147
|
-
planned.push(...(await planOneAttempt(attempt, destSnapDir, kinds, plannedSourceHashes
|
|
126
|
+
planned.push(...(await planOneAttempt(attempt, destSnapDir, kinds, plannedSourceHashes)));
|
|
148
127
|
}
|
|
149
128
|
|
|
150
129
|
const knownEvalIds = experimentOfSnapshot(snapshot)?.evalIds ?? fallbackUnion(selected, snapshot.experimentId);
|
|
151
|
-
const experiment =
|
|
152
|
-
snapshot.experiment !== undefined && redactor
|
|
153
|
-
? (redactExperimentInfo(snapshot.experiment as unknown as Record<string, unknown>, redactor) as unknown as SnapshotMeta["experiment"])
|
|
154
|
-
: snapshot.experiment;
|
|
155
130
|
const meta: SnapshotMeta = {
|
|
156
131
|
format: RESULTS_FORMAT,
|
|
157
132
|
schemaVersion: snapshot.schemaVersion,
|
|
158
133
|
producer: snapshot.producer,
|
|
159
134
|
experimentId: snapshot.experimentId,
|
|
160
|
-
...(experiment !== undefined ? { experiment } : {}),
|
|
135
|
+
...(snapshot.experiment !== undefined ? { experiment: snapshot.experiment } : {}),
|
|
161
136
|
agent: snapshot.agent,
|
|
162
137
|
...(snapshot.model !== undefined ? { model: snapshot.model } : {}),
|
|
163
138
|
startedAt: snapshot.startedAt,
|
|
164
139
|
...(snapshot.completedAt !== undefined ? { completedAt: snapshot.completedAt } : {}),
|
|
165
140
|
...(knownEvalIds.length ? { knownEvalIds } : {}),
|
|
166
|
-
...(snapshot.name !== undefined
|
|
167
|
-
? { name: redactor && typeof snapshot.name === "string" ? redactor(snapshot.name) : snapshot.name }
|
|
168
|
-
: {}),
|
|
169
|
-
// 发布根标记只声明流程,不证明结果:applied = 消毒函数对全部自由文本字段跑过;
|
|
170
|
-
// none = 作者显式的原文发布声明。view --out 的防呆据此分级。
|
|
171
|
-
publish: { redaction: redactor ? "applied" : "none" },
|
|
141
|
+
...(snapshot.name !== undefined ? { name: snapshot.name } : {}),
|
|
172
142
|
};
|
|
173
143
|
planned.push({ path: join(destSnapDir, SNAPSHOT_FILE), bytes: Buffer.from(JSON.stringify(meta, null, 2), "utf-8") });
|
|
174
144
|
return planned;
|
|
@@ -179,21 +149,18 @@ async function planOneAttempt(
|
|
|
179
149
|
destSnapDir: string,
|
|
180
150
|
kinds: ArtifactKind[],
|
|
181
151
|
plannedSourceHashes: Set<string>,
|
|
182
|
-
redactor: Redactor | undefined,
|
|
183
152
|
): Promise<PlannedFile[]> {
|
|
184
153
|
const destAttemptDir = join(destSnapDir, attempt.ref.attempt);
|
|
185
154
|
const planned: PlannedFile[] = [];
|
|
186
155
|
|
|
187
156
|
// sources 是唯一「两层」的 artifact(attempt 级引用 + 快照级去重仓库),不能像其它四类那样
|
|
188
157
|
// 单文件原字节完事——原字节只是引用,不带内容。走读取面已经会解引用+回退的 attempt.sources()
|
|
189
|
-
//
|
|
158
|
+
// 拿到完整内容,按内容哈希重新去重落盘——发布根里引用与内容永远一致,携带条目也被归拢进本快照。
|
|
190
159
|
const genericKinds = kinds.filter((k) => k !== "sources");
|
|
191
160
|
const files = await findArtifactFiles(attempt, genericKinds);
|
|
192
161
|
const copied = new Set(files.map((f) => f.kind));
|
|
193
162
|
for (const { kind, source } of files) {
|
|
194
|
-
|
|
195
|
-
const bytes = redactor ? redactArtifactBytes(kind, raw, redactor) : raw;
|
|
196
|
-
planned.push({ path: join(destAttemptDir, artifactFileOf(kind)), bytes, source });
|
|
163
|
+
planned.push({ path: join(destAttemptDir, artifactFileOf(kind)), bytes: await readFile(source), source });
|
|
197
164
|
}
|
|
198
165
|
|
|
199
166
|
if (kinds.includes("sources")) {
|
|
@@ -203,13 +170,12 @@ async function planOneAttempt(
|
|
|
203
170
|
const destStoreDir = join(destSnapDir, "sources");
|
|
204
171
|
const refs: { path: string; sha256: string }[] = [];
|
|
205
172
|
for (const src of sources) {
|
|
206
|
-
const
|
|
207
|
-
const sha256 = hashEvalSource(normalizeEvalSource(content));
|
|
173
|
+
const sha256 = hashEvalSource(normalizeEvalSource(src.content));
|
|
208
174
|
refs.push({ path: src.path, sha256 });
|
|
209
175
|
if (!plannedSourceHashes.has(sha256)) {
|
|
210
176
|
planned.push({
|
|
211
177
|
path: join(destStoreDir, `${sha256}.json`),
|
|
212
|
-
bytes: Buffer.from(JSON.stringify({ content }), "utf-8"),
|
|
178
|
+
bytes: Buffer.from(JSON.stringify({ content: src.content }), "utf-8"),
|
|
213
179
|
});
|
|
214
180
|
plannedSourceHashes.add(sha256);
|
|
215
181
|
}
|
|
@@ -221,40 +187,11 @@ async function planOneAttempt(
|
|
|
221
187
|
}
|
|
222
188
|
}
|
|
223
189
|
|
|
224
|
-
|
|
225
|
-
if (redactor) record = redactResultRecord(record, redactor);
|
|
190
|
+
const record = slimForCopy(attempt.result, copied);
|
|
226
191
|
planned.push({ path: join(destAttemptDir, RESULT_FILE), bytes: Buffer.from(JSON.stringify(record, null, 2), "utf-8") });
|
|
227
192
|
return planned;
|
|
228
193
|
}
|
|
229
194
|
|
|
230
|
-
/** 单个 artifact 文件的消毒:按种类解析 JSON、走各自的自由文本标注,重新序列化。 */
|
|
231
|
-
function redactArtifactBytes(kind: ArtifactKind, raw: Buffer, redactor: Redactor): Buffer {
|
|
232
|
-
try {
|
|
233
|
-
const parsed = JSON.parse(raw.toString("utf-8")) as unknown;
|
|
234
|
-
let next: unknown;
|
|
235
|
-
switch (kind) {
|
|
236
|
-
case "events":
|
|
237
|
-
next = redactEvents(parsed as StreamEvent[], redactor);
|
|
238
|
-
break;
|
|
239
|
-
case "trace":
|
|
240
|
-
next = redactSpans(parsed as TraceSpan[], redactor);
|
|
241
|
-
break;
|
|
242
|
-
// diff 的 before/after/内容与 o11y 的命令 / 错误 / URL 都是自由文本;路径键为结构字段。
|
|
243
|
-
case "diff":
|
|
244
|
-
case "o11y":
|
|
245
|
-
case "agentSetup":
|
|
246
|
-
next = redactJsonValue(parsed, redactor);
|
|
247
|
-
break;
|
|
248
|
-
default:
|
|
249
|
-
next = parsed;
|
|
250
|
-
}
|
|
251
|
-
return Buffer.from(JSON.stringify(next), "utf-8");
|
|
252
|
-
} catch {
|
|
253
|
-
// 解析不了的按原字节透传(malformed 数据不该在发布路径上被静默改写)。
|
|
254
|
-
return raw;
|
|
255
|
-
}
|
|
256
|
-
}
|
|
257
|
-
|
|
258
195
|
/** 目标目录非空即报错:盘上不该出现「我没写的东西被动过」的惊讶;发布脚本要幂等就自己先清目录。 */
|
|
259
196
|
async function assertEmptyDestination(dest: string): Promise<void> {
|
|
260
197
|
let entries: string[];
|
package/src/results/publish.ts
CHANGED
|
@@ -1,149 +1,7 @@
|
|
|
1
|
-
//
|
|
2
|
-
//
|
|
3
|
-
//
|
|
4
|
-
//
|
|
5
|
-
// redact 逐值消毒,范围由 schema 的自由文本标注决定:redactor 只对自由文本字段调用——
|
|
6
|
-
// 格式、判定、身份、路径、哈希这类结构字段永不经过它,发布根不会因 redact 变得不可读或
|
|
7
|
-
// 引用断裂。自由文本清单在下面的 FREE_TEXT 标注单点维护(与 AttemptList.redact 同一口径)。
|
|
8
|
-
|
|
9
|
-
import type { EvalResult, StreamEvent, TraceSpan } from "../types.ts";
|
|
1
|
+
// 发布预算(见 docs/feature/results/library.md「复制与瘦身:copySnapshots」)。
|
|
2
|
+
// 结果数据分两类:.niceeval/ 是本地事实根,不是默认可提交目录;任何要离开本机的拷贝是
|
|
3
|
+
// 发布拷贝,经 copySnapshots 这一条管线产出。管线只做选择、归拢与整文件大小预检,
|
|
4
|
+
// 不改写内容——保密边界由格式在采集侧划定(env 值与命令 stdout/stderr 不进结果文件)。
|
|
10
5
|
|
|
11
6
|
/** 发布前整文件预检的单文件上限(50 MiB,为 GitHub 100 MB 硬限保留余量);不是可调旋钮。 */
|
|
12
7
|
export const PUBLISH_FILE_MAX_BYTES = 50 * 1024 * 1024;
|
|
13
|
-
|
|
14
|
-
export type Redactor = (text: string) => string;
|
|
15
|
-
|
|
16
|
-
/**
|
|
17
|
-
* 结构字段键名(取值是格式、判定、身份、路径、哈希——redactor 永不触碰):
|
|
18
|
-
* 事件的 type/callId/status/role/tool/skill/requestId/optionId、span 的 ids/kind、
|
|
19
|
-
* 断言的 name/severity/outcome、错误的 code/phase、locator/artifactBase/fingerprint、
|
|
20
|
-
* 源码路径与 sha256、provider 名、时间戳等。新增字符串字段先判断体裁再决定进不进这张表。
|
|
21
|
-
*/
|
|
22
|
-
const STRUCTURAL_KEYS = new Set([
|
|
23
|
-
"type",
|
|
24
|
-
"callId",
|
|
25
|
-
"status",
|
|
26
|
-
"role",
|
|
27
|
-
"tool",
|
|
28
|
-
"requestId",
|
|
29
|
-
"optionId",
|
|
30
|
-
"id",
|
|
31
|
-
"traceId",
|
|
32
|
-
"spanId",
|
|
33
|
-
"parentSpanId",
|
|
34
|
-
"kind",
|
|
35
|
-
"format",
|
|
36
|
-
"verdict",
|
|
37
|
-
"severity",
|
|
38
|
-
"outcome",
|
|
39
|
-
"artifactBase",
|
|
40
|
-
"locator",
|
|
41
|
-
"fingerprint",
|
|
42
|
-
"sha256",
|
|
43
|
-
"path",
|
|
44
|
-
"file",
|
|
45
|
-
"provider",
|
|
46
|
-
"sandboxId",
|
|
47
|
-
"code",
|
|
48
|
-
"phase",
|
|
49
|
-
"level",
|
|
50
|
-
"agent",
|
|
51
|
-
"model",
|
|
52
|
-
"experimentId",
|
|
53
|
-
"startedAt",
|
|
54
|
-
"completedAt",
|
|
55
|
-
"schemaVersion",
|
|
56
|
-
"evalFilterFingerprint",
|
|
57
|
-
"reasoningEffort",
|
|
58
|
-
"skill",
|
|
59
|
-
"window",
|
|
60
|
-
"net",
|
|
61
|
-
"loc",
|
|
62
|
-
"dedupeKey",
|
|
63
|
-
]);
|
|
64
|
-
|
|
65
|
-
/** 深度遍历 JSON 值:字符串按「键名是否结构字段」决定过不过 redactor;结构键下整棵子树跳过。 */
|
|
66
|
-
export function redactJsonValue(value: unknown, redact: Redactor, key?: string): unknown {
|
|
67
|
-
if (typeof value === "string") {
|
|
68
|
-
if (key !== undefined && STRUCTURAL_KEYS.has(key)) return value;
|
|
69
|
-
return redact(value);
|
|
70
|
-
}
|
|
71
|
-
if (Array.isArray(value)) return value.map((v) => redactJsonValue(v, redact, key));
|
|
72
|
-
if (value !== null && typeof value === "object") {
|
|
73
|
-
const out: Record<string, unknown> = {};
|
|
74
|
-
for (const [k, v] of Object.entries(value)) {
|
|
75
|
-
if (STRUCTURAL_KEYS.has(k) && typeof v === "string") {
|
|
76
|
-
out[k] = v;
|
|
77
|
-
continue;
|
|
78
|
-
}
|
|
79
|
-
out[k] = redactJsonValue(v, redact, k);
|
|
80
|
-
}
|
|
81
|
-
return out;
|
|
82
|
-
}
|
|
83
|
-
return value;
|
|
84
|
-
}
|
|
85
|
-
|
|
86
|
-
/** events.json 的发布消毒:消息与工具入出参是自由文本,type/callId/status 等结构字段不动。 */
|
|
87
|
-
export function redactEvents(events: StreamEvent[], redact: Redactor): StreamEvent[] {
|
|
88
|
-
return redactJsonValue(events, redact) as StreamEvent[];
|
|
89
|
-
}
|
|
90
|
-
|
|
91
|
-
/** trace.json 的发布消毒:属性值与可携带动态内容的 span name 过 redactor,ids/kind 不动。 */
|
|
92
|
-
export function redactSpans(spans: TraceSpan[], redact: Redactor): TraceSpan[] {
|
|
93
|
-
return spans.map((span) => ({
|
|
94
|
-
...span,
|
|
95
|
-
name: redact(span.name),
|
|
96
|
-
...(span.attributes !== undefined
|
|
97
|
-
? { attributes: redactJsonValue(span.attributes, redact) as TraceSpan["attributes"] }
|
|
98
|
-
: {}),
|
|
99
|
-
}));
|
|
100
|
-
}
|
|
101
|
-
|
|
102
|
-
/** result.json 的发布消毒:断言 detail/evidence/expected/received、error/diagnostic 的
|
|
103
|
-
* message/cause/stack、skipReason、description 是自由文本;判定与身份字段不动。 */
|
|
104
|
-
export function redactResultRecord(record: Record<string, unknown>, redact: Redactor): Record<string, unknown> {
|
|
105
|
-
const r = record as Partial<EvalResult> & Record<string, unknown>;
|
|
106
|
-
const out: Record<string, unknown> = { ...record };
|
|
107
|
-
if (typeof r.description === "string") out.description = redact(r.description);
|
|
108
|
-
if (typeof r.skipReason === "string") out.skipReason = redact(r.skipReason);
|
|
109
|
-
if (Array.isArray(r.assertions)) {
|
|
110
|
-
out.assertions = r.assertions.map((a) => ({
|
|
111
|
-
...a,
|
|
112
|
-
...(a.detail !== undefined ? { detail: redact(a.detail) } : {}),
|
|
113
|
-
...(a.outcome !== "unavailable" && a.evidence !== undefined ? { evidence: redact(a.evidence) } : {}),
|
|
114
|
-
...(a.outcome !== "unavailable" && a.expected !== undefined ? { expected: redact(a.expected) } : {}),
|
|
115
|
-
...(a.outcome !== "unavailable" && a.received !== undefined ? { received: redact(a.received) } : {}),
|
|
116
|
-
}));
|
|
117
|
-
}
|
|
118
|
-
if (r.error !== undefined) {
|
|
119
|
-
out.error = {
|
|
120
|
-
...r.error,
|
|
121
|
-
message: redact(r.error.message),
|
|
122
|
-
...(r.error.stack !== undefined ? { stack: redact(r.error.stack) } : {}),
|
|
123
|
-
...(r.error.cause !== undefined ? { cause: { ...r.error.cause, message: redact(r.error.cause.message) } } : {}),
|
|
124
|
-
};
|
|
125
|
-
}
|
|
126
|
-
if (Array.isArray(r.diagnostics)) {
|
|
127
|
-
out.diagnostics = r.diagnostics.map((d) => ({
|
|
128
|
-
...d,
|
|
129
|
-
message: redact(d.message),
|
|
130
|
-
...(d.data !== undefined ? { data: redactJsonValue(d.data, redact) as typeof d.data } : {}),
|
|
131
|
-
}));
|
|
132
|
-
}
|
|
133
|
-
if (r.experiment !== undefined) {
|
|
134
|
-
out.experiment = redactExperimentInfo(r.experiment as unknown as Record<string, unknown>, redact);
|
|
135
|
-
}
|
|
136
|
-
return out;
|
|
137
|
-
}
|
|
138
|
-
|
|
139
|
-
/** ExperimentRunInfo 的发布消毒:description、flags 与 sandbox params 的字符串值。 */
|
|
140
|
-
export function redactExperimentInfo(info: Record<string, unknown>, redact: Redactor): Record<string, unknown> {
|
|
141
|
-
const out: Record<string, unknown> = { ...info };
|
|
142
|
-
if (typeof info.description === "string") out.description = redact(info.description);
|
|
143
|
-
if (info.flags !== undefined) out.flags = redactJsonValue(info.flags, redact);
|
|
144
|
-
const sandbox = info.sandbox as { provider: string; params?: Record<string, unknown> } | undefined;
|
|
145
|
-
if (sandbox?.params !== undefined) {
|
|
146
|
-
out.sandbox = { ...sandbox, params: redactJsonValue(sandbox.params, redact) };
|
|
147
|
-
}
|
|
148
|
-
return out;
|
|
149
|
-
}
|
|
@@ -552,7 +552,7 @@ describe("createResultsWriter", () => {
|
|
|
552
552
|
expect(await q2.agentSetup()).toBeNull();
|
|
553
553
|
|
|
554
554
|
const dest = join(await makeRoot(), "published");
|
|
555
|
-
await copySnapshots(results.latest(), dest, {
|
|
555
|
+
await copySnapshots(results.latest(), dest, { artifacts: ["agentSetup"] });
|
|
556
556
|
const copied = join(dest, "skill-ab_claude-effect", basename(snap.dir), "q1", "a1", "agent-setup.json");
|
|
557
557
|
expect(JSON.parse(await readFile(copied, "utf-8"))).toEqual(manifest);
|
|
558
558
|
});
|
|
@@ -737,7 +737,7 @@ describe("copySnapshots", () => {
|
|
|
737
737
|
|
|
738
738
|
const results = await openResults(root);
|
|
739
739
|
const dest = join(await makeRoot(), "site/data/run");
|
|
740
|
-
const copied = await copySnapshots(results.latest(), dest, {
|
|
740
|
+
const copied = await copySnapshots(results.latest(), dest, { artifacts: ["events"] });
|
|
741
741
|
|
|
742
742
|
expect(copied.warnings).toHaveLength(0);
|
|
743
743
|
expect(copied.dir).toBe(dest);
|
|
@@ -775,15 +775,15 @@ describe("copySnapshots", () => {
|
|
|
775
775
|
|
|
776
776
|
const occupied = await makeRoot();
|
|
777
777
|
await writeFile(join(occupied, "existing.txt"), "x", "utf-8");
|
|
778
|
-
await expect(copySnapshots(results.latest(), occupied
|
|
778
|
+
await expect(copySnapshots(results.latest(), occupied)).rejects.toThrow(/not empty/);
|
|
779
779
|
|
|
780
|
-
await expect(copySnapshots(results.latest(), join(await makeRoot(), "out"), {
|
|
780
|
+
await expect(copySnapshots(results.latest(), join(await makeRoot(), "out"), { artifacts: ["evnets" as never] })).rejects.toThrow(/Unknown artifact kind/);
|
|
781
781
|
|
|
782
|
-
await expect(copySnapshots([], join(await makeRoot(), "out")
|
|
782
|
+
await expect(copySnapshots([], join(await makeRoot(), "out"))).rejects.toThrow(/no snapshots/);
|
|
783
783
|
|
|
784
784
|
// 手工传入同一 experiment 的两个快照(未走 latest 去重):只带最新,记 warning。
|
|
785
785
|
const dest2 = join(await makeRoot(), "run2");
|
|
786
|
-
const collided = await copySnapshots(results.experiments[0].snapshots, dest2
|
|
786
|
+
const collided = await copySnapshots(results.experiments[0].snapshots, dest2);
|
|
787
787
|
expect(collided.warnings).toHaveLength(1);
|
|
788
788
|
expect(collided.warnings[0]).toMatch(/multiple snapshots selected/);
|
|
789
789
|
const destDirs = await readdir(join(dest2, "e"));
|
|
@@ -1005,7 +1005,7 @@ describe("AttemptLocator · 落盘 / 读取 / 携带 / 撞车", () => {
|
|
|
1005
1005
|
const locator1 = a1!.locator!;
|
|
1006
1006
|
|
|
1007
1007
|
const dest = join(await makeRoot(), "published");
|
|
1008
|
-
await copySnapshots(results.latest(), dest, {
|
|
1008
|
+
await copySnapshots(results.latest(), dest, { artifacts: [] });
|
|
1009
1009
|
|
|
1010
1010
|
const destResults = await openResults(dest);
|
|
1011
1011
|
expect(resolveLocator(destResults, locator0).result.attempt).toBe(0);
|
|
@@ -1192,7 +1192,7 @@ describe("sources · 快照级去重仓库", () => {
|
|
|
1192
1192
|
|
|
1193
1193
|
const results = await openResults(root);
|
|
1194
1194
|
const dest = join(await makeRoot(), "published");
|
|
1195
|
-
await copySnapshots(results.latest(), dest, {
|
|
1195
|
+
await copySnapshots(results.latest(), dest, { artifacts: ["sources"] });
|
|
1196
1196
|
|
|
1197
1197
|
const destSnapDir = join(dest, "e", basename(snap.dir));
|
|
1198
1198
|
const destStoreFiles = await readdir(join(destSnapDir, "sources"));
|
package/src/results/types.ts
CHANGED
|
@@ -44,11 +44,6 @@ export interface SnapshotMeta {
|
|
|
44
44
|
completedAt?: string;
|
|
45
45
|
/** 写入时刻该实验已知的 eval 并集 —— 残缺检测的分母随数据走(copySnapshots 自动补记,writer 可声明)。 */
|
|
46
46
|
knownEvalIds?: string[];
|
|
47
|
-
/**
|
|
48
|
-
* 发布拷贝的自描述标记:copySnapshots 补记,消毒函数 → "applied"、redact: false → "none";
|
|
49
|
-
* 本地事实根没有此字段。只声明流程,不证明无秘密;view --out 据此分级防呆。
|
|
50
|
-
*/
|
|
51
|
-
publish?: { redaction: "applied" | "none" };
|
|
52
47
|
/** 项目名(来自 config.name),透传给 `niceeval view` 顶部 hero 显示。 */
|
|
53
48
|
name?: LocalizedText;
|
|
54
49
|
}
|
|
@@ -130,8 +125,6 @@ export interface Snapshot {
|
|
|
130
125
|
dir: string;
|
|
131
126
|
/** 写入时刻该实验已知的 eval 并集(可选);copySnapshots 自动补记,writer.snapshot() 也可声明。 */
|
|
132
127
|
knownEvalIds?: string[];
|
|
133
|
-
/** 发布拷贝的自描述标记(见 SnapshotMeta.publish);本地事实根没有此字段。 */
|
|
134
|
-
publish?: { redaction: "applied" | "none" };
|
|
135
128
|
}
|
|
136
129
|
|
|
137
130
|
/** 一个实验的全部历史:同一 experiment id 的历次快照归在一起。 */
|
|
@@ -119,7 +119,7 @@ describe("index.ts · copyFetchedArtifacts(--out 静态导出)对 sources.json
|
|
|
119
119
|
await seedDedupedSnapshot(root);
|
|
120
120
|
|
|
121
121
|
const out = join(root, "site");
|
|
122
|
-
await buildView({ input: root, out
|
|
122
|
+
await buildView({ input: root, out });
|
|
123
123
|
|
|
124
124
|
const scan = await loadViewScan(root);
|
|
125
125
|
const byId = new Map(scan.viewData.snapshots.flatMap((s) => s.results.map((r) => [r.id, r])));
|
package/src/view/data.ts
CHANGED
|
@@ -57,12 +57,6 @@ export interface ViewScan {
|
|
|
57
57
|
reportPages: ViewReportPageHtml[];
|
|
58
58
|
/** 外壳注入资产(styles / scripts;{src} 已按路径纪律解析成 inline 内容),只进 web 面。 */
|
|
59
59
|
shellAssets: { styles: string[]; scripts: string[]; head: ResolvedHeadTag[] };
|
|
60
|
-
/**
|
|
61
|
-
* --out 的数据等级(见 docs/feature/reports/view.md「静态导出」):全部选中快照带
|
|
62
|
-
* publish:{redaction:"applied"} 才是 "applied",否则 "sensitive"(含本地事实根与
|
|
63
|
-
* redaction:"none"——上游声明过原文发布也不豁免导出时的确认)。
|
|
64
|
-
*/
|
|
65
|
-
publishState: "applied" | "sensitive";
|
|
66
60
|
}
|
|
67
61
|
|
|
68
62
|
/** view 宿主输入的组合语义(与 show 对齐,docs/feature/reports/architecture.md「Scope 是计算入口」)。 */
|
|
@@ -292,17 +286,12 @@ export async function loadViewScan(input?: string, opts: ViewScanOptions = {}):
|
|
|
292
286
|
skippedRuns: results.skipped.map(toSkippedNotice),
|
|
293
287
|
report: slot.meta,
|
|
294
288
|
};
|
|
295
|
-
const publishState =
|
|
296
|
-
selection.snapshots.length > 0 && selection.snapshots.every((snap) => snap.publish?.redaction === "applied")
|
|
297
|
-
? ("applied" as const)
|
|
298
|
-
: ("sensitive" as const);
|
|
299
289
|
return {
|
|
300
290
|
viewData,
|
|
301
291
|
artifactDirs,
|
|
302
292
|
attemptsByBase,
|
|
303
293
|
reportPages: slot.pages,
|
|
304
294
|
shellAssets: slot.shellAssets,
|
|
305
|
-
publishState,
|
|
306
295
|
};
|
|
307
296
|
}
|
|
308
297
|
|
package/src/view/index.ts
CHANGED
|
@@ -88,23 +88,12 @@ export async function buildView(opts: ViewOptions = {}): Promise<string> {
|
|
|
88
88
|
"--out exports the whole results root and cannot be combined with eval prefixes or --experiment.\n" +
|
|
89
89
|
"To publish a site for one experiment, build a narrower results root and export that:\n" +
|
|
90
90
|
' const results = await openResults(".niceeval");\n' +
|
|
91
|
-
' await copySnapshots(results.latest().filter((s) => s.experimentId.startsWith("<prefix>/")), "<publish-root>"
|
|
91
|
+
' await copySnapshots(results.latest().filter((s) => s.experimentId.startsWith("<prefix>/")), "<publish-root>");\n' +
|
|
92
92
|
"Then: niceeval view --results <publish-root> --out <site>",
|
|
93
93
|
);
|
|
94
94
|
}
|
|
95
95
|
// 静态导出保持「任一页失败整体失败」(pageFailure 缺省 "throw"),不产出半套站点。
|
|
96
96
|
const plan = await planSite(opts.input, opts.scan);
|
|
97
|
-
// --out 按发布防呆二分(见 docs/feature/reports/view.md「静态导出」):目标结果根的全部快照
|
|
98
|
-
// 带 publish:{redaction:"applied"}(copySnapshots 补记)时直接导出;redaction:"none"、
|
|
99
|
-
// 无标记结果或本地事实根,都必须显式传 --allow-sensitive-artifacts——静态站原样携带证据文件,
|
|
100
|
-
// 上游声明过原文发布也不豁免这里的确认。
|
|
101
|
-
if (!opts.allowSensitiveArtifacts && plan.scan.publishState !== "applied") {
|
|
102
|
-
throw new ViewInputError(
|
|
103
|
-
`--out refuses to export unredacted results: not every selected snapshot carries publish: { redaction: "applied" }. ` +
|
|
104
|
-
`Produce a publish root first with copySnapshots({ redact }) and export that (niceeval view --results <publish-root> --out <site>), ` +
|
|
105
|
-
`or pass --allow-sensitive-artifacts to explicitly export raw evidence (prompts, tool args, full outputs, sources).`,
|
|
106
|
-
);
|
|
107
|
-
}
|
|
108
97
|
await writeSite(plan, out);
|
|
109
98
|
return out;
|
|
110
99
|
}
|
package/src/view/server.ts
CHANGED
|
@@ -13,8 +13,6 @@ export interface ViewOptions {
|
|
|
13
13
|
input?: string;
|
|
14
14
|
out?: string;
|
|
15
15
|
port?: number;
|
|
16
|
-
/** `--out` 对非发布根(无 publish:applied 标记)导出时的显式确认;静态站原样携带证据文件。 */
|
|
17
|
-
allowSensitiveArtifacts?: boolean;
|
|
18
16
|
/** 报告槽的组合语义(位置前缀 / --experiment / --report),透传给站点管线。 */
|
|
19
17
|
scan?: ViewScanOptions;
|
|
20
18
|
}
|
|
@@ -54,7 +54,7 @@ describe("站点管线奇偶:server 与 --out 是同一份产物", () => {
|
|
|
54
54
|
await seedSnapshot(root, "exp", "q1");
|
|
55
55
|
|
|
56
56
|
const out = await makeDir("niceeval-parity-out-");
|
|
57
|
-
await buildView({ input: root, out
|
|
57
|
+
await buildView({ input: root, out });
|
|
58
58
|
server = await startViewServer({ input: root });
|
|
59
59
|
|
|
60
60
|
// 清单驱动遍历:计划里的每一个路径,写盘字节 ≡ 服务字节。
|
package/src/view/site.ts
CHANGED
|
@@ -21,7 +21,7 @@ export interface SiteFile {
|
|
|
21
21
|
export interface SitePlan {
|
|
22
22
|
/** path → SiteFile;写盘按它遍历,server 按它查表(查不到即 404,不存在旁路取数)。 */
|
|
23
23
|
files: Map<string, SiteFile>;
|
|
24
|
-
/** 构建这份产物用的扫描结果(
|
|
24
|
+
/** 构建这份产物用的扫描结果(宿主前置校验与调试用;不进产物)。 */
|
|
25
25
|
scan: ViewScan;
|
|
26
26
|
}
|
|
27
27
|
|
|
@@ -398,7 +398,7 @@ describe("buildView · --out 与 --report", () => {
|
|
|
398
398
|
await writeFile(join(artifactDir, "events.json"), "[]", "utf-8");
|
|
399
399
|
|
|
400
400
|
const out = join(root, "site");
|
|
401
|
-
await buildView({ input: root, out,
|
|
401
|
+
await buildView({ input: root, out, scan: { report: { path: EXAM_REPORT, cwd: root } } });
|
|
402
402
|
|
|
403
403
|
const html = await readFile(join(out, "index.html"), "utf-8");
|
|
404
404
|
// 双语两个 <template> 静态块都在,壳按界面语言摆放。
|
|
@@ -424,7 +424,7 @@ describe("buildView · --out 与 --report", () => {
|
|
|
424
424
|
await writeFile(join(artifactDir, "o11y.json"), "{}", "utf-8");
|
|
425
425
|
|
|
426
426
|
const out = join(root, "site");
|
|
427
|
-
await buildView({ input: root, out
|
|
427
|
+
await buildView({ input: root, out });
|
|
428
428
|
const exported = join(out, "artifact", "compare_codex/2026-07-09T10-00-00-000Z/weather/brooklyn/a0");
|
|
429
429
|
expect(existsSync(join(exported, "diff.json"))).toBe(true);
|
|
430
430
|
expect(existsSync(join(exported, "events.json"))).toBe(true);
|
|
@@ -435,10 +435,10 @@ describe("buildView · --out 与 --report", () => {
|
|
|
435
435
|
const root = await seedRoot();
|
|
436
436
|
const out = join(root, "site");
|
|
437
437
|
for (const scan of [{ patterns: ["weather"] }, { experiment: "compare/bub" }]) {
|
|
438
|
-
const attempt = buildView({ input: root, out,
|
|
438
|
+
const attempt = buildView({ input: root, out, scan });
|
|
439
439
|
await expect(attempt).rejects.toBeInstanceOf(ViewInputError);
|
|
440
440
|
await expect(
|
|
441
|
-
buildView({ input: root, out,
|
|
441
|
+
buildView({ input: root, out, scan }),
|
|
442
442
|
).rejects.toThrow(/copySnapshots/);
|
|
443
443
|
}
|
|
444
444
|
// 同参数不带 --out 时照常收窄报告槽(不报错)。
|
|
@@ -448,7 +448,7 @@ describe("buildView · --out 与 --report", () => {
|
|
|
448
448
|
it("默认导出(无 --report):报告槽填充 ExperimentComparison,双语块与增强 runtime 恒内联", async () => {
|
|
449
449
|
const root = await seedRoot();
|
|
450
450
|
const out = join(root, "site");
|
|
451
|
-
await buildView({ input: root, out
|
|
451
|
+
await buildView({ input: root, out });
|
|
452
452
|
const html = await readFile(join(out, "index.html"), "utf-8");
|
|
453
453
|
expect(html).toContain('<template id="niceeval-report-report-en">');
|
|
454
454
|
expect(html).toContain('<template id="niceeval-report-report-zh-CN">');
|