niceeval 0.5.1 → 0.5.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/docs-site/zh/concepts/adapter.mdx +6 -6
- package/docs-site/zh/concepts/experiment.mdx +6 -6
- package/docs-site/zh/concepts/overview.mdx +1 -1
- package/docs-site/zh/concepts/tier.mdx +6 -6
- package/docs-site/zh/guides/ci-integration.mdx +2 -0
- package/docs-site/zh/guides/connect-your-agent.mdx +7 -7
- package/docs-site/zh/guides/custom-reports.mdx +7 -7
- package/docs-site/zh/guides/experiments.mdx +3 -3
- package/docs-site/zh/guides/publish-report.mdx +91 -0
- package/docs-site/zh/guides/report-components.mdx +4 -4
- package/docs-site/zh/guides/reporters.mdx +1 -1
- package/docs-site/zh/guides/results-data.mdx +1 -1
- package/docs-site/zh/guides/runner.mdx +1 -1
- package/docs-site/zh/guides/sandbox-agent.mdx +2 -2
- package/docs-site/zh/guides/viewing-results.mdx +3 -19
- package/docs-site/zh/guides/write-experiment.mdx +9 -9
- package/docs-site/zh/guides/write-send.mdx +6 -6
- package/docs-site/zh/index.mdx +1 -1
- package/docs-site/zh/introduction.mdx +1 -1
- package/docs-site/zh/reference/builtin-agents.mdx +4 -0
- package/docs-site/zh/reference/cli.mdx +10 -6
- package/docs-site/zh/reference/define-agent.mdx +7 -3
- package/docs-site/zh/reference/define-config.mdx +3 -1
- package/docs-site/zh/reference/define-eval.mdx +9 -3
- package/docs-site/zh/reference/events.mdx +4 -0
- package/docs-site/zh/reference/expect.mdx +4 -0
- package/package.json +3 -2
- package/src/agents/ai-sdk.test.ts +1 -1
- package/src/agents/ai-sdk.ts +2 -2
- package/src/agents/claude-code.ts +1 -1
- package/src/agents/codex.ts +2 -2
- package/src/agents/streaming.test.ts +1 -1
- package/src/agents/types.ts +2 -2
- package/src/agents/ui-message-stream.test.ts +1 -1
- package/src/cli.ts +53 -15
- package/src/context/context.test.ts +1 -1
- package/src/context/context.ts +3 -3
- package/src/context/session.ts +2 -2
- package/src/context/types.ts +2 -2
- package/src/i18n/en.ts +2 -2
- package/src/i18n/zh-CN.ts +2 -2
- package/src/report/aggregate.ts +16 -16
- package/src/report/components.tsx +1 -1
- package/src/report/compute.ts +7 -7
- package/src/report/dual-face.test.tsx +1 -1
- package/src/report/{param.ts → flag.ts} +6 -6
- package/src/report/index.ts +3 -3
- package/src/report/react/MetricLine.tsx +1 -1
- package/src/report/react/fixtures.ts +1 -1
- package/src/report/report.test.ts +17 -17
- package/src/report/types.ts +11 -11
- package/src/results/results.test.ts +2 -2
- package/src/runner/attempt.ts +3 -3
- package/src/runner/fingerprint.ts +1 -1
- package/src/runner/reporters/braintrust.test.ts +2 -2
- package/src/runner/reporters/braintrust.ts +3 -3
- package/src/runner/run.ts +1 -1
- package/src/runner/types.ts +8 -8
- package/src/view/app/css.d.ts +3 -0
- package/src/view/app/i18n.ts +3 -3
- package/src/view/app/lib/rows.ts +4 -4
- package/src/view/client-dist/app.js +2 -2
- package/src/view/data.test.ts +25 -1
- package/src/view/data.ts +27 -3
package/src/report/aggregate.ts
CHANGED
|
@@ -18,15 +18,15 @@ import type {
|
|
|
18
18
|
Metric,
|
|
19
19
|
MetricCell,
|
|
20
20
|
MetricColumn,
|
|
21
|
-
|
|
21
|
+
FlagRef,
|
|
22
22
|
} from "./types.ts";
|
|
23
23
|
import { formatMetricValue } from "./format.ts";
|
|
24
24
|
|
|
25
25
|
// 复合键分隔符:NUL 不会出现在 eval id / experimentId / ISO 时间里,拼接键不会串味
|
|
26
26
|
const KEY_SEP = "\u0000";
|
|
27
27
|
|
|
28
|
-
/**
|
|
29
|
-
export const
|
|
28
|
+
/** flag 未声明时的组名:不猜,如实归一组。 */
|
|
29
|
+
export const FLAG_UNSET = "(unset)";
|
|
30
30
|
|
|
31
31
|
/** 计算函数的第一参:选集(warnings 随行)或手工挑的快照数组(没有挑选过程,自然无警告)。 */
|
|
32
32
|
export type SnapshotsInput = Selection | Snapshot[];
|
|
@@ -105,33 +105,33 @@ export function evalGroupOf(id: string): string {
|
|
|
105
105
|
return slash === -1 ? id : id.slice(0, slash);
|
|
106
106
|
}
|
|
107
107
|
|
|
108
|
-
function
|
|
109
|
-
return typeof dimension === "object" && "kind" in dimension && dimension.kind === "
|
|
108
|
+
function isFlagRef(dimension: DimensionInput): dimension is FlagRef {
|
|
109
|
+
return typeof dimension === "object" && "kind" in dimension && dimension.kind === "flag";
|
|
110
110
|
}
|
|
111
111
|
|
|
112
|
-
/** experiment 声明的
|
|
113
|
-
function
|
|
114
|
-
return attempt.result.experiment?.
|
|
112
|
+
/** experiment 声明的 flags(经 runner 原样透传进持久化字段 ExperimentRunInfo.flags)。 */
|
|
113
|
+
function flagsOf(attempt: AttemptHandle): Record<string, unknown> | undefined {
|
|
114
|
+
return attempt.result.experiment?.flags;
|
|
115
115
|
}
|
|
116
116
|
|
|
117
|
-
/**
|
|
118
|
-
export function
|
|
119
|
-
const value =
|
|
120
|
-
if (value === undefined) return
|
|
117
|
+
/** flag 声明值 → 组标签:label 函数优先,其余 String();未声明 → FLAG_UNSET。 */
|
|
118
|
+
export function flagGroupKey(ref: FlagRef, item: Item): string {
|
|
119
|
+
const value = flagsOf(item.attempt)?.[ref.name];
|
|
120
|
+
if (value === undefined) return FLAG_UNSET;
|
|
121
121
|
// 持久化字段是 Record<string, unknown>;声明侧的合法值就是这三种标量
|
|
122
122
|
if (typeof ref.label === "function") return ref.label(value as string | number | boolean);
|
|
123
123
|
return String(value);
|
|
124
124
|
}
|
|
125
125
|
|
|
126
|
-
/**
|
|
127
|
-
export function
|
|
128
|
-
const value =
|
|
126
|
+
/** flag 作轴:要求数值;未声明或非数值 → null(点不画,注脚报数)。 */
|
|
127
|
+
export function flagAxisValue(ref: FlagRef, item: Item): number | null {
|
|
128
|
+
const value = flagsOf(item.attempt)?.[ref.name];
|
|
129
129
|
return typeof value === "number" ? value : null;
|
|
130
130
|
}
|
|
131
131
|
|
|
132
132
|
export function dimensionKey(dimension: DimensionInput, item: Item): string {
|
|
133
133
|
if (typeof dimension !== "string") {
|
|
134
|
-
if (
|
|
134
|
+
if (isFlagRef(dimension)) return flagGroupKey(dimension, item);
|
|
135
135
|
return dimension.of(item.attempt);
|
|
136
136
|
}
|
|
137
137
|
const result = item.attempt.result;
|
|
@@ -159,7 +159,7 @@ export const MetricScatter = Object.assign(
|
|
|
159
159
|
);
|
|
160
160
|
MetricScatter.displayName = "MetricScatter";
|
|
161
161
|
|
|
162
|
-
/** 趋势线:x 是 experiment 声明的
|
|
162
|
+
/** 趋势线:x 是 experiment 声明的 flag(flag()),同系列按 x 排序连线。 */
|
|
163
163
|
export const MetricLine = Object.assign(
|
|
164
164
|
defineComponent<MetricLineProps>({
|
|
165
165
|
web: (props) => <MetricLineWeb {...props} />,
|
package/src/report/compute.ts
CHANGED
|
@@ -19,7 +19,7 @@ import type {
|
|
|
19
19
|
Metric,
|
|
20
20
|
MetricCell,
|
|
21
21
|
OverviewData,
|
|
22
|
-
|
|
22
|
+
FlagRef,
|
|
23
23
|
ScatterData,
|
|
24
24
|
ScoreboardData,
|
|
25
25
|
TableData,
|
|
@@ -39,7 +39,7 @@ import {
|
|
|
39
39
|
experimentIdOf,
|
|
40
40
|
filterItems,
|
|
41
41
|
groupItems,
|
|
42
|
-
|
|
42
|
+
flagAxisValue,
|
|
43
43
|
resolveInput,
|
|
44
44
|
snapshotKeyOf,
|
|
45
45
|
toColumn,
|
|
@@ -52,7 +52,7 @@ import { formatMetricValue, formatPlainNumber } from "./format.ts";
|
|
|
52
52
|
// ───────────────────────── MetricTable.data ─────────────────────────
|
|
53
53
|
|
|
54
54
|
export interface TableDataOptions<M extends readonly Metric[]> {
|
|
55
|
-
/** 行维度(内置 / 自定义 /
|
|
55
|
+
/** 行维度(内置 / 自定义 / flag())。 */
|
|
56
56
|
rows: DimensionInput;
|
|
57
57
|
/** 每列一个指标;列键 = metric.name 的字面量,拼错编译不过。 */
|
|
58
58
|
columns: M;
|
|
@@ -289,10 +289,10 @@ export async function scatterData(input: SnapshotsInput, opts: ScatterDataOption
|
|
|
289
289
|
// ───────────────────────── MetricLine.data ─────────────────────────
|
|
290
290
|
|
|
291
291
|
export interface LineDataOptions {
|
|
292
|
-
/** x 轴:experiment 声明的
|
|
293
|
-
x:
|
|
292
|
+
/** x 轴:experiment 声明的 flag(数值),不解析 experiment 命名。 */
|
|
293
|
+
x: FlagRef;
|
|
294
294
|
y: Metric;
|
|
295
|
-
/** 可选:每个系列一条线(
|
|
295
|
+
/** 可选:每个系列一条线(flag 或普通维度);省略 = 单系列。 */
|
|
296
296
|
series?: DimensionInput;
|
|
297
297
|
}
|
|
298
298
|
|
|
@@ -303,7 +303,7 @@ export async function lineData(input: SnapshotsInput, opts: LineDataOptions): Pr
|
|
|
303
303
|
const groups = groupItems(items, "experiment");
|
|
304
304
|
const rows: LineData["rows"] = [];
|
|
305
305
|
for (const [key, group] of groups) {
|
|
306
|
-
const x =
|
|
306
|
+
const x = flagAxisValue(opts.x, group[0]); // flag 是 experiment 级声明,组内一致
|
|
307
307
|
rows.push({
|
|
308
308
|
key,
|
|
309
309
|
series: opts.series ? dimensionKey(opts.series, group[0]) : undefined,
|
|
@@ -228,7 +228,7 @@ describe("MetricLine 双面", () => {
|
|
|
228
228
|
`);
|
|
229
229
|
});
|
|
230
230
|
|
|
231
|
-
it("两面同口径:未声明
|
|
231
|
+
it("两面同口径:未声明 flag 的点两面都不画、注脚同数", () => {
|
|
232
232
|
expect(html).toContain("1 point missing data");
|
|
233
233
|
expect(term).toContain("1 point missing data");
|
|
234
234
|
expect(html).not.toContain('data-key="ultra/legacy"'); // 注脚 title 如实列缺数据的点,但不画
|
|
@@ -1,18 +1,18 @@
|
|
|
1
|
-
//
|
|
1
|
+
// flag():把 experiment 声明的 flags 当维度或轴(docs/reports.md「flags 与新摆法」)。
|
|
2
2
|
// 变量来自配置,不来自命名 —— 报告不解析 experiment id 字符串抠变量。
|
|
3
3
|
|
|
4
|
-
import type {
|
|
4
|
+
import type { FlagRef } from "./types.ts";
|
|
5
5
|
|
|
6
|
-
export function
|
|
6
|
+
export function flag(
|
|
7
7
|
name: string,
|
|
8
8
|
opts?: {
|
|
9
9
|
/** 组标签 / 轴标签;函数形态把声明值折成组名(如 `(v) => \`${v} agents\``)。 */
|
|
10
10
|
label?: string | ((value: string | number | boolean) => string);
|
|
11
11
|
unit?: string;
|
|
12
12
|
},
|
|
13
|
-
):
|
|
13
|
+
): FlagRef {
|
|
14
14
|
if (typeof name !== "string" || name.length === 0) {
|
|
15
|
-
throw new Error("
|
|
15
|
+
throw new Error("flag: name must be a non-empty string (the key declared in the experiment's flags).");
|
|
16
16
|
}
|
|
17
|
-
return { kind: "
|
|
17
|
+
return { kind: "flag", name, label: opts?.label, unit: opts?.unit };
|
|
18
18
|
}
|
package/src/report/index.ts
CHANGED
|
@@ -8,9 +8,9 @@
|
|
|
8
8
|
// ./web.ts,只有那一侧 import react-dom。写报告文件的项目要装 react(.tsx 编译产物
|
|
9
9
|
// import react/jsx-runtime)。
|
|
10
10
|
|
|
11
|
-
// 指标与
|
|
11
|
+
// 指标与 flag
|
|
12
12
|
export { defineMetric, passRate, examScore, durationMs, tokens, costUSD } from "./metrics.ts";
|
|
13
|
-
export {
|
|
13
|
+
export { flag } from "./flag.ts";
|
|
14
14
|
|
|
15
15
|
// 报告基座与双面组件基座
|
|
16
16
|
export { defineReport, isReportDefinition, renderReportToText } from "./report.ts";
|
|
@@ -84,7 +84,7 @@ export type {
|
|
|
84
84
|
MetricCell,
|
|
85
85
|
MetricColumn,
|
|
86
86
|
OverviewData,
|
|
87
|
-
|
|
87
|
+
FlagRef,
|
|
88
88
|
ScatterData,
|
|
89
89
|
ScoreboardData,
|
|
90
90
|
SelectionWarning,
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
// MetricLine:趋势线——x 是 experiment 声明的
|
|
1
|
+
// MetricLine:趋势线——x 是 experiment 声明的 flag(有序变量),每个系列一条线。
|
|
2
2
|
// 与 MetricScatter 的分工:scatter 的两轴都是测出来的指标(找 frontier),
|
|
3
3
|
// line 的 x 是你配置的变量(看趋势)。x 轴正常升序;y 轴向随 better;
|
|
4
4
|
// x 或 y 缺数据的点不画,注脚如实报数;每个点带 <title> hover,可经 pointHref 下钻。
|
|
@@ -207,7 +207,7 @@ export const lineData: LineData = {
|
|
|
207
207
|
y: { value: 0.7, display: "70%", samples: 6, total: 6, refs: [] },
|
|
208
208
|
},
|
|
209
209
|
{
|
|
210
|
-
// 未声明
|
|
210
|
+
// 未声明 flag 的 experiment:作轴不画点,注脚报数
|
|
211
211
|
key: "ultra/legacy",
|
|
212
212
|
series: "1 agents",
|
|
213
213
|
x: null,
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// niceeval/report 计算层的单元测试:全部用内存 fake(Snapshot / AttemptHandle 按
|
|
2
2
|
// niceeval/results 的读取契约手工构造),专门覆盖 docs/reports.md 点名的坑 ——
|
|
3
3
|
// 两级聚合 vs 平铺、pass@k、examScore 空真、skipped 稀释、scoreboard 固定分母与
|
|
4
|
-
// 最长前缀、scatter/delta 的 null 语义、快照键对比、
|
|
4
|
+
// 最长前缀、scatter/delta 的 null 语义、快照键对比、flag 维度与轴、
|
|
5
5
|
// cases 的 redact/truncated、身份键去重、Selection warnings 随行。
|
|
6
6
|
|
|
7
7
|
import { describe, expect, it } from "vitest";
|
|
@@ -10,7 +10,7 @@ import type { AssertionResult, EvalResult, ResultOutcome, RunSummary } from "../
|
|
|
10
10
|
import type { AttemptHandle, RunDir, Selection, SelectionWarning, Snapshot } from "../results/index.ts";
|
|
11
11
|
import type { Dimension, MetricCell } from "./types.ts";
|
|
12
12
|
import { costUSD, defineMetric, durationMs, examScore, passRate, tokens } from "./metrics.ts";
|
|
13
|
-
import {
|
|
13
|
+
import { flag } from "./flag.ts";
|
|
14
14
|
import { formatMetricValue } from "./format.ts";
|
|
15
15
|
import {
|
|
16
16
|
CaseList,
|
|
@@ -465,22 +465,22 @@ describe("MetricScatter.data", () => {
|
|
|
465
465
|
});
|
|
466
466
|
});
|
|
467
467
|
|
|
468
|
-
// ─────────────────────────
|
|
468
|
+
// ───────────────────────── flag():维度与轴 ─────────────────────────
|
|
469
469
|
|
|
470
|
-
describe("
|
|
471
|
-
const
|
|
470
|
+
describe("flag()", () => {
|
|
471
|
+
const withFlags = (id: string, flags: Record<string, unknown> | undefined, outcome: ResultOutcome) =>
|
|
472
472
|
snap({
|
|
473
473
|
experimentId: id,
|
|
474
|
-
results: [res("A", outcome, { experimentId: id, experiment: { id,
|
|
474
|
+
results: [res("A", outcome, { experimentId: id, experiment: { id, flags } })],
|
|
475
475
|
});
|
|
476
476
|
|
|
477
|
-
it("MetricLine.data:x 收
|
|
478
|
-
const s1 =
|
|
479
|
-
const s2 =
|
|
480
|
-
const legacy =
|
|
477
|
+
it("MetricLine.data:x 收 flag、按 experiment 聚合;未声明的作轴 x=null 报数", async () => {
|
|
478
|
+
const s1 = withFlags("ultra/lat-100", { latencyMs: 100, agents: 1 }, "passed");
|
|
479
|
+
const s2 = withFlags("ultra/lat-300", { latencyMs: 300, agents: 1 }, "failed");
|
|
480
|
+
const legacy = withFlags("ultra/legacy", undefined, "passed");
|
|
481
481
|
const data = await MetricLine.data([s1, s2, legacy], {
|
|
482
|
-
x:
|
|
483
|
-
series:
|
|
482
|
+
x: flag("latencyMs", { label: "Simulated latency", unit: "ms" }),
|
|
483
|
+
series: flag("agents", { label: (v) => `${v} agents` }),
|
|
484
484
|
y: passRate,
|
|
485
485
|
});
|
|
486
486
|
expect(data.x).toEqual({ key: "latencyMs", label: "Simulated latency", unit: "ms" });
|
|
@@ -493,18 +493,18 @@ describe("param()", () => {
|
|
|
493
493
|
expect(p100.series).toBe("1 agents");
|
|
494
494
|
expect(p100.y.value).toBe(1);
|
|
495
495
|
|
|
496
|
-
// 未声明
|
|
496
|
+
// 未声明 flag 的 experiment 不猜:作轴 x=null(组件不画、注脚报数),分组归 (unset)
|
|
497
497
|
const legacyRow = data.rows.find((r) => r.key === "ultra/legacy")!;
|
|
498
498
|
expect(legacyRow.x).toBeNull();
|
|
499
499
|
expect(legacyRow.xDisplay).toBe("");
|
|
500
500
|
expect(legacyRow.series).toBe("(unset)");
|
|
501
501
|
});
|
|
502
502
|
|
|
503
|
-
it("
|
|
504
|
-
const s1 =
|
|
505
|
-
const s2 =
|
|
503
|
+
it("flag 当维度用:按声明值分组,label 函数折组名", async () => {
|
|
504
|
+
const s1 = withFlags("exp/a", { agents: 1 }, "passed");
|
|
505
|
+
const s2 = withFlags("exp/b", { agents: 16 }, "failed");
|
|
506
506
|
const data = await MetricTable.data([s1, s2], {
|
|
507
|
-
rows:
|
|
507
|
+
rows: flag("agents", { label: (v) => `${v} agents` }),
|
|
508
508
|
columns: [passRate],
|
|
509
509
|
});
|
|
510
510
|
expect(data.dimension).toBe("agents");
|
package/src/report/types.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
// niceeval/report 的公开类型:指标(Metric)、维度(Dimension /
|
|
1
|
+
// niceeval/report 的公开类型:指标(Metric)、维度(Dimension / flag())与计算函数
|
|
2
2
|
// 产物(即组件的 data props)。数据契约照 docs/reports.md「计算函数与数据契约」;
|
|
3
3
|
// 这些不是持久化格式,没有 format / schemaVersion 信封,兼容性跟随 npm 版本。
|
|
4
4
|
|
|
@@ -66,20 +66,20 @@ export type Dimension =
|
|
|
66
66
|
| { name: string; of: (attempt: AttemptHandle) => string };
|
|
67
67
|
|
|
68
68
|
/**
|
|
69
|
-
*
|
|
69
|
+
* flag() 的产物:把 experiment 声明的 flags 当维度(series / rows / columns / points
|
|
70
70
|
* 槽,按声明值分组)或轴(MetricLine 的 x 槽,要求数值并驱动刻度)。
|
|
71
|
-
* 未声明该
|
|
71
|
+
* 未声明该 flag 的 experiment 不猜:分组如实归「(unset)」,作轴不画点、注脚报数。
|
|
72
72
|
*/
|
|
73
|
-
export interface
|
|
74
|
-
readonly kind: "
|
|
73
|
+
export interface FlagRef {
|
|
74
|
+
readonly kind: "flag";
|
|
75
75
|
readonly name: string;
|
|
76
76
|
/** 组标签 / 轴标签;函数形态把声明值折成组名(如 `(v) => \`${v} agents\``)。 */
|
|
77
77
|
readonly label?: string | ((value: string | number | boolean) => string);
|
|
78
78
|
readonly unit?: string;
|
|
79
79
|
}
|
|
80
80
|
|
|
81
|
-
/** 维度槽的输入:内置/自定义维度,或 experiment 声明的
|
|
82
|
-
export type DimensionInput = Dimension |
|
|
81
|
+
/** 维度槽的输入:内置/自定义维度,或 experiment 声明的 flag。 */
|
|
82
|
+
export type DimensionInput = Dimension | FlagRef;
|
|
83
83
|
|
|
84
84
|
// ───────────────────────── 计算产物(组件 data props)─────────────────────────
|
|
85
85
|
|
|
@@ -174,9 +174,9 @@ export interface ScatterData {
|
|
|
174
174
|
}[];
|
|
175
175
|
}
|
|
176
176
|
|
|
177
|
-
/** MetricLine 的 x 轴:experiment 声明的
|
|
177
|
+
/** MetricLine 的 x 轴:experiment 声明的 flag,数值驱动刻度。 */
|
|
178
178
|
export interface LineAxis {
|
|
179
|
-
/**
|
|
179
|
+
/** flag 名。 */
|
|
180
180
|
key: string;
|
|
181
181
|
label: string;
|
|
182
182
|
unit?: string;
|
|
@@ -184,14 +184,14 @@ export interface LineAxis {
|
|
|
184
184
|
|
|
185
185
|
export interface LineData {
|
|
186
186
|
x: LineAxis;
|
|
187
|
-
/** 系列维度名(
|
|
187
|
+
/** 系列维度名(flag 或普通维度)。 */
|
|
188
188
|
series?: string;
|
|
189
189
|
y: MetricColumn;
|
|
190
190
|
rows: {
|
|
191
191
|
/** 点的键(experiment id):每个点 = 一个 experiment 的聚合。 */
|
|
192
192
|
key: string;
|
|
193
193
|
series?: string;
|
|
194
|
-
/**
|
|
194
|
+
/** flag 声明值;未声明或非数值 → null,点不画、注脚报数。 */
|
|
195
195
|
x: number | null;
|
|
196
196
|
/** 已格式化的 x("300 ms");x 为 null 时为空串。 */
|
|
197
197
|
xDisplay: string;
|
|
@@ -600,7 +600,7 @@ describe("Artifacts reporter(writer 薄壳)", () => {
|
|
|
600
600
|
const fresh: EvalResult = {
|
|
601
601
|
id: "algebra/q1",
|
|
602
602
|
experimentId: "compare/bub",
|
|
603
|
-
experiment: { id: "compare/bub",
|
|
603
|
+
experiment: { id: "compare/bub", flags: { style: "concise" } },
|
|
604
604
|
agent: "bub",
|
|
605
605
|
model: "gpt-5.4",
|
|
606
606
|
outcome: "passed",
|
|
@@ -707,7 +707,7 @@ describe("Artifacts reporter(writer 薄壳)", () => {
|
|
|
707
707
|
"experimentId": "compare/bub",
|
|
708
708
|
"experiment": {
|
|
709
709
|
"id": "compare/bub",
|
|
710
|
-
"
|
|
710
|
+
"flags": {
|
|
711
711
|
"style": "concise"
|
|
712
712
|
}
|
|
713
713
|
},
|
package/src/runner/attempt.ts
CHANGED
|
@@ -228,7 +228,7 @@ async function runAttemptBody(
|
|
|
228
228
|
signal,
|
|
229
229
|
model: run.model,
|
|
230
230
|
reasoningEffort: run.reasoningEffort,
|
|
231
|
-
|
|
231
|
+
flags: run.flags,
|
|
232
232
|
sandbox,
|
|
233
233
|
session: createAgentSession(),
|
|
234
234
|
telemetry,
|
|
@@ -270,7 +270,7 @@ async function runAttemptBody(
|
|
|
270
270
|
sandbox,
|
|
271
271
|
model: run.model,
|
|
272
272
|
reasoningEffort: run.reasoningEffort,
|
|
273
|
-
|
|
273
|
+
flags: run.flags,
|
|
274
274
|
signal,
|
|
275
275
|
log,
|
|
276
276
|
judge,
|
|
@@ -442,7 +442,7 @@ async function collectSources(
|
|
|
442
442
|
function experimentRunInfo(run: AgentRun): EvalResult["experiment"] {
|
|
443
443
|
return {
|
|
444
444
|
id: run.experimentId,
|
|
445
|
-
|
|
445
|
+
flags: run.flags,
|
|
446
446
|
runs: run.runs,
|
|
447
447
|
earlyExit: run.earlyExit,
|
|
448
448
|
sandbox: run.sandbox === undefined ? undefined : sandboxLabel(run.sandbox),
|
|
@@ -38,7 +38,7 @@ export async function computeFingerprint(
|
|
|
38
38
|
experimentId: run.experimentId,
|
|
39
39
|
agent: run.agent.name,
|
|
40
40
|
model: run.model,
|
|
41
|
-
|
|
41
|
+
flags: run.flags,
|
|
42
42
|
sandbox: run.sandbox === undefined ? undefined : sandboxLabel(run.sandbox),
|
|
43
43
|
timeoutMs: run.timeoutMs,
|
|
44
44
|
strict: run.strict,
|
|
@@ -85,7 +85,7 @@ describe("toBraintrustEvent", () => {
|
|
|
85
85
|
model: "gpt-5.2",
|
|
86
86
|
attempt: 1,
|
|
87
87
|
outcome: "failed",
|
|
88
|
-
experiment: { id: "compare/codex",
|
|
88
|
+
experiment: { id: "compare/codex", flags: { tape: true } },
|
|
89
89
|
assertions: [{ name: "compiles", severity: "gate", score: 0, passed: false, detail: "tsc failed" }],
|
|
90
90
|
}),
|
|
91
91
|
);
|
|
@@ -96,7 +96,7 @@ describe("toBraintrustEvent", () => {
|
|
|
96
96
|
outcome: "failed",
|
|
97
97
|
model: "gpt-5.2",
|
|
98
98
|
experiment: "compare/codex",
|
|
99
|
-
|
|
99
|
+
flags: { tape: true },
|
|
100
100
|
failedAssertions: [{ name: "compiles", detail: "tsc failed" }],
|
|
101
101
|
});
|
|
102
102
|
expect(event.id).toBe("compare/codex|algebra/quadratic|codex|gpt-5.2|a1");
|
|
@@ -113,7 +113,7 @@ export function Braintrust(config: BraintrustConfig = {}): Reporter {
|
|
|
113
113
|
* - scores:soft 断言按名字记分,gate 断言记在 `gate:` 前缀下 —— 实验 diff 里
|
|
114
114
|
* gate 回归和 soft 分数回归用同一套机制看。重名断言追加 `#n` 消歧,不静默覆盖。
|
|
115
115
|
* - metrics:start/end(Braintrust 由此算时长)+ token 用量 + 估算成本;缺就不写,不编 0。
|
|
116
|
-
* - metadata:身份维度(agent / model / experiment / attempt /
|
|
116
|
+
* - metadata:身份维度(agent / model / experiment / attempt / flags)+ 失败断言明细。
|
|
117
117
|
*/
|
|
118
118
|
export function toBraintrustEvent(result: EvalResult): BraintrustLogEvent {
|
|
119
119
|
const scores: Record<string, number> = {};
|
|
@@ -150,8 +150,8 @@ export function toBraintrustEvent(result: EvalResult): BraintrustLogEvent {
|
|
|
150
150
|
};
|
|
151
151
|
if (result.model !== undefined) metadata.model = result.model;
|
|
152
152
|
if (result.experimentId !== undefined) metadata.experiment = result.experimentId;
|
|
153
|
-
if (result.experiment?.
|
|
154
|
-
metadata.
|
|
153
|
+
if (result.experiment?.flags && Object.keys(result.experiment.flags).length > 0) {
|
|
154
|
+
metadata.flags = result.experiment.flags;
|
|
155
155
|
}
|
|
156
156
|
if (result.skipReason !== undefined) metadata.skipReason = result.skipReason;
|
|
157
157
|
const failed = result.assertions
|
package/src/runner/run.ts
CHANGED
|
@@ -104,7 +104,7 @@ export async function runEvals(opts: RunOptions): Promise<RunSummary> {
|
|
|
104
104
|
if (run.experimentId && priorRunKeys.has(`${run.experimentId}|${evalDef.id}`)) continue;
|
|
105
105
|
// key 标识「同一个运行配置下的同一条 eval」,earlyExit 的跳过/abort 只应作用于
|
|
106
106
|
// 同 key 的重试轮。experimentId 必须进 key:两个实验可以同 agent 同 model、只差
|
|
107
|
-
//
|
|
107
|
+
// flags(feature A/B 正是这种形状),漏掉它会让先过的实验把其它实验的同名 eval
|
|
108
108
|
// 整个跳掉——花了钱还丢结果。
|
|
109
109
|
const key = `${run.experimentId ?? ""}|${run.agent.name}|${run.model ?? ""}|${evalDef.id}`;
|
|
110
110
|
attempts.push({
|
package/src/runner/types.ts
CHANGED
|
@@ -12,7 +12,7 @@ import type { TestContext } from "../context/types.ts";
|
|
|
12
12
|
|
|
13
13
|
export interface ExperimentRunInfo {
|
|
14
14
|
id?: string;
|
|
15
|
-
|
|
15
|
+
flags?: Record<string, unknown>;
|
|
16
16
|
runs?: number;
|
|
17
17
|
earlyExit?: boolean;
|
|
18
18
|
sandbox?: string;
|
|
@@ -61,7 +61,7 @@ export interface EvalResult {
|
|
|
61
61
|
/** `summary.json` 的格式标记;把 niceeval 报告和其它工具的同名文件区分开。 */
|
|
62
62
|
export const RESULTS_FORMAT = "niceeval.results";
|
|
63
63
|
/** 结果格式版本,只在破坏兼容读取时递增;读取器只认相同版本,缺失按 1。见 docs/results-format.md。 */
|
|
64
|
-
export const RESULTS_SCHEMA_VERSION =
|
|
64
|
+
export const RESULTS_SCHEMA_VERSION = 3;
|
|
65
65
|
|
|
66
66
|
export interface RunSummary {
|
|
67
67
|
/** 恒为 "niceeval.results";和 schemaVersion、producer 一起构成持久化契约,永不移动或改名。 */
|
|
@@ -95,11 +95,11 @@ export interface RunSummary {
|
|
|
95
95
|
snapshots?: Record<string, { startedAt?: string; knownEvalIds?: string[] }>;
|
|
96
96
|
}
|
|
97
97
|
|
|
98
|
-
/** onRunStart 的运行规模:去重后 eval 数 × 配置(
|
|
98
|
+
/** onRunStart 的运行规模:去重后 eval 数 × 配置(agent×model×flags)数 → 总运行(attempt)数。 */
|
|
99
99
|
export interface RunShape {
|
|
100
100
|
/** 去重后实际要跑的 eval 数(= evals.length)。 */
|
|
101
101
|
evals: number;
|
|
102
|
-
/** (agent, model,
|
|
102
|
+
/** (agent, model, flags) 配置组合数;compare 多 agent 时 > 1。 */
|
|
103
103
|
configs: number;
|
|
104
104
|
/** 总 attempt 数(evals × configs × runs);逐行输出与汇总计数都按它。 */
|
|
105
105
|
totalRuns: number;
|
|
@@ -174,8 +174,8 @@ export interface ExperimentDef {
|
|
|
174
174
|
model?: string;
|
|
175
175
|
/** 模型推理努力程度(如 "low"/"medium"/"high",取值由具体模型/adapter 决定);省略=用 agent 原生默认。经 ctx.reasoningEffort 透给 adapter 与 eval。 */
|
|
176
176
|
reasoningEffort?: string;
|
|
177
|
-
/**
|
|
178
|
-
|
|
177
|
+
/** 实验条件(A/B 里的 feature flag),由实验文件声明;经 ctx.flags 透传给 adapter、t.flags 暴露给 eval。 */
|
|
178
|
+
flags?: Record<string, unknown>;
|
|
179
179
|
/** 同一 eval 重复跑几次(结果各计一条 attempt);省略/CLI `--runs` 覆盖时默认 1。 */
|
|
180
180
|
runs?: number;
|
|
181
181
|
/** 一次重复(runs > 1)里某次 attempt 失败后是否跳过剩余重复;省略默认 true(提前退出省钱)。 */
|
|
@@ -267,12 +267,12 @@ export function runWho(run: Pick<AgentRun, "agent" | "model" | "experimentId">):
|
|
|
267
267
|
return run.model ? `${run.agent.name}/${run.model}` : run.agent.name;
|
|
268
268
|
}
|
|
269
269
|
|
|
270
|
-
/** 一个 (agent, model,
|
|
270
|
+
/** 一个 (agent, model, flags) 的运行配置 —— 由 CLI / 实验展开。 */
|
|
271
271
|
export interface AgentRun {
|
|
272
272
|
agent: Agent;
|
|
273
273
|
model?: string;
|
|
274
274
|
reasoningEffort?: string;
|
|
275
|
-
|
|
275
|
+
flags: Record<string, unknown>;
|
|
276
276
|
runs: number;
|
|
277
277
|
earlyExit: boolean;
|
|
278
278
|
sandbox?: SandboxOption;
|
package/src/view/app/i18n.ts
CHANGED
|
@@ -58,7 +58,7 @@ export type MessageKey =
|
|
|
58
58
|
| "detail.rawSample"
|
|
59
59
|
| "detail.rawNote"
|
|
60
60
|
| "config.experiment"
|
|
61
|
-
| "config.
|
|
61
|
+
| "config.flagsNone"
|
|
62
62
|
| "config.default"
|
|
63
63
|
| "config.none"
|
|
64
64
|
| "config.notApplicable"
|
|
@@ -177,7 +177,7 @@ const dictionaries: Record<Locale, Dictionary> = {
|
|
|
177
177
|
"detail.rawSample": "Raw sample result",
|
|
178
178
|
"detail.rawNote": "debug JSON, defaults to first error/failure when available",
|
|
179
179
|
"config.experiment": "experiment",
|
|
180
|
-
"config.
|
|
180
|
+
"config.flagsNone": "none",
|
|
181
181
|
"config.default": "default",
|
|
182
182
|
"config.none": "none",
|
|
183
183
|
"config.notApplicable": "n/a",
|
|
@@ -293,7 +293,7 @@ const dictionaries: Record<Locale, Dictionary> = {
|
|
|
293
293
|
"detail.rawSample": "原始样例结果",
|
|
294
294
|
"detail.rawNote": "调试 JSON,默认选择第一条错误/失败",
|
|
295
295
|
"config.experiment": "实验",
|
|
296
|
-
"config.
|
|
296
|
+
"config.flagsNone": "无",
|
|
297
297
|
"config.default": "默认",
|
|
298
298
|
"config.none": "无",
|
|
299
299
|
"config.notApplicable": "不适用",
|
package/src/view/app/lib/rows.ts
CHANGED
|
@@ -127,9 +127,9 @@ export function valueFor(row: ViewRow, key: SortKey): string | number | null {
|
|
|
127
127
|
|
|
128
128
|
export function configChips(row: ViewRow, t: T): [string, ReactNode][] {
|
|
129
129
|
const exp = row.experiment || {};
|
|
130
|
-
const
|
|
131
|
-
? Object.entries(exp.
|
|
132
|
-
: t("config.
|
|
130
|
+
const flags = exp.flags && Object.keys(exp.flags).length
|
|
131
|
+
? Object.entries(exp.flags).map(([k, v]) => k + "=" + formatConfigValue(v)).join(", ")
|
|
132
|
+
: t("config.flagsNone");
|
|
133
133
|
return [
|
|
134
134
|
[t("config.experiment"), row.synthetic ? row.label : row.experimentId],
|
|
135
135
|
[t("table.model"), row.model || t("config.default")],
|
|
@@ -138,6 +138,6 @@ export function configChips(row: ViewRow, t: T): [string, ReactNode][] {
|
|
|
138
138
|
["earlyExit", exp.earlyExit === undefined ? t("config.notApplicable") : String(exp.earlyExit)],
|
|
139
139
|
["sandbox", exp.sandbox || t("config.default")],
|
|
140
140
|
["budget", exp.budget === undefined ? t("config.none") : "$" + exp.budget],
|
|
141
|
-
["
|
|
141
|
+
["flags", flags],
|
|
142
142
|
];
|
|
143
143
|
}
|