niceeval 0.5.1 → 0.5.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (64) hide show
  1. package/docs-site/zh/concepts/adapter.mdx +6 -6
  2. package/docs-site/zh/concepts/experiment.mdx +6 -6
  3. package/docs-site/zh/concepts/overview.mdx +1 -1
  4. package/docs-site/zh/concepts/tier.mdx +6 -6
  5. package/docs-site/zh/guides/ci-integration.mdx +2 -0
  6. package/docs-site/zh/guides/connect-your-agent.mdx +7 -7
  7. package/docs-site/zh/guides/custom-reports.mdx +7 -7
  8. package/docs-site/zh/guides/experiments.mdx +3 -3
  9. package/docs-site/zh/guides/publish-report.mdx +91 -0
  10. package/docs-site/zh/guides/report-components.mdx +4 -4
  11. package/docs-site/zh/guides/reporters.mdx +1 -1
  12. package/docs-site/zh/guides/results-data.mdx +1 -1
  13. package/docs-site/zh/guides/runner.mdx +1 -1
  14. package/docs-site/zh/guides/sandbox-agent.mdx +2 -2
  15. package/docs-site/zh/guides/viewing-results.mdx +3 -19
  16. package/docs-site/zh/guides/write-experiment.mdx +9 -9
  17. package/docs-site/zh/guides/write-send.mdx +6 -6
  18. package/docs-site/zh/index.mdx +1 -1
  19. package/docs-site/zh/introduction.mdx +1 -1
  20. package/docs-site/zh/reference/builtin-agents.mdx +4 -0
  21. package/docs-site/zh/reference/cli.mdx +10 -6
  22. package/docs-site/zh/reference/define-agent.mdx +7 -3
  23. package/docs-site/zh/reference/define-config.mdx +3 -1
  24. package/docs-site/zh/reference/define-eval.mdx +9 -3
  25. package/docs-site/zh/reference/events.mdx +4 -0
  26. package/docs-site/zh/reference/expect.mdx +4 -0
  27. package/package.json +3 -2
  28. package/src/agents/ai-sdk.test.ts +1 -1
  29. package/src/agents/ai-sdk.ts +2 -2
  30. package/src/agents/claude-code.ts +1 -1
  31. package/src/agents/codex.ts +2 -2
  32. package/src/agents/streaming.test.ts +1 -1
  33. package/src/agents/types.ts +2 -2
  34. package/src/agents/ui-message-stream.test.ts +1 -1
  35. package/src/cli.ts +53 -15
  36. package/src/context/context.test.ts +1 -1
  37. package/src/context/context.ts +3 -3
  38. package/src/context/session.ts +2 -2
  39. package/src/context/types.ts +2 -2
  40. package/src/i18n/en.ts +2 -2
  41. package/src/i18n/zh-CN.ts +2 -2
  42. package/src/report/aggregate.ts +16 -16
  43. package/src/report/components.tsx +1 -1
  44. package/src/report/compute.ts +7 -7
  45. package/src/report/dual-face.test.tsx +1 -1
  46. package/src/report/{param.ts → flag.ts} +6 -6
  47. package/src/report/index.ts +3 -3
  48. package/src/report/react/MetricLine.tsx +1 -1
  49. package/src/report/react/fixtures.ts +1 -1
  50. package/src/report/report.test.ts +17 -17
  51. package/src/report/types.ts +11 -11
  52. package/src/results/results.test.ts +2 -2
  53. package/src/runner/attempt.ts +3 -3
  54. package/src/runner/fingerprint.ts +1 -1
  55. package/src/runner/reporters/braintrust.test.ts +2 -2
  56. package/src/runner/reporters/braintrust.ts +3 -3
  57. package/src/runner/run.ts +1 -1
  58. package/src/runner/types.ts +8 -8
  59. package/src/view/app/css.d.ts +3 -0
  60. package/src/view/app/i18n.ts +3 -3
  61. package/src/view/app/lib/rows.ts +4 -4
  62. package/src/view/client-dist/app.js +2 -2
  63. package/src/view/data.test.ts +25 -1
  64. package/src/view/data.ts +27 -3
@@ -18,15 +18,15 @@ import type {
18
18
  Metric,
19
19
  MetricCell,
20
20
  MetricColumn,
21
- ParamRef,
21
+ FlagRef,
22
22
  } from "./types.ts";
23
23
  import { formatMetricValue } from "./format.ts";
24
24
 
25
25
  // 复合键分隔符:NUL 不会出现在 eval id / experimentId / ISO 时间里,拼接键不会串味
26
26
  const KEY_SEP = "\u0000";
27
27
 
28
- /** param 未声明时的组名:不猜,如实归一组。 */
29
- export const PARAM_UNSET = "(unset)";
28
+ /** flag 未声明时的组名:不猜,如实归一组。 */
29
+ export const FLAG_UNSET = "(unset)";
30
30
 
31
31
  /** 计算函数的第一参:选集(warnings 随行)或手工挑的快照数组(没有挑选过程,自然无警告)。 */
32
32
  export type SnapshotsInput = Selection | Snapshot[];
@@ -105,33 +105,33 @@ export function evalGroupOf(id: string): string {
105
105
  return slash === -1 ? id : id.slice(0, slash);
106
106
  }
107
107
 
108
- function isParamRef(dimension: DimensionInput): dimension is ParamRef {
109
- return typeof dimension === "object" && "kind" in dimension && dimension.kind === "param";
108
+ function isFlagRef(dimension: DimensionInput): dimension is FlagRef {
109
+ return typeof dimension === "object" && "kind" in dimension && dimension.kind === "flag";
110
110
  }
111
111
 
112
- /** experiment 声明的 params(经 runner 原样透传进持久化字段 ExperimentRunInfo.params)。 */
113
- function paramsOf(attempt: AttemptHandle): Record<string, unknown> | undefined {
114
- return attempt.result.experiment?.params;
112
+ /** experiment 声明的 flags(经 runner 原样透传进持久化字段 ExperimentRunInfo.flags)。 */
113
+ function flagsOf(attempt: AttemptHandle): Record<string, unknown> | undefined {
114
+ return attempt.result.experiment?.flags;
115
115
  }
116
116
 
117
- /** param 声明值 → 组标签:label 函数优先,其余 String();未声明 → PARAM_UNSET。 */
118
- export function paramGroupKey(ref: ParamRef, item: Item): string {
119
- const value = paramsOf(item.attempt)?.[ref.name];
120
- if (value === undefined) return PARAM_UNSET;
117
+ /** flag 声明值 → 组标签:label 函数优先,其余 String();未声明 → FLAG_UNSET。 */
118
+ export function flagGroupKey(ref: FlagRef, item: Item): string {
119
+ const value = flagsOf(item.attempt)?.[ref.name];
120
+ if (value === undefined) return FLAG_UNSET;
121
121
  // 持久化字段是 Record<string, unknown>;声明侧的合法值就是这三种标量
122
122
  if (typeof ref.label === "function") return ref.label(value as string | number | boolean);
123
123
  return String(value);
124
124
  }
125
125
 
126
- /** param 作轴:要求数值;未声明或非数值 → null(点不画,注脚报数)。 */
127
- export function paramAxisValue(ref: ParamRef, item: Item): number | null {
128
- const value = paramsOf(item.attempt)?.[ref.name];
126
+ /** flag 作轴:要求数值;未声明或非数值 → null(点不画,注脚报数)。 */
127
+ export function flagAxisValue(ref: FlagRef, item: Item): number | null {
128
+ const value = flagsOf(item.attempt)?.[ref.name];
129
129
  return typeof value === "number" ? value : null;
130
130
  }
131
131
 
132
132
  export function dimensionKey(dimension: DimensionInput, item: Item): string {
133
133
  if (typeof dimension !== "string") {
134
- if (isParamRef(dimension)) return paramGroupKey(dimension, item);
134
+ if (isFlagRef(dimension)) return flagGroupKey(dimension, item);
135
135
  return dimension.of(item.attempt);
136
136
  }
137
137
  const result = item.attempt.result;
@@ -159,7 +159,7 @@ export const MetricScatter = Object.assign(
159
159
  );
160
160
  MetricScatter.displayName = "MetricScatter";
161
161
 
162
- /** 趋势线:x 是 experiment 声明的 param(param()),同系列按 x 排序连线。 */
162
+ /** 趋势线:x 是 experiment 声明的 flag(flag()),同系列按 x 排序连线。 */
163
163
  export const MetricLine = Object.assign(
164
164
  defineComponent<MetricLineProps>({
165
165
  web: (props) => <MetricLineWeb {...props} />,
@@ -19,7 +19,7 @@ import type {
19
19
  Metric,
20
20
  MetricCell,
21
21
  OverviewData,
22
- ParamRef,
22
+ FlagRef,
23
23
  ScatterData,
24
24
  ScoreboardData,
25
25
  TableData,
@@ -39,7 +39,7 @@ import {
39
39
  experimentIdOf,
40
40
  filterItems,
41
41
  groupItems,
42
- paramAxisValue,
42
+ flagAxisValue,
43
43
  resolveInput,
44
44
  snapshotKeyOf,
45
45
  toColumn,
@@ -52,7 +52,7 @@ import { formatMetricValue, formatPlainNumber } from "./format.ts";
52
52
  // ───────────────────────── MetricTable.data ─────────────────────────
53
53
 
54
54
  export interface TableDataOptions<M extends readonly Metric[]> {
55
- /** 行维度(内置 / 自定义 / param())。 */
55
+ /** 行维度(内置 / 自定义 / flag())。 */
56
56
  rows: DimensionInput;
57
57
  /** 每列一个指标;列键 = metric.name 的字面量,拼错编译不过。 */
58
58
  columns: M;
@@ -289,10 +289,10 @@ export async function scatterData(input: SnapshotsInput, opts: ScatterDataOption
289
289
  // ───────────────────────── MetricLine.data ─────────────────────────
290
290
 
291
291
  export interface LineDataOptions {
292
- /** x 轴:experiment 声明的 param(数值),不解析 experiment 命名。 */
293
- x: ParamRef;
292
+ /** x 轴:experiment 声明的 flag(数值),不解析 experiment 命名。 */
293
+ x: FlagRef;
294
294
  y: Metric;
295
- /** 可选:每个系列一条线(param 或普通维度);省略 = 单系列。 */
295
+ /** 可选:每个系列一条线(flag 或普通维度);省略 = 单系列。 */
296
296
  series?: DimensionInput;
297
297
  }
298
298
 
@@ -303,7 +303,7 @@ export async function lineData(input: SnapshotsInput, opts: LineDataOptions): Pr
303
303
  const groups = groupItems(items, "experiment");
304
304
  const rows: LineData["rows"] = [];
305
305
  for (const [key, group] of groups) {
306
- const x = paramAxisValue(opts.x, group[0]); // param 是 experiment 级声明,组内一致
306
+ const x = flagAxisValue(opts.x, group[0]); // flag 是 experiment 级声明,组内一致
307
307
  rows.push({
308
308
  key,
309
309
  series: opts.series ? dimensionKey(opts.series, group[0]) : undefined,
@@ -228,7 +228,7 @@ describe("MetricLine 双面", () => {
228
228
  `);
229
229
  });
230
230
 
231
- it("两面同口径:未声明 param 的点两面都不画、注脚同数", () => {
231
+ it("两面同口径:未声明 flag 的点两面都不画、注脚同数", () => {
232
232
  expect(html).toContain("1 point missing data");
233
233
  expect(term).toContain("1 point missing data");
234
234
  expect(html).not.toContain('data-key="ultra/legacy"'); // 注脚 title 如实列缺数据的点,但不画
@@ -1,18 +1,18 @@
1
- // param():把 experiment 声明的 params 当维度或轴(docs/reports.md「params 与新摆法」)。
1
+ // flag():把 experiment 声明的 flags 当维度或轴(docs/reports.md「flags 与新摆法」)。
2
2
  // 变量来自配置,不来自命名 —— 报告不解析 experiment id 字符串抠变量。
3
3
 
4
- import type { ParamRef } from "./types.ts";
4
+ import type { FlagRef } from "./types.ts";
5
5
 
6
- export function param(
6
+ export function flag(
7
7
  name: string,
8
8
  opts?: {
9
9
  /** 组标签 / 轴标签;函数形态把声明值折成组名(如 `(v) => \`${v} agents\``)。 */
10
10
  label?: string | ((value: string | number | boolean) => string);
11
11
  unit?: string;
12
12
  },
13
- ): ParamRef {
13
+ ): FlagRef {
14
14
  if (typeof name !== "string" || name.length === 0) {
15
- throw new Error("param: name must be a non-empty string (the key declared in the experiment's params).");
15
+ throw new Error("flag: name must be a non-empty string (the key declared in the experiment's flags).");
16
16
  }
17
- return { kind: "param", name, label: opts?.label, unit: opts?.unit };
17
+ return { kind: "flag", name, label: opts?.label, unit: opts?.unit };
18
18
  }
@@ -8,9 +8,9 @@
8
8
  // ./web.ts,只有那一侧 import react-dom。写报告文件的项目要装 react(.tsx 编译产物
9
9
  // import react/jsx-runtime)。
10
10
 
11
- // 指标与 param
11
+ // 指标与 flag
12
12
  export { defineMetric, passRate, examScore, durationMs, tokens, costUSD } from "./metrics.ts";
13
- export { param } from "./param.ts";
13
+ export { flag } from "./flag.ts";
14
14
 
15
15
  // 报告基座与双面组件基座
16
16
  export { defineReport, isReportDefinition, renderReportToText } from "./report.ts";
@@ -84,7 +84,7 @@ export type {
84
84
  MetricCell,
85
85
  MetricColumn,
86
86
  OverviewData,
87
- ParamRef,
87
+ FlagRef,
88
88
  ScatterData,
89
89
  ScoreboardData,
90
90
  SelectionWarning,
@@ -1,4 +1,4 @@
1
- // MetricLine:趋势线——x 是 experiment 声明的 param(有序变量),每个系列一条线。
1
+ // MetricLine:趋势线——x 是 experiment 声明的 flag(有序变量),每个系列一条线。
2
2
  // 与 MetricScatter 的分工:scatter 的两轴都是测出来的指标(找 frontier),
3
3
  // line 的 x 是你配置的变量(看趋势)。x 轴正常升序;y 轴向随 better;
4
4
  // x 或 y 缺数据的点不画,注脚如实报数;每个点带 <title> hover,可经 pointHref 下钻。
@@ -207,7 +207,7 @@ export const lineData: LineData = {
207
207
  y: { value: 0.7, display: "70%", samples: 6, total: 6, refs: [] },
208
208
  },
209
209
  {
210
- // 未声明 param 的 experiment:作轴不画点,注脚报数
210
+ // 未声明 flag 的 experiment:作轴不画点,注脚报数
211
211
  key: "ultra/legacy",
212
212
  series: "1 agents",
213
213
  x: null,
@@ -1,7 +1,7 @@
1
1
  // niceeval/report 计算层的单元测试:全部用内存 fake(Snapshot / AttemptHandle 按
2
2
  // niceeval/results 的读取契约手工构造),专门覆盖 docs/reports.md 点名的坑 ——
3
3
  // 两级聚合 vs 平铺、pass@k、examScore 空真、skipped 稀释、scoreboard 固定分母与
4
- // 最长前缀、scatter/delta 的 null 语义、快照键对比、param 维度与轴、
4
+ // 最长前缀、scatter/delta 的 null 语义、快照键对比、flag 维度与轴、
5
5
  // cases 的 redact/truncated、身份键去重、Selection warnings 随行。
6
6
 
7
7
  import { describe, expect, it } from "vitest";
@@ -10,7 +10,7 @@ import type { AssertionResult, EvalResult, ResultOutcome, RunSummary } from "../
10
10
  import type { AttemptHandle, RunDir, Selection, SelectionWarning, Snapshot } from "../results/index.ts";
11
11
  import type { Dimension, MetricCell } from "./types.ts";
12
12
  import { costUSD, defineMetric, durationMs, examScore, passRate, tokens } from "./metrics.ts";
13
- import { param } from "./param.ts";
13
+ import { flag } from "./flag.ts";
14
14
  import { formatMetricValue } from "./format.ts";
15
15
  import {
16
16
  CaseList,
@@ -465,22 +465,22 @@ describe("MetricScatter.data", () => {
465
465
  });
466
466
  });
467
467
 
468
- // ───────────────────────── param():维度与轴 ─────────────────────────
468
+ // ───────────────────────── flag():维度与轴 ─────────────────────────
469
469
 
470
- describe("param()", () => {
471
- const withParams = (id: string, params: Record<string, unknown> | undefined, outcome: ResultOutcome) =>
470
+ describe("flag()", () => {
471
+ const withFlags = (id: string, flags: Record<string, unknown> | undefined, outcome: ResultOutcome) =>
472
472
  snap({
473
473
  experimentId: id,
474
- results: [res("A", outcome, { experimentId: id, experiment: { id, params } })],
474
+ results: [res("A", outcome, { experimentId: id, experiment: { id, flags } })],
475
475
  });
476
476
 
477
- it("MetricLine.data:x 收 param、按 experiment 聚合;未声明的作轴 x=null 报数", async () => {
478
- const s1 = withParams("ultra/lat-100", { latencyMs: 100, agents: 1 }, "passed");
479
- const s2 = withParams("ultra/lat-300", { latencyMs: 300, agents: 1 }, "failed");
480
- const legacy = withParams("ultra/legacy", undefined, "passed");
477
+ it("MetricLine.data:x 收 flag、按 experiment 聚合;未声明的作轴 x=null 报数", async () => {
478
+ const s1 = withFlags("ultra/lat-100", { latencyMs: 100, agents: 1 }, "passed");
479
+ const s2 = withFlags("ultra/lat-300", { latencyMs: 300, agents: 1 }, "failed");
480
+ const legacy = withFlags("ultra/legacy", undefined, "passed");
481
481
  const data = await MetricLine.data([s1, s2, legacy], {
482
- x: param("latencyMs", { label: "Simulated latency", unit: "ms" }),
483
- series: param("agents", { label: (v) => `${v} agents` }),
482
+ x: flag("latencyMs", { label: "Simulated latency", unit: "ms" }),
483
+ series: flag("agents", { label: (v) => `${v} agents` }),
484
484
  y: passRate,
485
485
  });
486
486
  expect(data.x).toEqual({ key: "latencyMs", label: "Simulated latency", unit: "ms" });
@@ -493,18 +493,18 @@ describe("param()", () => {
493
493
  expect(p100.series).toBe("1 agents");
494
494
  expect(p100.y.value).toBe(1);
495
495
 
496
- // 未声明 param 的 experiment 不猜:作轴 x=null(组件不画、注脚报数),分组归 (unset)
496
+ // 未声明 flag 的 experiment 不猜:作轴 x=null(组件不画、注脚报数),分组归 (unset)
497
497
  const legacyRow = data.rows.find((r) => r.key === "ultra/legacy")!;
498
498
  expect(legacyRow.x).toBeNull();
499
499
  expect(legacyRow.xDisplay).toBe("");
500
500
  expect(legacyRow.series).toBe("(unset)");
501
501
  });
502
502
 
503
- it("param 当维度用:按声明值分组,label 函数折组名", async () => {
504
- const s1 = withParams("exp/a", { agents: 1 }, "passed");
505
- const s2 = withParams("exp/b", { agents: 16 }, "failed");
503
+ it("flag 当维度用:按声明值分组,label 函数折组名", async () => {
504
+ const s1 = withFlags("exp/a", { agents: 1 }, "passed");
505
+ const s2 = withFlags("exp/b", { agents: 16 }, "failed");
506
506
  const data = await MetricTable.data([s1, s2], {
507
- rows: param("agents", { label: (v) => `${v} agents` }),
507
+ rows: flag("agents", { label: (v) => `${v} agents` }),
508
508
  columns: [passRate],
509
509
  });
510
510
  expect(data.dimension).toBe("agents");
@@ -1,4 +1,4 @@
1
- // niceeval/report 的公开类型:指标(Metric)、维度(Dimension / param())与计算函数
1
+ // niceeval/report 的公开类型:指标(Metric)、维度(Dimension / flag())与计算函数
2
2
  // 产物(即组件的 data props)。数据契约照 docs/reports.md「计算函数与数据契约」;
3
3
  // 这些不是持久化格式,没有 format / schemaVersion 信封,兼容性跟随 npm 版本。
4
4
 
@@ -66,20 +66,20 @@ export type Dimension =
66
66
  | { name: string; of: (attempt: AttemptHandle) => string };
67
67
 
68
68
  /**
69
- * param() 的产物:把 experiment 声明的 params 当维度(series / rows / columns / points
69
+ * flag() 的产物:把 experiment 声明的 flags 当维度(series / rows / columns / points
70
70
  * 槽,按声明值分组)或轴(MetricLine 的 x 槽,要求数值并驱动刻度)。
71
- * 未声明该 param 的 experiment 不猜:分组如实归「(unset)」,作轴不画点、注脚报数。
71
+ * 未声明该 flag 的 experiment 不猜:分组如实归「(unset)」,作轴不画点、注脚报数。
72
72
  */
73
- export interface ParamRef {
74
- readonly kind: "param";
73
+ export interface FlagRef {
74
+ readonly kind: "flag";
75
75
  readonly name: string;
76
76
  /** 组标签 / 轴标签;函数形态把声明值折成组名(如 `(v) => \`${v} agents\``)。 */
77
77
  readonly label?: string | ((value: string | number | boolean) => string);
78
78
  readonly unit?: string;
79
79
  }
80
80
 
81
- /** 维度槽的输入:内置/自定义维度,或 experiment 声明的 param。 */
82
- export type DimensionInput = Dimension | ParamRef;
81
+ /** 维度槽的输入:内置/自定义维度,或 experiment 声明的 flag。 */
82
+ export type DimensionInput = Dimension | FlagRef;
83
83
 
84
84
  // ───────────────────────── 计算产物(组件 data props)─────────────────────────
85
85
 
@@ -174,9 +174,9 @@ export interface ScatterData {
174
174
  }[];
175
175
  }
176
176
 
177
- /** MetricLine 的 x 轴:experiment 声明的 param,数值驱动刻度。 */
177
+ /** MetricLine 的 x 轴:experiment 声明的 flag,数值驱动刻度。 */
178
178
  export interface LineAxis {
179
- /** param 名。 */
179
+ /** flag 名。 */
180
180
  key: string;
181
181
  label: string;
182
182
  unit?: string;
@@ -184,14 +184,14 @@ export interface LineAxis {
184
184
 
185
185
  export interface LineData {
186
186
  x: LineAxis;
187
- /** 系列维度名(param 或普通维度)。 */
187
+ /** 系列维度名(flag 或普通维度)。 */
188
188
  series?: string;
189
189
  y: MetricColumn;
190
190
  rows: {
191
191
  /** 点的键(experiment id):每个点 = 一个 experiment 的聚合。 */
192
192
  key: string;
193
193
  series?: string;
194
- /** param 声明值;未声明或非数值 → null,点不画、注脚报数。 */
194
+ /** flag 声明值;未声明或非数值 → null,点不画、注脚报数。 */
195
195
  x: number | null;
196
196
  /** 已格式化的 x("300 ms");x 为 null 时为空串。 */
197
197
  xDisplay: string;
@@ -600,7 +600,7 @@ describe("Artifacts reporter(writer 薄壳)", () => {
600
600
  const fresh: EvalResult = {
601
601
  id: "algebra/q1",
602
602
  experimentId: "compare/bub",
603
- experiment: { id: "compare/bub", params: { style: "concise" } },
603
+ experiment: { id: "compare/bub", flags: { style: "concise" } },
604
604
  agent: "bub",
605
605
  model: "gpt-5.4",
606
606
  outcome: "passed",
@@ -707,7 +707,7 @@ describe("Artifacts reporter(writer 薄壳)", () => {
707
707
  "experimentId": "compare/bub",
708
708
  "experiment": {
709
709
  "id": "compare/bub",
710
- "params": {
710
+ "flags": {
711
711
  "style": "concise"
712
712
  }
713
713
  },
@@ -228,7 +228,7 @@ async function runAttemptBody(
228
228
  signal,
229
229
  model: run.model,
230
230
  reasoningEffort: run.reasoningEffort,
231
- params: run.params,
231
+ flags: run.flags,
232
232
  sandbox,
233
233
  session: createAgentSession(),
234
234
  telemetry,
@@ -270,7 +270,7 @@ async function runAttemptBody(
270
270
  sandbox,
271
271
  model: run.model,
272
272
  reasoningEffort: run.reasoningEffort,
273
- params: run.params,
273
+ flags: run.flags,
274
274
  signal,
275
275
  log,
276
276
  judge,
@@ -442,7 +442,7 @@ async function collectSources(
442
442
  function experimentRunInfo(run: AgentRun): EvalResult["experiment"] {
443
443
  return {
444
444
  id: run.experimentId,
445
- params: run.params,
445
+ flags: run.flags,
446
446
  runs: run.runs,
447
447
  earlyExit: run.earlyExit,
448
448
  sandbox: run.sandbox === undefined ? undefined : sandboxLabel(run.sandbox),
@@ -38,7 +38,7 @@ export async function computeFingerprint(
38
38
  experimentId: run.experimentId,
39
39
  agent: run.agent.name,
40
40
  model: run.model,
41
- params: run.params,
41
+ flags: run.flags,
42
42
  sandbox: run.sandbox === undefined ? undefined : sandboxLabel(run.sandbox),
43
43
  timeoutMs: run.timeoutMs,
44
44
  strict: run.strict,
@@ -85,7 +85,7 @@ describe("toBraintrustEvent", () => {
85
85
  model: "gpt-5.2",
86
86
  attempt: 1,
87
87
  outcome: "failed",
88
- experiment: { id: "compare/codex", params: { tape: true } },
88
+ experiment: { id: "compare/codex", flags: { tape: true } },
89
89
  assertions: [{ name: "compiles", severity: "gate", score: 0, passed: false, detail: "tsc failed" }],
90
90
  }),
91
91
  );
@@ -96,7 +96,7 @@ describe("toBraintrustEvent", () => {
96
96
  outcome: "failed",
97
97
  model: "gpt-5.2",
98
98
  experiment: "compare/codex",
99
- params: { tape: true },
99
+ flags: { tape: true },
100
100
  failedAssertions: [{ name: "compiles", detail: "tsc failed" }],
101
101
  });
102
102
  expect(event.id).toBe("compare/codex|algebra/quadratic|codex|gpt-5.2|a1");
@@ -113,7 +113,7 @@ export function Braintrust(config: BraintrustConfig = {}): Reporter {
113
113
  * - scores:soft 断言按名字记分,gate 断言记在 `gate:` 前缀下 —— 实验 diff 里
114
114
  * gate 回归和 soft 分数回归用同一套机制看。重名断言追加 `#n` 消歧,不静默覆盖。
115
115
  * - metrics:start/end(Braintrust 由此算时长)+ token 用量 + 估算成本;缺就不写,不编 0。
116
- * - metadata:身份维度(agent / model / experiment / attempt / params)+ 失败断言明细。
116
+ * - metadata:身份维度(agent / model / experiment / attempt / flags)+ 失败断言明细。
117
117
  */
118
118
  export function toBraintrustEvent(result: EvalResult): BraintrustLogEvent {
119
119
  const scores: Record<string, number> = {};
@@ -150,8 +150,8 @@ export function toBraintrustEvent(result: EvalResult): BraintrustLogEvent {
150
150
  };
151
151
  if (result.model !== undefined) metadata.model = result.model;
152
152
  if (result.experimentId !== undefined) metadata.experiment = result.experimentId;
153
- if (result.experiment?.params && Object.keys(result.experiment.params).length > 0) {
154
- metadata.params = result.experiment.params;
153
+ if (result.experiment?.flags && Object.keys(result.experiment.flags).length > 0) {
154
+ metadata.flags = result.experiment.flags;
155
155
  }
156
156
  if (result.skipReason !== undefined) metadata.skipReason = result.skipReason;
157
157
  const failed = result.assertions
package/src/runner/run.ts CHANGED
@@ -104,7 +104,7 @@ export async function runEvals(opts: RunOptions): Promise<RunSummary> {
104
104
  if (run.experimentId && priorRunKeys.has(`${run.experimentId}|${evalDef.id}`)) continue;
105
105
  // key 标识「同一个运行配置下的同一条 eval」,earlyExit 的跳过/abort 只应作用于
106
106
  // 同 key 的重试轮。experimentId 必须进 key:两个实验可以同 agent 同 model、只差
107
- // params(feature A/B 正是这种形状),漏掉它会让先过的实验把其它实验的同名 eval
107
+ // flags(feature A/B 正是这种形状),漏掉它会让先过的实验把其它实验的同名 eval
108
108
  // 整个跳掉——花了钱还丢结果。
109
109
  const key = `${run.experimentId ?? ""}|${run.agent.name}|${run.model ?? ""}|${evalDef.id}`;
110
110
  attempts.push({
@@ -12,7 +12,7 @@ import type { TestContext } from "../context/types.ts";
12
12
 
13
13
  export interface ExperimentRunInfo {
14
14
  id?: string;
15
- params?: Record<string, unknown>;
15
+ flags?: Record<string, unknown>;
16
16
  runs?: number;
17
17
  earlyExit?: boolean;
18
18
  sandbox?: string;
@@ -61,7 +61,7 @@ export interface EvalResult {
61
61
  /** `summary.json` 的格式标记;把 niceeval 报告和其它工具的同名文件区分开。 */
62
62
  export const RESULTS_FORMAT = "niceeval.results";
63
63
  /** 结果格式版本,只在破坏兼容读取时递增;读取器只认相同版本,缺失按 1。见 docs/results-format.md。 */
64
- export const RESULTS_SCHEMA_VERSION = 2;
64
+ export const RESULTS_SCHEMA_VERSION = 3;
65
65
 
66
66
  export interface RunSummary {
67
67
  /** 恒为 "niceeval.results";和 schemaVersion、producer 一起构成持久化契约,永不移动或改名。 */
@@ -95,11 +95,11 @@ export interface RunSummary {
95
95
  snapshots?: Record<string, { startedAt?: string; knownEvalIds?: string[] }>;
96
96
  }
97
97
 
98
- /** onRunStart 的运行规模:去重后 eval 数 × 配置(agent×model×params)数 → 总运行(attempt)数。 */
98
+ /** onRunStart 的运行规模:去重后 eval 数 × 配置(agent×model×flags)数 → 总运行(attempt)数。 */
99
99
  export interface RunShape {
100
100
  /** 去重后实际要跑的 eval 数(= evals.length)。 */
101
101
  evals: number;
102
- /** (agent, model, params) 配置组合数;compare 多 agent 时 > 1。 */
102
+ /** (agent, model, flags) 配置组合数;compare 多 agent 时 > 1。 */
103
103
  configs: number;
104
104
  /** 总 attempt 数(evals × configs × runs);逐行输出与汇总计数都按它。 */
105
105
  totalRuns: number;
@@ -174,8 +174,8 @@ export interface ExperimentDef {
174
174
  model?: string;
175
175
  /** 模型推理努力程度(如 "low"/"medium"/"high",取值由具体模型/adapter 决定);省略=用 agent 原生默认。经 ctx.reasoningEffort 透给 adapter 与 eval。 */
176
176
  reasoningEffort?: string;
177
- /** 传给每次 attempt params,经 t.params 暴露给 eval;与 CLI flag 合并(CLI 优先)。 */
178
- params?: Record<string, unknown>;
177
+ /** 实验条件(A/B 里的 feature flag),由实验文件声明;经 ctx.flags 透传给 adapter、t.flags 暴露给 eval。 */
178
+ flags?: Record<string, unknown>;
179
179
  /** 同一 eval 重复跑几次(结果各计一条 attempt);省略/CLI `--runs` 覆盖时默认 1。 */
180
180
  runs?: number;
181
181
  /** 一次重复(runs > 1)里某次 attempt 失败后是否跳过剩余重复;省略默认 true(提前退出省钱)。 */
@@ -267,12 +267,12 @@ export function runWho(run: Pick<AgentRun, "agent" | "model" | "experimentId">):
267
267
  return run.model ? `${run.agent.name}/${run.model}` : run.agent.name;
268
268
  }
269
269
 
270
- /** 一个 (agent, model, params) 的运行配置 —— 由 CLI / 实验展开。 */
270
+ /** 一个 (agent, model, flags) 的运行配置 —— 由 CLI / 实验展开。 */
271
271
  export interface AgentRun {
272
272
  agent: Agent;
273
273
  model?: string;
274
274
  reasoningEffort?: string;
275
- params: Record<string, unknown>;
275
+ flags: Record<string, unknown>;
276
276
  runs: number;
277
277
  earlyExit: boolean;
278
278
  sandbox?: SandboxOption;
@@ -0,0 +1,3 @@
1
+ // TS7 默认开启 noUncheckedSideEffectImports,CSS 副作用导入(main.tsx 引 styles.css,
2
+ // 由 Vite 处理)需要显式模块声明才能通过 typecheck。
3
+ declare module "*.css";
@@ -58,7 +58,7 @@ export type MessageKey =
58
58
  | "detail.rawSample"
59
59
  | "detail.rawNote"
60
60
  | "config.experiment"
61
- | "config.paramsNone"
61
+ | "config.flagsNone"
62
62
  | "config.default"
63
63
  | "config.none"
64
64
  | "config.notApplicable"
@@ -177,7 +177,7 @@ const dictionaries: Record<Locale, Dictionary> = {
177
177
  "detail.rawSample": "Raw sample result",
178
178
  "detail.rawNote": "debug JSON, defaults to first error/failure when available",
179
179
  "config.experiment": "experiment",
180
- "config.paramsNone": "none",
180
+ "config.flagsNone": "none",
181
181
  "config.default": "default",
182
182
  "config.none": "none",
183
183
  "config.notApplicable": "n/a",
@@ -293,7 +293,7 @@ const dictionaries: Record<Locale, Dictionary> = {
293
293
  "detail.rawSample": "原始样例结果",
294
294
  "detail.rawNote": "调试 JSON,默认选择第一条错误/失败",
295
295
  "config.experiment": "实验",
296
- "config.paramsNone": "无",
296
+ "config.flagsNone": "无",
297
297
  "config.default": "默认",
298
298
  "config.none": "无",
299
299
  "config.notApplicable": "不适用",
@@ -127,9 +127,9 @@ export function valueFor(row: ViewRow, key: SortKey): string | number | null {
127
127
 
128
128
  export function configChips(row: ViewRow, t: T): [string, ReactNode][] {
129
129
  const exp = row.experiment || {};
130
- const params = exp.params && Object.keys(exp.params).length
131
- ? Object.entries(exp.params).map(([k, v]) => k + "=" + formatConfigValue(v)).join(", ")
132
- : t("config.paramsNone");
130
+ const flags = exp.flags && Object.keys(exp.flags).length
131
+ ? Object.entries(exp.flags).map(([k, v]) => k + "=" + formatConfigValue(v)).join(", ")
132
+ : t("config.flagsNone");
133
133
  return [
134
134
  [t("config.experiment"), row.synthetic ? row.label : row.experimentId],
135
135
  [t("table.model"), row.model || t("config.default")],
@@ -138,6 +138,6 @@ export function configChips(row: ViewRow, t: T): [string, ReactNode][] {
138
138
  ["earlyExit", exp.earlyExit === undefined ? t("config.notApplicable") : String(exp.earlyExit)],
139
139
  ["sandbox", exp.sandbox || t("config.default")],
140
140
  ["budget", exp.budget === undefined ? t("config.none") : "$" + exp.budget],
141
- ["params", params],
141
+ ["flags", flags],
142
142
  ];
143
143
  }