niceeval 0.6.0 → 0.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agents/types.d.ts +5 -1
- package/dist/report/built-ins/experiment-comparison.d.ts +1 -0
- package/dist/report/built-ins/experiment-comparison.js +13 -0
- package/dist/report/built-ins/index.d.ts +1 -1
- package/dist/report/built-ins/index.js +1 -1
- package/dist/report/components.d.ts +3 -1
- package/dist/report/components.js +1 -1
- package/dist/report/compute.js +15 -33
- package/dist/report/format.d.ts +0 -6
- package/dist/report/format.js +1 -18
- package/dist/report/index.d.ts +2 -2
- package/dist/report/index.js +1 -1
- package/dist/report/locale.d.ts +13 -2
- package/dist/report/locale.js +26 -3
- package/dist/report/metrics.js +1 -1
- package/dist/report/react/AttemptList.d.ts +2 -2
- package/dist/report/react/AttemptList.js +3 -4
- package/dist/report/react/ExperimentList.d.ts +3 -2
- package/dist/report/react/ExperimentList.js +59 -7
- package/dist/report/react/MetricScatter.js +1 -5
- package/dist/report/react/fixtures.js +9 -4
- package/dist/report/react/format.d.ts +1 -1
- package/dist/report/react/format.js +1 -1
- package/dist/report/react/index.d.ts +1 -1
- package/dist/report/text/faces.js +71 -33
- package/dist/report/types.d.ts +3 -15
- package/docs-site/zh/concepts/adapter.mdx +2 -2
- package/docs-site/zh/concepts/assert.mdx +11 -10
- package/docs-site/zh/concepts/evals.mdx +7 -6
- package/docs-site/zh/concepts/overview.mdx +1 -1
- package/docs-site/zh/guides/agent-feedback-loop.mdx +9 -7
- package/docs-site/zh/guides/fixtures.mdx +3 -1
- package/docs-site/zh/guides/official-adapters.mdx +1 -1
- package/docs-site/zh/guides/publish-report.mdx +1 -1
- package/docs-site/zh/guides/report-components.mdx +13 -13
- package/docs-site/zh/guides/results-data.mdx +28 -2
- package/docs-site/zh/guides/sandbox-agent.mdx +1 -0
- package/docs-site/zh/guides/sandbox-providers.mdx +1 -1
- package/docs-site/zh/guides/viewing-results.mdx +6 -5
- package/docs-site/zh/guides/write-send.mdx +2 -1
- package/docs-site/zh/reference/cli.mdx +3 -3
- package/docs-site/zh/reference/define-agent.mdx +3 -1
- package/docs-site/zh/reference/events.mdx +2 -2
- package/docs-site/zh/reference/expect.mdx +10 -5
- package/package.json +1 -1
- package/src/agents/types.ts +5 -1
- package/src/i18n/en.ts +1 -2
- package/src/i18n/zh-CN.ts +1 -1
- package/src/report/built-in-user-parity.test.tsx +44 -47
- package/src/report/built-ins/experiment-comparison.tsx +19 -0
- package/src/report/built-ins/index.ts +1 -1
- package/src/report/components.tsx +3 -1
- package/src/report/compute.ts +12 -36
- package/src/report/dual-render.test.tsx +33 -29
- package/src/report/format.ts +1 -16
- package/src/report/index.ts +1 -2
- package/src/report/locale.ts +26 -3
- package/src/report/metrics.ts +1 -1
- package/src/report/react/AttemptList.tsx +4 -6
- package/src/report/react/ExperimentList.tsx +167 -55
- package/src/report/react/MetricScatter.tsx +1 -11
- package/src/report/react/enhance.js +39 -0
- package/src/report/react/fixtures.ts +10 -4
- package/src/report/react/format.ts +1 -1
- package/src/report/react/index.tsx +0 -1
- package/src/report/react/render.test.tsx +24 -8
- package/src/report/react/styles.css +40 -13
- package/src/report/report.test.ts +2 -7
- package/src/report/text/faces.ts +76 -36
- package/src/report/types.ts +3 -15
- package/src/results/host-equivalence.test.ts +3 -3
- package/src/show/index.ts +8 -15
- package/src/show/render.ts +7 -204
- package/src/show/show.test.ts +22 -23
- package/src/view/data.test.ts +1 -1
- package/src/view/data.ts +6 -6
- package/src/view/index.ts +1 -1
- package/src/view/shared/types.ts +1 -1
- package/src/view/view-report.test.ts +5 -5
- package/dist/o11y/execution-tree.d.ts +0 -103
- package/dist/o11y/otlp/select.d.ts +0 -22
- package/dist/report/built-ins/cost-pass-rate-comparison.d.ts +0 -1
- package/dist/report/built-ins/cost-pass-rate-comparison.js +0 -17
- package/dist/results/annotated-source.d.ts +0 -61
- package/dist/results/attempt-evidence.d.ts +0 -69
- package/dist/results/attempt-source.d.ts +0 -15
- package/src/report/built-ins/cost-pass-rate-comparison.tsx +0 -23
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
title: "标准事件流参考"
|
|
3
3
|
sidebarTitle: "事件流"
|
|
4
|
-
description: "StreamEvent
|
|
4
|
+
description: "StreamEvent 的十种事件:每种的字段、什么时候吐、哪些断言消费它。adapter 的核心工作就是产出这条流。"
|
|
5
5
|
---
|
|
6
6
|
|
|
7
7
|
adapter 的 `send` 返回一个 `Turn`,其中 `events: StreamEvent[]` 是**断言的唯一数据源**:`t.calledTool()`、`t.reply`、`toolOrder`、`noFailedActions`……全部从这条流上读。把你的 agent"这一轮做了什么"翻成这条流,整套断言就都能用。
|
|
@@ -79,7 +79,7 @@ token 用量或 OTel span 反推得到)。存在时优先于按价格表(`define
|
|
|
79
79
|
|
|
80
80
|
## StreamEvent 变体一览
|
|
81
81
|
|
|
82
|
-
`StreamEvent`
|
|
82
|
+
`StreamEvent` 的十种变体,逐字段列出(消费它们的断言 / 使用细节见下面的「事件总表」和「逐事件说明」):
|
|
83
83
|
|
|
84
84
|
{/* GENERATED:BEGIN stream-events */}
|
|
85
85
|
|
|
@@ -200,13 +200,18 @@ atLeast(threshold: number): ValueAssertion;
|
|
|
200
200
|
## 自定义 matcher
|
|
201
201
|
|
|
202
202
|
```ts
|
|
203
|
-
import { makeAssertion } from "niceeval/expect";
|
|
203
|
+
import { makeAssertion, type Assertion } from "niceeval/expect";
|
|
204
204
|
|
|
205
|
-
|
|
206
|
-
return
|
|
207
|
-
|
|
205
|
+
function validEmail(): Assertion {
|
|
206
|
+
return makeAssertion({
|
|
207
|
+
name: "valid email",
|
|
208
|
+
score(value) {
|
|
209
|
+
return typeof value === "string" && value.includes("@") ? 1 : 0;
|
|
210
|
+
},
|
|
211
|
+
});
|
|
212
|
+
}
|
|
208
213
|
|
|
209
214
|
t.check(t.reply, validEmail());
|
|
210
215
|
```
|
|
211
216
|
|
|
212
|
-
|
|
217
|
+
`makeAssertion` 接受单个 spec 对象(`name`、可选 `severity`/`threshold`、`score`),直接返回一个 `Assertion`;按惯例用一个同名函数包一层再调用,而不是把 `makeAssertion` 本身当工厂调用。
|
package/package.json
CHANGED
package/src/agents/types.ts
CHANGED
|
@@ -221,7 +221,11 @@ export interface AgentContext {
|
|
|
221
221
|
* 具体取值)是两个维度——这里只是「跑的是哪个实验」的稳定标识,不携带条件内容。
|
|
222
222
|
*/
|
|
223
223
|
readonly experimentId?: string;
|
|
224
|
-
/**
|
|
224
|
+
/**
|
|
225
|
+
* 所有 agent 都有:沙箱型是运行器按项目/experiment 配置备好的真实沙箱句柄,remote 型是
|
|
226
|
+
* `createRemoteSandbox()` 产出的 stub(仅含 `workdir`/`sandboxId`/`otlpHost`/`stop` 等
|
|
227
|
+
* 元信息,其余方法调用即抛错)。
|
|
228
|
+
*/
|
|
225
229
|
readonly sandbox: Sandbox;
|
|
226
230
|
readonly session: AgentSession;
|
|
227
231
|
/**
|
package/src/i18n/en.ts
CHANGED
|
@@ -54,8 +54,7 @@ export const en = {
|
|
|
54
54
|
" niceeval exp [group|experiment] [eval-id-prefix…] run experiments\n" +
|
|
55
55
|
" niceeval show [eval-id-prefix… | @<locator>] read results in the terminal\n" +
|
|
56
56
|
" bare: current verdicts per experiment (composed across runs), each row\n" +
|
|
57
|
-
" with a compact attempt index (locator +
|
|
58
|
-
" [E,X,⏱,D] + failure reason)\n" +
|
|
57
|
+
" with a compact attempt index (locator + failure reason)\n" +
|
|
59
58
|
" a single eval id: attempts + assertion details\n" +
|
|
60
59
|
" @<locator> exactly one attempt: no flag -> compact overview;\n" +
|
|
61
60
|
" with a flag -> that evidence slice\n" +
|
package/src/i18n/zh-CN.ts
CHANGED
|
@@ -52,7 +52,7 @@ export const zhCN = {
|
|
|
52
52
|
" niceeval exp [组|实验] [eval-id 前缀…] 跑实验\n" +
|
|
53
53
|
" niceeval show [eval-id 前缀… | @<locator>] 终端读结果\n" +
|
|
54
54
|
" 裸跑:每个 experiment 的现刻判定(跨 run 合成),每行带紧凑 attempt 索引\n" +
|
|
55
|
-
" (locator +
|
|
55
|
+
" (locator + 失败原因)\n" +
|
|
56
56
|
" 单个 eval id:attempt 与断言明细\n" +
|
|
57
57
|
" @<locator> 精确一个 attempt:无 flag → 紧凑全景;带 flag → 对应证据切面\n" +
|
|
58
58
|
" --eval 该 attempt 运行时保存的 Eval 源码,断言标回源码行\n" +
|
|
@@ -1,15 +1,15 @@
|
|
|
1
|
-
// Phase 4 验收:证明内置报告
|
|
1
|
+
// Phase 4 验收:证明内置报告 ExperimentComparison 与一份普通用户 --report 文件走的是
|
|
2
2
|
// 完全同一套机制 —— 不是长得像,而是同一条 build → resolveReportTree → validate → render 管线。
|
|
3
3
|
//
|
|
4
4
|
// 证据链(plan/built-in-reports-user-parity.md「Phase 4」):
|
|
5
|
-
// 1. test/fixtures/report/
|
|
5
|
+
// 1. test/fixtures/report/experiment-comparison-public-copy.tsx 只从公开 barrel import,
|
|
6
6
|
// 与 built-in 逐节点同构(见文件头)。
|
|
7
7
|
// 2. 同一 ReportContext 下分别 build + resolveReportTree,深比较 resolved 树:组件类型 / 顺序 /
|
|
8
8
|
// resolved data / props keys 全部结构化相等(不靠 definition 引用相等,两份是独立文件)。
|
|
9
9
|
// 3. 两份分别过 renderReportToText / renderReportToStaticHtml,比较事实(散点键 / 表格行 /
|
|
10
10
|
// attempt 深链),不做 HTML 字面 snapshot。
|
|
11
11
|
// 4. 覆盖 plan 的 9 个必测场景。
|
|
12
|
-
// 5. 架构不变量:renderer 源码零 "
|
|
12
|
+
// 5. 架构不变量:renderer 源码零 "ExperimentComparison" / "built-ins" 分支。
|
|
13
13
|
// 6. show / view 只在「无 --report」分支选内置报告,--report 只换 definition,不碰
|
|
14
14
|
// Selection / resolve / validate / render。
|
|
15
15
|
|
|
@@ -22,8 +22,8 @@ import type { AttemptHandle, Results, Selection, Snapshot } from "../results/ind
|
|
|
22
22
|
import type { ExperimentListItem, ScatterData } from "./types.ts";
|
|
23
23
|
import { encodeAttemptLocator } from "../results/locator.ts";
|
|
24
24
|
import { Col, ExperimentList, defineReport } from "./index.ts";
|
|
25
|
-
import {
|
|
26
|
-
import publicCopy from "../../test/fixtures/report/
|
|
25
|
+
import { ExperimentComparison } from "./built-ins/index.ts";
|
|
26
|
+
import publicCopy from "../../test/fixtures/report/experiment-comparison-public-copy.tsx";
|
|
27
27
|
import { renderReportToText } from "./report.ts";
|
|
28
28
|
import { renderReportToStaticHtml } from "./web.ts";
|
|
29
29
|
import {
|
|
@@ -278,15 +278,15 @@ function expectedLocatorFor(attempt: AttemptHandle): string {
|
|
|
278
278
|
|
|
279
279
|
describe("resolved 树结构化等价(built-in ≡ 包外 public-copy)", () => {
|
|
280
280
|
it("两份是独立 ReportDefinition,不是同一引用", () => {
|
|
281
|
-
expect(
|
|
282
|
-
expect(typeof
|
|
281
|
+
expect(ExperimentComparison).not.toBe(publicCopy);
|
|
282
|
+
expect(typeof ExperimentComparison.build).toBe("function");
|
|
283
283
|
expect(typeof publicCopy.build).toBe("function");
|
|
284
284
|
});
|
|
285
285
|
|
|
286
286
|
for (const { name, ctx } of SCENARIOS) {
|
|
287
287
|
it(`同一 ReportContext 下 resolved 树逐节点相等 — ${name}`, async () => {
|
|
288
288
|
const c = ctx();
|
|
289
|
-
const builtIn = await resolvedTreeOf(
|
|
289
|
+
const builtIn = await resolvedTreeOf(ExperimentComparison, c);
|
|
290
290
|
const user = await resolvedTreeOf(publicCopy, ctx()); // 各用一份等价 ctx,证明不依赖共享状态
|
|
291
291
|
|
|
292
292
|
// 组件类型与顺序:<Col> 包 [MetricScatter, ExperimentList],别无它物。
|
|
@@ -308,7 +308,7 @@ describe("resolved 树结构化等价(built-in ≡ 包外 public-copy)", () => {
|
|
|
308
308
|
|
|
309
309
|
describe("必测场景:resolved data 事实", () => {
|
|
310
310
|
it("场景 1+2:多 experiment/agent,成本与通过率都有值;缺成本的点如实 x=null,实验列表仍出项", async () => {
|
|
311
|
-
const tree = await resolvedTreeOf(
|
|
311
|
+
const tree = await resolvedTreeOf(ExperimentComparison, richContext());
|
|
312
312
|
const scatter = scatterOf(tree)!;
|
|
313
313
|
const experiments = experimentListOf(tree)!;
|
|
314
314
|
|
|
@@ -342,7 +342,7 @@ describe("必测场景:resolved data 事实", () => {
|
|
|
342
342
|
});
|
|
343
343
|
|
|
344
344
|
it("场景 3:恰好一个可画点(另一实验缺成本)", async () => {
|
|
345
|
-
const tree = await resolvedTreeOf(
|
|
345
|
+
const tree = await resolvedTreeOf(ExperimentComparison, oneDrawableContext());
|
|
346
346
|
const scatter = scatterOf(tree)!;
|
|
347
347
|
const drawable = scatter.rows.filter((r) => r.x.value !== null && r.y.value !== null);
|
|
348
348
|
expect(drawable).toHaveLength(1);
|
|
@@ -350,20 +350,20 @@ describe("必测场景:resolved data 事实", () => {
|
|
|
350
350
|
});
|
|
351
351
|
|
|
352
352
|
it("场景 4:空 Selection —— 散点无 rows、实验列表为空数组", async () => {
|
|
353
|
-
const tree = await resolvedTreeOf(
|
|
353
|
+
const tree = await resolvedTreeOf(ExperimentComparison, emptyContext());
|
|
354
354
|
expect(scatterOf(tree)!.rows).toEqual([]);
|
|
355
355
|
expect(experimentListOf(tree)).toEqual([]);
|
|
356
356
|
});
|
|
357
357
|
|
|
358
358
|
it("场景 4 变体:全 skipped —— 每个点 x/y 都不可测,0 可画点", async () => {
|
|
359
|
-
const tree = await resolvedTreeOf(
|
|
359
|
+
const tree = await resolvedTreeOf(ExperimentComparison, allSkippedContext());
|
|
360
360
|
const scatter = scatterOf(tree)!;
|
|
361
361
|
expect(scatter.rows.every((r) => r.x.value === null && r.y.value === null)).toBe(true);
|
|
362
362
|
expect(scatter.rows.filter((r) => r.x.value !== null && r.y.value !== null)).toHaveLength(0);
|
|
363
363
|
});
|
|
364
364
|
|
|
365
365
|
it("场景 5:failed/errored/skipped 混合 —— ExperimentList 展开区结果摘要正确(eval 级折叠计票)", async () => {
|
|
366
|
-
const tree = await resolvedTreeOf(
|
|
366
|
+
const tree = await resolvedTreeOf(ExperimentComparison, richContext());
|
|
367
367
|
const codex = experimentListOf(tree)!.find((e) => e.experimentId === "compare/codex-mid")!;
|
|
368
368
|
// algebra/x errored、algebra/y skipped、algebra/z passed → eval 级折叠 {passed:1, errored:1, skipped:1}
|
|
369
369
|
expect(codex.verdicts).toEqual({ passed: 1, failed: 0, errored: 1, skipped: 1 });
|
|
@@ -374,7 +374,7 @@ describe("必测场景:resolved data 事实", () => {
|
|
|
374
374
|
});
|
|
375
375
|
|
|
376
376
|
it("场景 6:同 experiment 多 eval、多 attempts —— 展开层与 attempt locator 正确", async () => {
|
|
377
|
-
const tree = await resolvedTreeOf(
|
|
377
|
+
const tree = await resolvedTreeOf(ExperimentComparison, richContext());
|
|
378
378
|
const bub = experimentListOf(tree)!.find((e) => e.experimentId === "compare/bub-low")!;
|
|
379
379
|
expect(bub.evalRows.map((e) => e.evalId).sort()).toEqual(["algebra/x", "algebra/y"]);
|
|
380
380
|
const multi = bub.evalRows.find((e) => e.evalId === "algebra/x")!;
|
|
@@ -394,12 +394,12 @@ describe("必测场景:resolved data 事实", () => {
|
|
|
394
394
|
describe("built-in 与 public-copy 渲染出的事实相同", () => {
|
|
395
395
|
for (const { name, ctx } of SCENARIOS) {
|
|
396
396
|
it(`text / web 事实一致 — ${name}`, async () => {
|
|
397
|
-
const builtinText = await renderReportToText(
|
|
397
|
+
const builtinText = await renderReportToText(ExperimentComparison, ctx(), { width: 100 });
|
|
398
398
|
const userText = await renderReportToText(publicCopy, ctx(), { width: 100 });
|
|
399
399
|
// 同一机制、同一 options:text 输出逐字相等(最强的事实级证据)。
|
|
400
400
|
expect(userText).toBe(builtinText);
|
|
401
401
|
|
|
402
|
-
const builtinHtml = await renderReportToStaticHtml(
|
|
402
|
+
const builtinHtml = await renderReportToStaticHtml(ExperimentComparison, ctx());
|
|
403
403
|
const userHtml = await renderReportToStaticHtml(publicCopy, ctx());
|
|
404
404
|
// 事实比较(不做整段 HTML snapshot):散点点键、attempt 深链集合相同。
|
|
405
405
|
expect(scatterKeysOf(userHtml)).toEqual(scatterKeysOf(builtinHtml));
|
|
@@ -411,7 +411,7 @@ describe("built-in 与 public-copy 渲染出的事实相同", () => {
|
|
|
411
411
|
const ctx = richContext();
|
|
412
412
|
// renderReportToStaticHtml 默认 attemptHref = view 的 `#/attempt/@<locator>`(不透明单段路由,
|
|
413
413
|
// 与 view/data.ts renderReportSlot 不传 attemptHref 时同一条默认路径)。
|
|
414
|
-
const builtinHtml = await renderReportToStaticHtml(
|
|
414
|
+
const builtinHtml = await renderReportToStaticHtml(ExperimentComparison, ctx);
|
|
415
415
|
const userHtml = await renderReportToStaticHtml(publicCopy, richContext());
|
|
416
416
|
const builtinHrefs = hrefsOf(builtinHtml);
|
|
417
417
|
const userHrefs = hrefsOf(userHtml);
|
|
@@ -437,31 +437,28 @@ describe("built-in 与 public-copy 渲染出的事实相同", () => {
|
|
|
437
437
|
|
|
438
438
|
describe("散点空态 / 单点 / 多点行为", () => {
|
|
439
439
|
it("0 可画点(空 Selection):text 说 No data、web 走 nre-scatter-empty nre-missing", async () => {
|
|
440
|
-
const text = await renderReportToText(
|
|
440
|
+
const text = await renderReportToText(ExperimentComparison, emptyContext(), { width: 100 });
|
|
441
441
|
expect(text).toContain("No data to plot");
|
|
442
442
|
expect(text).not.toContain("better → upper right");
|
|
443
|
-
const html = await renderReportToStaticHtml(
|
|
443
|
+
const html = await renderReportToStaticHtml(ExperimentComparison, emptyContext());
|
|
444
444
|
expect(html).toContain("nre-scatter-empty");
|
|
445
445
|
expect(html).toContain("nre-missing");
|
|
446
446
|
});
|
|
447
447
|
|
|
448
|
-
it("恰好 1 可画点:text/web
|
|
449
|
-
const text = await renderReportToText(
|
|
450
|
-
expect(text).toContain("
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
expect(html).toContain("nre-scatter-empty");
|
|
454
|
-
// 单点不是「缺数据」空态:散点空态本身不带 nre-missing 变体类(与 0 可画点的
|
|
455
|
-
// "nre-scatter-empty nre-missing" 区分开)。ExperimentList 展开区自己的缺成本格子
|
|
456
|
-
// 也合法渲染 nre-missing,不属于这条散点专属的断言范围。
|
|
448
|
+
it("恰好 1 可画点:text/web 都正常画单点", async () => {
|
|
449
|
+
const text = await renderReportToText(ExperimentComparison, oneDrawableContext(), { width: 100 });
|
|
450
|
+
expect(text).toContain("better → upper right");
|
|
451
|
+
const html = await renderReportToStaticHtml(ExperimentComparison, oneDrawableContext());
|
|
452
|
+
expect(html).toContain("nre-scatter-point");
|
|
453
|
+
expect(html).not.toContain("nre-scatter-empty");
|
|
457
454
|
expect(html).not.toContain("nre-scatter-empty nre-missing");
|
|
458
|
-
expect(html).
|
|
455
|
+
expect(html).toContain("<svg");
|
|
459
456
|
});
|
|
460
457
|
|
|
461
458
|
it("≥2 可画点:正常绘图(text 有 better → upper right,web 有 svg)", async () => {
|
|
462
|
-
const text = await renderReportToText(
|
|
459
|
+
const text = await renderReportToText(ExperimentComparison, richContext(), { width: 100 });
|
|
463
460
|
expect(text).toContain("better → upper right");
|
|
464
|
-
const html = await renderReportToStaticHtml(
|
|
461
|
+
const html = await renderReportToStaticHtml(ExperimentComparison, richContext());
|
|
465
462
|
expect(html).toContain("<svg");
|
|
466
463
|
expect(html).not.toContain("nre-scatter-empty");
|
|
467
464
|
});
|
|
@@ -472,7 +469,7 @@ describe("散点空态 / 单点 / 多点行为", () => {
|
|
|
472
469
|
describe("场景 7:en / zh-CN 数据 resolve 一次、chrome 分别本地化", () => {
|
|
473
470
|
it("resolve 一次的同一棵树,en / zh-CN 各渲染一遍:data 事实相同,只有 chrome 文案不同", async () => {
|
|
474
471
|
// 关键:build + resolveReportTree 只做一次,拿到唯一的 resolved 树;两个 locale 复用它。
|
|
475
|
-
const node = (await
|
|
472
|
+
const node = (await ExperimentComparison.build(richContext())) as ReportNode;
|
|
476
473
|
const resolved = await resolveReportTree(node);
|
|
477
474
|
validateReportTree(resolved);
|
|
478
475
|
|
|
@@ -518,7 +515,7 @@ describe("场景 7:en / zh-CN 数据 resolve 一次、chrome 分别本地化", (
|
|
|
518
515
|
// 注意:spy 挂在 ExperimentList.data 本身,不是 compute.experimentListData —— components.tsx
|
|
519
516
|
// 用 `Object.assign(component, { data: experimentListData })` 把函数值复制成一个普通属性,
|
|
520
517
|
// 那份拷贝在 components.tsx 求值时(模块加载期)就定死了,晚于此的 vi.spyOn(compute, …) патch
|
|
521
|
-
// 不到已经复制走的引用。直接调用方(built-ins/
|
|
518
|
+
// 不到已经复制走的引用。直接调用方(built-ins/experiment-comparison.tsx)读的是
|
|
522
519
|
// `ExperimentList.data` 这个属性,所以要在这个属性上打桩。MetricScatter 不受影响:它的
|
|
523
520
|
// resolve() 住在 components.tsx 里,调用的是 scatterData 这个 import 绑定本身(ESM 实时绑定),
|
|
524
521
|
// vi.spyOn(compute, "scatterData") 打得中。
|
|
@@ -526,7 +523,7 @@ describe("场景 7:en / zh-CN 数据 resolve 一次、chrome 分别本地化", (
|
|
|
526
523
|
try {
|
|
527
524
|
// ExperimentList 没有 resolve 面,.data() 直接在 build() 里 await——计算发生在 build,
|
|
528
525
|
// 不是 resolveReportTree;MetricScatter 仍是 selection-form,resolve 阶段才调它的 .data()。
|
|
529
|
-
const node = (await
|
|
526
|
+
const node = (await ExperimentComparison.build(richContext())) as ReportNode;
|
|
530
527
|
expect(experimentListSpy).toHaveBeenCalledTimes(1);
|
|
531
528
|
const resolved = await resolveReportTree(node);
|
|
532
529
|
expect(scatterSpy).toHaveBeenCalledTimes(1);
|
|
@@ -603,38 +600,38 @@ describe("场景 9:最小用户报告只放 <ExperimentList> —— 不计算散
|
|
|
603
600
|
describe("架构不变量:renderer 不认识 built-in", () => {
|
|
604
601
|
const RENDERER_FILES = ["./report.ts", "./web.ts", "./tree.ts"];
|
|
605
602
|
for (const rel of RENDERER_FILES) {
|
|
606
|
-
it(`${rel} 源码里没有 "
|
|
603
|
+
it(`${rel} 源码里没有 "ExperimentComparison" / "built-ins" 分支`, () => {
|
|
607
604
|
const src = readFileSync(new URL(rel, import.meta.url), "utf8");
|
|
608
|
-
expect(src).not.toContain("
|
|
605
|
+
expect(src).not.toContain("ExperimentComparison");
|
|
609
606
|
expect(src).not.toContain("built-ins");
|
|
610
607
|
});
|
|
611
608
|
}
|
|
612
609
|
});
|
|
613
610
|
|
|
614
|
-
// ═════════════════════════ 8. 宿主选择:show
|
|
611
|
+
// ═════════════════════════ 8. 宿主选择:show / view 共用默认 definition ═════════════════════════
|
|
615
612
|
|
|
616
|
-
describe("宿主选择:裸 show
|
|
617
|
-
it("show / view
|
|
613
|
+
describe("宿主选择:裸 show / view 共用 ExperimentComparison,--report 替换 definition", () => {
|
|
614
|
+
it("show / view 源码:Selection 先合成,随后选择内置或用户 definition", () => {
|
|
618
615
|
const showSrc = readFileSync(new URL("../show/index.ts", import.meta.url), "utf8");
|
|
619
|
-
expect(showSrc).toMatch(/
|
|
620
|
-
expect(showSrc).toMatch(/
|
|
621
|
-
expect(showSrc).not.
|
|
616
|
+
expect(showSrc).toMatch(/import \{ ExperimentComparison \} from "\.\.\/\.\.\/dist\/report\/built-ins\/index\.js"/);
|
|
617
|
+
expect(showSrc).toMatch(/flags\.report === undefined \? ExperimentComparison : await loadReportFile\(cwd, flags\.report\)/);
|
|
618
|
+
expect(showSrc).not.toContain("showIndexText(selection");
|
|
622
619
|
// Selection 在报告槽之前、与 report 无关地由 selectCurrentResults 合成。
|
|
623
620
|
expect(showSrc).toMatch(/selectCurrentResults\(results, \{ experiment: flags\.experiment, patterns \}\)/);
|
|
624
621
|
|
|
625
622
|
const viewSrc = readFileSync(new URL("../view/data.ts", import.meta.url), "utf8");
|
|
626
|
-
// view 的报告槽:report 有值走 loadReportFile,否则
|
|
627
|
-
expect(viewSrc).toMatch(/report\s*\?[\s\S]*loadReportFile[\s\S]*:\s*\(await import\([^)]*built-ins[^)]*\)\)\.
|
|
623
|
+
// view 的报告槽:report 有值走 loadReportFile,否则 ExperimentComparison;Selection 同样先于报告槽合成。
|
|
624
|
+
expect(viewSrc).toMatch(/report\s*\?[\s\S]*loadReportFile[\s\S]*:\s*\(await import\([^)]*built-ins[^)]*\)\)\.ExperimentComparison/);
|
|
628
625
|
expect(viewSrc).toMatch(/selectCurrentResults\(results, \{ experiment: opts\.experiment, patterns \}\)/);
|
|
629
626
|
});
|
|
630
627
|
|
|
631
|
-
it("
|
|
628
|
+
it("ExperimentComparison 与 public-copy 作为显式报告时事实完全相同", async () => {
|
|
632
629
|
for (const { ctx } of SCENARIOS) {
|
|
633
|
-
const bareText = await renderReportToText(
|
|
630
|
+
const bareText = await renderReportToText(ExperimentComparison, ctx(), { width: 100 });
|
|
634
631
|
const reportText = await renderReportToText(publicCopy, ctx(), { width: 100 });
|
|
635
632
|
expect(reportText).toBe(bareText);
|
|
636
633
|
|
|
637
|
-
const bareHtml = await renderReportToStaticHtml(
|
|
634
|
+
const bareHtml = await renderReportToStaticHtml(ExperimentComparison, ctx());
|
|
638
635
|
const reportHtml = await renderReportToStaticHtml(publicCopy, ctx());
|
|
639
636
|
expect(scatterKeysOf(reportHtml)).toEqual(scatterKeysOf(bareHtml));
|
|
640
637
|
expect(hrefsOf(reportHtml)).toEqual(hrefsOf(bareHtml));
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
// ExperimentComparison:niceeval view 裸跑时的内置 experiment 比较报告。
|
|
2
|
+
//
|
|
3
|
+
// 名字描述整份报告回答的问题,不绑定其中某一张图。报告由成本 × 通过率散点和固定列
|
|
4
|
+
// ExperimentList 组成;它是一份普通 ReportDefinition,没有 renderer 私有通道。
|
|
5
|
+
|
|
6
|
+
import { Col } from "../primitives.tsx";
|
|
7
|
+
import { ExperimentList, MetricScatter } from "../components.tsx";
|
|
8
|
+
import { costUSD, passRate } from "../metrics.ts";
|
|
9
|
+
import { defineReport } from "../report.ts";
|
|
10
|
+
|
|
11
|
+
export const ExperimentComparison = defineReport(async ({ selection }) => {
|
|
12
|
+
const experiments = await ExperimentList.data(selection);
|
|
13
|
+
return (
|
|
14
|
+
<Col>
|
|
15
|
+
<MetricScatter selection={selection} points="experiment" series="agent" x={costUSD} y={passRate} />
|
|
16
|
+
<ExperimentList items={experiments} filter />
|
|
17
|
+
</Col>
|
|
18
|
+
);
|
|
19
|
+
});
|
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
// 内置报告目录:一个文件一个按展示内容命名的普通 ReportDefinition,显式导出值,不建字符串 registry。
|
|
2
|
-
export {
|
|
2
|
+
export { ExperimentComparison } from "./experiment-comparison.tsx";
|
|
@@ -90,6 +90,8 @@ export interface GroupSummaryProps {
|
|
|
90
90
|
*/
|
|
91
91
|
export interface ExperimentListProps {
|
|
92
92
|
items: ExperimentListItem[];
|
|
93
|
+
/** web 面在比较表前显示实验过滤框;text 面忽略。 */
|
|
94
|
+
filter?: boolean;
|
|
93
95
|
/** chrome 文案 locale;省略时随宿主上下文(宿主外默认 "en")。 */
|
|
94
96
|
locale?: ReportLocale;
|
|
95
97
|
className?: string;
|
|
@@ -254,7 +256,7 @@ EvalList.displayName = "EvalList";
|
|
|
254
256
|
|
|
255
257
|
/**
|
|
256
258
|
* Attempt 列表:实体列表的叶子层,每项一个 Attempt,固定展示判定、断言、error、Judge 评语
|
|
257
|
-
* (assertions 的 detail/evidence)与证据引用(locator
|
|
259
|
+
* (assertions 的 detail/evidence)与证据引用(locator)。它不预设只看失败;
|
|
258
260
|
* 报告作者过滤 `AttemptListItem[]`、用 `.slice()` 限量,`total` 让渲染面如实报告剩余数量。
|
|
259
261
|
*/
|
|
260
262
|
export const AttemptList: ReportComponent<AttemptListProps> & { data: typeof attemptListData } = Object.assign(
|
package/src/report/compute.ts
CHANGED
|
@@ -11,7 +11,6 @@
|
|
|
11
11
|
// - core 中立:只认 Metric / Dimension 接口,不出现具体 agent 名的分支。
|
|
12
12
|
|
|
13
13
|
import type {
|
|
14
|
-
AttemptEvidenceCapabilities,
|
|
15
14
|
AttemptListItem,
|
|
16
15
|
AttemptLocator,
|
|
17
16
|
DeltaData,
|
|
@@ -248,34 +247,6 @@ export async function tableData<const M extends readonly Metric[]>(
|
|
|
248
247
|
// (docs/feature/reports/library.md「实体列表」)。AttemptListItem 是三者共用的叶子形状——
|
|
249
248
|
// ExperimentList / EvalList 的下钻数组直接复用它,不各自精简一份。
|
|
250
249
|
|
|
251
|
-
/**
|
|
252
|
-
* 一个 attempt 的证据能力标记,只读已有的瘦身摘要位(`hasSources` / `hasEvents` / `hasTrace`)
|
|
253
|
-
* 加一次 `diff()` 懒加载——不调用完整的 `loadAttemptEvidence`(那还会额外装配 Eval 源码标注
|
|
254
|
-
* 与 ExecutionTree,这里只要四个布尔位)。三位定义与 `AttemptEvidence.capabilities` 完全一致:
|
|
255
|
-
*
|
|
256
|
-
* - `eval`:`hasSources` 为真。真正的门槛是 `AnnotatedEvalSource` 非空且有内容(源码行或断言
|
|
257
|
-
* 非空),但 sources 存在时 `buildAnnotatedEvalSource` 恒产出至少一行源码,断言又永远来自
|
|
258
|
-
* `result.assertions`(不需要 IO)——`hasSources` 是这个条件的精确瘦身版,不是近似。
|
|
259
|
-
* - `execution`:`hasEvents` 为真,与 `loadAttemptEvidence` 的 `events !== null && events.length > 0`
|
|
260
|
-
* 同一个判定,写入面按同一条规则算的 `hasEvents` 就是它的瘦身镜像。
|
|
261
|
-
* - `timing`:`hasEvents && hasTrace` 为真——`execution` 成立且这次运行接入过 OTel
|
|
262
|
-
* (`ExecutionTree.timingAvailable` = `spans.length > 0`,`hasTrace` 正是这个判定的瘦身镜像)。
|
|
263
|
-
* - `diff`:唯一需要 IO 的一位,`attempt.diff()` 非空且至少改动/删除了一个文件——落盘没有
|
|
264
|
-
* 与 `hasEvents` 对应的瘦身摘要位,只能懒加载判定;成本是每个 item 一次 `diff()` 读取,
|
|
265
|
-
* 不是完整证据装配的四路 Promise.all。
|
|
266
|
-
*/
|
|
267
|
-
async function attemptCapabilities(attempt: AttemptHandle): Promise<AttemptEvidenceCapabilities> {
|
|
268
|
-
const result = attempt.result;
|
|
269
|
-
const diff = await attempt.diff();
|
|
270
|
-
const diffCapable = diff !== null && (Object.keys(diff.generatedFiles).length > 0 || diff.deletedFiles.length > 0);
|
|
271
|
-
return {
|
|
272
|
-
eval: result.hasSources === true,
|
|
273
|
-
execution: result.hasEvents === true,
|
|
274
|
-
timing: result.hasEvents === true && result.hasTrace === true,
|
|
275
|
-
diff: diffCapable,
|
|
276
|
-
};
|
|
277
|
-
}
|
|
278
|
-
|
|
279
250
|
/** 自由文本(error / 断言 detail / evidence)的发布消毒钩子;身份字段(name/severity/loc)不经它。 */
|
|
280
251
|
function redactAssertions(assertions: AssertionResult[], redact: (text: string) => string): AssertionResult[] {
|
|
281
252
|
if (assertions.length === 0) return assertions;
|
|
@@ -287,7 +258,7 @@ function redactAssertions(assertions: AssertionResult[], redact: (text: string)
|
|
|
287
258
|
}
|
|
288
259
|
|
|
289
260
|
/** AttemptList / ExperimentList / EvalList 共用的叶子构造:一个 Item → 一个 AttemptListItem。 */
|
|
290
|
-
|
|
261
|
+
function attemptListItemOf(item: Item, redact: (text: string) => string): AttemptListItem {
|
|
291
262
|
const result = item.attempt.result;
|
|
292
263
|
const cost = attemptCostUSD(result);
|
|
293
264
|
return {
|
|
@@ -301,7 +272,6 @@ async function attemptListItemOf(item: Item, redact: (text: string) => string):
|
|
|
301
272
|
durationMs: result.durationMs,
|
|
302
273
|
...(cost !== null ? { costUSD: cost } : {}),
|
|
303
274
|
locator: locatorOf(item),
|
|
304
|
-
capabilities: await attemptCapabilities(item.attempt),
|
|
305
275
|
};
|
|
306
276
|
}
|
|
307
277
|
|
|
@@ -320,7 +290,7 @@ export async function attemptListData(
|
|
|
320
290
|
const { snapshots } = resolveInput(input);
|
|
321
291
|
const redact = opts?.redact ?? identityRedact;
|
|
322
292
|
const items = collectItems(snapshots);
|
|
323
|
-
return
|
|
293
|
+
return items.map((item) => attemptListItemOf(item, redact));
|
|
324
294
|
}
|
|
325
295
|
|
|
326
296
|
/** `EvalList.data(selection)`:每个 `experimentId + evalId` 一项,按 evalId 再按 experimentId 升序。 */
|
|
@@ -339,7 +309,7 @@ export async function evalListData(input: SnapshotsInput): Promise<EvalListItem[
|
|
|
339
309
|
const sorted = [...group].sort((a, b) => a.attempt.result.attempt - b.attempt.result.attempt);
|
|
340
310
|
const verdict = foldEvalVerdict(sorted.map((item) => item.attempt.result));
|
|
341
311
|
const representative = sorted.find((item) => item.attempt.result.verdict === verdict) ?? sorted[0]!;
|
|
342
|
-
const attempts =
|
|
312
|
+
const attempts = sorted.map((item) => attemptListItemOf(item, identityRedact));
|
|
343
313
|
out.push({
|
|
344
314
|
evalId: evalIdOf(sorted[0]!),
|
|
345
315
|
experimentId: experimentIdOf(sorted[0]!),
|
|
@@ -370,7 +340,7 @@ export async function experimentListData(input: SnapshotsInput): Promise<Experim
|
|
|
370
340
|
const sorted = [...evalItems].sort((a, b) => a.attempt.result.attempt - b.attempt.result.attempt);
|
|
371
341
|
const verdict = foldEvalVerdict(sorted.map((item) => item.attempt.result));
|
|
372
342
|
const representative = sorted.find((item) => item.attempt.result.verdict === verdict) ?? sorted[0]!;
|
|
373
|
-
const attempts =
|
|
343
|
+
const attempts = sorted.map((item) => attemptListItemOf(item, identityRedact));
|
|
374
344
|
evalRows.push({
|
|
375
345
|
evalId,
|
|
376
346
|
verdict,
|
|
@@ -400,7 +370,14 @@ export async function experimentListData(input: SnapshotsInput): Promise<Experim
|
|
|
400
370
|
evalRows,
|
|
401
371
|
});
|
|
402
372
|
}
|
|
403
|
-
|
|
373
|
+
// ExperimentList 是默认实验比较表:初始态按成功率从高到低,缺数据沉底;
|
|
374
|
+
// 同分时按 experiment id 稳定排序。web 增强可临时重排,text 面沿用同一基准顺序。
|
|
375
|
+
out.sort((a, b) => {
|
|
376
|
+
if (a.passRate.value === null && b.passRate.value === null) return a.experimentId.localeCompare(b.experimentId);
|
|
377
|
+
if (a.passRate.value === null) return 1;
|
|
378
|
+
if (b.passRate.value === null) return -1;
|
|
379
|
+
return b.passRate.value - a.passRate.value || a.experimentId.localeCompare(b.experimentId);
|
|
380
|
+
});
|
|
404
381
|
return out;
|
|
405
382
|
}
|
|
406
383
|
|
|
@@ -772,4 +749,3 @@ function deltaDisplay(metric: Metric, delta: number | null): string {
|
|
|
772
749
|
const text = displayValue(metric, delta); // 负号由格式化自带
|
|
773
750
|
return delta > 0 ? `+${text}` : text;
|
|
774
751
|
}
|
|
775
|
-
|