niceeval 0.8.0 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/INDEX.md +77 -45
- package/dist/agents/types.d.ts +28 -6
- package/dist/i18n/en.d.ts +2 -0
- package/dist/i18n/zh-CN.d.ts +2 -0
- package/dist/report/built-in/index.d.ts +3 -2
- package/dist/report/built-in/index.js +7 -8
- package/dist/report/built-in/standard.d.ts +1 -0
- package/dist/report/built-in/standard.js +30 -0
- package/dist/report/components.d.ts +72 -6
- package/dist/report/components.js +159 -10
- package/dist/report/compute.d.ts +31 -4
- package/dist/report/compute.js +131 -12
- package/dist/report/index.d.ts +4 -4
- package/dist/report/index.js +3 -2
- package/dist/report/locale.d.ts +39 -1
- package/dist/report/locale.js +69 -0
- package/dist/report/react/AttemptList.d.ts +3 -1
- package/dist/report/react/AttemptList.js +3 -3
- package/dist/report/react/CopyFixPrompt.d.ts +12 -0
- package/dist/report/react/CopyFixPrompt.js +12 -0
- package/dist/report/react/HeroCard.d.ts +13 -0
- package/dist/report/react/HeroCard.js +35 -0
- package/dist/report/react/PoweredBy.d.ts +5 -0
- package/dist/report/react/PoweredBy.js +7 -0
- package/dist/report/react/ScopeWarnings.d.ts +12 -0
- package/dist/report/react/ScopeWarnings.js +18 -0
- package/dist/report/react/TraceWaterfall.d.ts +14 -0
- package/dist/report/react/TraceWaterfall.js +22 -0
- package/dist/report/react/index.d.ts +6 -1
- package/dist/report/react/index.js +6 -0
- package/dist/report/report.d.ts +22 -6
- package/dist/report/report.js +66 -53
- package/dist/report/scope-warnings.d.ts +28 -0
- package/dist/report/scope-warnings.js +101 -0
- package/dist/report/text/faces.d.ts +19 -1
- package/dist/report/text/faces.js +61 -0
- package/dist/report/tree.js +6 -1
- package/dist/report/types.d.ts +45 -10
- package/dist/report/web.d.ts +5 -4
- package/dist/report/web.js +7 -22
- package/dist/results/select.d.ts +15 -2
- package/dist/results/select.js +75 -10
- package/dist/results/types.d.ts +26 -11
- package/dist/runner/fingerprint.d.ts +3 -3
- package/dist/runner/sandbox-selection.d.ts +12 -0
- package/dist/runner/types.d.ts +18 -6
- package/dist/sandbox/types.d.ts +12 -0
- package/docs-site/zh/explanation/evals.mdx +2 -1
- package/docs-site/zh/explanation/experiment.mdx +2 -0
- package/docs-site/zh/how-to/custom-reports.mdx +22 -6
- package/docs-site/zh/how-to/publish-report.mdx +6 -8
- package/docs-site/zh/how-to/viewing-results.mdx +3 -3
- package/docs-site/zh/how-to/write-experiment.mdx +40 -1
- package/docs-site/zh/reference/builtin-agents.mdx +40 -3
- package/docs-site/zh/reference/cli.mdx +1 -2
- package/docs-site/zh/reference/define-eval.mdx +8 -0
- package/docs-site/zh/reference/official-adapters.mdx +10 -5
- package/docs-site/zh/reference/report-components.mdx +2 -2
- package/docs-site/zh/reference/results-data.mdx +2 -4
- package/package.json +2 -1
- package/src/agents/bub.ts +13 -1
- package/src/agents/claude-code.test.ts +43 -1
- package/src/agents/claude-code.ts +32 -14
- package/src/agents/codex.test.ts +168 -1
- package/src/agents/codex.ts +51 -15
- package/src/agents/mcp.ts +31 -0
- package/src/agents/post-setup.ts +33 -0
- package/src/agents/types.ts +28 -7
- package/src/cli.ts +9 -11
- package/src/context/context.ts +11 -5
- package/src/define.ts +3 -0
- package/src/i18n/en.ts +5 -2
- package/src/i18n/zh-CN.ts +5 -1
- package/src/index.ts +1 -0
- package/src/report/built-in/index.tsx +8 -7
- package/src/report/built-in/standard.tsx +59 -0
- package/src/report/components.tsx +231 -17
- package/src/report/compute.ts +146 -21
- package/src/report/dual-render.test.tsx +139 -12
- package/src/report/index.ts +20 -1
- package/src/report/locale.ts +83 -1
- package/src/report/react/AttemptList.tsx +13 -1
- package/src/report/react/CopyFixPrompt.tsx +37 -0
- package/src/report/react/HeroCard.tsx +59 -0
- package/src/report/react/PoweredBy.tsx +20 -0
- package/src/report/react/ScopeWarnings.tsx +74 -0
- package/src/report/react/TraceWaterfall.tsx +78 -0
- package/src/report/react/enhance.js +14 -0
- package/src/report/react/index.tsx +11 -0
- package/src/report/react/styles.css +187 -7
- package/src/report/report.test.ts +2 -20
- package/src/report/report.ts +97 -62
- package/src/report/scope-warnings.ts +155 -0
- package/src/report/site-components.test.tsx +526 -0
- package/src/report/text/faces.ts +66 -0
- package/src/report/tree.ts +8 -1
- package/src/report/types.ts +51 -11
- package/src/report/web.ts +7 -40
- package/src/results/copy.ts +15 -78
- package/src/results/host-equivalence.test.ts +5 -1
- package/src/results/open.ts +5 -4
- package/src/results/publish.ts +4 -146
- package/src/results/results.test.ts +86 -9
- package/src/results/select.ts +78 -10
- package/src/results/types.ts +27 -7
- package/src/runner/attempt.ts +10 -9
- package/src/runner/discover.test.ts +9 -1
- package/src/runner/discover.ts +3 -3
- package/src/runner/fingerprint.ts +9 -4
- package/src/runner/ledger.test.ts +30 -1
- package/src/runner/ledger.ts +26 -4
- package/src/runner/run.ts +5 -1
- package/src/runner/sandbox-selection.test.ts +131 -0
- package/src/runner/sandbox-selection.ts +110 -0
- package/src/runner/types.ts +19 -2
- package/src/sandbox/types.ts +6 -0
- package/src/show/index.ts +17 -10
- package/src/show/render.ts +11 -11
- package/src/show/report-host.test.ts +32 -15
- package/src/show/report-host.ts +5 -4
- package/src/show/show.test.ts +140 -3
- package/src/view/app/App.test.tsx +78 -17
- package/src/view/app/App.tsx +17 -78
- package/src/view/app/components/CopyControls.tsx +4 -42
- package/src/view/app/i18n.ts +5 -227
- package/src/view/app/lib/rows.ts +3 -21
- package/src/view/app/shared.ts +1 -3
- package/src/view/app/types.ts +2 -2
- package/src/view/artifact-serving.test.ts +1 -1
- package/src/view/client-dist/app.css +1 -1
- package/src/view/client-dist/app.js +20 -20
- package/src/view/data.ts +2 -13
- package/src/view/index.ts +1 -12
- package/src/view/server.ts +0 -2
- package/src/view/shared/types.ts +11 -6
- package/src/view/site-parity.test.ts +1 -1
- package/src/view/site.ts +1 -1
- package/src/view/styles.css +6 -266
- package/src/view/view-report.test.ts +64 -27
- package/src/view/app/components/LazyArtifact.tsx +0 -51
- package/src/view/app/components/SkippedRunsBanner.tsx +0 -140
- package/src/view/app/pages/AttemptsPage.tsx +0 -80
- package/src/view/app/pages/TracesPage.tsx +0 -35
package/src/report/types.ts
CHANGED
|
@@ -299,6 +299,57 @@ export interface ScopeSummaryData {
|
|
|
299
299
|
totalCostUSD: MetricCell;
|
|
300
300
|
}
|
|
301
301
|
|
|
302
|
+
// ───────────────────────── 站点组件(Hero / CopyFixPrompt / TraceWaterfall)─────────────────────────
|
|
303
|
+
|
|
304
|
+
/**
|
|
305
|
+
* `HeroCard` 的数据(docs/feature/reports/library/site-components.md):站点标题区的
|
|
306
|
+
* 运行 meta——最后运行时间与快照合成来源。标题不在 data 里,它是站点声明与 Scope 的合成物,
|
|
307
|
+
* 经 `HeroCardProps.title` 传入。
|
|
308
|
+
*/
|
|
309
|
+
export interface HeroData {
|
|
310
|
+
/** Scope 中最新快照的开始时间;空 Scope 为 null,不编造当前时间。 */
|
|
311
|
+
latestStartedAt: string | null;
|
|
312
|
+
/** 贡献当前水位的快照数;大于 1 时 web 面标注「由 N 次运行合成」。 */
|
|
313
|
+
snapshots: number;
|
|
314
|
+
}
|
|
315
|
+
|
|
316
|
+
/**
|
|
317
|
+
* `CopyFixPrompt` 的数据:resolve 期算好的修复 prompt 全文与参与的失败数
|
|
318
|
+
* (docs/feature/reports/library/site-components.md)。
|
|
319
|
+
*/
|
|
320
|
+
export interface CopyFixPromptData {
|
|
321
|
+
/** 修复 prompt 全文;失败逐条含 eval id、主失败摘要与 attempt 下钻命令。 */
|
|
322
|
+
prompt: string;
|
|
323
|
+
/** 参与 prompt 的失败 attempt 数(verdict 为 failed / errored)。 */
|
|
324
|
+
failures: number;
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
/** `TraceWaterfall` 一行里的一个顶层 span 摘要(canonical OTel 字段归一后的形态)。 */
|
|
328
|
+
export interface TraceSpanSummary {
|
|
329
|
+
name: string;
|
|
330
|
+
/** 归一后的语义角色;turn 归入 agent,未识别落 other。 */
|
|
331
|
+
kind: "agent" | "model" | "tool" | "other";
|
|
332
|
+
/** 相对该 attempt trace 起点的偏移(毫秒)。 */
|
|
333
|
+
startOffsetMs: number;
|
|
334
|
+
durationMs: number;
|
|
335
|
+
/** span status 为 error 时 true(web 面失败标记的来源)。 */
|
|
336
|
+
failed: boolean;
|
|
337
|
+
}
|
|
338
|
+
|
|
339
|
+
/**
|
|
340
|
+
* `TraceWaterfall` 一行 = 一次 attempt 的执行时间瀑布摘要。只画被测 agent 的原始 span
|
|
341
|
+
* (trace.json);runner 生命周期节点(`result.phases`)不进瀑布,组合视图归 attempt 详情。
|
|
342
|
+
*/
|
|
343
|
+
export interface TraceWaterfallRow {
|
|
344
|
+
experimentId: string;
|
|
345
|
+
evalId: string;
|
|
346
|
+
locator: AttemptLocator;
|
|
347
|
+
/** trace.json 缺失或为空时 null;行照常出现,证据位置如实显示缺失,不猜值。 */
|
|
348
|
+
durationMs: number | null;
|
|
349
|
+
/** 顶层 span 摘要,按 startOffsetMs 升序。 */
|
|
350
|
+
spans: readonly TraceSpanSummary[];
|
|
351
|
+
}
|
|
352
|
+
|
|
302
353
|
/** 一个可比组的数据;三个子块都只消费本组快照,不能含其它父目录的引用。 */
|
|
303
354
|
export interface ExperimentComparisonGroupData {
|
|
304
355
|
/** experiment id 的完整父路径;根目录 experiment 使用完整 id。 */
|
|
@@ -394,14 +445,3 @@ export interface ExperimentListItem {
|
|
|
394
445
|
lastRunAt: string;
|
|
395
446
|
evalRows: ExperimentListEvalRow[];
|
|
396
447
|
}
|
|
397
|
-
|
|
398
|
-
/** 三个实体列表共用的计算选项。 */
|
|
399
|
-
export interface EntityListDataOptions {
|
|
400
|
-
/**
|
|
401
|
-
* 展示层遮蔽:只改写这次组件数据中的自由文本——条目本身与任何嵌套 attempt 条目的
|
|
402
|
-
* `failureSummary`;身份与分类字段(experimentId、evalId、locator、数值指标)不经它。
|
|
403
|
-
* 只作用于这次计算产出的组件数据,不改盘上或任何导出目录里的 artifact;
|
|
404
|
-
* 发布 artifact 的脱敏用 copySnapshots({ redact })。
|
|
405
|
-
*/
|
|
406
|
-
redact?: (text: string) => string;
|
|
407
|
-
}
|
package/src/report/web.ts
CHANGED
|
@@ -2,7 +2,6 @@
|
|
|
2
2
|
// HTML 烘进查看器的报告槽。只有这一侧真正 import react-dom(import 边界即运行时边界),
|
|
3
3
|
// 所以本文件不从 niceeval/report 的入口 re-export —— 宿主与测试按源路径 import。
|
|
4
4
|
|
|
5
|
-
import * as React from "react";
|
|
6
5
|
import type { ReactNode } from "react";
|
|
7
6
|
import { renderToStaticMarkup } from "react-dom/server";
|
|
8
7
|
import type { AttemptLocator } from "../results/locator.ts";
|
|
@@ -26,39 +25,11 @@ export interface StaticHtmlOptions {
|
|
|
26
25
|
locale?: ReportLocale;
|
|
27
26
|
}
|
|
28
27
|
|
|
29
|
-
/**
|
|
30
|
-
* 挑选警告的 HTML 形态:宿主级前置块(`.nre nre-report-warnings` 外壳内一个
|
|
31
|
-
* `ul.nre-warnings` + `li.nre-warning[data-kind]`,复用 styles.css 已有样式)。
|
|
32
|
-
* 带 `command` 的警告把命令渲染为可复制块(`.nre-warning-command`);无 command 的
|
|
33
|
-
* 只显示 message,不硬造动作。经 renderToStaticMarkup 走 React,文本自动转义。
|
|
34
|
-
*/
|
|
35
|
-
function renderScopeWarningsHtml(scope: Scope): string {
|
|
36
|
-
return renderToStaticMarkup(
|
|
37
|
-
React.createElement(
|
|
38
|
-
"div",
|
|
39
|
-
{ className: "nre nre-report-warnings" },
|
|
40
|
-
React.createElement(
|
|
41
|
-
"ul",
|
|
42
|
-
{ className: "nre-warnings" },
|
|
43
|
-
scope.warnings.map((w, i) =>
|
|
44
|
-
React.createElement(
|
|
45
|
-
"li",
|
|
46
|
-
{ key: i, className: "nre-warning", "data-kind": w.kind },
|
|
47
|
-
w.message,
|
|
48
|
-
"command" in w && w.command
|
|
49
|
-
? React.createElement("code", { className: "nre-warning-command", "data-nre-copy": w.command }, w.command)
|
|
50
|
-
: null,
|
|
51
|
-
),
|
|
52
|
-
),
|
|
53
|
-
),
|
|
54
|
-
),
|
|
55
|
-
);
|
|
56
|
-
}
|
|
57
|
-
|
|
58
28
|
/**
|
|
59
29
|
* web 宿主的装载语义:选页 → resolve(组合展开 + spec 取数,唯一的 await 边界)→
|
|
60
|
-
* 树校验(与 text 宿主同一遍)→ 静态渲染 web
|
|
61
|
-
*
|
|
30
|
+
* 树校验(与 text 宿主同一遍)→ 静态渲染 web 面。宿主不在报告树外另设警告通道——
|
|
31
|
+
* 挑选警告的呈现件是 `ScopeWarnings` 组件,内建报告每页都放它,自定义报告放不放是
|
|
32
|
+
* 作者义务(docs/feature/reports/architecture.md「Scope 是计算入口」)。
|
|
62
33
|
*/
|
|
63
34
|
export async function renderReportToStaticHtml(
|
|
64
35
|
definition: ReportDefinition,
|
|
@@ -78,15 +49,13 @@ export async function renderReportToStaticHtml(
|
|
|
78
49
|
attemptHref: options?.attemptHref ?? ((locator) => `#/attempt/${locator}`),
|
|
79
50
|
locale: options?.locale ?? DEFAULT_REPORT_LOCALE,
|
|
80
51
|
};
|
|
81
|
-
|
|
82
|
-
const warnings = ctx.scope.warnings.length > 0 ? renderScopeWarningsHtml(ctx.scope) : "";
|
|
83
|
-
return warnings + body;
|
|
52
|
+
return runWithWebContext(webCtx, () => renderToStaticMarkup(resolved as ReactNode));
|
|
84
53
|
}
|
|
85
54
|
|
|
86
55
|
/**
|
|
87
56
|
* 渲染一页报告树的 web 面(宿主逐页调用;页选择归宿主):resolve → validate → 静态渲染。
|
|
88
|
-
*
|
|
89
|
-
*
|
|
57
|
+
* 挑选警告由页内的 `ScopeWarnings` 组件呈现,宿主不前置任何树外块。
|
|
58
|
+
* ctx.report 是宿主规范化后的声明。
|
|
90
59
|
*/
|
|
91
60
|
export async function renderReportTreeToStaticHtml(
|
|
92
61
|
tree: import("./tree.ts").ReportNode,
|
|
@@ -104,7 +73,5 @@ export async function renderReportTreeToStaticHtml(
|
|
|
104
73
|
attemptHref: options?.attemptHref ?? ((locator) => `#/attempt/${locator}`),
|
|
105
74
|
locale: options?.locale ?? DEFAULT_REPORT_LOCALE,
|
|
106
75
|
};
|
|
107
|
-
|
|
108
|
-
const warnings = ctx.scope.warnings.length > 0 ? renderScopeWarningsHtml(ctx.scope) : "";
|
|
109
|
-
return warnings + body;
|
|
76
|
+
return runWithWebContext(webCtx, () => renderToStaticMarkup(resolved as ReactNode));
|
|
110
77
|
}
|
package/src/results/copy.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// copySnapshots:把选中快照按格式感知地复制到另一个目录(定稿见 docs/feature/results/library.md「复制与瘦身」)。
|
|
2
2
|
//
|
|
3
3
|
// 发布场景的原语:只带指定 artifact、只带选中快照的全部 attempt,布局知识不外泄。
|
|
4
|
-
// artifact 复制忠实于源(
|
|
4
|
+
// artifact 复制忠实于源(原字节,不重新序列化、不改写);snapshot.json / result.json
|
|
5
5
|
// 按选中条目重建,版本元数据保留。产物是一个标准结果根目录(同布局),openResults /
|
|
6
6
|
// `niceeval view` 直接能读。唯一随行补记的是挑选时的覆盖事实:每个复制出的快照带上
|
|
7
7
|
// knownEvalIds(复制时刻该实验已知的 eval 并集),发布目录上重新 openResults().latest(),
|
|
@@ -9,21 +9,13 @@
|
|
|
9
9
|
|
|
10
10
|
import { mkdir, readdir, readFile, stat, writeFile } from "node:fs/promises";
|
|
11
11
|
import { basename, dirname, join, resolve } from "node:path";
|
|
12
|
-
import type { EvalResult
|
|
12
|
+
import type { EvalResult } from "../types.ts";
|
|
13
13
|
import { RESULTS_FORMAT } from "../types.ts";
|
|
14
14
|
import { RESULT_FILE, SNAPSHOT_FILE, artifactFileOf, experimentDirOf } from "./format.ts";
|
|
15
15
|
import { experimentOfSnapshot } from "./open.ts";
|
|
16
16
|
import { isNewerSnapshot } from "./select.ts";
|
|
17
17
|
import { hashEvalSource, normalizeEvalSource } from "./source-hash.ts";
|
|
18
|
-
import {
|
|
19
|
-
PUBLISH_FILE_MAX_BYTES,
|
|
20
|
-
redactEvents,
|
|
21
|
-
redactExperimentInfo,
|
|
22
|
-
redactJsonValue,
|
|
23
|
-
redactResultRecord,
|
|
24
|
-
redactSpans,
|
|
25
|
-
type Redactor,
|
|
26
|
-
} from "./publish.ts";
|
|
18
|
+
import { PUBLISH_FILE_MAX_BYTES } from "./publish.ts";
|
|
27
19
|
import type { ArtifactKind, AttemptHandle, Scope, Snapshot, SnapshotMeta } from "./types.ts";
|
|
28
20
|
|
|
29
21
|
/** 缺省携带的 artifact:events / trace / o11y / agentSetup / sources;diff 不截断、可达百 MB,缺省不带。 */
|
|
@@ -33,12 +25,6 @@ const VALID_ARTIFACTS: ArtifactKind[] = ["events", "trace", "o11y", "agentSetup"
|
|
|
33
25
|
export interface CopySnapshotsOptions {
|
|
34
26
|
/** 要带上的 artifact 种类;缺省带 events / trace / o11y / agentSetup / sources,不带 diff。 */
|
|
35
27
|
artifacts?: ArtifactKind[];
|
|
36
|
-
/**
|
|
37
|
-
* 发布消毒(必填,没有隐式默认):传函数逐值消毒自由文本字段,或显式传 `false` 声明
|
|
38
|
-
* 「这批数据可以原文发布」——「不消毒」必须是写在代码里的选择,不是忘了传参数的副作用。
|
|
39
|
-
* 结构字段(格式、判定、身份、路径、哈希)永不经过 redactor。
|
|
40
|
-
*/
|
|
41
|
-
redact: Redactor | false;
|
|
42
28
|
}
|
|
43
29
|
|
|
44
30
|
export interface CopySnapshotsResult {
|
|
@@ -56,7 +42,7 @@ export interface CopySnapshotsResult {
|
|
|
56
42
|
export async function copySnapshots(
|
|
57
43
|
scope: Scope | readonly Snapshot[],
|
|
58
44
|
destDir: string,
|
|
59
|
-
opts: CopySnapshotsOptions,
|
|
45
|
+
opts: CopySnapshotsOptions = {},
|
|
60
46
|
): Promise<CopySnapshotsResult> {
|
|
61
47
|
const selected = Array.isArray(scope) ? (scope as readonly Snapshot[]) : (scope as Scope).snapshots;
|
|
62
48
|
if (selected.length === 0) {
|
|
@@ -64,12 +50,6 @@ export async function copySnapshots(
|
|
|
64
50
|
"copySnapshots got no snapshots to copy. Check the experiments filter, or pass snapshots from openResults().latest().",
|
|
65
51
|
);
|
|
66
52
|
}
|
|
67
|
-
if (opts?.redact === undefined) {
|
|
68
|
-
throw new Error(
|
|
69
|
-
'copySnapshots requires an explicit "redact" option: pass a (text) => string sanitizer, or the literal false to declare this data safe to publish verbatim.',
|
|
70
|
-
);
|
|
71
|
-
}
|
|
72
|
-
const redactor: Redactor | undefined = opts.redact === false ? undefined : opts.redact;
|
|
73
53
|
const kinds = opts.artifacts ?? [...DEFAULT_PUBLISH_ARTIFACTS];
|
|
74
54
|
for (const kind of kinds) {
|
|
75
55
|
if (!VALID_ARTIFACTS.includes(kind)) {
|
|
@@ -97,11 +77,11 @@ export async function copySnapshots(
|
|
|
97
77
|
if (isNewerSnapshot(snapshot, existing)) byExperiment.set(snapshot.experimentId, snapshot);
|
|
98
78
|
}
|
|
99
79
|
|
|
100
|
-
//
|
|
101
|
-
//
|
|
80
|
+
// 发布前整文件预检:先规划并序列化全部目标文件,任一文件超过 PUBLISH_FILE_MAX_BYTES
|
|
81
|
+
// 就整体失败,不留半成品目标目录。
|
|
102
82
|
const planned: PlannedFile[] = [];
|
|
103
83
|
for (const snapshot of byExperiment.values()) {
|
|
104
|
-
planned.push(...(await planOneSnapshot(snapshot, [...selected], dest, kinds
|
|
84
|
+
planned.push(...(await planOneSnapshot(snapshot, [...selected], dest, kinds)));
|
|
105
85
|
}
|
|
106
86
|
const oversized = planned.filter((f) => f.bytes.byteLength > PUBLISH_FILE_MAX_BYTES);
|
|
107
87
|
if (oversized.length > 0) {
|
|
@@ -135,7 +115,6 @@ async function planOneSnapshot(
|
|
|
135
115
|
selected: Snapshot[],
|
|
136
116
|
destRoot: string,
|
|
137
117
|
kinds: ArtifactKind[],
|
|
138
|
-
redactor: Redactor | undefined,
|
|
139
118
|
): Promise<PlannedFile[]> {
|
|
140
119
|
const destSnapDir = join(destRoot, experimentDirOf(snapshot.experimentId), basename(snapshot.dir));
|
|
141
120
|
const planned: PlannedFile[] = [];
|
|
@@ -144,31 +123,22 @@ async function planOneSnapshot(
|
|
|
144
123
|
// 复制到目的地也只应该有一份——这个 Set 记录本快照已经规划过的 hash,整快照的 attempt 共享。
|
|
145
124
|
const plannedSourceHashes = new Set<string>();
|
|
146
125
|
for (const attempt of snapshot.attempts) {
|
|
147
|
-
planned.push(...(await planOneAttempt(attempt, destSnapDir, kinds, plannedSourceHashes
|
|
126
|
+
planned.push(...(await planOneAttempt(attempt, destSnapDir, kinds, plannedSourceHashes)));
|
|
148
127
|
}
|
|
149
128
|
|
|
150
129
|
const knownEvalIds = experimentOfSnapshot(snapshot)?.evalIds ?? fallbackUnion(selected, snapshot.experimentId);
|
|
151
|
-
const experiment =
|
|
152
|
-
snapshot.experiment !== undefined && redactor
|
|
153
|
-
? (redactExperimentInfo(snapshot.experiment as unknown as Record<string, unknown>, redactor) as unknown as SnapshotMeta["experiment"])
|
|
154
|
-
: snapshot.experiment;
|
|
155
130
|
const meta: SnapshotMeta = {
|
|
156
131
|
format: RESULTS_FORMAT,
|
|
157
132
|
schemaVersion: snapshot.schemaVersion,
|
|
158
133
|
producer: snapshot.producer,
|
|
159
134
|
experimentId: snapshot.experimentId,
|
|
160
|
-
...(experiment !== undefined ? { experiment } : {}),
|
|
135
|
+
...(snapshot.experiment !== undefined ? { experiment: snapshot.experiment } : {}),
|
|
161
136
|
agent: snapshot.agent,
|
|
162
137
|
...(snapshot.model !== undefined ? { model: snapshot.model } : {}),
|
|
163
138
|
startedAt: snapshot.startedAt,
|
|
164
139
|
...(snapshot.completedAt !== undefined ? { completedAt: snapshot.completedAt } : {}),
|
|
165
140
|
...(knownEvalIds.length ? { knownEvalIds } : {}),
|
|
166
|
-
...(snapshot.name !== undefined
|
|
167
|
-
? { name: redactor && typeof snapshot.name === "string" ? redactor(snapshot.name) : snapshot.name }
|
|
168
|
-
: {}),
|
|
169
|
-
// 发布根标记只声明流程,不证明结果:applied = 消毒函数对全部自由文本字段跑过;
|
|
170
|
-
// none = 作者显式的原文发布声明。view --out 的防呆据此分级。
|
|
171
|
-
publish: { redaction: redactor ? "applied" : "none" },
|
|
141
|
+
...(snapshot.name !== undefined ? { name: snapshot.name } : {}),
|
|
172
142
|
};
|
|
173
143
|
planned.push({ path: join(destSnapDir, SNAPSHOT_FILE), bytes: Buffer.from(JSON.stringify(meta, null, 2), "utf-8") });
|
|
174
144
|
return planned;
|
|
@@ -179,21 +149,18 @@ async function planOneAttempt(
|
|
|
179
149
|
destSnapDir: string,
|
|
180
150
|
kinds: ArtifactKind[],
|
|
181
151
|
plannedSourceHashes: Set<string>,
|
|
182
|
-
redactor: Redactor | undefined,
|
|
183
152
|
): Promise<PlannedFile[]> {
|
|
184
153
|
const destAttemptDir = join(destSnapDir, attempt.ref.attempt);
|
|
185
154
|
const planned: PlannedFile[] = [];
|
|
186
155
|
|
|
187
156
|
// sources 是唯一「两层」的 artifact(attempt 级引用 + 快照级去重仓库),不能像其它四类那样
|
|
188
157
|
// 单文件原字节完事——原字节只是引用,不带内容。走读取面已经会解引用+回退的 attempt.sources()
|
|
189
|
-
//
|
|
158
|
+
// 拿到完整内容,按内容哈希重新去重落盘——发布根里引用与内容永远一致,携带条目也被归拢进本快照。
|
|
190
159
|
const genericKinds = kinds.filter((k) => k !== "sources");
|
|
191
160
|
const files = await findArtifactFiles(attempt, genericKinds);
|
|
192
161
|
const copied = new Set(files.map((f) => f.kind));
|
|
193
162
|
for (const { kind, source } of files) {
|
|
194
|
-
|
|
195
|
-
const bytes = redactor ? redactArtifactBytes(kind, raw, redactor) : raw;
|
|
196
|
-
planned.push({ path: join(destAttemptDir, artifactFileOf(kind)), bytes, source });
|
|
163
|
+
planned.push({ path: join(destAttemptDir, artifactFileOf(kind)), bytes: await readFile(source), source });
|
|
197
164
|
}
|
|
198
165
|
|
|
199
166
|
if (kinds.includes("sources")) {
|
|
@@ -203,13 +170,12 @@ async function planOneAttempt(
|
|
|
203
170
|
const destStoreDir = join(destSnapDir, "sources");
|
|
204
171
|
const refs: { path: string; sha256: string }[] = [];
|
|
205
172
|
for (const src of sources) {
|
|
206
|
-
const
|
|
207
|
-
const sha256 = hashEvalSource(normalizeEvalSource(content));
|
|
173
|
+
const sha256 = hashEvalSource(normalizeEvalSource(src.content));
|
|
208
174
|
refs.push({ path: src.path, sha256 });
|
|
209
175
|
if (!plannedSourceHashes.has(sha256)) {
|
|
210
176
|
planned.push({
|
|
211
177
|
path: join(destStoreDir, `${sha256}.json`),
|
|
212
|
-
bytes: Buffer.from(JSON.stringify({ content }), "utf-8"),
|
|
178
|
+
bytes: Buffer.from(JSON.stringify({ content: src.content }), "utf-8"),
|
|
213
179
|
});
|
|
214
180
|
plannedSourceHashes.add(sha256);
|
|
215
181
|
}
|
|
@@ -221,40 +187,11 @@ async function planOneAttempt(
|
|
|
221
187
|
}
|
|
222
188
|
}
|
|
223
189
|
|
|
224
|
-
|
|
225
|
-
if (redactor) record = redactResultRecord(record, redactor);
|
|
190
|
+
const record = slimForCopy(attempt.result, copied);
|
|
226
191
|
planned.push({ path: join(destAttemptDir, RESULT_FILE), bytes: Buffer.from(JSON.stringify(record, null, 2), "utf-8") });
|
|
227
192
|
return planned;
|
|
228
193
|
}
|
|
229
194
|
|
|
230
|
-
/** 单个 artifact 文件的消毒:按种类解析 JSON、走各自的自由文本标注,重新序列化。 */
|
|
231
|
-
function redactArtifactBytes(kind: ArtifactKind, raw: Buffer, redactor: Redactor): Buffer {
|
|
232
|
-
try {
|
|
233
|
-
const parsed = JSON.parse(raw.toString("utf-8")) as unknown;
|
|
234
|
-
let next: unknown;
|
|
235
|
-
switch (kind) {
|
|
236
|
-
case "events":
|
|
237
|
-
next = redactEvents(parsed as StreamEvent[], redactor);
|
|
238
|
-
break;
|
|
239
|
-
case "trace":
|
|
240
|
-
next = redactSpans(parsed as TraceSpan[], redactor);
|
|
241
|
-
break;
|
|
242
|
-
// diff 的 before/after/内容与 o11y 的命令 / 错误 / URL 都是自由文本;路径键为结构字段。
|
|
243
|
-
case "diff":
|
|
244
|
-
case "o11y":
|
|
245
|
-
case "agentSetup":
|
|
246
|
-
next = redactJsonValue(parsed, redactor);
|
|
247
|
-
break;
|
|
248
|
-
default:
|
|
249
|
-
next = parsed;
|
|
250
|
-
}
|
|
251
|
-
return Buffer.from(JSON.stringify(next), "utf-8");
|
|
252
|
-
} catch {
|
|
253
|
-
// 解析不了的按原字节透传(malformed 数据不该在发布路径上被静默改写)。
|
|
254
|
-
return raw;
|
|
255
|
-
}
|
|
256
|
-
}
|
|
257
|
-
|
|
258
195
|
/** 目标目录非空即报错:盘上不该出现「我没写的东西被动过」的惊讶;发布脚本要幂等就自己先清目录。 */
|
|
259
196
|
async function assertEmptyDestination(dest: string): Promise<void> {
|
|
260
197
|
let entries: string[];
|
|
@@ -126,7 +126,8 @@ interface NormExperiment {
|
|
|
126
126
|
type NormWarning =
|
|
127
127
|
| { kind: "partial-coverage"; experimentId: string; covered: number; total: number }
|
|
128
128
|
| { kind: "stale-snapshot"; experimentId: string; startedAt: string; latestStartedAt: string }
|
|
129
|
-
| { kind: "unfinished-snapshot"; experimentId: string; startedAt: string }
|
|
129
|
+
| { kind: "unfinished-snapshot"; experimentId: string; startedAt: string }
|
|
130
|
+
| { kind: "unreadable-snapshot"; reason: string };
|
|
130
131
|
interface NormSelection {
|
|
131
132
|
warnings: NormWarning[];
|
|
132
133
|
experiments: NormExperiment[];
|
|
@@ -141,6 +142,9 @@ function normalizeWarning(w: ScopeWarning): NormWarning {
|
|
|
141
142
|
case "unfinished-snapshot":
|
|
142
143
|
// dir 是宿主机绝对路径,归一化掉;身份靠 experimentId + startedAt。
|
|
143
144
|
return { kind: w.kind, experimentId: w.experimentId, startedAt: w.startedAt };
|
|
145
|
+
case "unreadable-snapshot":
|
|
146
|
+
// dir 是宿主机绝对路径,归一化掉;这个 kind 本就非实验作用域,没有 experimentId 可比。
|
|
147
|
+
return { kind: w.kind, reason: w.reason };
|
|
144
148
|
}
|
|
145
149
|
}
|
|
146
150
|
|
package/src/results/open.ts
CHANGED
|
@@ -150,7 +150,7 @@ export async function openResults(dir: string): Promise<Results> {
|
|
|
150
150
|
try {
|
|
151
151
|
targetStat = await stat(target);
|
|
152
152
|
} catch {
|
|
153
|
-
return makeResults([], []);
|
|
153
|
+
return makeResults([], [], target);
|
|
154
154
|
}
|
|
155
155
|
|
|
156
156
|
if (targetStat.isFile()) {
|
|
@@ -165,17 +165,18 @@ export async function openResults(dir: string): Promise<Results> {
|
|
|
165
165
|
// 建 locator 索引:必须在全部快照扫完、Experiment 归组完成之后,返回 Results 之前——
|
|
166
166
|
// 撞车(LocatorCollisionError)在这里抛,不静默吞、不拖到消费方第一次 resolveLocator() 才发现。
|
|
167
167
|
const locatorIndex = buildAttemptLocatorIndex(experiments);
|
|
168
|
-
const results = makeResults(experiments, state.skipped);
|
|
168
|
+
const results = makeResults(experiments, state.skipped, target);
|
|
169
169
|
locatorIndexByResults.set(results, locatorIndex);
|
|
170
170
|
return results;
|
|
171
171
|
}
|
|
172
172
|
|
|
173
|
-
function makeResults(experiments: Experiment[], skipped: SkippedDir[]): Results {
|
|
173
|
+
function makeResults(experiments: Experiment[], skipped: SkippedDir[], root: string): Results {
|
|
174
174
|
const results: Results = {
|
|
175
|
+
root,
|
|
175
176
|
experiments,
|
|
176
177
|
skipped,
|
|
177
178
|
latest(opts?: { experiments?: string | string[] }): Scope {
|
|
178
|
-
return selectLatest(
|
|
179
|
+
return selectLatest(results, opts);
|
|
179
180
|
},
|
|
180
181
|
current(opts?: { experiments?: string | string[] }): Scope {
|
|
181
182
|
return selectCurrentResults(results, { experiment: opts?.experiments });
|
package/src/results/publish.ts
CHANGED
|
@@ -1,149 +1,7 @@
|
|
|
1
|
-
//
|
|
2
|
-
//
|
|
3
|
-
//
|
|
4
|
-
//
|
|
5
|
-
// redact 逐值消毒,范围由 schema 的自由文本标注决定:redactor 只对自由文本字段调用——
|
|
6
|
-
// 格式、判定、身份、路径、哈希这类结构字段永不经过它,发布根不会因 redact 变得不可读或
|
|
7
|
-
// 引用断裂。自由文本清单在下面的 FREE_TEXT 标注单点维护(与 AttemptList.redact 同一口径)。
|
|
8
|
-
|
|
9
|
-
import type { EvalResult, StreamEvent, TraceSpan } from "../types.ts";
|
|
1
|
+
// 发布预算(见 docs/feature/results/library.md「复制与瘦身:copySnapshots」)。
|
|
2
|
+
// 结果数据分两类:.niceeval/ 是本地事实根,不是默认可提交目录;任何要离开本机的拷贝是
|
|
3
|
+
// 发布拷贝,经 copySnapshots 这一条管线产出。管线只做选择、归拢与整文件大小预检,
|
|
4
|
+
// 不改写内容——保密边界由格式在采集侧划定(env 值与命令 stdout/stderr 不进结果文件)。
|
|
10
5
|
|
|
11
6
|
/** 发布前整文件预检的单文件上限(50 MiB,为 GitHub 100 MB 硬限保留余量);不是可调旋钮。 */
|
|
12
7
|
export const PUBLISH_FILE_MAX_BYTES = 50 * 1024 * 1024;
|
|
13
|
-
|
|
14
|
-
export type Redactor = (text: string) => string;
|
|
15
|
-
|
|
16
|
-
/**
|
|
17
|
-
* 结构字段键名(取值是格式、判定、身份、路径、哈希——redactor 永不触碰):
|
|
18
|
-
* 事件的 type/callId/status/role/tool/skill/requestId/optionId、span 的 ids/kind、
|
|
19
|
-
* 断言的 name/severity/outcome、错误的 code/phase、locator/artifactBase/fingerprint、
|
|
20
|
-
* 源码路径与 sha256、provider 名、时间戳等。新增字符串字段先判断体裁再决定进不进这张表。
|
|
21
|
-
*/
|
|
22
|
-
const STRUCTURAL_KEYS = new Set([
|
|
23
|
-
"type",
|
|
24
|
-
"callId",
|
|
25
|
-
"status",
|
|
26
|
-
"role",
|
|
27
|
-
"tool",
|
|
28
|
-
"requestId",
|
|
29
|
-
"optionId",
|
|
30
|
-
"id",
|
|
31
|
-
"traceId",
|
|
32
|
-
"spanId",
|
|
33
|
-
"parentSpanId",
|
|
34
|
-
"kind",
|
|
35
|
-
"format",
|
|
36
|
-
"verdict",
|
|
37
|
-
"severity",
|
|
38
|
-
"outcome",
|
|
39
|
-
"artifactBase",
|
|
40
|
-
"locator",
|
|
41
|
-
"fingerprint",
|
|
42
|
-
"sha256",
|
|
43
|
-
"path",
|
|
44
|
-
"file",
|
|
45
|
-
"provider",
|
|
46
|
-
"sandboxId",
|
|
47
|
-
"code",
|
|
48
|
-
"phase",
|
|
49
|
-
"level",
|
|
50
|
-
"agent",
|
|
51
|
-
"model",
|
|
52
|
-
"experimentId",
|
|
53
|
-
"startedAt",
|
|
54
|
-
"completedAt",
|
|
55
|
-
"schemaVersion",
|
|
56
|
-
"evalFilterFingerprint",
|
|
57
|
-
"reasoningEffort",
|
|
58
|
-
"skill",
|
|
59
|
-
"window",
|
|
60
|
-
"net",
|
|
61
|
-
"loc",
|
|
62
|
-
"dedupeKey",
|
|
63
|
-
]);
|
|
64
|
-
|
|
65
|
-
/** 深度遍历 JSON 值:字符串按「键名是否结构字段」决定过不过 redactor;结构键下整棵子树跳过。 */
|
|
66
|
-
export function redactJsonValue(value: unknown, redact: Redactor, key?: string): unknown {
|
|
67
|
-
if (typeof value === "string") {
|
|
68
|
-
if (key !== undefined && STRUCTURAL_KEYS.has(key)) return value;
|
|
69
|
-
return redact(value);
|
|
70
|
-
}
|
|
71
|
-
if (Array.isArray(value)) return value.map((v) => redactJsonValue(v, redact, key));
|
|
72
|
-
if (value !== null && typeof value === "object") {
|
|
73
|
-
const out: Record<string, unknown> = {};
|
|
74
|
-
for (const [k, v] of Object.entries(value)) {
|
|
75
|
-
if (STRUCTURAL_KEYS.has(k) && typeof v === "string") {
|
|
76
|
-
out[k] = v;
|
|
77
|
-
continue;
|
|
78
|
-
}
|
|
79
|
-
out[k] = redactJsonValue(v, redact, k);
|
|
80
|
-
}
|
|
81
|
-
return out;
|
|
82
|
-
}
|
|
83
|
-
return value;
|
|
84
|
-
}
|
|
85
|
-
|
|
86
|
-
/** events.json 的发布消毒:消息与工具入出参是自由文本,type/callId/status 等结构字段不动。 */
|
|
87
|
-
export function redactEvents(events: StreamEvent[], redact: Redactor): StreamEvent[] {
|
|
88
|
-
return redactJsonValue(events, redact) as StreamEvent[];
|
|
89
|
-
}
|
|
90
|
-
|
|
91
|
-
/** trace.json 的发布消毒:属性值与可携带动态内容的 span name 过 redactor,ids/kind 不动。 */
|
|
92
|
-
export function redactSpans(spans: TraceSpan[], redact: Redactor): TraceSpan[] {
|
|
93
|
-
return spans.map((span) => ({
|
|
94
|
-
...span,
|
|
95
|
-
name: redact(span.name),
|
|
96
|
-
...(span.attributes !== undefined
|
|
97
|
-
? { attributes: redactJsonValue(span.attributes, redact) as TraceSpan["attributes"] }
|
|
98
|
-
: {}),
|
|
99
|
-
}));
|
|
100
|
-
}
|
|
101
|
-
|
|
102
|
-
/** result.json 的发布消毒:断言 detail/evidence/expected/received、error/diagnostic 的
|
|
103
|
-
* message/cause/stack、skipReason、description 是自由文本;判定与身份字段不动。 */
|
|
104
|
-
export function redactResultRecord(record: Record<string, unknown>, redact: Redactor): Record<string, unknown> {
|
|
105
|
-
const r = record as Partial<EvalResult> & Record<string, unknown>;
|
|
106
|
-
const out: Record<string, unknown> = { ...record };
|
|
107
|
-
if (typeof r.description === "string") out.description = redact(r.description);
|
|
108
|
-
if (typeof r.skipReason === "string") out.skipReason = redact(r.skipReason);
|
|
109
|
-
if (Array.isArray(r.assertions)) {
|
|
110
|
-
out.assertions = r.assertions.map((a) => ({
|
|
111
|
-
...a,
|
|
112
|
-
...(a.detail !== undefined ? { detail: redact(a.detail) } : {}),
|
|
113
|
-
...(a.outcome !== "unavailable" && a.evidence !== undefined ? { evidence: redact(a.evidence) } : {}),
|
|
114
|
-
...(a.outcome !== "unavailable" && a.expected !== undefined ? { expected: redact(a.expected) } : {}),
|
|
115
|
-
...(a.outcome !== "unavailable" && a.received !== undefined ? { received: redact(a.received) } : {}),
|
|
116
|
-
}));
|
|
117
|
-
}
|
|
118
|
-
if (r.error !== undefined) {
|
|
119
|
-
out.error = {
|
|
120
|
-
...r.error,
|
|
121
|
-
message: redact(r.error.message),
|
|
122
|
-
...(r.error.stack !== undefined ? { stack: redact(r.error.stack) } : {}),
|
|
123
|
-
...(r.error.cause !== undefined ? { cause: { ...r.error.cause, message: redact(r.error.cause.message) } } : {}),
|
|
124
|
-
};
|
|
125
|
-
}
|
|
126
|
-
if (Array.isArray(r.diagnostics)) {
|
|
127
|
-
out.diagnostics = r.diagnostics.map((d) => ({
|
|
128
|
-
...d,
|
|
129
|
-
message: redact(d.message),
|
|
130
|
-
...(d.data !== undefined ? { data: redactJsonValue(d.data, redact) as typeof d.data } : {}),
|
|
131
|
-
}));
|
|
132
|
-
}
|
|
133
|
-
if (r.experiment !== undefined) {
|
|
134
|
-
out.experiment = redactExperimentInfo(r.experiment as unknown as Record<string, unknown>, redact);
|
|
135
|
-
}
|
|
136
|
-
return out;
|
|
137
|
-
}
|
|
138
|
-
|
|
139
|
-
/** ExperimentRunInfo 的发布消毒:description、flags 与 sandbox params 的字符串值。 */
|
|
140
|
-
export function redactExperimentInfo(info: Record<string, unknown>, redact: Redactor): Record<string, unknown> {
|
|
141
|
-
const out: Record<string, unknown> = { ...info };
|
|
142
|
-
if (typeof info.description === "string") out.description = redact(info.description);
|
|
143
|
-
if (info.flags !== undefined) out.flags = redactJsonValue(info.flags, redact);
|
|
144
|
-
const sandbox = info.sandbox as { provider: string; params?: Record<string, unknown> } | undefined;
|
|
145
|
-
if (sandbox?.params !== undefined) {
|
|
146
|
-
out.sandbox = { ...sandbox, params: redactJsonValue(sandbox.params, redact) };
|
|
147
|
-
}
|
|
148
|
-
return out;
|
|
149
|
-
}
|