niceeval 0.6.1 → 0.6.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agents/types.d.ts +67 -5
- package/dist/context/types.d.ts +32 -12
- package/dist/i18n/en.d.ts +54 -0
- package/dist/i18n/zh-CN.d.ts +55 -1
- package/dist/o11y/types.d.ts +16 -2
- package/dist/report/aggregate.d.ts +5 -3
- package/dist/report/aggregate.js +32 -5
- package/dist/report/built-ins/experiment-comparison.d.ts +39 -1
- package/dist/report/built-ins/experiment-comparison.js +116 -10
- package/dist/report/built-ins/index.d.ts +1 -0
- package/dist/report/built-ins/index.js +1 -1
- package/dist/report/components.d.ts +8 -2
- package/dist/report/components.js +3 -3
- package/dist/report/compute.d.ts +11 -18
- package/dist/report/compute.js +54 -34
- package/dist/report/flag.d.ts +16 -1
- package/dist/report/flag.js +19 -1
- package/dist/report/format.d.ts +16 -8
- package/dist/report/format.js +27 -12
- package/dist/report/index.d.ts +4 -3
- package/dist/report/index.js +5 -4
- package/dist/report/locale.d.ts +11 -2
- package/dist/report/locale.js +23 -5
- package/dist/report/metrics.d.ts +13 -1
- package/dist/report/metrics.js +65 -14
- package/dist/report/primitives.d.ts +6 -0
- package/dist/report/react/AttemptList.d.ts +2 -2
- package/dist/report/react/AttemptList.js +5 -6
- package/dist/report/react/EvalList.d.ts +1 -1
- package/dist/report/react/EvalList.js +0 -0
- package/dist/report/react/ExperimentComparison.d.ts +8 -0
- package/dist/report/react/ExperimentComparison.js +11 -0
- package/dist/report/react/ExperimentList.d.ts +2 -1
- package/dist/report/react/ExperimentList.js +8 -10
- package/dist/report/react/MetricScatter.js +5 -11
- package/dist/report/react/chart-math.d.ts +23 -6
- package/dist/report/react/chart-math.js +71 -19
- package/dist/report/react/fixtures.d.ts +3 -3
- package/dist/report/react/fixtures.js +21 -14
- package/dist/report/report.d.ts +5 -1
- package/dist/report/report.js +6 -2
- package/dist/report/text/faces.d.ts +1 -1
- package/dist/report/text/faces.js +42 -41
- package/dist/report/text/table.js +36 -5
- package/dist/report/types.d.ts +39 -21
- package/dist/results/types.d.ts +11 -0
- package/dist/runner/feedback/sink.d.ts +110 -0
- package/dist/runner/types.d.ts +513 -22
- package/dist/sandbox/docker.d.ts +23 -2
- package/dist/sandbox/e2b.d.ts +15 -1
- package/dist/sandbox/errors.d.ts +30 -3
- package/dist/sandbox/io-retry.d.ts +17 -0
- package/dist/sandbox/registry.d.ts +2 -0
- package/dist/sandbox/resolve.d.ts +18 -5
- package/dist/sandbox/retry.d.ts +11 -1
- package/dist/sandbox/types.d.ts +39 -5
- package/dist/sandbox/vercel.d.ts +7 -1
- package/dist/scoring/coverage.d.ts +30 -0
- package/dist/scoring/display.d.ts +21 -0
- package/dist/scoring/display.js +120 -0
- package/dist/scoring/types.d.ts +103 -20
- package/dist/shared/aggregate.d.ts +1 -0
- package/dist/shared/aggregate.js +3 -3
- package/dist/shared/types.d.ts +28 -0
- package/dist/tty-line.d.ts +0 -4
- package/dist/util.d.ts +23 -0
- package/docs-site/zh/concepts/adapter.mdx +22 -4
- package/docs-site/zh/concepts/experiment.mdx +1 -1
- package/docs-site/zh/concepts/overview.mdx +6 -6
- package/docs-site/zh/guides/agent-feedback-loop.mdx +28 -26
- package/docs-site/zh/guides/authoring.mdx +33 -0
- package/docs-site/zh/guides/ci-integration.mdx +23 -12
- package/docs-site/zh/guides/connect-your-agent.mdx +29 -3
- package/docs-site/zh/guides/custom-reports.mdx +29 -34
- package/docs-site/zh/guides/dataset-fanout.mdx +25 -3
- package/docs-site/zh/guides/debug-sandbox.mdx +57 -0
- package/docs-site/zh/guides/debugging.mdx +210 -0
- package/docs-site/zh/guides/experiments.mdx +10 -3
- package/docs-site/zh/guides/official-adapters.mdx +26 -2
- package/docs-site/zh/guides/publish-report.mdx +30 -16
- package/docs-site/zh/guides/report-components.mdx +42 -30
- package/docs-site/zh/guides/reporters.mdx +2 -2
- package/docs-site/zh/guides/results-data.mdx +17 -9
- package/docs-site/zh/guides/runner.mdx +17 -7
- package/docs-site/zh/guides/sandbox-agent.mdx +56 -7
- package/docs-site/zh/guides/sandbox-providers.mdx +257 -9
- package/docs-site/zh/guides/scoring-guide.mdx +4 -4
- package/docs-site/zh/guides/viewing-results.mdx +79 -36
- package/docs-site/zh/guides/write-experiment.mdx +5 -3
- package/docs-site/zh/guides/write-send.mdx +17 -1
- package/docs-site/zh/index.mdx +1 -1
- package/docs-site/zh/reference/builtin-agents.mdx +27 -0
- package/docs-site/zh/reference/capabilities.mdx +2 -2
- package/docs-site/zh/reference/cli.mdx +33 -7
- package/docs-site/zh/reference/define-agent.mdx +57 -4
- package/docs-site/zh/reference/define-config.mdx +1 -1
- package/docs-site/zh/reference/define-eval.mdx +42 -9
- package/docs-site/zh/reference/expect.mdx +26 -1
- package/package.json +5 -1
- package/src/agents/ai-sdk-otel.test.ts +1 -0
- package/src/agents/ai-sdk.test.ts +3 -0
- package/src/agents/ai-sdk.ts +3 -0
- package/src/agents/bub-install-spec.test.ts +34 -0
- package/src/agents/bub-install-spec.ts +32 -0
- package/src/agents/bub.ts +31 -32
- package/src/agents/claude-code.test.ts +130 -9
- package/src/agents/claude-code.ts +76 -4
- package/src/agents/codex.test.ts +189 -40
- package/src/agents/codex.ts +155 -14
- package/src/agents/coding-cli-versions.test.ts +15 -0
- package/src/agents/coding-cli-versions.ts +3 -0
- package/src/agents/index.ts +11 -0
- package/src/agents/langgraph.test.ts +204 -0
- package/src/agents/langgraph.ts +495 -0
- package/src/agents/marketplace.ts +85 -0
- package/src/agents/native-config.test.ts +179 -0
- package/src/agents/native-config.ts +267 -0
- package/src/agents/openai-compat.test.ts +1 -0
- package/src/agents/openclaw.test.ts +31 -0
- package/src/agents/openclaw.ts +171 -0
- package/src/agents/plugin-config.test.ts +1 -0
- package/src/agents/sdk-streams.test.ts +79 -0
- package/src/agents/sdk-streams.ts +55 -10
- package/src/agents/skills.test.ts +1 -0
- package/src/agents/streaming.test.ts +3 -9
- package/src/agents/types.ts +68 -5
- package/src/agents/ui-message-stream.test.ts +3 -0
- package/src/cli.ts +411 -108
- package/src/context/context.test.ts +51 -12
- package/src/context/context.ts +161 -29
- package/src/context/session.test.ts +1 -0
- package/src/context/session.ts +114 -6
- package/src/context/types.ts +30 -12
- package/src/define.test.ts +13 -8
- package/src/define.ts +25 -4
- package/src/expect/index.ts +53 -23
- package/src/i18n/en.ts +64 -2
- package/src/i18n/zh-CN.ts +65 -3
- package/src/o11y/cost.test.ts +1 -0
- package/src/o11y/execution-tree.test.ts +1 -20
- package/src/o11y/otlp/mappers/claude-code.test.ts +1 -0
- package/src/o11y/otlp/parse.test.ts +1 -0
- package/src/o11y/otlp/turn-otel.test.ts +1 -0
- package/src/o11y/parsers/bub.test.ts +1 -0
- package/src/o11y/parsers/claude-code.test.ts +1 -34
- package/src/o11y/parsers/openclaw.test.ts +154 -0
- package/src/o11y/parsers/openclaw.ts +310 -0
- package/src/o11y/prices.json +746 -311
- package/src/o11y/tool-names.test.ts +1 -0
- package/src/o11y/types.ts +16 -2
- package/src/report/aggregate.ts +34 -5
- package/src/report/built-in-user-parity.test.tsx +110 -153
- package/src/report/built-ins/experiment-comparison.tsx +173 -13
- package/src/report/built-ins/index.ts +6 -1
- package/src/report/components.tsx +9 -3
- package/src/report/compute.ts +70 -40
- package/src/report/dual-render.test.tsx +194 -67
- package/src/report/flag.ts +30 -2
- package/src/report/format.ts +35 -11
- package/src/report/index.ts +22 -4
- package/src/report/locale.ts +25 -5
- package/src/report/metrics.ts +67 -14
- package/src/report/primitives.tsx +6 -0
- package/src/report/react/AttemptList.tsx +6 -31
- package/src/report/react/EvalList.tsx +0 -0
- package/src/report/react/ExperimentComparison.tsx +68 -0
- package/src/report/react/ExperimentList.tsx +15 -9
- package/src/report/react/MetricScatter.tsx +12 -14
- package/src/report/react/chart-math.test.ts +85 -0
- package/src/report/react/chart-math.ts +101 -22
- package/src/report/react/enhance.js +33 -1
- package/src/report/react/fixtures.ts +24 -17
- package/src/report/react/render.test.tsx +9 -64
- package/src/report/react/styles.css +73 -2
- package/src/report/report.test.ts +306 -98
- package/src/report/report.ts +6 -2
- package/src/report/text/faces.ts +47 -43
- package/src/report/text/table.ts +42 -5
- package/src/report/types.ts +41 -21
- package/src/results/annotated-source.test.ts +62 -9
- package/src/results/annotated-source.ts +64 -6
- package/src/results/attempt-evidence.test.ts +9 -7
- package/src/results/attempt-evidence.ts +15 -8
- package/src/results/attempt-source.ts +6 -3
- package/src/results/copy.ts +145 -55
- package/src/results/host-equivalence.test.ts +8 -6
- package/src/results/index.ts +2 -0
- package/src/results/locator.test.ts +1 -22
- package/src/results/open.ts +7 -1
- package/src/results/publish.ts +149 -0
- package/src/results/results.test.ts +85 -51
- package/src/results/truncate.ts +90 -0
- package/src/results/types.ts +7 -0
- package/src/results/writer.ts +31 -13
- package/src/runner/attempt.test.ts +138 -7
- package/src/runner/attempt.ts +603 -104
- package/src/runner/discover.test.ts +47 -0
- package/src/runner/discover.ts +36 -2
- package/src/runner/eval-source.test.ts +1 -27
- package/src/runner/feedback/agent.test.ts +504 -0
- package/src/runner/feedback/agent.ts +409 -0
- package/src/runner/feedback/ci.test.ts +562 -0
- package/src/runner/feedback/ci.ts +401 -0
- package/src/runner/feedback/coordinator.test.ts +317 -0
- package/src/runner/feedback/coordinator.ts +397 -0
- package/src/runner/feedback/failure.ts +40 -0
- package/src/runner/feedback/human.test.ts +616 -0
- package/src/runner/feedback/human.ts +535 -0
- package/src/runner/feedback/index.ts +66 -0
- package/src/runner/feedback/io.ts +78 -0
- package/src/runner/feedback/profile.test.ts +50 -0
- package/src/runner/feedback/profile.ts +58 -0
- package/src/runner/feedback/reducer.test.ts +395 -0
- package/src/runner/feedback/reducer.ts +260 -0
- package/src/runner/feedback/renderer.ts +82 -0
- package/src/runner/feedback/sink.ts +203 -0
- package/src/runner/feedback/testing.ts +106 -0
- package/src/runner/ledger.test.ts +230 -0
- package/src/runner/ledger.ts +329 -0
- package/src/runner/report.test.ts +128 -3
- package/src/runner/report.ts +33 -9
- package/src/runner/reporters/artifacts.ts +8 -2
- package/src/runner/reporters/braintrust.test.ts +8 -7
- package/src/runner/reporters/braintrust.ts +9 -2
- package/src/runner/reporters/index.ts +2 -2
- package/src/runner/reporters/json.test.ts +162 -0
- package/src/runner/reporters/json.ts +35 -8
- package/src/runner/reporters/shared.ts +1 -5
- package/src/runner/run.test.ts +760 -3
- package/src/runner/run.ts +242 -36
- package/src/runner/sandbox-prep.ts +3 -42
- package/src/runner/timing.ts +158 -0
- package/src/runner/types.ts +518 -22
- package/src/sandbox/checkpoint.test.ts +55 -0
- package/src/sandbox/checkpoint.ts +29 -8
- package/src/sandbox/cli-commands.ts +407 -0
- package/src/sandbox/docker.ts +115 -16
- package/src/sandbox/e2b-agent-template.test.ts +56 -0
- package/src/sandbox/e2b-agent-template.ts +94 -0
- package/src/sandbox/e2b.ts +74 -9
- package/src/sandbox/errors.ts +111 -4
- package/src/sandbox/index.ts +2 -0
- package/src/sandbox/io-retry.test.ts +58 -0
- package/src/sandbox/io-retry.ts +45 -0
- package/src/sandbox/keep-registry.test.ts +86 -0
- package/src/sandbox/keep-registry.ts +142 -0
- package/src/sandbox/keep.ts +178 -0
- package/src/sandbox/paths.test.ts +1 -0
- package/src/sandbox/paths.ts +19 -8
- package/src/sandbox/registry.ts +20 -3
- package/src/sandbox/resolve.ts +76 -11
- package/src/sandbox/retry.test.ts +70 -0
- package/src/sandbox/retry.ts +46 -4
- package/src/sandbox/types.ts +44 -6
- package/src/sandbox/vercel.ts +43 -20
- package/src/scoring/collector.ts +60 -17
- package/src/scoring/coverage.ts +95 -0
- package/src/scoring/diff.ts +81 -0
- package/src/scoring/display.test.ts +121 -0
- package/src/scoring/display.ts +133 -0
- package/src/scoring/evidence.test.ts +189 -0
- package/src/scoring/judge.test.ts +142 -0
- package/src/scoring/judge.ts +15 -18
- package/src/scoring/scoped.ts +217 -50
- package/src/scoring/types.ts +117 -20
- package/src/scoring/verdict.ts +16 -4
- package/src/shared/aggregate.ts +3 -2
- package/src/shared/types.ts +31 -0
- package/src/show/compose.ts +2 -2
- package/src/show/index.ts +21 -1
- package/src/show/render.ts +619 -104
- package/src/show/show.test.ts +235 -19
- package/src/tty-line.ts +8 -26
- package/src/util.test.ts +1 -0
- package/src/util.ts +41 -0
- package/src/view/app/components/AttemptModal.tsx +153 -2
- package/src/view/app/components/CodeView.tsx +32 -11
- package/src/view/app/components/CopyControls.tsx +2 -2
- package/src/view/app/i18n.ts +6 -0
- package/src/view/app/lib/attempt-route.test.ts +1 -0
- package/src/view/app/lib/verdict.ts +7 -9
- package/src/view/artifact-serving.test.ts +2 -1
- package/src/view/client-dist/app.css +1 -1
- package/src/view/client-dist/app.js +17 -17
- package/src/view/data.test.ts +1 -0
- package/src/view/data.ts +11 -1
- package/src/view/index.ts +11 -0
- package/src/view/server.ts +2 -0
- package/src/view/styles.css +3 -0
- package/src/view/view-report.test.ts +6 -5
- package/src/runner/reporters/console.ts +0 -70
- package/src/runner/reporters/live.test.ts +0 -56
- package/src/runner/reporters/live.ts +0 -247
- package/src/runner/reporters/quiet.test.ts +0 -66
- package/src/runner/reporters/quiet.ts +0 -49
- package/src/runner/reporters/table.ts +0 -277
|
@@ -6,16 +6,33 @@
|
|
|
6
6
|
//
|
|
7
7
|
// 泛化自 src/view/app/lib/transcript-data.tsx 的 indexAsserts()(Map<string, Assertion[]>
|
|
8
8
|
// + noloc 兜底桶的先例)——同一套"按 loc 分桶、没有 loc 进兜底"的思路,换成源码行数组
|
|
9
|
-
// (不是 Map)存放,外加 sourceSha256 / summary
|
|
10
|
-
//
|
|
9
|
+
// (不是 Map)存放,外加 sourceSha256 / summary 计数。轮次的完整展开(events → ExecutionTree)
|
|
10
|
+
// 仍是 ExecutionTree 的地盘;这个模型只把每轮的头行事实标回 send 调用行(见 SendAnnotation),
|
|
11
|
+
// 作为源码页指向 --execution 的跨面指针。
|
|
11
12
|
//
|
|
12
13
|
// 本模块不 import react/jsx,纯数据 + 纯函数,可以在任何 Node 语境(CLI 的 show、
|
|
13
14
|
// view 的 server 端数据准备)调用。
|
|
14
15
|
|
|
15
|
-
import type { AssertionResult } from "../types.ts";
|
|
16
|
+
import type { AssertionResult, PhaseTiming, SourceLoc, StreamEvent } from "../types.ts";
|
|
16
17
|
import { hashEvalSource, normalizeEvalSource } from "./source-hash.ts";
|
|
17
18
|
|
|
18
|
-
/**
|
|
19
|
+
/**
|
|
20
|
+
* 标回 `t.send(...)` 调用行的一轮 turn 头行事实(契约见 docs/feature/reports/show.md
|
|
21
|
+
* 「--eval:把断言放回源码」)。身份标签与 --execution / --timing / diff windows 同一套;
|
|
22
|
+
* 回复全文与轮内卡片不进这个模型——源码页只回答「这行代码对应哪一轮、这一轮成了没成」。
|
|
23
|
+
*/
|
|
24
|
+
export interface SendAnnotation {
|
|
25
|
+
/** `s<session>/t<turn>`。 */
|
|
26
|
+
label: string;
|
|
27
|
+
/** 轮的终态;时间树只记 failed 位,waiting 需要事件流佐证时由派生方给。 */
|
|
28
|
+
status: "completed" | "failed" | "waiting";
|
|
29
|
+
/** 该轮墙钟;时间树缺这一轮的节点时省略。 */
|
|
30
|
+
durationMs?: number;
|
|
31
|
+
/** send 调用位置(用户消息事件的 loc)。 */
|
|
32
|
+
loc: SourceLoc;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/** 一行源码 + 映射到这一行的全部断言与 send 标注(保持原始顺序;可以是空数组)。 */
|
|
19
36
|
export interface AnnotatedSourceLine {
|
|
20
37
|
/** 1-indexed 行号,与 SourceLoc.line 同一坐标系。 */
|
|
21
38
|
line: number;
|
|
@@ -28,6 +45,11 @@ export interface AnnotatedSourceLine {
|
|
|
28
45
|
* 领域 model 的职责(与 indexAsserts() 的先例一致,它同样只分桶、不折叠)。
|
|
29
46
|
*/
|
|
30
47
|
assertions: AssertionResult[];
|
|
48
|
+
/**
|
|
49
|
+
* 映射到这一行的 send 标注,按轮次顺序;循环里的 send 一行多轮。与断言的 never-drop
|
|
50
|
+
* 契约不同,定位不到行的轮不进任何兜底桶——轮次的全量面是 --execution,这里只是指针。
|
|
51
|
+
*/
|
|
52
|
+
sends: SendAnnotation[];
|
|
31
53
|
}
|
|
32
54
|
|
|
33
55
|
export interface AnnotatedEvalSourceSummary {
|
|
@@ -67,6 +89,34 @@ export interface AnnotatedEvalSource {
|
|
|
67
89
|
summary: AnnotatedEvalSourceSummary;
|
|
68
90
|
}
|
|
69
91
|
|
|
92
|
+
/**
|
|
93
|
+
* 从标准事件流与阶段时间树派生 send 标注:第 i 条用户消息开第 i 轮(与 --execution 的
|
|
94
|
+
* 分轮边界同一条规则,见 show/render.ts::executionText),头行事实取 `eval.run` 下第 i 个
|
|
95
|
+
* turn 节点;用户消息没有 loc 的轮不产出标注。纯函数,无 IO。
|
|
96
|
+
*/
|
|
97
|
+
export function deriveSendAnnotations(
|
|
98
|
+
events: readonly StreamEvent[] | null,
|
|
99
|
+
phases: readonly PhaseTiming[] | undefined,
|
|
100
|
+
): SendAnnotation[] {
|
|
101
|
+
if (!events || events.length === 0) return [];
|
|
102
|
+
const turnNodes = (phases ?? []).flatMap((p) => p.children ?? []).filter((n) => n.kind === "turn");
|
|
103
|
+
const out: SendAnnotation[] = [];
|
|
104
|
+
let turnIndex = -1;
|
|
105
|
+
for (const event of events) {
|
|
106
|
+
if (event.type !== "message" || event.role !== "user") continue;
|
|
107
|
+
turnIndex += 1;
|
|
108
|
+
if (!event.loc) continue;
|
|
109
|
+
const turn = turnNodes[turnIndex];
|
|
110
|
+
out.push({
|
|
111
|
+
label: turn?.label ?? `t${turnIndex + 1}`,
|
|
112
|
+
status: turn?.failed ? "failed" : "completed",
|
|
113
|
+
...(turn !== undefined ? { durationMs: turn.durationMs } : {}),
|
|
114
|
+
loc: event.loc,
|
|
115
|
+
});
|
|
116
|
+
}
|
|
117
|
+
return out;
|
|
118
|
+
}
|
|
119
|
+
|
|
70
120
|
/**
|
|
71
121
|
* 纯函数:给定一份源码文本(未归一化也可以,内部会 normalizeEvalSource)与一批断言,
|
|
72
122
|
* 产出标注好的 AnnotatedEvalSource。幂等 / 无 IO —— 可以在渲染路径上安全调用。
|
|
@@ -77,13 +127,21 @@ export interface AnnotatedEvalSource {
|
|
|
77
127
|
export function buildAnnotatedEvalSource(
|
|
78
128
|
source: { path: string; content: string },
|
|
79
129
|
assertions: readonly AssertionResult[],
|
|
130
|
+
sends: readonly SendAnnotation[] = [],
|
|
80
131
|
): AnnotatedEvalSource {
|
|
81
132
|
const normalized = normalizeEvalSource(source.content);
|
|
82
133
|
// 末尾的单个换行符不产出幻影空行(源码文件几乎总以换行符收尾);中间的空行原样保留为一行。
|
|
83
134
|
const body = normalized.endsWith("\n") ? normalized.slice(0, -1) : normalized;
|
|
84
135
|
const rawLines = body.length === 0 && normalized.length === 0 ? [""] : body.split("\n");
|
|
85
136
|
|
|
86
|
-
const lines: AnnotatedSourceLine[] = rawLines.map((text, i) => ({ line: i + 1, text, assertions: [] }));
|
|
137
|
+
const lines: AnnotatedSourceLine[] = rawLines.map((text, i) => ({ line: i + 1, text, assertions: [], sends: [] }));
|
|
138
|
+
|
|
139
|
+
for (const send of sends) {
|
|
140
|
+
// 与断言同一条映射规则;不满足的轮直接丢(全量面在 --execution,见 SendAnnotation)。
|
|
141
|
+
if (send.loc.file === source.path && send.loc.line >= 1 && send.loc.line <= lines.length) {
|
|
142
|
+
lines[send.loc.line - 1]!.sends.push(send);
|
|
143
|
+
}
|
|
144
|
+
}
|
|
87
145
|
|
|
88
146
|
const unmapped: AssertionResult[] = [];
|
|
89
147
|
let mappedAssertions = 0;
|
|
@@ -93,7 +151,7 @@ export function buildAnnotatedEvalSource(
|
|
|
93
151
|
let soft = 0;
|
|
94
152
|
|
|
95
153
|
for (const assertion of assertions) {
|
|
96
|
-
if (assertion.passed) passed++;
|
|
154
|
+
if (assertion.outcome === "passed") passed++;
|
|
97
155
|
else failed++;
|
|
98
156
|
if (assertion.severity === "gate") gate++;
|
|
99
157
|
else soft++;
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
// cases: docs/engineering/unit-tests/results/cases.md
|
|
1
2
|
// loadAttemptEvidence 单测(定稿契约见 plan/attempt-evidence-feedback-loop.md「中性数据准备」、
|
|
2
3
|
// src/results/attempt-evidence.ts 的头注)。用真实 createResultsWriter → openResults 的读写链路
|
|
3
4
|
// 落一份最小 fixture(不手写 JSON 文件,理由同 loadAnnotatedEvalSource 的既有端到端测试:
|
|
@@ -7,6 +8,7 @@ import { afterEach, describe, expect, it } from "vitest";
|
|
|
7
8
|
import { mkdtemp, rm } from "node:fs/promises";
|
|
8
9
|
import { tmpdir } from "node:os";
|
|
9
10
|
import { join } from "node:path";
|
|
11
|
+
import type { DiffArtifact } from "../types.ts";
|
|
10
12
|
import {
|
|
11
13
|
createResultsWriter,
|
|
12
14
|
loadAttemptEvidence,
|
|
@@ -34,7 +36,7 @@ const SOURCE_PATH = "evals/a.eval.ts";
|
|
|
34
36
|
const SOURCE_CONTENT = 'import { defineEval } from "niceeval";\nexport default defineEval({\n test() {},\n});\n';
|
|
35
37
|
|
|
36
38
|
const ASSERTIONS: EvalResult["assertions"] = [
|
|
37
|
-
{ name: "check-1",
|
|
39
|
+
{ name: "check-1", outcome: "passed" as const, severity: "gate", score: 1, loc: { file: SOURCE_PATH, line: 3 } },
|
|
38
40
|
];
|
|
39
41
|
|
|
40
42
|
const EVENTS: StreamEvent[] = [
|
|
@@ -47,8 +49,8 @@ const TRACE: TraceSpan[] = [
|
|
|
47
49
|
{ traceId: "t1", spanId: "s1", name: "tool.get_weather", startMs: 0, endMs: 10, attributes: { call_id: "c1" } },
|
|
48
50
|
];
|
|
49
51
|
|
|
50
|
-
const NONEMPTY_DIFF:
|
|
51
|
-
const EMPTY_DIFF:
|
|
52
|
+
const NONEMPTY_DIFF: DiffArtifact = [{ window: "s1/t1", changes: { "a.txt": { status: "added", after: "hello" } } }];
|
|
53
|
+
const EMPTY_DIFF: DiffArtifact = [{ window: "s1/t1", changes: {} }];
|
|
52
54
|
|
|
53
55
|
/** 起一个 writer,写一条 attempt,finish,再从头 openResults 读回它的 AttemptHandle。 */
|
|
54
56
|
async function seedAttempt(
|
|
@@ -72,7 +74,7 @@ describe("loadAttemptEvidence", () => {
|
|
|
72
74
|
const root = await makeRoot();
|
|
73
75
|
const attempt = await seedAttempt(
|
|
74
76
|
root,
|
|
75
|
-
{ id: "weather/brooklyn", assertions: ASSERTIONS },
|
|
77
|
+
{ id: "weather/brooklyn", assertions: ASSERTIONS, phases: [{ name: "eval.run", durationMs: 900 }] },
|
|
76
78
|
{ events: EVENTS, trace: TRACE, diff: NONEMPTY_DIFF, sources: [{ path: SOURCE_PATH, content: SOURCE_CONTENT }] },
|
|
77
79
|
);
|
|
78
80
|
|
|
@@ -86,7 +88,7 @@ describe("loadAttemptEvidence", () => {
|
|
|
86
88
|
const actionNode = evidence.execution!.nodes.find((n) => n.kind === "action");
|
|
87
89
|
expect(actionNode).toBeDefined();
|
|
88
90
|
expect((actionNode as { span?: TraceSpan }).span).toBeDefined();
|
|
89
|
-
expect(evidence.diff).toEqual(NONEMPTY_DIFF);
|
|
91
|
+
expect(evidence.diff?.windows).toEqual(NONEMPTY_DIFF);
|
|
90
92
|
|
|
91
93
|
expect(evidence.capabilities).toEqual({ eval: true, execution: true, timing: true, diff: true });
|
|
92
94
|
});
|
|
@@ -103,7 +105,7 @@ describe("loadAttemptEvidence", () => {
|
|
|
103
105
|
expect(evidence.capabilities).toEqual({ eval: false, execution: false, timing: false, diff: false });
|
|
104
106
|
});
|
|
105
107
|
|
|
106
|
-
it("有 events 没有
|
|
108
|
+
it("有 events 没有 phases(旧 runner 产出):execution 为真、timing 为假", async () => {
|
|
107
109
|
const root = await makeRoot();
|
|
108
110
|
const attempt = await seedAttempt(root, { id: "weather/albany" }, { events: EVENTS });
|
|
109
111
|
|
|
@@ -122,7 +124,7 @@ describe("loadAttemptEvidence", () => {
|
|
|
122
124
|
|
|
123
125
|
const evidence = await loadAttemptEvidence(attempt);
|
|
124
126
|
|
|
125
|
-
expect(evidence.diff).toEqual(EMPTY_DIFF);
|
|
127
|
+
expect(evidence.diff?.windows).toEqual(EMPTY_DIFF);
|
|
126
128
|
expect(evidence.capabilities.diff).toBe(false);
|
|
127
129
|
});
|
|
128
130
|
|
|
@@ -31,7 +31,7 @@ import { join } from "node:path";
|
|
|
31
31
|
import type { AttemptHandle } from "./types.ts";
|
|
32
32
|
import { type AttemptIdentity, type AttemptLocator, encodeAttemptLocator } from "./locator.ts";
|
|
33
33
|
import { loadAnnotatedEvalSource } from "./attempt-source.ts";
|
|
34
|
-
import type
|
|
34
|
+
import { deriveSendAnnotations, type AnnotatedEvalSource } from "./annotated-source.ts";
|
|
35
35
|
import { buildExecutionTree, type ExecutionTree } from "../o11y/execution-tree.ts";
|
|
36
36
|
import type { DiffData, EvalResult } from "../types.ts";
|
|
37
37
|
|
|
@@ -63,7 +63,7 @@ export interface AttemptEvidenceCapabilities {
|
|
|
63
63
|
eval: boolean;
|
|
64
64
|
/** events 非空数组——有骨架可看。 */
|
|
65
65
|
execution: boolean;
|
|
66
|
-
/**
|
|
66
|
+
/** result.json 带 phases(阶段计时)——`--timing` 时间树可渲染。 */
|
|
67
67
|
timing: boolean;
|
|
68
68
|
/** diff 非空且至少改动/删除了一个文件。 */
|
|
69
69
|
diff: boolean;
|
|
@@ -86,6 +86,8 @@ export interface AttemptEvidence {
|
|
|
86
86
|
execution: ExecutionTree | null;
|
|
87
87
|
/** 被测 Agent 对 Sandbox 工作区的文件变化(`--diff` 切面);原样透传 attempt.diff()。 */
|
|
88
88
|
diff: DiffData | null;
|
|
89
|
+
/** OTel spans(`--timing` 把 turn 节点下的 agent/model/tool spans 挂回时间树);没有 trace 就是 null。 */
|
|
90
|
+
trace: import("../types.ts").TraceSpan[] | null;
|
|
89
91
|
artifactPaths: EvidencePaths;
|
|
90
92
|
capabilities: AttemptEvidenceCapabilities;
|
|
91
93
|
}
|
|
@@ -110,12 +112,17 @@ export async function loadAttemptEvidence(attempt: AttemptHandle): Promise<Attem
|
|
|
110
112
|
};
|
|
111
113
|
const locator = attempt.locator ?? encodeAttemptLocator(identity);
|
|
112
114
|
|
|
113
|
-
const [
|
|
114
|
-
loadAnnotatedEvalSource(attempt),
|
|
115
|
+
const [events, trace, diff] = await Promise.all([
|
|
115
116
|
attempt.events(),
|
|
116
117
|
attempt.trace(),
|
|
117
118
|
attempt.diff(),
|
|
118
119
|
]);
|
|
120
|
+
// send 标注要拿事件流里用户消息的 loc 与阶段时间树里的 turn 节点配对,所以在 events
|
|
121
|
+
// 就绪后再装配 evalSource;派生与分桶都是纯函数(annotated-source.ts),这里只接线。
|
|
122
|
+
const evalSource = await loadAnnotatedEvalSource(
|
|
123
|
+
attempt,
|
|
124
|
+
deriveSendAnnotations(events, attempt.result.phases),
|
|
125
|
+
);
|
|
119
126
|
|
|
120
127
|
// execution 的 null 边界:没有非空事件骨架就是 null,即使 trace 里有 span(那些 span 在
|
|
121
128
|
// buildExecutionTree 里只会全部落进 telemetry-only 桶,产出一棵没有骨架、只有遥测的树——
|
|
@@ -124,9 +131,8 @@ export async function loadAttemptEvidence(attempt: AttemptHandle): Promise<Attem
|
|
|
124
131
|
const execution = hasEvents ? buildExecutionTree(events, trace ?? []) : null;
|
|
125
132
|
|
|
126
133
|
const evalCapable = evalSource !== null && (evalSource.lines.length > 0 || evalSource.summary.totalAssertions > 0);
|
|
127
|
-
// 与 show/render.ts::diffText
|
|
128
|
-
|
|
129
|
-
const diffCapable = diff !== null && (Object.keys(diff.generatedFiles).length > 0 || diff.deletedFiles.length > 0);
|
|
134
|
+
// 与 show/render.ts::diffText 同一口径:任一窗口触及过任一文件才算"有变化"。
|
|
135
|
+
const diffCapable = diff !== null && Object.keys(diff.files).length > 0;
|
|
130
136
|
|
|
131
137
|
return {
|
|
132
138
|
locator,
|
|
@@ -135,11 +141,12 @@ export async function loadAttemptEvidence(attempt: AttemptHandle): Promise<Attem
|
|
|
135
141
|
evalSource,
|
|
136
142
|
execution,
|
|
137
143
|
diff,
|
|
144
|
+
trace,
|
|
138
145
|
artifactPaths: { dir: attemptArtifactDir(attempt) },
|
|
139
146
|
capabilities: {
|
|
140
147
|
eval: evalCapable,
|
|
141
148
|
execution: execution !== null,
|
|
142
|
-
timing:
|
|
149
|
+
timing: (attempt.result.phases?.length ?? 0) > 0,
|
|
143
150
|
diff: diffCapable,
|
|
144
151
|
},
|
|
145
152
|
};
|
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
// 或参照。
|
|
9
9
|
|
|
10
10
|
import type { AttemptHandle } from "./types.ts";
|
|
11
|
-
import { buildAnnotatedEvalSource, type AnnotatedEvalSource } from "./annotated-source.ts";
|
|
11
|
+
import { buildAnnotatedEvalSource, type AnnotatedEvalSource, type SendAnnotation } from "./annotated-source.ts";
|
|
12
12
|
|
|
13
13
|
/**
|
|
14
14
|
* 给定一个 attempt,取回它的 eval 源码(经 sources() 解引用,可能来自本快照或
|
|
@@ -22,7 +22,10 @@ import { buildAnnotatedEvalSource, type AnnotatedEvalSource } from "./annotated-
|
|
|
22
22
|
* 只服务这条打通链路的验证目的,不是最终选择策略(后续阶段如需要为每份引用到的文件
|
|
23
23
|
* 都出一份 AnnotatedEvalSource,在这基础上很容易扩成返回数组)。
|
|
24
24
|
*/
|
|
25
|
-
export async function loadAnnotatedEvalSource(
|
|
25
|
+
export async function loadAnnotatedEvalSource(
|
|
26
|
+
attempt: AttemptHandle,
|
|
27
|
+
sends: readonly SendAnnotation[] = [],
|
|
28
|
+
): Promise<AnnotatedEvalSource | null> {
|
|
26
29
|
const sources = await attempt.sources();
|
|
27
30
|
if (!sources || sources.length === 0) return null;
|
|
28
31
|
|
|
@@ -44,5 +47,5 @@ export async function loadAnnotatedEvalSource(attempt: AttemptHandle): Promise<A
|
|
|
44
47
|
}
|
|
45
48
|
}
|
|
46
49
|
|
|
47
|
-
return buildAnnotatedEvalSource(primary, assertions);
|
|
50
|
+
return buildAnnotatedEvalSource(primary, assertions, sends);
|
|
48
51
|
}
|
package/src/results/copy.ts
CHANGED
|
@@ -7,20 +7,38 @@
|
|
|
7
7
|
// knownEvalIds(复制时刻该实验已知的 eval 并集),发布目录上重新 openResults().latest(),
|
|
8
8
|
// 残缺警告被同一套机制重新算出来,不靠发布者转述。
|
|
9
9
|
|
|
10
|
-
import {
|
|
10
|
+
import { mkdir, readdir, readFile, stat, writeFile } from "node:fs/promises";
|
|
11
11
|
import { basename, dirname, join, resolve } from "node:path";
|
|
12
|
-
import type { EvalResult } from "../types.ts";
|
|
12
|
+
import type { EvalResult, StreamEvent, TraceSpan } from "../types.ts";
|
|
13
13
|
import { RESULTS_FORMAT } from "../types.ts";
|
|
14
14
|
import { RESULT_FILE, SNAPSHOT_FILE, artifactFileOf, experimentDirOf } from "./format.ts";
|
|
15
15
|
import { experimentOfSnapshot } from "./open.ts";
|
|
16
16
|
import { isNewerSnapshot } from "./select.ts";
|
|
17
17
|
import { hashEvalSource, normalizeEvalSource } from "./source-hash.ts";
|
|
18
|
+
import {
|
|
19
|
+
PUBLISH_FILE_MAX_BYTES,
|
|
20
|
+
redactEvents,
|
|
21
|
+
redactExperimentInfo,
|
|
22
|
+
redactJsonValue,
|
|
23
|
+
redactResultRecord,
|
|
24
|
+
redactSpans,
|
|
25
|
+
type Redactor,
|
|
26
|
+
} from "./publish.ts";
|
|
18
27
|
import type { ArtifactKind, AttemptHandle, Selection, Snapshot, SnapshotMeta } from "./types.ts";
|
|
19
|
-
|
|
28
|
+
|
|
29
|
+
/** 缺省携带的 artifact:events / trace / o11y / agentSetup / sources;diff 不截断、可达百 MB,缺省不带。 */
|
|
30
|
+
const DEFAULT_PUBLISH_ARTIFACTS: ArtifactKind[] = ["events", "trace", "o11y", "agentSetup", "sources"];
|
|
31
|
+
const VALID_ARTIFACTS: ArtifactKind[] = ["events", "trace", "o11y", "agentSetup", "diff", "sources"];
|
|
20
32
|
|
|
21
33
|
export interface CopySnapshotsOptions {
|
|
22
|
-
/** 要带上的 artifact
|
|
34
|
+
/** 要带上的 artifact 种类;缺省带 events / trace / o11y / agentSetup / sources,不带 diff。 */
|
|
23
35
|
artifacts?: ArtifactKind[];
|
|
36
|
+
/**
|
|
37
|
+
* 发布消毒(必填,没有隐式默认):传函数逐值消毒自由文本字段,或显式传 `false` 声明
|
|
38
|
+
* 「这批数据可以原文发布」——「不消毒」必须是写在代码里的选择,不是忘了传参数的副作用。
|
|
39
|
+
* 结构字段(格式、判定、身份、路径、哈希)永不经过 redactor。
|
|
40
|
+
*/
|
|
41
|
+
redact: Redactor | false;
|
|
24
42
|
}
|
|
25
43
|
|
|
26
44
|
export interface CopySnapshotsResult {
|
|
@@ -38,7 +56,7 @@ export interface CopySnapshotsResult {
|
|
|
38
56
|
export async function copySnapshots(
|
|
39
57
|
selection: Selection | Snapshot[],
|
|
40
58
|
destDir: string,
|
|
41
|
-
opts
|
|
59
|
+
opts: CopySnapshotsOptions,
|
|
42
60
|
): Promise<CopySnapshotsResult> {
|
|
43
61
|
const selected = Array.isArray(selection) ? selection : selection.snapshots;
|
|
44
62
|
if (selected.length === 0) {
|
|
@@ -46,11 +64,17 @@ export async function copySnapshots(
|
|
|
46
64
|
"copySnapshots got no snapshots to copy. Check the experiments filter, or pass snapshots from openResults().latest().",
|
|
47
65
|
);
|
|
48
66
|
}
|
|
49
|
-
|
|
67
|
+
if (opts?.redact === undefined) {
|
|
68
|
+
throw new Error(
|
|
69
|
+
'copySnapshots requires an explicit "redact" option: pass a (text) => string sanitizer, or the literal false to declare this data safe to publish verbatim.',
|
|
70
|
+
);
|
|
71
|
+
}
|
|
72
|
+
const redactor: Redactor | undefined = opts.redact === false ? undefined : opts.redact;
|
|
73
|
+
const kinds = opts.artifacts ?? [...DEFAULT_PUBLISH_ARTIFACTS];
|
|
50
74
|
for (const kind of kinds) {
|
|
51
|
-
if (!
|
|
75
|
+
if (!VALID_ARTIFACTS.includes(kind)) {
|
|
52
76
|
throw new Error(
|
|
53
|
-
`Unknown artifact kind "${String(kind)}" in copySnapshots options. Valid kinds: ${
|
|
77
|
+
`Unknown artifact kind "${String(kind)}" in copySnapshots options. Valid kinds: ${VALID_ARTIFACTS.join(", ")}.`,
|
|
54
78
|
);
|
|
55
79
|
}
|
|
56
80
|
}
|
|
@@ -73,96 +97,162 @@ export async function copySnapshots(
|
|
|
73
97
|
if (isNewerSnapshot(snapshot, existing)) byExperiment.set(snapshot.experimentId, snapshot);
|
|
74
98
|
}
|
|
75
99
|
|
|
76
|
-
|
|
100
|
+
// 发布前整文件预检:先规划并序列化全部目标文件(消毒发生在序列化后、写盘前),任一文件
|
|
101
|
+
// 超过 PUBLISH_FILE_MAX_BYTES 就整体失败,不留半成品目标目录。
|
|
102
|
+
const planned: PlannedFile[] = [];
|
|
77
103
|
for (const snapshot of byExperiment.values()) {
|
|
78
|
-
await
|
|
104
|
+
planned.push(...(await planOneSnapshot(snapshot, selected, dest, kinds, redactor)));
|
|
105
|
+
}
|
|
106
|
+
const oversized = planned.filter((f) => f.bytes.byteLength > PUBLISH_FILE_MAX_BYTES);
|
|
107
|
+
if (oversized.length > 0) {
|
|
108
|
+
const lines = oversized.map(
|
|
109
|
+
(f) =>
|
|
110
|
+
` ${f.source ?? f.path}: ${f.bytes.byteLength} bytes (limit ${PUBLISH_FILE_MAX_BYTES}). Exclude that artifact kind from "artifacts", or regenerate the history with the current writer.`,
|
|
111
|
+
);
|
|
112
|
+
throw new Error(`copySnapshots publish precheck failed; ${oversized.length} file(s) exceed the 50 MiB publish budget:\n${lines.join("\n")}`);
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
await mkdir(dest, { recursive: true });
|
|
116
|
+
for (const file of planned) {
|
|
117
|
+
await mkdir(dirname(file.path), { recursive: true });
|
|
118
|
+
await writeFile(file.path, file.bytes);
|
|
79
119
|
}
|
|
80
120
|
|
|
81
121
|
return { dir: dest, warnings };
|
|
82
122
|
}
|
|
83
123
|
|
|
84
|
-
|
|
124
|
+
interface PlannedFile {
|
|
125
|
+
/** 目标绝对路径。 */
|
|
126
|
+
path: string;
|
|
127
|
+
/** 序列化(且已消毒)后的字节。 */
|
|
128
|
+
bytes: Buffer;
|
|
129
|
+
/** 源文件路径(预检报错定位用);重建的 json 没有单一源。 */
|
|
130
|
+
source?: string;
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
async function planOneSnapshot(
|
|
134
|
+
snapshot: Snapshot,
|
|
135
|
+
selected: Snapshot[],
|
|
136
|
+
destRoot: string,
|
|
137
|
+
kinds: ArtifactKind[],
|
|
138
|
+
redactor: Redactor | undefined,
|
|
139
|
+
): Promise<PlannedFile[]> {
|
|
85
140
|
const destSnapDir = join(destRoot, experimentDirOf(snapshot.experimentId), basename(snapshot.dir));
|
|
86
|
-
|
|
141
|
+
const planned: PlannedFile[] = [];
|
|
87
142
|
|
|
88
143
|
// sources 的去重仓库(sources/<sha256>.json)是快照级的:同一份源码被多个 attempt 引用时,
|
|
89
|
-
// 复制到目的地也只应该有一份——这个 Set
|
|
90
|
-
const
|
|
144
|
+
// 复制到目的地也只应该有一份——这个 Set 记录本快照已经规划过的 hash,整快照的 attempt 共享。
|
|
145
|
+
const plannedSourceHashes = new Set<string>();
|
|
91
146
|
for (const attempt of snapshot.attempts) {
|
|
92
|
-
await
|
|
147
|
+
planned.push(...(await planOneAttempt(attempt, destSnapDir, kinds, plannedSourceHashes, redactor)));
|
|
93
148
|
}
|
|
94
149
|
|
|
95
150
|
const knownEvalIds = experimentOfSnapshot(snapshot)?.evalIds ?? fallbackUnion(selected, snapshot.experimentId);
|
|
151
|
+
const experiment =
|
|
152
|
+
snapshot.experiment !== undefined && redactor
|
|
153
|
+
? (redactExperimentInfo(snapshot.experiment as unknown as Record<string, unknown>, redactor) as unknown as SnapshotMeta["experiment"])
|
|
154
|
+
: snapshot.experiment;
|
|
96
155
|
const meta: SnapshotMeta = {
|
|
97
156
|
format: RESULTS_FORMAT,
|
|
98
157
|
schemaVersion: snapshot.schemaVersion,
|
|
99
158
|
producer: snapshot.producer,
|
|
100
159
|
experimentId: snapshot.experimentId,
|
|
101
|
-
...(
|
|
160
|
+
...(experiment !== undefined ? { experiment } : {}),
|
|
102
161
|
agent: snapshot.agent,
|
|
103
162
|
...(snapshot.model !== undefined ? { model: snapshot.model } : {}),
|
|
104
163
|
startedAt: snapshot.startedAt,
|
|
105
164
|
...(snapshot.completedAt !== undefined ? { completedAt: snapshot.completedAt } : {}),
|
|
106
165
|
...(knownEvalIds.length ? { knownEvalIds } : {}),
|
|
107
|
-
...(snapshot.name !== undefined
|
|
166
|
+
...(snapshot.name !== undefined
|
|
167
|
+
? { name: redactor && typeof snapshot.name === "string" ? redactor(snapshot.name) : snapshot.name }
|
|
168
|
+
: {}),
|
|
169
|
+
// 发布根标记只声明流程,不证明结果:applied = 消毒函数对全部自由文本字段跑过;
|
|
170
|
+
// none = 作者显式的原文发布声明。view --out 的防呆据此分级。
|
|
171
|
+
publish: { redaction: redactor ? "applied" : "none" },
|
|
108
172
|
};
|
|
109
|
-
|
|
173
|
+
planned.push({ path: join(destSnapDir, SNAPSHOT_FILE), bytes: Buffer.from(JSON.stringify(meta, null, 2), "utf-8") });
|
|
174
|
+
return planned;
|
|
110
175
|
}
|
|
111
176
|
|
|
112
|
-
async function
|
|
177
|
+
async function planOneAttempt(
|
|
113
178
|
attempt: AttemptHandle,
|
|
114
179
|
destSnapDir: string,
|
|
115
180
|
kinds: ArtifactKind[],
|
|
116
|
-
|
|
117
|
-
|
|
181
|
+
plannedSourceHashes: Set<string>,
|
|
182
|
+
redactor: Redactor | undefined,
|
|
183
|
+
): Promise<PlannedFile[]> {
|
|
118
184
|
const destAttemptDir = join(destSnapDir, attempt.ref.attempt);
|
|
119
|
-
|
|
185
|
+
const planned: PlannedFile[] = [];
|
|
120
186
|
|
|
121
187
|
// sources 是唯一「两层」的 artifact(attempt 级引用 + 快照级去重仓库),不能像其它四类那样
|
|
122
|
-
//
|
|
123
|
-
//
|
|
124
|
-
// 回退链(携带条目复制后,原快照可能不在目的地里,必须此刻把内容落到自己脚下)。
|
|
188
|
+
// 单文件原字节完事——原字节只是引用,不带内容。走读取面已经会解引用+回退的 attempt.sources()
|
|
189
|
+
// 拿到完整内容,消毒后按(消毒后)内容哈希重新去重落盘——发布根里引用与内容永远一致。
|
|
125
190
|
const genericKinds = kinds.filter((k) => k !== "sources");
|
|
126
191
|
const files = await findArtifactFiles(attempt, genericKinds);
|
|
127
|
-
await Promise.all(files.map(({ kind, source }) => copyFile(source, join(destAttemptDir, artifactFileOf(kind)))));
|
|
128
192
|
const copied = new Set(files.map((f) => f.kind));
|
|
193
|
+
for (const { kind, source } of files) {
|
|
194
|
+
const raw = await readFile(source);
|
|
195
|
+
const bytes = redactor ? redactArtifactBytes(kind, raw, redactor) : raw;
|
|
196
|
+
planned.push({ path: join(destAttemptDir, artifactFileOf(kind)), bytes, source });
|
|
197
|
+
}
|
|
129
198
|
|
|
130
199
|
if (kinds.includes("sources")) {
|
|
131
|
-
const
|
|
132
|
-
if (
|
|
200
|
+
const sources = await attempt.sources();
|
|
201
|
+
if (sources && sources.length > 0) {
|
|
202
|
+
copied.add("sources");
|
|
203
|
+
const destStoreDir = join(destSnapDir, "sources");
|
|
204
|
+
const refs: { path: string; sha256: string }[] = [];
|
|
205
|
+
for (const src of sources) {
|
|
206
|
+
const content = redactor ? redactor(src.content) : src.content;
|
|
207
|
+
const sha256 = hashEvalSource(normalizeEvalSource(content));
|
|
208
|
+
refs.push({ path: src.path, sha256 });
|
|
209
|
+
if (!plannedSourceHashes.has(sha256)) {
|
|
210
|
+
planned.push({
|
|
211
|
+
path: join(destStoreDir, `${sha256}.json`),
|
|
212
|
+
bytes: Buffer.from(JSON.stringify({ content }), "utf-8"),
|
|
213
|
+
});
|
|
214
|
+
plannedSourceHashes.add(sha256);
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
planned.push({
|
|
218
|
+
path: join(destAttemptDir, artifactFileOf("sources")),
|
|
219
|
+
bytes: Buffer.from(JSON.stringify(refs), "utf-8"),
|
|
220
|
+
});
|
|
221
|
+
}
|
|
133
222
|
}
|
|
134
223
|
|
|
135
|
-
|
|
136
|
-
|
|
224
|
+
let record = slimForCopy(attempt.result, copied);
|
|
225
|
+
if (redactor) record = redactResultRecord(record, redactor);
|
|
226
|
+
planned.push({ path: join(destAttemptDir, RESULT_FILE), bytes: Buffer.from(JSON.stringify(record, null, 2), "utf-8") });
|
|
227
|
+
return planned;
|
|
137
228
|
}
|
|
138
229
|
|
|
139
|
-
/**
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
await mkdir(destStoreDir, { recursive: true });
|
|
160
|
-
await writeFile(join(destStoreDir, `${sha256}.json`), JSON.stringify({ content: src.content }), "utf-8");
|
|
161
|
-
copiedSourceHashes.add(sha256);
|
|
230
|
+
/** 单个 artifact 文件的消毒:按种类解析 JSON、走各自的自由文本标注,重新序列化。 */
|
|
231
|
+
function redactArtifactBytes(kind: ArtifactKind, raw: Buffer, redactor: Redactor): Buffer {
|
|
232
|
+
try {
|
|
233
|
+
const parsed = JSON.parse(raw.toString("utf-8")) as unknown;
|
|
234
|
+
let next: unknown;
|
|
235
|
+
switch (kind) {
|
|
236
|
+
case "events":
|
|
237
|
+
next = redactEvents(parsed as StreamEvent[], redactor);
|
|
238
|
+
break;
|
|
239
|
+
case "trace":
|
|
240
|
+
next = redactSpans(parsed as TraceSpan[], redactor);
|
|
241
|
+
break;
|
|
242
|
+
// diff 的 before/after/内容与 o11y 的命令 / 错误 / URL 都是自由文本;路径键为结构字段。
|
|
243
|
+
case "diff":
|
|
244
|
+
case "o11y":
|
|
245
|
+
case "agentSetup":
|
|
246
|
+
next = redactJsonValue(parsed, redactor);
|
|
247
|
+
break;
|
|
248
|
+
default:
|
|
249
|
+
next = parsed;
|
|
162
250
|
}
|
|
251
|
+
return Buffer.from(JSON.stringify(next), "utf-8");
|
|
252
|
+
} catch {
|
|
253
|
+
// 解析不了的按原字节透传(malformed 数据不该在发布路径上被静默改写)。
|
|
254
|
+
return raw;
|
|
163
255
|
}
|
|
164
|
-
await writeFile(join(destAttemptDir, artifactFileOf("sources")), JSON.stringify(refs), "utf-8");
|
|
165
|
-
return true;
|
|
166
256
|
}
|
|
167
257
|
|
|
168
258
|
/** 目标目录非空即报错:盘上不该出现「我没写的东西被动过」的惊讶;发布脚本要幂等就自己先清目录。 */
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
// cases: docs/engineering/unit-tests/results/cases.md
|
|
1
2
|
// show / view 宿主等价契约(设计契约:plan/show-view-equivalence.md 完成定义;
|
|
2
3
|
// docs/feature/reports/architecture.md「Selection 是计算入口」;docs/feature/results/library.md「选择快照」)。
|
|
3
4
|
//
|
|
@@ -339,7 +340,7 @@ describe("selectCurrentResults · 现刻水位结构化身份", () => {
|
|
|
339
340
|
const rootA = await makeRoot();
|
|
340
341
|
const rootB = await makeRoot();
|
|
341
342
|
// 失败 eval 让 id 现于两面(通过 eval 只进折叠子行,不便断言);qa / qb 各根独有。
|
|
342
|
-
const boom = { assertions: [{ name: "succeeded()", severity: "gate" as const, score: 0,
|
|
343
|
+
const boom = { assertions: [{ name: "succeeded()", severity: "gate" as const, score: 0, outcome: "failed" as const, detail: "boom" }] };
|
|
343
344
|
await writeSnapshot(rootA, "2026-07-01T08-00-00-000Z", { experimentId: "onlyA/bub", startedAt: "2026-07-01T08:00:00.000Z" }, [res("qa", "failed", boom)]);
|
|
344
345
|
await writeSnapshot(rootB, "2026-07-02T08-00-00-000Z", { experimentId: "onlyB/bub", startedAt: "2026-07-02T08:00:00.000Z" }, [res("qb", "failed", boom)]);
|
|
345
346
|
const normB = normalizeSelection(selectCurrentResults(await openResults(rootB)));
|
|
@@ -389,7 +390,7 @@ async function seedPartialRerun(): Promise<string> {
|
|
|
389
390
|
const root = await makeRoot();
|
|
390
391
|
await writeSnapshot(root, "2026-07-01T08-00-00-000Z", { experimentId: "compare/bub", startedAt: "2026-07-01T08:00:00.000Z" }, [
|
|
391
392
|
res("q1", "passed"),
|
|
392
|
-
res("q2", "failed", { assertions: [{ name: 'fileChanged("q2.tsx")', severity: "gate", score: 0,
|
|
393
|
+
res("q2", "failed", { assertions: [{ name: 'fileChanged("q2.tsx")', severity: "gate", score: 0, outcome: "failed" as const, detail: "file was not modified" }] }),
|
|
393
394
|
]);
|
|
394
395
|
await writeSnapshot(root, "2026-07-02T08-00-00-000Z", { experimentId: "compare/bub", startedAt: "2026-07-02T08:00:00.000Z" }, [
|
|
395
396
|
res("q1", "passed"),
|
|
@@ -404,7 +405,7 @@ async function seedPartialCoverage(): Promise<string> {
|
|
|
404
405
|
root,
|
|
405
406
|
"2026-07-01T08-00-00-000Z",
|
|
406
407
|
{ experimentId: "compare/bub", startedAt: "2026-07-01T08:00:00.000Z", knownEvalIds: ["q1", "q2"] },
|
|
407
|
-
[res("q1", "failed", { assertions: [{ name: "succeeded()", severity: "gate", score: 0,
|
|
408
|
+
[res("q1", "failed", { assertions: [{ name: "succeeded()", severity: "gate", score: 0, outcome: "failed" as const, detail: "boom" }] })],
|
|
408
409
|
);
|
|
409
410
|
return root;
|
|
410
411
|
}
|
|
@@ -415,9 +416,10 @@ describe("宿主接线 · show text 面与 view web 面反映同一批事实", (
|
|
|
415
416
|
const text = await showText(root, []);
|
|
416
417
|
const html = await viewHtml(root);
|
|
417
418
|
for (const face of [text, html]) {
|
|
418
|
-
expect(face).toContain("compare/bub");
|
|
419
419
|
expect(face).toContain("q2"); // 周一补齐的失败 eval,若宿主回退 results.latest() 就会消失
|
|
420
420
|
}
|
|
421
|
+
expect(text).toMatch(/\bbub\s+default\s+bub\s+1s\s+50%/);
|
|
422
|
+
expect(html).toContain('data-sort-value="compare/bub"');
|
|
421
423
|
expect(text).toContain("1 passed / 1 failed");
|
|
422
424
|
expect(html).toContain("50%");
|
|
423
425
|
// 现刻水位覆盖齐全 → 两面都不出 partial-coverage(text: "verdicts cover"; html: data-kind)。
|
|
@@ -431,7 +433,7 @@ describe("宿主接线 · show text 面与 view web 面反映同一批事实", (
|
|
|
431
433
|
it("位置前缀收窄:两扇门都只见范围内的 eval,范围外一致排除", async () => {
|
|
432
434
|
const root = await makeRoot();
|
|
433
435
|
await writeSnapshot(root, "2026-07-01T08-00-00-000Z", { experimentId: "compare/bub", startedAt: "2026-07-01T08:00:00.000Z" }, [
|
|
434
|
-
res("weather/brooklyn", "failed", { assertions: [{ name: "succeeded()", severity: "gate", score: 0,
|
|
436
|
+
res("weather/brooklyn", "failed", { assertions: [{ name: "succeeded()", severity: "gate", score: 0, outcome: "failed" as const, detail: "boom" }] }),
|
|
435
437
|
res("weather/queens", "passed"),
|
|
436
438
|
res("algebra/quadratic", "passed"),
|
|
437
439
|
]);
|
|
@@ -497,7 +499,7 @@ function hasPartialCoverageHtml(html: string): boolean {
|
|
|
497
499
|
it("通过计数一致:同一 fixture 两面反映 3 过 1 败", async () => {
|
|
498
500
|
const root = await makeRoot();
|
|
499
501
|
await writeSnapshot(root, "2026-07-01T08-00-00-000Z", { experimentId: "compare/bub", startedAt: "2026-07-01T08:00:00.000Z" }, [
|
|
500
|
-
res("q1", "failed", { durationMs: 40_000, estimatedCostUSD: 0.04, assertions: [{ name: "succeeded()", severity: "gate", score: 0,
|
|
502
|
+
res("q1", "failed", { durationMs: 40_000, estimatedCostUSD: 0.04, assertions: [{ name: "succeeded()", severity: "gate", score: 0, outcome: "failed" as const, detail: "boom" }] }),
|
|
501
503
|
res("q2", "passed", { durationMs: 2_000, estimatedCostUSD: 0.02 }),
|
|
502
504
|
res("q3", "passed", { durationMs: 2_000, estimatedCostUSD: 0.02 }),
|
|
503
505
|
res("q4", "passed", { durationMs: 2_000, estimatedCostUSD: 0.02 }),
|
package/src/results/index.ts
CHANGED
|
@@ -29,9 +29,11 @@ export {
|
|
|
29
29
|
} from "./locator.ts";
|
|
30
30
|
export {
|
|
31
31
|
buildAnnotatedEvalSource,
|
|
32
|
+
deriveSendAnnotations,
|
|
32
33
|
type AnnotatedEvalSource,
|
|
33
34
|
type AnnotatedEvalSourceSummary,
|
|
34
35
|
type AnnotatedSourceLine,
|
|
36
|
+
type SendAnnotation,
|
|
35
37
|
} from "./annotated-source.ts";
|
|
36
38
|
export { loadAnnotatedEvalSource } from "./attempt-source.ts";
|
|
37
39
|
export {
|