niceeval 0.6.1 → 0.6.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agents/types.d.ts +67 -5
- package/dist/context/types.d.ts +32 -12
- package/dist/i18n/en.d.ts +54 -0
- package/dist/i18n/zh-CN.d.ts +55 -1
- package/dist/o11y/types.d.ts +16 -2
- package/dist/report/aggregate.d.ts +5 -3
- package/dist/report/aggregate.js +32 -5
- package/dist/report/built-ins/experiment-comparison.d.ts +39 -1
- package/dist/report/built-ins/experiment-comparison.js +116 -10
- package/dist/report/built-ins/index.d.ts +1 -0
- package/dist/report/built-ins/index.js +1 -1
- package/dist/report/components.d.ts +8 -2
- package/dist/report/components.js +3 -3
- package/dist/report/compute.d.ts +11 -18
- package/dist/report/compute.js +54 -34
- package/dist/report/flag.d.ts +16 -1
- package/dist/report/flag.js +19 -1
- package/dist/report/format.d.ts +16 -8
- package/dist/report/format.js +27 -12
- package/dist/report/index.d.ts +4 -3
- package/dist/report/index.js +5 -4
- package/dist/report/locale.d.ts +11 -2
- package/dist/report/locale.js +23 -5
- package/dist/report/metrics.d.ts +13 -1
- package/dist/report/metrics.js +65 -14
- package/dist/report/primitives.d.ts +6 -0
- package/dist/report/react/AttemptList.d.ts +2 -2
- package/dist/report/react/AttemptList.js +5 -6
- package/dist/report/react/EvalList.d.ts +1 -1
- package/dist/report/react/EvalList.js +0 -0
- package/dist/report/react/ExperimentComparison.d.ts +8 -0
- package/dist/report/react/ExperimentComparison.js +11 -0
- package/dist/report/react/ExperimentList.d.ts +2 -1
- package/dist/report/react/ExperimentList.js +8 -10
- package/dist/report/react/MetricScatter.js +5 -11
- package/dist/report/react/chart-math.d.ts +23 -6
- package/dist/report/react/chart-math.js +71 -19
- package/dist/report/react/fixtures.d.ts +3 -3
- package/dist/report/react/fixtures.js +21 -14
- package/dist/report/report.d.ts +5 -1
- package/dist/report/report.js +6 -2
- package/dist/report/text/faces.d.ts +1 -1
- package/dist/report/text/faces.js +42 -41
- package/dist/report/text/table.js +36 -5
- package/dist/report/types.d.ts +39 -21
- package/dist/results/types.d.ts +11 -0
- package/dist/runner/feedback/sink.d.ts +110 -0
- package/dist/runner/types.d.ts +513 -22
- package/dist/sandbox/docker.d.ts +23 -2
- package/dist/sandbox/e2b.d.ts +15 -1
- package/dist/sandbox/errors.d.ts +30 -3
- package/dist/sandbox/io-retry.d.ts +17 -0
- package/dist/sandbox/registry.d.ts +2 -0
- package/dist/sandbox/resolve.d.ts +18 -5
- package/dist/sandbox/retry.d.ts +11 -1
- package/dist/sandbox/types.d.ts +39 -5
- package/dist/sandbox/vercel.d.ts +7 -1
- package/dist/scoring/coverage.d.ts +30 -0
- package/dist/scoring/display.d.ts +21 -0
- package/dist/scoring/display.js +120 -0
- package/dist/scoring/types.d.ts +103 -20
- package/dist/shared/aggregate.d.ts +1 -0
- package/dist/shared/aggregate.js +3 -3
- package/dist/shared/types.d.ts +28 -0
- package/dist/tty-line.d.ts +0 -4
- package/dist/util.d.ts +23 -0
- package/docs-site/zh/concepts/adapter.mdx +22 -4
- package/docs-site/zh/concepts/experiment.mdx +1 -1
- package/docs-site/zh/concepts/overview.mdx +6 -6
- package/docs-site/zh/guides/agent-feedback-loop.mdx +28 -26
- package/docs-site/zh/guides/authoring.mdx +33 -0
- package/docs-site/zh/guides/ci-integration.mdx +23 -12
- package/docs-site/zh/guides/connect-your-agent.mdx +29 -3
- package/docs-site/zh/guides/custom-reports.mdx +29 -34
- package/docs-site/zh/guides/dataset-fanout.mdx +25 -3
- package/docs-site/zh/guides/debug-sandbox.mdx +57 -0
- package/docs-site/zh/guides/debugging.mdx +210 -0
- package/docs-site/zh/guides/experiments.mdx +10 -3
- package/docs-site/zh/guides/official-adapters.mdx +26 -2
- package/docs-site/zh/guides/publish-report.mdx +30 -16
- package/docs-site/zh/guides/report-components.mdx +42 -30
- package/docs-site/zh/guides/reporters.mdx +2 -2
- package/docs-site/zh/guides/results-data.mdx +17 -9
- package/docs-site/zh/guides/runner.mdx +17 -7
- package/docs-site/zh/guides/sandbox-agent.mdx +56 -7
- package/docs-site/zh/guides/sandbox-providers.mdx +257 -9
- package/docs-site/zh/guides/scoring-guide.mdx +4 -4
- package/docs-site/zh/guides/viewing-results.mdx +79 -36
- package/docs-site/zh/guides/write-experiment.mdx +5 -3
- package/docs-site/zh/guides/write-send.mdx +17 -1
- package/docs-site/zh/index.mdx +1 -1
- package/docs-site/zh/reference/builtin-agents.mdx +27 -0
- package/docs-site/zh/reference/capabilities.mdx +2 -2
- package/docs-site/zh/reference/cli.mdx +33 -7
- package/docs-site/zh/reference/define-agent.mdx +57 -4
- package/docs-site/zh/reference/define-config.mdx +1 -1
- package/docs-site/zh/reference/define-eval.mdx +42 -9
- package/docs-site/zh/reference/expect.mdx +26 -1
- package/package.json +5 -1
- package/src/agents/ai-sdk-otel.test.ts +1 -0
- package/src/agents/ai-sdk.test.ts +3 -0
- package/src/agents/ai-sdk.ts +3 -0
- package/src/agents/bub-install-spec.test.ts +34 -0
- package/src/agents/bub-install-spec.ts +32 -0
- package/src/agents/bub.ts +31 -32
- package/src/agents/claude-code.test.ts +130 -9
- package/src/agents/claude-code.ts +76 -4
- package/src/agents/codex.test.ts +189 -40
- package/src/agents/codex.ts +155 -14
- package/src/agents/coding-cli-versions.test.ts +15 -0
- package/src/agents/coding-cli-versions.ts +3 -0
- package/src/agents/index.ts +11 -0
- package/src/agents/langgraph.test.ts +204 -0
- package/src/agents/langgraph.ts +495 -0
- package/src/agents/marketplace.ts +85 -0
- package/src/agents/native-config.test.ts +179 -0
- package/src/agents/native-config.ts +267 -0
- package/src/agents/openai-compat.test.ts +1 -0
- package/src/agents/openclaw.test.ts +31 -0
- package/src/agents/openclaw.ts +171 -0
- package/src/agents/plugin-config.test.ts +1 -0
- package/src/agents/sdk-streams.test.ts +79 -0
- package/src/agents/sdk-streams.ts +55 -10
- package/src/agents/skills.test.ts +1 -0
- package/src/agents/streaming.test.ts +3 -9
- package/src/agents/types.ts +68 -5
- package/src/agents/ui-message-stream.test.ts +3 -0
- package/src/cli.ts +411 -108
- package/src/context/context.test.ts +51 -12
- package/src/context/context.ts +161 -29
- package/src/context/session.test.ts +1 -0
- package/src/context/session.ts +114 -6
- package/src/context/types.ts +30 -12
- package/src/define.test.ts +13 -8
- package/src/define.ts +25 -4
- package/src/expect/index.ts +53 -23
- package/src/i18n/en.ts +64 -2
- package/src/i18n/zh-CN.ts +65 -3
- package/src/o11y/cost.test.ts +1 -0
- package/src/o11y/execution-tree.test.ts +1 -20
- package/src/o11y/otlp/mappers/claude-code.test.ts +1 -0
- package/src/o11y/otlp/parse.test.ts +1 -0
- package/src/o11y/otlp/turn-otel.test.ts +1 -0
- package/src/o11y/parsers/bub.test.ts +1 -0
- package/src/o11y/parsers/claude-code.test.ts +1 -34
- package/src/o11y/parsers/openclaw.test.ts +154 -0
- package/src/o11y/parsers/openclaw.ts +310 -0
- package/src/o11y/prices.json +746 -311
- package/src/o11y/tool-names.test.ts +1 -0
- package/src/o11y/types.ts +16 -2
- package/src/report/aggregate.ts +34 -5
- package/src/report/built-in-user-parity.test.tsx +110 -153
- package/src/report/built-ins/experiment-comparison.tsx +173 -13
- package/src/report/built-ins/index.ts +6 -1
- package/src/report/components.tsx +9 -3
- package/src/report/compute.ts +70 -40
- package/src/report/dual-render.test.tsx +194 -67
- package/src/report/flag.ts +30 -2
- package/src/report/format.ts +35 -11
- package/src/report/index.ts +22 -4
- package/src/report/locale.ts +25 -5
- package/src/report/metrics.ts +67 -14
- package/src/report/primitives.tsx +6 -0
- package/src/report/react/AttemptList.tsx +6 -31
- package/src/report/react/EvalList.tsx +0 -0
- package/src/report/react/ExperimentComparison.tsx +68 -0
- package/src/report/react/ExperimentList.tsx +15 -9
- package/src/report/react/MetricScatter.tsx +12 -14
- package/src/report/react/chart-math.test.ts +85 -0
- package/src/report/react/chart-math.ts +101 -22
- package/src/report/react/enhance.js +33 -1
- package/src/report/react/fixtures.ts +24 -17
- package/src/report/react/render.test.tsx +9 -64
- package/src/report/react/styles.css +73 -2
- package/src/report/report.test.ts +306 -98
- package/src/report/report.ts +6 -2
- package/src/report/text/faces.ts +47 -43
- package/src/report/text/table.ts +42 -5
- package/src/report/types.ts +41 -21
- package/src/results/annotated-source.test.ts +62 -9
- package/src/results/annotated-source.ts +64 -6
- package/src/results/attempt-evidence.test.ts +9 -7
- package/src/results/attempt-evidence.ts +15 -8
- package/src/results/attempt-source.ts +6 -3
- package/src/results/copy.ts +145 -55
- package/src/results/host-equivalence.test.ts +8 -6
- package/src/results/index.ts +2 -0
- package/src/results/locator.test.ts +1 -22
- package/src/results/open.ts +7 -1
- package/src/results/publish.ts +149 -0
- package/src/results/results.test.ts +85 -51
- package/src/results/truncate.ts +90 -0
- package/src/results/types.ts +7 -0
- package/src/results/writer.ts +31 -13
- package/src/runner/attempt.test.ts +138 -7
- package/src/runner/attempt.ts +603 -104
- package/src/runner/discover.test.ts +47 -0
- package/src/runner/discover.ts +36 -2
- package/src/runner/eval-source.test.ts +1 -27
- package/src/runner/feedback/agent.test.ts +504 -0
- package/src/runner/feedback/agent.ts +409 -0
- package/src/runner/feedback/ci.test.ts +562 -0
- package/src/runner/feedback/ci.ts +401 -0
- package/src/runner/feedback/coordinator.test.ts +317 -0
- package/src/runner/feedback/coordinator.ts +397 -0
- package/src/runner/feedback/failure.ts +40 -0
- package/src/runner/feedback/human.test.ts +616 -0
- package/src/runner/feedback/human.ts +535 -0
- package/src/runner/feedback/index.ts +66 -0
- package/src/runner/feedback/io.ts +78 -0
- package/src/runner/feedback/profile.test.ts +50 -0
- package/src/runner/feedback/profile.ts +58 -0
- package/src/runner/feedback/reducer.test.ts +395 -0
- package/src/runner/feedback/reducer.ts +260 -0
- package/src/runner/feedback/renderer.ts +82 -0
- package/src/runner/feedback/sink.ts +203 -0
- package/src/runner/feedback/testing.ts +106 -0
- package/src/runner/ledger.test.ts +230 -0
- package/src/runner/ledger.ts +329 -0
- package/src/runner/report.test.ts +128 -3
- package/src/runner/report.ts +33 -9
- package/src/runner/reporters/artifacts.ts +8 -2
- package/src/runner/reporters/braintrust.test.ts +8 -7
- package/src/runner/reporters/braintrust.ts +9 -2
- package/src/runner/reporters/index.ts +2 -2
- package/src/runner/reporters/json.test.ts +162 -0
- package/src/runner/reporters/json.ts +35 -8
- package/src/runner/reporters/shared.ts +1 -5
- package/src/runner/run.test.ts +760 -3
- package/src/runner/run.ts +242 -36
- package/src/runner/sandbox-prep.ts +3 -42
- package/src/runner/timing.ts +158 -0
- package/src/runner/types.ts +518 -22
- package/src/sandbox/checkpoint.test.ts +55 -0
- package/src/sandbox/checkpoint.ts +29 -8
- package/src/sandbox/cli-commands.ts +407 -0
- package/src/sandbox/docker.ts +115 -16
- package/src/sandbox/e2b-agent-template.test.ts +56 -0
- package/src/sandbox/e2b-agent-template.ts +94 -0
- package/src/sandbox/e2b.ts +74 -9
- package/src/sandbox/errors.ts +111 -4
- package/src/sandbox/index.ts +2 -0
- package/src/sandbox/io-retry.test.ts +58 -0
- package/src/sandbox/io-retry.ts +45 -0
- package/src/sandbox/keep-registry.test.ts +86 -0
- package/src/sandbox/keep-registry.ts +142 -0
- package/src/sandbox/keep.ts +178 -0
- package/src/sandbox/paths.test.ts +1 -0
- package/src/sandbox/paths.ts +19 -8
- package/src/sandbox/registry.ts +20 -3
- package/src/sandbox/resolve.ts +76 -11
- package/src/sandbox/retry.test.ts +70 -0
- package/src/sandbox/retry.ts +46 -4
- package/src/sandbox/types.ts +44 -6
- package/src/sandbox/vercel.ts +43 -20
- package/src/scoring/collector.ts +60 -17
- package/src/scoring/coverage.ts +95 -0
- package/src/scoring/diff.ts +81 -0
- package/src/scoring/display.test.ts +121 -0
- package/src/scoring/display.ts +133 -0
- package/src/scoring/evidence.test.ts +189 -0
- package/src/scoring/judge.test.ts +142 -0
- package/src/scoring/judge.ts +15 -18
- package/src/scoring/scoped.ts +217 -50
- package/src/scoring/types.ts +117 -20
- package/src/scoring/verdict.ts +16 -4
- package/src/shared/aggregate.ts +3 -2
- package/src/shared/types.ts +31 -0
- package/src/show/compose.ts +2 -2
- package/src/show/index.ts +21 -1
- package/src/show/render.ts +619 -104
- package/src/show/show.test.ts +235 -19
- package/src/tty-line.ts +8 -26
- package/src/util.test.ts +1 -0
- package/src/util.ts +41 -0
- package/src/view/app/components/AttemptModal.tsx +153 -2
- package/src/view/app/components/CodeView.tsx +32 -11
- package/src/view/app/components/CopyControls.tsx +2 -2
- package/src/view/app/i18n.ts +6 -0
- package/src/view/app/lib/attempt-route.test.ts +1 -0
- package/src/view/app/lib/verdict.ts +7 -9
- package/src/view/artifact-serving.test.ts +2 -1
- package/src/view/client-dist/app.css +1 -1
- package/src/view/client-dist/app.js +17 -17
- package/src/view/data.test.ts +1 -0
- package/src/view/data.ts +11 -1
- package/src/view/index.ts +11 -0
- package/src/view/server.ts +2 -0
- package/src/view/styles.css +3 -0
- package/src/view/view-report.test.ts +6 -5
- package/src/runner/reporters/console.ts +0 -70
- package/src/runner/reporters/live.test.ts +0 -56
- package/src/runner/reporters/live.ts +0 -247
- package/src/runner/reporters/quiet.test.ts +0 -66
- package/src/runner/reporters/quiet.ts +0 -49
- package/src/runner/reporters/table.ts +0 -277
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
// cases: docs/engineering/unit-tests/results/cases.md
|
|
1
2
|
// AttemptLocator 单测(定稿见 docs/feature/results/library.md「按 locator 寻址一个 attempt」、docs/concepts.md「Attempt 定位符」):
|
|
2
3
|
// 确定性编码、字段敏感(任一身份字段变化都换 locator)、@ 前缀与定长 base36 格式、decode 的语法
|
|
3
4
|
// 校验分支、大批量下的实际无撞车、buildLocatorIndex 的撞车检测(用可注入的 encode 强制制造撞车,
|
|
@@ -79,10 +80,6 @@ describe("encodeAttemptLocator · 确定性与格式", () => {
|
|
|
79
80
|
});
|
|
80
81
|
|
|
81
82
|
describe("decodeAttemptLocator · 语法校验", () => {
|
|
82
|
-
it("接受一个真实编码出来的 locator", () => {
|
|
83
|
-
expect(decodeAttemptLocator(encodeAttemptLocator(id()))).toEqual({ valid: true, scheme: 1 });
|
|
84
|
-
});
|
|
85
|
-
|
|
86
83
|
it.each([
|
|
87
84
|
["", "空字符串"],
|
|
88
85
|
["7k2m9qz", "缺 @ 前缀"],
|
|
@@ -149,24 +146,6 @@ describe("buildLocatorIndex", () => {
|
|
|
149
146
|
expect(err.message).toContain("collision");
|
|
150
147
|
});
|
|
151
148
|
|
|
152
|
-
it("撞车中若第三个身份与已记录身份相同,不重复抛错(只有真正不同的身份对才算撞车)", () => {
|
|
153
|
-
const sameEncode = (() => {
|
|
154
|
-
let calls = 0;
|
|
155
|
-
return (_identity: AttemptIdentity) => {
|
|
156
|
-
calls++;
|
|
157
|
-
return `@1samesame` as ReturnType<typeof encodeAttemptLocator>;
|
|
158
|
-
};
|
|
159
|
-
})();
|
|
160
|
-
const identity = id();
|
|
161
|
-
const index = buildLocatorIndex(
|
|
162
|
-
[
|
|
163
|
-
{ identity, handle: "first" },
|
|
164
|
-
{ identity, handle: "second" },
|
|
165
|
-
],
|
|
166
|
-
sameEncode,
|
|
167
|
-
);
|
|
168
|
-
expect(index.size).toBe(1);
|
|
169
|
-
});
|
|
170
149
|
});
|
|
171
150
|
|
|
172
151
|
describe("resolveAttemptLocator", () => {
|
package/src/results/open.ts
CHANGED
|
@@ -11,6 +11,7 @@ import { basename, dirname, join, relative, resolve, sep } from "node:path";
|
|
|
11
11
|
import type { EvalResult } from "../types.ts";
|
|
12
12
|
import type { O11ySummary, StreamEvent, TraceSpan } from "../types.ts";
|
|
13
13
|
import type { AgentSetupManifest, DiffData, SourceArtifact } from "../types.ts";
|
|
14
|
+
import { deriveDiffData } from "../scoring/diff.ts";
|
|
14
15
|
import { RESULT_FILE, SNAPSHOT_FILE, artifactFileOf, classifySnapshot } from "./format.ts";
|
|
15
16
|
import { isNewerSnapshot, selectLatest } from "./select.ts";
|
|
16
17
|
import {
|
|
@@ -360,7 +361,12 @@ function makeAttempt(snapshot: Snapshot, snapshotDir: string, attemptDir: string
|
|
|
360
361
|
trace: lazyArtifact<TraceSpan[]>(candidates, "trace", record.trace),
|
|
361
362
|
o11y: lazyArtifact<O11ySummary>(candidates, "o11y", record.o11y),
|
|
362
363
|
agentSetup: lazyArtifact<AgentSetupManifest>(candidates, "agentSetup", record.agentSetup),
|
|
363
|
-
diff:
|
|
364
|
+
diff: (() => {
|
|
365
|
+
// diff.json 落盘的是逐窗口 delta 序列(DiffArtifact);文件级视图在读取面派生。
|
|
366
|
+
const raw = lazyArtifact<import("../types.ts").DiffArtifact>(candidates, "diff", record.diff);
|
|
367
|
+
let memo: Promise<DiffData | null> | undefined;
|
|
368
|
+
return () => (memo ??= raw().then((windows) => (windows === null ? null : deriveDiffData(windows))));
|
|
369
|
+
})(),
|
|
364
370
|
sources: lazySources(candidates, candidateSnapshotRoots, record.sources),
|
|
365
371
|
};
|
|
366
372
|
}
|
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
// 发布消毒与发布预算(见 docs/feature/results/library.md「复制与瘦身:copySnapshots」)。
|
|
2
|
+
// 结果数据分两个等级:.niceeval/ 是本地事实根(未消毒);任何要离开本机的拷贝是发布拷贝,
|
|
3
|
+
// 只能经 copySnapshots 这一条管线产出,消毒在这里做且没有隐式默认(redact 必填)。
|
|
4
|
+
//
|
|
5
|
+
// redact 逐值消毒,范围由 schema 的自由文本标注决定:redactor 只对自由文本字段调用——
|
|
6
|
+
// 格式、判定、身份、路径、哈希这类结构字段永不经过它,发布根不会因 redact 变得不可读或
|
|
7
|
+
// 引用断裂。自由文本清单在下面的 FREE_TEXT 标注单点维护(与 AttemptList.redact 同一口径)。
|
|
8
|
+
|
|
9
|
+
import type { EvalResult, StreamEvent, TraceSpan } from "../types.ts";
|
|
10
|
+
|
|
11
|
+
/** 发布前整文件预检的单文件上限(50 MiB,为 GitHub 100 MB 硬限保留余量);不是可调旋钮。 */
|
|
12
|
+
export const PUBLISH_FILE_MAX_BYTES = 50 * 1024 * 1024;
|
|
13
|
+
|
|
14
|
+
export type Redactor = (text: string) => string;
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* 结构字段键名(取值是格式、判定、身份、路径、哈希——redactor 永不触碰):
|
|
18
|
+
* 事件的 type/callId/status/role/tool/skill/requestId/optionId、span 的 ids/kind、
|
|
19
|
+
* 断言的 name/severity/outcome、错误的 code/phase、locator/artifactBase/fingerprint、
|
|
20
|
+
* 源码路径与 sha256、provider 名、时间戳等。新增字符串字段先判断体裁再决定进不进这张表。
|
|
21
|
+
*/
|
|
22
|
+
const STRUCTURAL_KEYS = new Set([
|
|
23
|
+
"type",
|
|
24
|
+
"callId",
|
|
25
|
+
"status",
|
|
26
|
+
"role",
|
|
27
|
+
"tool",
|
|
28
|
+
"requestId",
|
|
29
|
+
"optionId",
|
|
30
|
+
"id",
|
|
31
|
+
"traceId",
|
|
32
|
+
"spanId",
|
|
33
|
+
"parentSpanId",
|
|
34
|
+
"kind",
|
|
35
|
+
"format",
|
|
36
|
+
"verdict",
|
|
37
|
+
"severity",
|
|
38
|
+
"outcome",
|
|
39
|
+
"artifactBase",
|
|
40
|
+
"locator",
|
|
41
|
+
"fingerprint",
|
|
42
|
+
"sha256",
|
|
43
|
+
"path",
|
|
44
|
+
"file",
|
|
45
|
+
"provider",
|
|
46
|
+
"sandboxId",
|
|
47
|
+
"code",
|
|
48
|
+
"phase",
|
|
49
|
+
"level",
|
|
50
|
+
"agent",
|
|
51
|
+
"model",
|
|
52
|
+
"experimentId",
|
|
53
|
+
"startedAt",
|
|
54
|
+
"completedAt",
|
|
55
|
+
"schemaVersion",
|
|
56
|
+
"evalFilterFingerprint",
|
|
57
|
+
"reasoningEffort",
|
|
58
|
+
"skill",
|
|
59
|
+
"window",
|
|
60
|
+
"net",
|
|
61
|
+
"loc",
|
|
62
|
+
"dedupeKey",
|
|
63
|
+
]);
|
|
64
|
+
|
|
65
|
+
/** 深度遍历 JSON 值:字符串按「键名是否结构字段」决定过不过 redactor;结构键下整棵子树跳过。 */
|
|
66
|
+
export function redactJsonValue(value: unknown, redact: Redactor, key?: string): unknown {
|
|
67
|
+
if (typeof value === "string") {
|
|
68
|
+
if (key !== undefined && STRUCTURAL_KEYS.has(key)) return value;
|
|
69
|
+
return redact(value);
|
|
70
|
+
}
|
|
71
|
+
if (Array.isArray(value)) return value.map((v) => redactJsonValue(v, redact, key));
|
|
72
|
+
if (value !== null && typeof value === "object") {
|
|
73
|
+
const out: Record<string, unknown> = {};
|
|
74
|
+
for (const [k, v] of Object.entries(value)) {
|
|
75
|
+
if (STRUCTURAL_KEYS.has(k) && typeof v === "string") {
|
|
76
|
+
out[k] = v;
|
|
77
|
+
continue;
|
|
78
|
+
}
|
|
79
|
+
out[k] = redactJsonValue(v, redact, k);
|
|
80
|
+
}
|
|
81
|
+
return out;
|
|
82
|
+
}
|
|
83
|
+
return value;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
/** events.json 的发布消毒:消息与工具入出参是自由文本,type/callId/status 等结构字段不动。 */
|
|
87
|
+
export function redactEvents(events: StreamEvent[], redact: Redactor): StreamEvent[] {
|
|
88
|
+
return redactJsonValue(events, redact) as StreamEvent[];
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/** trace.json 的发布消毒:属性值与可携带动态内容的 span name 过 redactor,ids/kind 不动。 */
|
|
92
|
+
export function redactSpans(spans: TraceSpan[], redact: Redactor): TraceSpan[] {
|
|
93
|
+
return spans.map((span) => ({
|
|
94
|
+
...span,
|
|
95
|
+
name: redact(span.name),
|
|
96
|
+
...(span.attributes !== undefined
|
|
97
|
+
? { attributes: redactJsonValue(span.attributes, redact) as TraceSpan["attributes"] }
|
|
98
|
+
: {}),
|
|
99
|
+
}));
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/** result.json 的发布消毒:断言 detail/evidence/expected/received、error/diagnostic 的
|
|
103
|
+
* message/cause/stack、skipReason、description 是自由文本;判定与身份字段不动。 */
|
|
104
|
+
export function redactResultRecord(record: Record<string, unknown>, redact: Redactor): Record<string, unknown> {
|
|
105
|
+
const r = record as Partial<EvalResult> & Record<string, unknown>;
|
|
106
|
+
const out: Record<string, unknown> = { ...record };
|
|
107
|
+
if (typeof r.description === "string") out.description = redact(r.description);
|
|
108
|
+
if (typeof r.skipReason === "string") out.skipReason = redact(r.skipReason);
|
|
109
|
+
if (Array.isArray(r.assertions)) {
|
|
110
|
+
out.assertions = r.assertions.map((a) => ({
|
|
111
|
+
...a,
|
|
112
|
+
...(a.detail !== undefined ? { detail: redact(a.detail) } : {}),
|
|
113
|
+
...(a.outcome !== "unavailable" && a.evidence !== undefined ? { evidence: redact(a.evidence) } : {}),
|
|
114
|
+
...(a.outcome !== "unavailable" && a.expected !== undefined ? { expected: redact(a.expected) } : {}),
|
|
115
|
+
...(a.outcome !== "unavailable" && a.received !== undefined ? { received: redact(a.received) } : {}),
|
|
116
|
+
}));
|
|
117
|
+
}
|
|
118
|
+
if (r.error !== undefined) {
|
|
119
|
+
out.error = {
|
|
120
|
+
...r.error,
|
|
121
|
+
message: redact(r.error.message),
|
|
122
|
+
...(r.error.stack !== undefined ? { stack: redact(r.error.stack) } : {}),
|
|
123
|
+
...(r.error.cause !== undefined ? { cause: { ...r.error.cause, message: redact(r.error.cause.message) } } : {}),
|
|
124
|
+
};
|
|
125
|
+
}
|
|
126
|
+
if (Array.isArray(r.diagnostics)) {
|
|
127
|
+
out.diagnostics = r.diagnostics.map((d) => ({
|
|
128
|
+
...d,
|
|
129
|
+
message: redact(d.message),
|
|
130
|
+
...(d.data !== undefined ? { data: redactJsonValue(d.data, redact) as typeof d.data } : {}),
|
|
131
|
+
}));
|
|
132
|
+
}
|
|
133
|
+
if (r.experiment !== undefined) {
|
|
134
|
+
out.experiment = redactExperimentInfo(r.experiment as unknown as Record<string, unknown>, redact);
|
|
135
|
+
}
|
|
136
|
+
return out;
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
/** ExperimentRunInfo 的发布消毒:description、flags 与 sandbox params 的字符串值。 */
|
|
140
|
+
export function redactExperimentInfo(info: Record<string, unknown>, redact: Redactor): Record<string, unknown> {
|
|
141
|
+
const out: Record<string, unknown> = { ...info };
|
|
142
|
+
if (typeof info.description === "string") out.description = redact(info.description);
|
|
143
|
+
if (info.flags !== undefined) out.flags = redactJsonValue(info.flags, redact);
|
|
144
|
+
const sandbox = info.sandbox as { provider: string; params?: Record<string, unknown> } | undefined;
|
|
145
|
+
if (sandbox?.params !== undefined) {
|
|
146
|
+
out.sandbox = { ...sandbox, params: redactJsonValue(sandbox.params, redact) };
|
|
147
|
+
}
|
|
148
|
+
return out;
|
|
149
|
+
}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
// cases: docs/engineering/unit-tests/results/cases.md
|
|
1
2
|
// niceeval/results 的单测:临时目录里构造最小 snapshot.json / result.json / artifact fixture,
|
|
2
3
|
// 覆盖定稿契约(docs/feature/results/library.md、docs/feature/results/architecture.md):分层读取(含快照级字段注入)、
|
|
3
4
|
// 懒加载与 artifactBase 回退、skipped 三种原因、unfinished-snapshot、latest() 三种警告、
|
|
@@ -464,7 +465,7 @@ describe("createResultsWriter", () => {
|
|
|
464
465
|
await snapA.writeAttempt({ id: "q2", verdict: "failed", attempt: 1, durationMs: 50, assertions: [] });
|
|
465
466
|
|
|
466
467
|
const snapB = await writer.snapshot({ experimentId: "compare/b", agent: "codex", startedAt: "2026-07-02T09:00:00.000Z" });
|
|
467
|
-
await snapB.writeAttempt({ id: "q1", verdict: "passed", attempt: 1, durationMs: 80, assertions: [] }, { diff: {
|
|
468
|
+
await snapB.writeAttempt({ id: "q1", verdict: "passed", attempt: 1, durationMs: 80, assertions: [] }, { diff: [{ window: "s1/t1", changes: { "a.txt": { status: "added", after: "1" } } }] });
|
|
468
469
|
|
|
469
470
|
await writer.finish();
|
|
470
471
|
expect(writer.snapshotDirs().map((s) => s.experimentId).sort()).toEqual(["compare/a", "compare/b"]);
|
|
@@ -506,7 +507,9 @@ describe("createResultsWriter", () => {
|
|
|
506
507
|
|
|
507
508
|
const b = results.experiments[1].latest;
|
|
508
509
|
expect(b.model).toBeUndefined();
|
|
509
|
-
|
|
510
|
+
const bDiff = await b.attempts[0].diff();
|
|
511
|
+
expect(bDiff?.windows).toEqual([{ window: "s1/t1", changes: { "a.txt": { status: "added", after: "1" } } }]);
|
|
512
|
+
expect(bDiff?.get("a.txt")).toBe("1");
|
|
510
513
|
|
|
511
514
|
const partial = results.latest().warnings.find((w) => w.kind === "partial-coverage")!;
|
|
512
515
|
expect(partial).toMatchObject({ experimentId: "compare/a", covered: 2, total: 3 });
|
|
@@ -549,7 +552,7 @@ describe("createResultsWriter", () => {
|
|
|
549
552
|
expect(await q2.agentSetup()).toBeNull();
|
|
550
553
|
|
|
551
554
|
const dest = join(await makeRoot(), "published");
|
|
552
|
-
await copySnapshots(results.latest(), dest, { artifacts: ["agentSetup"] });
|
|
555
|
+
await copySnapshots(results.latest(), dest, { ...{ artifacts: ["agentSetup"] }, redact: false });
|
|
553
556
|
const copied = join(dest, "skill-ab_claude-effect", basename(snap.dir), "q1", "a1", "agent-setup.json");
|
|
554
557
|
expect(JSON.parse(await readFile(copied, "utf-8"))).toEqual(manifest);
|
|
555
558
|
});
|
|
@@ -590,27 +593,32 @@ describe("createResultsWriter", () => {
|
|
|
590
593
|
|
|
591
594
|
it("快照目录独占创建:撞名换随机后缀重试直到成功(EEXIST 不会覆盖已有目录)", async () => {
|
|
592
595
|
const root = await makeRoot();
|
|
593
|
-
|
|
594
|
-
|
|
596
|
+
// 受控随机序列:writer B 面对同一身份时重置序列,首次尝试重放 writer A 消耗过的
|
|
597
|
+
// 随机值,必然撞上 A 的目录;重试消耗序列后续值,得到不同后缀。测试不假设一个
|
|
598
|
+
// 后缀消耗几次 random,也不假设数值到后缀字符的映射。
|
|
595
599
|
let call = 0;
|
|
596
600
|
const randomSpy = vi.spyOn(Math, "random").mockImplementation(() => {
|
|
597
601
|
call += 1;
|
|
598
|
-
return call
|
|
602
|
+
return ((call * 37) % 977) / 977;
|
|
599
603
|
});
|
|
600
604
|
try {
|
|
601
|
-
const
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
const
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
605
|
+
const identity = { experimentId: "e", agent: "bub", startedAt: "2026-07-01T08:00:00.000Z" } as const;
|
|
606
|
+
const writerA = createResultsWriter(root, { producer: { name: "x" } });
|
|
607
|
+
const snapA = await writerA.snapshot(identity);
|
|
608
|
+
const canary = join(snapA.dir, "canary.txt");
|
|
609
|
+
await writeFile(canary, "occupied", "utf8");
|
|
610
|
+
|
|
611
|
+
call = 0; // 重放同一序列,让 writer B 的首次尝试与 snapA 同名
|
|
612
|
+
const writerB = createResultsWriter(root, { producer: { name: "x" } });
|
|
613
|
+
const snapB = await writerB.snapshot(identity);
|
|
614
|
+
|
|
615
|
+
expect(snapB.dir).not.toBe(snapA.dir);
|
|
616
|
+
expect(await exists(join(snapB.dir, "snapshot.json"))).toBe(true);
|
|
617
|
+
// 被占用的目录内容原样保留 —— 独占创建不会覆盖已有内容。
|
|
618
|
+
expect(await exists(canary)).toBe(true);
|
|
619
|
+
expect(await exists(join(snapB.dir, "canary.txt"))).toBe(false);
|
|
611
620
|
} finally {
|
|
612
621
|
randomSpy.mockRestore();
|
|
613
|
-
vi.useRealTimers();
|
|
614
622
|
}
|
|
615
623
|
});
|
|
616
624
|
|
|
@@ -621,7 +629,7 @@ describe("createResultsWriter", () => {
|
|
|
621
629
|
const fresh: EvalResult = {
|
|
622
630
|
id: "algebra/q1",
|
|
623
631
|
experimentId: "compare/bub",
|
|
624
|
-
experiment: {
|
|
632
|
+
experiment: { flags: { style: "concise" }, runs: 1, earlyExit: true, selectedEvalIds: ["algebra/q1"] },
|
|
625
633
|
agent: "bub",
|
|
626
634
|
model: "gpt-5.4",
|
|
627
635
|
verdict: "passed",
|
|
@@ -636,7 +644,7 @@ describe("createResultsWriter", () => {
|
|
|
636
644
|
sources: [{ path: "evals/a.ts", content: "x" }],
|
|
637
645
|
trace: [{ name: "turn", kind: "turn" } as never],
|
|
638
646
|
o11y: { toolCalls: 2 } as never,
|
|
639
|
-
diff: {
|
|
647
|
+
diff: [{ window: "s1/t1", changes: { "a.txt": { status: "added", after: "1" } } }],
|
|
640
648
|
rawTranscript: "raw",
|
|
641
649
|
};
|
|
642
650
|
const carried: EvalResult = {
|
|
@@ -714,7 +722,7 @@ describe("copySnapshots", () => {
|
|
|
714
722
|
await writeResultFile(monday, "q1/a1", record({ id: "q1", attempt: 1, hasEvents: true, hasTrace: true }));
|
|
715
723
|
await writeArtifactFile(monday, "q1/a1", "events.json", [{ n: 1 }]);
|
|
716
724
|
await writeArtifactFile(monday, "q1/a1", "trace.json", [{ name: "turn" }]);
|
|
717
|
-
await writeArtifactFile(monday, "q1/a1", "diff.json", {
|
|
725
|
+
await writeArtifactFile(monday, "q1/a1", "diff.json", [{ window: "s1/t1", changes: {} }]);
|
|
718
726
|
await writeResultFile(monday, "q2/a1", record({ id: "q2", attempt: 1 }));
|
|
719
727
|
|
|
720
728
|
// 周五只重跑了 q1:最新快照残缺。
|
|
@@ -729,7 +737,7 @@ describe("copySnapshots", () => {
|
|
|
729
737
|
|
|
730
738
|
const results = await openResults(root);
|
|
731
739
|
const dest = join(await makeRoot(), "site/data/run");
|
|
732
|
-
const copied = await copySnapshots(results.latest(), dest, { artifacts: ["events"] });
|
|
740
|
+
const copied = await copySnapshots(results.latest(), dest, { ...{ artifacts: ["events"] }, redact: false });
|
|
733
741
|
|
|
734
742
|
expect(copied.warnings).toHaveLength(0);
|
|
735
743
|
expect(copied.dir).toBe(dest);
|
|
@@ -767,15 +775,15 @@ describe("copySnapshots", () => {
|
|
|
767
775
|
|
|
768
776
|
const occupied = await makeRoot();
|
|
769
777
|
await writeFile(join(occupied, "existing.txt"), "x", "utf-8");
|
|
770
|
-
await expect(copySnapshots(results.latest(), occupied)).rejects.toThrow(/not empty/);
|
|
778
|
+
await expect(copySnapshots(results.latest(), occupied, { redact: false })).rejects.toThrow(/not empty/);
|
|
771
779
|
|
|
772
|
-
await expect(copySnapshots(results.latest(), join(await makeRoot(), "out"), { artifacts: ["evnets" as never] })).rejects.toThrow(/Unknown artifact kind/);
|
|
780
|
+
await expect(copySnapshots(results.latest(), join(await makeRoot(), "out"), { ...{ artifacts: ["evnets" as never] }, redact: false })).rejects.toThrow(/Unknown artifact kind/);
|
|
773
781
|
|
|
774
|
-
await expect(copySnapshots([], join(await makeRoot(), "out"))).rejects.toThrow(/no snapshots/);
|
|
782
|
+
await expect(copySnapshots([], join(await makeRoot(), "out"), { redact: false })).rejects.toThrow(/no snapshots/);
|
|
775
783
|
|
|
776
784
|
// 手工传入同一 experiment 的两个快照(未走 latest 去重):只带最新,记 warning。
|
|
777
785
|
const dest2 = join(await makeRoot(), "run2");
|
|
778
|
-
const collided = await copySnapshots(results.experiments[0].snapshots, dest2);
|
|
786
|
+
const collided = await copySnapshots(results.experiments[0].snapshots, dest2, { redact: false });
|
|
779
787
|
expect(collided.warnings).toHaveLength(1);
|
|
780
788
|
expect(collided.warnings[0]).toMatch(/multiple snapshots selected/);
|
|
781
789
|
const destDirs = await readdir(join(dest2, "e"));
|
|
@@ -996,48 +1004,74 @@ describe("AttemptLocator · 落盘 / 读取 / 携带 / 撞车", () => {
|
|
|
996
1004
|
const locator1 = a1!.locator!;
|
|
997
1005
|
|
|
998
1006
|
const dest = join(await makeRoot(), "published");
|
|
999
|
-
await copySnapshots(results.latest(), dest, { artifacts: [] });
|
|
1007
|
+
await copySnapshots(results.latest(), dest, { ...{ artifacts: [] }, redact: false });
|
|
1000
1008
|
|
|
1001
1009
|
const destResults = await openResults(dest);
|
|
1002
1010
|
expect(resolveLocator(destResults, locator0).result.attempt).toBe(0);
|
|
1003
1011
|
expect(resolveLocator(destResults, locator1).result.attempt).toBe(1);
|
|
1004
1012
|
});
|
|
1005
|
-
});
|
|
1006
|
-
|
|
1007
|
-
// ───────────────────────── sources 去重仓库 ─────────────────────────
|
|
1008
1013
|
|
|
1009
|
-
|
|
1010
|
-
it("两个 attempt 共享字节相同的 eval 源码:去重仓库只落一份 blob(文件数 = 1)", async () => {
|
|
1014
|
+
it("createResultsWriter({ snapshotStartedAt }):writeAttemptFor() 的隐式声明统一用这个锚点,不再按各 result 自己的 attempt startedAt 分别猜", async () => {
|
|
1011
1015
|
const root = await makeRoot();
|
|
1012
|
-
const
|
|
1013
|
-
const
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1016
|
+
const snapshotStartedAt = "2026-07-10T00:00:00.000Z";
|
|
1017
|
+
const writer = createResultsWriter(root, {
|
|
1018
|
+
producer: { name: "niceeval", version: "1.0.0" },
|
|
1019
|
+
snapshotStartedAt,
|
|
1020
|
+
});
|
|
1021
|
+
|
|
1022
|
+
// 两个不同 experiment,各自的 result.startedAt 与 writer 级锚点都不同 —— 如果还在按
|
|
1023
|
+
// 「该 experiment 首条落盘结果的 attempt 时刻」猜,两份 snapshot.json 的 startedAt 会
|
|
1024
|
+
// 分别变成各自 result 的 attempt 时刻,而不是这里统一传入的 snapshotStartedAt。
|
|
1025
|
+
await writer.writeAttemptFor({
|
|
1026
|
+
id: "q1", experimentId: "e1", agent: "bub", verdict: "passed", attempt: 0,
|
|
1027
|
+
startedAt: "2020-01-01T00:00:00.000Z", durationMs: 1, assertions: [],
|
|
1028
|
+
});
|
|
1029
|
+
await writer.writeAttemptFor({
|
|
1030
|
+
id: "q1", experimentId: "e2", agent: "bub", verdict: "passed", attempt: 0,
|
|
1031
|
+
startedAt: "2021-06-15T00:00:00.000Z", durationMs: 1, assertions: [],
|
|
1032
|
+
});
|
|
1017
1033
|
await writer.finish();
|
|
1018
1034
|
|
|
1019
|
-
const
|
|
1020
|
-
|
|
1035
|
+
const results = await openResults(root);
|
|
1036
|
+
const expE1 = results.experiments.find((e) => e.id === "e1")!;
|
|
1037
|
+
const expE2 = results.experiments.find((e) => e.id === "e2")!;
|
|
1038
|
+
expect(expE1.latest.startedAt).toBe(snapshotStartedAt);
|
|
1039
|
+
expect(expE2.latest.startedAt).toBe(snapshotStartedAt); // 两个 experiment 共用同一个锚点,不碰撞
|
|
1040
|
+
|
|
1041
|
+
// attempt 级 startedAt(墙钟事实)依然各自独立保留,没有被锚点覆盖 ——
|
|
1042
|
+
// 快照身份锚点与 attempt 墙钟事实是两回事,继续分别保存。
|
|
1043
|
+
const attempt1 = expE1.latest.evals[0]!.attempts[0]!;
|
|
1044
|
+
const attempt2 = expE2.latest.evals[0]!.attempts[0]!;
|
|
1045
|
+
expect(attempt1.result.startedAt).toBe("2020-01-01T00:00:00.000Z");
|
|
1046
|
+
expect(attempt2.result.startedAt).toBe("2021-06-15T00:00:00.000Z");
|
|
1047
|
+
|
|
1048
|
+
// locator 由统一锚点 + experimentId 派生,两边不因为共享锚点而碰撞。
|
|
1049
|
+
expect(attempt1.locator).not.toBe(attempt2.locator);
|
|
1050
|
+
expect(attempt1.locator).toBe(
|
|
1051
|
+
encodeAttemptLocator({ experimentId: "e1", snapshotStartedAt, evalId: "q1", attempt: 0 }),
|
|
1052
|
+
);
|
|
1053
|
+
expect(attempt2.locator).toBe(
|
|
1054
|
+
encodeAttemptLocator({ experimentId: "e2", snapshotStartedAt, evalId: "q1", attempt: 0 }),
|
|
1055
|
+
);
|
|
1021
1056
|
});
|
|
1022
1057
|
|
|
1023
|
-
it("
|
|
1058
|
+
it("createResultsWriter() 省略 snapshotStartedAt 时保留旧的按首条 result.startedAt 猜测行为(第三方直调兼容)", async () => {
|
|
1024
1059
|
const root = await makeRoot();
|
|
1025
1060
|
const writer = createResultsWriter(root, { producer: { name: "niceeval", version: "1.0.0" } });
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
);
|
|
1031
|
-
await snap.writeAttempt(
|
|
1032
|
-
{ id: "q2", verdict: "passed", attempt: 0, durationMs: 1, assertions: [] },
|
|
1033
|
-
{ sources: [{ path: "evals/b.eval.ts", content: "export default { test() { /* different */ } };\n" }] },
|
|
1034
|
-
);
|
|
1061
|
+
await writer.writeAttemptFor({
|
|
1062
|
+
id: "q1", experimentId: "e", agent: "bub", verdict: "passed", attempt: 0,
|
|
1063
|
+
startedAt: "2020-03-03T00:00:00.000Z", durationMs: 1, assertions: [],
|
|
1064
|
+
});
|
|
1035
1065
|
await writer.finish();
|
|
1036
1066
|
|
|
1037
|
-
const
|
|
1038
|
-
expect(
|
|
1067
|
+
const results = await openResults(root);
|
|
1068
|
+
expect(results.experiments[0]!.latest.startedAt).toBe("2020-03-03T00:00:00.000Z");
|
|
1039
1069
|
});
|
|
1070
|
+
});
|
|
1040
1071
|
|
|
1072
|
+
// ───────────────────────── sources 去重仓库 ─────────────────────────
|
|
1073
|
+
|
|
1074
|
+
describe("sources · 快照级去重仓库", () => {
|
|
1041
1075
|
it("经真实 --resume carry 流程(writeAttemptFor 的 artifactBase 分支)携带的 attempt,其 sources() 引用在新快照里依然能解到原快照内容", async () => {
|
|
1042
1076
|
const root = await makeRoot();
|
|
1043
1077
|
const content = "export default { test() {} };\n";
|
|
@@ -1157,7 +1191,7 @@ describe("sources · 快照级去重仓库", () => {
|
|
|
1157
1191
|
|
|
1158
1192
|
const results = await openResults(root);
|
|
1159
1193
|
const dest = join(await makeRoot(), "published");
|
|
1160
|
-
await copySnapshots(results.latest(), dest, { artifacts: ["sources"] });
|
|
1194
|
+
await copySnapshots(results.latest(), dest, { ...{ artifacts: ["sources"] }, redact: false });
|
|
1161
1195
|
|
|
1162
1196
|
const destSnapDir = join(dest, "e", basename(snap.dir));
|
|
1163
1197
|
const destStoreFiles = await readdir(join(destSnapDir, "sources"));
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
// 落盘大值截断(见 docs/feature/results/architecture.md「大值截断」)。
|
|
2
|
+
// 运行时全量,落盘截断:落点唯一在 snap.writeAttempt(writer.ts)——不在 adapter、不在 OTLP
|
|
3
|
+
// 解析、不在事件归一化里做。适用范围:events.json 的事件字段与 trace.json 的 span 属性里的
|
|
4
|
+
// 任意字符串值。没有 flag、没有配置项;截断永远不影响判决(落盘是证据,不是评分输入)。
|
|
5
|
+
|
|
6
|
+
import type { StreamEvent, TraceSpan, Truncation } from "../types.ts";
|
|
7
|
+
|
|
8
|
+
/** 每个字符串值的落盘上限(UTF-8 字节)。 */
|
|
9
|
+
export const ARTIFACT_VALUE_MAX_BYTES = 256 * 1024;
|
|
10
|
+
|
|
11
|
+
const encoder = new TextEncoder();
|
|
12
|
+
|
|
13
|
+
/** 截断到 maxBytes(按 UTF-8 字符边界回退,不切断多字节字符),末尾追加人可读 marker。 */
|
|
14
|
+
function truncateString(value: string, originalBytes: number): string {
|
|
15
|
+
// 按字节预算切:先按「字节数 ≥ 字符数」粗切,再逐步回退到不超预算的字符边界。
|
|
16
|
+
let sliceLen = Math.min(value.length, ARTIFACT_VALUE_MAX_BYTES);
|
|
17
|
+
let head = value.slice(0, sliceLen);
|
|
18
|
+
while (encoder.encode(head).length > ARTIFACT_VALUE_MAX_BYTES && sliceLen > 0) {
|
|
19
|
+
// 超出量 ÷ 4(UTF-8 最长 4 字节)是安全的最小回退步长;循环兜底处理边界。
|
|
20
|
+
sliceLen -= Math.max(1, Math.ceil((encoder.encode(head).length - ARTIFACT_VALUE_MAX_BYTES) / 4));
|
|
21
|
+
head = value.slice(0, sliceLen);
|
|
22
|
+
}
|
|
23
|
+
const kept = encoder.encode(head).length;
|
|
24
|
+
return `${head}\n[niceeval] truncated ${originalBytes} → ${kept} bytes`;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* 深度遍历一个 JSON 值,截断其中超限的字符串;返回(可能新建的)值与收集到的截断记录。
|
|
29
|
+
* `path` 用点分段(数组下标同样入段),事件里从字段名起、span 属性里从 attribute key 起。
|
|
30
|
+
*/
|
|
31
|
+
function truncateJsonValue(value: unknown, path: string, out: Truncation[]): unknown {
|
|
32
|
+
if (typeof value === "string") {
|
|
33
|
+
const bytes = encoder.encode(value).length;
|
|
34
|
+
if (bytes <= ARTIFACT_VALUE_MAX_BYTES) return value;
|
|
35
|
+
out.push({ path, originalBytes: bytes });
|
|
36
|
+
return truncateString(value, bytes);
|
|
37
|
+
}
|
|
38
|
+
if (Array.isArray(value)) {
|
|
39
|
+
let changed = false;
|
|
40
|
+
const next = value.map((v, i) => {
|
|
41
|
+
const r = truncateJsonValue(v, `${path}.${i}`, out);
|
|
42
|
+
if (r !== v) changed = true;
|
|
43
|
+
return r;
|
|
44
|
+
});
|
|
45
|
+
return changed ? next : value;
|
|
46
|
+
}
|
|
47
|
+
if (value !== null && typeof value === "object") {
|
|
48
|
+
let changed = false;
|
|
49
|
+
const next: Record<string, unknown> = {};
|
|
50
|
+
for (const [k, v] of Object.entries(value)) {
|
|
51
|
+
const r = truncateJsonValue(v, path ? `${path}.${k}` : k, out);
|
|
52
|
+
if (r !== v) changed = true;
|
|
53
|
+
next[k] = r;
|
|
54
|
+
}
|
|
55
|
+
return changed ? next : value;
|
|
56
|
+
}
|
|
57
|
+
return value;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/** events.json 落盘前的截断:逐事件处理全部字符串字段,超限的加结构化 `truncated` 标记。 */
|
|
61
|
+
export function truncateEvents(events: readonly StreamEvent[]): StreamEvent[] {
|
|
62
|
+
return events.map((event) => {
|
|
63
|
+
const out: Truncation[] = [];
|
|
64
|
+
const next: Record<string, unknown> = {};
|
|
65
|
+
let changed = false;
|
|
66
|
+
for (const [k, v] of Object.entries(event)) {
|
|
67
|
+
if (k === "type" || k === "truncated") {
|
|
68
|
+
next[k] = v;
|
|
69
|
+
continue;
|
|
70
|
+
}
|
|
71
|
+
const r = truncateJsonValue(v, k, out);
|
|
72
|
+
if (r !== v) changed = true;
|
|
73
|
+
next[k] = r;
|
|
74
|
+
}
|
|
75
|
+
if (!changed) return event;
|
|
76
|
+
return { ...next, truncated: [...(event.truncated ?? []), ...out] } as StreamEvent;
|
|
77
|
+
});
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/** trace.json 落盘前的截断:span 属性里的任意字符串值(name 一并覆盖——可携带动态内容)。 */
|
|
81
|
+
export function truncateSpans(spans: readonly TraceSpan[]): TraceSpan[] {
|
|
82
|
+
return spans.map((span) => {
|
|
83
|
+
const out: Truncation[] = [];
|
|
84
|
+
const attributes = span.attributes
|
|
85
|
+
? (truncateJsonValue(span.attributes, "", out) as TraceSpan["attributes"])
|
|
86
|
+
: span.attributes;
|
|
87
|
+
if (out.length === 0) return span;
|
|
88
|
+
return { ...span, attributes, truncated: [...(span.truncated ?? []), ...out] };
|
|
89
|
+
});
|
|
90
|
+
}
|
package/src/results/types.ts
CHANGED
|
@@ -44,6 +44,11 @@ export interface SnapshotMeta {
|
|
|
44
44
|
completedAt?: string;
|
|
45
45
|
/** 写入时刻该实验已知的 eval 并集 —— 残缺检测的分母随数据走(copySnapshots 自动补记,writer 可声明)。 */
|
|
46
46
|
knownEvalIds?: string[];
|
|
47
|
+
/**
|
|
48
|
+
* 发布拷贝的自描述标记:copySnapshots 补记,消毒函数 → "applied"、redact: false → "none";
|
|
49
|
+
* 本地事实根没有此字段。只声明流程,不证明无秘密;view --out 据此分级防呆。
|
|
50
|
+
*/
|
|
51
|
+
publish?: { redaction: "applied" | "none" };
|
|
47
52
|
/** 项目名(来自 config.name),透传给 `niceeval view` 顶部 hero 显示。 */
|
|
48
53
|
name?: LocalizedText;
|
|
49
54
|
}
|
|
@@ -125,6 +130,8 @@ export interface Snapshot {
|
|
|
125
130
|
dir: string;
|
|
126
131
|
/** 写入时刻该实验已知的 eval 并集(可选);copySnapshots 自动补记,writer.snapshot() 也可声明。 */
|
|
127
132
|
knownEvalIds?: string[];
|
|
133
|
+
/** 发布拷贝的自描述标记(见 SnapshotMeta.publish);本地事实根没有此字段。 */
|
|
134
|
+
publish?: { redaction: "applied" | "none" };
|
|
128
135
|
}
|
|
129
136
|
|
|
130
137
|
/** 一个实验的全部历史:同一 experiment id 的历次快照归在一起。 */
|