niceeval 0.6.1 → 0.6.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agents/types.d.ts +67 -5
- package/dist/context/types.d.ts +32 -12
- package/dist/i18n/en.d.ts +54 -0
- package/dist/i18n/zh-CN.d.ts +55 -1
- package/dist/o11y/types.d.ts +16 -2
- package/dist/report/aggregate.d.ts +5 -3
- package/dist/report/aggregate.js +32 -5
- package/dist/report/built-ins/experiment-comparison.d.ts +39 -1
- package/dist/report/built-ins/experiment-comparison.js +116 -10
- package/dist/report/built-ins/index.d.ts +1 -0
- package/dist/report/built-ins/index.js +1 -1
- package/dist/report/components.d.ts +8 -2
- package/dist/report/components.js +3 -3
- package/dist/report/compute.d.ts +11 -18
- package/dist/report/compute.js +54 -34
- package/dist/report/flag.d.ts +16 -1
- package/dist/report/flag.js +19 -1
- package/dist/report/format.d.ts +16 -8
- package/dist/report/format.js +27 -12
- package/dist/report/index.d.ts +4 -3
- package/dist/report/index.js +5 -4
- package/dist/report/locale.d.ts +11 -2
- package/dist/report/locale.js +23 -5
- package/dist/report/metrics.d.ts +13 -1
- package/dist/report/metrics.js +65 -14
- package/dist/report/primitives.d.ts +6 -0
- package/dist/report/react/AttemptList.d.ts +2 -2
- package/dist/report/react/AttemptList.js +5 -6
- package/dist/report/react/EvalList.d.ts +1 -1
- package/dist/report/react/EvalList.js +0 -0
- package/dist/report/react/ExperimentComparison.d.ts +8 -0
- package/dist/report/react/ExperimentComparison.js +11 -0
- package/dist/report/react/ExperimentList.d.ts +2 -1
- package/dist/report/react/ExperimentList.js +8 -10
- package/dist/report/react/MetricScatter.js +5 -11
- package/dist/report/react/chart-math.d.ts +23 -6
- package/dist/report/react/chart-math.js +71 -19
- package/dist/report/react/fixtures.d.ts +3 -3
- package/dist/report/react/fixtures.js +21 -14
- package/dist/report/report.d.ts +5 -1
- package/dist/report/report.js +6 -2
- package/dist/report/text/faces.d.ts +1 -1
- package/dist/report/text/faces.js +42 -41
- package/dist/report/text/table.js +36 -5
- package/dist/report/types.d.ts +39 -21
- package/dist/results/types.d.ts +11 -0
- package/dist/runner/feedback/sink.d.ts +110 -0
- package/dist/runner/types.d.ts +513 -22
- package/dist/sandbox/docker.d.ts +23 -2
- package/dist/sandbox/e2b.d.ts +15 -1
- package/dist/sandbox/errors.d.ts +30 -3
- package/dist/sandbox/io-retry.d.ts +17 -0
- package/dist/sandbox/registry.d.ts +2 -0
- package/dist/sandbox/resolve.d.ts +18 -5
- package/dist/sandbox/retry.d.ts +11 -1
- package/dist/sandbox/types.d.ts +39 -5
- package/dist/sandbox/vercel.d.ts +7 -1
- package/dist/scoring/coverage.d.ts +30 -0
- package/dist/scoring/display.d.ts +21 -0
- package/dist/scoring/display.js +120 -0
- package/dist/scoring/types.d.ts +103 -20
- package/dist/shared/aggregate.d.ts +1 -0
- package/dist/shared/aggregate.js +3 -3
- package/dist/shared/types.d.ts +28 -0
- package/dist/tty-line.d.ts +0 -4
- package/dist/util.d.ts +23 -0
- package/docs-site/zh/concepts/adapter.mdx +22 -4
- package/docs-site/zh/concepts/experiment.mdx +1 -1
- package/docs-site/zh/concepts/overview.mdx +6 -6
- package/docs-site/zh/guides/agent-feedback-loop.mdx +28 -26
- package/docs-site/zh/guides/authoring.mdx +33 -0
- package/docs-site/zh/guides/ci-integration.mdx +23 -12
- package/docs-site/zh/guides/connect-your-agent.mdx +29 -3
- package/docs-site/zh/guides/custom-reports.mdx +29 -34
- package/docs-site/zh/guides/dataset-fanout.mdx +25 -3
- package/docs-site/zh/guides/debug-sandbox.mdx +57 -0
- package/docs-site/zh/guides/debugging.mdx +210 -0
- package/docs-site/zh/guides/experiments.mdx +10 -3
- package/docs-site/zh/guides/official-adapters.mdx +26 -2
- package/docs-site/zh/guides/publish-report.mdx +30 -16
- package/docs-site/zh/guides/report-components.mdx +42 -30
- package/docs-site/zh/guides/reporters.mdx +2 -2
- package/docs-site/zh/guides/results-data.mdx +17 -9
- package/docs-site/zh/guides/runner.mdx +17 -7
- package/docs-site/zh/guides/sandbox-agent.mdx +56 -7
- package/docs-site/zh/guides/sandbox-providers.mdx +257 -9
- package/docs-site/zh/guides/scoring-guide.mdx +4 -4
- package/docs-site/zh/guides/viewing-results.mdx +79 -36
- package/docs-site/zh/guides/write-experiment.mdx +5 -3
- package/docs-site/zh/guides/write-send.mdx +17 -1
- package/docs-site/zh/index.mdx +1 -1
- package/docs-site/zh/reference/builtin-agents.mdx +27 -0
- package/docs-site/zh/reference/capabilities.mdx +2 -2
- package/docs-site/zh/reference/cli.mdx +33 -7
- package/docs-site/zh/reference/define-agent.mdx +57 -4
- package/docs-site/zh/reference/define-config.mdx +1 -1
- package/docs-site/zh/reference/define-eval.mdx +42 -9
- package/docs-site/zh/reference/expect.mdx +26 -1
- package/package.json +5 -1
- package/src/agents/ai-sdk-otel.test.ts +1 -0
- package/src/agents/ai-sdk.test.ts +3 -0
- package/src/agents/ai-sdk.ts +3 -0
- package/src/agents/bub-install-spec.test.ts +34 -0
- package/src/agents/bub-install-spec.ts +32 -0
- package/src/agents/bub.ts +31 -32
- package/src/agents/claude-code.test.ts +130 -9
- package/src/agents/claude-code.ts +76 -4
- package/src/agents/codex.test.ts +189 -40
- package/src/agents/codex.ts +155 -14
- package/src/agents/coding-cli-versions.test.ts +15 -0
- package/src/agents/coding-cli-versions.ts +3 -0
- package/src/agents/index.ts +11 -0
- package/src/agents/langgraph.test.ts +204 -0
- package/src/agents/langgraph.ts +495 -0
- package/src/agents/marketplace.ts +85 -0
- package/src/agents/native-config.test.ts +179 -0
- package/src/agents/native-config.ts +267 -0
- package/src/agents/openai-compat.test.ts +1 -0
- package/src/agents/openclaw.test.ts +31 -0
- package/src/agents/openclaw.ts +171 -0
- package/src/agents/plugin-config.test.ts +1 -0
- package/src/agents/sdk-streams.test.ts +79 -0
- package/src/agents/sdk-streams.ts +55 -10
- package/src/agents/skills.test.ts +1 -0
- package/src/agents/streaming.test.ts +3 -9
- package/src/agents/types.ts +68 -5
- package/src/agents/ui-message-stream.test.ts +3 -0
- package/src/cli.ts +411 -108
- package/src/context/context.test.ts +51 -12
- package/src/context/context.ts +161 -29
- package/src/context/session.test.ts +1 -0
- package/src/context/session.ts +114 -6
- package/src/context/types.ts +30 -12
- package/src/define.test.ts +13 -8
- package/src/define.ts +25 -4
- package/src/expect/index.ts +53 -23
- package/src/i18n/en.ts +64 -2
- package/src/i18n/zh-CN.ts +65 -3
- package/src/o11y/cost.test.ts +1 -0
- package/src/o11y/execution-tree.test.ts +1 -20
- package/src/o11y/otlp/mappers/claude-code.test.ts +1 -0
- package/src/o11y/otlp/parse.test.ts +1 -0
- package/src/o11y/otlp/turn-otel.test.ts +1 -0
- package/src/o11y/parsers/bub.test.ts +1 -0
- package/src/o11y/parsers/claude-code.test.ts +1 -34
- package/src/o11y/parsers/openclaw.test.ts +154 -0
- package/src/o11y/parsers/openclaw.ts +310 -0
- package/src/o11y/prices.json +746 -311
- package/src/o11y/tool-names.test.ts +1 -0
- package/src/o11y/types.ts +16 -2
- package/src/report/aggregate.ts +34 -5
- package/src/report/built-in-user-parity.test.tsx +110 -153
- package/src/report/built-ins/experiment-comparison.tsx +173 -13
- package/src/report/built-ins/index.ts +6 -1
- package/src/report/components.tsx +9 -3
- package/src/report/compute.ts +70 -40
- package/src/report/dual-render.test.tsx +194 -67
- package/src/report/flag.ts +30 -2
- package/src/report/format.ts +35 -11
- package/src/report/index.ts +22 -4
- package/src/report/locale.ts +25 -5
- package/src/report/metrics.ts +67 -14
- package/src/report/primitives.tsx +6 -0
- package/src/report/react/AttemptList.tsx +6 -31
- package/src/report/react/EvalList.tsx +0 -0
- package/src/report/react/ExperimentComparison.tsx +68 -0
- package/src/report/react/ExperimentList.tsx +15 -9
- package/src/report/react/MetricScatter.tsx +12 -14
- package/src/report/react/chart-math.test.ts +85 -0
- package/src/report/react/chart-math.ts +101 -22
- package/src/report/react/enhance.js +33 -1
- package/src/report/react/fixtures.ts +24 -17
- package/src/report/react/render.test.tsx +9 -64
- package/src/report/react/styles.css +73 -2
- package/src/report/report.test.ts +306 -98
- package/src/report/report.ts +6 -2
- package/src/report/text/faces.ts +47 -43
- package/src/report/text/table.ts +42 -5
- package/src/report/types.ts +41 -21
- package/src/results/annotated-source.test.ts +62 -9
- package/src/results/annotated-source.ts +64 -6
- package/src/results/attempt-evidence.test.ts +9 -7
- package/src/results/attempt-evidence.ts +15 -8
- package/src/results/attempt-source.ts +6 -3
- package/src/results/copy.ts +145 -55
- package/src/results/host-equivalence.test.ts +8 -6
- package/src/results/index.ts +2 -0
- package/src/results/locator.test.ts +1 -22
- package/src/results/open.ts +7 -1
- package/src/results/publish.ts +149 -0
- package/src/results/results.test.ts +85 -51
- package/src/results/truncate.ts +90 -0
- package/src/results/types.ts +7 -0
- package/src/results/writer.ts +31 -13
- package/src/runner/attempt.test.ts +138 -7
- package/src/runner/attempt.ts +603 -104
- package/src/runner/discover.test.ts +47 -0
- package/src/runner/discover.ts +36 -2
- package/src/runner/eval-source.test.ts +1 -27
- package/src/runner/feedback/agent.test.ts +504 -0
- package/src/runner/feedback/agent.ts +409 -0
- package/src/runner/feedback/ci.test.ts +562 -0
- package/src/runner/feedback/ci.ts +401 -0
- package/src/runner/feedback/coordinator.test.ts +317 -0
- package/src/runner/feedback/coordinator.ts +397 -0
- package/src/runner/feedback/failure.ts +40 -0
- package/src/runner/feedback/human.test.ts +616 -0
- package/src/runner/feedback/human.ts +535 -0
- package/src/runner/feedback/index.ts +66 -0
- package/src/runner/feedback/io.ts +78 -0
- package/src/runner/feedback/profile.test.ts +50 -0
- package/src/runner/feedback/profile.ts +58 -0
- package/src/runner/feedback/reducer.test.ts +395 -0
- package/src/runner/feedback/reducer.ts +260 -0
- package/src/runner/feedback/renderer.ts +82 -0
- package/src/runner/feedback/sink.ts +203 -0
- package/src/runner/feedback/testing.ts +106 -0
- package/src/runner/ledger.test.ts +230 -0
- package/src/runner/ledger.ts +329 -0
- package/src/runner/report.test.ts +128 -3
- package/src/runner/report.ts +33 -9
- package/src/runner/reporters/artifacts.ts +8 -2
- package/src/runner/reporters/braintrust.test.ts +8 -7
- package/src/runner/reporters/braintrust.ts +9 -2
- package/src/runner/reporters/index.ts +2 -2
- package/src/runner/reporters/json.test.ts +162 -0
- package/src/runner/reporters/json.ts +35 -8
- package/src/runner/reporters/shared.ts +1 -5
- package/src/runner/run.test.ts +760 -3
- package/src/runner/run.ts +242 -36
- package/src/runner/sandbox-prep.ts +3 -42
- package/src/runner/timing.ts +158 -0
- package/src/runner/types.ts +518 -22
- package/src/sandbox/checkpoint.test.ts +55 -0
- package/src/sandbox/checkpoint.ts +29 -8
- package/src/sandbox/cli-commands.ts +407 -0
- package/src/sandbox/docker.ts +115 -16
- package/src/sandbox/e2b-agent-template.test.ts +56 -0
- package/src/sandbox/e2b-agent-template.ts +94 -0
- package/src/sandbox/e2b.ts +74 -9
- package/src/sandbox/errors.ts +111 -4
- package/src/sandbox/index.ts +2 -0
- package/src/sandbox/io-retry.test.ts +58 -0
- package/src/sandbox/io-retry.ts +45 -0
- package/src/sandbox/keep-registry.test.ts +86 -0
- package/src/sandbox/keep-registry.ts +142 -0
- package/src/sandbox/keep.ts +178 -0
- package/src/sandbox/paths.test.ts +1 -0
- package/src/sandbox/paths.ts +19 -8
- package/src/sandbox/registry.ts +20 -3
- package/src/sandbox/resolve.ts +76 -11
- package/src/sandbox/retry.test.ts +70 -0
- package/src/sandbox/retry.ts +46 -4
- package/src/sandbox/types.ts +44 -6
- package/src/sandbox/vercel.ts +43 -20
- package/src/scoring/collector.ts +60 -17
- package/src/scoring/coverage.ts +95 -0
- package/src/scoring/diff.ts +81 -0
- package/src/scoring/display.test.ts +121 -0
- package/src/scoring/display.ts +133 -0
- package/src/scoring/evidence.test.ts +189 -0
- package/src/scoring/judge.test.ts +142 -0
- package/src/scoring/judge.ts +15 -18
- package/src/scoring/scoped.ts +217 -50
- package/src/scoring/types.ts +117 -20
- package/src/scoring/verdict.ts +16 -4
- package/src/shared/aggregate.ts +3 -2
- package/src/shared/types.ts +31 -0
- package/src/show/compose.ts +2 -2
- package/src/show/index.ts +21 -1
- package/src/show/render.ts +619 -104
- package/src/show/show.test.ts +235 -19
- package/src/tty-line.ts +8 -26
- package/src/util.test.ts +1 -0
- package/src/util.ts +41 -0
- package/src/view/app/components/AttemptModal.tsx +153 -2
- package/src/view/app/components/CodeView.tsx +32 -11
- package/src/view/app/components/CopyControls.tsx +2 -2
- package/src/view/app/i18n.ts +6 -0
- package/src/view/app/lib/attempt-route.test.ts +1 -0
- package/src/view/app/lib/verdict.ts +7 -9
- package/src/view/artifact-serving.test.ts +2 -1
- package/src/view/client-dist/app.css +1 -1
- package/src/view/client-dist/app.js +17 -17
- package/src/view/data.test.ts +1 -0
- package/src/view/data.ts +11 -1
- package/src/view/index.ts +11 -0
- package/src/view/server.ts +2 -0
- package/src/view/styles.css +3 -0
- package/src/view/view-report.test.ts +6 -5
- package/src/runner/reporters/console.ts +0 -70
- package/src/runner/reporters/live.test.ts +0 -56
- package/src/runner/reporters/live.ts +0 -247
- package/src/runner/reporters/quiet.test.ts +0 -66
- package/src/runner/reporters/quiet.ts +0 -49
- package/src/runner/reporters/table.ts +0 -277
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
// cases: docs/engineering/unit-tests/adapters/cases.md
|
|
2
|
+
// 原生配置文件共享实现(native-config.ts)的单测:项目相对路径的接受/拒绝口径、原始字节
|
|
3
|
+
// 与 SHA-256、JSON / TOML 的语法验收与保留键判定、checkpoint key 条目随字节变化。
|
|
4
|
+
// 契约来源:docs/feature/adapters/architecture/coding-agent-extensions.md「类型边界」,
|
|
5
|
+
// 场景矩阵:docs/engineering/unit-tests/adapters/cases.md「Coding Agent 扩展安装」。
|
|
6
|
+
|
|
7
|
+
import { afterEach, beforeEach, describe, expect, it } from "vitest";
|
|
8
|
+
import { createHash } from "node:crypto";
|
|
9
|
+
import { mkdir, mkdtemp, rm, symlink, writeFile } from "node:fs/promises";
|
|
10
|
+
import { tmpdir } from "node:os";
|
|
11
|
+
import { join } from "node:path";
|
|
12
|
+
import {
|
|
13
|
+
assertJsonNativeConfig,
|
|
14
|
+
assertTomlNativeConfig,
|
|
15
|
+
loadNativeConfigFile,
|
|
16
|
+
nativeConfigCheckpointItem,
|
|
17
|
+
scanTomlTopLevel,
|
|
18
|
+
type LoadedNativeConfig,
|
|
19
|
+
} from "./native-config.ts";
|
|
20
|
+
|
|
21
|
+
let root: string;
|
|
22
|
+
let outside: string;
|
|
23
|
+
|
|
24
|
+
beforeEach(async () => {
|
|
25
|
+
root = await mkdtemp(join(tmpdir(), "niceeval-native-config-root-"));
|
|
26
|
+
outside = await mkdtemp(join(tmpdir(), "niceeval-native-config-outside-"));
|
|
27
|
+
});
|
|
28
|
+
|
|
29
|
+
afterEach(async () => {
|
|
30
|
+
await rm(root, { recursive: true, force: true });
|
|
31
|
+
await rm(outside, { recursive: true, force: true });
|
|
32
|
+
});
|
|
33
|
+
|
|
34
|
+
const load = (path: string) =>
|
|
35
|
+
loadNativeConfigFile({ agent: "codex", field: "configFile", path, projectRoot: root });
|
|
36
|
+
|
|
37
|
+
describe("loadNativeConfigFile · 路径口径", () => {
|
|
38
|
+
it("普通相对路径与 ./ 前缀都合法,manifest 路径规范化(./ 剥掉)", async () => {
|
|
39
|
+
await mkdir(join(root, "configs"), { recursive: true });
|
|
40
|
+
await writeFile(join(root, "configs/no-web.toml"), 'web_search = "disabled"\n');
|
|
41
|
+
|
|
42
|
+
const plain = await load("configs/no-web.toml");
|
|
43
|
+
const dotted = await load("./configs/no-web.toml");
|
|
44
|
+
expect(plain.path).toBe("configs/no-web.toml");
|
|
45
|
+
expect(dotted.path).toBe("configs/no-web.toml");
|
|
46
|
+
expect(plain.sha256).toBe(dotted.sha256);
|
|
47
|
+
});
|
|
48
|
+
|
|
49
|
+
it("原始字节逐字节读回,SHA-256 是字节哈希;改一个字节 → 哈希与 checkpoint 条目都变", async () => {
|
|
50
|
+
const body = '#:schema https://example.com/schema.json\nweb_search = "disabled"\n';
|
|
51
|
+
await writeFile(join(root, "a.toml"), body);
|
|
52
|
+
const a = await load("a.toml");
|
|
53
|
+
expect(a.bytes.toString("utf8")).toBe(body);
|
|
54
|
+
expect(a.sha256).toBe(createHash("sha256").update(body).digest("hex"));
|
|
55
|
+
|
|
56
|
+
await writeFile(join(root, "b.toml"), body.replace("disabled", "disablee"));
|
|
57
|
+
const b = await load("b.toml");
|
|
58
|
+
expect(b.sha256).not.toBe(a.sha256);
|
|
59
|
+
expect(nativeConfigCheckpointItem("codex", b)).not.toBe(nativeConfigCheckpointItem("codex", a));
|
|
60
|
+
expect(nativeConfigCheckpointItem("codex", a)).toContain(a.sha256);
|
|
61
|
+
});
|
|
62
|
+
|
|
63
|
+
it("包含 `..` 的路径、绝对路径与 `~` 路径都拒绝", async () => {
|
|
64
|
+
await expect(load("../a.toml")).rejects.toThrow(/相对路径|relative/);
|
|
65
|
+
await expect(load("configs/../../a.toml")).rejects.toThrow(/相对路径|relative/);
|
|
66
|
+
await expect(load(join(root, "a.toml"))).rejects.toThrow(/相对路径|relative/);
|
|
67
|
+
await expect(load("~/a.toml")).rejects.toThrow(/相对路径|relative/);
|
|
68
|
+
await expect(load("")).rejects.toThrow(/相对路径|relative/);
|
|
69
|
+
});
|
|
70
|
+
|
|
71
|
+
it("路径不存在时报错点名原路径与解析后的绝对路径", async () => {
|
|
72
|
+
await expect(load("configs/nope.toml")).rejects.toThrow(/configs\/nope\.toml/);
|
|
73
|
+
});
|
|
74
|
+
|
|
75
|
+
it("符号链接解析后逃出项目根 → 拒绝", async () => {
|
|
76
|
+
await writeFile(join(outside, "escape.toml"), 'web_search = "disabled"\n');
|
|
77
|
+
await symlink(join(outside, "escape.toml"), join(root, "escape.toml"));
|
|
78
|
+
await expect(load("escape.toml")).rejects.toThrow(/项目根之外|outside the project root/);
|
|
79
|
+
});
|
|
80
|
+
|
|
81
|
+
it("指向目录 → 拒绝(不是普通文件)", async () => {
|
|
82
|
+
await mkdir(join(root, "configs"), { recursive: true });
|
|
83
|
+
await expect(load("configs")).rejects.toThrow(/普通文件|regular file/);
|
|
84
|
+
});
|
|
85
|
+
});
|
|
86
|
+
|
|
87
|
+
const cfgOf = (text: string): LoadedNativeConfig => ({
|
|
88
|
+
path: "configs/x",
|
|
89
|
+
bytes: Buffer.from(text, "utf8"),
|
|
90
|
+
sha256: createHash("sha256").update(text).digest("hex"),
|
|
91
|
+
});
|
|
92
|
+
|
|
93
|
+
describe("assertJsonNativeConfig(claude-code settings.json)", () => {
|
|
94
|
+
const opts = { agent: "claude-code", field: "settingsFile", reservedKeys: ["model", "env"] };
|
|
95
|
+
|
|
96
|
+
it("合法 JSON 对象 + 无保留键 → 通过($schema 等标记不受影响)", () => {
|
|
97
|
+
const cfg = cfgOf('{ "$schema": "https://x", "permissions": { "deny": ["WebSearch"] } }');
|
|
98
|
+
expect(() => assertJsonNativeConfig(cfg, opts)).not.toThrow();
|
|
99
|
+
});
|
|
100
|
+
|
|
101
|
+
it("语法错误 → 报错点名文件与 JSON", () => {
|
|
102
|
+
expect(() => assertJsonNativeConfig(cfgOf("{ nope"), opts)).toThrow(/JSON/);
|
|
103
|
+
});
|
|
104
|
+
|
|
105
|
+
it("顶层不是对象(数组)→ 语法验收失败", () => {
|
|
106
|
+
expect(() => assertJsonNativeConfig(cfgOf("[1, 2]"), opts)).toThrow(/JSON/);
|
|
107
|
+
});
|
|
108
|
+
|
|
109
|
+
it("顶层保留键(model / env)→ 报错点名冲突键", () => {
|
|
110
|
+
const cfg = cfgOf('{ "model": "opus", "env": { "A": "1" }, "permissions": {} }');
|
|
111
|
+
expect(() => assertJsonNativeConfig(cfg, opts)).toThrow(/model, env/);
|
|
112
|
+
});
|
|
113
|
+
|
|
114
|
+
it("保留键只看顶层:嵌套对象里的同名键不算", () => {
|
|
115
|
+
const cfg = cfgOf('{ "permissions": { "model": "x", "env": {} } }');
|
|
116
|
+
expect(() => assertJsonNativeConfig(cfg, opts)).not.toThrow();
|
|
117
|
+
});
|
|
118
|
+
});
|
|
119
|
+
|
|
120
|
+
describe("assertTomlNativeConfig(codex config.toml)", () => {
|
|
121
|
+
const opts = {
|
|
122
|
+
agent: "codex",
|
|
123
|
+
field: "configFile",
|
|
124
|
+
reservedKeys: ["model", "model_provider", "model_providers", "model_reasoning_effort", "mcp_servers", "otel"],
|
|
125
|
+
};
|
|
126
|
+
|
|
127
|
+
it("合法 TOML + 无保留键 → 通过(schema 注释、普通表都保留判定之外)", () => {
|
|
128
|
+
const cfg = cfgOf(
|
|
129
|
+
'#:schema https://developers.openai.com/codex/config-schema.json\nweb_search = "disabled"\n\n[sandbox_workspace_write]\nnetwork_access = false\n',
|
|
130
|
+
);
|
|
131
|
+
expect(() => assertTomlNativeConfig(cfg, opts)).not.toThrow();
|
|
132
|
+
});
|
|
133
|
+
|
|
134
|
+
it("顶层保留键赋值(model = …)→ 报错点名冲突键", () => {
|
|
135
|
+
expect(() => assertTomlNativeConfig(cfgOf('model = "gpt-5"\n'), opts)).toThrow(/model/);
|
|
136
|
+
});
|
|
137
|
+
|
|
138
|
+
it("保留表头([mcp_servers.x] / [otel])→ 报错点名首段", () => {
|
|
139
|
+
expect(() => assertTomlNativeConfig(cfgOf('[mcp_servers.browser]\ncommand = "npx"\n'), opts)).toThrow(
|
|
140
|
+
/mcp_servers/,
|
|
141
|
+
);
|
|
142
|
+
expect(() => assertTomlNativeConfig(cfgOf("[otel]\nenvironment = 'x'\n"), opts)).toThrow(/otel/);
|
|
143
|
+
});
|
|
144
|
+
|
|
145
|
+
it("顶层点分键(otel.environment = …)按首段判保留", () => {
|
|
146
|
+
expect(() => assertTomlNativeConfig(cfgOf('otel.environment = "x"\n'), opts)).toThrow(/otel/);
|
|
147
|
+
});
|
|
148
|
+
|
|
149
|
+
it("非保留表**里面**的同名键不算顶层保留键([profiles.x] 下的 model 属于 profiles)", () => {
|
|
150
|
+
const cfg = cfgOf('[profiles.fast]\nmodel = "gpt-5"\n');
|
|
151
|
+
expect(() => assertTomlNativeConfig(cfg, opts)).not.toThrow();
|
|
152
|
+
});
|
|
153
|
+
|
|
154
|
+
it("字符串与多行数组里的「model =」不误判", () => {
|
|
155
|
+
const cfg = cfgOf('note = "model = fake"\nallow = [\n "model",\n]\n');
|
|
156
|
+
expect(() => assertTomlNativeConfig(cfg, opts)).not.toThrow();
|
|
157
|
+
});
|
|
158
|
+
|
|
159
|
+
it("语法错误(既不是键赋值也不是表头的行)→ 报错", () => {
|
|
160
|
+
expect(() => assertTomlNativeConfig(cfgOf("just some words\n"), opts)).toThrow(/TOML/);
|
|
161
|
+
});
|
|
162
|
+
|
|
163
|
+
it("未闭合的多行字符串 → 报错", () => {
|
|
164
|
+
expect(() => assertTomlNativeConfig(cfgOf('a = """\nnever closed\n'), opts)).toThrow(/TOML/);
|
|
165
|
+
});
|
|
166
|
+
});
|
|
167
|
+
|
|
168
|
+
describe("scanTomlTopLevel", () => {
|
|
169
|
+
it("多行字符串里的表头形状不进入键集合", () => {
|
|
170
|
+
const scan = scanTomlTopLevel('a = """\n[mcp_servers.fake]\n"""\nb = 1\n');
|
|
171
|
+
expect(scan.error).toBeUndefined();
|
|
172
|
+
expect([...scan.rootKeys].sort()).toEqual(["a", "b"]);
|
|
173
|
+
});
|
|
174
|
+
|
|
175
|
+
it("带引号的表头首段剥引号", () => {
|
|
176
|
+
const scan = scanTomlTopLevel('["otel".sub]\nx = 1\n');
|
|
177
|
+
expect(scan.rootKeys.has("otel")).toBe(true);
|
|
178
|
+
});
|
|
179
|
+
});
|
|
@@ -0,0 +1,267 @@
|
|
|
1
|
+
// 官方原生配置文件(Claude Code settings.json / Codex config.toml)的共享实现:
|
|
2
|
+
// 「从本地项目根安全取原始字节」+「按各 Agent 的保留键清单验收」+「原样落进沙箱」。
|
|
3
|
+
// 定稿见 docs/feature/adapters/architecture/coding-agent-extensions.md「类型边界」「安装顺序」。
|
|
4
|
+
//
|
|
5
|
+
// 关键契约:文件是完整用户配置层,不是 patch —— 只解析以验证官方语法和检查保留键,
|
|
6
|
+
// 验证后写入的仍是**原始字节**(JSON Schema 标记、TOML 注释与官方编辑器支持全保留),
|
|
7
|
+
// 不继承宿主机配置,不拼接、deep merge 或重新序列化。字段名不跨 Agent 统一
|
|
8
|
+
// (`settingsFile` / `configFile`),但路径解析、字节承诺和保留键语义是同一套,收在这里。
|
|
9
|
+
|
|
10
|
+
import { createHash } from "node:crypto";
|
|
11
|
+
import { readFile, realpath, stat } from "node:fs/promises";
|
|
12
|
+
import { isAbsolute, resolve, sep } from "node:path";
|
|
13
|
+
import { posix } from "node:path";
|
|
14
|
+
import { t } from "../i18n/index.ts";
|
|
15
|
+
import type { Sandbox } from "../types.ts";
|
|
16
|
+
|
|
17
|
+
/** 本地读到的一份原生配置文件:规范化的项目相对路径、原始字节与字节 SHA-256。 */
|
|
18
|
+
export interface LoadedNativeConfig {
|
|
19
|
+
/** 规范化后的项目相对路径(manifest 记录用;`./a/b` → `a/b`)。 */
|
|
20
|
+
path: string;
|
|
21
|
+
/** 原始字节 —— 上传沙箱时原样写入,不重新序列化。 */
|
|
22
|
+
bytes: Buffer;
|
|
23
|
+
/** 原始字节的 SHA-256(hex):进安装 checkpoint key 与 manifest;正文不落任何 artifact。 */
|
|
24
|
+
sha256: string;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
export interface LoadNativeConfigOptions {
|
|
28
|
+
/** 报错归属的 agent 名(如 "claude-code")。 */
|
|
29
|
+
agent: string;
|
|
30
|
+
/** 报错归属的 config 字段名(如 "settingsFile" / "configFile")。 */
|
|
31
|
+
field: string;
|
|
32
|
+
/** 用户配置的路径(必须是项目根内的相对路径)。 */
|
|
33
|
+
path: string;
|
|
34
|
+
/** 解析根;省略用跑 niceeval 的项目根(process.cwd()),与 SkillSpec 本地路径同一口径。 */
|
|
35
|
+
projectRoot?: string;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* 从本地项目根解析并读取一份原生配置文件的原始字节。
|
|
40
|
+
* 路径契约(docs 定稿):只接受项目根内的相对路径 —— 普通相对路径与 `./` 前缀合法;
|
|
41
|
+
* 包含 `..` 的路径、绝对路径、`~` 路径,以及解析符号链接后逃出项目根的路径,全部在
|
|
42
|
+
* setup 阶段抛错(attempt errored,不伪装成断言失败)。
|
|
43
|
+
*/
|
|
44
|
+
export async function loadNativeConfigFile(opts: LoadNativeConfigOptions): Promise<LoadedNativeConfig> {
|
|
45
|
+
const { agent, field, path } = opts;
|
|
46
|
+
const rejectPath = () => new Error(t("nativeConfig.pathNotProjectRelative", { agent, field, path }));
|
|
47
|
+
if (!path || path.startsWith("~") || isAbsolute(path) || /^[A-Za-z]:[\\/]/.test(path)) throw rejectPath();
|
|
48
|
+
if (path.split(/[\\/]/).includes("..")) throw rejectPath();
|
|
49
|
+
|
|
50
|
+
const root = opts.projectRoot ?? process.cwd();
|
|
51
|
+
const abs = resolve(root, path);
|
|
52
|
+
const info = await stat(abs).catch(() => undefined);
|
|
53
|
+
if (!info) throw new Error(t("nativeConfig.missing", { agent, field, path, resolved: abs }));
|
|
54
|
+
if (!info.isFile()) throw new Error(t("nativeConfig.notFile", { agent, field, path }));
|
|
55
|
+
|
|
56
|
+
// 符号链接逃逸检查按真实路径做:文件与项目根都 realpath 之后再验包含关系。
|
|
57
|
+
const [realFile, realRoot] = await Promise.all([realpath(abs), realpath(root)]);
|
|
58
|
+
if (!realFile.startsWith(realRoot + sep)) {
|
|
59
|
+
throw new Error(t("nativeConfig.escapesRoot", { agent, field, path, resolved: realFile }));
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
const bytes = await readFile(abs);
|
|
63
|
+
return {
|
|
64
|
+
path: posix.normalize(path.split(/[\\/]/).join("/")),
|
|
65
|
+
bytes,
|
|
66
|
+
sha256: createHash("sha256").update(bytes).digest("hex"),
|
|
67
|
+
};
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
// ───────────────────────── 验收:语法 + 保留键 ─────────────────────────
|
|
71
|
+
|
|
72
|
+
export interface NativeConfigValidationOptions {
|
|
73
|
+
agent: string;
|
|
74
|
+
field: string;
|
|
75
|
+
/** 该 Agent 的保留键清单(experiment / Adapter 拥有,出现在用户文件里即报错)。 */
|
|
76
|
+
reservedKeys: readonly string[];
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/** JSON 形态(Claude Code settings.json):整文件必须是合法 JSON 对象,顶层不得含保留键。 */
|
|
80
|
+
export function assertJsonNativeConfig(cfg: LoadedNativeConfig, opts: NativeConfigValidationOptions): void {
|
|
81
|
+
let parsed: unknown;
|
|
82
|
+
try {
|
|
83
|
+
parsed = JSON.parse(cfg.bytes.toString("utf8"));
|
|
84
|
+
} catch (e) {
|
|
85
|
+
throw invalidSyntax(cfg, opts, "JSON", e instanceof Error ? e.message : String(e));
|
|
86
|
+
}
|
|
87
|
+
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
|
|
88
|
+
throw invalidSyntax(cfg, opts, "JSON", "top level must be a JSON object");
|
|
89
|
+
}
|
|
90
|
+
const hit = opts.reservedKeys.filter((k) => Object.prototype.hasOwnProperty.call(parsed, k));
|
|
91
|
+
if (hit.length) throw reservedKeys(cfg, opts, hit);
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
/**
|
|
95
|
+
* TOML 形态(Codex config.toml):行级语法验收 + 顶层保留键检查。
|
|
96
|
+
* 保留键的判定口径:根上下文的键首段(`otel.environment = …` 算 `otel`)与所有表头首段
|
|
97
|
+
* (`[mcp_servers.browser]` 算 `mcp_servers`);非保留表**里面**的同名键不算
|
|
98
|
+
* (`[profiles.x]` 下的 `model` 属于 profiles,不占顶层 `model`)。
|
|
99
|
+
*/
|
|
100
|
+
export function assertTomlNativeConfig(cfg: LoadedNativeConfig, opts: NativeConfigValidationOptions): void {
|
|
101
|
+
const scan = scanTomlTopLevel(cfg.bytes.toString("utf8"));
|
|
102
|
+
if (scan.error) throw invalidSyntax(cfg, opts, "TOML", scan.error);
|
|
103
|
+
const hit = opts.reservedKeys.filter((k) => scan.rootKeys.has(k));
|
|
104
|
+
if (hit.length) throw reservedKeys(cfg, opts, hit);
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
function invalidSyntax(
|
|
108
|
+
cfg: LoadedNativeConfig,
|
|
109
|
+
opts: NativeConfigValidationOptions,
|
|
110
|
+
format: string,
|
|
111
|
+
detail: string,
|
|
112
|
+
): Error {
|
|
113
|
+
return new Error(
|
|
114
|
+
t("nativeConfig.invalidSyntax", { agent: opts.agent, field: opts.field, path: cfg.path, format, detail }),
|
|
115
|
+
);
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
function reservedKeys(cfg: LoadedNativeConfig, opts: NativeConfigValidationOptions, keys: string[]): Error {
|
|
119
|
+
return new Error(
|
|
120
|
+
t("nativeConfig.reservedKeys", { agent: opts.agent, field: opts.field, path: cfg.path, keys: keys.join(", ") }),
|
|
121
|
+
);
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
// ───────────────────────── TOML 顶层扫描 ─────────────────────────
|
|
125
|
+
|
|
126
|
+
type StrMode = "none" | "basic" | "literal";
|
|
127
|
+
|
|
128
|
+
/**
|
|
129
|
+
* 轻量 TOML 顶层扫描:抠出「根上下文的键首段 + 表头首段」供保留键检查,顺带做行级语法
|
|
130
|
+
* 验收(每个非空非注释行必须是表头或键赋值)。不是完整 TOML 解析器 —— 验证后写回沙箱的
|
|
131
|
+
* 是原始字节,解析结果只用于验收,不值得为它引入解析依赖;多行字符串(""" / ''')与
|
|
132
|
+
* 多行数组的跨行状态有覆盖,足以对官方 config.toml 做保留键判定。
|
|
133
|
+
*/
|
|
134
|
+
export function scanTomlTopLevel(text: string): { rootKeys: Set<string>; error?: string } {
|
|
135
|
+
const rootKeys = new Set<string>();
|
|
136
|
+
let currentTable: string | null = null;
|
|
137
|
+
let mode: StrMode = "none";
|
|
138
|
+
let depth = 0;
|
|
139
|
+
const lines = text.replace(/^\uFEFF/, "").split(/\r?\n/);
|
|
140
|
+
for (let n = 0; n < lines.length; n++) {
|
|
141
|
+
const line = lines[n]!;
|
|
142
|
+
if (mode !== "none") {
|
|
143
|
+
const close = line.indexOf(mode === "basic" ? '"""' : "'''");
|
|
144
|
+
if (close === -1) continue;
|
|
145
|
+
({ mode, depth } = scanValue(line.slice(close + 3), depth));
|
|
146
|
+
continue;
|
|
147
|
+
}
|
|
148
|
+
if (depth > 0) {
|
|
149
|
+
({ mode, depth } = scanValue(line, depth));
|
|
150
|
+
continue;
|
|
151
|
+
}
|
|
152
|
+
const trimmed = line.trim();
|
|
153
|
+
if (!trimmed || trimmed.startsWith("#")) continue;
|
|
154
|
+
if (trimmed.startsWith("[")) {
|
|
155
|
+
const header = /^\[\[?\s*([^\]]+?)\s*\]\]?\s*(?:#.*)?$/.exec(trimmed);
|
|
156
|
+
if (!header) return { rootKeys, error: `line ${n + 1}: invalid table header: ${trimmed}` };
|
|
157
|
+
currentTable = firstKeySegment(header[1]!);
|
|
158
|
+
rootKeys.add(currentTable);
|
|
159
|
+
continue;
|
|
160
|
+
}
|
|
161
|
+
const assign = /^((?:"[^"]*"|'[^']*'|[\w-]+)(?:\s*\.\s*(?:"[^"]*"|'[^']*'|[\w-]+))*)\s*=(.*)$/.exec(trimmed);
|
|
162
|
+
if (!assign) {
|
|
163
|
+
return { rootKeys, error: `line ${n + 1}: expected a key assignment or table header, got: ${trimmed}` };
|
|
164
|
+
}
|
|
165
|
+
if (currentTable === null) rootKeys.add(firstKeySegment(assign[1]!));
|
|
166
|
+
({ mode, depth } = scanValue(assign[2]!, depth));
|
|
167
|
+
}
|
|
168
|
+
if (mode !== "none") return { rootKeys, error: "unterminated multi-line string" };
|
|
169
|
+
if (depth > 0) return { rootKeys, error: "unbalanced array brackets" };
|
|
170
|
+
return { rootKeys };
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
/** 扫过一段 value 文本,返回跨行状态(未闭合的多行字符串 / 未配平的数组括号)。 */
|
|
174
|
+
function scanValue(s: string, depth: number): { mode: StrMode; depth: number } {
|
|
175
|
+
let i = 0;
|
|
176
|
+
while (i < s.length) {
|
|
177
|
+
if (s.startsWith('"""', i)) {
|
|
178
|
+
const close = s.indexOf('"""', i + 3);
|
|
179
|
+
if (close === -1) return { mode: "basic", depth };
|
|
180
|
+
i = close + 3;
|
|
181
|
+
continue;
|
|
182
|
+
}
|
|
183
|
+
if (s.startsWith("'''", i)) {
|
|
184
|
+
const close = s.indexOf("'''", i + 3);
|
|
185
|
+
if (close === -1) return { mode: "literal", depth };
|
|
186
|
+
i = close + 3;
|
|
187
|
+
continue;
|
|
188
|
+
}
|
|
189
|
+
const ch = s[i]!;
|
|
190
|
+
if (ch === '"') {
|
|
191
|
+
i += 1;
|
|
192
|
+
while (i < s.length && s[i] !== '"') i += s[i] === "\\" ? 2 : 1;
|
|
193
|
+
i += 1;
|
|
194
|
+
continue;
|
|
195
|
+
}
|
|
196
|
+
if (ch === "'") {
|
|
197
|
+
i += 1;
|
|
198
|
+
while (i < s.length && s[i] !== "'") i += 1;
|
|
199
|
+
i += 1;
|
|
200
|
+
continue;
|
|
201
|
+
}
|
|
202
|
+
if (ch === "#") break;
|
|
203
|
+
if (ch === "[") depth += 1;
|
|
204
|
+
else if (ch === "]") depth -= 1;
|
|
205
|
+
i += 1;
|
|
206
|
+
}
|
|
207
|
+
return { mode: "none", depth };
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
/** 点分键的首段:`mcp_servers.browser` → `mcp_servers`;带引号的首段剥引号。 */
|
|
211
|
+
function firstKeySegment(dotted: string): string {
|
|
212
|
+
const s = dotted.trim();
|
|
213
|
+
const quoted = /^"([^"]*)"|^'([^']*)'/.exec(s);
|
|
214
|
+
if (quoted) return quoted[1] ?? quoted[2] ?? "";
|
|
215
|
+
return /^[\w-]+/.exec(s)?.[0] ?? s;
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
// ───────────────────────── 落进沙箱 ─────────────────────────
|
|
219
|
+
|
|
220
|
+
function tmpUploadPath(): string {
|
|
221
|
+
return `/tmp/niceeval-native-config-${Date.now().toString(36)}-${Math.floor(Math.random() * 1e9).toString(36)}`;
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
/**
|
|
225
|
+
* 原始字节整文件替换沙箱内 destPath(claude-code:用户级 `~/.claude/settings.json`)。
|
|
226
|
+
* 走 uploadFile + mv 而不是 heredoc:heredoc 会给内容补换行,破坏「原始字节」承诺。
|
|
227
|
+
* destPath 不加引号,以便 shell 展开 `~`(受信内部路径,同 shared.writeFile 的约定)。
|
|
228
|
+
*/
|
|
229
|
+
export async function uploadNativeConfigFile(sb: Sandbox, cfg: LoadedNativeConfig, destPath: string): Promise<void> {
|
|
230
|
+
const tmp = tmpUploadPath();
|
|
231
|
+
await sb.uploadFile(tmp, cfg.bytes);
|
|
232
|
+
const res = await sb.runShell(`mkdir -p $(dirname ${destPath}) && mv ${tmp} ${destPath}`);
|
|
233
|
+
if (res.exitCode !== 0) {
|
|
234
|
+
throw new Error(t("nativeConfig.uploadFailed", { path: cfg.path, dest: destPath, tail: outputTail(res) }));
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
/**
|
|
239
|
+
* 原始字节追加到沙箱内 destPath 末尾,前后各补一个换行(补的换行在用户字节之外,用户内容
|
|
240
|
+
* 本身逐字节保留)。codex 专用:codex 只读一份用户级 config.toml,Adapter 生成层与用户层
|
|
241
|
+
* 只能同文件分段共存(见 codex.ts setup 的布局说明)。
|
|
242
|
+
*/
|
|
243
|
+
export async function appendNativeConfigFile(sb: Sandbox, cfg: LoadedNativeConfig, destPath: string): Promise<void> {
|
|
244
|
+
const tmp = tmpUploadPath();
|
|
245
|
+
await sb.uploadFile(tmp, cfg.bytes);
|
|
246
|
+
const res = await sb.runShell(
|
|
247
|
+
`printf '\\n' >> ${destPath} && cat ${tmp} >> ${destPath} && printf '\\n' >> ${destPath} && rm -f ${tmp}`,
|
|
248
|
+
);
|
|
249
|
+
if (res.exitCode !== 0) {
|
|
250
|
+
throw new Error(t("nativeConfig.uploadFailed", { path: cfg.path, dest: destPath, tail: outputTail(res) }));
|
|
251
|
+
}
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
/**
|
|
255
|
+
* 原生配置文件在安装 checkpoint key 里的条目。契约(docs/feature/adapters/architecture/
|
|
256
|
+
* coding-agent-extensions.md「可复现性」):安装 checkpoint key 必须包含配置文件原始字节的
|
|
257
|
+
* SHA-256,内容不同的两个配置文件不复用同一份安装缓存。claude-code / codex 当前没有跨沙箱
|
|
258
|
+
* 安装缓存(每个沙箱的 setup 全量执行,不存在错配复用的通道);本函数是该条目的单一事实源
|
|
259
|
+
* —— 为这两个 adapter 引入任何安装缓存(如 bub 式 checkpoint)时,key 必须并入本条目。
|
|
260
|
+
*/
|
|
261
|
+
export function nativeConfigCheckpointItem(agent: string, cfg: Pick<LoadedNativeConfig, "sha256">): string {
|
|
262
|
+
return `--native-config(${agent}:sha256:${cfg.sha256})`;
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
function outputTail(res: { stdout: string; stderr: string }, n = 12): string {
|
|
266
|
+
return (res.stdout + res.stderr).trim().split("\n").slice(-n).join("\n");
|
|
267
|
+
}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
// cases: docs/engineering/unit-tests/adapters/cases.md
|
|
2
|
+
import { describe, expect, it } from "vitest";
|
|
3
|
+
|
|
4
|
+
import { openClawAgent } from "./openclaw.ts";
|
|
5
|
+
|
|
6
|
+
describe("openClawAgent", () => {
|
|
7
|
+
it("复用 defineSandboxAgent:kind sandbox + canonical mapper + tracing 面齐全", () => {
|
|
8
|
+
const agent = openClawAgent();
|
|
9
|
+
expect(agent.name).toBe("openclaw");
|
|
10
|
+
expect(agent.kind).toBe("sandbox");
|
|
11
|
+
expect(typeof agent.send).toBe("function");
|
|
12
|
+
expect(typeof agent.setup).toBe("function");
|
|
13
|
+
expect(typeof agent.spanMapper).toBe("function");
|
|
14
|
+
expect(agent.tracing?.protocol).toBe("http/protobuf");
|
|
15
|
+
// tracing 是 env-based 标准 OTEL_* 注入
|
|
16
|
+
expect(agent.tracing?.env?.("http://127.0.0.1:4318/v1/traces")).toMatchObject({
|
|
17
|
+
OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: "http://127.0.0.1:4318/v1/traces",
|
|
18
|
+
});
|
|
19
|
+
});
|
|
20
|
+
|
|
21
|
+
it("常态证据覆盖只声明 partial(fixture 未验证不声明 complete),负断言由覆盖声明限制", () => {
|
|
22
|
+
const agent = openClawAgent();
|
|
23
|
+
expect(agent.coverage?.events?.status).toBe("partial");
|
|
24
|
+
expect(agent.coverage?.actions?.status).toBe("partial");
|
|
25
|
+
expect(agent.coverage?.messages?.status).toBe("partial");
|
|
26
|
+
expect(agent.coverage?.usage?.status).toBe("partial");
|
|
27
|
+
// 不声明 complete
|
|
28
|
+
const statuses = Object.values(agent.coverage ?? {}).map((c) => c?.status);
|
|
29
|
+
expect(statuses).not.toContain("complete");
|
|
30
|
+
});
|
|
31
|
+
});
|
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
import { defineSandboxAgent } from "../define.ts";
|
|
2
|
+
import { requireEnv } from "../util.ts";
|
|
3
|
+
import { shared } from "./shared.ts";
|
|
4
|
+
import {
|
|
5
|
+
appendProjectInstruction,
|
|
6
|
+
installSkills,
|
|
7
|
+
installedSkillNames,
|
|
8
|
+
skillDiscoveryInstruction,
|
|
9
|
+
} from "./skills.ts";
|
|
10
|
+
import { writeAgentSetupManifest } from "./manifest.ts";
|
|
11
|
+
import { mapGenericSpans } from "../o11y/otlp/canonical.ts";
|
|
12
|
+
import { parseOpenClawTranscript, parseOpenClawRunJson } from "../o11y/parsers/openclaw.ts";
|
|
13
|
+
import { randomUUID } from "node:crypto";
|
|
14
|
+
import type { Agent, AgentSetupManifest, EvidenceCoverage, SkillSpec, StreamEvent } from "../types.ts";
|
|
15
|
+
|
|
16
|
+
// ───────────────────────────────────────────────────────────────────────────
|
|
17
|
+
// OpenClaw 的 agent adapter(沙箱型)。
|
|
18
|
+
//
|
|
19
|
+
// 连接方式:在沙箱里 spawn `openclaw agent --local --json`(嵌入式 agent 循环,不依赖
|
|
20
|
+
// 长驻 gateway),读回结果封包 + session transcript → 标准事件流。方言解析全部住
|
|
21
|
+
// src/o11y/parsers/openclaw.ts,不进 core(契约见 docs/feature/adapters/sdk/openclaw/README.md)。
|
|
22
|
+
//
|
|
23
|
+
// 行为轨优先级(collection.md):session transcript(完整工具轨迹)优先;transcript 拿不到
|
|
24
|
+
// 时只保留 `--json` 封包的最终回复,不从最终文本猜工具行为——此时负断言不可信,send 会
|
|
25
|
+
// 经 ctx.log 明确记录这个限制。
|
|
26
|
+
// ───────────────────────────────────────────────────────────────────────────
|
|
27
|
+
|
|
28
|
+
// 契约要求实现前用真实 CLI fixture 钉版本;fixture 固定前先跟 npm latest 走,
|
|
29
|
+
// 钉死后归口 coding-cli-versions.ts(与 codex / claude-code 同源管理)。
|
|
30
|
+
const DEFAULT_OPENCLAW_CLI_VERSION = "latest";
|
|
31
|
+
|
|
32
|
+
/** OpenClaw 的 skill 目录(`skills` 生态的「通用」目录);发现靠 AGENTS.md 指引,不依赖原生扫描。 */
|
|
33
|
+
const SKILL_DIR = ".agents/skills";
|
|
34
|
+
|
|
35
|
+
export interface OpenClawConfig {
|
|
36
|
+
/** 模型 API key(OpenClaw 默认走 Anthropic)。省略时读 ANTHROPIC_API_KEY env。 */
|
|
37
|
+
apiKey?: string;
|
|
38
|
+
/** 固定安装的 openclaw npm 版本(如 "1.2.3");省略时用内置默认。 */
|
|
39
|
+
version?: string;
|
|
40
|
+
/**
|
|
41
|
+
* 装进沙箱的 Skill(本地目录/文件,或 repo + 可钉 ref + 可选启用集)。
|
|
42
|
+
* 落在 `.agents/skills/<name>/`,并写一段发现指引进 AGENTS.md。
|
|
43
|
+
*/
|
|
44
|
+
skills?: SkillSpec[];
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* OpenClaw 的内置 sandbox Agent 工厂。复用 `defineSandboxAgent`、`shared` 安装工具、
|
|
49
|
+
* `ctx.session` 存取器与 canonical OTel 通用 mapper(`mapGenericSpans`);OpenClaw 方言
|
|
50
|
+
* (transcript / `agent --json` 字段)只住 `src/o11y/parsers/openclaw.ts`,不进 core。
|
|
51
|
+
*
|
|
52
|
+
* 会话契约:首轮显式发一个全新 session id 并 `ctx.session.capture()`(不依赖 OpenClaw 的
|
|
53
|
+
* 默认主会话——否则相邻 attempt 会静默共享历史),后续轮用 `ctx.session.id` resume;
|
|
54
|
+
* `t.newSession()` 后的新会话线自然拿到新 id,session 之间互相隔离。
|
|
55
|
+
*/
|
|
56
|
+
export function openClawAgent(config?: OpenClawConfig): Agent {
|
|
57
|
+
const getApiKey = () => config?.apiKey ?? requireEnv("ANTHROPIC_API_KEY");
|
|
58
|
+
const version = config?.version ?? DEFAULT_OPENCLAW_CLI_VERSION;
|
|
59
|
+
|
|
60
|
+
// 契约(sdk/openclaw/README.md):只有 fixture 证明完整的行为才进入公开能力。transcript
|
|
61
|
+
// 完整性尚未经真实 CLI fixture 验证,常态覆盖只声明 partial(不是 complete)——负断言在
|
|
62
|
+
// 非 complete 通道上一律 unavailable,「明确限制负断言」由覆盖声明落实,不靠口头承诺。
|
|
63
|
+
const FIXTURE_UNVERIFIED = "OpenClaw transcript completeness is not yet fixture-verified";
|
|
64
|
+
const defaultCoverage: EvidenceCoverage = {
|
|
65
|
+
events: { status: "partial", reason: FIXTURE_UNVERIFIED },
|
|
66
|
+
actions: { status: "partial", reason: FIXTURE_UNVERIFIED },
|
|
67
|
+
messages: { status: "partial", reason: FIXTURE_UNVERIFIED },
|
|
68
|
+
usage: { status: "partial", reason: FIXTURE_UNVERIFIED },
|
|
69
|
+
};
|
|
70
|
+
|
|
71
|
+
return defineSandboxAgent({
|
|
72
|
+
name: "openclaw",
|
|
73
|
+
coverage: defaultCoverage,
|
|
74
|
+
// OpenClaw 没有专属 span 方言 mapper:原生 span 走 canonical 通用 heuristic。
|
|
75
|
+
// OTel 内容采集关闭时只影响 trace 证据面;行为轨(下面的 transcript 解析)不受影响。
|
|
76
|
+
spanMapper: mapGenericSpans,
|
|
77
|
+
|
|
78
|
+
tracing: {
|
|
79
|
+
protocol: "http/protobuf",
|
|
80
|
+
env: (endpoint) => ({
|
|
81
|
+
OTEL_TRACES_EXPORTER: "otlp",
|
|
82
|
+
OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: endpoint,
|
|
83
|
+
OTEL_EXPORTER_OTLP_TRACES_PROTOCOL: "http/protobuf",
|
|
84
|
+
}),
|
|
85
|
+
},
|
|
86
|
+
|
|
87
|
+
async setup(sb) {
|
|
88
|
+
// 预制模板已把 openclaw 烘焙进镜像(PATH 上)就跳过安装;否则 npm 全局装。
|
|
89
|
+
await sb.runShell(
|
|
90
|
+
`command -v openclaw >/dev/null 2>&1 || npm install -g openclaw@${version}`,
|
|
91
|
+
);
|
|
92
|
+
|
|
93
|
+
const manifest: AgentSetupManifest = { skills: [] };
|
|
94
|
+
if (config?.skills?.length) {
|
|
95
|
+
manifest.skills = await installSkills(sb, config.skills, { dir: SKILL_DIR });
|
|
96
|
+
// 发现指引跟着一起写:不提示 = 白装。
|
|
97
|
+
await appendProjectInstruction(
|
|
98
|
+
sb,
|
|
99
|
+
skillDiscoveryInstruction(SKILL_DIR, installedSkillNames(manifest.skills)),
|
|
100
|
+
);
|
|
101
|
+
}
|
|
102
|
+
if (manifest.skills.length) {
|
|
103
|
+
await writeAgentSetupManifest(sb, manifest);
|
|
104
|
+
}
|
|
105
|
+
},
|
|
106
|
+
|
|
107
|
+
async send(input, ctx) {
|
|
108
|
+
const sb = ctx.sandbox;
|
|
109
|
+
// 会话契约:新会话线显式发新 session id(隔离);后续轮 resume 记录的 id。
|
|
110
|
+
const sessionId = ctx.session.id ?? `niceeval-${sb.sandboxId}-${randomUUID().slice(0, 8)}`;
|
|
111
|
+
ctx.session.capture(sessionId);
|
|
112
|
+
|
|
113
|
+
const args = ["agent", "--local", "--session-id", sessionId, "--message", input.text, "--json"];
|
|
114
|
+
const env: Record<string, string> = {
|
|
115
|
+
ANTHROPIC_API_KEY: getApiKey(),
|
|
116
|
+
...ctx.telemetry?.env,
|
|
117
|
+
};
|
|
118
|
+
const res = await sb.runCommand("openclaw", args, { env, stream: true });
|
|
119
|
+
|
|
120
|
+
const runJson = parseOpenClawRunJson(res.stdout);
|
|
121
|
+
// 封包若带回服务端分配的 session key,后续轮以它为准(capture first-writer-wins,
|
|
122
|
+
// 首轮已用自发 id 落地时不覆盖)。
|
|
123
|
+
ctx.session.capture(runJson.sessionId);
|
|
124
|
+
|
|
125
|
+
// 完整工具轨迹的唯一来源:session transcript。「最新 jsonl」而非按 session id 精确定位
|
|
126
|
+
// (同 claude-code 的裁决:send 串行,最新的一定是刚跑完的这次;超时 fallback 若产生
|
|
127
|
+
// 第二条 run,也落在同一个最新 transcript 里,单次读取不重复采集)。
|
|
128
|
+
const raw = await shared.captureLatestJsonl(sb, "~/.openclaw/agents");
|
|
129
|
+
const parsed = parseOpenClawTranscript(raw);
|
|
130
|
+
const events: StreamEvent[] = [...parsed.events];
|
|
131
|
+
|
|
132
|
+
// transcript 缺失 / 有解析不了的行:这一轮的工具轨迹不可信,coverage 降级说出来
|
|
133
|
+
// (负断言由此落 unavailable,而不是在空流上假通过),不从最终文本猜工具行为。
|
|
134
|
+
let turnCoverage: EvidenceCoverage | undefined;
|
|
135
|
+
if (raw === undefined || parsed.events.length === 0) {
|
|
136
|
+
const reason = "session transcript unavailable; only the --json final reply was collected";
|
|
137
|
+
turnCoverage = {
|
|
138
|
+
events: { status: "unavailable", reason },
|
|
139
|
+
actions: { status: "unavailable", reason },
|
|
140
|
+
usage: { status: "unavailable", reason },
|
|
141
|
+
};
|
|
142
|
+
ctx.log("openclaw transcript unavailable: tool trajectory missing for this turn, negative assertions are unreliable");
|
|
143
|
+
if (runJson.text) events.push({ type: "message", role: "assistant", text: runJson.text });
|
|
144
|
+
} else if (!parsed.parseSuccess) {
|
|
145
|
+
const reason = "some transcript lines could not be parsed";
|
|
146
|
+
turnCoverage = {
|
|
147
|
+
events: { status: "partial", reason },
|
|
148
|
+
actions: { status: "partial", reason },
|
|
149
|
+
};
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
const failed = res.exitCode !== 0 || runJson.failed;
|
|
153
|
+
if (failed) events.push({ type: "error", message: shared.diagnoseFailure(res, parsed.events, raw) });
|
|
154
|
+
|
|
155
|
+
// 用量:transcript 逐消息累加优先;transcript 没报时用封包摘要,都没有就是零值。
|
|
156
|
+
const usage =
|
|
157
|
+
parsed.usage.inputTokens > 0 || parsed.usage.outputTokens > 0
|
|
158
|
+
? parsed.usage
|
|
159
|
+
: (runJson.usage ?? parsed.usage);
|
|
160
|
+
|
|
161
|
+
return {
|
|
162
|
+
events,
|
|
163
|
+
usage,
|
|
164
|
+
status: failed ? "failed" : "completed",
|
|
165
|
+
...(turnCoverage ? { coverage: turnCoverage } : {}),
|
|
166
|
+
};
|
|
167
|
+
},
|
|
168
|
+
});
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
export default openClawAgent();
|