niceeval 0.1.1 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -8
- package/README.zh.md +13 -13
- package/docs/README.md +119 -0
- package/docs/adapters/README.md +60 -0
- package/docs/adapters/authoring.md +179 -0
- package/docs/adapters/coding-agent-skills-plugins.md +324 -0
- package/docs/adapters/collection.md +128 -0
- package/docs/adapters/contract.md +263 -0
- package/docs/adapters/reference/agent-eval.md +215 -0
- package/docs/adapters/reference/agent-loop-apis.md +69 -0
- package/docs/adapters/reference/claude-code-otel-telemetry.md +100 -0
- package/docs/adapters/reference/eve-protocol.md +127 -0
- package/docs/adapters/reference/otel-genai.md +107 -0
- package/docs/adapters/reference/otel-instrumentation.md +65 -0
- package/docs/adapters/targets.md +99 -0
- package/docs/architecture.md +140 -0
- package/docs/assertions.md +388 -0
- package/docs/capabilities-by-construction.md +48 -0
- package/docs/cli.md +171 -0
- package/docs/concepts.md +106 -0
- package/docs/e2e-ci.md +295 -0
- package/docs/eval-authoring.md +319 -0
- package/docs/experiments.md +154 -0
- package/docs/getting-started.md +224 -0
- package/docs/multi-agent.md +145 -0
- package/docs/observability.md +333 -0
- package/docs/origin-integration.md +195 -0
- package/docs/references.md +35 -0
- package/docs/results-format.md +228 -0
- package/docs/runner.md +104 -0
- package/docs/sandbox.md +276 -0
- package/docs/scoring.md +119 -0
- package/docs/source-map.md +106 -0
- package/docs/tier-sync.md +177 -0
- package/docs/view.md +157 -0
- package/docs/vision.md +88 -0
- package/package.json +39 -5
- package/src/agents/ai-sdk-otel.test.ts +22 -0
- package/src/agents/ai-sdk-otel.ts +62 -0
- package/src/agents/ai-sdk.test.ts +462 -0
- package/src/agents/ai-sdk.ts +575 -0
- package/src/agents/bub.ts +30 -37
- package/src/agents/claude-code.ts +7 -4
- package/src/agents/codex.ts +15 -10
- package/src/agents/index.ts +48 -1
- package/src/agents/sdk-streams.test.ts +145 -0
- package/src/agents/sdk-streams.ts +457 -0
- package/src/agents/shared.ts +77 -39
- package/src/agents/streaming.test.ts +152 -0
- package/src/agents/streaming.ts +195 -0
- package/src/agents/types.ts +218 -0
- package/src/agents/ui-message-stream.test.ts +241 -0
- package/src/agents/ui-message-stream.ts +368 -0
- package/src/cli.ts +124 -77
- package/src/context/context.test.ts +203 -7
- package/src/context/context.ts +158 -49
- package/src/context/session.test.ts +96 -0
- package/src/context/session.ts +119 -9
- package/src/context/types.ts +205 -0
- package/src/define.ts +4 -14
- package/src/expect/index.ts +8 -71
- package/src/i18n/core.ts +28 -0
- package/src/i18n/en.ts +47 -5
- package/src/i18n/index.ts +5 -17
- package/src/i18n/zh-CN.ts +47 -5
- package/src/index.ts +12 -0
- package/src/loaders/index.ts +8 -39
- package/src/o11y/otlp/canonical.ts +7 -6
- package/src/o11y/otlp/mappers/codex.ts +10 -8
- package/src/o11y/otlp/mappers/index.ts +9 -21
- package/src/o11y/otlp/receiver.ts +25 -7
- package/src/o11y/otlp/sandbox-receiver.ts +57 -21
- package/src/o11y/otlp/turn-otel.test.ts +110 -0
- package/src/o11y/otlp/turn-otel.ts +125 -0
- package/src/o11y/parsers/bub.ts +13 -11
- package/src/o11y/parsers/claude-code.ts +27 -51
- package/src/o11y/parsers/codex.ts +29 -46
- package/src/o11y/parsers/index.ts +5 -14
- package/src/o11y/tool-names.test.ts +61 -0
- package/src/o11y/tool-names.ts +74 -0
- package/src/o11y/types.ts +135 -0
- package/src/runner/attempt.ts +456 -0
- package/src/runner/fingerprint.ts +59 -0
- package/src/runner/remote-sandbox.ts +53 -0
- package/src/runner/report.ts +53 -0
- package/src/runner/reporters/artifacts.ts +25 -4
- package/src/runner/reporters/console.ts +2 -8
- package/src/runner/reporters/live.ts +2 -8
- package/src/runner/reporters/shared.ts +15 -0
- package/src/runner/reporters/table.ts +10 -62
- package/src/runner/run.ts +114 -619
- package/src/runner/types.ts +237 -0
- package/src/sandbox/checkpoint.ts +15 -13
- package/src/sandbox/docker-stream.ts +87 -0
- package/src/sandbox/docker.ts +42 -120
- package/src/sandbox/e2b.ts +24 -75
- package/src/sandbox/local-files.ts +33 -0
- package/src/sandbox/paths.test.ts +85 -0
- package/src/sandbox/paths.ts +45 -0
- package/src/sandbox/resolve.ts +10 -34
- package/src/sandbox/shell.ts +17 -0
- package/src/sandbox/source-files.ts +42 -2
- package/src/sandbox/types.ts +158 -0
- package/src/sandbox/vercel.ts +55 -71
- package/src/scoring/collector.ts +3 -2
- package/src/scoring/judge.ts +72 -146
- package/src/scoring/match.ts +79 -0
- package/src/scoring/scoped.ts +66 -18
- package/src/scoring/types.ts +76 -0
- package/src/shared/aggregate.ts +48 -0
- package/src/shared/format.ts +20 -0
- package/src/shared/outcome.ts +48 -0
- package/src/shared/types.ts +37 -0
- package/src/types.ts +20 -911
- package/src/view/aggregate.ts +103 -0
- package/src/view/app/App.tsx +26 -5
- package/src/view/app/components/AttemptModal.tsx +12 -3
- package/src/view/app/components/CodeView.tsx +16 -19
- package/src/view/app/components/CopyControls.tsx +4 -4
- package/src/view/app/components/ExperimentTable.tsx +5 -5
- package/src/view/app/i18n.ts +31 -7
- package/src/view/app/lib/format.ts +17 -20
- package/src/view/app/lib/outcome.ts +4 -16
- package/src/view/app/lib/transcript-data.tsx +20 -4
- package/src/view/app/main.tsx +3 -5
- package/src/view/app/pages/RunsPage.tsx +2 -2
- package/src/view/app/pages/TracesPage.tsx +2 -2
- package/src/view/app/shared.ts +2 -1
- package/src/view/app/types.ts +9 -44
- package/src/view/client-dist/app.css +1 -1
- package/src/view/client-dist/app.js +18 -18
- package/src/view/index.ts +24 -401
- package/src/view/loader.ts +234 -0
- package/src/view/server.ts +136 -0
- package/src/view/shared/types.ts +64 -0
- package/src/view/styles.css +60 -9
package/src/sandbox/e2b.ts
CHANGED
|
@@ -5,8 +5,6 @@
|
|
|
5
5
|
// 模板:opts.template 选 e2b 模板名/ID;省略用 e2b 默认 "base"。预制模板(烘焙好
|
|
6
6
|
// codex/claude-code/bub 的 "niceeval-agents")见 sandbox/e2b/。
|
|
7
7
|
|
|
8
|
-
import { readdir, readFile, stat } from "node:fs/promises";
|
|
9
|
-
import { join, relative, sep } from "node:path";
|
|
10
8
|
import { Sandbox as E2BSdkSandbox, CommandExitError } from "e2b";
|
|
11
9
|
import type {
|
|
12
10
|
Sandbox,
|
|
@@ -16,11 +14,10 @@ import type {
|
|
|
16
14
|
SourceFiles,
|
|
17
15
|
ReadSourceFilesOptions,
|
|
18
16
|
} from "../types.ts";
|
|
19
|
-
import {
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
const DEFAULT_IGNORE_FILES = ["EVAL.ts", "PROMPT.md"];
|
|
17
|
+
import { readSourceFilesByList } from "./source-files.ts";
|
|
18
|
+
import { collectLocalFiles } from "./local-files.ts";
|
|
19
|
+
import { shellQuote } from "./shell.ts";
|
|
20
|
+
import { resolveSandboxPath } from "./paths.ts";
|
|
24
21
|
|
|
25
22
|
// e2b 默认用户 "user",home 在 /home/user;工作区放其下。
|
|
26
23
|
const E2B_WORKDIR = "/home/user/workspace";
|
|
@@ -30,12 +27,8 @@ const DEFAULT_COMMAND_TIMEOUT_MS = 600_000;
|
|
|
30
27
|
// 沙箱存活上限(到点 e2b 自动回收)。给足空间跑完 setup + agent + 测试脚本。
|
|
31
28
|
const SESSION_TIMEOUT_MS = 1_800_000;
|
|
32
29
|
|
|
33
|
-
/** 单引号包裹 + 转义,把一个参数安全嵌进 shell 命令串。 */
|
|
34
|
-
function shellQuote(s: string): string {
|
|
35
|
-
return `'${s.replace(/'/g, "'\\''")}'`;
|
|
36
|
-
}
|
|
37
|
-
|
|
38
30
|
export class E2BSandbox implements Sandbox {
|
|
31
|
+
readonly workdir = E2B_WORKDIR;
|
|
39
32
|
readonly otlpHost = null;
|
|
40
33
|
private sbx: E2BSdkSandbox;
|
|
41
34
|
private commandTimeoutMs: number;
|
|
@@ -73,7 +66,7 @@ export class E2BSandbox implements Sandbox {
|
|
|
73
66
|
// 否则用模板默认(非 root)用户 —— 跨后端语义一致(见 types.ts 的 CommandOptions.root)。
|
|
74
67
|
try {
|
|
75
68
|
const res = await this.sbx.commands.run(script, {
|
|
76
|
-
cwd: opts.cwd
|
|
69
|
+
cwd: resolveSandboxPath(this.workdir, opts.cwd),
|
|
77
70
|
envs: opts.env,
|
|
78
71
|
user: opts.root ? "root" : undefined,
|
|
79
72
|
timeoutMs: this.commandTimeoutMs,
|
|
@@ -90,11 +83,7 @@ export class E2BSandbox implements Sandbox {
|
|
|
90
83
|
}
|
|
91
84
|
|
|
92
85
|
private abs(path: string): string {
|
|
93
|
-
return
|
|
94
|
-
}
|
|
95
|
-
|
|
96
|
-
private targetPath(path: string, targetDir: string): string {
|
|
97
|
-
return path.startsWith("/") ? path : `${targetDir.replace(/\/$/, "")}/${path}`;
|
|
86
|
+
return resolveSandboxPath(this.workdir, path);
|
|
98
87
|
}
|
|
99
88
|
|
|
100
89
|
async readFile(path: string): Promise<string> {
|
|
@@ -111,53 +100,35 @@ export class E2BSandbox implements Sandbox {
|
|
|
111
100
|
}
|
|
112
101
|
|
|
113
102
|
async readSourceFiles(opts: ReadSourceFilesOptions = {}): Promise<SourceFiles> {
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
.map((p) => p.trim().replace(/^\.\//, ""))
|
|
128
|
-
.filter((p) => p && !ignoreFiles.has(p.split("/").at(-1) ?? ""));
|
|
129
|
-
|
|
130
|
-
const files: { path: string; content: string }[] = [];
|
|
131
|
-
await Promise.all(
|
|
132
|
-
paths.map(async (path) => {
|
|
133
|
-
try {
|
|
134
|
-
const content = await this.sbx.files.read(`${E2B_WORKDIR}/${path}`, { format: "text" });
|
|
135
|
-
files.push({ path, content });
|
|
136
|
-
} catch {
|
|
137
|
-
// skip unreadable files (binary, permissions, etc.)
|
|
138
|
-
}
|
|
139
|
-
}),
|
|
140
|
-
);
|
|
141
|
-
return makeSourceFiles(files);
|
|
142
|
-
}
|
|
143
|
-
|
|
144
|
-
async writeFiles(files: Record<string, string>, targetDir = E2B_WORKDIR): Promise<void> {
|
|
145
|
-
const entries = Object.entries(files).map(([p, data]) => ({ path: this.targetPath(p, targetDir), data }));
|
|
103
|
+
// find 列路径 + 逐文件 files.read —— 与 vercel 后端共用同一两阶段模板。
|
|
104
|
+
return readSourceFilesByList({
|
|
105
|
+
options: opts,
|
|
106
|
+
runShell: (script) => this.runShell(script),
|
|
107
|
+
readOne: (path) => this.sbx.files.read(`${E2B_WORKDIR}/${path}`, { format: "text" }),
|
|
108
|
+
});
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
// targetDir 已由 paths.ts 的 normalizeSandboxPaths 解析成绝对路径;这里再解析一次
|
|
112
|
+
// 只是对直接使用后端实例(未包 normalize)的幂等防御,提到 map 外只算一次。
|
|
113
|
+
async writeFiles(files: Record<string, string>, targetDir?: string): Promise<void> {
|
|
114
|
+
const base = resolveSandboxPath(this.workdir, targetDir);
|
|
115
|
+
const entries = Object.entries(files).map(([p, data]) => ({ path: resolveSandboxPath(base, p), data }));
|
|
146
116
|
if (entries.length === 0) return;
|
|
147
117
|
await this.sbx.files.write(entries);
|
|
148
118
|
}
|
|
149
119
|
|
|
150
|
-
async uploadFiles(files: SandboxFile[], targetDir
|
|
120
|
+
async uploadFiles(files: SandboxFile[], targetDir?: string): Promise<void> {
|
|
151
121
|
if (files.length === 0) return;
|
|
122
|
+
const base = resolveSandboxPath(this.workdir, targetDir);
|
|
152
123
|
await this.sbx.files.write(
|
|
153
124
|
files.map((f) => ({
|
|
154
|
-
path:
|
|
125
|
+
path: resolveSandboxPath(base, f.path),
|
|
155
126
|
data: Buffer.isBuffer(f.content) ? toArrayBuffer(f.content) : f.content,
|
|
156
127
|
})),
|
|
157
128
|
);
|
|
158
129
|
}
|
|
159
130
|
|
|
160
|
-
async uploadDirectory(localDir: string, targetDir
|
|
131
|
+
async uploadDirectory(localDir: string, targetDir?: string, opts: { ignore?: string[] } = {}): Promise<void> {
|
|
161
132
|
await this.uploadFiles(await collectLocalFiles(localDir, opts.ignore), targetDir);
|
|
162
133
|
}
|
|
163
134
|
|
|
@@ -175,28 +146,6 @@ export class E2BSandbox implements Sandbox {
|
|
|
175
146
|
}
|
|
176
147
|
}
|
|
177
148
|
|
|
178
|
-
async function collectLocalFiles(localDir: string, ignore: readonly string[] = []): Promise<SandboxFile[]> {
|
|
179
|
-
const ignored = new Set(ignore);
|
|
180
|
-
const out: SandboxFile[] = [];
|
|
181
|
-
async function walk(dir: string): Promise<void> {
|
|
182
|
-
for (const entry of await readdir(dir)) {
|
|
183
|
-
if (ignored.has(entry)) continue;
|
|
184
|
-
const abs = join(dir, entry);
|
|
185
|
-
const st = await stat(abs);
|
|
186
|
-
if (st.isDirectory()) {
|
|
187
|
-
await walk(abs);
|
|
188
|
-
} else if (st.isFile()) {
|
|
189
|
-
out.push({
|
|
190
|
-
path: relative(localDir, abs).split(sep).join("/"),
|
|
191
|
-
content: await readFile(abs),
|
|
192
|
-
});
|
|
193
|
-
}
|
|
194
|
-
}
|
|
195
|
-
}
|
|
196
|
-
await walk(localDir);
|
|
197
|
-
return out;
|
|
198
|
-
}
|
|
199
|
-
|
|
200
149
|
/** Buffer → ArrayBuffer(e2b files.write 接受 string | ArrayBuffer | Blob | ReadableStream)。 */
|
|
201
150
|
function toArrayBuffer(buf: Buffer): ArrayBuffer {
|
|
202
151
|
return buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength) as ArrayBuffer;
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
// 本地目录文件采集(uploadDirectory 用)与 readSourceFiles 的默认过滤常量,
|
|
2
|
+
// docker / vercel / e2b 三后端共用一份。
|
|
3
|
+
|
|
4
|
+
import { readdir, readFile, stat } from "node:fs/promises";
|
|
5
|
+
import { join, relative, sep } from "node:path";
|
|
6
|
+
import type { SandboxFile } from "../types.ts";
|
|
7
|
+
|
|
8
|
+
export const DEFAULT_SOURCE_EXTENSIONS = ["ts", "tsx", "js", "jsx"];
|
|
9
|
+
export const DEFAULT_IGNORE_DIRS = [".git", ".next", "node_modules", "dist", "build", "coverage"];
|
|
10
|
+
export const DEFAULT_IGNORE_FILES = ["EVAL.ts", "PROMPT.md"];
|
|
11
|
+
|
|
12
|
+
/** 递归收集本地目录下的全部文件;路径统一转成 POSIX 分隔的相对路径,供上传沙箱用。 */
|
|
13
|
+
export async function collectLocalFiles(localDir: string, ignore: readonly string[] = []): Promise<SandboxFile[]> {
|
|
14
|
+
const ignored = new Set(ignore);
|
|
15
|
+
const out: SandboxFile[] = [];
|
|
16
|
+
async function walk(dir: string): Promise<void> {
|
|
17
|
+
for (const entry of await readdir(dir)) {
|
|
18
|
+
if (ignored.has(entry)) continue;
|
|
19
|
+
const abs = join(dir, entry);
|
|
20
|
+
const st = await stat(abs);
|
|
21
|
+
if (st.isDirectory()) {
|
|
22
|
+
await walk(abs);
|
|
23
|
+
} else if (st.isFile()) {
|
|
24
|
+
out.push({
|
|
25
|
+
path: relative(localDir, abs).split(sep).join("/"),
|
|
26
|
+
content: await readFile(abs),
|
|
27
|
+
});
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
await walk(localDir);
|
|
32
|
+
return out;
|
|
33
|
+
}
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
import { describe, expect, it } from "vitest";
|
|
2
|
+
import { normalizeSandboxPaths, resolveLocalPath, resolveSandboxPath } from "./paths.ts";
|
|
3
|
+
import type { Sandbox } from "../types.ts";
|
|
4
|
+
|
|
5
|
+
function fakeSandbox(): Sandbox & { calls: string[] } {
|
|
6
|
+
const calls: string[] = [];
|
|
7
|
+
return {
|
|
8
|
+
workdir: "/work",
|
|
9
|
+
sandboxId: "fake",
|
|
10
|
+
otlpHost: null,
|
|
11
|
+
runCommand: async (_cmd, _args, opts) => {
|
|
12
|
+
calls.push(`cwd:${opts?.cwd ?? ""}`);
|
|
13
|
+
return { stdout: "", stderr: "", exitCode: 0 };
|
|
14
|
+
},
|
|
15
|
+
runShell: async (_script, opts) => {
|
|
16
|
+
calls.push(`shell-cwd:${opts?.cwd ?? ""}`);
|
|
17
|
+
return { stdout: "", stderr: "", exitCode: 0 };
|
|
18
|
+
},
|
|
19
|
+
readFile: async (path) => {
|
|
20
|
+
calls.push(`read:${path}`);
|
|
21
|
+
return "";
|
|
22
|
+
},
|
|
23
|
+
fileExists: async (path) => {
|
|
24
|
+
calls.push(`exists:${path}`);
|
|
25
|
+
return true;
|
|
26
|
+
},
|
|
27
|
+
readSourceFiles: async () => Object.assign([], {
|
|
28
|
+
text: () => "",
|
|
29
|
+
code: () => "",
|
|
30
|
+
fileMatching: () => undefined,
|
|
31
|
+
fileMatchingAll: () => undefined,
|
|
32
|
+
hasPath: () => false,
|
|
33
|
+
}),
|
|
34
|
+
writeFiles: async (_files, targetDir) => {
|
|
35
|
+
calls.push(`write:${targetDir}`);
|
|
36
|
+
},
|
|
37
|
+
uploadFiles: async (_files, targetDir) => {
|
|
38
|
+
calls.push(`upload:${targetDir}`);
|
|
39
|
+
},
|
|
40
|
+
uploadDirectory: async (_localDir, targetDir) => {
|
|
41
|
+
calls.push(`upload-dir:${targetDir}`);
|
|
42
|
+
},
|
|
43
|
+
stop: async () => {},
|
|
44
|
+
downloadFile: async (path) => {
|
|
45
|
+
calls.push(`download:${path}`);
|
|
46
|
+
return Buffer.from("");
|
|
47
|
+
},
|
|
48
|
+
uploadFile: async (path) => {
|
|
49
|
+
calls.push(`upload-file:${path}`);
|
|
50
|
+
},
|
|
51
|
+
calls,
|
|
52
|
+
};
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
describe("sandbox path helpers", () => {
|
|
56
|
+
it("resolves sandbox paths relative to workdir", () => {
|
|
57
|
+
expect(resolveSandboxPath("/work", undefined)).toBe("/work");
|
|
58
|
+
expect(resolveSandboxPath("/work", "src/app.ts")).toBe("/work/src/app.ts");
|
|
59
|
+
expect(resolveSandboxPath("/work", "/tmp/out")).toBe("/tmp/out");
|
|
60
|
+
});
|
|
61
|
+
|
|
62
|
+
it("resolves local paths relative to eval directories", () => {
|
|
63
|
+
expect(resolveLocalPath("/repo/evals/auth", "../fixtures/app")).toBe("/repo/evals/fixtures/app");
|
|
64
|
+
expect(resolveLocalPath("/repo/evals/auth", "/tmp/app")).toBe("/tmp/app");
|
|
65
|
+
});
|
|
66
|
+
|
|
67
|
+
it("normalizes paths for custom sandbox implementations", async () => {
|
|
68
|
+
const sandbox = fakeSandbox();
|
|
69
|
+
const normalized = normalizeSandboxPaths(sandbox);
|
|
70
|
+
|
|
71
|
+
await normalized.runCommand("npm", ["test"], { cwd: "packages/api" });
|
|
72
|
+
await normalized.readFile("src/app.ts");
|
|
73
|
+
await normalized.uploadFiles([], "fixtures");
|
|
74
|
+
await normalized.uploadDirectory("/host/app");
|
|
75
|
+
await normalized.downloadFile("dist/out.txt");
|
|
76
|
+
|
|
77
|
+
expect(sandbox.calls).toEqual([
|
|
78
|
+
"cwd:/work/packages/api",
|
|
79
|
+
"read:/work/src/app.ts",
|
|
80
|
+
"upload:/work/fixtures",
|
|
81
|
+
"upload-dir:/work",
|
|
82
|
+
"download:/work/dist/out.txt",
|
|
83
|
+
]);
|
|
84
|
+
});
|
|
85
|
+
});
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import { posix } from "node:path";
|
|
2
|
+
import { isAbsolute, resolve } from "node:path";
|
|
3
|
+
import type { CommandOptions, Sandbox } from "../types.ts";
|
|
4
|
+
|
|
5
|
+
export function resolveSandboxPath(workdir: string, path?: string): string {
|
|
6
|
+
if (!path || path === ".") return workdir;
|
|
7
|
+
return path.startsWith("/") ? path : posix.join(workdir, path);
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
export function resolveLocalPath(baseDir: string | undefined, path: string): string {
|
|
11
|
+
if (!baseDir || isAbsolute(path)) return path;
|
|
12
|
+
return resolve(baseDir, path);
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
function resolveCommandOptions(workdir: string, opts: CommandOptions | undefined): CommandOptions | undefined {
|
|
16
|
+
if (!opts?.cwd) return opts;
|
|
17
|
+
return { ...opts, cwd: resolveSandboxPath(workdir, opts.cwd) };
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
export function normalizeSandboxPaths(sandbox: Sandbox): Sandbox {
|
|
21
|
+
return {
|
|
22
|
+
get workdir() {
|
|
23
|
+
return sandbox.workdir;
|
|
24
|
+
},
|
|
25
|
+
get sandboxId() {
|
|
26
|
+
return sandbox.sandboxId;
|
|
27
|
+
},
|
|
28
|
+
get otlpHost() {
|
|
29
|
+
return sandbox.otlpHost;
|
|
30
|
+
},
|
|
31
|
+
runCommand: (cmd, args, opts) => sandbox.runCommand(cmd, args, resolveCommandOptions(sandbox.workdir, opts)),
|
|
32
|
+
runShell: (script, opts) => sandbox.runShell(script, resolveCommandOptions(sandbox.workdir, opts)),
|
|
33
|
+
readFile: (path) => sandbox.readFile(resolveSandboxPath(sandbox.workdir, path)),
|
|
34
|
+
fileExists: (path) => sandbox.fileExists(resolveSandboxPath(sandbox.workdir, path)),
|
|
35
|
+
readSourceFiles: (opts) => sandbox.readSourceFiles(opts),
|
|
36
|
+
writeFiles: (files, targetDir) => sandbox.writeFiles(files, resolveSandboxPath(sandbox.workdir, targetDir)),
|
|
37
|
+
uploadFiles: (files, targetDir) => sandbox.uploadFiles(files, resolveSandboxPath(sandbox.workdir, targetDir)),
|
|
38
|
+
uploadDirectory: (localDir, targetDir, opts) =>
|
|
39
|
+
sandbox.uploadDirectory(localDir, resolveSandboxPath(sandbox.workdir, targetDir), opts),
|
|
40
|
+
stop: () => sandbox.stop(),
|
|
41
|
+
appendLog: sandbox.appendLog ? (line) => sandbox.appendLog!(line) : undefined,
|
|
42
|
+
downloadFile: (path) => sandbox.downloadFile(resolveSandboxPath(sandbox.workdir, path)),
|
|
43
|
+
uploadFile: (path, content) => sandbox.uploadFile(resolveSandboxPath(sandbox.workdir, path), content),
|
|
44
|
+
};
|
|
45
|
+
}
|
package/src/sandbox/resolve.ts
CHANGED
|
@@ -1,10 +1,13 @@
|
|
|
1
|
-
// 沙箱后端解析:把
|
|
2
|
-
//
|
|
1
|
+
// 沙箱后端解析:把 config.sandbox / experiment.sandbox(工厂函数产出的 spec 数据结构)
|
|
2
|
+
// 折叠成一个具体后端 + 参数,并按需创建实例。`sandbox` 没有默认值也没有按名字选的入口——
|
|
3
|
+
// 沙箱型 agent 必须显式给 dockerSandbox() / vercelSandbox() / e2bSandbox() / defineSandbox(),
|
|
4
|
+
// 省略时 resolveSandbox() 直接抛错,不猜环境、不兜底。
|
|
3
5
|
// 后端名的行为分支只允许出现在 sandbox/ 内(见 docs/architecture.md)。
|
|
4
6
|
|
|
5
7
|
import { Effect } from "effect";
|
|
6
|
-
import type { CustomSandboxSpec, Sandbox,
|
|
8
|
+
import type { CustomSandboxSpec, Sandbox, SandboxOption, SandboxRuntime } from "../types.ts";
|
|
7
9
|
import { registerSandbox, stopSandbox } from "./registry.ts";
|
|
10
|
+
import { normalizeSandboxPaths } from "./paths.ts";
|
|
8
11
|
import { t } from "../i18n/index.ts";
|
|
9
12
|
|
|
10
13
|
/** 归一化后的沙箱描述:确定的后端 + 各后端参数(只有对应后端用得上的会有值)。 */
|
|
@@ -22,37 +25,10 @@ export interface ResolvedSandbox {
|
|
|
22
25
|
recommendedConcurrency?: number;
|
|
23
26
|
}
|
|
24
27
|
|
|
25
|
-
/**
|
|
26
|
-
* 决定用哪个后端:
|
|
27
|
-
* - 显式指定且非 "auto" → 直接用。
|
|
28
|
-
* - 否则:env 里有 VERCEL_TOKEN / VERCEL_OIDC_TOKEN → "vercel"。
|
|
29
|
-
* - 再否则:env 里有 E2B_API_KEY → "e2b"。
|
|
30
|
-
* - 再否则 → "docker"。
|
|
31
|
-
*/
|
|
32
|
-
export function resolveBackend(opts: { backend?: SandboxBackend }): SandboxBackend {
|
|
33
|
-
const { backend } = opts;
|
|
34
|
-
if (backend && backend !== "auto") {
|
|
35
|
-
return backend;
|
|
36
|
-
}
|
|
37
|
-
if (process.env.VERCEL_TOKEN || process.env.VERCEL_OIDC_TOKEN) {
|
|
38
|
-
return "vercel";
|
|
39
|
-
}
|
|
40
|
-
if (process.env.E2B_API_KEY) {
|
|
41
|
-
return "e2b";
|
|
42
|
-
}
|
|
43
|
-
return "docker";
|
|
44
|
-
}
|
|
45
|
-
|
|
46
|
-
/** 把字符串后端名 或 spec 数据结构 归一化成 ResolvedSandbox(spec 的 backend 已是具体值,直接用)。 */
|
|
28
|
+
/** 把 spec 数据结构归一化成 ResolvedSandbox;省略(undefined)直接报错——没有默认后端。 */
|
|
47
29
|
export function resolveSandbox(opt: SandboxOption | undefined, runtimeDefault?: SandboxRuntime): ResolvedSandbox {
|
|
48
|
-
if (opt
|
|
49
|
-
|
|
50
|
-
}
|
|
51
|
-
const backend = resolveBackend({ backend: opt });
|
|
52
|
-
if (backend !== "docker" && backend !== "vercel" && backend !== "e2b") {
|
|
53
|
-
throw new Error(t("sandbox.backendNotImplemented", { backend }));
|
|
54
|
-
}
|
|
55
|
-
return { backend, runtime: runtimeDefault };
|
|
30
|
+
if (!opt) throw new Error(t("sandbox.missingSpec"));
|
|
31
|
+
return { ...opt, runtime: opt.runtime ?? runtimeDefault };
|
|
56
32
|
}
|
|
57
33
|
|
|
58
34
|
/**
|
|
@@ -94,7 +70,7 @@ export function createSandbox(opts: {
|
|
|
94
70
|
Effect.promise<Sandbox>(async () => {
|
|
95
71
|
// 起好就登记:让 cli 的兜底强清(二次 Ctrl+C / 看门狗超时)能直接停到它,不只靠下面的
|
|
96
72
|
// release。即便本 fiber 创建后立刻被中断、release 还没来得及跑,登记表也已认得这个沙箱。
|
|
97
|
-
const sb = await createBackend(r, opts.timeout);
|
|
73
|
+
const sb = normalizeSandboxPaths(await createBackend(r, opts.timeout));
|
|
98
74
|
registerSandbox(sb);
|
|
99
75
|
return sb;
|
|
100
76
|
}),
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
// shell 拼接工具:单引号转义与 find 脚本构造,docker / vercel / e2b / checkpoint 共用。
|
|
2
|
+
|
|
3
|
+
/** 单引号包裹 + 转义,把一个参数安全嵌进 shell 命令串。 */
|
|
4
|
+
export function shellQuote(s: string): string {
|
|
5
|
+
return `'${s.replace(/'/g, "'\\''")}'`;
|
|
6
|
+
}
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* 构造 readSourceFiles 用的 find 脚本:按目录名(任意深度)剪枝、按扩展名收文件,
|
|
10
|
+
* 输出 `./` 前缀的相对路径。ignoreDirs / extensions 可能来自 eval 作者输入,
|
|
11
|
+
* 一律走 shellQuote 转义后再拼进脚本,防止特殊字符破坏脚本结构。
|
|
12
|
+
*/
|
|
13
|
+
export function buildFindScript(opts: { extensions: readonly string[]; ignoreDirs: readonly string[] }): string {
|
|
14
|
+
const dirPrune = opts.ignoreDirs.map((d) => `-name ${shellQuote(d)}`).join(" -o ");
|
|
15
|
+
const nameTests = opts.extensions.map((e) => `-name ${shellQuote(`*.${e}`)}`).join(" -o ");
|
|
16
|
+
return `find . \\( -type d \\( ${dirPrune} \\) \\) -prune -o -type f \\( ${nameTests} \\) -print`;
|
|
17
|
+
}
|
|
@@ -1,8 +1,11 @@
|
|
|
1
|
-
// 把裸 SourceFile[] 包装成带便利方法的 SourceFiles(仍是真数组)
|
|
1
|
+
// 把裸 SourceFile[] 包装成带便利方法的 SourceFiles(仍是真数组),
|
|
2
|
+
// 以及 vercel / e2b 共用的「find 列文件 + 逐文件读」两阶段读取模板。
|
|
2
3
|
// 方法用 defineProperties 挂成 non-enumerable:.filter/.map/展开/迭代/JSON 都不受影响。
|
|
3
4
|
|
|
4
|
-
import type { SourceFile, SourceFiles } from "../types.ts";
|
|
5
|
+
import type { CommandResult, ReadSourceFilesOptions, SourceFile, SourceFiles } from "../types.ts";
|
|
5
6
|
import { stripComments } from "../util.ts";
|
|
7
|
+
import { buildFindScript } from "./shell.ts";
|
|
8
|
+
import { DEFAULT_IGNORE_DIRS, DEFAULT_IGNORE_FILES, DEFAULT_SOURCE_EXTENSIONS } from "./local-files.ts";
|
|
6
9
|
|
|
7
10
|
export function makeSourceFiles(files: SourceFile[]): SourceFiles {
|
|
8
11
|
const arr = files.slice();
|
|
@@ -28,3 +31,40 @@ export function makeSourceFiles(files: SourceFile[]): SourceFiles {
|
|
|
28
31
|
|
|
29
32
|
return arr as unknown as SourceFiles;
|
|
30
33
|
}
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* 两阶段 readSourceFiles 模板(vercel / e2b 共用):
|
|
37
|
+
* Phase 1 只做 find(列路径,短命令快速结束);Phase 2 经 readOne 逐文件独立读取,
|
|
38
|
+
* 不依赖长命令输出流 —— 即使 session 快到平台上限,后半段读取也不会被截断。
|
|
39
|
+
* readOne 返回 null 表示该文件跳过(二进制、无权限等)。
|
|
40
|
+
*/
|
|
41
|
+
export async function readSourceFilesByList(opts: {
|
|
42
|
+
options: ReadSourceFilesOptions;
|
|
43
|
+
runShell: (script: string) => Promise<CommandResult>;
|
|
44
|
+
readOne: (path: string) => Promise<string | null>;
|
|
45
|
+
}): Promise<SourceFiles> {
|
|
46
|
+
const extensions = opts.options.extensions ?? DEFAULT_SOURCE_EXTENSIONS;
|
|
47
|
+
const ignoreDirs = opts.options.ignoreDirs ?? DEFAULT_IGNORE_DIRS;
|
|
48
|
+
const ignoreFiles = new Set(opts.options.ignoreFiles ?? DEFAULT_IGNORE_FILES);
|
|
49
|
+
|
|
50
|
+
const result = await opts.runShell(buildFindScript({ extensions, ignoreDirs }));
|
|
51
|
+
|
|
52
|
+
const paths = result.stdout
|
|
53
|
+
.trim()
|
|
54
|
+
.split("\n")
|
|
55
|
+
.map((p) => p.trim().replace(/^\.\//, ""))
|
|
56
|
+
.filter((p) => p && !ignoreFiles.has(p.split("/").at(-1) ?? ""));
|
|
57
|
+
|
|
58
|
+
const files: SourceFile[] = [];
|
|
59
|
+
await Promise.all(
|
|
60
|
+
paths.map(async (path) => {
|
|
61
|
+
try {
|
|
62
|
+
const content = await opts.readOne(path);
|
|
63
|
+
if (content !== null) files.push({ path, content });
|
|
64
|
+
} catch {
|
|
65
|
+
// 跳过读不了的文件(二进制、权限等)。
|
|
66
|
+
}
|
|
67
|
+
}),
|
|
68
|
+
);
|
|
69
|
+
return makeSourceFiles(files);
|
|
70
|
+
}
|
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
// sandbox 域类型:Sandbox 接口、后端 spec(可辨识联合)、命令与文件 IO 的形状。
|
|
2
|
+
// 「在哪里跑、如何隔离」的全部契约在这里;后端实现见本目录各文件,分发见 resolve.ts。
|
|
3
|
+
|
|
4
|
+
export interface CommandResult {
|
|
5
|
+
stdout: string;
|
|
6
|
+
stderr: string;
|
|
7
|
+
exitCode: number;
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
export interface SandboxFile {
|
|
11
|
+
path: string;
|
|
12
|
+
content: string | Buffer;
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
/** 一个源码文件:相对工作区根的路径 + 文本内容。readSourceFiles 的返回元素。 */
|
|
16
|
+
export interface SourceFile {
|
|
17
|
+
path: string;
|
|
18
|
+
content: string;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* readSourceFiles 的返回值:仍是一个 SourceFile 数组(.filter/.some/.map 照用),
|
|
23
|
+
* 额外挂上整体匹配 / 按文件匹配的便利方法,省掉 eval 目录里手写的 source-helpers。
|
|
24
|
+
*/
|
|
25
|
+
export interface SourceFiles extends ReadonlyArray<SourceFile> {
|
|
26
|
+
/** 全部文件内容拼接(每段前带 `// path` 注释),用于整体 regex。 */
|
|
27
|
+
text(): string;
|
|
28
|
+
/** 同 text() 但先剥注释,只看真实代码。 */
|
|
29
|
+
code(): string;
|
|
30
|
+
/** 第一个内容命中 pattern 的文件。 */
|
|
31
|
+
fileMatching(pattern: RegExp): SourceFile | undefined;
|
|
32
|
+
/** 第一个内容命中全部 patterns 的文件(同文件共现,per-file 而非拼接源码)。 */
|
|
33
|
+
fileMatchingAll(patterns: RegExp[]): SourceFile | undefined;
|
|
34
|
+
/** 是否存在路径命中 pattern 的文件。 */
|
|
35
|
+
hasPath(pattern: RegExp): boolean;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/** readSourceFiles 的可选项;不传则用一套合理默认。 */
|
|
39
|
+
export interface ReadSourceFilesOptions {
|
|
40
|
+
/** 文件扩展名(不带点)。默认 ts/tsx/js/jsx。 */
|
|
41
|
+
extensions?: string[];
|
|
42
|
+
/** 按目录名(任意深度)剪枝。默认 .git/.next/node_modules/dist/build/coverage。 */
|
|
43
|
+
ignoreDirs?: string[];
|
|
44
|
+
/** 按文件 basename 忽略。默认 EVAL.ts/PROMPT.md。 */
|
|
45
|
+
ignoreFiles?: string[];
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/** 内置后端名;只在 sandbox/resolve.ts 内部做环境探测和 CLI `--sandbox` 解析用,不出现在 `sandbox` 字段的类型里。 */
|
|
49
|
+
export type SandboxBackend = "docker" | "vercel" | "e2b";
|
|
50
|
+
|
|
51
|
+
/** 镜像/模板里的 Node 运行时版本。 */
|
|
52
|
+
export type SandboxRuntime = "node20" | "node24";
|
|
53
|
+
|
|
54
|
+
/**
|
|
55
|
+
* Sandbox 的「数据结构」定义 —— 与 agent 一样可带参数(见 docs/sandbox.md)。
|
|
56
|
+
* 必须用工厂函数构造(`dockerSandbox()` / `vercelSandbox()` / `e2bSandbox()` / `defineSandbox()`),
|
|
57
|
+
* 放进 config / experiment 的 `sandbox` 字段 —— 字段类型只接受这个数据结构,不接受裸字符串,
|
|
58
|
+
* 省略字段 = 按环境自动探测后端。各后端的参数互不相同 —— 这是个按 `backend` 区分的可辨识联合(discriminated union)。
|
|
59
|
+
*/
|
|
60
|
+
export interface DockerSandboxSpec {
|
|
61
|
+
readonly backend: "docker";
|
|
62
|
+
/** 覆盖默认镜像;默认按 runtime 选 `node:*-slim`。预制模板:传烘焙好 agent CLI 的镜像名。 */
|
|
63
|
+
readonly image?: string;
|
|
64
|
+
readonly runtime?: SandboxRuntime;
|
|
65
|
+
}
|
|
66
|
+
export interface VercelSandboxSpec {
|
|
67
|
+
readonly backend: "vercel";
|
|
68
|
+
/** 从已有快照起 microVM。预制模板:烘焙好 agent CLI 的 snapshotId。 */
|
|
69
|
+
readonly snapshotId?: string;
|
|
70
|
+
readonly runtime?: SandboxRuntime;
|
|
71
|
+
}
|
|
72
|
+
export interface E2BSandboxSpec {
|
|
73
|
+
readonly backend: "e2b";
|
|
74
|
+
/** e2b 模板名/ID。预制模板:烘焙好 agent CLI 的模板(如 `"niceeval-agents"`)。省略用 e2b 默认 `"base"`。 */
|
|
75
|
+
readonly template?: string;
|
|
76
|
+
/** 仅作记录;e2b 的 node 版本由模板决定,不在创建时选。 */
|
|
77
|
+
readonly runtime?: SandboxRuntime;
|
|
78
|
+
}
|
|
79
|
+
/**
|
|
80
|
+
* 用户自定义后端:`create` 直接产出一个 `Sandbox` 实例,不经 resolve.ts 的内置 backend switch。
|
|
81
|
+
* 用 `defineSandbox()` 构造(见 src/define.ts)。`backend` 只用于展示 / 日志,不参与分发。
|
|
82
|
+
*/
|
|
83
|
+
export interface CustomSandboxSpec {
|
|
84
|
+
readonly backend: string;
|
|
85
|
+
readonly runtime?: SandboxRuntime;
|
|
86
|
+
readonly recommendedConcurrency?: number;
|
|
87
|
+
readonly create: (opts: { timeout?: number; runtime?: SandboxRuntime }) => Promise<Sandbox>;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
export type SandboxSpec = DockerSandboxSpec | VercelSandboxSpec | E2BSandboxSpec | CustomSandboxSpec;
|
|
91
|
+
|
|
92
|
+
/** config / experiment 的 `sandbox` 字段:必须是工厂函数产出的 spec 数据结构,省略 = 自动探测后端。 */
|
|
93
|
+
export type SandboxOption = SandboxSpec;
|
|
94
|
+
|
|
95
|
+
export interface CommandOptions {
|
|
96
|
+
env?: Record<string, string>;
|
|
97
|
+
cwd?: string;
|
|
98
|
+
/**
|
|
99
|
+
* 把本命令的输出也送进沙箱的「原生日志流」(于是 `docker logs` / Docker UI 的 Logs
|
|
100
|
+
* 标签页能实时看到它)。给 agent 命令(codex exec / bub run / claude)开它,就能在容器
|
|
101
|
+
* 日志里看到 agent 的【原始输出】。后端各自实现(docker:tee 到 PID1 tail 的文件;
|
|
102
|
+
* 不支持的后端忽略)—— 日志怎么浮现是 backend 的事,adapter 只声明意图。
|
|
103
|
+
*/
|
|
104
|
+
stream?: boolean;
|
|
105
|
+
/**
|
|
106
|
+
* 以 root 跑本命令。默认 `false` —— 命令以沙箱的标准**非 root** 用户跑(agent 的自然环境)。
|
|
107
|
+
* 给 setup 阶段装系统依赖用(`apt-get install …`、`pip install --break-system-packages …`)。
|
|
108
|
+
*
|
|
109
|
+
* 语义跨后端一致:"本命令以 root 跑,否则以标准非 root 用户跑"。各后端映射到自己的原生机制
|
|
110
|
+
* (docker:`exec --user root`;E2B:`{ user: "root" }`;Vercel:`{ sudo: true }`;Daytona:`{ user }`)。
|
|
111
|
+
* 本就全程 root 的后端(如 Modal)视作 no-op;完全无法提权的后端可不支持(抛错)—— 但**默认值与
|
|
112
|
+
* 语义保持一致**,不因后端而变。
|
|
113
|
+
*/
|
|
114
|
+
root?: boolean;
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
export interface Sandbox {
|
|
118
|
+
readonly workdir: string;
|
|
119
|
+
runCommand(cmd: string, args?: string[], opts?: CommandOptions): Promise<CommandResult>;
|
|
120
|
+
runShell(script: string, opts?: CommandOptions): Promise<CommandResult>;
|
|
121
|
+
readFile(path: string): Promise<string>;
|
|
122
|
+
fileExists(path: string): Promise<boolean>;
|
|
123
|
+
/**
|
|
124
|
+
* 一次 shell 往返读全部源码文件(按扩展名收、按目录/文件名忽略)。
|
|
125
|
+
* 取代每个 eval 目录里手写的 find + 逐文件 readFile。
|
|
126
|
+
*/
|
|
127
|
+
readSourceFiles(opts?: ReadSourceFilesOptions): Promise<SourceFiles>;
|
|
128
|
+
writeFiles(files: Record<string, string>, targetDir?: string): Promise<void>;
|
|
129
|
+
uploadFiles(files: SandboxFile[], targetDir?: string): Promise<void>;
|
|
130
|
+
uploadDirectory(localDir: string, targetDir?: string, opts?: { ignore?: string[] }): Promise<void>;
|
|
131
|
+
stop(): Promise<void>;
|
|
132
|
+
readonly sandboxId: string;
|
|
133
|
+
/**
|
|
134
|
+
* 本地 OTLP 接收器的目标 host。
|
|
135
|
+
* - `string`:沙箱内可通过该 hostname 回连宿主 OTLP 端口(如 docker 的 `host.docker.internal`)。
|
|
136
|
+
* - `null`:沙箱运行在远程云端(如 e2b/vercel),无法访问宿主本地端口 → 跳过 tracing。
|
|
137
|
+
* 可通过环境变量 `NICEEVAL_OTLP_HOST` 强制覆盖(如配置 tunnel 时)。
|
|
138
|
+
*/
|
|
139
|
+
readonly otlpHost: string | null;
|
|
140
|
+
|
|
141
|
+
/**
|
|
142
|
+
* 可选:把一行写进容器的「主日志」(PID1 在 tail 它)——于是 `docker logs` /
|
|
143
|
+
* Docker UI 的 Logs 标签页能实时看到 agent 逐轮活动。docker 后端实现,其它可省略。
|
|
144
|
+
*/
|
|
145
|
+
appendLog?(line: string): Promise<void>;
|
|
146
|
+
|
|
147
|
+
/**
|
|
148
|
+
* 从沙箱内任意路径读取文件,返回二进制 Buffer。
|
|
149
|
+
* 对应各 backend:Docker getArchive / Vercel readFileToBuffer / e2b files.read(bytes) / …
|
|
150
|
+
*/
|
|
151
|
+
downloadFile(path: string): Promise<Buffer>;
|
|
152
|
+
|
|
153
|
+
/**
|
|
154
|
+
* 向沙箱内任意路径写入文件(二进制)。
|
|
155
|
+
* 对应各 backend:Docker putArchive / Vercel fs.writeFile(Buffer) / e2b files.write / …
|
|
156
|
+
*/
|
|
157
|
+
uploadFile(path: string, content: Buffer): Promise<void>;
|
|
158
|
+
}
|