niceeval 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (94) hide show
  1. package/README.md +164 -0
  2. package/README.zh.md +160 -0
  3. package/bin/niceeval.js +11 -0
  4. package/package.json +96 -0
  5. package/src/agents/bub.ts +223 -0
  6. package/src/agents/builtin.ts +6 -0
  7. package/src/agents/claude-code.ts +96 -0
  8. package/src/agents/codex.ts +122 -0
  9. package/src/agents/index.ts +25 -0
  10. package/src/agents/shared.ts +145 -0
  11. package/src/cli.ts +421 -0
  12. package/src/context/context.test.ts +96 -0
  13. package/src/context/context.ts +427 -0
  14. package/src/context/control-flow.ts +27 -0
  15. package/src/context/session.ts +124 -0
  16. package/src/define.ts +112 -0
  17. package/src/expect/index.ts +243 -0
  18. package/src/i18n/en.ts +131 -0
  19. package/src/i18n/index.ts +39 -0
  20. package/src/i18n/zh-CN.ts +132 -0
  21. package/src/index.ts +39 -0
  22. package/src/loaders/index.ts +56 -0
  23. package/src/o11y/cost.ts +57 -0
  24. package/src/o11y/derive.ts +304 -0
  25. package/src/o11y/otlp/canonical.ts +112 -0
  26. package/src/o11y/otlp/mappers/bub.ts +30 -0
  27. package/src/o11y/otlp/mappers/codex.ts +38 -0
  28. package/src/o11y/otlp/mappers/index.ts +25 -0
  29. package/src/o11y/otlp/parse.ts +330 -0
  30. package/src/o11y/otlp/receiver.ts +100 -0
  31. package/src/o11y/otlp/sandbox-receiver.ts +113 -0
  32. package/src/o11y/otlp/select.ts +82 -0
  33. package/src/o11y/parsers/bub.ts +240 -0
  34. package/src/o11y/parsers/claude-code.ts +270 -0
  35. package/src/o11y/parsers/codex.ts +480 -0
  36. package/src/o11y/parsers/index.ts +37 -0
  37. package/src/o11y/prices.json +9873 -0
  38. package/src/runner/discover.ts +77 -0
  39. package/src/runner/reporters/artifacts.ts +81 -0
  40. package/src/runner/reporters/console.ts +76 -0
  41. package/src/runner/reporters/index.ts +5 -0
  42. package/src/runner/reporters/json.ts +52 -0
  43. package/src/runner/reporters/live.ts +178 -0
  44. package/src/runner/reporters/table.ts +328 -0
  45. package/src/runner/run.ts +860 -0
  46. package/src/runner/sandbox-prep.ts +91 -0
  47. package/src/sandbox/checkpoint.ts +39 -0
  48. package/src/sandbox/docker.ts +539 -0
  49. package/src/sandbox/e2b.ts +203 -0
  50. package/src/sandbox/index.ts +25 -0
  51. package/src/sandbox/registry.ts +60 -0
  52. package/src/sandbox/resolve.ts +131 -0
  53. package/src/sandbox/source-files.ts +30 -0
  54. package/src/sandbox/vercel.ts +236 -0
  55. package/src/scoring/collector.ts +113 -0
  56. package/src/scoring/judge.ts +236 -0
  57. package/src/scoring/scoped.ts +289 -0
  58. package/src/scoring/verdict.ts +20 -0
  59. package/src/source-loc.ts +53 -0
  60. package/src/types.ts +913 -0
  61. package/src/util.test.ts +31 -0
  62. package/src/util.ts +50 -0
  63. package/src/view/app/App.tsx +189 -0
  64. package/src/view/app/components/AttemptModal.tsx +66 -0
  65. package/src/view/app/components/CodeView.tsx +272 -0
  66. package/src/view/app/components/CopyControls.tsx +89 -0
  67. package/src/view/app/components/ExperimentTable.tsx +266 -0
  68. package/src/view/app/components/GroupSelector.tsx +61 -0
  69. package/src/view/app/components/LazyArtifact.tsx +50 -0
  70. package/src/view/app/components/Trace.tsx +100 -0
  71. package/src/view/app/components/Transcript.tsx +130 -0
  72. package/src/view/app/components/primitives.tsx +43 -0
  73. package/src/view/app/components/ui/badge.tsx +21 -0
  74. package/src/view/app/components/ui/dialog.tsx +34 -0
  75. package/src/view/app/components/ui/tabs.tsx +30 -0
  76. package/src/view/app/i18n.ts +341 -0
  77. package/src/view/app/index.html +13 -0
  78. package/src/view/app/lib/cn.ts +7 -0
  79. package/src/view/app/lib/format.ts +73 -0
  80. package/src/view/app/lib/guards.ts +61 -0
  81. package/src/view/app/lib/outcome.ts +96 -0
  82. package/src/view/app/lib/rows.ts +63 -0
  83. package/src/view/app/lib/transcript-data.tsx +121 -0
  84. package/src/view/app/main.tsx +17 -0
  85. package/src/view/app/pages/RunsPage.tsx +83 -0
  86. package/src/view/app/pages/TracesPage.tsx +40 -0
  87. package/src/view/app/shared.ts +10 -0
  88. package/src/view/app/types.ts +114 -0
  89. package/src/view/app/vite.config.ts +26 -0
  90. package/src/view/client-dist/app.css +2 -0
  91. package/src/view/client-dist/app.js +56 -0
  92. package/src/view/index.ts +406 -0
  93. package/src/view/styles.css +1074 -0
  94. package/src/view/template.html +15 -0
@@ -0,0 +1,203 @@
1
+ // E2B 沙箱后端:用 e2b SDK 把 E2B microVM 当隔离工作区跑 eval。
2
+ // 契约对齐 ../types.ts 的 Sandbox 接口,与 DockerSandbox / VercelSandbox 可互换。
3
+ //
4
+ // 鉴权:E2B_API_KEY(team 级,e2b CLI `e2b auth login` 后也写在 ~/.e2b)。
5
+ // 模板:opts.template 选 e2b 模板名/ID;省略用 e2b 默认 "base"。预制模板(烘焙好
6
+ // codex/claude-code/bub 的 "niceeval-agents")见 sandbox/e2b/。
7
+
8
+ import { readdir, readFile, stat } from "node:fs/promises";
9
+ import { join, relative, sep } from "node:path";
10
+ import { Sandbox as E2BSdkSandbox, CommandExitError } from "e2b";
11
+ import type {
12
+ Sandbox,
13
+ CommandResult,
14
+ CommandOptions,
15
+ SandboxFile,
16
+ SourceFiles,
17
+ ReadSourceFilesOptions,
18
+ } from "../types.ts";
19
+ import { makeSourceFiles } from "./source-files.ts";
20
+
21
+ const DEFAULT_SOURCE_EXTENSIONS = ["ts", "tsx", "js", "jsx"];
22
+ const DEFAULT_IGNORE_DIRS = [".git", ".next", "node_modules", "dist", "build", "coverage"];
23
+ const DEFAULT_IGNORE_FILES = ["EVAL.ts", "PROMPT.md"];
24
+
25
+ // e2b 默认用户 "user",home 在 /home/user;工作区放其下。
26
+ const E2B_WORKDIR = "/home/user/workspace";
27
+
28
+ // 单条命令默认超时(10 分钟),防止长跑的 build/install 被截断。
29
+ const DEFAULT_COMMAND_TIMEOUT_MS = 600_000;
30
+ // 沙箱存活上限(到点 e2b 自动回收)。给足空间跑完 setup + agent + 测试脚本。
31
+ const SESSION_TIMEOUT_MS = 1_800_000;
32
+
33
+ /** 单引号包裹 + 转义,把一个参数安全嵌进 shell 命令串。 */
34
+ function shellQuote(s: string): string {
35
+ return `'${s.replace(/'/g, "'\\''")}'`;
36
+ }
37
+
38
+ export class E2BSandbox implements Sandbox {
39
+ readonly otlpHost = null;
40
+ private sbx: E2BSdkSandbox;
41
+ private commandTimeoutMs: number;
42
+ readonly sandboxId: string;
43
+
44
+ private constructor(sbx: E2BSdkSandbox, id: string, commandTimeoutMs: number) {
45
+ this.sbx = sbx;
46
+ this.sandboxId = id;
47
+ this.commandTimeoutMs = commandTimeoutMs;
48
+ }
49
+
50
+ static async create(
51
+ opts: { timeout?: number; runtime?: "node20" | "node24"; template?: string } = {},
52
+ ): Promise<E2BSandbox> {
53
+ const commandTimeoutMs = opts.timeout ?? DEFAULT_COMMAND_TIMEOUT_MS;
54
+ // e2b 的 node 版本由模板决定,runtime 仅作记录(不在创建时选)。
55
+ const apiKey = process.env.E2B_API_KEY;
56
+ const sdkOpts = { apiKey, timeoutMs: SESSION_TIMEOUT_MS } as const;
57
+ // 有 template 就从模板起,否则用 e2b 默认 "base"。
58
+ const sbx = opts.template
59
+ ? await E2BSdkSandbox.create(opts.template, sdkOpts)
60
+ : await E2BSdkSandbox.create(sdkOpts);
61
+ // 备好工作区目录(模板默认 cwd 是 home,workspace 子目录可能不存在)。
62
+ await sbx.commands.run(`mkdir -p ${E2B_WORKDIR}`);
63
+ return new E2BSandbox(sbx, sbx.sandboxId, commandTimeoutMs);
64
+ }
65
+
66
+ async runCommand(cmd: string, args: string[] = [], opts: CommandOptions = {}): Promise<CommandResult> {
67
+ const line = [cmd, ...args.map(shellQuote)].join(" ");
68
+ return this.runShell(line, opts);
69
+ }
70
+
71
+ async runShell(script: string, opts: CommandOptions = {}): Promise<CommandResult> {
72
+ // e2b commands.run 经 bash 执行 → 支持 && / 管道 / $()。root 用户映射到 { user: "root" },
73
+ // 否则用模板默认(非 root)用户 —— 跨后端语义一致(见 types.ts 的 CommandOptions.root)。
74
+ try {
75
+ const res = await this.sbx.commands.run(script, {
76
+ cwd: opts.cwd ?? E2B_WORKDIR,
77
+ envs: opts.env,
78
+ user: opts.root ? "root" : undefined,
79
+ timeoutMs: this.commandTimeoutMs,
80
+ });
81
+ return { stdout: res.stdout, stderr: res.stderr, exitCode: res.exitCode };
82
+ } catch (e) {
83
+ // e2b 在退出码非 0 时【抛】CommandExitError;但 Sandbox 契约要求【返回】带 exitCode 的结果
84
+ // (与 docker / vercel 一致)——否则 agent 命令 / build / 测试一旦非 0 退出就会炸,而不是被判分。
85
+ if (e instanceof CommandExitError) {
86
+ return { stdout: e.stdout, stderr: e.stderr, exitCode: e.exitCode };
87
+ }
88
+ throw e;
89
+ }
90
+ }
91
+
92
+ private abs(path: string): string {
93
+ return path.startsWith("/") ? path : `${E2B_WORKDIR}/${path}`;
94
+ }
95
+
96
+ private targetPath(path: string, targetDir: string): string {
97
+ return path.startsWith("/") ? path : `${targetDir.replace(/\/$/, "")}/${path}`;
98
+ }
99
+
100
+ async readFile(path: string): Promise<string> {
101
+ return this.sbx.files.read(this.abs(path), { format: "text" });
102
+ }
103
+
104
+ async fileExists(path: string): Promise<boolean> {
105
+ try {
106
+ await this.sbx.files.read(this.abs(path), { format: "bytes" });
107
+ return true;
108
+ } catch {
109
+ return false;
110
+ }
111
+ }
112
+
113
+ async readSourceFiles(opts: ReadSourceFilesOptions = {}): Promise<SourceFiles> {
114
+ const extensions = opts.extensions ?? DEFAULT_SOURCE_EXTENSIONS;
115
+ const ignoreDirs = opts.ignoreDirs ?? DEFAULT_IGNORE_DIRS;
116
+ const ignoreFiles = new Set(opts.ignoreFiles ?? DEFAULT_IGNORE_FILES);
117
+
118
+ // 一次 find 列路径,再逐文件 files.read —— 与 vercel 后端同形。
119
+ const dirPrune = ignoreDirs.map((d) => `-name '${d}'`).join(" -o ");
120
+ const nameTests = extensions.map((e) => `-name '*.${e}'`).join(" -o ");
121
+ const listScript = `find . \\( -type d \\( ${dirPrune} \\) \\) -prune -o -type f \\( ${nameTests} \\) -print`;
122
+ const result = await this.runShell(listScript);
123
+
124
+ const paths = result.stdout
125
+ .trim()
126
+ .split("\n")
127
+ .map((p) => p.trim().replace(/^\.\//, ""))
128
+ .filter((p) => p && !ignoreFiles.has(p.split("/").at(-1) ?? ""));
129
+
130
+ const files: { path: string; content: string }[] = [];
131
+ await Promise.all(
132
+ paths.map(async (path) => {
133
+ try {
134
+ const content = await this.sbx.files.read(`${E2B_WORKDIR}/${path}`, { format: "text" });
135
+ files.push({ path, content });
136
+ } catch {
137
+ // skip unreadable files (binary, permissions, etc.)
138
+ }
139
+ }),
140
+ );
141
+ return makeSourceFiles(files);
142
+ }
143
+
144
+ async writeFiles(files: Record<string, string>, targetDir = E2B_WORKDIR): Promise<void> {
145
+ const entries = Object.entries(files).map(([p, data]) => ({ path: this.targetPath(p, targetDir), data }));
146
+ if (entries.length === 0) return;
147
+ await this.sbx.files.write(entries);
148
+ }
149
+
150
+ async uploadFiles(files: SandboxFile[], targetDir = E2B_WORKDIR): Promise<void> {
151
+ if (files.length === 0) return;
152
+ await this.sbx.files.write(
153
+ files.map((f) => ({
154
+ path: this.targetPath(f.path, targetDir),
155
+ data: Buffer.isBuffer(f.content) ? toArrayBuffer(f.content) : f.content,
156
+ })),
157
+ );
158
+ }
159
+
160
+ async uploadDirectory(localDir: string, targetDir: string, opts: { ignore?: string[] } = {}): Promise<void> {
161
+ await this.uploadFiles(await collectLocalFiles(localDir, opts.ignore), targetDir);
162
+ }
163
+
164
+ async stop(): Promise<void> {
165
+ await this.sbx.kill();
166
+ }
167
+
168
+ async downloadFile(path: string): Promise<Buffer> {
169
+ const bytes = await this.sbx.files.read(this.abs(path), { format: "bytes" });
170
+ return Buffer.from(bytes);
171
+ }
172
+
173
+ async uploadFile(path: string, content: Buffer): Promise<void> {
174
+ await this.sbx.files.write(this.abs(path), toArrayBuffer(content));
175
+ }
176
+ }
177
+
178
+ async function collectLocalFiles(localDir: string, ignore: readonly string[] = []): Promise<SandboxFile[]> {
179
+ const ignored = new Set(ignore);
180
+ const out: SandboxFile[] = [];
181
+ async function walk(dir: string): Promise<void> {
182
+ for (const entry of await readdir(dir)) {
183
+ if (ignored.has(entry)) continue;
184
+ const abs = join(dir, entry);
185
+ const st = await stat(abs);
186
+ if (st.isDirectory()) {
187
+ await walk(abs);
188
+ } else if (st.isFile()) {
189
+ out.push({
190
+ path: relative(localDir, abs).split(sep).join("/"),
191
+ content: await readFile(abs),
192
+ });
193
+ }
194
+ }
195
+ }
196
+ await walk(localDir);
197
+ return out;
198
+ }
199
+
200
+ /** Buffer → ArrayBuffer(e2b files.write 接受 string | ArrayBuffer | Blob | ReadableStream)。 */
201
+ function toArrayBuffer(buf: Buffer): ArrayBuffer {
202
+ return buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength) as ArrayBuffer;
203
+ }
@@ -0,0 +1,25 @@
1
+ // niceeval/sandbox 公开导出:「在哪里跑」相关的类型 + 工厂 + 扩展点。
2
+ // 具体后端实现类(DockerSandbox / VercelSandbox / E2BSandbox)是内部实现细节,不在此导出——
3
+ // 需要自定义后端时用 defineSandbox(),不需要绕开 resolve.ts 直接 new 内置类。
4
+
5
+ export { dockerSandbox, vercelSandbox, e2bSandbox, defineSandbox } from "../define.ts";
6
+ export { createCheckpoint, restoreCheckpoint } from "./checkpoint.ts";
7
+
8
+ export type {
9
+ Sandbox,
10
+ SandboxHandle,
11
+ SandboxFile,
12
+ SourceFile,
13
+ SourceFiles,
14
+ ReadSourceFilesOptions,
15
+ SandboxBackend,
16
+ SandboxOption,
17
+ SandboxSpec,
18
+ SandboxRuntime,
19
+ DockerSandboxSpec,
20
+ VercelSandboxSpec,
21
+ E2BSandboxSpec,
22
+ CustomSandboxSpec,
23
+ CommandResult,
24
+ CommandOptions,
25
+ } from "../types.ts";
@@ -0,0 +1,60 @@
1
+ // 活动沙箱登记表 + 强制清理兜底。
2
+ //
3
+ // 中断(Ctrl+C)时,正常路径靠 Effect 的 Scope finalizer 跑 sb.stop() 停容器。但 finalizer
4
+ // 这条路不是 100% 可靠:vsb.stop() 这类远端调用可能慢/挂,用户等不及再按一次 Ctrl+C,进程
5
+ // 就被硬退了 —— 沙箱成了孤儿(只能等后端 session/TTL 过期)。
6
+ //
7
+ // 这里维护一份独立于 Effect 的登记表,让 cli 在「二次中断 / graceful 清理超时 / 正常返回后」
8
+ // 都能直接、带超时地强停所有还活着的沙箱。stop 不再静默吞异常(原 `.catch(() => {})`),失败打到
9
+ // stderr,这样孤儿至少留下痕迹可查。
10
+
11
+ import type { Sandbox } from "../types.ts";
12
+ import { t } from "../i18n/index.ts";
13
+
14
+ const live = new Set<Sandbox>();
15
+
16
+ // 单个 stop 的默认超时:vsb.stop() 偶发慢/挂,清理不能无限等。到点就放弃、记一笔,
17
+ // 让流程继续走到退出 —— 没停掉的沙箱靠后端 session/TTL 兜底过期。
18
+ const DEFAULT_STOP_TIMEOUT_MS = 8_000;
19
+
20
+ export function registerSandbox(sb: Sandbox): void {
21
+ live.add(sb);
22
+ }
23
+
24
+ export function liveSandboxCount(): number {
25
+ return live.size;
26
+ }
27
+
28
+ /**
29
+ * 带超时地停单个沙箱。成功 / 失败 / 超时都从登记表移除(失败的靠后端过期兜底),
30
+ * 异常打到 stderr 不再静默吞。供 Scope finalizer 与兜底强清共用,避免重复实现 stop 语义。
31
+ */
32
+ export async function stopSandbox(sb: Sandbox, timeoutMs = DEFAULT_STOP_TIMEOUT_MS): Promise<void> {
33
+ let timer: ReturnType<typeof setTimeout> | undefined;
34
+ try {
35
+ await Promise.race([
36
+ sb.stop(),
37
+ new Promise<never>((_, reject) => {
38
+ timer = setTimeout(() => reject(new Error(t("sandbox.stopTimeout", { timeoutMs }))), timeoutMs);
39
+ }),
40
+ ]);
41
+ } catch (e) {
42
+ const msg = e instanceof Error ? e.message : String(e);
43
+ process.stderr.write(t("sandbox.stopFailed", { id: sb.sandboxId, message: msg }));
44
+ } finally {
45
+ if (timer) clearTimeout(timer);
46
+ live.delete(sb);
47
+ }
48
+ }
49
+
50
+ /**
51
+ * 兜底强清:并发停掉所有还登记着的沙箱(各自带超时、各自兜错,绝不抛)。返回尝试停的数量。
52
+ * cli 在二次中断 / graceful 清理超时 / 正常返回后调用 —— 正常跑完时登记表已空,是个 no-op。
53
+ */
54
+ export async function stopAllSandboxes(timeoutMs = DEFAULT_STOP_TIMEOUT_MS): Promise<number> {
55
+ const all = [...live];
56
+ if (all.length === 0) return 0;
57
+ process.stderr.write(t("sandbox.forceCleanup", { count: all.length }));
58
+ await Promise.allSettled(all.map((sb) => stopSandbox(sb, timeoutMs)));
59
+ return all.length;
60
+ }
@@ -0,0 +1,131 @@
1
+ // 沙箱后端解析:把 --sandbox / config.sandbox / experiment.sandbox(字符串后端名 或
2
+ // spec 数据结构)折叠成一个具体后端 + 参数,并按需创建实例。
3
+ // 后端名的行为分支只允许出现在 sandbox/ 内(见 docs/architecture.md)。
4
+
5
+ import { Effect } from "effect";
6
+ import type { CustomSandboxSpec, Sandbox, SandboxBackend, SandboxOption, SandboxRuntime } from "../types.ts";
7
+ import { registerSandbox, stopSandbox } from "./registry.ts";
8
+ import { t } from "../i18n/index.ts";
9
+
10
+ /** 归一化后的沙箱描述:确定的后端 + 各后端参数(只有对应后端用得上的会有值)。 */
11
+ export interface ResolvedSandbox {
12
+ backend: string;
13
+ runtime?: SandboxRuntime;
14
+ /** docker */
15
+ image?: string;
16
+ /** vercel */
17
+ snapshotId?: string;
18
+ /** e2b */
19
+ template?: string;
20
+ /** 自定义后端(defineSandbox):有它就直接调用,跳过下面的内置 backend switch。 */
21
+ create?: CustomSandboxSpec["create"];
22
+ recommendedConcurrency?: number;
23
+ }
24
+
25
+ /**
26
+ * 决定用哪个后端:
27
+ * - 显式指定且非 "auto" → 直接用。
28
+ * - 否则:env 里有 VERCEL_TOKEN / VERCEL_OIDC_TOKEN → "vercel"。
29
+ * - 再否则:env 里有 E2B_API_KEY → "e2b"。
30
+ * - 再否则 → "docker"。
31
+ */
32
+ export function resolveBackend(opts: { backend?: SandboxBackend }): SandboxBackend {
33
+ const { backend } = opts;
34
+ if (backend && backend !== "auto") {
35
+ return backend;
36
+ }
37
+ if (process.env.VERCEL_TOKEN || process.env.VERCEL_OIDC_TOKEN) {
38
+ return "vercel";
39
+ }
40
+ if (process.env.E2B_API_KEY) {
41
+ return "e2b";
42
+ }
43
+ return "docker";
44
+ }
45
+
46
+ /** 把字符串后端名 或 spec 数据结构 归一化成 ResolvedSandbox(spec 的 backend 已是具体值,直接用)。 */
47
+ export function resolveSandbox(opt: SandboxOption | undefined, runtimeDefault?: SandboxRuntime): ResolvedSandbox {
48
+ if (opt && typeof opt === "object") {
49
+ return { ...opt, runtime: opt.runtime ?? runtimeDefault };
50
+ }
51
+ const backend = resolveBackend({ backend: opt });
52
+ if (backend !== "docker" && backend !== "vercel" && backend !== "e2b") {
53
+ throw new Error(t("sandbox.backendNotImplemented", { backend }));
54
+ }
55
+ return { backend, runtime: runtimeDefault };
56
+ }
57
+
58
+ /**
59
+ * 各后端的推荐默认并发数。反映的是后端侧约束(daemon 容量、API quota、session 池大小),
60
+ * 不是用户侧的 agent API 限速——后者由用户通过 --max-concurrency 或 config.maxConcurrency 设置。
61
+ * docker:本地 daemon 创建容器有开销,10 是经验上稳健的上限。
62
+ * e2b:云服务,20 是默认账户并发配额的保守估计。
63
+ * vercel:sandbox session 有严格的并发限制,1 避免 429。
64
+ */
65
+ export function sandboxRecommendedConcurrency(opt: SandboxOption | undefined): number {
66
+ if (!opt) return 10;
67
+ const r = resolveSandbox(opt);
68
+ switch (r.backend) {
69
+ case "docker": return 10;
70
+ case "e2b": return 20;
71
+ case "vercel": return 1;
72
+ default: return r.recommendedConcurrency ?? 5;
73
+ }
74
+ }
75
+
76
+ /** 报告 / 日志用的简短标签:后端名,带上区分性的参数(镜像 / 快照 / 模板)。 */
77
+ export function sandboxLabel(opt: SandboxOption | undefined): string {
78
+ const r = resolveSandbox(opt);
79
+ const detail = r.image ?? r.snapshotId ?? r.template;
80
+ return detail ? `${r.backend}:${detail}` : r.backend;
81
+ }
82
+
83
+ /**
84
+ * 按解析出的后端 + 参数创建沙箱,并把 stop() 注册为 Scope 回收动作。
85
+ * 在 Effect.scoped / Effect.gen 里 yield* 即可;成功/失败/中断都保证 stop。
86
+ */
87
+ export function createSandbox(opts: {
88
+ sandbox?: SandboxOption;
89
+ timeout?: number;
90
+ runtime?: SandboxRuntime;
91
+ }) {
92
+ const r = resolveSandbox(opts.sandbox, opts.runtime);
93
+ return Effect.acquireRelease(
94
+ Effect.promise<Sandbox>(async () => {
95
+ // 起好就登记:让 cli 的兜底强清(二次 Ctrl+C / 看门狗超时)能直接停到它,不只靠下面的
96
+ // release。即便本 fiber 创建后立刻被中断、release 还没来得及跑,登记表也已认得这个沙箱。
97
+ const sb = await createBackend(r, opts.timeout);
98
+ registerSandbox(sb);
99
+ return sb;
100
+ }),
101
+ // release:成功 / 失败 / 中断都跑。带超时 + 失败不静默(stopSandbox 内做),并把它移出登记表。
102
+ (sb) => Effect.promise(() => stopSandbox(sb)),
103
+ );
104
+ }
105
+
106
+ async function createBackend(r: ResolvedSandbox, timeout?: number): Promise<Sandbox> {
107
+ // 自定义后端(defineSandbox):不认后端名,直接调用用户给的 create()。
108
+ if (r.create) return r.create({ timeout, runtime: r.runtime });
109
+ switch (r.backend) {
110
+ case "docker": {
111
+ const { DockerSandbox } = await import("./docker.ts").catch(() => {
112
+ throw new Error(t("sandbox.dependencyMissing.docker"));
113
+ });
114
+ return DockerSandbox.create({ timeout, runtime: r.runtime, image: r.image });
115
+ }
116
+ case "vercel": {
117
+ const { VercelSandbox } = await import("./vercel.ts").catch(() => {
118
+ throw new Error(t("sandbox.dependencyMissing.vercel"));
119
+ });
120
+ return VercelSandbox.create({ timeout, runtime: r.runtime, snapshotId: r.snapshotId });
121
+ }
122
+ case "e2b": {
123
+ const { E2BSandbox } = await import("./e2b.ts").catch(() => {
124
+ throw new Error(t("sandbox.dependencyMissing.e2b"));
125
+ });
126
+ return E2BSandbox.create({ timeout, runtime: r.runtime, template: r.template });
127
+ }
128
+ default:
129
+ throw new Error(t("sandbox.backendNotImplemented", { backend: r.backend }));
130
+ }
131
+ }
@@ -0,0 +1,30 @@
1
+ // 把裸 SourceFile[] 包装成带便利方法的 SourceFiles(仍是真数组)。
2
+ // 方法用 defineProperties 挂成 non-enumerable:.filter/.map/展开/迭代/JSON 都不受影响。
3
+
4
+ import type { SourceFile, SourceFiles } from "../types.ts";
5
+ import { stripComments } from "../util.ts";
6
+
7
+ export function makeSourceFiles(files: SourceFile[]): SourceFiles {
8
+ const arr = files.slice();
9
+ const text = (): string => arr.map((f) => `\n// ${f.path}\n${f.content}`).join("\n");
10
+
11
+ Object.defineProperties(arr, {
12
+ text: { value: text, enumerable: false },
13
+ code: { value: (): string => stripComments(text()), enumerable: false },
14
+ fileMatching: {
15
+ value: (pattern: RegExp): SourceFile | undefined => arr.find((f) => pattern.test(f.content)),
16
+ enumerable: false,
17
+ },
18
+ fileMatchingAll: {
19
+ value: (patterns: RegExp[]): SourceFile | undefined =>
20
+ arr.find((f) => patterns.every((p) => p.test(f.content))),
21
+ enumerable: false,
22
+ },
23
+ hasPath: {
24
+ value: (pattern: RegExp): boolean => arr.some((f) => pattern.test(f.path)),
25
+ enumerable: false,
26
+ },
27
+ });
28
+
29
+ return arr as unknown as SourceFiles;
30
+ }
@@ -0,0 +1,236 @@
1
+ // Vercel Sandbox 后端:用 @vercel/sandbox SDK 把 Vercel microVM 当隔离工作区跑 eval。
2
+ // 契约对齐 ../types.ts 的 Sandbox 接口,与 DockerSandbox 可互换。
3
+
4
+ import { readdir, readFile, stat } from "node:fs/promises";
5
+ import { join, relative, sep } from "node:path";
6
+ import { Sandbox as VSandbox } from "@vercel/sandbox";
7
+ import type {
8
+ Sandbox,
9
+ CommandResult,
10
+ CommandOptions,
11
+ SandboxFile,
12
+ SourceFiles,
13
+ ReadSourceFilesOptions,
14
+ } from "../types.ts";
15
+ import { makeSourceFiles } from "./source-files.ts";
16
+ import { t } from "../i18n/index.ts";
17
+
18
+ const DEFAULT_SOURCE_EXTENSIONS = ["ts", "tsx", "js", "jsx"];
19
+ const DEFAULT_IGNORE_DIRS = [".git", ".next", "node_modules", "dist", "build", "coverage"];
20
+ const DEFAULT_IGNORE_FILES = ["EVAL.ts", "PROMPT.md"];
21
+
22
+ // Vercel Sandbox 的默认工作区路径(SDK writeFiles 默认落这里)。
23
+ const VERCEL_WORKDIR = "/vercel/sandbox";
24
+
25
+ // 单条命令的默认超时:设为沙箱 session 生命周期(10 分钟),防止长时间 build/install 被截断。
26
+ const DEFAULT_COMMAND_TIMEOUT_MS = 600_000;
27
+ // Rotate session when it has been alive >270s to stay under the plan cap (~360-390s).
28
+ const ROTATE_THRESHOLD_MS = 270_000;
29
+ const SESSION_TIMEOUT_MS = 1_200_000;
30
+
31
+ export class VercelSandbox implements Sandbox {
32
+ readonly otlpHost = null;
33
+ private vsb: InstanceType<typeof VSandbox>;
34
+ private commandTimeoutMs: number;
35
+ private sessionCreatedAt: number;
36
+ private runtime: string;
37
+ readonly sandboxId: string;
38
+
39
+ private constructor(vsb: InstanceType<typeof VSandbox>, id: string, commandTimeoutMs: number, runtime: string) {
40
+ this.vsb = vsb;
41
+ this.sandboxId = id;
42
+ this.commandTimeoutMs = commandTimeoutMs;
43
+ this.sessionCreatedAt = Date.now();
44
+ this.runtime = runtime;
45
+ }
46
+
47
+ static async create(
48
+ opts: { timeout?: number; runtime?: "node20" | "node24"; snapshotId?: string } = {},
49
+ ): Promise<VercelSandbox> {
50
+ // Vercel 支持 node22/node24/node26/python3.13;node20 回退到 node22。
51
+ const runtime = opts.runtime === "node20" ? "node22" : (opts.runtime ?? "node24");
52
+ const commandTimeoutMs = opts.timeout ?? DEFAULT_COMMAND_TIMEOUT_MS;
53
+
54
+ // 凭据:优先从 env 显式传入(绕过 OIDC flow,非 TTY 环境也能用)。
55
+ // 需要同时设 VERCEL_API_TOKEN + VERCEL_TEAM_ID + VERCEL_PROJECT_ID 三个。
56
+ const token = process.env.VERCEL_API_TOKEN;
57
+ const teamId = process.env.VERCEL_TEAM_ID;
58
+ const projectId = process.env.VERCEL_PROJECT_ID ?? "vercel-sandbox-default-project";
59
+ const credParams = token && teamId ? { token, teamId, projectId } : {};
60
+
61
+ // 给了 snapshotId 就从快照起 microVM(预制模板:烘焙好 agent CLI 的快照)。
62
+ const sourceParams = opts.snapshotId ? { source: { type: "snapshot", snapshotId: opts.snapshotId } } : {};
63
+
64
+ // session timeout 固定为 1200000ms (20 min)。不随 commandTimeoutMs 放大:
65
+ // 实测发现超大的 timeout 值(>1200s)会导致 Vercel 返回实际更短的 session。
66
+ // 1200000ms 是经验证能跑完 ~355s eval 的上限。
67
+ const vsb = await VSandbox.create({ runtime, timeout: SESSION_TIMEOUT_MS, ...sourceParams, ...credParams } as Parameters<typeof VSandbox.create>[0]);
68
+ const id = vsb.currentSession().sessionId;
69
+ return new VercelSandbox(vsb, id, commandTimeoutMs, runtime);
70
+ }
71
+
72
+ // 当 session 存活超过 ROTATE_THRESHOLD_MS 时,拍快照并换用新 session。
73
+ // 这绕过了 Vercel plan 对 extendTimeout 的限制(始终返回 HTTP 400)。
74
+ private async rotateIfNeeded(): Promise<void> {
75
+ const elapsed = Date.now() - this.sessionCreatedAt;
76
+ if (elapsed < ROTATE_THRESHOLD_MS) return;
77
+
78
+ const token = process.env.VERCEL_API_TOKEN;
79
+ const teamId = process.env.VERCEL_TEAM_ID;
80
+ const projectId = process.env.VERCEL_PROJECT_ID ?? "vercel-sandbox-default-project";
81
+ const credParams = token && teamId ? { token, teamId, projectId } : {};
82
+
83
+ try {
84
+ const snap = await this.vsb.snapshot();
85
+ const snapshotId = snap.snapshotId;
86
+ const newVsb = await VSandbox.create({
87
+ runtime: this.runtime,
88
+ timeout: SESSION_TIMEOUT_MS,
89
+ source: { type: "snapshot", snapshotId },
90
+ ...credParams,
91
+ } as Parameters<typeof VSandbox.create>[0]);
92
+ this.vsb = newVsb;
93
+ this.sessionCreatedAt = Date.now();
94
+ console.error(t("vercel.rotated", {
95
+ seconds: Math.round(elapsed / 1000),
96
+ sessionId: newVsb.currentSession().sessionId,
97
+ }));
98
+ } catch (err) {
99
+ console.error(t("vercel.rotateFailed", {
100
+ seconds: Math.round(elapsed / 1000),
101
+ error: String(err),
102
+ }));
103
+ }
104
+ }
105
+
106
+ async runCommand(cmd: string, args: string[] = [], opts: CommandOptions = {}): Promise<CommandResult> {
107
+ await this.rotateIfNeeded();
108
+ const finished = await this.vsb.runCommand({
109
+ cmd,
110
+ args,
111
+ cwd: opts.cwd ?? VERCEL_WORKDIR,
112
+ env: opts.env,
113
+ sudo: opts.root ?? false,
114
+ // 显式设 per-command timeout 防止长跑命令(npm build/install)被流截断。
115
+ timeoutMs: this.commandTimeoutMs,
116
+ });
117
+ return {
118
+ stdout: await finished.stdout(),
119
+ stderr: await finished.stderr(),
120
+ exitCode: finished.exitCode,
121
+ };
122
+ }
123
+
124
+ async runShell(script: string, opts: CommandOptions = {}): Promise<CommandResult> {
125
+ return this.runCommand("bash", ["-c", script], opts);
126
+ }
127
+
128
+ async readFile(path: string): Promise<string> {
129
+ const absPath = path.startsWith("/") ? path : `${VERCEL_WORKDIR}/${path}`;
130
+ const buf = await this.vsb.readFileToBuffer({ path: absPath });
131
+ if (!buf) throw new Error(t("vercel.fileNotFound", { path: absPath }));
132
+ return buf.toString("utf8");
133
+ }
134
+
135
+ async fileExists(path: string): Promise<boolean> {
136
+ const absPath = path.startsWith("/") ? path : `${VERCEL_WORKDIR}/${path}`;
137
+ const buf = await this.vsb.readFileToBuffer({ path: absPath });
138
+ return buf !== null;
139
+ }
140
+
141
+ async readSourceFiles(opts: ReadSourceFilesOptions = {}): Promise<SourceFiles> {
142
+ const extensions = opts.extensions ?? DEFAULT_SOURCE_EXTENSIONS;
143
+ const ignoreDirs = opts.ignoreDirs ?? DEFAULT_IGNORE_DIRS;
144
+ const ignoreFiles = new Set(opts.ignoreFiles ?? DEFAULT_IGNORE_FILES);
145
+
146
+ // 两阶段读取:Phase 1 只做 find(列路径,NDJSON 流短命令快速结束);
147
+ // Phase 2 逐文件用 readFileToBuffer(独立 HTTP GET,不依赖 NDJSON 流)。
148
+ // 这样即使 session 快到 plan 上限,后半段读取也不会被截断。
149
+ const dirPrune = ignoreDirs.map((d) => `-name '${d}'`).join(" -o ");
150
+ const nameTests = extensions.map((e) => `-name '*.${e}'`).join(" -o ");
151
+ const listScript = `find . \\( -type d \\( ${dirPrune} \\) \\) -prune -o -type f \\( ${nameTests} \\) -print`;
152
+ const result = await this.runShell(listScript);
153
+
154
+ const paths = result.stdout
155
+ .trim()
156
+ .split("\n")
157
+ .map((p) => p.trim().replace(/^\.\//, ""))
158
+ .filter((p) => p && !ignoreFiles.has(p.split("/").at(-1) ?? ""));
159
+
160
+ const files: { path: string; content: string }[] = [];
161
+ await Promise.all(
162
+ paths.map(async (path) => {
163
+ const absPath = `${VERCEL_WORKDIR}/${path}`;
164
+ try {
165
+ const buf = await this.vsb.readFileToBuffer({ path: absPath });
166
+ if (buf) files.push({ path, content: buf.toString("utf8") });
167
+ } catch {
168
+ // skip unreadable files (binary, permissions, etc.)
169
+ }
170
+ }),
171
+ );
172
+ return makeSourceFiles(files);
173
+ }
174
+
175
+ async writeFiles(files: Record<string, string>, targetDir = VERCEL_WORKDIR): Promise<void> {
176
+ const entries = Object.entries(files).map(([p, content]) => ({
177
+ // 相对路径 → Vercel SDK 自动 prepend /vercel/sandbox;绝对路径原样传。
178
+ path: p.startsWith("/") ? p : `${targetDir.replace(/\/$/, "")}/${p}`,
179
+ content,
180
+ }));
181
+ if (entries.length === 0) return;
182
+ await this.vsb.writeFiles(entries);
183
+ }
184
+
185
+ async uploadFiles(files: SandboxFile[], targetDir = VERCEL_WORKDIR): Promise<void> {
186
+ if (files.length === 0) return;
187
+ await this.vsb.writeFiles(
188
+ files.map((f) => ({
189
+ path: f.path.startsWith("/") ? f.path : `${targetDir.replace(/\/$/, "")}/${f.path}`,
190
+ content: f.content,
191
+ })),
192
+ );
193
+ }
194
+
195
+ async uploadDirectory(localDir: string, targetDir: string, opts: { ignore?: string[] } = {}): Promise<void> {
196
+ await this.uploadFiles(await collectLocalFiles(localDir, opts.ignore), targetDir);
197
+ }
198
+
199
+ async stop(): Promise<void> {
200
+ await this.vsb.stop();
201
+ }
202
+
203
+ async downloadFile(path: string): Promise<Buffer> {
204
+ const absPath = path.startsWith("/") ? path : `${VERCEL_WORKDIR}/${path}`;
205
+ const buf = await this.vsb.readFileToBuffer({ path: absPath });
206
+ if (!buf) throw new Error(t("vercel.fileNotFound", { path: absPath }));
207
+ return buf;
208
+ }
209
+
210
+ async uploadFile(path: string, content: Buffer): Promise<void> {
211
+ const absPath = path.startsWith("/") ? path : `${VERCEL_WORKDIR}/${path}`;
212
+ await this.vsb.writeFiles([{ path: absPath, content }]);
213
+ }
214
+ }
215
+
216
+ async function collectLocalFiles(localDir: string, ignore: readonly string[] = []): Promise<SandboxFile[]> {
217
+ const ignored = new Set(ignore);
218
+ const out: SandboxFile[] = [];
219
+ async function walk(dir: string): Promise<void> {
220
+ for (const entry of await readdir(dir)) {
221
+ if (ignored.has(entry)) continue;
222
+ const abs = join(dir, entry);
223
+ const st = await stat(abs);
224
+ if (st.isDirectory()) {
225
+ await walk(abs);
226
+ } else if (st.isFile()) {
227
+ out.push({
228
+ path: relative(localDir, abs).split(sep).join("/"),
229
+ content: await readFile(abs),
230
+ });
231
+ }
232
+ }
233
+ }
234
+ await walk(localDir);
235
+ return out;
236
+ }