@sandboxrouter/sdk 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +260 -2
- package/dist/bench-store.d.ts +12 -0
- package/dist/bench-store.js +23 -0
- package/dist/bench.d.ts +24 -0
- package/dist/bench.js +65 -0
- package/dist/catalog.d.ts +35 -0
- package/dist/catalog.js +198 -0
- package/dist/cli.d.ts +2 -0
- package/dist/cli.js +195 -0
- package/dist/computers.d.ts +124 -0
- package/dist/computers.js +66 -0
- package/dist/index.d.ts +17 -0
- package/dist/index.js +17 -0
- package/dist/leaderboard.d.ts +39 -0
- package/dist/leaderboard.js +184 -0
- package/dist/providers/blaxel.d.ts +21 -0
- package/dist/providers/blaxel.js +152 -0
- package/dist/providers/daytona.d.ts +13 -0
- package/dist/providers/daytona.js +121 -0
- package/dist/providers/e2b.d.ts +10 -0
- package/dist/providers/e2b.js +69 -0
- package/dist/providers/maritime.d.ts +38 -0
- package/dist/providers/maritime.js +240 -0
- package/dist/providers/modal.d.ts +10 -0
- package/dist/providers/modal.js +82 -0
- package/dist/providers/morph.d.ts +19 -0
- package/dist/providers/morph.js +181 -0
- package/dist/providers/runloop.d.ts +16 -0
- package/dist/providers/runloop.js +145 -0
- package/dist/providers/sprites.d.ts +15 -0
- package/dist/providers/sprites.js +135 -0
- package/dist/providers/tensorlake.d.ts +14 -0
- package/dist/providers/tensorlake.js +192 -0
- package/dist/providers/vercel.d.ts +12 -0
- package/dist/providers/vercel.js +87 -0
- package/dist/router.d.ts +65 -0
- package/dist/router.js +180 -0
- package/dist/server.d.ts +31 -0
- package/dist/server.js +257 -0
- package/dist/sse.d.ts +6 -0
- package/dist/sse.js +33 -0
- package/dist/types.d.ts +83 -0
- package/dist/types.js +19 -0
- package/dist/util.d.ts +11 -0
- package/dist/util.js +37 -0
- package/package.json +113 -4
package/dist/cli.js
ADDED
|
@@ -0,0 +1,195 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import { parseArgs } from "node:util";
|
|
3
|
+
import { readFileSync, writeFileSync } from "node:fs";
|
|
4
|
+
import { bench, benchMarkdown } from "./bench.js";
|
|
5
|
+
import { loadReports, makeReport, renderLeaderboard } from "./leaderboard.js";
|
|
6
|
+
import { CATALOG, hourlyCost } from "./catalog.js";
|
|
7
|
+
import { Router } from "./router.js";
|
|
8
|
+
import { createApp } from "./server.js";
|
|
9
|
+
import { RouterError } from "./types.js";
|
|
10
|
+
const HELP = `sandboxrouter: one API for every sandbox provider
|
|
11
|
+
|
|
12
|
+
Usage:
|
|
13
|
+
sandboxrouter providers [--vcpu N] [--memory GB] price table + which keys are set
|
|
14
|
+
sandboxrouter route [spec flags] [route flags] dry run: the order providers would be tried
|
|
15
|
+
sandboxrouter run "<cmd>" [spec flags] [route flags] [--keep]
|
|
16
|
+
sandboxrouter bench [--runs 3] [--provider a,b] [--json] [--share [--label "where you ran it"]]
|
|
17
|
+
sandboxrouter leaderboard [results-dir] [--out leaderboard.html]
|
|
18
|
+
sandboxrouter desktop [--os linux|windows|android] [--user ID] [--control]
|
|
19
|
+
get or create a computer and print a link to watch (or take over) its screen
|
|
20
|
+
sandboxrouter screenshot <provider:id> [--out screen.png]
|
|
21
|
+
sandboxrouter serve [--port 8787] [--host 127.0.0.1] [--cors https://site.example,http://localhost:3000]
|
|
22
|
+
|
|
23
|
+
Spec flags: --vcpu N --memory GB --gpu T4 --timeout MIN
|
|
24
|
+
Route flags: --provider maritime[,e2b,...] --strategy default|cheapest|fastest
|
|
25
|
+
--no-fallback --require gpu,custom-image
|
|
26
|
+
|
|
27
|
+
Keys: MARITIME_API_KEY, E2B_API_KEY, DAYTONA_API_KEY, MODAL_TOKEN_ID+MODAL_TOKEN_SECRET,
|
|
28
|
+
VERCEL_OIDC_TOKEN (or VERCEL_TOKEN+VERCEL_TEAM_ID+VERCEL_PROJECT_ID)`;
|
|
29
|
+
const { values, positionals } = parseArgs({
|
|
30
|
+
allowPositionals: true,
|
|
31
|
+
options: {
|
|
32
|
+
vcpu: { type: "string" },
|
|
33
|
+
memory: { type: "string" },
|
|
34
|
+
gpu: { type: "string" },
|
|
35
|
+
timeout: { type: "string" },
|
|
36
|
+
provider: { type: "string" },
|
|
37
|
+
strategy: { type: "string" },
|
|
38
|
+
"no-fallback": { type: "boolean" },
|
|
39
|
+
require: { type: "string" },
|
|
40
|
+
keep: { type: "boolean" },
|
|
41
|
+
runs: { type: "string" },
|
|
42
|
+
json: { type: "boolean" },
|
|
43
|
+
port: { type: "string" },
|
|
44
|
+
host: { type: "string" },
|
|
45
|
+
help: { type: "boolean", short: "h" },
|
|
46
|
+
os: { type: "string" },
|
|
47
|
+
user: { type: "string" },
|
|
48
|
+
control: { type: "boolean" },
|
|
49
|
+
out: { type: "string" },
|
|
50
|
+
share: { type: "boolean" },
|
|
51
|
+
cors: { type: "string" },
|
|
52
|
+
label: { type: "string" },
|
|
53
|
+
},
|
|
54
|
+
});
|
|
55
|
+
const list = (s) => (s ? s.split(",").map((x) => x.trim()).filter(Boolean) : undefined);
|
|
56
|
+
const num = (s) => (s === undefined ? undefined : Number(s));
|
|
57
|
+
const spec = {
|
|
58
|
+
vcpu: num(values.vcpu),
|
|
59
|
+
memoryGb: num(values.memory),
|
|
60
|
+
gpu: values.gpu,
|
|
61
|
+
timeoutMinutes: num(values.timeout),
|
|
62
|
+
};
|
|
63
|
+
const providers = list(values.provider);
|
|
64
|
+
const route = {
|
|
65
|
+
provider: providers,
|
|
66
|
+
strategy: values.strategy,
|
|
67
|
+
fallback: values["no-fallback"] ? false : undefined,
|
|
68
|
+
require: list(values.require),
|
|
69
|
+
};
|
|
70
|
+
async function main() {
|
|
71
|
+
const [cmd, ...rest] = positionals;
|
|
72
|
+
const router = new Router();
|
|
73
|
+
if (!cmd || values.help)
|
|
74
|
+
return console.log(HELP);
|
|
75
|
+
if (cmd === "providers") {
|
|
76
|
+
const rows = CATALOG.map((c) => ({
|
|
77
|
+
provider: c.name,
|
|
78
|
+
id: c.id,
|
|
79
|
+
"$/hr": `$${hourlyCost(c.id, spec).toFixed(4)}`,
|
|
80
|
+
configured: router.isConfigured(c.id) ? "yes" : "no",
|
|
81
|
+
capabilities: c.capabilities.join(", ") || "-",
|
|
82
|
+
}));
|
|
83
|
+
if (values.json)
|
|
84
|
+
return console.log(JSON.stringify(rows, null, 2));
|
|
85
|
+
console.log(`Estimated list price for ${spec.vcpu ?? 2} vCPU, ${spec.memoryGb ?? 4} GB while running:\n`);
|
|
86
|
+
console.table(rows);
|
|
87
|
+
return;
|
|
88
|
+
}
|
|
89
|
+
if (cmd === "route") {
|
|
90
|
+
const plan = router.plan(spec, route);
|
|
91
|
+
if (values.json)
|
|
92
|
+
return console.log(JSON.stringify(plan, null, 2));
|
|
93
|
+
let n = 0;
|
|
94
|
+
for (const c of plan) {
|
|
95
|
+
const mark = c.skip ? " -" : `${++n}.`.padStart(3);
|
|
96
|
+
console.log(`${mark} ${c.provider.padEnd(11)} $${c.estimatedHourlyUsd.toFixed(4)}/hr${c.skip ? ` (skipped: ${c.skip})` : ""}`);
|
|
97
|
+
}
|
|
98
|
+
return;
|
|
99
|
+
}
|
|
100
|
+
if (cmd === "run") {
|
|
101
|
+
const command = rest.join(" ");
|
|
102
|
+
if (!command)
|
|
103
|
+
throw new RouterError('usage: sandboxrouter run "<cmd>"');
|
|
104
|
+
const sbx = await router.create(spec, route);
|
|
105
|
+
for (const a of sbx.attempts.filter((a) => !a.ok))
|
|
106
|
+
console.error(`! ${a.provider} failed, fell back: ${a.error}`);
|
|
107
|
+
console.error(`> ${sbx.id} on ${sbx.provider} (~$${sbx.estimatedHourlyUsd}/hr, started in ${sbx.attempts.at(-1)?.ms} ms)`);
|
|
108
|
+
try {
|
|
109
|
+
const r = await sbx.exec(command, {
|
|
110
|
+
timeoutSeconds: 600,
|
|
111
|
+
onStdout: (c) => process.stdout.write(c),
|
|
112
|
+
onStderr: (c) => process.stderr.write(c),
|
|
113
|
+
});
|
|
114
|
+
process.exitCode = r.exitCode;
|
|
115
|
+
}
|
|
116
|
+
finally {
|
|
117
|
+
if (values.keep)
|
|
118
|
+
console.error(`> kept alive: ${sbx.id}`);
|
|
119
|
+
else
|
|
120
|
+
await sbx.kill();
|
|
121
|
+
}
|
|
122
|
+
return;
|
|
123
|
+
}
|
|
124
|
+
if (cmd === "bench") {
|
|
125
|
+
const rows = await bench({
|
|
126
|
+
router,
|
|
127
|
+
providers,
|
|
128
|
+
runs: num(values.runs) ?? 3,
|
|
129
|
+
spec: { ...spec, timeoutMinutes: spec.timeoutMinutes ?? 5 },
|
|
130
|
+
onProgress: (m) => console.error(m),
|
|
131
|
+
});
|
|
132
|
+
if (rows.length === 0)
|
|
133
|
+
return console.error("No providers configured. Set at least one API key (see --help).");
|
|
134
|
+
if (values.json)
|
|
135
|
+
return console.log(JSON.stringify(rows, null, 2));
|
|
136
|
+
console.log(`\nsandboxrouter bench: ${new Date().toISOString().slice(0, 10)}, ${rows[0].runs} runs each\n`);
|
|
137
|
+
console.log(benchMarkdown(rows));
|
|
138
|
+
for (const r of rows.filter((r) => r.lastError))
|
|
139
|
+
console.error(`\n${r.provider} last error: ${r.lastError}`);
|
|
140
|
+
if (values.share) {
|
|
141
|
+
const version = JSON.parse(readFileSync(new URL("../package.json", import.meta.url), "utf8")).version;
|
|
142
|
+
const report = makeReport(rows, { version, runs: rows[0].runs, spec, label: values.label });
|
|
143
|
+
const file = values.out ?? `sandboxrouter-bench-${report.date.slice(0, 10)}-${Math.random().toString(36).slice(2, 8)}.json`;
|
|
144
|
+
writeFileSync(file, JSON.stringify(report, null, 2) + "\n");
|
|
145
|
+
console.log(`\nwrote ${file}. Add it to results/ in a PR to github.com/gigialc/sandboxrouter to put it on the leaderboard.`);
|
|
146
|
+
}
|
|
147
|
+
return;
|
|
148
|
+
}
|
|
149
|
+
if (cmd === "leaderboard") {
|
|
150
|
+
const dir = rest[0] ?? "results";
|
|
151
|
+
const reports = loadReports(dir);
|
|
152
|
+
const out = values.out ?? "leaderboard.html";
|
|
153
|
+
writeFileSync(out, renderLeaderboard(reports));
|
|
154
|
+
console.log(`wrote ${out} from ${reports.length} report${reports.length === 1 ? "" : "s"} in ${dir}/`);
|
|
155
|
+
return;
|
|
156
|
+
}
|
|
157
|
+
if (cmd === "desktop") {
|
|
158
|
+
const os = (values.os ?? "linux");
|
|
159
|
+
const c = await router.computer({ os, externalUserId: values.user }, { provider: providers });
|
|
160
|
+
const info = await c.info();
|
|
161
|
+
const link = await c.viewer(values.control ? "control" : "watch");
|
|
162
|
+
console.log(`${c.id} ${info.os} ${info.status}${info.screen ? ` ${info.screen.width}x${info.screen.height}` : ""}`);
|
|
163
|
+
console.log(`${values.control ? "take over" : "watch"}: ${link.url} (expires ${link.expiresAt})`);
|
|
164
|
+
return;
|
|
165
|
+
}
|
|
166
|
+
if (cmd === "screenshot") {
|
|
167
|
+
const id = rest[0];
|
|
168
|
+
if (!id)
|
|
169
|
+
throw new RouterError("usage: sandboxrouter screenshot <provider:id> [--out screen.png]");
|
|
170
|
+
const img = await (await router.getComputer(id)).screenshot();
|
|
171
|
+
const out = values.out ?? "screen.png";
|
|
172
|
+
(await import("node:fs")).writeFileSync(out, img);
|
|
173
|
+
console.log(`wrote ${out} (${img.byteLength} bytes)`);
|
|
174
|
+
return;
|
|
175
|
+
}
|
|
176
|
+
if (cmd === "serve") {
|
|
177
|
+
const port = num(values.port) ?? Number(process.env.PORT ?? 8787);
|
|
178
|
+
const host = values.host ?? "127.0.0.1";
|
|
179
|
+
const origins = list(values.cors);
|
|
180
|
+
createApp(router, undefined, origins).listen(port, host, () => {
|
|
181
|
+
console.log(`sandboxrouter listening on http://${host}:${port}`);
|
|
182
|
+
if (origins?.length)
|
|
183
|
+
console.log(`browser access allowed from: ${origins.join(", ")}`);
|
|
184
|
+
if (!process.env.SANDBOXROUTER_TOKEN)
|
|
185
|
+
console.log("warning: SANDBOXROUTER_TOKEN is not set, so the API is unauthenticated");
|
|
186
|
+
});
|
|
187
|
+
return;
|
|
188
|
+
}
|
|
189
|
+
console.error(`unknown command: ${cmd}\n\n${HELP}`);
|
|
190
|
+
process.exitCode = 2;
|
|
191
|
+
}
|
|
192
|
+
main().catch((err) => {
|
|
193
|
+
console.error(`error: ${err.message}`);
|
|
194
|
+
process.exitCode = 1;
|
|
195
|
+
});
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
import { type ProviderId } from "./types.js";
|
|
2
|
+
/**
|
|
3
|
+
* Computers: machines with a screen that a model drives with clicks and keys (computer use).
|
|
4
|
+
* Linux and Windows desktops, Android phones. Maritime is the first implementation.
|
|
5
|
+
*/
|
|
6
|
+
export type ComputerOs = "linux" | "windows" | "android";
|
|
7
|
+
export interface ComputerSpec {
|
|
8
|
+
os?: ComputerOs;
|
|
9
|
+
/** Your user's id. The same id returns the same computer (files and logins kept). */
|
|
10
|
+
externalUserId?: string;
|
|
11
|
+
name?: string;
|
|
12
|
+
/** Linux only, each side 640 to 1920. */
|
|
13
|
+
screen?: {
|
|
14
|
+
width: number;
|
|
15
|
+
height: number;
|
|
16
|
+
};
|
|
17
|
+
/** Android only. */
|
|
18
|
+
arch?: "x86_64" | "arm64";
|
|
19
|
+
}
|
|
20
|
+
export type ComputerActionName = "screenshot" | "left_click" | "right_click" | "middle_click" | "double_click" | "triple_click" | "mouse_move" | "left_click_drag" | "left_mouse_down" | "left_mouse_up" | "scroll" | "type" | "key" | "hold_key" | "wait" | "zoom" | "cursor_position";
|
|
21
|
+
/** The canonical action (Anthropic computer-use shaped). Coordinates are in the last screenshot's frame. */
|
|
22
|
+
export interface ComputerAction {
|
|
23
|
+
action: ComputerActionName;
|
|
24
|
+
coordinate?: [number, number];
|
|
25
|
+
startCoordinate?: [number, number];
|
|
26
|
+
/** `type`: text to type. `key` / `hold_key`: an xdotool key name like "Return" or "ctrl+s". */
|
|
27
|
+
text?: string;
|
|
28
|
+
modifier?: string;
|
|
29
|
+
repeat?: number;
|
|
30
|
+
scrollDirection?: "up" | "down" | "left" | "right";
|
|
31
|
+
scrollAmount?: number;
|
|
32
|
+
duration?: number;
|
|
33
|
+
region?: [number, number, number, number];
|
|
34
|
+
noScreenshot?: boolean;
|
|
35
|
+
}
|
|
36
|
+
export interface ActionResult {
|
|
37
|
+
action?: string;
|
|
38
|
+
ok?: boolean;
|
|
39
|
+
error?: string;
|
|
40
|
+
frameId?: number;
|
|
41
|
+
/** Post-action screenshot, base64. */
|
|
42
|
+
imageB64?: string;
|
|
43
|
+
mime?: string;
|
|
44
|
+
width?: number;
|
|
45
|
+
height?: number;
|
|
46
|
+
[key: string]: unknown;
|
|
47
|
+
}
|
|
48
|
+
export interface ComputerInfo {
|
|
49
|
+
id: string;
|
|
50
|
+
os: ComputerOs;
|
|
51
|
+
status: string;
|
|
52
|
+
screen?: {
|
|
53
|
+
width: number;
|
|
54
|
+
height: number;
|
|
55
|
+
modelWidth?: number;
|
|
56
|
+
};
|
|
57
|
+
externalUserId?: string | null;
|
|
58
|
+
}
|
|
59
|
+
export interface ProviderComputer {
|
|
60
|
+
readonly nativeId: string;
|
|
61
|
+
info(): Promise<ComputerInfo>;
|
|
62
|
+
act(action: ComputerAction): Promise<ActionResult>;
|
|
63
|
+
actBatch(actions: ComputerAction[]): Promise<ActionResult[]>;
|
|
64
|
+
screenshot(opts?: {
|
|
65
|
+
format?: "png" | "jpeg";
|
|
66
|
+
quality?: number;
|
|
67
|
+
}): Promise<Uint8Array>;
|
|
68
|
+
exec(command: string, timeoutSeconds?: number): Promise<{
|
|
69
|
+
exitCode: number;
|
|
70
|
+
stdout: string;
|
|
71
|
+
stderr: string;
|
|
72
|
+
}>;
|
|
73
|
+
readFile(path: string): Promise<Uint8Array>;
|
|
74
|
+
writeFile(path: string, content: string | Uint8Array): Promise<void>;
|
|
75
|
+
/** A link for a person: `watch` is view-only, `control` hands them the desktop (logins, CAPTCHAs). */
|
|
76
|
+
viewer(mode?: "watch" | "control", opts?: {
|
|
77
|
+
ttlSeconds?: number;
|
|
78
|
+
reason?: string;
|
|
79
|
+
}): Promise<{
|
|
80
|
+
url: string;
|
|
81
|
+
expiresAt: string;
|
|
82
|
+
}>;
|
|
83
|
+
sleep(): Promise<void>;
|
|
84
|
+
delete(): Promise<void>;
|
|
85
|
+
}
|
|
86
|
+
export interface ComputerProvider {
|
|
87
|
+
readonly id: ProviderId;
|
|
88
|
+
readonly computerOs: ComputerOs[];
|
|
89
|
+
configured(): boolean;
|
|
90
|
+
createComputer(spec: ComputerSpec): Promise<ProviderComputer>;
|
|
91
|
+
connectComputer(nativeId: string): Promise<ProviderComputer>;
|
|
92
|
+
}
|
|
93
|
+
export declare class RoutedComputer {
|
|
94
|
+
readonly provider: ProviderId;
|
|
95
|
+
private c;
|
|
96
|
+
readonly id: string;
|
|
97
|
+
constructor(provider: ProviderId, c: ProviderComputer);
|
|
98
|
+
get nativeId(): string;
|
|
99
|
+
info(): Promise<ComputerInfo>;
|
|
100
|
+
act(action: ComputerAction): Promise<ActionResult>;
|
|
101
|
+
actBatch(actions: ComputerAction[]): Promise<ActionResult[]>;
|
|
102
|
+
screenshot(opts?: {
|
|
103
|
+
format?: "png" | "jpeg";
|
|
104
|
+
quality?: number;
|
|
105
|
+
}): Promise<Uint8Array<ArrayBufferLike>>;
|
|
106
|
+
exec(command: string, timeoutSeconds?: number): Promise<{
|
|
107
|
+
exitCode: number;
|
|
108
|
+
stdout: string;
|
|
109
|
+
stderr: string;
|
|
110
|
+
}>;
|
|
111
|
+
readFile(path: string): Promise<Uint8Array<ArrayBufferLike>>;
|
|
112
|
+
writeFile(path: string, content: string | Uint8Array): Promise<void>;
|
|
113
|
+
viewer(mode?: "watch" | "control", opts?: {
|
|
114
|
+
ttlSeconds?: number;
|
|
115
|
+
reason?: string;
|
|
116
|
+
}): Promise<{
|
|
117
|
+
url: string;
|
|
118
|
+
expiresAt: string;
|
|
119
|
+
}>;
|
|
120
|
+
sleep(): Promise<void>;
|
|
121
|
+
delete(): Promise<void>;
|
|
122
|
+
}
|
|
123
|
+
/** Pick a computer provider that offers this OS and has credentials, in the given order. */
|
|
124
|
+
export declare function createComputer(providers: ComputerProvider[], spec?: ComputerSpec, prefer?: ProviderId[]): Promise<RoutedComputer>;
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
import { RouterError } from "./types.js";
|
|
2
|
+
export class RoutedComputer {
|
|
3
|
+
provider;
|
|
4
|
+
c;
|
|
5
|
+
id;
|
|
6
|
+
constructor(provider, c) {
|
|
7
|
+
this.provider = provider;
|
|
8
|
+
this.c = c;
|
|
9
|
+
this.id = `${provider}:${c.nativeId}`;
|
|
10
|
+
}
|
|
11
|
+
get nativeId() {
|
|
12
|
+
return this.c.nativeId;
|
|
13
|
+
}
|
|
14
|
+
info() {
|
|
15
|
+
return this.c.info();
|
|
16
|
+
}
|
|
17
|
+
act(action) {
|
|
18
|
+
return this.c.act(action);
|
|
19
|
+
}
|
|
20
|
+
actBatch(actions) {
|
|
21
|
+
if (actions.length > 50)
|
|
22
|
+
throw new RouterError("at most 50 actions per batch");
|
|
23
|
+
return this.c.actBatch(actions);
|
|
24
|
+
}
|
|
25
|
+
screenshot(opts) {
|
|
26
|
+
return this.c.screenshot(opts);
|
|
27
|
+
}
|
|
28
|
+
exec(command, timeoutSeconds) {
|
|
29
|
+
return this.c.exec(command, timeoutSeconds);
|
|
30
|
+
}
|
|
31
|
+
readFile(path) {
|
|
32
|
+
return this.c.readFile(path);
|
|
33
|
+
}
|
|
34
|
+
writeFile(path, content) {
|
|
35
|
+
return this.c.writeFile(path, content);
|
|
36
|
+
}
|
|
37
|
+
viewer(mode, opts) {
|
|
38
|
+
return this.c.viewer(mode, opts);
|
|
39
|
+
}
|
|
40
|
+
sleep() {
|
|
41
|
+
return this.c.sleep();
|
|
42
|
+
}
|
|
43
|
+
delete() {
|
|
44
|
+
return this.c.delete();
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
/** Pick a computer provider that offers this OS and has credentials, in the given order. */
|
|
48
|
+
export async function createComputer(providers, spec = {}, prefer) {
|
|
49
|
+
const os = spec.os ?? "linux";
|
|
50
|
+
const ordered = prefer ? prefer.map((id) => providers.find((p) => p.id === id)).filter((p) => !!p) : providers;
|
|
51
|
+
const usable = ordered.filter((p) => p.computerOs.includes(os) && p.configured());
|
|
52
|
+
if (usable.length === 0) {
|
|
53
|
+
const offering = providers.filter((p) => p.computerOs.includes(os)).map((p) => p.id);
|
|
54
|
+
throw new RouterError(offering.length ? `no configured provider for a ${os} computer (offered by: ${offering.join(", ")})` : `no provider offers a ${os} computer`);
|
|
55
|
+
}
|
|
56
|
+
const errors = [];
|
|
57
|
+
for (const p of usable) {
|
|
58
|
+
try {
|
|
59
|
+
return new RoutedComputer(p.id, await p.createComputer({ ...spec, os }));
|
|
60
|
+
}
|
|
61
|
+
catch (err) {
|
|
62
|
+
errors.push(`${p.id}: ${err.message}`);
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
throw new RouterError(`all computer providers failed: ${errors.join(" | ")}`);
|
|
66
|
+
}
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
export { Router, RoutedSandbox, defaultProviders, parseId, type Candidate, type RouterOptions } from "./router.js";
|
|
2
|
+
export { CATALOG, hourlyCost, unfit, info, type ProviderInfo } from "./catalog.js";
|
|
3
|
+
export { createApp } from "./server.js";
|
|
4
|
+
export { RoutedComputer, createComputer, type ActionResult, type ComputerAction, type ComputerActionName, type ComputerInfo, type ComputerOs, type ComputerProvider, type ComputerSpec, type ProviderComputer, } from "./computers.js";
|
|
5
|
+
export { bench, benchMarkdown, type BenchRow } from "./bench.js";
|
|
6
|
+
export { aggregate, loadReports, makeReport, renderLeaderboard, type BenchReport, type LeaderRow } from "./leaderboard.js";
|
|
7
|
+
export { MaritimeProvider } from "./providers/maritime.js";
|
|
8
|
+
export { E2BProvider } from "./providers/e2b.js";
|
|
9
|
+
export { DaytonaProvider } from "./providers/daytona.js";
|
|
10
|
+
export { ModalProvider } from "./providers/modal.js";
|
|
11
|
+
export { VercelProvider } from "./providers/vercel.js";
|
|
12
|
+
export { MorphProvider } from "./providers/morph.js";
|
|
13
|
+
export { SpritesProvider } from "./providers/sprites.js";
|
|
14
|
+
export { BlaxelProvider } from "./providers/blaxel.js";
|
|
15
|
+
export { TensorlakeProvider } from "./providers/tensorlake.js";
|
|
16
|
+
export { RunloopProvider } from "./providers/runloop.js";
|
|
17
|
+
export * from "./types.js";
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
export { Router, RoutedSandbox, defaultProviders, parseId } from "./router.js";
|
|
2
|
+
export { CATALOG, hourlyCost, unfit, info } from "./catalog.js";
|
|
3
|
+
export { createApp } from "./server.js";
|
|
4
|
+
export { RoutedComputer, createComputer, } from "./computers.js";
|
|
5
|
+
export { bench, benchMarkdown } from "./bench.js";
|
|
6
|
+
export { aggregate, loadReports, makeReport, renderLeaderboard } from "./leaderboard.js";
|
|
7
|
+
export { MaritimeProvider } from "./providers/maritime.js";
|
|
8
|
+
export { E2BProvider } from "./providers/e2b.js";
|
|
9
|
+
export { DaytonaProvider } from "./providers/daytona.js";
|
|
10
|
+
export { ModalProvider } from "./providers/modal.js";
|
|
11
|
+
export { VercelProvider } from "./providers/vercel.js";
|
|
12
|
+
export { MorphProvider } from "./providers/morph.js";
|
|
13
|
+
export { SpritesProvider } from "./providers/sprites.js";
|
|
14
|
+
export { BlaxelProvider } from "./providers/blaxel.js";
|
|
15
|
+
export { TensorlakeProvider } from "./providers/tensorlake.js";
|
|
16
|
+
export { RunloopProvider } from "./providers/runloop.js";
|
|
17
|
+
export * from "./types.js";
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import type { BenchRow } from "./bench.js";
|
|
2
|
+
import type { ProviderId, SandboxSpec } from "./types.js";
|
|
3
|
+
/**
|
|
4
|
+
* A shareable bench result. People submit these as PRs to results/, and `sandboxrouter
|
|
5
|
+
* leaderboard` renders every submission into one static page. No server, nothing to trust but
|
|
6
|
+
* the files in the repo.
|
|
7
|
+
*/
|
|
8
|
+
export interface BenchReport {
|
|
9
|
+
tool: "sandboxrouter";
|
|
10
|
+
version: string;
|
|
11
|
+
date: string;
|
|
12
|
+
/** Free text from the submitter, e.g. "us-east-1 c7g.large" or "Berlin, home fiber". */
|
|
13
|
+
label?: string;
|
|
14
|
+
runs: number;
|
|
15
|
+
spec: SandboxSpec;
|
|
16
|
+
node: string;
|
|
17
|
+
platform: string;
|
|
18
|
+
rows: BenchRow[];
|
|
19
|
+
}
|
|
20
|
+
export declare function makeReport(rows: BenchRow[], opts: {
|
|
21
|
+
version: string;
|
|
22
|
+
runs: number;
|
|
23
|
+
spec: SandboxSpec;
|
|
24
|
+
label?: string;
|
|
25
|
+
}): BenchReport;
|
|
26
|
+
export declare function loadReports(dir: string): BenchReport[];
|
|
27
|
+
export interface LeaderRow {
|
|
28
|
+
provider: ProviderId;
|
|
29
|
+
name: string;
|
|
30
|
+
submissions: number;
|
|
31
|
+
medianCreateMs?: number;
|
|
32
|
+
medianExecMs?: number;
|
|
33
|
+
successRate: number;
|
|
34
|
+
hourlyUsd: number;
|
|
35
|
+
}
|
|
36
|
+
/** Median of each submission's medians, so one heavy submitter can't skew a provider. */
|
|
37
|
+
export declare function aggregate(reports: BenchReport[]): LeaderRow[];
|
|
38
|
+
/** One self-contained HTML page: a sorted table with an inline bar per measure, each on its own scale. */
|
|
39
|
+
export declare function renderLeaderboard(reports: BenchReport[]): string;
|
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
import { readdirSync, readFileSync } from "node:fs";
|
|
2
|
+
import { join } from "node:path";
|
|
3
|
+
import { CATALOG, hourlyCost } from "./catalog.js";
|
|
4
|
+
import { median } from "./bench-store.js";
|
|
5
|
+
export function makeReport(rows, opts) {
|
|
6
|
+
return {
|
|
7
|
+
tool: "sandboxrouter",
|
|
8
|
+
version: opts.version,
|
|
9
|
+
date: new Date().toISOString(),
|
|
10
|
+
label: opts.label,
|
|
11
|
+
runs: opts.runs,
|
|
12
|
+
// Only the shape matters for comparison; never ship env or metadata.
|
|
13
|
+
spec: { vcpu: opts.spec.vcpu, memoryGb: opts.spec.memoryGb },
|
|
14
|
+
node: process.version,
|
|
15
|
+
platform: `${process.platform}-${process.arch}`,
|
|
16
|
+
rows: rows.map(({ lastError: _drop, ...r }) => r),
|
|
17
|
+
};
|
|
18
|
+
}
|
|
19
|
+
export function loadReports(dir) {
|
|
20
|
+
let files;
|
|
21
|
+
try {
|
|
22
|
+
files = readdirSync(dir).filter((f) => f.endsWith(".json"));
|
|
23
|
+
}
|
|
24
|
+
catch {
|
|
25
|
+
return [];
|
|
26
|
+
}
|
|
27
|
+
const out = [];
|
|
28
|
+
for (const f of files.sort()) {
|
|
29
|
+
try {
|
|
30
|
+
const r = JSON.parse(readFileSync(join(dir, f), "utf8"));
|
|
31
|
+
if (r?.tool === "sandboxrouter" && Array.isArray(r.rows))
|
|
32
|
+
out.push(r);
|
|
33
|
+
}
|
|
34
|
+
catch {
|
|
35
|
+
/* skip unreadable files */
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
return out;
|
|
39
|
+
}
|
|
40
|
+
const KNOWN = new Set(CATALOG.map((c) => c.id));
|
|
41
|
+
/** Median of each submission's medians, so one heavy submitter can't skew a provider. */
|
|
42
|
+
export function aggregate(reports) {
|
|
43
|
+
const by = new Map();
|
|
44
|
+
for (const rep of reports)
|
|
45
|
+
for (const r of rep.rows)
|
|
46
|
+
if (KNOWN.has(r.provider))
|
|
47
|
+
by.set(r.provider, [...(by.get(r.provider) ?? []), r]);
|
|
48
|
+
const rows = [...by.entries()].map(([id, rs]) => {
|
|
49
|
+
const runs = rs.reduce((n, r) => n + r.runs, 0);
|
|
50
|
+
const ok = rs.reduce((n, r) => n + (r.runs - r.failures), 0);
|
|
51
|
+
return {
|
|
52
|
+
provider: id,
|
|
53
|
+
name: CATALOG.find((c) => c.id === id).name,
|
|
54
|
+
submissions: rs.length,
|
|
55
|
+
medianCreateMs: median(rs.map((r) => r.medianCreateMs).filter((n) => typeof n === "number")),
|
|
56
|
+
medianExecMs: median(rs.map((r) => r.medianExecMs).filter((n) => typeof n === "number")),
|
|
57
|
+
successRate: runs ? ok / runs : 0,
|
|
58
|
+
hourlyUsd: hourlyCost(id),
|
|
59
|
+
};
|
|
60
|
+
});
|
|
61
|
+
return rows.sort((a, b) => (a.medianCreateMs ?? Infinity) - (b.medianCreateMs ?? Infinity));
|
|
62
|
+
}
|
|
63
|
+
const esc = (s) => String(s).replace(/[&<>"']/g, (c) => ({ "&": "&", "<": "<", ">": ">", '"': """, "'": "'" })[c]);
|
|
64
|
+
function bar(value, max, text, tip) {
|
|
65
|
+
if (value === undefined)
|
|
66
|
+
return `<td class="num muted">n/a</td>`;
|
|
67
|
+
const pct = max > 0 ? Math.max(1.5, (value / max) * 100) : 0;
|
|
68
|
+
return `<td class="barcell" tabindex="0" data-tip="${esc(tip)}" aria-label="${esc(tip)}">
|
|
69
|
+
<div class="track"><div class="bar" style="width:${pct.toFixed(1)}%"></div></div><span class="val">${esc(text)}</span></td>`;
|
|
70
|
+
}
|
|
71
|
+
/** One self-contained HTML page: a sorted table with an inline bar per measure, each on its own scale. */
|
|
72
|
+
export function renderLeaderboard(reports) {
|
|
73
|
+
const rows = aggregate(reports);
|
|
74
|
+
const maxCreate = Math.max(0, ...rows.map((r) => r.medianCreateMs ?? 0));
|
|
75
|
+
const maxPrice = Math.max(0, ...rows.map((r) => r.hourlyUsd));
|
|
76
|
+
const dates = reports.map((r) => r.date).sort();
|
|
77
|
+
const span = dates.length ? `${dates[0].slice(0, 10)} to ${dates.at(-1).slice(0, 10)}` : "no submissions yet";
|
|
78
|
+
const body = rows
|
|
79
|
+
.map((r, i) => `<tr>
|
|
80
|
+
<td class="num rank">${i + 1}</td>
|
|
81
|
+
<th scope="row">${esc(r.name)}</th>
|
|
82
|
+
${bar(r.medianCreateMs, maxCreate, r.medianCreateMs === undefined ? "" : `${r.medianCreateMs.toLocaleString("en-US")} ms`, `${r.name}: median cold start ${r.medianCreateMs} ms across ${r.submissions} submission${r.submissions === 1 ? "" : "s"}`)}
|
|
83
|
+
<td class="num">${r.medianExecMs === undefined ? "n/a" : `${r.medianExecMs} ms`}</td>
|
|
84
|
+
${bar(r.hourlyUsd, maxPrice, `$${r.hourlyUsd.toFixed(3)}`, `${r.name}: $${r.hourlyUsd.toFixed(4)} per hour at list price, 2 vCPU / 4 GB`)}
|
|
85
|
+
<td class="num">${(r.successRate * 100).toFixed(0)}%</td>
|
|
86
|
+
<td class="num muted">${r.submissions}</td>
|
|
87
|
+
</tr>`)
|
|
88
|
+
.join("\n");
|
|
89
|
+
const subs = reports
|
|
90
|
+
.map((r) => `<li>${esc(r.date.slice(0, 10))} · ${esc(r.label ?? "unlabeled")} · ${esc(r.platform)} · ${r.runs} runs · ${r.rows.length} providers</li>`)
|
|
91
|
+
.join("\n");
|
|
92
|
+
return `<!DOCTYPE html>
|
|
93
|
+
<html lang="en">
|
|
94
|
+
<head>
|
|
95
|
+
<meta charset="utf-8">
|
|
96
|
+
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
97
|
+
<title>sandboxrouter leaderboard</title>
|
|
98
|
+
<style>
|
|
99
|
+
.viz-root {
|
|
100
|
+
color-scheme: light;
|
|
101
|
+
--page: #f9f9f7; --surface-1: #fcfcfb; --text-primary: #0b0b0b; --text-secondary: #52514e;
|
|
102
|
+
--text-muted: #898781; --grid: #e1e0d9; --baseline: #c3c2b7; --border: rgba(11,11,11,0.10);
|
|
103
|
+
--series-1: #2a78d6;
|
|
104
|
+
}
|
|
105
|
+
@media (prefers-color-scheme: dark) {
|
|
106
|
+
:root:where(:not([data-theme="light"])) .viz-root {
|
|
107
|
+
color-scheme: dark;
|
|
108
|
+
--page: #0d0d0d; --surface-1: #1a1a19; --text-primary: #f0efec; --text-secondary: #c3c2b7;
|
|
109
|
+
--text-muted: #898781; --grid: #2c2c2a; --baseline: #383835; --border: rgba(255,255,255,0.10);
|
|
110
|
+
--series-1: #3987e5;
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
:root[data-theme="dark"] .viz-root {
|
|
114
|
+
color-scheme: dark;
|
|
115
|
+
--page: #0d0d0d; --surface-1: #1a1a19; --text-primary: #f0efec; --text-secondary: #c3c2b7;
|
|
116
|
+
--text-muted: #898781; --grid: #2c2c2a; --baseline: #383835; --border: rgba(255,255,255,0.10);
|
|
117
|
+
--series-1: #3987e5;
|
|
118
|
+
}
|
|
119
|
+
* { box-sizing: border-box; }
|
|
120
|
+
body { margin: 0; }
|
|
121
|
+
.viz-root { background: var(--page); color: var(--text-primary); min-height: 100vh;
|
|
122
|
+
font: 15px/1.5 system-ui, -apple-system, "Segoe UI", sans-serif; padding: 48px 24px; }
|
|
123
|
+
main { max-width: 1040px; margin: 0 auto; }
|
|
124
|
+
h1 { font-size: 28px; margin: 0 0 4px; letter-spacing: -0.01em; }
|
|
125
|
+
.sub { color: var(--text-secondary); margin: 0 0 24px; }
|
|
126
|
+
.card { background: var(--surface-1); border: 1px solid var(--border); border-radius: 12px; padding: 8px 16px; overflow-x: auto; }
|
|
127
|
+
table { width: 100%; border-collapse: collapse; }
|
|
128
|
+
caption { text-align: left; color: var(--text-muted); font-size: 13px; padding: 12px 0 4px; caption-side: bottom; }
|
|
129
|
+
th, td { padding: 10px 8px; border-bottom: 1px solid var(--grid); text-align: left; white-space: nowrap; }
|
|
130
|
+
thead th { color: var(--text-muted); font-weight: 500; font-size: 13px; border-bottom-color: var(--baseline); }
|
|
131
|
+
tbody tr:last-child > * { border-bottom: 0; }
|
|
132
|
+
tbody th { font-weight: 600; }
|
|
133
|
+
.num { font-variant-numeric: tabular-nums; }
|
|
134
|
+
.rank, .muted { color: var(--text-muted); }
|
|
135
|
+
.barcell { min-width: 220px; position: relative; outline: none; }
|
|
136
|
+
.barcell { display: table-cell; }
|
|
137
|
+
.track { display: inline-block; vertical-align: middle; width: 140px; height: 12px; border-left: 1px solid var(--baseline); }
|
|
138
|
+
.bar { height: 12px; background: var(--series-1); border-radius: 0 4px 4px 0; }
|
|
139
|
+
.val { display: inline-block; vertical-align: middle; margin-left: 8px; color: var(--text-primary); font-variant-numeric: tabular-nums; }
|
|
140
|
+
.barcell:hover .bar, .barcell:focus .bar { filter: brightness(1.12); }
|
|
141
|
+
#tip { position: fixed; pointer-events: none; background: var(--surface-1); color: var(--text-primary);
|
|
142
|
+
border: 1px solid var(--border); border-radius: 8px; padding: 6px 10px; font-size: 13px;
|
|
143
|
+
box-shadow: 0 4px 16px rgba(0,0,0,0.12); opacity: 0; transition: opacity 80ms; max-width: 320px; white-space: normal; }
|
|
144
|
+
.notes { color: var(--text-secondary); font-size: 14px; margin-top: 24px; }
|
|
145
|
+
.notes ul { padding-left: 18px; color: var(--text-muted); }
|
|
146
|
+
a { color: inherit; }
|
|
147
|
+
</style>
|
|
148
|
+
</head>
|
|
149
|
+
<body class="viz-root">
|
|
150
|
+
<main>
|
|
151
|
+
<h1>Sandbox cold starts</h1>
|
|
152
|
+
<p class="sub">Time from <code>create()</code> to a usable sandbox, measured by <a href="https://github.com/gigialc/sandboxrouter">sandboxrouter bench</a>. ${reports.length} submission${reports.length === 1 ? "" : "s"}, ${esc(span)}.</p>
|
|
153
|
+
<div class="card">
|
|
154
|
+
<table>
|
|
155
|
+
<thead><tr>
|
|
156
|
+
<th>#</th><th>Provider</th><th>Cold start (median)</th><th>Exec round trip</th>
|
|
157
|
+
<th>List price, 2 vCPU / 4 GB</th><th>Success</th><th>Submissions</th>
|
|
158
|
+
</tr></thead>
|
|
159
|
+
<tbody>
|
|
160
|
+
${body || `<tr><td colspan="7" class="muted">No results yet. Run <code>npx @sandboxrouter/sdk bench --share</code> and open a PR.</td></tr>`}
|
|
161
|
+
</tbody>
|
|
162
|
+
</table>
|
|
163
|
+
</div>
|
|
164
|
+
<div class="notes">
|
|
165
|
+
<p>Sorted by cold start. Each bar column has its own scale. Cold start and exec are the median of each submission's medians, so one submitter can't dominate a provider. Latency includes the network between the submitter and each provider. Prices are list prices from each provider's pricing page (sources in <code>src/catalog.ts</code>); some bill active CPU only, so idle work costs less.</p>
|
|
166
|
+
<p>Submissions:</p>
|
|
167
|
+
<ul>${subs}</ul>
|
|
168
|
+
</div>
|
|
169
|
+
</main>
|
|
170
|
+
<div id="tip" role="tooltip"></div>
|
|
171
|
+
<script>
|
|
172
|
+
const tip = document.getElementById("tip");
|
|
173
|
+
const show = (el, x, y) => { tip.textContent = el.dataset.tip; tip.style.left = (x + 14) + "px"; tip.style.top = (y + 14) + "px"; tip.style.opacity = 1; };
|
|
174
|
+
for (const el of document.querySelectorAll("[data-tip]")) {
|
|
175
|
+
el.addEventListener("mousemove", (e) => show(el, e.clientX, e.clientY));
|
|
176
|
+
el.addEventListener("mouseleave", () => (tip.style.opacity = 0));
|
|
177
|
+
el.addEventListener("focus", () => { const r = el.getBoundingClientRect(); show(el, r.left, r.bottom - 8); });
|
|
178
|
+
el.addEventListener("blur", () => (tip.style.opacity = 0));
|
|
179
|
+
}
|
|
180
|
+
</script>
|
|
181
|
+
</body>
|
|
182
|
+
</html>
|
|
183
|
+
`;
|
|
184
|
+
}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import { type Provider, type ProviderSandbox, type SandboxSpec } from "../types.js";
|
|
2
|
+
/**
|
|
3
|
+
* Blaxel sandboxes via `@blaxel/core`. Auth: BL_API_KEY + BL_WORKSPACE (or `bl login`).
|
|
4
|
+
* Region: BL_REGION, else us-pdx-1. Blaxel sizes by memory only and gives 1 vCPU per 2 GB,
|
|
5
|
+
* so memory is max(memoryGb, 2 × vcpu) GB. Idle sandboxes drop to standby on their own and
|
|
6
|
+
* wake on the next call; pause() archives explicitly.
|
|
7
|
+
*/
|
|
8
|
+
export declare class BlaxelProvider implements Provider {
|
|
9
|
+
private apiKey;
|
|
10
|
+
private workspace;
|
|
11
|
+
private region;
|
|
12
|
+
readonly id: "blaxel";
|
|
13
|
+
private ready?;
|
|
14
|
+
constructor(apiKey?: string | undefined, workspace?: string | undefined, region?: string);
|
|
15
|
+
configured(): boolean;
|
|
16
|
+
/** Load the SDK once and point its (global) settings at our credentials. */
|
|
17
|
+
private sdk;
|
|
18
|
+
create(spec: SandboxSpec): Promise<ProviderSandbox>;
|
|
19
|
+
/** Reattach by sandbox name; an archived (paused) sandbox is unarchived first. */
|
|
20
|
+
connect(nativeId: string): Promise<ProviderSandbox>;
|
|
21
|
+
}
|