@sandboxrouter/sdk 0.3.5 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -3
- package/dist/bench-store.d.ts +2 -1
- package/dist/bench-store.js +6 -3
- package/dist/billing.d.ts +5 -0
- package/dist/billing.js +3 -0
- package/dist/check.d.ts +5 -0
- package/dist/check.js +12 -1
- package/dist/cli.js +19 -6
- package/dist/client.d.ts +33 -11
- package/dist/client.js +29 -13
- package/dist/cost.d.ts +11 -0
- package/dist/cost.js +43 -1
- package/dist/index.d.ts +6 -3
- package/dist/index.js +5 -2
- package/dist/ledger.d.ts +8 -1
- package/dist/router.d.ts +71 -5
- package/dist/router.js +136 -39
- package/dist/server.d.ts +21 -7
- package/dist/server.js +72 -30
- package/dist/types.d.ts +37 -9
- package/dist/types.js +3 -1
- package/dist/util.js +2 -1
- package/dist/validate.d.ts +17 -0
- package/dist/validate.js +66 -0
- package/dist/verified.d.ts +2 -0
- package/dist/verified.js +2 -0
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -21,7 +21,7 @@ try {
|
|
|
21
21
|
|
|
22
22
|
Runs as plain JavaScript on Node 20+. (In TypeScript 5.2+, or Node 24+, `await using sbx = ...` kills the sandbox for you.)
|
|
23
23
|
|
|
24
|
-
Set the keys of the providers you use. `create()` starts the sandbox on the cheapest one that fits
|
|
24
|
+
Set the keys of the providers you use. `create()` starts the sandbox on the cheapest one that fits: list price for the size you asked for, corrected where we measured what a provider really bills. Providers within 10% of each other on price go fastest first, by start times measured where you run the router. Providers that failed their last real-account check go last. Ties are alphabetical, and no provider pays for placement. If that provider is down, busy or out of credit, the next one takes it, and every attempt is on `sbx.attempts`. The sandbox stays on that provider until you `kill()` it.
|
|
25
25
|
|
|
26
26
|
Site and docs: [sandboxrouter.vercel.app](https://sandboxrouter.vercel.app) · npm: [`@sandboxrouter/sdk`](https://www.npmjs.com/package/@sandboxrouter/sdk)
|
|
27
27
|
|
|
@@ -30,14 +30,15 @@ Site and docs: [sandboxrouter.vercel.app](https://sandboxrouter.vercel.app) · n
|
|
|
30
30
|
AI agents need a disposable computer to run code a model wrote: install packages, run tests, do an RL rollout. Dozens of providers sell one, each with its own SDK and auth, and the same 2 vCPU / 4 GB sandbox costs from under a cent to over thirty cents an hour. sandboxrouter is one interface over all of them:
|
|
31
31
|
|
|
32
32
|
- **Same code everywhere.** `create`, `exec` (streamed), `writeFile`, `readFile`, `pause`, `kill`. Switch providers by changing which keys are set.
|
|
33
|
-
- **
|
|
33
|
+
- **Cheapest real cost, then speed, with fallback.** Or `cheapest` for list price alone, `fastest` by start times, `verified` for tested providers first, or a list of providers you name. `requireVerified` limits a router to providers that passed the real-account check.
|
|
34
|
+
- **Errors you can act on.** Every failure has a `code` (`no_provider`, `setup`, `all_failed`, `timeout`, ...), an HTTP status, and the list of providers tried.
|
|
34
35
|
- **Your keys.** Your accounts and your bills; nothing is proxied through anyone else.
|
|
35
36
|
- **Cleanup.** A local ledger of every sandbox started, so `sandboxrouter gc` kills what a crashed script left behind.
|
|
36
37
|
- **TypeScript, Python, HTTP and a CLI.** Plus desktops and phones for computer-use models.
|
|
37
38
|
|
|
38
39
|
## Prices
|
|
39
40
|
|
|
40
|
-
List price while running, 2 vCPU / 4 GB, cheapest first; waitlisted sign-ups follow. The router
|
|
41
|
+
List price while running, 2 vCPU / 4 GB, cheapest first; waitlisted sign-ups follow. The router ranks by these, corrected where we measured what a provider really bills, and tries providers whose last real-account check failed last. "Up to" = CPU billed only while busy, shown with every core busy. "Not run yet" = built from the provider's docs and tested against mocks only. Click a provider for its billing details; more in [Reading the price table](https://sandboxrouter.vercel.app/docs#price-table).
|
|
41
42
|
|
|
42
43
|
| Provider | $/hr | Plan needed | Real-account check | Source |
|
|
43
44
|
|---|---|---|---|---|
|
package/dist/bench-store.d.ts
CHANGED
|
@@ -6,7 +6,8 @@ export interface BenchSample {
|
|
|
6
6
|
at: string;
|
|
7
7
|
}
|
|
8
8
|
export type BenchData = Partial<Record<ProviderId, BenchSample>>;
|
|
9
|
-
|
|
9
|
+
/** $SANDBOXROUTER_BENCH_FILE, else bench.json next to the ledger: $SANDBOXROUTER_HOME, else ~/.sandboxrouter/. */
|
|
10
|
+
export declare function defaultBenchFile(): string;
|
|
10
11
|
export declare function loadBench(file?: string): BenchData;
|
|
11
12
|
export declare function saveBench(data: BenchData, file?: string): void;
|
|
12
13
|
export declare function median(xs?: number[]): number | undefined;
|
package/dist/bench-store.js
CHANGED
|
@@ -1,8 +1,11 @@
|
|
|
1
1
|
import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
|
2
2
|
import { homedir } from "node:os";
|
|
3
3
|
import { dirname, join } from "node:path";
|
|
4
|
-
|
|
5
|
-
export function
|
|
4
|
+
/** $SANDBOXROUTER_BENCH_FILE, else bench.json next to the ledger: $SANDBOXROUTER_HOME, else ~/.sandboxrouter/. */
|
|
5
|
+
export function defaultBenchFile() {
|
|
6
|
+
return process.env.SANDBOXROUTER_BENCH_FILE || join(process.env.SANDBOXROUTER_HOME || join(homedir(), ".sandboxrouter"), "bench.json");
|
|
7
|
+
}
|
|
8
|
+
export function loadBench(file = defaultBenchFile()) {
|
|
6
9
|
try {
|
|
7
10
|
return JSON.parse(readFileSync(file, "utf8"));
|
|
8
11
|
}
|
|
@@ -10,7 +13,7 @@ export function loadBench(file = DEFAULT_BENCH_FILE) {
|
|
|
10
13
|
return {};
|
|
11
14
|
}
|
|
12
15
|
}
|
|
13
|
-
export function saveBench(data, file =
|
|
16
|
+
export function saveBench(data, file = defaultBenchFile()) {
|
|
14
17
|
mkdirSync(dirname(file), { recursive: true });
|
|
15
18
|
writeFileSync(file, JSON.stringify(data, null, 2));
|
|
16
19
|
}
|
package/dist/billing.js
ADDED
package/dist/check.d.ts
CHANGED
|
@@ -15,6 +15,11 @@ export interface CheckReport {
|
|
|
15
15
|
results: CheckResult[];
|
|
16
16
|
/** True when no step failed. */
|
|
17
17
|
ok: boolean;
|
|
18
|
+
/**
|
|
19
|
+
* Set when the run says nothing about the provider: the account running it couldn't pay (402: no
|
|
20
|
+
* balance, plan not started). Such a run isn't saved, so the provider keeps its last real result.
|
|
21
|
+
*/
|
|
22
|
+
inconclusive?: string;
|
|
18
23
|
}
|
|
19
24
|
/**
|
|
20
25
|
* One real pass over a provider: create, exec (stdout, stderr, exit code, env, cwd), streaming,
|
package/dist/check.js
CHANGED
|
@@ -42,10 +42,21 @@ export async function checkProvider(router, provider, opts = {}) {
|
|
|
42
42
|
}
|
|
43
43
|
};
|
|
44
44
|
let sbx;
|
|
45
|
+
let unpaid;
|
|
45
46
|
await step("create", async () => {
|
|
46
|
-
|
|
47
|
+
try {
|
|
48
|
+
sbx = await router.create({ timeoutMinutes: 10, ...fixedSize(provider), ...opts.spec }, { provider, fallback: false });
|
|
49
|
+
}
|
|
50
|
+
catch (err) {
|
|
51
|
+
if (err.attempts?.at(-1)?.status !== 402)
|
|
52
|
+
throw err;
|
|
53
|
+
unpaid = err.message;
|
|
54
|
+
return { status: "skip", detail: `the account can't pay (402), so this run says nothing about ${provider}: ${unpaid}` };
|
|
55
|
+
}
|
|
47
56
|
return `${sbx.id}, ~$${sbx.estimatedHourlyUsd}/hr`;
|
|
48
57
|
});
|
|
58
|
+
if (unpaid)
|
|
59
|
+
return { provider, results, ok: false, inconclusive: "the account running the check can't pay (402)" };
|
|
49
60
|
if (!sbx)
|
|
50
61
|
return { provider, results, ok: false };
|
|
51
62
|
const box = sbx;
|
package/dist/cli.js
CHANGED
|
@@ -39,7 +39,7 @@ Usage:
|
|
|
39
39
|
sandboxrouter serve [--port 8787] [--host 127.0.0.1] [--cors https://site.example,http://localhost:3000]
|
|
40
40
|
|
|
41
41
|
Spec flags: --vcpu N --memory GB --gpu T4 --timeout MIN
|
|
42
|
-
Route flags: --provider e2b[,modal,...] --strategy verified|cheapest|fastest (default
|
|
42
|
+
Route flags: --provider e2b[,modal,...] --strategy balanced|verified|cheapest|fastest (default balanced: cheapest real cost, speed breaks near-ties)
|
|
43
43
|
--no-fallback --fallback-any (also on setup errors like a bad key)
|
|
44
44
|
--create-timeout SEC (per provider, default 120) --require gpu,custom-image
|
|
45
45
|
|
|
@@ -156,8 +156,13 @@ async function main() {
|
|
|
156
156
|
let n = 0;
|
|
157
157
|
for (const c of plan) {
|
|
158
158
|
const mark = c.skip ? " -" : `${++n}.`.padStart(3);
|
|
159
|
-
const tested = c.verified ? "tested
|
|
160
|
-
|
|
159
|
+
const tested = c.failing ? "check failed" : c.verified ? "tested" : "";
|
|
160
|
+
// The list price only shows where a measured bill moved the estimate off it.
|
|
161
|
+
const list = c.listHourlyUsd !== c.estimatedHourlyUsd ? ` (list $${c.listHourlyUsd.toFixed(4)})` : "";
|
|
162
|
+
const size = `${c.size.vcpu} vCPU / ${c.size.memoryGb} GB`;
|
|
163
|
+
const speed = c.medianCreateMs !== undefined ? `${c.medianCreateMs} ms` : "";
|
|
164
|
+
const cols = [`$${c.estimatedHourlyUsd.toFixed(4)}/hr${list}`, size, speed, tested].filter(Boolean).join(" ");
|
|
165
|
+
console.log(`${mark} ${c.provider.padEnd(11)} ${cols}${c.skip ? ` (skipped: ${c.skip})` : ""}`);
|
|
161
166
|
}
|
|
162
167
|
return;
|
|
163
168
|
}
|
|
@@ -214,6 +219,11 @@ async function main() {
|
|
|
214
219
|
},
|
|
215
220
|
});
|
|
216
221
|
reports.push(report);
|
|
222
|
+
if (report.inconclusive) {
|
|
223
|
+
if (!values.json)
|
|
224
|
+
console.log(` not saved: ${report.inconclusive}; ${p} keeps its last saved result`);
|
|
225
|
+
continue;
|
|
226
|
+
}
|
|
217
227
|
if (saveDir) {
|
|
218
228
|
mkdirSync(saveDir, { recursive: true });
|
|
219
229
|
const saved = publishableCheck({ tool: "sandboxrouter-check", version, date: new Date().toISOString(), platform: `${platform()}-${arch()}`, ...(by ? { by } : {}), ...report });
|
|
@@ -222,9 +232,12 @@ async function main() {
|
|
|
222
232
|
}
|
|
223
233
|
if (values.json)
|
|
224
234
|
console.log(JSON.stringify(reports, null, 2));
|
|
225
|
-
else
|
|
226
|
-
|
|
227
|
-
|
|
235
|
+
else {
|
|
236
|
+
const unpaid = reports.filter((r) => r.inconclusive).length;
|
|
237
|
+
const ran = reports.length - unpaid;
|
|
238
|
+
console.log(`\n${reports.filter((r) => r.ok).length}/${ran} passed${unpaid ? `; ${unpaid} not run (the account can't pay)` : ""}`);
|
|
239
|
+
}
|
|
240
|
+
if (reports.some((r) => !r.ok && !r.inconclusive))
|
|
228
241
|
process.exitCode = 1;
|
|
229
242
|
return;
|
|
230
243
|
}
|
package/dist/client.d.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type { Candidate, Size } from "./router.js";
|
|
2
|
+
import type { Attempt, ExecOptions, ExecResult, RouteOptions, RouterErrorCode, SandboxSpec } from "./types.js";
|
|
2
3
|
/**
|
|
3
4
|
* Client for the sandboxrouter HTTP API: the hosted service (one key, prepaid credits) or your own
|
|
4
5
|
* `sandboxrouter serve`. Provider keys live on the server; this side only needs the one key.
|
|
@@ -20,18 +21,16 @@ export declare class Client {
|
|
|
20
21
|
/** @internal */
|
|
21
22
|
request(method: string, path: string, body?: unknown, accept?: string): Promise<Response>;
|
|
22
23
|
private json;
|
|
23
|
-
/** Providers this server can route to, with
|
|
24
|
+
/** Providers this server can route to, with prices at the default size. */
|
|
24
25
|
providers(): Promise<{
|
|
25
26
|
id: string;
|
|
26
27
|
configured: boolean;
|
|
27
28
|
estimatedHourlyUsd: number;
|
|
29
|
+
listHourlyUsd: number;
|
|
30
|
+
size: Size;
|
|
28
31
|
}[]>;
|
|
29
|
-
/** Dry run: which providers would be tried, in order. */
|
|
30
|
-
route(spec?: SandboxSpec, route?: RouteOptions): Promise<
|
|
31
|
-
provider: string;
|
|
32
|
-
estimatedHourlyUsd: number;
|
|
33
|
-
skip?: string;
|
|
34
|
-
}[]>;
|
|
32
|
+
/** Dry run: which providers would be tried, in order, each with its price, size and any reason it's skipped. */
|
|
33
|
+
route(spec?: SandboxSpec, route?: RouteOptions): Promise<Candidate[]>;
|
|
35
34
|
/** Start a sandbox on the first provider that works, as the server routes it. */
|
|
36
35
|
create(spec?: SandboxSpec, route?: RouteOptions): Promise<RemoteSandbox>;
|
|
37
36
|
/** A handle on a sandbox you created earlier. */
|
|
@@ -40,19 +39,42 @@ export declare class Client {
|
|
|
40
39
|
export declare class ClientError extends Error {
|
|
41
40
|
readonly status: number;
|
|
42
41
|
readonly attempts: Attempt[];
|
|
43
|
-
|
|
42
|
+
/** The server's RouterErrorCode, when the router refused or failed (e.g. "no_provider", "all_failed"). */
|
|
43
|
+
readonly code?: RouterErrorCode | undefined;
|
|
44
|
+
constructor(status: number, message: string, attempts?: Attempt[],
|
|
45
|
+
/** The server's RouterErrorCode, when the router refused or failed (e.g. "no_provider", "all_failed"). */
|
|
46
|
+
code?: RouterErrorCode | undefined);
|
|
47
|
+
}
|
|
48
|
+
/** What POST /v1/sandboxes answers. A hosted service may add fields (e.g. what it charges). */
|
|
49
|
+
export interface CreateResponse {
|
|
50
|
+
id: string;
|
|
51
|
+
provider: string;
|
|
52
|
+
size?: Size;
|
|
53
|
+
estimatedHourlyUsd?: number;
|
|
54
|
+
listHourlyUsd?: number;
|
|
55
|
+
attempts: Attempt[];
|
|
56
|
+
[field: string]: unknown;
|
|
44
57
|
}
|
|
45
58
|
export declare class RemoteSandbox {
|
|
46
59
|
private readonly client;
|
|
47
60
|
readonly id: string;
|
|
48
61
|
readonly provider: string;
|
|
49
|
-
|
|
62
|
+
/** The full create response, including any fields the server added. Empty for `client.sandbox(id)`. */
|
|
63
|
+
readonly details: Partial<CreateResponse>;
|
|
64
|
+
readonly estimatedHourlyUsd?: number;
|
|
65
|
+
readonly listHourlyUsd?: number;
|
|
66
|
+
readonly size?: Size;
|
|
50
67
|
readonly attempts: Attempt[];
|
|
51
|
-
constructor(client: Client, id: string, provider: string,
|
|
68
|
+
constructor(client: Client, id: string, provider: string,
|
|
69
|
+
/** The full create response, including any fields the server added. Empty for `client.sandbox(id)`. */
|
|
70
|
+
details?: Partial<CreateResponse>);
|
|
52
71
|
private path;
|
|
53
72
|
exec(cmd: string, opts?: ExecOptions): Promise<ExecResult>;
|
|
54
73
|
writeFile(path: string, content: string | Uint8Array): Promise<void>;
|
|
55
74
|
readFile(path: string): Promise<Uint8Array>;
|
|
75
|
+
readText(path: string): Promise<string>;
|
|
56
76
|
pause(): Promise<void>;
|
|
57
77
|
kill(): Promise<void>;
|
|
78
|
+
/** `await using sbx = await client.create()` kills it when the block ends. */
|
|
79
|
+
[Symbol.asyncDispose](): Promise<void>;
|
|
58
80
|
}
|
package/dist/client.js
CHANGED
|
@@ -35,28 +35,25 @@ export class Client {
|
|
|
35
35
|
parsed = JSON.parse(text);
|
|
36
36
|
}
|
|
37
37
|
catch { }
|
|
38
|
-
throw new ClientError(res.status, parsed.error ?? (text || res.statusText), parsed.attempts ?? []);
|
|
38
|
+
throw new ClientError(res.status, parsed.error ?? (text || res.statusText), parsed.attempts ?? [], parsed.code);
|
|
39
39
|
}
|
|
40
40
|
return res;
|
|
41
41
|
}
|
|
42
42
|
async json(method, path, body) {
|
|
43
43
|
return (await this.request(method, path, body)).json();
|
|
44
44
|
}
|
|
45
|
-
/** Providers this server can route to, with
|
|
45
|
+
/** Providers this server can route to, with prices at the default size. */
|
|
46
46
|
async providers() {
|
|
47
47
|
return (await this.json("GET", "/v1/providers")).providers;
|
|
48
48
|
}
|
|
49
|
-
/** Dry run: which providers would be tried, in order. */
|
|
49
|
+
/** Dry run: which providers would be tried, in order, each with its price, size and any reason it's skipped. */
|
|
50
50
|
async route(spec = {}, route = {}) {
|
|
51
51
|
return (await this.json("POST", "/v1/route", { ...spec, ...route })).candidates;
|
|
52
52
|
}
|
|
53
53
|
/** Start a sandbox on the first provider that works, as the server routes it. */
|
|
54
54
|
async create(spec = {}, route = {}) {
|
|
55
|
-
const r = await this.json("POST", "/v1/sandboxes", {
|
|
56
|
-
|
|
57
|
-
...route,
|
|
58
|
-
});
|
|
59
|
-
return new RemoteSandbox(this, r.id, r.provider, r.estimatedHourlyUsd, r.attempts);
|
|
55
|
+
const r = await this.json("POST", "/v1/sandboxes", { ...spec, ...route });
|
|
56
|
+
return new RemoteSandbox(this, r.id, r.provider, r);
|
|
60
57
|
}
|
|
61
58
|
/** A handle on a sandbox you created earlier. */
|
|
62
59
|
sandbox(id) {
|
|
@@ -67,10 +64,14 @@ const DEFAULT_URL = "http://127.0.0.1:8787";
|
|
|
67
64
|
export class ClientError extends Error {
|
|
68
65
|
status;
|
|
69
66
|
attempts;
|
|
70
|
-
|
|
67
|
+
code;
|
|
68
|
+
constructor(status, message, attempts = [],
|
|
69
|
+
/** The server's RouterErrorCode, when the router refused or failed (e.g. "no_provider", "all_failed"). */
|
|
70
|
+
code) {
|
|
71
71
|
super(`${status}: ${message}`);
|
|
72
72
|
this.status = status;
|
|
73
73
|
this.attempts = attempts;
|
|
74
|
+
this.code = code;
|
|
74
75
|
this.name = "ClientError";
|
|
75
76
|
}
|
|
76
77
|
}
|
|
@@ -78,14 +79,22 @@ export class RemoteSandbox {
|
|
|
78
79
|
client;
|
|
79
80
|
id;
|
|
80
81
|
provider;
|
|
82
|
+
details;
|
|
81
83
|
estimatedHourlyUsd;
|
|
84
|
+
listHourlyUsd;
|
|
85
|
+
size;
|
|
82
86
|
attempts;
|
|
83
|
-
constructor(client, id, provider,
|
|
87
|
+
constructor(client, id, provider,
|
|
88
|
+
/** The full create response, including any fields the server added. Empty for `client.sandbox(id)`. */
|
|
89
|
+
details = {}) {
|
|
84
90
|
this.client = client;
|
|
85
91
|
this.id = id;
|
|
86
92
|
this.provider = provider;
|
|
87
|
-
this.
|
|
88
|
-
this.
|
|
93
|
+
this.details = details;
|
|
94
|
+
this.estimatedHourlyUsd = details.estimatedHourlyUsd;
|
|
95
|
+
this.listHourlyUsd = details.listHourlyUsd;
|
|
96
|
+
this.size = details.size;
|
|
97
|
+
this.attempts = details.attempts ?? [];
|
|
89
98
|
}
|
|
90
99
|
path(suffix = "") {
|
|
91
100
|
return `/v1/sandboxes/${encodeURIComponent(this.id)}${suffix}`;
|
|
@@ -118,7 +127,7 @@ export class RemoteSandbox {
|
|
|
118
127
|
else if (event === "result")
|
|
119
128
|
return payload;
|
|
120
129
|
else if (event === "error")
|
|
121
|
-
throw new ClientError(502, payload.error ?? data);
|
|
130
|
+
throw new ClientError(payload.status ?? 502, payload.error ?? data, [], payload.code);
|
|
122
131
|
}
|
|
123
132
|
}
|
|
124
133
|
throw new ClientError(502, "stream ended before the command finished");
|
|
@@ -131,10 +140,17 @@ export class RemoteSandbox {
|
|
|
131
140
|
const res = await this.client.request("GET", this.path(`/files?path=${encodeURIComponent(path)}`), undefined, "application/octet-stream");
|
|
132
141
|
return new Uint8Array(await res.arrayBuffer());
|
|
133
142
|
}
|
|
143
|
+
async readText(path) {
|
|
144
|
+
return new TextDecoder().decode(await this.readFile(path));
|
|
145
|
+
}
|
|
134
146
|
async pause() {
|
|
135
147
|
await this.client.request("POST", this.path("/pause"));
|
|
136
148
|
}
|
|
137
149
|
async kill() {
|
|
138
150
|
await this.client.request("DELETE", this.path());
|
|
139
151
|
}
|
|
152
|
+
/** `await using sbx = await client.create()` kills it when the block ends. */
|
|
153
|
+
async [Symbol.asyncDispose]() {
|
|
154
|
+
await this.kill();
|
|
155
|
+
}
|
|
140
156
|
}
|
package/dist/cost.d.ts
CHANGED
|
@@ -74,4 +74,15 @@ type Reader = (e: CostEntry) => Promise<Omit<Bill, "at">>;
|
|
|
74
74
|
export declare const BILL_READERS: Partial<Record<ProviderId, Reader>>;
|
|
75
75
|
export declare function settle(run: CostRun): Promise<CostRun>;
|
|
76
76
|
export declare function costMarkdown(run: CostRun): string;
|
|
77
|
+
/**
|
|
78
|
+
* Billed / predicted per provider, from the saved cost runs in `dir` (default results/cost): the latest
|
|
79
|
+
* entry whose bill is precise enough to compare. A bill rounded to whole cents only counts when the
|
|
80
|
+
* prediction is at least $1, so rounding can't move the ratio by more than 1%. Providers whose terms
|
|
81
|
+
* forbid publishing benchmarks are left out, like everywhere else results are published.
|
|
82
|
+
*/
|
|
83
|
+
export declare function billedVsList(dir?: string): {
|
|
84
|
+
provider: ProviderId;
|
|
85
|
+
ratio: number;
|
|
86
|
+
date: string;
|
|
87
|
+
}[];
|
|
77
88
|
export {};
|
package/dist/cost.js
CHANGED
|
@@ -1,4 +1,6 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { readdirSync, readFileSync } from "node:fs";
|
|
2
|
+
import { join } from "node:path";
|
|
3
|
+
import { benchmarksPublishable, fittingSpec, hourlyCost } from "./catalog.js";
|
|
2
4
|
import { Router } from "./router.js";
|
|
3
5
|
/**
|
|
4
6
|
* The size the sandbox really has, as "<vcpus> <memoryMb>". nproc and MemTotal can show the whole host
|
|
@@ -163,3 +165,43 @@ export function costMarkdown(run) {
|
|
|
163
165
|
}
|
|
164
166
|
return lines.join("\n");
|
|
165
167
|
}
|
|
168
|
+
/**
|
|
169
|
+
* Billed / predicted per provider, from the saved cost runs in `dir` (default results/cost): the latest
|
|
170
|
+
* entry whose bill is precise enough to compare. A bill rounded to whole cents only counts when the
|
|
171
|
+
* prediction is at least $1, so rounding can't move the ratio by more than 1%. Providers whose terms
|
|
172
|
+
* forbid publishing benchmarks are left out, like everywhere else results are published.
|
|
173
|
+
*/
|
|
174
|
+
export function billedVsList(dir = "results/cost") {
|
|
175
|
+
let files;
|
|
176
|
+
try {
|
|
177
|
+
files = readdirSync(dir).filter((f) => f.endsWith(".json"));
|
|
178
|
+
}
|
|
179
|
+
catch {
|
|
180
|
+
return [];
|
|
181
|
+
}
|
|
182
|
+
const latest = new Map();
|
|
183
|
+
for (const f of files) {
|
|
184
|
+
let run;
|
|
185
|
+
try {
|
|
186
|
+
run = JSON.parse(readFileSync(join(dir, f), "utf8"));
|
|
187
|
+
}
|
|
188
|
+
catch {
|
|
189
|
+
continue;
|
|
190
|
+
}
|
|
191
|
+
if (run?.tool !== "sandboxrouter-cost" || !Array.isArray(run.entries))
|
|
192
|
+
continue;
|
|
193
|
+
for (const e of run.entries) {
|
|
194
|
+
const usd = e.bill?.usd;
|
|
195
|
+
const predicted = e.predictedUsd;
|
|
196
|
+
if (!usd || !predicted || usd <= 0 || predicted <= 0 || !benchmarksPublishable(e.provider))
|
|
197
|
+
continue;
|
|
198
|
+
if (/integer cents/i.test(e.bill.source) && predicted < 1)
|
|
199
|
+
continue;
|
|
200
|
+
const date = run.date.slice(0, 10);
|
|
201
|
+
const prev = latest.get(e.provider);
|
|
202
|
+
if (!prev || date >= prev.date)
|
|
203
|
+
latest.set(e.provider, { ratio: Math.round((usd / predicted) * 1000) / 1000, date });
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
return [...latest].map(([provider, v]) => ({ provider, ...v })).sort((a, b) => a.provider.localeCompare(b.provider));
|
|
207
|
+
}
|
package/dist/index.d.ts
CHANGED
|
@@ -2,10 +2,13 @@ export { checkProvider, CHECK_STEPS, type CheckReport, type CheckResult, type Ch
|
|
|
2
2
|
export { Ledger, defaultLedgerFile, type LedgerEntry } from "./ledger.js";
|
|
3
3
|
export { conformanceTable, loadConformance, type ConformanceRow, type SavedCheck } from "./conformance.js";
|
|
4
4
|
export { gc, listOpen, isGone, type GcResult, type OpenSandbox } from "./gc.js";
|
|
5
|
-
export { Router, RoutedSandbox, defaultProviders, defaultComputerProviders, parseId, type Candidate, type RouterOptions } from "./router.js";
|
|
5
|
+
export { Router, RoutedSandbox, defaultProviders, defaultComputerProviders, parseId, balanced, sizeOf, PRICE_TIE, DEFAULT_CREATE_TIMEOUT_S, DEFAULT_CREATE_DEADLINE_S, type Candidate, type RouterOptions, type SandboxPrice, type Size, } from "./router.js";
|
|
6
|
+
export { VERIFIED, FAILING } from "./verified.js";
|
|
7
|
+
export { BILLED_VS_LIST } from "./billing.js";
|
|
8
|
+
export { validate, splitRequest, SPEC_KEYS, ROUTE_KEYS, STRATEGIES } from "./validate.js";
|
|
6
9
|
export { CATALOG, hourlyCost, unfit, info, type ProviderInfo } from "./catalog.js";
|
|
7
|
-
export { createApp, HttpError, type ServerHooks } from "./server.js";
|
|
8
|
-
export { Client, ClientError, RemoteSandbox } from "./client.js";
|
|
10
|
+
export { createApp, HttpError, ERROR_STATUS, type ServerHooks } from "./server.js";
|
|
11
|
+
export { Client, ClientError, RemoteSandbox, type CreateResponse } from "./client.js";
|
|
9
12
|
export { RoutedComputer, createComputer, planComputer, computerCatalog, computerHourly, COMPUTER_ENTRIES, type ComputerCatalogEntry, type ComputerCandidate, type ComputerProviderId, type ActionResult, type ComputerAction, type ComputerActionName, type ComputerInfo, type ComputerOs, type ComputerProvider, type ComputerSpec, type ProviderComputer, } from "./computers.js";
|
|
10
13
|
export { bench, benchMarkdown, type BenchRow } from "./bench.js";
|
|
11
14
|
export { aggregate, loadReports, makeReport, renderLeaderboard, type BenchReport, type LeaderRow } from "./leaderboard.js";
|
package/dist/index.js
CHANGED
|
@@ -2,9 +2,12 @@ export { checkProvider, CHECK_STEPS } from "./check.js";
|
|
|
2
2
|
export { Ledger, defaultLedgerFile } from "./ledger.js";
|
|
3
3
|
export { conformanceTable, loadConformance } from "./conformance.js";
|
|
4
4
|
export { gc, listOpen, isGone } from "./gc.js";
|
|
5
|
-
export { Router, RoutedSandbox, defaultProviders, defaultComputerProviders, parseId } from "./router.js";
|
|
5
|
+
export { Router, RoutedSandbox, defaultProviders, defaultComputerProviders, parseId, balanced, sizeOf, PRICE_TIE, DEFAULT_CREATE_TIMEOUT_S, DEFAULT_CREATE_DEADLINE_S, } from "./router.js";
|
|
6
|
+
export { VERIFIED, FAILING } from "./verified.js";
|
|
7
|
+
export { BILLED_VS_LIST } from "./billing.js";
|
|
8
|
+
export { validate, splitRequest, SPEC_KEYS, ROUTE_KEYS, STRATEGIES } from "./validate.js";
|
|
6
9
|
export { CATALOG, hourlyCost, unfit, info } from "./catalog.js";
|
|
7
|
-
export { createApp, HttpError } from "./server.js";
|
|
10
|
+
export { createApp, HttpError, ERROR_STATUS } from "./server.js";
|
|
8
11
|
export { Client, ClientError, RemoteSandbox } from "./client.js";
|
|
9
12
|
export { RoutedComputer, createComputer, planComputer, computerCatalog, computerHourly, COMPUTER_ENTRIES, } from "./computers.js";
|
|
10
13
|
export { bench, benchMarkdown } from "./bench.js";
|
package/dist/ledger.d.ts
CHANGED
|
@@ -8,8 +8,15 @@ export interface LedgerEntry {
|
|
|
8
8
|
createdAt: string;
|
|
9
9
|
/** ISO time its requested lifetime (spec.timeoutMinutes) runs out. */
|
|
10
10
|
expiresAt: string;
|
|
11
|
-
/**
|
|
11
|
+
/** Expected cost per hour while it runs (list price x measured billed ratio). */
|
|
12
12
|
usdHour: number;
|
|
13
|
+
/** List price per hour. Missing in entries written before 0.5.0. */
|
|
14
|
+
listUsdHour?: number;
|
|
15
|
+
/** Size it was created at. Missing in entries written before 0.5.0. */
|
|
16
|
+
size?: {
|
|
17
|
+
vcpu: number;
|
|
18
|
+
memoryGb: number;
|
|
19
|
+
};
|
|
13
20
|
}
|
|
14
21
|
/**
|
|
15
22
|
* Where the default ledger lives: $SANDBOXROUTER_HOME/sandboxes.jsonl, else ~/.sandboxrouter/.
|
package/dist/router.d.ts
CHANGED
|
@@ -1,19 +1,43 @@
|
|
|
1
1
|
import { RoutedComputer, type ComputerOs, type ComputerProvider, type ComputerProviderId, type ComputerSpec } from "./computers.js";
|
|
2
2
|
import { Ledger } from "./ledger.js";
|
|
3
3
|
import { type Attempt, type ExecOptions, type ExecResult, type Provider, type ProviderId, type ProviderSandbox, type RouteOptions, type SandboxSpec } from "./types.js";
|
|
4
|
+
/** vCPUs and memory a sandbox gets. */
|
|
5
|
+
export interface Size {
|
|
6
|
+
vcpu: number;
|
|
7
|
+
memoryGb: number;
|
|
8
|
+
}
|
|
4
9
|
export interface Candidate {
|
|
5
10
|
provider: ProviderId;
|
|
11
|
+
/**
|
|
12
|
+
* What the sandbox is expected to cost, USD/hour: the provider's list price for this size, times what
|
|
13
|
+
* it really billed against list price where we measured that (`billedVsList`). The default strategy
|
|
14
|
+
* ranks by this, and a service reselling sandboxes should charge from it.
|
|
15
|
+
*/
|
|
6
16
|
estimatedHourlyUsd: number;
|
|
17
|
+
/** The provider's published list price for this size, USD/hour. */
|
|
18
|
+
listHourlyUsd: number;
|
|
19
|
+
/** Billed / list-price ratio behind `estimatedHourlyUsd`, when measured (`BILLED_VS_LIST`). */
|
|
20
|
+
billedVsList?: number;
|
|
21
|
+
/**
|
|
22
|
+
* The size the sandbox gets: the provider's fixed default image size where it has one, else what was
|
|
23
|
+
* asked for (2 vCPU / 4 GB when nothing was). Some providers round up to their next size.
|
|
24
|
+
*/
|
|
25
|
+
size: Size;
|
|
26
|
+
/** Median create time: this router's own recent creates when it has a few, else your `bench` file. */
|
|
7
27
|
medianCreateMs?: number;
|
|
8
28
|
/** Passed the real-account check (as of this release, or `RouterOptions.verified`). */
|
|
9
29
|
verified: boolean;
|
|
30
|
+
/** Failed a step of its last real-account check (as of this release, or `RouterOptions.failing`). */
|
|
31
|
+
failing: boolean;
|
|
10
32
|
/** Why this provider won't be tried, if it won't. */
|
|
11
33
|
skip?: string;
|
|
12
34
|
}
|
|
35
|
+
/** Providers within this share of the cheapest remaining price count as a tie, broken by speed. */
|
|
36
|
+
export declare const PRICE_TIE = 0.1;
|
|
13
37
|
export interface RouterOptions {
|
|
14
38
|
/** Replace the built-in adapters (tests, custom providers). */
|
|
15
39
|
providers?: Provider[];
|
|
16
|
-
/** Path of the bench results file
|
|
40
|
+
/** Path of the bench results file: start times for the speed tie-break and strategy "fastest". */
|
|
17
41
|
benchFile?: string;
|
|
18
42
|
/** Desktop-only computer providers (sandbox providers that also sell computers are found automatically). */
|
|
19
43
|
computers?: ComputerProvider[];
|
|
@@ -24,10 +48,21 @@ export interface RouterOptions {
|
|
|
24
48
|
*/
|
|
25
49
|
ledger?: string | false;
|
|
26
50
|
/**
|
|
27
|
-
* Providers the
|
|
28
|
-
* passed as of this release (`VERIFIED`). Pass your own
|
|
51
|
+
* Providers that passed the real-account check: tried first by strategy "verified", and the only ones
|
|
52
|
+
* used with `requireVerified`. Default: those that passed as of this release (`VERIFIED`). Pass your own
|
|
53
|
+
* list, e.g. after running `check` yourself.
|
|
29
54
|
*/
|
|
30
55
|
verified?: ProviderId[];
|
|
56
|
+
/** Providers whose last real-account check failed. The default strategy tries them last. Default: `FAILING`. */
|
|
57
|
+
failing?: ProviderId[];
|
|
58
|
+
/**
|
|
59
|
+
* Only route to providers in `verified`; the rest are skipped. For services that run sandboxes on their own
|
|
60
|
+
* accounts, where a provider that misbehaves without failing (keeps billing after kill, garbles output)
|
|
61
|
+
* costs money. Default false.
|
|
62
|
+
*/
|
|
63
|
+
requireVerified?: boolean;
|
|
64
|
+
/** Billed / list-price ratio per provider, behind `estimatedHourlyUsd`. Default: the ratios in `BILLED_VS_LIST`, from saved cost runs. */
|
|
65
|
+
billedVsList?: Partial<Record<ProviderId, number>>;
|
|
31
66
|
}
|
|
32
67
|
/** Desktop-only providers. Filled in as adapters are added. */
|
|
33
68
|
export declare function defaultComputerProviders(): ComputerProvider[];
|
|
@@ -36,6 +71,11 @@ export declare class Router {
|
|
|
36
71
|
private providers;
|
|
37
72
|
private benchFile?;
|
|
38
73
|
private readonly verified;
|
|
74
|
+
private readonly failing;
|
|
75
|
+
private readonly requireVerified;
|
|
76
|
+
private readonly billed;
|
|
77
|
+
/** Create times this router measured, most recent last. */
|
|
78
|
+
private readonly live;
|
|
39
79
|
private computerOnly;
|
|
40
80
|
/** The local record of sandboxes this router created, or undefined when turned off. */
|
|
41
81
|
readonly ledger?: Ledger;
|
|
@@ -54,6 +94,8 @@ export declare class Router {
|
|
|
54
94
|
private whyNone;
|
|
55
95
|
/** Start a sandbox on the best available provider, falling back down the list on failure. */
|
|
56
96
|
create(spec?: SandboxSpec, route?: RouteOptions): Promise<RoutedSandbox>;
|
|
97
|
+
/** Remember a successful create's time, for the speed tie-break and strategy "fastest". */
|
|
98
|
+
private observe;
|
|
57
99
|
private record;
|
|
58
100
|
private discard;
|
|
59
101
|
/** Every provider that sells computers (desktops/phones): sandbox providers that do, plus desktop-only ones. */
|
|
@@ -70,18 +112,31 @@ export declare class Router {
|
|
|
70
112
|
/** Reattach to a sandbox by its routed id (`provider:nativeId`). */
|
|
71
113
|
get(id: string): Promise<RoutedSandbox>;
|
|
72
114
|
}
|
|
115
|
+
/** What a sandbox costs and its size, as the router priced it when it was created. */
|
|
116
|
+
export interface SandboxPrice {
|
|
117
|
+
/** Expected cost, USD/hour: list price x what the provider really billed against it, where measured. */
|
|
118
|
+
estimatedHourlyUsd: number;
|
|
119
|
+
/** The provider's published list price for this size, USD/hour. */
|
|
120
|
+
listHourlyUsd: number;
|
|
121
|
+
size: Size;
|
|
122
|
+
}
|
|
73
123
|
export declare class RoutedSandbox {
|
|
74
124
|
readonly provider: ProviderId;
|
|
75
125
|
private box;
|
|
76
126
|
/** Every create attempt, including failed fallbacks. */
|
|
77
127
|
readonly attempts: Attempt[];
|
|
78
|
-
readonly estimatedHourlyUsd: number;
|
|
79
128
|
private ledger?;
|
|
80
129
|
/** Stable id across processes: `provider:nativeId`. */
|
|
81
130
|
readonly id: string;
|
|
131
|
+
/** Expected cost, USD/hour (see `Candidate.estimatedHourlyUsd`). Undefined for a handle from `get()` this machine didn't create. */
|
|
132
|
+
readonly estimatedHourlyUsd?: number;
|
|
133
|
+
/** The provider's list price for this size, USD/hour. */
|
|
134
|
+
readonly listHourlyUsd?: number;
|
|
135
|
+
/** The size it was created at (see `Candidate.size`). */
|
|
136
|
+
readonly size?: Size;
|
|
82
137
|
constructor(provider: ProviderId, box: ProviderSandbox,
|
|
83
138
|
/** Every create attempt, including failed fallbacks. */
|
|
84
|
-
attempts: Attempt[],
|
|
139
|
+
attempts: Attempt[], price?: SandboxPrice, ledger?: Ledger | undefined);
|
|
85
140
|
get nativeId(): string;
|
|
86
141
|
/**
|
|
87
142
|
* Run a command. If it outlives `opts.timeoutSeconds` the adapter stops it and this rejects with
|
|
@@ -103,7 +158,18 @@ export declare function parseId(id: string): {
|
|
|
103
158
|
provider: ProviderId;
|
|
104
159
|
nativeId: string;
|
|
105
160
|
};
|
|
161
|
+
/**
|
|
162
|
+
* The default order. Providers whose last real-account check failed go last. The rest go cheapest
|
|
163
|
+
* first by effective price (list price x what the provider really billed, where measured), except that
|
|
164
|
+
* providers within PRICE_TIE of the cheapest remaining one count as a tie and go fastest first (by
|
|
165
|
+
* median create time; unknown speed after known), then cheapest, then alphabetical.
|
|
166
|
+
*/
|
|
167
|
+
export declare function balanced(list: Candidate[]): Candidate[];
|
|
106
168
|
/** Seconds to wait for one provider's create before trying the next (RouteOptions.createTimeoutSeconds). */
|
|
107
169
|
export declare const DEFAULT_CREATE_TIMEOUT_S = 120;
|
|
170
|
+
/** Seconds a whole create may take over every provider it tries (RouteOptions.deadlineSeconds). */
|
|
171
|
+
export declare const DEFAULT_CREATE_DEADLINE_S = 240;
|
|
108
172
|
/** Seconds past a command's timeout before the router stops waiting for the adapter. */
|
|
109
173
|
export declare const EXEC_GRACE_S = 30;
|
|
174
|
+
/** The size a provider gives for a spec: its fixed default image size, else what was asked for. */
|
|
175
|
+
export declare function sizeOf(id: ProviderId, spec?: SandboxSpec): Size;
|