@sandboxrouter/sdk 0.3.5 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/bench-store.d.ts +2 -1
- package/dist/bench-store.js +6 -3
- package/dist/billing.d.ts +5 -0
- package/dist/billing.js +3 -0
- package/dist/check.d.ts +5 -0
- package/dist/check.js +12 -1
- package/dist/cli.js +17 -6
- package/dist/cost.d.ts +11 -0
- package/dist/cost.js +43 -1
- package/dist/router.d.ts +39 -3
- package/dist/router.js +72 -13
- package/dist/types.d.ts +11 -6
- package/dist/verified.d.ts +2 -0
- package/dist/verified.js +2 -0
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -21,7 +21,7 @@ try {
|
|
|
21
21
|
|
|
22
22
|
Runs as plain JavaScript on Node 20+. (In TypeScript 5.2+, or Node 24+, `await using sbx = ...` kills the sandbox for you.)
|
|
23
23
|
|
|
24
|
-
Set the keys of the providers you use. `create()` starts the sandbox on the cheapest one that fits
|
|
24
|
+
Set the keys of the providers you use. `create()` starts the sandbox on the cheapest one that fits: list price for the size you asked for, corrected where we measured what a provider really bills. Providers within 10% of each other on price go fastest first, by start times measured where you run the router. Providers that failed their last real-account check go last. Ties are alphabetical, and no provider pays for placement. If that provider is down, busy or out of credit, the next one takes it, and every attempt is on `sbx.attempts`. The sandbox stays on that provider until you `kill()` it.
|
|
25
25
|
|
|
26
26
|
Site and docs: [sandboxrouter.vercel.app](https://sandboxrouter.vercel.app) · npm: [`@sandboxrouter/sdk`](https://www.npmjs.com/package/@sandboxrouter/sdk)
|
|
27
27
|
|
|
@@ -30,7 +30,7 @@ Site and docs: [sandboxrouter.vercel.app](https://sandboxrouter.vercel.app) · n
|
|
|
30
30
|
AI agents need a disposable computer to run code a model wrote: install packages, run tests, do an RL rollout. Dozens of providers sell one, each with its own SDK and auth, and the same 2 vCPU / 4 GB sandbox costs from under a cent to over thirty cents an hour. sandboxrouter is one interface over all of them:
|
|
31
31
|
|
|
32
32
|
- **Same code everywhere.** `create`, `exec` (streamed), `writeFile`, `readFile`, `pause`, `kill`. Switch providers by changing which keys are set.
|
|
33
|
-
- **
|
|
33
|
+
- **Cheapest real cost, then speed, with fallback.** Or `cheapest` for list price alone, `fastest` by start times, `verified` for tested providers first, or a list of providers you name.
|
|
34
34
|
- **Your keys.** Your accounts and your bills; nothing is proxied through anyone else.
|
|
35
35
|
- **Cleanup.** A local ledger of every sandbox started, so `sandboxrouter gc` kills what a crashed script left behind.
|
|
36
36
|
- **TypeScript, Python, HTTP and a CLI.** Plus desktops and phones for computer-use models.
|
package/dist/bench-store.d.ts
CHANGED
|
@@ -6,7 +6,8 @@ export interface BenchSample {
|
|
|
6
6
|
at: string;
|
|
7
7
|
}
|
|
8
8
|
export type BenchData = Partial<Record<ProviderId, BenchSample>>;
|
|
9
|
-
|
|
9
|
+
/** $SANDBOXROUTER_BENCH_FILE, else bench.json next to the ledger: $SANDBOXROUTER_HOME, else ~/.sandboxrouter/. */
|
|
10
|
+
export declare function defaultBenchFile(): string;
|
|
10
11
|
export declare function loadBench(file?: string): BenchData;
|
|
11
12
|
export declare function saveBench(data: BenchData, file?: string): void;
|
|
12
13
|
export declare function median(xs?: number[]): number | undefined;
|
package/dist/bench-store.js
CHANGED
|
@@ -1,8 +1,11 @@
|
|
|
1
1
|
import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
|
2
2
|
import { homedir } from "node:os";
|
|
3
3
|
import { dirname, join } from "node:path";
|
|
4
|
-
|
|
5
|
-
export function
|
|
4
|
+
/** $SANDBOXROUTER_BENCH_FILE, else bench.json next to the ledger: $SANDBOXROUTER_HOME, else ~/.sandboxrouter/. */
|
|
5
|
+
export function defaultBenchFile() {
|
|
6
|
+
return process.env.SANDBOXROUTER_BENCH_FILE || join(process.env.SANDBOXROUTER_HOME || join(homedir(), ".sandboxrouter"), "bench.json");
|
|
7
|
+
}
|
|
8
|
+
export function loadBench(file = defaultBenchFile()) {
|
|
6
9
|
try {
|
|
7
10
|
return JSON.parse(readFileSync(file, "utf8"));
|
|
8
11
|
}
|
|
@@ -10,7 +13,7 @@ export function loadBench(file = DEFAULT_BENCH_FILE) {
|
|
|
10
13
|
return {};
|
|
11
14
|
}
|
|
12
15
|
}
|
|
13
|
-
export function saveBench(data, file =
|
|
16
|
+
export function saveBench(data, file = defaultBenchFile()) {
|
|
14
17
|
mkdirSync(dirname(file), { recursive: true });
|
|
15
18
|
writeFileSync(file, JSON.stringify(data, null, 2));
|
|
16
19
|
}
|
package/dist/billing.js
ADDED
package/dist/check.d.ts
CHANGED
|
@@ -15,6 +15,11 @@ export interface CheckReport {
|
|
|
15
15
|
results: CheckResult[];
|
|
16
16
|
/** True when no step failed. */
|
|
17
17
|
ok: boolean;
|
|
18
|
+
/**
|
|
19
|
+
* Set when the run says nothing about the provider: the account running it couldn't pay (402: no
|
|
20
|
+
* balance, plan not started). Such a run isn't saved, so the provider keeps its last real result.
|
|
21
|
+
*/
|
|
22
|
+
inconclusive?: string;
|
|
18
23
|
}
|
|
19
24
|
/**
|
|
20
25
|
* One real pass over a provider: create, exec (stdout, stderr, exit code, env, cwd), streaming,
|
package/dist/check.js
CHANGED
|
@@ -42,10 +42,21 @@ export async function checkProvider(router, provider, opts = {}) {
|
|
|
42
42
|
}
|
|
43
43
|
};
|
|
44
44
|
let sbx;
|
|
45
|
+
let unpaid;
|
|
45
46
|
await step("create", async () => {
|
|
46
|
-
|
|
47
|
+
try {
|
|
48
|
+
sbx = await router.create({ timeoutMinutes: 10, ...fixedSize(provider), ...opts.spec }, { provider, fallback: false });
|
|
49
|
+
}
|
|
50
|
+
catch (err) {
|
|
51
|
+
if (err.attempts?.at(-1)?.status !== 402)
|
|
52
|
+
throw err;
|
|
53
|
+
unpaid = err.message;
|
|
54
|
+
return { status: "skip", detail: `the account can't pay (402), so this run says nothing about ${provider}: ${unpaid}` };
|
|
55
|
+
}
|
|
47
56
|
return `${sbx.id}, ~$${sbx.estimatedHourlyUsd}/hr`;
|
|
48
57
|
});
|
|
58
|
+
if (unpaid)
|
|
59
|
+
return { provider, results, ok: false, inconclusive: "the account running the check can't pay (402)" };
|
|
49
60
|
if (!sbx)
|
|
50
61
|
return { provider, results, ok: false };
|
|
51
62
|
const box = sbx;
|
package/dist/cli.js
CHANGED
|
@@ -39,7 +39,7 @@ Usage:
|
|
|
39
39
|
sandboxrouter serve [--port 8787] [--host 127.0.0.1] [--cors https://site.example,http://localhost:3000]
|
|
40
40
|
|
|
41
41
|
Spec flags: --vcpu N --memory GB --gpu T4 --timeout MIN
|
|
42
|
-
Route flags: --provider e2b[,modal,...] --strategy verified|cheapest|fastest (default
|
|
42
|
+
Route flags: --provider e2b[,modal,...] --strategy balanced|verified|cheapest|fastest (default balanced: cheapest real cost, speed breaks near-ties)
|
|
43
43
|
--no-fallback --fallback-any (also on setup errors like a bad key)
|
|
44
44
|
--create-timeout SEC (per provider, default 120) --require gpu,custom-image
|
|
45
45
|
|
|
@@ -156,8 +156,11 @@ async function main() {
|
|
|
156
156
|
let n = 0;
|
|
157
157
|
for (const c of plan) {
|
|
158
158
|
const mark = c.skip ? " -" : `${++n}.`.padStart(3);
|
|
159
|
-
const tested = c.verified ? "tested
|
|
160
|
-
|
|
159
|
+
const tested = c.failing ? "check failed " : c.verified ? "tested " : " ";
|
|
160
|
+
// The billed column only shows where a measured bill moved it off list price.
|
|
161
|
+
const real = c.effectiveHourlyUsd !== c.estimatedHourlyUsd ? ` (billed ~$${c.effectiveHourlyUsd.toFixed(4)})` : "";
|
|
162
|
+
const speed = c.medianCreateMs !== undefined ? `${c.medianCreateMs} ms ` : "";
|
|
163
|
+
console.log(`${mark} ${c.provider.padEnd(11)} $${c.estimatedHourlyUsd.toFixed(4)}/hr${real} ${speed}${tested}${c.skip ? `(skipped: ${c.skip})` : ""}`.trimEnd());
|
|
161
164
|
}
|
|
162
165
|
return;
|
|
163
166
|
}
|
|
@@ -214,6 +217,11 @@ async function main() {
|
|
|
214
217
|
},
|
|
215
218
|
});
|
|
216
219
|
reports.push(report);
|
|
220
|
+
if (report.inconclusive) {
|
|
221
|
+
if (!values.json)
|
|
222
|
+
console.log(` not saved: ${report.inconclusive}; ${p} keeps its last saved result`);
|
|
223
|
+
continue;
|
|
224
|
+
}
|
|
217
225
|
if (saveDir) {
|
|
218
226
|
mkdirSync(saveDir, { recursive: true });
|
|
219
227
|
const saved = publishableCheck({ tool: "sandboxrouter-check", version, date: new Date().toISOString(), platform: `${platform()}-${arch()}`, ...(by ? { by } : {}), ...report });
|
|
@@ -222,9 +230,12 @@ async function main() {
|
|
|
222
230
|
}
|
|
223
231
|
if (values.json)
|
|
224
232
|
console.log(JSON.stringify(reports, null, 2));
|
|
225
|
-
else
|
|
226
|
-
|
|
227
|
-
|
|
233
|
+
else {
|
|
234
|
+
const unpaid = reports.filter((r) => r.inconclusive).length;
|
|
235
|
+
const ran = reports.length - unpaid;
|
|
236
|
+
console.log(`\n${reports.filter((r) => r.ok).length}/${ran} passed${unpaid ? `; ${unpaid} not run (the account can't pay)` : ""}`);
|
|
237
|
+
}
|
|
238
|
+
if (reports.some((r) => !r.ok && !r.inconclusive))
|
|
228
239
|
process.exitCode = 1;
|
|
229
240
|
return;
|
|
230
241
|
}
|
package/dist/cost.d.ts
CHANGED
|
@@ -74,4 +74,15 @@ type Reader = (e: CostEntry) => Promise<Omit<Bill, "at">>;
|
|
|
74
74
|
export declare const BILL_READERS: Partial<Record<ProviderId, Reader>>;
|
|
75
75
|
export declare function settle(run: CostRun): Promise<CostRun>;
|
|
76
76
|
export declare function costMarkdown(run: CostRun): string;
|
|
77
|
+
/**
|
|
78
|
+
* Billed / predicted per provider, from the saved cost runs in `dir` (default results/cost): the latest
|
|
79
|
+
* entry whose bill is precise enough to compare. A bill rounded to whole cents only counts when the
|
|
80
|
+
* prediction is at least $1, so rounding can't move the ratio by more than 1%. Providers whose terms
|
|
81
|
+
* forbid publishing benchmarks are left out, like everywhere else results are published.
|
|
82
|
+
*/
|
|
83
|
+
export declare function billedVsList(dir?: string): {
|
|
84
|
+
provider: ProviderId;
|
|
85
|
+
ratio: number;
|
|
86
|
+
date: string;
|
|
87
|
+
}[];
|
|
77
88
|
export {};
|
package/dist/cost.js
CHANGED
|
@@ -1,4 +1,6 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { readdirSync, readFileSync } from "node:fs";
|
|
2
|
+
import { join } from "node:path";
|
|
3
|
+
import { benchmarksPublishable, fittingSpec, hourlyCost } from "./catalog.js";
|
|
2
4
|
import { Router } from "./router.js";
|
|
3
5
|
/**
|
|
4
6
|
* The size the sandbox really has, as "<vcpus> <memoryMb>". nproc and MemTotal can show the whole host
|
|
@@ -163,3 +165,43 @@ export function costMarkdown(run) {
|
|
|
163
165
|
}
|
|
164
166
|
return lines.join("\n");
|
|
165
167
|
}
|
|
168
|
+
/**
|
|
169
|
+
* Billed / predicted per provider, from the saved cost runs in `dir` (default results/cost): the latest
|
|
170
|
+
* entry whose bill is precise enough to compare. A bill rounded to whole cents only counts when the
|
|
171
|
+
* prediction is at least $1, so rounding can't move the ratio by more than 1%. Providers whose terms
|
|
172
|
+
* forbid publishing benchmarks are left out, like everywhere else results are published.
|
|
173
|
+
*/
|
|
174
|
+
export function billedVsList(dir = "results/cost") {
|
|
175
|
+
let files;
|
|
176
|
+
try {
|
|
177
|
+
files = readdirSync(dir).filter((f) => f.endsWith(".json"));
|
|
178
|
+
}
|
|
179
|
+
catch {
|
|
180
|
+
return [];
|
|
181
|
+
}
|
|
182
|
+
const latest = new Map();
|
|
183
|
+
for (const f of files) {
|
|
184
|
+
let run;
|
|
185
|
+
try {
|
|
186
|
+
run = JSON.parse(readFileSync(join(dir, f), "utf8"));
|
|
187
|
+
}
|
|
188
|
+
catch {
|
|
189
|
+
continue;
|
|
190
|
+
}
|
|
191
|
+
if (run?.tool !== "sandboxrouter-cost" || !Array.isArray(run.entries))
|
|
192
|
+
continue;
|
|
193
|
+
for (const e of run.entries) {
|
|
194
|
+
const usd = e.bill?.usd;
|
|
195
|
+
const predicted = e.predictedUsd;
|
|
196
|
+
if (!usd || !predicted || usd <= 0 || predicted <= 0 || !benchmarksPublishable(e.provider))
|
|
197
|
+
continue;
|
|
198
|
+
if (/integer cents/i.test(e.bill.source) && predicted < 1)
|
|
199
|
+
continue;
|
|
200
|
+
const date = run.date.slice(0, 10);
|
|
201
|
+
const prev = latest.get(e.provider);
|
|
202
|
+
if (!prev || date >= prev.date)
|
|
203
|
+
latest.set(e.provider, { ratio: Math.round((usd / predicted) * 1000) / 1000, date });
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
return [...latest].map(([provider, v]) => ({ provider, ...v })).sort((a, b) => a.provider.localeCompare(b.provider));
|
|
207
|
+
}
|
package/dist/router.d.ts
CHANGED
|
@@ -3,17 +3,28 @@ import { Ledger } from "./ledger.js";
|
|
|
3
3
|
import { type Attempt, type ExecOptions, type ExecResult, type Provider, type ProviderId, type ProviderSandbox, type RouteOptions, type SandboxSpec } from "./types.js";
|
|
4
4
|
export interface Candidate {
|
|
5
5
|
provider: ProviderId;
|
|
6
|
+
/** List price for the spec, USD/hour. What the sandbox is expected to cost at the provider's published rates. */
|
|
6
7
|
estimatedHourlyUsd: number;
|
|
8
|
+
/**
|
|
9
|
+
* List price x what the provider really billed against list price, where that was measured
|
|
10
|
+
* (`BILLED_VS_LIST`, or `RouterOptions.billedVsList`); the list price otherwise. The default strategy ranks by this.
|
|
11
|
+
*/
|
|
12
|
+
effectiveHourlyUsd: number;
|
|
13
|
+
/** Median create time: this router's own recent creates when it has a few, else your `bench` file. */
|
|
7
14
|
medianCreateMs?: number;
|
|
8
15
|
/** Passed the real-account check (as of this release, or `RouterOptions.verified`). */
|
|
9
16
|
verified: boolean;
|
|
17
|
+
/** Failed a step of its last real-account check (as of this release, or `RouterOptions.failing`). */
|
|
18
|
+
failing: boolean;
|
|
10
19
|
/** Why this provider won't be tried, if it won't. */
|
|
11
20
|
skip?: string;
|
|
12
21
|
}
|
|
22
|
+
/** Providers within this share of the cheapest remaining price count as a tie, broken by speed. */
|
|
23
|
+
export declare const PRICE_TIE = 0.1;
|
|
13
24
|
export interface RouterOptions {
|
|
14
25
|
/** Replace the built-in adapters (tests, custom providers). */
|
|
15
26
|
providers?: Provider[];
|
|
16
|
-
/** Path of the bench results file
|
|
27
|
+
/** Path of the bench results file: start times for the speed tie-break and strategy "fastest". */
|
|
17
28
|
benchFile?: string;
|
|
18
29
|
/** Desktop-only computer providers (sandbox providers that also sell computers are found automatically). */
|
|
19
30
|
computers?: ComputerProvider[];
|
|
@@ -24,10 +35,21 @@ export interface RouterOptions {
|
|
|
24
35
|
*/
|
|
25
36
|
ledger?: string | false;
|
|
26
37
|
/**
|
|
27
|
-
* Providers the
|
|
28
|
-
* passed as of this release (`VERIFIED`). Pass your own
|
|
38
|
+
* Providers that passed the real-account check: tried first by strategy "verified", and the only ones
|
|
39
|
+
* used with `requireVerified`. Default: those that passed as of this release (`VERIFIED`). Pass your own
|
|
40
|
+
* list, e.g. after running `check` yourself.
|
|
29
41
|
*/
|
|
30
42
|
verified?: ProviderId[];
|
|
43
|
+
/** Providers whose last real-account check failed. The default strategy tries them last. Default: `FAILING`. */
|
|
44
|
+
failing?: ProviderId[];
|
|
45
|
+
/**
|
|
46
|
+
* Only route to providers in `verified`; the rest are skipped. For services that run sandboxes on their own
|
|
47
|
+
* accounts, where a provider that misbehaves without failing (keeps billing after kill, garbles output)
|
|
48
|
+
* costs money. Default false.
|
|
49
|
+
*/
|
|
50
|
+
requireVerified?: boolean;
|
|
51
|
+
/** Billed / list-price ratio per provider, for the default ranking. Default: `BILLED_VS_LIST` from saved cost runs. */
|
|
52
|
+
billedVsList?: Partial<Record<ProviderId, number>>;
|
|
31
53
|
}
|
|
32
54
|
/** Desktop-only providers. Filled in as adapters are added. */
|
|
33
55
|
export declare function defaultComputerProviders(): ComputerProvider[];
|
|
@@ -36,6 +58,11 @@ export declare class Router {
|
|
|
36
58
|
private providers;
|
|
37
59
|
private benchFile?;
|
|
38
60
|
private readonly verified;
|
|
61
|
+
private readonly failing;
|
|
62
|
+
private readonly requireVerified;
|
|
63
|
+
private readonly billed;
|
|
64
|
+
/** Create times this router measured, most recent last. */
|
|
65
|
+
private readonly live;
|
|
39
66
|
private computerOnly;
|
|
40
67
|
/** The local record of sandboxes this router created, or undefined when turned off. */
|
|
41
68
|
readonly ledger?: Ledger;
|
|
@@ -54,6 +81,8 @@ export declare class Router {
|
|
|
54
81
|
private whyNone;
|
|
55
82
|
/** Start a sandbox on the best available provider, falling back down the list on failure. */
|
|
56
83
|
create(spec?: SandboxSpec, route?: RouteOptions): Promise<RoutedSandbox>;
|
|
84
|
+
/** Remember a successful create's time, for the speed tie-break and strategy "fastest". */
|
|
85
|
+
private observe;
|
|
57
86
|
private record;
|
|
58
87
|
private discard;
|
|
59
88
|
/** Every provider that sells computers (desktops/phones): sandbox providers that do, plus desktop-only ones. */
|
|
@@ -103,6 +132,13 @@ export declare function parseId(id: string): {
|
|
|
103
132
|
provider: ProviderId;
|
|
104
133
|
nativeId: string;
|
|
105
134
|
};
|
|
135
|
+
/**
|
|
136
|
+
* The default order. Providers whose last real-account check failed go last. The rest go cheapest
|
|
137
|
+
* first by effective price (list price x what the provider really billed, where measured), except that
|
|
138
|
+
* providers within PRICE_TIE of the cheapest remaining one count as a tie and go fastest first (by
|
|
139
|
+
* median create time; unknown speed after known), then cheapest, then alphabetical.
|
|
140
|
+
*/
|
|
141
|
+
export declare function balanced(list: Candidate[]): Candidate[];
|
|
106
142
|
/** Seconds to wait for one provider's create before trying the next (RouteOptions.createTimeoutSeconds). */
|
|
107
143
|
export declare const DEFAULT_CREATE_TIMEOUT_S = 120;
|
|
108
144
|
/** Seconds past a command's timeout before the router stops waiting for the adapter. */
|
package/dist/router.js
CHANGED
|
@@ -3,7 +3,8 @@ import { RoutedComputer, createComputer, planComputer, } from "./computers.js";
|
|
|
3
3
|
import { loadBench, median } from "./bench-store.js";
|
|
4
4
|
import { Ledger, defaultLedgerFile } from "./ledger.js";
|
|
5
5
|
import { DEFAULT_EXEC_TIMEOUT_S, errorStatus, timeoutError } from "./util.js";
|
|
6
|
-
import {
|
|
6
|
+
import { BILLED_VS_LIST } from "./billing.js";
|
|
7
|
+
import { FAILING, VERIFIED } from "./verified.js";
|
|
7
8
|
import { DaytonaProvider } from "./providers/daytona.js";
|
|
8
9
|
import { E2BProvider } from "./providers/e2b.js";
|
|
9
10
|
import { MaritimeProvider } from "./providers/maritime.js";
|
|
@@ -43,6 +44,12 @@ import { MobilerunProvider } from "./providers/mobilerun.js";
|
|
|
43
44
|
import { LimrunProvider } from "./providers/limrun.js";
|
|
44
45
|
import { WorkspacesAgentProvider } from "./providers/workspaces.js";
|
|
45
46
|
import { ProviderError, SetupError, RouterError, } from "./types.js";
|
|
47
|
+
/** Providers within this share of the cheapest remaining price count as a tie, broken by speed. */
|
|
48
|
+
export const PRICE_TIE = 0.1;
|
|
49
|
+
/** Creates this router must have seen for a provider before its own timings replace the bench file's. */
|
|
50
|
+
const LIVE_MIN_SAMPLES = 3;
|
|
51
|
+
/** Recent creates kept per provider for the live median. */
|
|
52
|
+
const LIVE_SAMPLES = 20;
|
|
46
53
|
/** Desktop-only providers. Filled in as adapters are added. */
|
|
47
54
|
export function defaultComputerProviders() {
|
|
48
55
|
return [new OrgoProvider(), new MobilerunProvider(), new LimrunProvider(), new WorkspacesAgentProvider()];
|
|
@@ -89,6 +96,11 @@ export class Router {
|
|
|
89
96
|
providers;
|
|
90
97
|
benchFile;
|
|
91
98
|
verified;
|
|
99
|
+
failing;
|
|
100
|
+
requireVerified;
|
|
101
|
+
billed;
|
|
102
|
+
/** Create times this router measured, most recent last. */
|
|
103
|
+
live = new Map();
|
|
92
104
|
computerOnly;
|
|
93
105
|
/** The local record of sandboxes this router created, or undefined when turned off. */
|
|
94
106
|
ledger;
|
|
@@ -99,6 +111,10 @@ export class Router {
|
|
|
99
111
|
this.providers = new Map((opts.providers ?? defaultProviders()).map((p) => [p.id, p]));
|
|
100
112
|
this.benchFile = opts.benchFile;
|
|
101
113
|
this.verified = new Set(opts.verified ?? Object.keys(VERIFIED));
|
|
114
|
+
this.failing = new Set(opts.failing ?? Object.keys(FAILING));
|
|
115
|
+
this.requireVerified = opts.requireVerified ?? false;
|
|
116
|
+
this.billed =
|
|
117
|
+
opts.billedVsList ?? Object.fromEntries(Object.entries(BILLED_VS_LIST).map(([id, b]) => [id, b.ratio]));
|
|
102
118
|
this.computerOnly = opts.computers ?? (opts.providers ? [] : defaultComputerProviders());
|
|
103
119
|
}
|
|
104
120
|
/** Registered and has credentials. False for providers this router doesn't know. */
|
|
@@ -121,40 +137,52 @@ export class Router {
|
|
|
121
137
|
// Local providers (Docker) are never picked automatically: they always "cost" $0 and isolate
|
|
122
138
|
// less than a cloud micro-VM. Name them explicitly to use them.
|
|
123
139
|
const ids = explicit ?? CATALOG.filter((c) => !c.local && this.providers.has(c.id)).map((c) => c.id);
|
|
124
|
-
const
|
|
140
|
+
const strategy = route.strategy ?? "default";
|
|
141
|
+
const speed = strategy === "default" || strategy === "balanced" || strategy === "fastest";
|
|
142
|
+
const bench = speed ? loadBench(this.benchFile) : {};
|
|
125
143
|
const list = ids.map((id) => {
|
|
126
144
|
// An id the catalog has never heard of (a typo in an explicit list) has no price to look up.
|
|
127
145
|
if (!CATALOG.some((c) => c.id === id))
|
|
128
|
-
return { provider: id, estimatedHourlyUsd: 0, verified: false, skip: "unknown provider" };
|
|
146
|
+
return { provider: id, estimatedHourlyUsd: 0, effectiveHourlyUsd: 0, verified: false, failing: false, skip: "unknown provider" };
|
|
129
147
|
const p = this.providers.get(id);
|
|
148
|
+
const list = round(hourlyCost(id, spec));
|
|
149
|
+
const live = this.live.get(id);
|
|
130
150
|
const c = {
|
|
131
151
|
provider: id,
|
|
132
|
-
estimatedHourlyUsd:
|
|
133
|
-
|
|
152
|
+
estimatedHourlyUsd: list,
|
|
153
|
+
effectiveHourlyUsd: round(list * (this.billed[id] ?? 1)),
|
|
154
|
+
medianCreateMs: speed ? (live && live.length >= LIVE_MIN_SAMPLES ? median(live) : median(bench[id]?.createMs)) : undefined,
|
|
134
155
|
verified: this.verified.has(id),
|
|
156
|
+
failing: this.failing.has(id),
|
|
135
157
|
};
|
|
136
158
|
if (!p)
|
|
137
159
|
c.skip = "no adapter registered";
|
|
138
160
|
else {
|
|
139
161
|
const gated = accessLabel(id);
|
|
140
|
-
c.skip =
|
|
162
|
+
c.skip =
|
|
163
|
+
unfit(id, spec, route.require) ??
|
|
164
|
+
(p.configured() ? undefined : gated ? `not configured; ${gated}` : "not configured") ??
|
|
165
|
+
(this.requireVerified && !c.verified ? "not tested (this router only uses providers that passed the real-account check)" : undefined);
|
|
141
166
|
}
|
|
142
167
|
return c;
|
|
143
168
|
});
|
|
144
|
-
// Neutral by construction: the same rule for every provider, ties alphabetical by id.
|
|
145
|
-
//
|
|
146
|
-
// An explicit
|
|
169
|
+
// Neutral by construction: the same rule for every provider, ties alphabetical by id. Nothing here
|
|
170
|
+
// depends on who the provider is, only on its price, what it really billed, how fast it started for
|
|
171
|
+
// this router and whether its last real-account check failed. An explicit list keeps the caller's order.
|
|
147
172
|
if (!explicit) {
|
|
148
173
|
const byPrice = (a, b) => a.estimatedHourlyUsd - b.estimatedHourlyUsd || a.provider.localeCompare(b.provider);
|
|
149
|
-
if (
|
|
174
|
+
if (strategy === "fastest") {
|
|
150
175
|
list.sort((a, b) => (a.medianCreateMs ?? Infinity) - (b.medianCreateMs ?? Infinity) || byPrice(a, b));
|
|
151
176
|
}
|
|
152
|
-
else if (
|
|
177
|
+
else if (strategy === "cheapest") {
|
|
153
178
|
list.sort(byPrice);
|
|
154
179
|
}
|
|
155
|
-
else {
|
|
180
|
+
else if (strategy === "verified") {
|
|
156
181
|
list.sort((a, b) => Number(b.verified) - Number(a.verified) || byPrice(a, b));
|
|
157
182
|
}
|
|
183
|
+
else {
|
|
184
|
+
return balanced(list);
|
|
185
|
+
}
|
|
158
186
|
}
|
|
159
187
|
return list;
|
|
160
188
|
}
|
|
@@ -205,6 +233,7 @@ export class Router {
|
|
|
205
233
|
return new ProviderError(c.provider, `create timed out after ${limitS}s`);
|
|
206
234
|
});
|
|
207
235
|
attempts.push({ provider: c.provider, ok: true, ms: Date.now() - t0 });
|
|
236
|
+
this.observe(c.provider, Date.now() - t0);
|
|
208
237
|
const sbx = new RoutedSandbox(c.provider, box, attempts, c.estimatedHourlyUsd, this.ledger);
|
|
209
238
|
this.record(sbx, c.estimatedHourlyUsd, spec.timeoutMinutes);
|
|
210
239
|
return sbx;
|
|
@@ -214,8 +243,8 @@ export class Router {
|
|
|
214
243
|
// so it can't bill unseen. If the kill fails, the ledger entry is left for `gc`.
|
|
215
244
|
if (timedOut)
|
|
216
245
|
pending.then((late) => this.discard(c.provider, late, c.estimatedHourlyUsd, spec.timeoutMinutes), () => { });
|
|
217
|
-
attempts.push({ provider: c.provider, ok: false, ms: Date.now() - t0, error: err.message });
|
|
218
246
|
const status = errorStatus(err);
|
|
247
|
+
attempts.push({ provider: c.provider, ok: false, ms: Date.now() - t0, error: err.message, ...(status ? { status } : {}) });
|
|
219
248
|
const setup = err instanceof SetupError || (status !== undefined && SETUP_ERRORS.has(status));
|
|
220
249
|
if (route.fallback !== "any" && setup) {
|
|
221
250
|
const msg = err.message.replace(/[.\s]+$/, "");
|
|
@@ -226,6 +255,14 @@ export class Router {
|
|
|
226
255
|
}
|
|
227
256
|
throw new RouterError(`all providers failed: ${attempts.map((a) => `${a.provider}: ${a.error}`).join(" | ")}`, attempts);
|
|
228
257
|
}
|
|
258
|
+
/** Remember a successful create's time, for the speed tie-break and strategy "fastest". */
|
|
259
|
+
observe(id, ms) {
|
|
260
|
+
const xs = this.live.get(id) ?? [];
|
|
261
|
+
xs.push(ms);
|
|
262
|
+
if (xs.length > LIVE_SAMPLES)
|
|
263
|
+
xs.shift();
|
|
264
|
+
this.live.set(id, xs);
|
|
265
|
+
}
|
|
229
266
|
record(sbx, usdHour, timeoutMinutes) {
|
|
230
267
|
const now = Date.now();
|
|
231
268
|
this.ledger?.open({
|
|
@@ -342,6 +379,28 @@ export function parseId(id) {
|
|
|
342
379
|
function round(n) {
|
|
343
380
|
return Math.round(n * 10000) / 10000;
|
|
344
381
|
}
|
|
382
|
+
/**
|
|
383
|
+
* The default order. Providers whose last real-account check failed go last. The rest go cheapest
|
|
384
|
+
* first by effective price (list price x what the provider really billed, where measured), except that
|
|
385
|
+
* providers within PRICE_TIE of the cheapest remaining one count as a tie and go fastest first (by
|
|
386
|
+
* median create time; unknown speed after known), then cheapest, then alphabetical.
|
|
387
|
+
*/
|
|
388
|
+
export function balanced(list) {
|
|
389
|
+
const byCost = (a, b) => a.effectiveHourlyUsd - b.effectiveHourlyUsd || a.provider.localeCompare(b.provider);
|
|
390
|
+
const bySpeed = (a, b) => (a.medianCreateMs ?? Infinity) - (b.medianCreateMs ?? Infinity) || byCost(a, b);
|
|
391
|
+
const order = (group) => {
|
|
392
|
+
const rest = [...group].sort(byCost);
|
|
393
|
+
const out = [];
|
|
394
|
+
while (rest.length) {
|
|
395
|
+
const limit = rest[0].effectiveHourlyUsd * (1 + PRICE_TIE);
|
|
396
|
+
const tie = rest.filter((c) => c.effectiveHourlyUsd <= limit);
|
|
397
|
+
out.push(...tie.sort(bySpeed));
|
|
398
|
+
rest.splice(0, tie.length);
|
|
399
|
+
}
|
|
400
|
+
return out;
|
|
401
|
+
};
|
|
402
|
+
return [...order(list.filter((c) => !c.failing)), ...order(list.filter((c) => c.failing))];
|
|
403
|
+
}
|
|
345
404
|
/** Seconds to wait for one provider's create before trying the next (RouteOptions.createTimeoutSeconds). */
|
|
346
405
|
export const DEFAULT_CREATE_TIMEOUT_S = 120;
|
|
347
406
|
/**
|
package/dist/types.d.ts
CHANGED
|
@@ -22,17 +22,20 @@ export interface SandboxSpec {
|
|
|
22
22
|
metadata?: Record<string, string>;
|
|
23
23
|
}
|
|
24
24
|
/**
|
|
25
|
-
* "
|
|
26
|
-
*
|
|
27
|
-
*
|
|
25
|
+
* "balanced" (and "default", the same thing): cheapest first by effective price (list price x what the
|
|
26
|
+
* provider really billed, where measured; `BILLED_VS_LIST`), with providers within 10% of each other on
|
|
27
|
+
* price ordered by speed (this router's own recent create times, else your `bench` file). Providers
|
|
28
|
+
* whose last real-account check failed (`FAILING`) go last.
|
|
29
|
+
* "verified": providers that passed the real-account check first (`VERIFIED`), then the rest; cheapest
|
|
30
|
+
* list price first within each.
|
|
28
31
|
* "cheapest": lowest list price for the spec, tested or not, ties alphabetical.
|
|
29
|
-
* "fastest": lowest median
|
|
32
|
+
* "fastest": lowest median create time (this router's own, else your `bench` file), then cheapest.
|
|
30
33
|
*/
|
|
31
|
-
export type Strategy = "default" | "verified" | "cheapest" | "fastest";
|
|
34
|
+
export type Strategy = "default" | "balanced" | "verified" | "cheapest" | "fastest";
|
|
32
35
|
export interface RouteOptions {
|
|
33
36
|
/** Pin to one provider, or give an ordered list to try. Omit to let the router choose. */
|
|
34
37
|
provider?: ProviderId | ProviderId[];
|
|
35
|
-
/** How to order candidates when no explicit list is given. Default: "
|
|
38
|
+
/** How to order candidates when no explicit list is given. Default: "balanced" (cheapest real cost, speed breaks near-ties). */
|
|
36
39
|
strategy?: Strategy;
|
|
37
40
|
/**
|
|
38
41
|
* What to do when a create fails. `true` (default): try the next candidate when the provider
|
|
@@ -86,6 +89,8 @@ export interface Attempt {
|
|
|
86
89
|
ok: boolean;
|
|
87
90
|
ms: number;
|
|
88
91
|
error?: string;
|
|
92
|
+
/** HTTP status of the failure, when the provider gave one (402: the account can't pay). */
|
|
93
|
+
status?: number;
|
|
89
94
|
}
|
|
90
95
|
export declare class RouterError extends Error {
|
|
91
96
|
readonly attempts: Attempt[];
|
package/dist/verified.d.ts
CHANGED
|
@@ -1,2 +1,4 @@
|
|
|
1
1
|
import type { ProviderId } from "./types.js";
|
|
2
2
|
export declare const VERIFIED: Partial<Record<ProviderId, string>>;
|
|
3
|
+
/** Providers whose last saved real-account check failed a step. The default strategy tries them last. */
|
|
4
|
+
export declare const FAILING: Partial<Record<ProviderId, string>>;
|
package/dist/verified.js
CHANGED
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@sandboxrouter/sdk",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.4.0",
|
|
4
4
|
"description": "One API for every sandbox provider: route AI agents' code by price, speed or capability, with automatic fallback. Free and open source.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "dist/index.js",
|