@claxedo/sandbox-manager 0.7.0 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +245 -42
- package/dist/checkpoint-manager.d.ts +29 -4
- package/dist/checkpoint-manager.mjs +133 -24
- package/dist/command.d.ts +9 -2
- package/dist/command.mjs +8 -8
- package/dist/contract.d.ts +687 -0
- package/dist/driver-billing.d.ts +29 -0
- package/dist/driver-catalog.d.ts +43 -58
- package/dist/driver-catalog.mjs +192 -229
- package/dist/drivers/boat-client.d.ts +95 -0
- package/dist/drivers/boat-named-snapshots.d.ts +31 -0
- package/dist/drivers/boat-template.d.ts +49 -0
- package/dist/drivers/boat-template.mjs +1196 -0
- package/dist/drivers/boat.d.ts +64 -0
- package/dist/drivers/boat.mjs +990 -0
- package/dist/drivers/cloudflare.d.ts +5 -8
- package/dist/drivers/cloudflare.mjs +364 -213
- package/dist/drivers/docker.d.ts +4 -3
- package/dist/drivers/docker.mjs +244 -165
- package/dist/drivers/local-brokering.d.ts +14 -0
- package/dist/drivers/local-brokering.mjs +1029 -0
- package/dist/drivers/modal.d.ts +4 -7
- package/dist/drivers/modal.mjs +220 -155
- package/dist/drivers/transient-error.d.ts +13 -0
- package/dist/drivers/vercel.d.ts +21 -23
- package/dist/drivers/vercel.mjs +279 -169
- package/dist/egress-policy.d.ts +134 -0
- package/dist/egress-warnings.d.ts +22 -0
- package/dist/garbage-collect.d.ts +14 -0
- package/dist/hosted-network-policy.d.ts +11 -4
- package/dist/hosted-network-policy.mjs +221 -4
- package/dist/image-name.d.ts +12 -2
- package/dist/image-name.mjs +43 -0
- package/dist/image.d.ts +1 -20
- package/dist/image.mjs +11 -66
- package/dist/index.d.ts +8 -533
- package/dist/index.mjs +1420 -315
- package/dist/json.d.ts +13 -0
- package/dist/lease-failure.d.ts +173 -0
- package/dist/lease-lifecycle.d.ts +65 -0
- package/dist/lease-retry-wait.d.ts +8 -0
- package/dist/lease-start.d.ts +21 -0
- package/dist/lease-types.d.ts +15 -5
- package/dist/manager.d.ts +2 -0
- package/dist/runtime-env.d.ts +15 -2
- package/dist/runtime-env.mjs +50 -1
- package/dist/runtime-snapshot.d.ts +49 -0
- package/dist/runtime-version.d.ts +9 -0
- package/dist/runtime-version.mjs +2 -0
- package/dist/service-url-exposure.d.ts +1 -3
- package/dist/service-url-exposure.mjs +1 -26
- package/dist/start-fresh.d.ts +17 -0
- package/dist/start-telemetry.d.ts +37 -0
- package/dist/stores/memory.d.ts +1 -1
- package/dist/stores/memory.mjs +65 -54
- package/package.json +67 -81
- package/dist/daytona-allow-list.d.ts +0 -16
- package/dist/daytona-allow-list.mjs +0 -11
- package/dist/drivers/box.d.ts +0 -52
- package/dist/drivers/box.mjs +0 -499
- package/dist/drivers/cloudflare-egress.d.ts +0 -40
- package/dist/drivers/cloudflare-egress.mjs +0 -113
- package/dist/drivers/daytona.d.ts +0 -127
- package/dist/drivers/daytona.mjs +0 -541
- package/dist/drivers/exe.d.ts +0 -26
- package/dist/drivers/exe.mjs +0 -469
- package/dist/drivers/fetch-bridge.d.ts +0 -10
- package/dist/drivers/fetch-bridge.mjs +0 -111
- package/dist/lease-policy.d.ts +0 -48
- package/dist/lease-policy.mjs +0 -280
package/dist/index.mjs
CHANGED
|
@@ -1,21 +1,649 @@
|
|
|
1
1
|
// src/constants.ts
|
|
2
2
|
var DEFAULT_WORKSPACE_RUNTIME_PORT = 2593;
|
|
3
3
|
|
|
4
|
+
// src/contract.ts
|
|
5
|
+
function validateSandboxPersistenceCapabilities(input) {
|
|
6
|
+
if (input.resume === "replacement-restore" && input.capture === "none") {
|
|
7
|
+
return { valid: false, reason: "replacement restore requires a capture source" };
|
|
8
|
+
}
|
|
9
|
+
if (input.capture === "none" && input.captureSource !== "not-applicable") {
|
|
10
|
+
return { valid: false, reason: "capture source behavior requires capture support" };
|
|
11
|
+
}
|
|
12
|
+
if (input.capture !== "none" && input.captureSource === "not-applicable") {
|
|
13
|
+
return { valid: false, reason: "capture support requires source behavior" };
|
|
14
|
+
}
|
|
15
|
+
if (input.capture === "same-resource" && input.restoreMount !== "same-resource") {
|
|
16
|
+
return { valid: false, reason: "same-resource capture requires same-resource restore" };
|
|
17
|
+
}
|
|
18
|
+
return { valid: true };
|
|
19
|
+
}
|
|
20
|
+
function sandboxLeaseHeldHost(lease) {
|
|
21
|
+
return lease?.sandboxId !== void 0 && (lease.status === "stopped" || lease.status === "unavailable") ? lease : void 0;
|
|
22
|
+
}
|
|
23
|
+
function applySandboxLeasePatch(current, patch, updatedAt) {
|
|
24
|
+
return {
|
|
25
|
+
...current,
|
|
26
|
+
...patch.status === void 0 ? {} : { status: patch.status },
|
|
27
|
+
...patch.retryCount === void 0 ? {} : { retryCount: patch.retryCount },
|
|
28
|
+
...patch.lastHeartbeatAt === void 0 ? {} : { lastHeartbeatAt: patch.lastHeartbeatAt },
|
|
29
|
+
...patch.lastActivityAt === void 0 ? {} : { lastActivityAt: patch.lastActivityAt },
|
|
30
|
+
...patch.lastHealthFailureAt === void 0 ? {} : { lastHealthFailureAt: patch.lastHealthFailureAt },
|
|
31
|
+
...patch.nextRetryAt === void 0 ? {} : { nextRetryAt: patch.nextRetryAt ?? void 0 },
|
|
32
|
+
...patch.lastError === void 0 ? {} : { lastError: patch.lastError ?? void 0 },
|
|
33
|
+
...patch.checkpoint === void 0 ? {} : { checkpoint: patch.checkpoint ?? void 0 },
|
|
34
|
+
...patch.restore === void 0 ? {} : { restore: patch.restore ?? void 0 },
|
|
35
|
+
...patch.start === void 0 ? {} : { start: patch.start },
|
|
36
|
+
updatedAt
|
|
37
|
+
};
|
|
38
|
+
}
|
|
39
|
+
function applySandboxLeaseFailure(current, error, nextRetryAt, failedAt) {
|
|
40
|
+
return {
|
|
41
|
+
...current,
|
|
42
|
+
status: "unavailable",
|
|
43
|
+
retryCount: current.retryCount + 1,
|
|
44
|
+
lastError: error,
|
|
45
|
+
nextRetryAt,
|
|
46
|
+
lastHealthFailureAt: failedAt,
|
|
47
|
+
updatedAt: failedAt,
|
|
48
|
+
...current.start ? { start: { ...current.start, failures: [...current.start.failures ?? [], { epoch: current.epoch, failedAt, error }] } } : {}
|
|
49
|
+
};
|
|
50
|
+
}
|
|
51
|
+
function applySandboxProvisionedTarget(current, target, updatedAt) {
|
|
52
|
+
const sameTarget = current.status === "ready" && current.url === target.url && current.sandboxId === target.sandboxId && current.hostId === target.hostId && current.driverResourceId === target.driverResourceId;
|
|
53
|
+
return {
|
|
54
|
+
...current,
|
|
55
|
+
status: target.url === void 0 ? "acquiring" : "ready",
|
|
56
|
+
routingId: sameTarget && current.routingId ? current.routingId : crypto.randomUUID(),
|
|
57
|
+
sandboxId: target.sandboxId,
|
|
58
|
+
url: target.url,
|
|
59
|
+
hostId: target.hostId,
|
|
60
|
+
driverResourceId: target.driverResourceId,
|
|
61
|
+
labels: target.labels,
|
|
62
|
+
...target.persistence === void 0 ? {} : { persistence: target.persistence },
|
|
63
|
+
retryCount: 0,
|
|
64
|
+
nextRetryAt: void 0,
|
|
65
|
+
lastError: void 0,
|
|
66
|
+
updatedAt
|
|
67
|
+
};
|
|
68
|
+
}
|
|
69
|
+
var SANDBOX_IMAGE_LABEL = "image";
|
|
70
|
+
function isSandboxListingUnsupported(err) {
|
|
71
|
+
return !!err && typeof err === "object" && err.listingUnsupported === true;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
// src/egress-policy.ts
|
|
75
|
+
function sandboxNetworkPolicyWithBrokeredHosts(policy, secrets) {
|
|
76
|
+
if (policy.mode !== "restricted" || !secrets?.length) return policy;
|
|
77
|
+
const hosts = [.../* @__PURE__ */ new Set([
|
|
78
|
+
...policy.hosts ?? [],
|
|
79
|
+
...secrets.flatMap((secret) => secret.hosts)
|
|
80
|
+
])].toSorted();
|
|
81
|
+
return {
|
|
82
|
+
...policy,
|
|
83
|
+
...hosts.length ? { hosts } : {}
|
|
84
|
+
};
|
|
85
|
+
}
|
|
86
|
+
function sandboxEgressDisposition(control, net) {
|
|
87
|
+
if (!net || net.mode === "allow-all") return { action: "enforce" };
|
|
88
|
+
if (control === "none") return { action: "withhold", reason: "sandbox_egress_uncontained" };
|
|
89
|
+
const hosts = net.hosts?.length ?? 0;
|
|
90
|
+
const cidrs = net.cidrs?.length ?? 0;
|
|
91
|
+
if (hosts === 0 && cidrs === 0) return { action: "enforce" };
|
|
92
|
+
if (hosts > 0) return { action: "enforce" };
|
|
93
|
+
return control === "hosts-and-cidrs" ? { action: "enforce" } : { action: "refuse", reason: "sandbox_egress_policy_unenforceable" };
|
|
94
|
+
}
|
|
95
|
+
function brokeredSecretPlaceholder(name) {
|
|
96
|
+
return `claxedo-broker:${name}`;
|
|
97
|
+
}
|
|
98
|
+
function brokeredPlaceholderEnv(secrets) {
|
|
99
|
+
return Object.fromEntries((secrets ?? []).map((secret) => [secret.name, brokeredSecretPlaceholder(secret.name)]));
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
// src/driver-billing.ts
|
|
103
|
+
var HOUR_SECONDS = 3600;
|
|
104
|
+
var boatBilling = {
|
|
105
|
+
source: { url: "https://docs.boat.dev/pricing", readOn: "2026-10-12" },
|
|
106
|
+
meter: "Boat bills machine time per second while a sandbox runs, at an hourly rate per size; a stopped sandbox costs nothing.",
|
|
107
|
+
rates: {
|
|
108
|
+
small: { size: "small", vcpu: 2, memoryGb: 4, usdPerHour: 0.018 },
|
|
109
|
+
default: { size: "default", vcpu: 4, memoryGb: 8, usdPerHour: 0.036 },
|
|
110
|
+
large: { size: "large", vcpu: 8, memoryGb: 16, usdPerHour: 0.072 }
|
|
111
|
+
}
|
|
112
|
+
};
|
|
113
|
+
var CLOUDFLARE_USD_PER_GIB_SECOND_MEMORY = 25e-7;
|
|
114
|
+
var CLOUDFLARE_USD_PER_GB_SECOND_DISK = 7e-8;
|
|
115
|
+
var CLOUDFLARE_USD_PER_VCPU_SECOND_ACTIVE = 2e-5;
|
|
116
|
+
var CLOUDFLARE_STANDARD_1 = { size: "standard-1", vcpu: 0.5, memoryGb: 4, diskGb: 8 };
|
|
117
|
+
var cloudflareBilling = {
|
|
118
|
+
source: { url: "https://developers.cloudflare.com/containers/pricing/", readOn: "2026-10-12" },
|
|
119
|
+
meter: "Cloudflare bills a container for every 10 ms it runs: memory at $0.0000025 per GiB-second and disk at $0.00000007 per GB-second for the whole run, plus CPU at $0.00002 per vCPU-second of active use.",
|
|
120
|
+
rates: {
|
|
121
|
+
default: {
|
|
122
|
+
size: CLOUDFLARE_STANDARD_1.size,
|
|
123
|
+
vcpu: CLOUDFLARE_STANDARD_1.vcpu,
|
|
124
|
+
memoryGb: CLOUDFLARE_STANDARD_1.memoryGb,
|
|
125
|
+
usdPerHour: (CLOUDFLARE_STANDARD_1.memoryGb * CLOUDFLARE_USD_PER_GIB_SECOND_MEMORY + CLOUDFLARE_STANDARD_1.diskGb * CLOUDFLARE_USD_PER_GB_SECOND_DISK) * HOUR_SECONDS
|
|
126
|
+
}
|
|
127
|
+
},
|
|
128
|
+
excludes: `active CPU, billed at $${CLOUDFLARE_USD_PER_VCPU_SECOND_ACTIVE} per vCPU-second and not measured by Claxedo, and the monthly allowance a Workers Paid plan includes`
|
|
129
|
+
};
|
|
130
|
+
function sandboxMachineRate(billing, machineClass) {
|
|
131
|
+
if (!billing) return void 0;
|
|
132
|
+
return billing.rates[machineClass ?? "default"] ?? (Object.keys(billing.rates).length === 1 ? billing.rates.default : void 0);
|
|
133
|
+
}
|
|
134
|
+
function sandboxMachineCostUsd(rate, activeMs) {
|
|
135
|
+
return rate.usdPerHour * activeMs / (HOUR_SECONDS * 1e3);
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
// ../sandbox-contract/dist/index.mjs
|
|
139
|
+
function ipv4Number(part) {
|
|
140
|
+
let value;
|
|
141
|
+
if (/^0[xX][0-9a-fA-F]+$/.test(part))
|
|
142
|
+
value = Number.parseInt(part.slice(2), 16);
|
|
143
|
+
else if (/^0[0-7]+$/.test(part))
|
|
144
|
+
value = Number.parseInt(part.slice(1), 8);
|
|
145
|
+
else if (/^\d+$/.test(part))
|
|
146
|
+
value = Number.parseInt(part, 10);
|
|
147
|
+
else
|
|
148
|
+
return;
|
|
149
|
+
return Number.isSafeInteger(value) ? value : void 0;
|
|
150
|
+
}
|
|
151
|
+
function parseIpv4(input) {
|
|
152
|
+
const parts = input.split(".");
|
|
153
|
+
if (parts.length > 1 && parts[parts.length - 1] === "")
|
|
154
|
+
parts.pop();
|
|
155
|
+
if (parts.length < 1 || parts.length > 4)
|
|
156
|
+
return;
|
|
157
|
+
const numbers = [];
|
|
158
|
+
for (const part of parts) {
|
|
159
|
+
const value = ipv4Number(part);
|
|
160
|
+
if (value === void 0)
|
|
161
|
+
return;
|
|
162
|
+
numbers.push(value);
|
|
163
|
+
}
|
|
164
|
+
const last = numbers[numbers.length - 1];
|
|
165
|
+
if (last >= 256 ** (5 - numbers.length))
|
|
166
|
+
return;
|
|
167
|
+
for (const value of numbers.slice(0, -1))
|
|
168
|
+
if (value > 255)
|
|
169
|
+
return;
|
|
170
|
+
let address = last;
|
|
171
|
+
for (let i = 0; i < numbers.length - 1; i++)
|
|
172
|
+
address += numbers[i] * 256 ** (3 - i);
|
|
173
|
+
return address >>> 0;
|
|
174
|
+
}
|
|
175
|
+
function hextet(part) {
|
|
176
|
+
return /^[0-9a-fA-F]{1,4}$/.test(part) ? Number.parseInt(part, 16) : void 0;
|
|
177
|
+
}
|
|
178
|
+
function embeddedQuad(input) {
|
|
179
|
+
const parts = input.split(".");
|
|
180
|
+
if (parts.length !== 4)
|
|
181
|
+
return;
|
|
182
|
+
const octets = [];
|
|
183
|
+
for (const part of parts) {
|
|
184
|
+
if (!/^\d{1,3}$/.test(part))
|
|
185
|
+
return;
|
|
186
|
+
const value = Number.parseInt(part, 10);
|
|
187
|
+
if (value > 255)
|
|
188
|
+
return;
|
|
189
|
+
octets.push(value);
|
|
190
|
+
}
|
|
191
|
+
return octets;
|
|
192
|
+
}
|
|
193
|
+
function parseIpv6(input) {
|
|
194
|
+
let body = input;
|
|
195
|
+
let tail;
|
|
196
|
+
const lastColon = body.lastIndexOf(":");
|
|
197
|
+
if (lastColon >= 0 && body.includes(".")) {
|
|
198
|
+
tail = embeddedQuad(body.slice(lastColon + 1));
|
|
199
|
+
if (!tail)
|
|
200
|
+
return;
|
|
201
|
+
body = body.slice(0, lastColon + 1) + "0:0";
|
|
202
|
+
}
|
|
203
|
+
if ((body.match(/::/g) ?? []).length > 1)
|
|
204
|
+
return;
|
|
205
|
+
let halves;
|
|
206
|
+
if (body.includes("::")) {
|
|
207
|
+
const [left = "", right = ""] = body.split("::");
|
|
208
|
+
const leftParts = left === "" ? [] : left.split(":");
|
|
209
|
+
const rightParts = right === "" ? [] : right.split(":");
|
|
210
|
+
const missing = 8 - leftParts.length - rightParts.length;
|
|
211
|
+
if (missing < 1)
|
|
212
|
+
return;
|
|
213
|
+
halves = [...leftParts, ...Array.from({ length: missing }, () => "0"), ...rightParts];
|
|
214
|
+
} else {
|
|
215
|
+
halves = body.split(":");
|
|
216
|
+
if (halves.length !== 8)
|
|
217
|
+
return;
|
|
218
|
+
}
|
|
219
|
+
if (halves.length !== 8)
|
|
220
|
+
return;
|
|
221
|
+
const segments = [];
|
|
222
|
+
for (const part of halves) {
|
|
223
|
+
const value = hextet(part);
|
|
224
|
+
if (value === void 0)
|
|
225
|
+
return;
|
|
226
|
+
segments.push(value);
|
|
227
|
+
}
|
|
228
|
+
if (tail) {
|
|
229
|
+
segments[6] = tail[0] * 256 + tail[1];
|
|
230
|
+
segments[7] = tail[2] * 256 + tail[3];
|
|
231
|
+
}
|
|
232
|
+
return segments;
|
|
233
|
+
}
|
|
234
|
+
function parseIpAddress(input) {
|
|
235
|
+
if (!input)
|
|
236
|
+
return;
|
|
237
|
+
let host = input.trim().toLowerCase();
|
|
238
|
+
if (host.startsWith("[") && host.endsWith("]") && host.includes(":"))
|
|
239
|
+
host = host.slice(1, -1);
|
|
240
|
+
if (host.includes(":")) {
|
|
241
|
+
const segments = parseIpv6(host);
|
|
242
|
+
return segments ? { kind: "v6", segments } : void 0;
|
|
243
|
+
}
|
|
244
|
+
const value = parseIpv4(host);
|
|
245
|
+
return value === void 0 ? void 0 : { kind: "v4", value };
|
|
246
|
+
}
|
|
247
|
+
var V4_PRIVATE = [
|
|
248
|
+
[0, 8],
|
|
249
|
+
[167772160, 8],
|
|
250
|
+
[1681915904, 10],
|
|
251
|
+
[2130706432, 8],
|
|
252
|
+
[2851995648, 16],
|
|
253
|
+
[2886729728, 12],
|
|
254
|
+
[3221225472, 24],
|
|
255
|
+
[3221225984, 24],
|
|
256
|
+
[3223307264, 24],
|
|
257
|
+
[3224682752, 24],
|
|
258
|
+
[3227017984, 24],
|
|
259
|
+
[3232235520, 16],
|
|
260
|
+
[3232706560, 24],
|
|
261
|
+
[3323068416, 15],
|
|
262
|
+
[3325256704, 24],
|
|
263
|
+
[3405803776, 24],
|
|
264
|
+
[3758096384, 4],
|
|
265
|
+
[4026531840, 4]
|
|
266
|
+
];
|
|
267
|
+
function inCidr(value, base, bits) {
|
|
268
|
+
return (value ^ base) >>> 32 - bits === 0;
|
|
269
|
+
}
|
|
270
|
+
function isPrivateIpv4(value) {
|
|
271
|
+
return V4_PRIVATE.some(([base, bits]) => inCidr(value, base, bits));
|
|
272
|
+
}
|
|
273
|
+
function embeddedIpv4(segments) {
|
|
274
|
+
const [a, b, c, d, e, f, g, h] = segments;
|
|
275
|
+
if (a === 0 && b === 0 && c === 0 && d === 0 && e === 0) {
|
|
276
|
+
if (f === 65535 || f === 0)
|
|
277
|
+
return g * 65536 + h >>> 0;
|
|
278
|
+
}
|
|
279
|
+
if (a === 100 && b === 65435 && c === 0 && d === 0 && e === 0 && f === 0)
|
|
280
|
+
return g * 65536 + h >>> 0;
|
|
281
|
+
if (a === 8194)
|
|
282
|
+
return b * 65536 + c >>> 0;
|
|
283
|
+
if (a === 8193 && b === 0)
|
|
284
|
+
return (g ^ 65535) * 65536 + (h ^ 65535) >>> 0;
|
|
285
|
+
return;
|
|
286
|
+
}
|
|
287
|
+
function isUnspecified(segments) {
|
|
288
|
+
return segments.every((segment) => segment === 0);
|
|
289
|
+
}
|
|
290
|
+
function isIpv6Loopback(segments) {
|
|
291
|
+
return segments.slice(0, 7).every((segment) => segment === 0) && segments[7] === 1;
|
|
292
|
+
}
|
|
293
|
+
function isPrivateIpAddress(ip) {
|
|
294
|
+
if (ip.kind === "v4")
|
|
295
|
+
return isPrivateIpv4(ip.value);
|
|
296
|
+
const segments = ip.segments;
|
|
297
|
+
if (isUnspecified(segments) || isIpv6Loopback(segments))
|
|
298
|
+
return true;
|
|
299
|
+
const first = segments[0];
|
|
300
|
+
if ((first & 65280) === 65280)
|
|
301
|
+
return true;
|
|
302
|
+
if ((first & 65024) === 64512)
|
|
303
|
+
return true;
|
|
304
|
+
if ((first & 65472) === 65152)
|
|
305
|
+
return true;
|
|
306
|
+
if ((first & 65472) === 65216)
|
|
307
|
+
return true;
|
|
308
|
+
if (first === 256 && segments[1] === 0 && segments[2] === 0 && segments[3] === 0)
|
|
309
|
+
return true;
|
|
310
|
+
if (first === 8193 && segments[1] === 3512)
|
|
311
|
+
return true;
|
|
312
|
+
const embedded = embeddedIpv4(segments);
|
|
313
|
+
if (embedded !== void 0)
|
|
314
|
+
return isPrivateIpv4(embedded);
|
|
315
|
+
return false;
|
|
316
|
+
}
|
|
317
|
+
var sandboxCheckpointUnusableCauses = ["expired", "runtime_outdated", "runtime_modified", "not_restorable"];
|
|
318
|
+
function isSandboxCheckpointUnusableCause(value) {
|
|
319
|
+
return sandboxCheckpointUnusableCauses.some((cause) => cause === value);
|
|
320
|
+
}
|
|
321
|
+
var sandboxDriverIds = ["modal", "vercel", "cloudflare", "boat", "docker"];
|
|
322
|
+
var sandboxDriverLabels = {
|
|
323
|
+
modal: "Modal",
|
|
324
|
+
vercel: "Vercel",
|
|
325
|
+
cloudflare: "Cloudflare",
|
|
326
|
+
boat: "Boat",
|
|
327
|
+
docker: "Docker"
|
|
328
|
+
};
|
|
329
|
+
function isSandboxDriverID(input) {
|
|
330
|
+
return !!input && sandboxDriverIds.includes(input);
|
|
331
|
+
}
|
|
332
|
+
function safeRepoUrl(input) {
|
|
333
|
+
try {
|
|
334
|
+
const url = new URL(input);
|
|
335
|
+
return url.protocol === "https:" || url.protocol === "http:" || url.protocol === "ssh:" ? input : void 0;
|
|
336
|
+
} catch {
|
|
337
|
+
return /^[\w.-]+@[\w.-]+:[\w./-]+$/.test(input) ? input : void 0;
|
|
338
|
+
}
|
|
339
|
+
}
|
|
340
|
+
function repoUrlHost(input) {
|
|
341
|
+
if (!safeRepoUrl(input))
|
|
342
|
+
return;
|
|
343
|
+
try {
|
|
344
|
+
const host = new URL(input).hostname;
|
|
345
|
+
return host || void 0;
|
|
346
|
+
} catch {
|
|
347
|
+
return /^[\w.-]+@([\w.-]+):[\w./-]+$/.exec(input)?.[1]?.toLowerCase();
|
|
348
|
+
}
|
|
349
|
+
}
|
|
350
|
+
function bareRepoHost(host) {
|
|
351
|
+
const bare = host.trim().toLowerCase();
|
|
352
|
+
return bare.startsWith("[") && bare.endsWith("]") && bare.includes(":") ? bare.slice(1, -1) : bare;
|
|
353
|
+
}
|
|
354
|
+
function loopbackRepoName(bare) {
|
|
355
|
+
return bare === "localhost" || bare.endsWith(".localhost");
|
|
356
|
+
}
|
|
357
|
+
function publicRepoHost(repoUrl) {
|
|
358
|
+
const host = repoUrlHost(repoUrl);
|
|
359
|
+
if (!host)
|
|
360
|
+
return;
|
|
361
|
+
const bare = bareRepoHost(host);
|
|
362
|
+
if (loopbackRepoName(bare))
|
|
363
|
+
return;
|
|
364
|
+
const ip = parseIpAddress(bare);
|
|
365
|
+
if (ip && isPrivateIpAddress(ip))
|
|
366
|
+
return;
|
|
367
|
+
return host;
|
|
368
|
+
}
|
|
369
|
+
|
|
370
|
+
// src/lease-failure.ts
|
|
371
|
+
function isSandboxRuntimeBootFailure(err) {
|
|
372
|
+
return !!err && typeof err === "object" && err.runtimeBootFailed === true;
|
|
373
|
+
}
|
|
374
|
+
var SandboxRuntimeBootError = class extends Error {
|
|
375
|
+
runtimeBootFailed = true;
|
|
376
|
+
constructor(reason) {
|
|
377
|
+
super(reason);
|
|
378
|
+
this.name = "SandboxRuntimeBootError";
|
|
379
|
+
}
|
|
380
|
+
};
|
|
381
|
+
function isSandboxCheckpointUnusable(err) {
|
|
382
|
+
return !!err && typeof err === "object" && isSandboxCheckpointUnusableCause(err.checkpointUnusable);
|
|
383
|
+
}
|
|
384
|
+
var SandboxCheckpointUnusableError = class extends Error {
|
|
385
|
+
constructor(checkpointUnusable, reason) {
|
|
386
|
+
super(reason);
|
|
387
|
+
this.checkpointUnusable = checkpointUnusable;
|
|
388
|
+
this.name = "SandboxCheckpointUnusableError";
|
|
389
|
+
}
|
|
390
|
+
};
|
|
391
|
+
function isSandboxHostGone(err) {
|
|
392
|
+
return !!err && typeof err === "object" && err.hostGone === true;
|
|
393
|
+
}
|
|
394
|
+
var SandboxHostGoneError = class extends Error {
|
|
395
|
+
hostGone = true;
|
|
396
|
+
constructor(reason) {
|
|
397
|
+
super(reason);
|
|
398
|
+
this.name = "SandboxHostGoneError";
|
|
399
|
+
}
|
|
400
|
+
};
|
|
401
|
+
function isSandboxProviderKeyRefused(err) {
|
|
402
|
+
return !!err && typeof err === "object" && err.providerKeyRefused === true;
|
|
403
|
+
}
|
|
404
|
+
var SandboxProviderKeyRefusedError = class extends Error {
|
|
405
|
+
providerKeyRefused = true;
|
|
406
|
+
constructor(reason) {
|
|
407
|
+
super(reason);
|
|
408
|
+
this.name = "SandboxProviderKeyRefusedError";
|
|
409
|
+
}
|
|
410
|
+
};
|
|
411
|
+
function isSandboxStartRefusal(err) {
|
|
412
|
+
return !!err && typeof err === "object" && err.startRefused === true;
|
|
413
|
+
}
|
|
414
|
+
var SandboxStartRefusedError = class extends Error {
|
|
415
|
+
startRefused = true;
|
|
416
|
+
constructor(reason) {
|
|
417
|
+
super(reason);
|
|
418
|
+
this.name = "SandboxStartRefusedError";
|
|
419
|
+
}
|
|
420
|
+
};
|
|
421
|
+
function sandboxDriverLabel(driver) {
|
|
422
|
+
return isSandboxDriverID(driver) ? sandboxDriverLabels[driver] : driver;
|
|
423
|
+
}
|
|
424
|
+
var PROVIDER_KEY_REFUSED = "provider_key_refused:";
|
|
425
|
+
var PROVIDER_FAILED = "provider_failed:";
|
|
426
|
+
function sandboxProviderKeyRefusedError(driver, message) {
|
|
427
|
+
return `${PROVIDER_KEY_REFUSED}${driver}: ${message}`;
|
|
428
|
+
}
|
|
429
|
+
function sandboxProviderFailedError(driver, message) {
|
|
430
|
+
return `${PROVIDER_FAILED}${driver}: ${message}`;
|
|
431
|
+
}
|
|
432
|
+
function codedDriver(error, prefix) {
|
|
433
|
+
if (!error.startsWith(prefix)) return void 0;
|
|
434
|
+
const rest = error.slice(prefix.length);
|
|
435
|
+
const end = rest.indexOf(": ");
|
|
436
|
+
return end < 0 ? rest : rest.slice(0, end);
|
|
437
|
+
}
|
|
438
|
+
var SANDBOX_START_WINDOWS = ["minute", "hour", "day"];
|
|
439
|
+
function isSandboxProviderRateLimited(err) {
|
|
440
|
+
const limit = !!err && typeof err === "object" ? err.providerRateLimited : void 0;
|
|
441
|
+
return !!limit && typeof limit === "object" && SANDBOX_START_WINDOWS.some((window) => window === limit.window);
|
|
442
|
+
}
|
|
443
|
+
var SandboxProviderRateLimitedError = class extends Error {
|
|
444
|
+
constructor(providerRateLimited, reason) {
|
|
445
|
+
super(reason);
|
|
446
|
+
this.providerRateLimited = providerRateLimited;
|
|
447
|
+
this.name = "SandboxProviderRateLimitedError";
|
|
448
|
+
}
|
|
449
|
+
};
|
|
450
|
+
function sandboxStartLimitRetryMs(window) {
|
|
451
|
+
return window === "minute" ? 6e4 : window === "hour" ? 36e5 : void 0;
|
|
452
|
+
}
|
|
453
|
+
var PROVIDER_RATE_LIMITED = "provider_rate_limited:";
|
|
454
|
+
var START_LIMIT_NAMES = { minute: "per-minute", hour: "hourly", day: "daily" };
|
|
455
|
+
function sandboxProviderRateLimitedError(driver, limit) {
|
|
456
|
+
const used = `${sandboxDriverLabel(driver)}'s ${START_LIMIT_NAMES[limit.window]} start limit (${limit.limit}) is used up`;
|
|
457
|
+
return `${PROVIDER_RATE_LIMITED}${limit.window}: ${limit.window === "day" ? `${used}; try again later.` : `${used}.`}`;
|
|
458
|
+
}
|
|
459
|
+
function sandboxProviderRateLimit(error) {
|
|
460
|
+
if (!error?.startsWith(PROVIDER_RATE_LIMITED)) return void 0;
|
|
461
|
+
const window = error.slice(PROVIDER_RATE_LIMITED.length).split(":", 1)[0];
|
|
462
|
+
return SANDBOX_START_WINDOWS.find((name) => name === window);
|
|
463
|
+
}
|
|
464
|
+
function providerRateLimitedReason(error) {
|
|
465
|
+
return sandboxProviderRateLimit(error) === void 0 ? void 0 : error.slice(error.indexOf(": ") + 2);
|
|
466
|
+
}
|
|
467
|
+
var RUNTIME_BOOT_FAILED = "runtime_boot_failed: ";
|
|
468
|
+
var CHECKPOINT_UNUSABLE = "checkpoint_unusable:";
|
|
469
|
+
var RUNTIME_HOST_GONE = "runtime_host_gone: ";
|
|
470
|
+
function sandboxRuntimeBootFailedError(reason) {
|
|
471
|
+
return `${RUNTIME_BOOT_FAILED}${reason}`;
|
|
472
|
+
}
|
|
473
|
+
function sandboxCheckpointUnusableError(cause, reason) {
|
|
474
|
+
return `${CHECKPOINT_UNUSABLE}${cause}: ${reason}`;
|
|
475
|
+
}
|
|
476
|
+
function sandboxCheckpointUnusable(error) {
|
|
477
|
+
if (!error?.startsWith(CHECKPOINT_UNUSABLE)) return void 0;
|
|
478
|
+
const rest = error.slice(CHECKPOINT_UNUSABLE.length);
|
|
479
|
+
const end = rest.indexOf(": ");
|
|
480
|
+
const cause = end < 0 ? void 0 : rest.slice(0, end);
|
|
481
|
+
return isSandboxCheckpointUnusableCause(cause) ? { cause, message: rest.slice(end + 2) } : void 0;
|
|
482
|
+
}
|
|
483
|
+
function sandboxHostGoneError(reason) {
|
|
484
|
+
return `${RUNTIME_HOST_GONE}${reason}`;
|
|
485
|
+
}
|
|
486
|
+
var RUNTIME_UNRESPONSIVE = "runtime_unresponsive:";
|
|
487
|
+
function sandboxRuntimeUnresponsiveError(forMs) {
|
|
488
|
+
return `${RUNTIME_UNRESPONSIVE}${Math.max(1, Math.round(forMs / 6e4))}`;
|
|
489
|
+
}
|
|
490
|
+
function sandboxStopReason(error) {
|
|
491
|
+
if (!error?.startsWith(RUNTIME_UNRESPONSIVE)) return void 0;
|
|
492
|
+
const minutes = Number(error.slice(RUNTIME_UNRESPONSIVE.length));
|
|
493
|
+
return `The workspace's runtime stopped answering; it was stopped after ${minutes} ${minutes === 1 ? "minute" : "minutes"}.`;
|
|
494
|
+
}
|
|
495
|
+
var WORKSPACE_RUNTIME_BOOT_FAILED_LINE = "workspace_runtime_boot_failed: ";
|
|
496
|
+
function workspaceRuntimeBootReason(diagnostic) {
|
|
497
|
+
const at = diagnostic.lastIndexOf(WORKSPACE_RUNTIME_BOOT_FAILED_LINE);
|
|
498
|
+
if (at < 0) return void 0;
|
|
499
|
+
const [reason = ""] = diagnostic.slice(at + WORKSPACE_RUNTIME_BOOT_FAILED_LINE.length).split(/\n\s+at /);
|
|
500
|
+
return reason.trim().replace(/^Error: /, "") || void 0;
|
|
501
|
+
}
|
|
502
|
+
function runtimeBootDiagnostic(log, maxChars) {
|
|
503
|
+
const trimmed = log.trim();
|
|
504
|
+
const reason = workspaceRuntimeBootReason(trimmed);
|
|
505
|
+
if (reason === void 0) return trimmed.slice(-maxChars);
|
|
506
|
+
return `${WORKSPACE_RUNTIME_BOOT_FAILED_LINE}${reason}`.slice(0, maxChars);
|
|
507
|
+
}
|
|
508
|
+
function sandboxRuntimeBootFailure(error) {
|
|
509
|
+
const unusable = sandboxCheckpointUnusable(error);
|
|
510
|
+
if (unusable) return unusable.message;
|
|
511
|
+
if (!error?.startsWith(RUNTIME_BOOT_FAILED)) return void 0;
|
|
512
|
+
const diagnostic = error.slice(RUNTIME_BOOT_FAILED.length);
|
|
513
|
+
return workspaceRuntimeBootReason(diagnostic) ?? diagnostic;
|
|
514
|
+
}
|
|
515
|
+
var SANDBOX_RUNTIME_UNHEALTHY = "runtime_unhealthy";
|
|
516
|
+
var SANDBOX_RETRY_CAP_EXCEEDED = "runtime_retry_cap_exceeded";
|
|
517
|
+
var CODED_REASONS = /* @__PURE__ */ new Map([
|
|
518
|
+
[SANDBOX_RUNTIME_UNHEALTHY, "The workspace machine stopped answering."],
|
|
519
|
+
[SANDBOX_RETRY_CAP_EXCEEDED, "Several tries in a row failed; try again later."]
|
|
520
|
+
]);
|
|
521
|
+
function firstLine(text) {
|
|
522
|
+
return text.split("\n", 1)[0]?.trim() ?? text;
|
|
523
|
+
}
|
|
524
|
+
function sandboxStartFailureReason(error) {
|
|
525
|
+
const boot = sandboxRuntimeBootFailure(error);
|
|
526
|
+
if (boot !== void 0) return firstLine(boot);
|
|
527
|
+
const limited = providerRateLimitedReason(error);
|
|
528
|
+
if (limited !== void 0) return limited;
|
|
529
|
+
const keyRefused = codedDriver(error, PROVIDER_KEY_REFUSED);
|
|
530
|
+
if (keyRefused !== void 0) return `${sandboxDriverLabel(keyRefused)} refused the sandbox provider key; fix it in Settings \u2192 Sandbox providers.`;
|
|
531
|
+
const failed = codedDriver(error, PROVIDER_FAILED);
|
|
532
|
+
if (failed !== void 0) return `${sandboxDriverLabel(failed)} returned an error.`;
|
|
533
|
+
return CODED_REASONS.get(error) ?? firstLine(error);
|
|
534
|
+
}
|
|
535
|
+
var LEASE_ENDED_ERRORS = /* @__PURE__ */ new Set(["runtime_lease_changed", "runtime_lease_stopped", "runtime_lease_destroyed"]);
|
|
536
|
+
function sandboxLeaseEndedUnderStart(error) {
|
|
537
|
+
return error !== void 0 && LEASE_ENDED_ERRORS.has(error);
|
|
538
|
+
}
|
|
539
|
+
function sandboxLeaseFailure(lease, input) {
|
|
540
|
+
if (lease.status !== "unavailable" || !lease.lastError) return void 0;
|
|
541
|
+
const unusable = sandboxCheckpointUnusable(lease.lastError);
|
|
542
|
+
if (unusable) return { kind: "checkpoint_unusable", cause: unusable.cause, message: firstLine(unusable.message), retrying: false };
|
|
543
|
+
const kind = sandboxRuntimeBootFailure(lease.lastError) !== void 0 ? "boot" : lease.lastError === SANDBOX_RUNTIME_UNHEALTHY ? "unhealthy" : "provider";
|
|
544
|
+
const retrying = kind !== "boot" && lease.retryCount < input.maxRetryCount && lease.nextRetryAt !== void 0 && lease.nextRetryAt > input.now;
|
|
545
|
+
return { kind, message: sandboxStartFailureReason(lease.lastError), retrying };
|
|
546
|
+
}
|
|
547
|
+
|
|
548
|
+
// src/lease-retry-wait.ts
|
|
549
|
+
async function liftSandboxRetryWait(leaseStore, workspaceId) {
|
|
550
|
+
const lease = await leaseStore.get(workspaceId);
|
|
551
|
+
if (lease?.status !== "unavailable" || lease.retryCount === 0 && lease.nextRetryAt === void 0) return false;
|
|
552
|
+
return !!await leaseStore.update(workspaceId, lease.epoch, { retryCount: 0, nextRetryAt: null }, "unavailable");
|
|
553
|
+
}
|
|
554
|
+
|
|
555
|
+
// ../claxedo-helpers/dist/api-error.mjs
|
|
556
|
+
var PUBLIC_API_ERRORS = {
|
|
557
|
+
repository_provider_unauthorized: { status: 502, message: "Repository provider refused authorization" },
|
|
558
|
+
repository_provider_unavailable: { status: 502, message: "Repository provider is unavailable" },
|
|
559
|
+
host_tunnel_timeout: { status: 503, retryable: true, message: "Host tunnel timed out" },
|
|
560
|
+
host_tunnel_response_body_too_large: { status: 413, message: "Host tunnel response body exceeds the relay limit" },
|
|
561
|
+
host_tunnel_unavailable: { status: 503, retryable: true, message: "Host tunnel is unavailable" },
|
|
562
|
+
workspace_checkpoint_unavailable: { status: 503, retryable: true, message: "Workspace checkpoint is unavailable" },
|
|
563
|
+
workspace_checkpoint_conflict: { status: 409, message: "Workspace checkpoint conflicted" },
|
|
564
|
+
invalid_input: { family: "access", status: 400, message: "The request names an invalid value" },
|
|
565
|
+
organization_policy_denied: { family: "access", status: 403, message: "Organization creation is disabled for this deployment" },
|
|
566
|
+
org_admin_required: { family: "access", status: 403, message: "Organization administrator authority is required" },
|
|
567
|
+
org_owner_required: { family: "access", status: 403, message: "Only an organization owner may grant, change or remove the owner role" },
|
|
568
|
+
org_owner_protected: { family: "access", status: 409, message: "The organization's founding owner cannot be removed or demoted" },
|
|
569
|
+
team_member_org_membership_required: { family: "access", status: 403, message: "The team member must belong to the team organization" },
|
|
570
|
+
project_member_org_membership_required: { family: "access", status: 403, message: "The project member must belong to the project organization" },
|
|
571
|
+
org_membership_required: { family: "access", status: 403, message: "Organization membership is required" },
|
|
572
|
+
project_admin_required: { family: "access", status: 403, message: "Project administrator authority is required" },
|
|
573
|
+
team_not_allowed_on_personal_org: { family: "access", status: 400, message: "Personal organizations cannot contain teams" },
|
|
574
|
+
team_member_target_required: { family: "access", status: 400, message: "Exactly one team member target is required" },
|
|
575
|
+
org_invitation_invalid: { family: "access", status: 409, message: "Invitation is invalid or unavailable" },
|
|
576
|
+
org_invitation_pending: { family: "access", status: 409, message: "An invitation to this address is already pending" },
|
|
577
|
+
org_invitation_rate_limited: { family: "access", status: 429, message: "Organization invitation limit exceeded" },
|
|
578
|
+
org_invitation_email_mismatch: { family: "access", status: 403, message: "Sign in with the invited verified email address" },
|
|
579
|
+
org_invitation_delivery_unavailable: { family: "access", status: 503, message: "Invitation delivery is unavailable" },
|
|
580
|
+
team_not_found: { family: "access", status: 404, message: "Team not found" },
|
|
581
|
+
team_member_not_found: { family: "access", status: 404, message: "Team member not found" },
|
|
582
|
+
org_member_not_found: { family: "access", status: 404, message: "Organization member not found" },
|
|
583
|
+
project_not_found: { family: "access", status: 404, message: "Project not found" },
|
|
584
|
+
project_member_not_found: { family: "access", status: 404, message: "Project member not found" },
|
|
585
|
+
project_member_owner_immutable: { family: "access", status: 409, message: "The project owner's access cannot be changed" },
|
|
586
|
+
resource_conflict: { family: "access", status: 409, message: "Organization or team authority changed concurrently" },
|
|
587
|
+
session_not_found: { family: "session_share", status: 404, message: "This session is not on the control plane, so it cannot be shared from People yet." },
|
|
588
|
+
session_share_admin_required: { family: "session_share", status: 403, message: "Only the person who started this session can manage its People." },
|
|
589
|
+
session_share_target_required: { family: "session_share", status: 400, message: "A session is shared with exactly one person" },
|
|
590
|
+
session_share_target_not_found: { family: "session_share", status: 404, message: "That person was not found in this organization." },
|
|
591
|
+
session_share_level_invalid: { family: "session_share", status: 400, message: "A share level is either follow or send." },
|
|
592
|
+
workspace_backing_conflict: { status: 409, message: "Workspace is provisioned in the cloud and cannot be placed on a machine" },
|
|
593
|
+
invalid_session_list_cursor: { status: 400, message: "Session list cursor does not match this query" },
|
|
594
|
+
BAD_REQUEST: { status: 400, message: "Invalid request" },
|
|
595
|
+
INTERNAL_SERVER_ERROR: { status: 500, message: "Internal error" },
|
|
596
|
+
CONTROL_PLANE_UNAVAILABLE: { status: 503, message: "Control Plane services are not configured" },
|
|
597
|
+
credential_verification_unsupported: { status: 400, message: "Credential provider does not support verification" },
|
|
598
|
+
credential_shape_invalid: { status: 400, message: "Credential secret has an unsupported shape" },
|
|
599
|
+
credential_provider_unavailable: { status: 502, retryable: true, message: "Credential provider request failed" },
|
|
600
|
+
credential_verification_failed: { status: 502, message: "Credential provider verification failed" },
|
|
601
|
+
credential_redirect_denied: { status: 502, message: "Sandbox provider redirects are not allowed" },
|
|
602
|
+
credential_endpoint_invalid: { status: 400, message: "Invalid credential provider endpoint" },
|
|
603
|
+
host_attestation_denied: { status: 403, message: "host_attestation_denied" },
|
|
604
|
+
signature_replayed: { status: 409, message: "signature_replayed" },
|
|
605
|
+
host_enrollment_not_found: { status: 404, message: "host_enrollment_not_found" },
|
|
606
|
+
enrollment_revoked: { status: 403, message: "enrollment_revoked" },
|
|
607
|
+
enrollment_paused: { status: 403, message: "enrollment_paused" },
|
|
608
|
+
enrollment_owner_ineligible: { status: 403, message: "enrollment_owner_ineligible" },
|
|
609
|
+
enrollment_key_version_mismatch: { status: 403, message: "enrollment_key_version_mismatch" },
|
|
610
|
+
invitation_invalid: { status: 403, message: "invitation_invalid" },
|
|
611
|
+
invitation_expired: { status: 410, message: "invitation_expired" },
|
|
612
|
+
invitation_revoked: { status: 410, message: "invitation_revoked" },
|
|
613
|
+
invitation_redeemed: { status: 409, message: "invitation_redeemed" },
|
|
614
|
+
invitation_host_conflict: { status: 409, message: "invitation_host_conflict" },
|
|
615
|
+
enrollment_generation_superseded: { status: 409, message: "enrollment_generation_superseded" },
|
|
616
|
+
host_assignment_outside_scope: { status: 400, message: "host_assignment_outside_scope" },
|
|
617
|
+
host_sealing_key_undeclared: { status: 409, message: "host_sealing_key_undeclared" },
|
|
618
|
+
host_provider_config_revision_stale: { status: 409, message: "host_provider_config_revision_stale" },
|
|
619
|
+
identity_conflict: { status: 409, message: "identity_conflict" },
|
|
620
|
+
registration_transition_denied: { status: 403, message: "registration_transition_denied" },
|
|
621
|
+
session_reservation_spent: { status: 409, message: "This session's create was undone; reserve it again to retry" },
|
|
622
|
+
actor_authorization_denied: { status: 403, message: "actor_authorization_denied" }
|
|
623
|
+
};
|
|
624
|
+
function publicApiErrorShape(code) {
|
|
625
|
+
const entry = PUBLIC_API_ERRORS[code];
|
|
626
|
+
return { status: entry.status, retryable: entry.retryable === true };
|
|
627
|
+
}
|
|
628
|
+
function publicApiFailure(code, message = PUBLIC_API_ERRORS[code].message) {
|
|
629
|
+
return Object.assign(new Error(message), { code, ...publicApiErrorShape(code) });
|
|
630
|
+
}
|
|
631
|
+
|
|
4
632
|
// src/checkpoint-manager.ts
|
|
5
633
|
async function captureSandboxCheckpoint(input) {
|
|
6
634
|
const lease = await requireLease(input.leaseStore, input.workspaceId);
|
|
7
|
-
if (lease.status !== "ready") throw
|
|
635
|
+
if (lease.status !== "ready") throw publicApiFailure("workspace_checkpoint_conflict", "workspace_checkpoint_lease_not_ready");
|
|
8
636
|
const persistence = lease.persistence;
|
|
9
|
-
if (!persistence || persistence.capture === "none") throw
|
|
637
|
+
if (!persistence || persistence.capture === "none") throw publicApiFailure("workspace_checkpoint_conflict", "workspace_checkpoint_unsupported");
|
|
10
638
|
const target = await input.target(input.workspaceId);
|
|
11
|
-
if (target.status !== "ready") throw
|
|
12
|
-
|
|
639
|
+
if (target.status !== "ready") throw publicApiFailure("workspace_checkpoint_unavailable", target.reason);
|
|
640
|
+
const discard = persistence.capture === "same-resource" ? void 0 : input.deleteSnapshot;
|
|
13
641
|
let capturedSource = false;
|
|
642
|
+
let result;
|
|
14
643
|
try {
|
|
15
|
-
await input.request.
|
|
16
|
-
await input.
|
|
17
|
-
|
|
18
|
-
if (!captured.ok) throw new Error(captured.reason);
|
|
644
|
+
const frozen = await freezeForCapture(input.request, input.endsLease !== void 0, input.workspaceId);
|
|
645
|
+
const captured = persistence.capture === "same-resource" ? { ok: true, snapshotId: target.driverResourceId ?? target.sandboxId } : await input.snapshot(input.workspaceId, (await input.leaseStore.get(input.workspaceId))?.checkpoint?.providerReference);
|
|
646
|
+
if (!captured.ok) throw publicApiFailure("workspace_checkpoint_conflict", captured.reason);
|
|
19
647
|
capturedSource = true;
|
|
20
648
|
const capturedAt = (input.now ?? Date.now)();
|
|
21
649
|
const checkpoint = {
|
|
@@ -26,28 +654,59 @@ async function captureSandboxCheckpoint(input) {
|
|
|
26
654
|
metadata: {
|
|
27
655
|
scope: persistence.capture,
|
|
28
656
|
sourceBehavior: requireCaptureSource(persistence.captureSource),
|
|
29
|
-
|
|
30
|
-
|
|
657
|
+
restoreMount: requireRestoreMount(persistence.restoreMount),
|
|
658
|
+
...frozen ? {} : { unfrozen: true }
|
|
31
659
|
}
|
|
32
660
|
};
|
|
661
|
+
const replaced = (await input.leaseStore.get(input.workspaceId))?.checkpoint;
|
|
33
662
|
const updated = await input.leaseStore.update(input.workspaceId, lease.epoch, {
|
|
34
663
|
checkpoint,
|
|
35
664
|
restore: null,
|
|
36
|
-
...
|
|
665
|
+
...input.endsLease?.stoppedError ? { lastError: input.endsLease.stoppedError } : {},
|
|
666
|
+
...input.endsLease || persistence.captureSource === "stopped" || persistence.captureSource === "deleted" ? { status: "stopped" } : {}
|
|
37
667
|
});
|
|
38
|
-
if (!updated)
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
668
|
+
if (!updated) {
|
|
669
|
+
await discardSnapshot(discard, target, captured.snapshotId);
|
|
670
|
+
throw publicApiFailure("workspace_checkpoint_conflict", "workspace_checkpoint_epoch_fenced");
|
|
671
|
+
}
|
|
672
|
+
if (replaced && replaced.providerReference !== captured.snapshotId) {
|
|
673
|
+
await discardSnapshot(discard, target, replaced.providerReference);
|
|
43
674
|
}
|
|
675
|
+
result = { status: "ready", lease: updated, checkpoint };
|
|
676
|
+
} catch (error) {
|
|
677
|
+
if (!capturedSource || persistence.captureSource === "preserved") await thawAfterFailure(input.request.runtime, input.workspaceId);
|
|
678
|
+
throw error;
|
|
679
|
+
}
|
|
680
|
+
if (!input.endsLease && persistence.captureSource === "preserved") await input.request.runtime.resume();
|
|
681
|
+
return result;
|
|
682
|
+
}
|
|
683
|
+
async function freezeForCapture(request, endsLease, workspaceId) {
|
|
684
|
+
try {
|
|
685
|
+
await request.runtime.freeze(request.policy ?? "drain", { idleBefore: request.idleBefore });
|
|
686
|
+
await request.runtime.flush();
|
|
687
|
+
await request.runtime.scrub();
|
|
688
|
+
return true;
|
|
689
|
+
} catch (error) {
|
|
690
|
+
if (!endsLease || request.idleBefore !== void 0) throw error;
|
|
691
|
+
console.warn("A stopping sandbox's runtime did not freeze; its files are captured as they are", { workspaceId, error: String(error) });
|
|
692
|
+
return false;
|
|
44
693
|
}
|
|
45
694
|
}
|
|
695
|
+
async function thawAfterFailure(runtime, workspaceId) {
|
|
696
|
+
await runtime.resume().catch((error) => {
|
|
697
|
+
console.warn("A runtime was not thawed after its capture failed", { workspaceId, error: String(error) });
|
|
698
|
+
});
|
|
699
|
+
}
|
|
700
|
+
async function discardSnapshot(discard, target, snapshotId) {
|
|
701
|
+
await discard?.(target, snapshotId).catch((error) => {
|
|
702
|
+
console.warn("A superseded sandbox snapshot could not be deleted", { workspaceId: target.workspaceId, snapshotId, error: String(error) });
|
|
703
|
+
});
|
|
704
|
+
}
|
|
46
705
|
async function restoreSandboxCheckpoint(input) {
|
|
47
706
|
const lease = await requireLease(input.leaseStore, input.workspaceId);
|
|
48
707
|
const checkpoint = lease.checkpoint;
|
|
49
708
|
if (!checkpoint || input.request.checkpointId && checkpoint.id !== input.request.checkpointId) {
|
|
50
|
-
throw
|
|
709
|
+
throw publicApiFailure("workspace_checkpoint_conflict", "workspace_checkpoint_not_found");
|
|
51
710
|
}
|
|
52
711
|
if (lease.restore?.state === "ready" && lease.restore.checkpointId === checkpoint.id) {
|
|
53
712
|
return { status: "ready", lease, checkpoint };
|
|
@@ -61,7 +720,7 @@ async function restoreSandboxCheckpoint(input) {
|
|
|
61
720
|
startedAt: (input.now ?? Date.now)()
|
|
62
721
|
};
|
|
63
722
|
const staged = lease.status === "ready" && lease.epoch === checkpoint.sourceEpoch ? await input.leaseStore.update(input.workspaceId, lease.epoch, { status: "stopped", restore: restoring }) : await input.leaseStore.update(input.workspaceId, lease.epoch, { restore: restoring });
|
|
64
|
-
if (!staged) throw
|
|
723
|
+
if (!staged) throw publicApiFailure("workspace_checkpoint_conflict", "workspace_restore_epoch_fenced");
|
|
65
724
|
const result = await input.ensure(input.workspaceId, {
|
|
66
725
|
homeRegion: staged.homeRegion,
|
|
67
726
|
...input.request.ensure,
|
|
@@ -84,7 +743,7 @@ async function restoreSandboxCheckpoint(input) {
|
|
|
84
743
|
error: result.error ?? "workspace_restore_unavailable"
|
|
85
744
|
}
|
|
86
745
|
});
|
|
87
|
-
throw
|
|
746
|
+
throw publicApiFailure("workspace_checkpoint_unavailable", result.error ?? "workspace_restore_unavailable");
|
|
88
747
|
}
|
|
89
748
|
await input.request.runtime.reconcile({ epoch: result.epoch, checkpointId: checkpoint.id });
|
|
90
749
|
const ready = await input.leaseStore.update(input.workspaceId, result.epoch, {
|
|
@@ -97,21 +756,21 @@ async function restoreSandboxCheckpoint(input) {
|
|
|
97
756
|
completedAt: (input.now ?? Date.now)()
|
|
98
757
|
}
|
|
99
758
|
});
|
|
100
|
-
if (!ready) throw
|
|
759
|
+
if (!ready) throw publicApiFailure("workspace_checkpoint_conflict", "workspace_restore_epoch_fenced");
|
|
101
760
|
return { status: "ready", lease: ready, checkpoint };
|
|
102
761
|
}
|
|
103
762
|
async function requireLease(store, workspaceId) {
|
|
104
763
|
const lease = await store.get(workspaceId);
|
|
105
|
-
if (!lease) throw
|
|
764
|
+
if (!lease) throw publicApiFailure("workspace_checkpoint_conflict", "workspace_checkpoint_lease_missing");
|
|
106
765
|
return lease;
|
|
107
766
|
}
|
|
108
767
|
function requireCaptureSource(value) {
|
|
109
768
|
if (value === "preserved" || value === "stopped" || value === "deleted") return value;
|
|
110
|
-
throw
|
|
769
|
+
throw publicApiFailure("workspace_checkpoint_conflict", "workspace_checkpoint_capabilities_invalid");
|
|
111
770
|
}
|
|
112
771
|
function requireRestoreMount(value) {
|
|
113
|
-
if (value === "same-resource" || value === "
|
|
114
|
-
throw
|
|
772
|
+
if (value === "same-resource" || value === "new-resource") return value;
|
|
773
|
+
throw publicApiFailure("workspace_checkpoint_conflict", "workspace_checkpoint_capabilities_invalid");
|
|
115
774
|
}
|
|
116
775
|
|
|
117
776
|
// src/hosted-network-policy.ts
|
|
@@ -141,17 +800,17 @@ var SANDBOX_BASELINE_EGRESS_HOSTS = [
|
|
|
141
800
|
];
|
|
142
801
|
function hostOf(value) {
|
|
143
802
|
const raw = value?.trim();
|
|
144
|
-
if (!raw) return;
|
|
803
|
+
if (!raw) return void 0;
|
|
145
804
|
try {
|
|
146
805
|
const host = new URL(raw).hostname.trim().toLowerCase();
|
|
147
806
|
return host ? host : void 0;
|
|
148
807
|
} catch {
|
|
149
|
-
return;
|
|
808
|
+
return void 0;
|
|
150
809
|
}
|
|
151
810
|
}
|
|
152
811
|
function sandboxSourceHost(source) {
|
|
153
|
-
if (!source || source.kind !== "git") return;
|
|
154
|
-
return
|
|
812
|
+
if (!source || source.kind !== "git") return void 0;
|
|
813
|
+
return publicRepoHost(source.repoUrl);
|
|
155
814
|
}
|
|
156
815
|
function hostedSandboxNetworkPolicy(input) {
|
|
157
816
|
const hosts = /* @__PURE__ */ new Set();
|
|
@@ -169,70 +828,319 @@ function hostedSandboxNetworkPolicy(input) {
|
|
|
169
828
|
return { mode: "restricted", hosts: [...hosts] };
|
|
170
829
|
}
|
|
171
830
|
|
|
172
|
-
// src/
|
|
173
|
-
function
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
if (
|
|
178
|
-
|
|
831
|
+
// src/runtime-snapshot.ts
|
|
832
|
+
async function applySandboxRuntimeSnapshot(input) {
|
|
833
|
+
const { leaseStore, workspaceId, snapshot } = input;
|
|
834
|
+
const liveness = input.liveness ?? { kind: "demote" };
|
|
835
|
+
const current = await leaseStore.get(workspaceId);
|
|
836
|
+
if (!current) return { ok: false, reason: "runtime_lease_missing" };
|
|
837
|
+
if (current.epoch !== snapshot.epoch) return { ok: false, reason: "runtime_lease_epoch_mismatch" };
|
|
838
|
+
if (!current.sandboxId || !current.url || !current.hostId) {
|
|
839
|
+
return { ok: false, reason: "runtime_lease_not_provisioned" };
|
|
179
840
|
}
|
|
180
|
-
if (
|
|
181
|
-
return {
|
|
841
|
+
if (current.status === "stopped" || current.status === "destroyed") {
|
|
842
|
+
return { ok: false, reason: "runtime_lease_not_serving" };
|
|
182
843
|
}
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
844
|
+
const timestamp = snapshot.now ?? (input.now ?? Date.now)();
|
|
845
|
+
const retry = liveness.kind === "retry-budget";
|
|
846
|
+
const updated = await leaseStore.update(workspaceId, snapshot.epoch, {
|
|
847
|
+
status: snapshot.ok ? "ready" : "unavailable",
|
|
848
|
+
lastHeartbeatAt: timestamp,
|
|
849
|
+
...snapshot.active ? { lastActivityAt: timestamp } : {},
|
|
850
|
+
...snapshot.ok ? { lastError: null } : {},
|
|
851
|
+
...retry && snapshot.ok ? { retryCount: 0, nextRetryAt: null } : {},
|
|
852
|
+
...retry && !snapshot.ok ? {
|
|
853
|
+
retryCount: current.retryCount + 1,
|
|
854
|
+
nextRetryAt: liveness.nextRetryAt({ retryCount: current.retryCount + 1, now: timestamp }) ?? null,
|
|
855
|
+
lastError: SANDBOX_RUNTIME_UNHEALTHY
|
|
856
|
+
} : {}
|
|
857
|
+
}, current.status);
|
|
858
|
+
if (!updated) return { ok: false, reason: await refusedReason(leaseStore, workspaceId, snapshot.epoch) };
|
|
859
|
+
return { ok: true, status: updated.status };
|
|
190
860
|
}
|
|
191
|
-
function
|
|
192
|
-
|
|
193
|
-
if (
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
if (hosts === 0 && cidrs === 0) return { action: "enforce" };
|
|
197
|
-
if (hosts > 0) return { action: "enforce" };
|
|
198
|
-
return control === "hosts-and-cidrs" ? { action: "enforce" } : { action: "refuse", reason: "sandbox_egress_policy_unenforceable" };
|
|
861
|
+
async function refusedReason(leaseStore, workspaceId, epoch) {
|
|
862
|
+
const current = await leaseStore.get(workspaceId);
|
|
863
|
+
if (!current) return "runtime_lease_missing";
|
|
864
|
+
if (current.epoch !== epoch) return "runtime_lease_epoch_mismatch";
|
|
865
|
+
return "runtime_lease_not_serving";
|
|
199
866
|
}
|
|
867
|
+
|
|
868
|
+
// src/egress-warnings.ts
|
|
200
869
|
function egressUnenforcedMessage(input) {
|
|
201
870
|
const scope = input.workspaceId ? `workspace ${input.workspaceId}` : "every sandbox it provisions";
|
|
202
871
|
const withheld = input.requested ? ` The requested allowlist (${(input.requested.hosts ?? []).length} host(s), ${(input.requested.cidrs ?? []).length} cidr(s)) was withheld, not applied.` : "";
|
|
203
|
-
return `[sandbox-manager] SANDBOX EGRESS IS UNRESTRICTED: driver "${input.driver}" declares egressControl: "none", so ${scope} can reach ANY host on the internet, including attacker-controlled buckets \u2014 agent-authored code inside the sandbox has an unmonitored exfiltration path.${withheld} Select a driver that can enforce egress (
|
|
872
|
+
return `[sandbox-manager] SANDBOX EGRESS IS UNRESTRICTED: driver "${input.driver}" declares egressControl: "none", so ${scope} can reach ANY host on the internet, including attacker-controlled buckets \u2014 agent-authored code inside the sandbox has an unmonitored exfiltration path.${withheld} Select a driver that can enforce egress (vercel) to close it. See public-docs/sandbox-egress.md.`;
|
|
204
873
|
}
|
|
205
874
|
var compositionEgressWarnings = /* @__PURE__ */ new Set();
|
|
875
|
+
var ensureEgressWarnings = /* @__PURE__ */ new Set();
|
|
206
876
|
function defaultEgressUnenforcedSink(event) {
|
|
207
877
|
console.warn(event.message);
|
|
208
878
|
}
|
|
209
|
-
function
|
|
879
|
+
function createEgressWarnings(input) {
|
|
880
|
+
const { id, egressControl } = input.driver;
|
|
881
|
+
const sink = input.sink ?? defaultEgressUnenforcedSink;
|
|
882
|
+
const key = `${id}|${egressControl}`;
|
|
883
|
+
if (egressControl === "none" && !compositionEgressWarnings.has(key)) {
|
|
884
|
+
compositionEgressWarnings.add(key);
|
|
885
|
+
sink({
|
|
886
|
+
phase: "composition",
|
|
887
|
+
reason: "sandbox_egress_uncontained",
|
|
888
|
+
driver: id,
|
|
889
|
+
egressControl,
|
|
890
|
+
message: egressUnenforcedMessage({ phase: "composition", driver: id })
|
|
891
|
+
});
|
|
892
|
+
}
|
|
210
893
|
return {
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
...patch.restore === void 0 ? {} : { restore: patch.restore ?? void 0 },
|
|
226
|
-
updatedAt
|
|
894
|
+
withheld(start) {
|
|
895
|
+
if (ensureEgressWarnings.has(key)) return;
|
|
896
|
+
ensureEgressWarnings.add(key);
|
|
897
|
+
sink({
|
|
898
|
+
phase: "ensure",
|
|
899
|
+
reason: "sandbox_egress_uncontained",
|
|
900
|
+
driver: id,
|
|
901
|
+
egressControl,
|
|
902
|
+
message: egressUnenforcedMessage({ phase: "ensure", driver: id, workspaceId: start.workspaceId, requested: start.requested }),
|
|
903
|
+
workspaceId: start.workspaceId,
|
|
904
|
+
...start.labels ? { labels: start.labels } : {},
|
|
905
|
+
requested: start.requested
|
|
906
|
+
});
|
|
907
|
+
}
|
|
227
908
|
};
|
|
228
909
|
}
|
|
229
|
-
|
|
230
|
-
|
|
910
|
+
|
|
911
|
+
// src/garbage-collect.ts
|
|
912
|
+
async function sweepSandboxes(input) {
|
|
913
|
+
const { driver, leaseStore, appLabel } = input;
|
|
914
|
+
const blind = { destroyed: [], kept: [], skipped: [], failed: [], listingUnsupported: true, driver: driver.id };
|
|
915
|
+
if (!driver.list) return blind;
|
|
916
|
+
let listed;
|
|
917
|
+
try {
|
|
918
|
+
listed = await driver.list();
|
|
919
|
+
} catch (err) {
|
|
920
|
+
if (!isSandboxListingUnsupported(err)) throw err;
|
|
921
|
+
return blind;
|
|
922
|
+
}
|
|
923
|
+
const leases = new Map((await leaseStore.list()).map((lease) => [lease.workspaceId, lease]));
|
|
924
|
+
const result = { destroyed: [], kept: [], skipped: [], failed: [] };
|
|
925
|
+
for (const target of listed) {
|
|
926
|
+
if (target.labels?.app !== appLabel) {
|
|
927
|
+
result.skipped.push({ target, reason: "unmanaged_app_label" });
|
|
928
|
+
continue;
|
|
929
|
+
}
|
|
930
|
+
const workspaceId = target.labels.workspaceId;
|
|
931
|
+
const epoch = target.labels.epoch;
|
|
932
|
+
if (!workspaceId || !epoch) {
|
|
933
|
+
result.skipped.push({ target, reason: "missing_runtime_labels" });
|
|
934
|
+
continue;
|
|
935
|
+
}
|
|
936
|
+
const lease = leases.get(workspaceId);
|
|
937
|
+
const leaseOwnsResource = lease !== void 0 && lease.status !== "destroyed" && (lease.sandboxId === target.sandboxId || lease.driverResourceId !== void 0 && lease.driverResourceId === target.driverResourceId);
|
|
938
|
+
if (leaseOwnsResource) {
|
|
939
|
+
result.kept.push(target);
|
|
940
|
+
continue;
|
|
941
|
+
}
|
|
942
|
+
if (lease?.status === "acquiring" && String(lease.epoch) === epoch) {
|
|
943
|
+
result.kept.push(target);
|
|
944
|
+
continue;
|
|
945
|
+
}
|
|
946
|
+
if (!driver.destroy) {
|
|
947
|
+
result.skipped.push({ target, reason: "destroy_unsupported" });
|
|
948
|
+
continue;
|
|
949
|
+
}
|
|
950
|
+
try {
|
|
951
|
+
await driver.destroy(target);
|
|
952
|
+
result.destroyed.push(target);
|
|
953
|
+
} catch (err) {
|
|
954
|
+
result.failed.push({ target, error: err instanceof Error ? err.message : String(err) });
|
|
955
|
+
}
|
|
956
|
+
}
|
|
957
|
+
return result;
|
|
958
|
+
}
|
|
959
|
+
|
|
960
|
+
// src/start-telemetry.ts
|
|
961
|
+
function createSandboxStartRecorder(options) {
|
|
962
|
+
function emit(lease, progress, timing, extra) {
|
|
963
|
+
try {
|
|
964
|
+
options.onStartPhase?.({
|
|
965
|
+
...timing,
|
|
966
|
+
sinceStartMs: Math.max(0, options.now() - progress.requestedAt),
|
|
967
|
+
workspaceId: lease.workspaceId,
|
|
968
|
+
epoch: lease.epoch,
|
|
969
|
+
driver: options.driver,
|
|
970
|
+
homeRegion: lease.homeRegion,
|
|
971
|
+
bootMode: progress.bootMode,
|
|
972
|
+
labels: extra.labels,
|
|
973
|
+
...extra.repoSizeBytes === void 0 ? {} : { repoSizeBytes: extra.repoSizeBytes }
|
|
974
|
+
});
|
|
975
|
+
} catch {
|
|
976
|
+
}
|
|
977
|
+
}
|
|
978
|
+
async function append(lease, progress, timings, extra) {
|
|
979
|
+
const fresh = timings.filter((timing, index) => !progress.phases.includes(timing.phase) && timings.findIndex((other) => other.phase === timing.phase) === index);
|
|
980
|
+
if (!fresh.length) return void 0;
|
|
981
|
+
const next = {
|
|
982
|
+
...progress,
|
|
983
|
+
markedAt: extra.markedAt ?? progress.markedAt,
|
|
984
|
+
phases: [...progress.phases, ...fresh.map((timing) => timing.phase)]
|
|
985
|
+
};
|
|
986
|
+
if (!await options.leaseStore.update(lease.workspaceId, lease.epoch, { start: next })) return void 0;
|
|
987
|
+
for (const timing of fresh) emit(lease, next, timing, extra);
|
|
988
|
+
return next;
|
|
989
|
+
}
|
|
990
|
+
async function started(workspaceId, epoch) {
|
|
991
|
+
const lease = await options.leaseStore.get(workspaceId);
|
|
992
|
+
return lease?.epoch === epoch && lease.start ? { lease, progress: lease.start } : void 0;
|
|
993
|
+
}
|
|
994
|
+
return {
|
|
995
|
+
/**
|
|
996
|
+
* The phases one provision call observes, each timed and written into the
|
|
997
|
+
* lease as it ends, so a poller reading the lease mid-boot sees how far the
|
|
998
|
+
* start has come. `opened` writes the epoch's `lease_decision` before the
|
|
999
|
+
* driver runs, so a stop that lands mid-boot finds the boot it ends on the
|
|
1000
|
+
* row. A serving lease is not starting. An epoch acquired with a start it
|
|
1001
|
+
* continues has that start and no phase yet; one acquired without begins
|
|
1002
|
+
* its own here.
|
|
1003
|
+
*/
|
|
1004
|
+
observe(lease, input) {
|
|
1005
|
+
if (lease.status === "ready") return void 0;
|
|
1006
|
+
let progress = lease.start ?? { requestedAt: input.enteredAt, startedAt: input.enteredAt, bootMode: input.bootMode, markedAt: input.enteredAt, phases: [] };
|
|
1007
|
+
const ended = [];
|
|
1008
|
+
let markedAt = progress.markedAt;
|
|
1009
|
+
const mark = (phase) => {
|
|
1010
|
+
if (progress.phases.includes(phase) || ended.some((timing) => timing.phase === phase)) return;
|
|
1011
|
+
const at = options.now();
|
|
1012
|
+
ended.push({ phase, durationMs: Math.max(0, at - markedAt) });
|
|
1013
|
+
markedAt = at;
|
|
1014
|
+
};
|
|
1015
|
+
const write = async () => {
|
|
1016
|
+
const written = await append(lease, progress, ended.splice(0), { labels: input.labels, markedAt }).catch(() => void 0);
|
|
1017
|
+
if (written) progress = written;
|
|
1018
|
+
};
|
|
1019
|
+
if (!progress.phases.length) mark("lease_decision");
|
|
1020
|
+
return {
|
|
1021
|
+
opened: write,
|
|
1022
|
+
end: async (phase) => {
|
|
1023
|
+
mark(phase);
|
|
1024
|
+
await write();
|
|
1025
|
+
}
|
|
1026
|
+
};
|
|
1027
|
+
},
|
|
1028
|
+
async report(workspaceId, input) {
|
|
1029
|
+
const start = await started(workspaceId, input.epoch);
|
|
1030
|
+
if (!start) return;
|
|
1031
|
+
await append(start.lease, start.progress, input.phases, {
|
|
1032
|
+
labels: start.lease.labels ?? {},
|
|
1033
|
+
...input.repoSizeBytes === void 0 ? {} : { repoSizeBytes: input.repoSizeBytes }
|
|
1034
|
+
});
|
|
1035
|
+
},
|
|
1036
|
+
async markAfter(workspaceId, input) {
|
|
1037
|
+
const start = await started(workspaceId, input.epoch);
|
|
1038
|
+
if (!start || input.notBefore < start.progress.startedAt) return;
|
|
1039
|
+
const at = options.now();
|
|
1040
|
+
await append(start.lease, start.progress, [{ phase: input.phase, durationMs: Math.max(0, at - start.progress.markedAt) }], {
|
|
1041
|
+
labels: start.lease.labels ?? {},
|
|
1042
|
+
markedAt: at
|
|
1043
|
+
});
|
|
1044
|
+
}
|
|
1045
|
+
};
|
|
231
1046
|
}
|
|
1047
|
+
|
|
1048
|
+
// src/lease-lifecycle.ts
|
|
1049
|
+
function createLifecycleReporter(sink, now) {
|
|
1050
|
+
const base = (lease) => ({ workspaceId: lease.workspaceId, epoch: lease.epoch, driver: lease.driver, labels: lease.labels ?? {} });
|
|
1051
|
+
const active = (lease, endedAt = now()) => lease.status === "ready" && lease.start ? { activeMs: Math.max(0, endedAt - lease.start.startedAt) } : {};
|
|
1052
|
+
const report = (event) => {
|
|
1053
|
+
try {
|
|
1054
|
+
sink?.(event);
|
|
1055
|
+
} catch {
|
|
1056
|
+
}
|
|
1057
|
+
};
|
|
1058
|
+
return {
|
|
1059
|
+
stopped: (lease, cause, outcome = {}) => report({ ...base(lease), kind: "stopped", cause, ...active(lease), ...outcome }),
|
|
1060
|
+
/** `endedAt` is when the host was last proven serving: a stop the provider made on its own is learned after the fact. */
|
|
1061
|
+
hostStopped: (lease, endedAt) => report({ ...base(lease), kind: "stopped", cause: "provider", ...active(lease, endedAt) }),
|
|
1062
|
+
destroyed: (lease) => report({ ...base(lease), kind: "destroyed", hostGone: false, ...active(lease) }),
|
|
1063
|
+
/** `endedAt` is when the host was last proven serving: the provider's loss is learned after the fact. */
|
|
1064
|
+
hostGone: (lease, endedAt) => report({ ...base(lease), kind: "destroyed", hostGone: true, ...active(lease, endedAt) }),
|
|
1065
|
+
failed: (lease, failure) => report({ ...base(lease), kind: "failed", ...failure }),
|
|
1066
|
+
startedFresh: (lease, checkpointUnusable) => report({ ...base(lease), kind: "startedFresh", ...checkpointUnusable ? { checkpointUnusable } : {} })
|
|
1067
|
+
};
|
|
1068
|
+
}
|
|
1069
|
+
|
|
1070
|
+
// src/lease-start.ts
|
|
1071
|
+
function leaseBootMode(lease, driver) {
|
|
1072
|
+
const restoring = Boolean(lease.checkpoint) && lease.status !== "ready" && driver.metadata.persistence.resume === "replacement-restore";
|
|
1073
|
+
if (restoring) return "restore";
|
|
1074
|
+
if (lease.sandboxId && lease.url && driver.resumeHost) return "resume";
|
|
1075
|
+
return "cold-start";
|
|
1076
|
+
}
|
|
1077
|
+
function continuedStart(lease, input) {
|
|
1078
|
+
if (!lease?.start) return void 0;
|
|
1079
|
+
const failure = sandboxLeaseFailure(lease, input);
|
|
1080
|
+
const continued = lease.status === "acquiring" || failure?.kind === "provider" && lease.nextRetryAt !== void 0 && lease.retryCount < input.maxRetryCount;
|
|
1081
|
+
if (!continued) return void 0;
|
|
1082
|
+
const { requestedAt, bootMode, failures } = lease.start;
|
|
1083
|
+
return { requestedAt, bootMode, startedAt: input.now, markedAt: input.now, phases: [], ...failures ? { failures } : {} };
|
|
1084
|
+
}
|
|
1085
|
+
|
|
1086
|
+
// src/start-fresh.ts
|
|
1087
|
+
function startsFreshFromImage(persistence) {
|
|
1088
|
+
return persistence.capture === "filesystem" && persistence.restoreMount === "new-resource";
|
|
1089
|
+
}
|
|
1090
|
+
async function startSandboxFresh(input) {
|
|
1091
|
+
if (!startsFreshFromImage(input.driver.metadata.persistence)) return { ok: false, reason: "start_fresh_unsupported" };
|
|
1092
|
+
const lease = await input.leaseStore.get(input.workspaceId);
|
|
1093
|
+
if (!lease) return { ok: false, reason: "runtime_lease_missing" };
|
|
1094
|
+
const failure = sandboxLeaseFailure(lease, { now: input.now, maxRetryCount: input.maxRetryCount });
|
|
1095
|
+
if (lease.status === "ready" || lease.status === "acquiring" || failure?.retrying) return { ok: false, reason: "workspace_running" };
|
|
1096
|
+
const status = lease.status === "destroyed" ? "destroyed" : "stopped";
|
|
1097
|
+
const updated = await input.leaseStore.update(input.workspaceId, lease.epoch, {
|
|
1098
|
+
status,
|
|
1099
|
+
checkpoint: null,
|
|
1100
|
+
restore: null,
|
|
1101
|
+
lastError: null,
|
|
1102
|
+
nextRetryAt: null,
|
|
1103
|
+
retryCount: 0
|
|
1104
|
+
}, lease.status);
|
|
1105
|
+
if (!updated) return { ok: false, reason: "runtime_lease_changed" };
|
|
1106
|
+
const resource = input.resource(lease);
|
|
1107
|
+
if (lease.checkpoint && resource) await discardSnapshot(input.driver.deleteSnapshot, resource, lease.checkpoint.providerReference);
|
|
1108
|
+
input.lifecycle.startedFresh(lease, failure?.kind === "checkpoint_unusable" ? failure.cause : void 0);
|
|
1109
|
+
return { ok: true, status };
|
|
1110
|
+
}
|
|
1111
|
+
|
|
1112
|
+
// src/runtime-env.ts
|
|
1113
|
+
var WORKSPACE_RUNTIME_IDENTITY_ENV_KEYS = [
|
|
1114
|
+
"WORKSPACE_RUNTIME_WORKSPACE_ID",
|
|
1115
|
+
"WORKSPACE_RUNTIME_HOST_ID",
|
|
1116
|
+
"WORKSPACE_RUNTIME_RELAY_WORKSPACE_IDS"
|
|
1117
|
+
];
|
|
1118
|
+
function workspaceRuntimeIdentityEnvConflicts(env) {
|
|
1119
|
+
if (!env) return [];
|
|
1120
|
+
return WORKSPACE_RUNTIME_IDENTITY_ENV_KEYS.filter((key) => env[key] !== void 0);
|
|
1121
|
+
}
|
|
1122
|
+
var WORKSPACE_RUNTIME_PUBLIC_ENV_KEYS = /* @__PURE__ */ new Set([
|
|
1123
|
+
...WORKSPACE_RUNTIME_IDENTITY_ENV_KEYS,
|
|
1124
|
+
"WORKSPACE_RUNTIME_DIRECTORY",
|
|
1125
|
+
"WORKSPACE_RUNTIME_PORT",
|
|
1126
|
+
"WORKSPACE_RUNTIME_HOST",
|
|
1127
|
+
"WORKSPACE_RUNTIME_SOURCE_KIND",
|
|
1128
|
+
"WORKSPACE_RUNTIME_GIT_REPO_URL",
|
|
1129
|
+
"WORKSPACE_RUNTIME_GIT_BRANCH",
|
|
1130
|
+
"WORKSPACE_RUNTIME_NATIVE_HARNESS",
|
|
1131
|
+
"WORKSPACE_RUNTIME_RELAY_JWKS_URL",
|
|
1132
|
+
"WORKSPACE_RUNTIME_MANAGEMENT_JWKS_URL",
|
|
1133
|
+
"WORKSPACE_RUNTIME_SESSION_AUTHORITY_URL"
|
|
1134
|
+
]);
|
|
1135
|
+
|
|
1136
|
+
// src/manager.ts
|
|
232
1137
|
var DEFAULT_APP_LABEL = "claxedo";
|
|
233
1138
|
function sandboxId(driver, workspaceId) {
|
|
234
1139
|
return `${driver}-${workspaceId}`;
|
|
235
1140
|
}
|
|
1141
|
+
function leaseHostId(driver, workspaceId, lease, managerInput) {
|
|
1142
|
+
return managerInput?.hostId ?? lease.hostId ?? sandboxId(driver.id, workspaceId);
|
|
1143
|
+
}
|
|
236
1144
|
function ensureHostInput(input) {
|
|
237
1145
|
const egress = sandboxEgressDisposition(input.driver.metadata.egressControl, input.managerInput?.net);
|
|
238
1146
|
const labels = {
|
|
@@ -244,7 +1152,7 @@ function ensureHostInput(input) {
|
|
|
244
1152
|
};
|
|
245
1153
|
return {
|
|
246
1154
|
workspaceId: input.workspaceId,
|
|
247
|
-
hostId: input.
|
|
1155
|
+
hostId: leaseHostId(input.driver, input.workspaceId, input.lease, input.managerInput),
|
|
248
1156
|
homeRegion: input.homeRegion,
|
|
249
1157
|
epoch: input.lease.epoch,
|
|
250
1158
|
labels,
|
|
@@ -252,16 +1160,20 @@ function ensureHostInput(input) {
|
|
|
252
1160
|
workspaceRoot: input.managerInput?.workspaceRoot ?? "/workspace",
|
|
253
1161
|
runtimeCwd: input.managerInput?.runtimeCwd,
|
|
254
1162
|
workspaceRuntimePort: input.managerInput?.workspaceRuntimePort ?? DEFAULT_WORKSPACE_RUNTIME_PORT,
|
|
255
|
-
env: input.managerInput?.env
|
|
1163
|
+
env: { ...input.managerInput?.env, ...input.leaseEnv },
|
|
256
1164
|
// Brokered secrets ride their own channel — NEVER merged into labels or env.
|
|
257
|
-
|
|
1165
|
+
// Presence of the key, not its length, is the signal: `[]` is the caller
|
|
1166
|
+
// withdrawing every brokered secret, and dropping it would reach the driver
|
|
1167
|
+
// as "preserve what you have".
|
|
1168
|
+
...input.managerInput?.secrets ? { secrets: input.managerInput.secrets } : {},
|
|
258
1169
|
source: input.managerInput?.source,
|
|
1170
|
+
...input.lease.machineClass ?? input.managerInput?.machineClass ? { machineClass: input.lease.machineClass ?? input.managerInput?.machineClass } : {},
|
|
259
1171
|
exposure: input.managerInput?.exposure ?? (input.driver.metadata.targetAccess === "loopback" ? { kind: "loopback" } : { kind: "relay" }),
|
|
260
1172
|
// Withheld, never downgraded: `undefined` here is the same shape a caller
|
|
261
1173
|
// that asked for no containment produces, so the throwing drivers take
|
|
262
1174
|
// their allow-all path and their throws stay intact for anyone who hands
|
|
263
1175
|
// them a policy directly.
|
|
264
|
-
...egress.action === "enforce" && input.managerInput?.net ? { net: input.managerInput.net } : {},
|
|
1176
|
+
...egress.action === "enforce" && input.managerInput?.net ? { net: sandboxNetworkPolicyWithBrokeredHosts(input.managerInput.net, input.managerInput.secrets) } : {},
|
|
265
1177
|
snapshot: input.managerInput?.snapshot
|
|
266
1178
|
};
|
|
267
1179
|
}
|
|
@@ -274,80 +1186,97 @@ function createSandboxManager(options) {
|
|
|
274
1186
|
const retryDelayMs = options.retryDelayMs ?? ((retryCount) => Math.min(6e4, 1e3 * 2 ** Math.max(0, retryCount - 1)));
|
|
275
1187
|
const maxRetryCount = options.maxRetryCount ?? Number.POSITIVE_INFINITY;
|
|
276
1188
|
const retryCapCooldownMs = options.retryCapCooldownMs ?? 10 * 6e4;
|
|
277
|
-
const
|
|
278
|
-
|
|
1189
|
+
const egressWarnings = createEgressWarnings({
|
|
1190
|
+
driver: { id: options.driver.id, egressControl: options.driver.metadata.egressControl },
|
|
1191
|
+
...options.onEgressUnenforced ? { sink: options.onEgressUnenforced } : {}
|
|
1192
|
+
});
|
|
279
1193
|
const lifecycleOperations = /* @__PURE__ */ new Map();
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
1194
|
+
const lifecycle = createLifecycleReporter(options.onLifecycle, now);
|
|
1195
|
+
const starts = createSandboxStartRecorder({
|
|
1196
|
+
leaseStore: options.leaseStore,
|
|
1197
|
+
driver: options.driver.id,
|
|
1198
|
+
now,
|
|
1199
|
+
...options.onStartPhase ? { onStartPhase: options.onStartPhase } : {}
|
|
1200
|
+
});
|
|
1201
|
+
function lifecycleQueue(current) {
|
|
1202
|
+
return current?.promise.catch(() => void 0) ?? Promise.resolve();
|
|
1203
|
+
}
|
|
1204
|
+
function lifecycleSettle(workspaceId, entry, operation) {
|
|
1205
|
+
lifecycleOperations.set(workspaceId, entry);
|
|
1206
|
+
return operation.finally(() => {
|
|
1207
|
+
if (lifecycleOperations.get(workspaceId) === entry) lifecycleOperations.delete(workspaceId);
|
|
294
1208
|
});
|
|
295
1209
|
}
|
|
296
|
-
|
|
297
|
-
const
|
|
298
|
-
if (
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
phase: "composition",
|
|
302
|
-
reason: "sandbox_egress_uncontained",
|
|
303
|
-
driver: options.driver.id,
|
|
304
|
-
egressControl,
|
|
305
|
-
message: egressUnenforcedMessage({ phase: "composition", driver: options.driver.id })
|
|
306
|
-
});
|
|
307
|
-
}
|
|
1210
|
+
function checkpointLifecycle(workspaceId, kind, run) {
|
|
1211
|
+
const current = lifecycleOperations.get(workspaceId);
|
|
1212
|
+
if (current?.kind === kind) return current.promise;
|
|
1213
|
+
const operation = lifecycleQueue(current).then(run);
|
|
1214
|
+
return lifecycleSettle(workspaceId, { kind, promise: operation }, operation);
|
|
308
1215
|
}
|
|
309
|
-
function
|
|
1216
|
+
function mutationLifecycle(workspaceId, kind, run) {
|
|
310
1217
|
const current = lifecycleOperations.get(workspaceId);
|
|
311
1218
|
if (current?.kind === kind) return current.promise;
|
|
312
|
-
const operation = (current
|
|
313
|
-
|
|
314
|
-
return operation.finally(() => {
|
|
315
|
-
if (lifecycleOperations.get(workspaceId)?.promise === operation) {
|
|
316
|
-
lifecycleOperations.delete(workspaceId);
|
|
317
|
-
}
|
|
318
|
-
});
|
|
1219
|
+
const operation = lifecycleQueue(current).then(run);
|
|
1220
|
+
return lifecycleSettle(workspaceId, { kind, promise: operation }, operation);
|
|
319
1221
|
}
|
|
320
|
-
|
|
321
|
-
if (
|
|
322
|
-
return { status: "unavailable", reason: "runtime_lease_not_ready" };
|
|
323
|
-
}
|
|
1222
|
+
function leaseResource(lease) {
|
|
1223
|
+
if (!lease.sandboxId || !lease.hostId) return void 0;
|
|
324
1224
|
return {
|
|
325
|
-
status: "ready",
|
|
326
1225
|
sandboxId: lease.sandboxId,
|
|
327
1226
|
url: lease.url,
|
|
328
1227
|
workspaceId: lease.workspaceId,
|
|
329
1228
|
hostId: lease.hostId,
|
|
330
1229
|
driverResourceId: lease.driverResourceId,
|
|
331
1230
|
labels: lease.labels,
|
|
332
|
-
driver: {
|
|
333
|
-
id: lease.driver,
|
|
334
|
-
resourceId: lease.driverResourceId ?? lease.sandboxId
|
|
335
|
-
},
|
|
1231
|
+
driver: { id: lease.driver, resourceId: lease.driverResourceId ?? lease.sandboxId },
|
|
336
1232
|
epoch: lease.epoch,
|
|
337
1233
|
homeRegion: lease.homeRegion
|
|
338
1234
|
};
|
|
339
1235
|
}
|
|
340
|
-
async function
|
|
1236
|
+
async function leaseTarget(lease) {
|
|
1237
|
+
const resource = leaseResource(lease);
|
|
1238
|
+
if (lease.status !== "ready" || !resource || !resource.url || !lease.routingId) {
|
|
1239
|
+
const failure = sandboxLeaseFailure(lease, { now: now(), maxRetryCount });
|
|
1240
|
+
const stopReason = lease.status === "stopped" ? sandboxStopReason(lease.lastError) : void 0;
|
|
1241
|
+
return {
|
|
1242
|
+
status: "unavailable",
|
|
1243
|
+
reason: "runtime_lease_not_ready",
|
|
1244
|
+
leaseStatus: lease.status,
|
|
1245
|
+
...lease.nextRetryAt !== void 0 ? { retryAfterMs: Math.max(0, lease.nextRetryAt - now()) } : {},
|
|
1246
|
+
...failure ? { failure } : {},
|
|
1247
|
+
...stopReason ? { stopReason } : {},
|
|
1248
|
+
...lease.start && (lease.status === "acquiring" || lease.status === "unavailable") ? { start: { ...lease.start, epoch: lease.epoch } } : {},
|
|
1249
|
+
...lease.status === "acquiring" && lease.hostId ? { booting: { epoch: lease.epoch, hostId: lease.hostId } } : {}
|
|
1250
|
+
};
|
|
1251
|
+
}
|
|
1252
|
+
const booted = resource.labels?.[SANDBOX_IMAGE_LABEL];
|
|
1253
|
+
const imageOutdated = options.driver.image !== void 0 && booted !== void 0 && booted !== options.driver.image;
|
|
1254
|
+
return {
|
|
1255
|
+
...resource,
|
|
1256
|
+
status: "ready",
|
|
1257
|
+
url: resource.url,
|
|
1258
|
+
routingId: lease.routingId,
|
|
1259
|
+
...imageOutdated ? { imageOutdated: true } : {},
|
|
1260
|
+
...lease.start ? { start: { ...lease.start, epoch: lease.epoch } } : {}
|
|
1261
|
+
};
|
|
1262
|
+
}
|
|
1263
|
+
async function provisionLease(workspaceId, lease, homeRegion, managerInput, enteredAt) {
|
|
341
1264
|
try {
|
|
1265
|
+
const leaseEnv = await managerInput.leaseEnv?.({ epoch: lease.epoch, hostId: leaseHostId(options.driver, workspaceId, lease, managerInput) });
|
|
1266
|
+
const identityConflicts = workspaceRuntimeIdentityEnvConflicts(leaseEnv);
|
|
1267
|
+
if (identityConflicts.length) {
|
|
1268
|
+
return { status: "unavailable", error: `sandbox env cannot set runtime identity: ${identityConflicts.join(", ")}`, epoch: lease.epoch, homeRegion };
|
|
1269
|
+
}
|
|
342
1270
|
const ensure = ensureHostInput({
|
|
343
1271
|
driver: options.driver,
|
|
344
1272
|
workspaceId,
|
|
345
1273
|
homeRegion,
|
|
346
1274
|
lease,
|
|
347
1275
|
managerInput,
|
|
1276
|
+
...leaseEnv ? { leaseEnv } : {},
|
|
348
1277
|
appLabel: options.appLabel ?? DEFAULT_APP_LABEL
|
|
349
1278
|
});
|
|
350
|
-
if (ensure.secrets?.length && options.driver.metadata.secretBrokering
|
|
1279
|
+
if (ensure.secrets?.length && options.driver.metadata.secretBrokering !== "native") {
|
|
351
1280
|
return {
|
|
352
1281
|
status: "unavailable",
|
|
353
1282
|
error: "secret_brokering_unsupported",
|
|
@@ -355,33 +1284,57 @@ function createSandboxManager(options) {
|
|
|
355
1284
|
homeRegion
|
|
356
1285
|
};
|
|
357
1286
|
}
|
|
358
|
-
const
|
|
1287
|
+
const resuming = Boolean(lease.sandboxId && ensure.bootSource?.kind === "default" && options.driver.resumeHost);
|
|
1288
|
+
const bootMode = resuming && lease.url ? "resume" : ensure.bootSource?.kind === "driver-snapshot" ? "restore" : "cold-start";
|
|
1289
|
+
const start = starts.observe(lease, { bootMode, enteredAt, labels: ensure.labels });
|
|
1290
|
+
await start?.opened();
|
|
1291
|
+
ensure.onResource = async (resource) => {
|
|
1292
|
+
const recorded = await options.leaseStore.recordTarget(workspaceId, lease.epoch, {
|
|
1293
|
+
...resource,
|
|
1294
|
+
labels: { ...resource.labels, ...ensure.labels },
|
|
1295
|
+
persistence: options.driver.metadata.persistence
|
|
1296
|
+
});
|
|
1297
|
+
if (!recorded) throw new Error("runtime_lease_changed");
|
|
1298
|
+
await start?.end("provider_ready");
|
|
1299
|
+
};
|
|
1300
|
+
ensure.onMachineReady = async () => start?.end("machine_ready");
|
|
1301
|
+
ensure.onImageReady = async () => start?.end("image_ready");
|
|
1302
|
+
const target = resuming ? await options.driver.resumeHost({ lease, ensure }) : await options.driver.ensureHost(ensure);
|
|
359
1303
|
if ("provisioning" in target) {
|
|
360
|
-
await options.leaseStore.update(workspaceId, lease.epoch, {
|
|
1304
|
+
const updated2 = await options.leaseStore.update(workspaceId, lease.epoch, {
|
|
361
1305
|
status: "acquiring",
|
|
362
1306
|
nextRetryAt: now() + target.retryAfterMs
|
|
363
|
-
});
|
|
1307
|
+
}, lease.status);
|
|
1308
|
+
if (!updated2) return { status: "unavailable", error: "runtime_lease_changed", epoch: lease.epoch, homeRegion };
|
|
364
1309
|
return {
|
|
365
1310
|
status: "provisioning",
|
|
366
1311
|
retryAfterMs: target.retryAfterMs,
|
|
367
1312
|
epoch: lease.epoch,
|
|
368
|
-
homeRegion
|
|
1313
|
+
homeRegion,
|
|
1314
|
+
// Honest progress for the connect UI: which of the three boot paths
|
|
1315
|
+
// this cycle is on. Derived from the same inputs that CHOSE the path
|
|
1316
|
+
// above, so it cannot drift from what actually ran.
|
|
1317
|
+
bootMode
|
|
369
1318
|
};
|
|
370
1319
|
}
|
|
371
|
-
const updated = await options.leaseStore.
|
|
372
|
-
status: "ready",
|
|
1320
|
+
const updated = await options.leaseStore.recordTarget(workspaceId, lease.epoch, {
|
|
373
1321
|
sandboxId: target.sandboxId,
|
|
374
1322
|
url: target.url,
|
|
375
1323
|
hostId: target.hostId,
|
|
376
1324
|
driverResourceId: target.driverResourceId,
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
1325
|
+
// The placement labels win over anything the driver echoed back: `app`,
|
|
1326
|
+
// `workspaceId` and `epoch` are what garbage collection reads to decide
|
|
1327
|
+
// whether a live sandbox belongs to this deployment, and a driver that
|
|
1328
|
+
// returned its own label set would otherwise decide that answer.
|
|
1329
|
+
labels: { ...target.labels, ...ensure.labels },
|
|
381
1330
|
persistence: options.driver.metadata.persistence
|
|
382
1331
|
});
|
|
383
|
-
|
|
384
|
-
|
|
1332
|
+
if (!updated) return { status: "unavailable", error: "runtime_lease_changed", epoch: lease.epoch, homeRegion };
|
|
1333
|
+
const resolved = await leaseTarget(updated);
|
|
1334
|
+
if (resolved.status === "ready") {
|
|
1335
|
+
await start?.end("runtime_ready");
|
|
1336
|
+
return resolved;
|
|
1337
|
+
}
|
|
385
1338
|
return {
|
|
386
1339
|
status: "unavailable",
|
|
387
1340
|
error: resolved.reason,
|
|
@@ -389,91 +1342,211 @@ function createSandboxManager(options) {
|
|
|
389
1342
|
homeRegion
|
|
390
1343
|
};
|
|
391
1344
|
} catch (err) {
|
|
392
|
-
const
|
|
393
|
-
if (lease.
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
1345
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
1346
|
+
if (isSandboxHostGone(err) && lease.sandboxId) {
|
|
1347
|
+
return hostGone(workspaceId, lease, managerInput, message, enteredAt);
|
|
1348
|
+
}
|
|
1349
|
+
const checkpointUnusable = isSandboxCheckpointUnusable(err) ? err.checkpointUnusable : void 0;
|
|
1350
|
+
const bootFailed = checkpointUnusable !== void 0 || isSandboxRuntimeBootFailure(err);
|
|
1351
|
+
const startLimit = !bootFailed && isSandboxProviderRateLimited(err) ? err.providerRateLimited : void 0;
|
|
1352
|
+
const error = checkpointUnusable ? sandboxCheckpointUnusableError(checkpointUnusable, message) : bootFailed ? sandboxRuntimeBootFailedError(message) : startLimit ? sandboxProviderRateLimitedError(options.driver.id, startLimit) : isSandboxProviderKeyRefused(err) ? sandboxProviderKeyRefusedError(options.driver.id, message) : isSandboxStartRefusal(err) || sandboxLeaseEndedUnderStart(message) ? message : sandboxProviderFailedError(options.driver.id, message);
|
|
1353
|
+
if (lease.status === "ready" && !bootFailed) {
|
|
1354
|
+
const updated = await options.leaseStore.update(workspaceId, lease.epoch, { lastError: error }, "ready");
|
|
1355
|
+
if (updated) {
|
|
1356
|
+
const resolved = await leaseTarget(updated);
|
|
1357
|
+
if (resolved.status === "ready") return { ...resolved, stale: true };
|
|
1358
|
+
}
|
|
397
1359
|
}
|
|
398
1360
|
const nextRetryCount = lease.retryCount + 1;
|
|
399
|
-
const
|
|
400
|
-
const
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
1361
|
+
const startLimitRetryMs = startLimit ? sandboxStartLimitRetryMs(startLimit.window) : void 0;
|
|
1362
|
+
const dayLimited = startLimit !== void 0 && startLimitRetryMs === void 0;
|
|
1363
|
+
const retryCapped = dayLimited || nextRetryCount >= maxRetryCount;
|
|
1364
|
+
const hostLeftRunning = bootFailed || retryCapped ? !await stopHeldHost(workspaceId, lease.epoch) : false;
|
|
1365
|
+
lifecycle.failed(lease, {
|
|
1366
|
+
bootFailed,
|
|
1367
|
+
retrying: !bootFailed && !retryCapped,
|
|
1368
|
+
...startLimit ? { startLimit: startLimit.window } : {},
|
|
1369
|
+
...hostLeftRunning ? { hostLeftRunning: true } : {},
|
|
1370
|
+
...checkpointUnusable ? { checkpointUnusable } : {}
|
|
1371
|
+
});
|
|
1372
|
+
const backoffMs = retryCapped ? retryCapCooldownMs : retryDelayMs(nextRetryCount);
|
|
1373
|
+
const retryInMs = dayLimited ? void 0 : Math.max(backoffMs, startLimitRetryMs ?? 0);
|
|
1374
|
+
const failed = await options.leaseStore.recordFailure(workspaceId, lease.epoch, error, retryInMs === void 0 ? void 0 : now() + retryInMs);
|
|
406
1375
|
return {
|
|
407
1376
|
status: "unavailable",
|
|
408
|
-
retryAfterMs: failed?.nextRetryAt ? Math.max(0, failed.nextRetryAt - now()) : retryAfterMs,
|
|
1377
|
+
...dayLimited ? {} : { retryAfterMs: failed?.nextRetryAt ? Math.max(0, failed.nextRetryAt - now()) : retryAfterMs },
|
|
409
1378
|
error,
|
|
410
1379
|
epoch: lease.epoch,
|
|
411
|
-
homeRegion
|
|
1380
|
+
homeRegion,
|
|
1381
|
+
...retryCapped ? { exhausted: true } : {}
|
|
1382
|
+
};
|
|
1383
|
+
}
|
|
1384
|
+
}
|
|
1385
|
+
async function hostGone(workspaceId, lease, managerInput, message, enteredAt) {
|
|
1386
|
+
if (!await endGoneHostLease(workspaceId, lease, message)) return { status: "unavailable", error: "runtime_lease_changed", epoch: lease.epoch, homeRegion: lease.homeRegion };
|
|
1387
|
+
const admitted = await admit(workspaceId, managerInput);
|
|
1388
|
+
if (!("lease" in admitted)) return admitted;
|
|
1389
|
+
return provisionLease(workspaceId, admitted.lease, admitted.lease.homeRegion, managerInput, enteredAt);
|
|
1390
|
+
}
|
|
1391
|
+
async function endGoneHostLease(workspaceId, lease, message) {
|
|
1392
|
+
const endedAt = hostLastProvenAliveAt(lease);
|
|
1393
|
+
const ended = await options.leaseStore.update(workspaceId, lease.epoch, {
|
|
1394
|
+
status: "destroyed",
|
|
1395
|
+
lastError: sandboxHostGoneError(message),
|
|
1396
|
+
...endedAt === void 0 ? {} : { endedAt }
|
|
1397
|
+
});
|
|
1398
|
+
if (!ended) return false;
|
|
1399
|
+
lifecycle.hostGone(lease, endedAt);
|
|
1400
|
+
return true;
|
|
1401
|
+
}
|
|
1402
|
+
async function endStoppedHostLease(workspaceId, lease) {
|
|
1403
|
+
const endedAt = hostLastProvenAliveAt(lease);
|
|
1404
|
+
const ended = await options.leaseStore.update(workspaceId, lease.epoch, { status: "stopped", ...endedAt === void 0 ? {} : { endedAt } });
|
|
1405
|
+
if (!ended) return false;
|
|
1406
|
+
lifecycle.hostStopped(lease, endedAt);
|
|
1407
|
+
return true;
|
|
1408
|
+
}
|
|
1409
|
+
function hostLastProvenAliveAt(lease) {
|
|
1410
|
+
if (lease.status !== "ready" || !lease.start) return void 0;
|
|
1411
|
+
return Math.max(lease.start.markedAt, lease.lastHeartbeatAt ?? 0, lease.lastActivityAt ?? 0);
|
|
1412
|
+
}
|
|
1413
|
+
async function hostStopped(stopHost, resource) {
|
|
1414
|
+
try {
|
|
1415
|
+
await stopHost?.(resource);
|
|
1416
|
+
return "stopped";
|
|
1417
|
+
} catch (err) {
|
|
1418
|
+
if (!isSandboxHostGone(err)) throw err;
|
|
1419
|
+
return "destroyed";
|
|
1420
|
+
}
|
|
1421
|
+
}
|
|
1422
|
+
async function stopHeldHost(workspaceId, epoch) {
|
|
1423
|
+
const current = await options.leaseStore.get(workspaceId);
|
|
1424
|
+
const resource = current?.epoch === epoch ? leaseResource(current) : void 0;
|
|
1425
|
+
const stopHost = options.driver.suspend ?? options.driver.stop;
|
|
1426
|
+
if (!current || !resource || !stopHost) return true;
|
|
1427
|
+
try {
|
|
1428
|
+
await stopHost({ ...resource, ...current.checkpoint ? { checkpoint: current.checkpoint.providerReference } : {} });
|
|
1429
|
+
return true;
|
|
1430
|
+
} catch (stopError) {
|
|
1431
|
+
console.warn("A sandbox host was not stopped after its start failed", {
|
|
1432
|
+
workspaceId,
|
|
1433
|
+
epoch,
|
|
1434
|
+
sandboxId: resource.sandboxId,
|
|
1435
|
+
error: stopError instanceof Error ? stopError.message : String(stopError)
|
|
1436
|
+
});
|
|
1437
|
+
return false;
|
|
1438
|
+
}
|
|
1439
|
+
}
|
|
1440
|
+
function capture(manager, workspaceId, request, endsLease) {
|
|
1441
|
+
return captureSandboxCheckpoint({
|
|
1442
|
+
workspaceId,
|
|
1443
|
+
request,
|
|
1444
|
+
leaseStore: options.leaseStore,
|
|
1445
|
+
target: (id) => manager.target(id),
|
|
1446
|
+
snapshot: (id, committed) => manager.snapshot(id, committed),
|
|
1447
|
+
...options.driver.deleteSnapshot ? { deleteSnapshot: options.driver.deleteSnapshot } : {},
|
|
1448
|
+
...endsLease ? { endsLease } : {},
|
|
1449
|
+
now
|
|
1450
|
+
});
|
|
1451
|
+
}
|
|
1452
|
+
function wanted(lease, afterMs = retryAfterMs) {
|
|
1453
|
+
return { status: "provisioning", retryAfterMs: afterMs, epoch: lease.epoch, homeRegion: lease.homeRegion, bootMode: leaseBootMode(lease, options.driver) };
|
|
1454
|
+
}
|
|
1455
|
+
function waiting(existing) {
|
|
1456
|
+
if (!existing) return void 0;
|
|
1457
|
+
if (existing.nextRetryAt && existing.nextRetryAt > now()) {
|
|
1458
|
+
if (existing.status === "acquiring") return wanted(existing, existing.nextRetryAt - now());
|
|
1459
|
+
return {
|
|
1460
|
+
status: "unavailable",
|
|
1461
|
+
retryAfterMs: existing.nextRetryAt - now(),
|
|
1462
|
+
error: existing.lastError,
|
|
1463
|
+
epoch: existing.epoch,
|
|
1464
|
+
homeRegion: existing.homeRegion,
|
|
1465
|
+
...existing.retryCount >= maxRetryCount ? { exhausted: true } : {}
|
|
412
1466
|
};
|
|
413
1467
|
}
|
|
1468
|
+
if (existing.status === "unavailable" && existing.retryCount >= maxRetryCount && existing.nextRetryAt === void 0 && sandboxProviderRateLimit(existing.lastError) !== "day") {
|
|
1469
|
+
return {
|
|
1470
|
+
status: "unavailable",
|
|
1471
|
+
error: existing.lastError ?? SANDBOX_RETRY_CAP_EXCEEDED,
|
|
1472
|
+
epoch: existing.epoch,
|
|
1473
|
+
homeRegion: existing.homeRegion
|
|
1474
|
+
};
|
|
1475
|
+
}
|
|
1476
|
+
return void 0;
|
|
1477
|
+
}
|
|
1478
|
+
function continues(lease) {
|
|
1479
|
+
if (lease.status === "ready") return true;
|
|
1480
|
+
if (lease.status !== "acquiring") return false;
|
|
1481
|
+
const stale = now() - lease.updatedAt >= staleAfterMs;
|
|
1482
|
+
return lease.sandboxId ? lease.nextRetryAt !== void 0 || stale : lease.nextRetryAt !== void 0 && !stale;
|
|
1483
|
+
}
|
|
1484
|
+
async function admit(workspaceId, input) {
|
|
1485
|
+
const egress = sandboxEgressDisposition(options.driver.metadata.egressControl, input.net);
|
|
1486
|
+
if (egress.action === "withhold" && input.net) {
|
|
1487
|
+
egressWarnings.withheld({ workspaceId, ...input.labels ? { labels: input.labels } : {}, requested: input.net });
|
|
1488
|
+
}
|
|
1489
|
+
if (egress.action === "refuse") {
|
|
1490
|
+
return { status: "unavailable", error: egress.reason, homeRegion: input.homeRegion };
|
|
1491
|
+
}
|
|
1492
|
+
const identityConflicts = workspaceRuntimeIdentityEnvConflicts(input.env);
|
|
1493
|
+
if (identityConflicts.length) {
|
|
1494
|
+
return {
|
|
1495
|
+
status: "unavailable",
|
|
1496
|
+
error: `sandbox env cannot set runtime identity: ${identityConflicts.join(", ")}`,
|
|
1497
|
+
homeRegion: input.homeRegion
|
|
1498
|
+
};
|
|
1499
|
+
}
|
|
1500
|
+
if (input.machineClass && !options.driver.metadata.machineClasses?.includes(input.machineClass)) {
|
|
1501
|
+
return { status: "unavailable", error: `sandbox_machine_class_unsupported: ${options.driver.id} has no ${input.machineClass} machines`, homeRegion: input.homeRegion };
|
|
1502
|
+
}
|
|
1503
|
+
const existing = await options.leaseStore.get(workspaceId);
|
|
1504
|
+
const wait = waiting(existing);
|
|
1505
|
+
if (wait) return wait;
|
|
1506
|
+
if (existing && continues(existing)) return { lease: existing };
|
|
1507
|
+
const start = continuedStart(existing, { now: now(), maxRetryCount });
|
|
1508
|
+
const acquired = await options.leaseStore.acquire(workspaceId, {
|
|
1509
|
+
homeRegion: input.homeRegion,
|
|
1510
|
+
driver: options.driver.id,
|
|
1511
|
+
staleAfterMs,
|
|
1512
|
+
now: now(),
|
|
1513
|
+
...input.labels ? { labels: input.labels } : {},
|
|
1514
|
+
...input.machineClass ? { machineClass: input.machineClass } : {},
|
|
1515
|
+
...start ? { start } : {}
|
|
1516
|
+
});
|
|
1517
|
+
if (!acquired.acquired) return wanted(acquired.lease, acquired.retryAfterMs);
|
|
1518
|
+
if (!existing) input.onLeaseOpened?.();
|
|
1519
|
+
return { lease: acquired.lease };
|
|
414
1520
|
}
|
|
415
1521
|
return {
|
|
416
1522
|
async ensure(workspaceId, input) {
|
|
417
|
-
const
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
homeRegion: existing.homeRegion
|
|
432
|
-
};
|
|
433
|
-
}
|
|
434
|
-
return {
|
|
435
|
-
status: "unavailable",
|
|
436
|
-
retryAfterMs: existing.nextRetryAt - now(),
|
|
437
|
-
error: existing.lastError,
|
|
438
|
-
epoch: existing.epoch,
|
|
439
|
-
homeRegion: existing.homeRegion
|
|
440
|
-
};
|
|
441
|
-
}
|
|
442
|
-
if (existing?.status === "unavailable" && existing.retryCount >= maxRetryCount && existing.nextRetryAt === void 0) {
|
|
443
|
-
return {
|
|
444
|
-
status: "unavailable",
|
|
445
|
-
error: existing.lastError ?? "runtime_retry_cap_exceeded",
|
|
446
|
-
epoch: existing.epoch,
|
|
447
|
-
homeRegion: existing.homeRegion
|
|
448
|
-
};
|
|
449
|
-
}
|
|
450
|
-
if (existing?.status === "ready") {
|
|
451
|
-
return provision(workspaceId, existing, existing.homeRegion, input);
|
|
1523
|
+
const enteredAt = now();
|
|
1524
|
+
const admitted = await admit(workspaceId, input);
|
|
1525
|
+
if (!("lease" in admitted)) return admitted;
|
|
1526
|
+
return provisionLease(workspaceId, admitted.lease, admitted.lease.homeRegion, input, enteredAt);
|
|
1527
|
+
},
|
|
1528
|
+
async acquire(workspaceId, input) {
|
|
1529
|
+
const admitted = await admit(workspaceId, input);
|
|
1530
|
+
return "lease" in admitted ? wanted(admitted.lease) : admitted;
|
|
1531
|
+
},
|
|
1532
|
+
async provision(workspaceId, epoch, input) {
|
|
1533
|
+
const enteredAt = now();
|
|
1534
|
+
const lease = await options.leaseStore.get(workspaceId);
|
|
1535
|
+
if (!lease || lease.epoch !== epoch) {
|
|
1536
|
+
return { status: "unavailable", error: "runtime_lease_changed", epoch, homeRegion: input.homeRegion };
|
|
452
1537
|
}
|
|
453
|
-
if (
|
|
454
|
-
return
|
|
1538
|
+
if (lease.status === "stopped" || lease.status === "destroyed") {
|
|
1539
|
+
return { status: "unavailable", error: `runtime_lease_${lease.status}`, epoch, homeRegion: lease.homeRegion };
|
|
455
1540
|
}
|
|
456
|
-
const
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
staleAfterMs,
|
|
460
|
-
now: now()
|
|
461
|
-
});
|
|
462
|
-
if (!acquired.acquired) {
|
|
463
|
-
return {
|
|
464
|
-
status: "provisioning",
|
|
465
|
-
retryAfterMs: acquired.retryAfterMs,
|
|
466
|
-
epoch: acquired.lease.epoch,
|
|
467
|
-
homeRegion: acquired.lease.homeRegion
|
|
468
|
-
};
|
|
469
|
-
}
|
|
470
|
-
return provision(workspaceId, acquired.lease, input.homeRegion, input);
|
|
1541
|
+
const admitted = lease.status === "unavailable" ? await admit(workspaceId, input) : waiting(lease) ?? { lease };
|
|
1542
|
+
if (!("lease" in admitted)) return admitted;
|
|
1543
|
+
return provisionLease(workspaceId, admitted.lease, admitted.lease.homeRegion, input, enteredAt);
|
|
471
1544
|
},
|
|
472
1545
|
async register(workspaceId, input) {
|
|
473
|
-
return await
|
|
1546
|
+
return await runtimeSnapshot(workspaceId, input);
|
|
474
1547
|
},
|
|
475
1548
|
async heartbeat(workspaceId, input) {
|
|
476
|
-
return await
|
|
1549
|
+
return await runtimeSnapshot(workspaceId, input);
|
|
477
1550
|
},
|
|
478
1551
|
async target(workspaceId) {
|
|
479
1552
|
const lease = await options.leaseStore.get(workspaceId);
|
|
@@ -486,24 +1559,17 @@ function createSandboxManager(options) {
|
|
|
486
1559
|
await options.driver.touch?.(target);
|
|
487
1560
|
return { touched: true, status: "ready" };
|
|
488
1561
|
},
|
|
489
|
-
async snapshot(workspaceId) {
|
|
1562
|
+
async snapshot(workspaceId, committed) {
|
|
490
1563
|
const target = await this.target(workspaceId);
|
|
491
1564
|
if (target.status !== "ready") return { ok: false, reason: target.reason };
|
|
492
1565
|
if (!options.driver.snapshot) return { ok: false, reason: "snapshot_unsupported" };
|
|
493
|
-
return { ok: true, ...await options.driver.snapshot(target) };
|
|
1566
|
+
return { ok: true, ...await options.driver.snapshot(target, committed) };
|
|
494
1567
|
},
|
|
495
1568
|
async checkpoint(workspaceId, input) {
|
|
496
|
-
return await
|
|
497
|
-
workspaceId,
|
|
498
|
-
request: input,
|
|
499
|
-
leaseStore: options.leaseStore,
|
|
500
|
-
target: (id) => this.target(id),
|
|
501
|
-
snapshot: (id) => this.snapshot(id),
|
|
502
|
-
now
|
|
503
|
-
}));
|
|
1569
|
+
return await checkpointLifecycle(workspaceId, "checkpoint", () => capture(this, workspaceId, input));
|
|
504
1570
|
},
|
|
505
1571
|
async restore(workspaceId, input) {
|
|
506
|
-
return await
|
|
1572
|
+
return await checkpointLifecycle(workspaceId, "restore", () => restoreSandboxCheckpoint({
|
|
507
1573
|
workspaceId,
|
|
508
1574
|
request: input,
|
|
509
1575
|
leaseStore: options.leaseStore,
|
|
@@ -511,139 +1577,178 @@ function createSandboxManager(options) {
|
|
|
511
1577
|
now
|
|
512
1578
|
}));
|
|
513
1579
|
},
|
|
514
|
-
async stop(workspaceId) {
|
|
515
|
-
return await
|
|
1580
|
+
async stop(workspaceId, input = {}) {
|
|
1581
|
+
return await mutationLifecycle(workspaceId, "stop", async () => {
|
|
516
1582
|
const lease = await options.leaseStore.get(workspaceId);
|
|
517
|
-
if (lease?.
|
|
1583
|
+
if (input.expectedEpoch !== void 0 && lease?.epoch !== input.expectedEpoch) return { ok: false, reason: "runtime_lease_changed" };
|
|
1584
|
+
const stopped = { ok: true, status: "stopped" };
|
|
1585
|
+
if (lease?.status === "stopped") return stopped;
|
|
518
1586
|
const target = await this.target(workspaceId);
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
1587
|
+
const stopHost = input.hostStopsItself ? void 0 : options.driver.suspend ?? options.driver.stop;
|
|
1588
|
+
const cause = input.idleBefore !== void 0 ? "idle" : input.unresponsiveForMs !== void 0 ? "unresponsive" : "explicit";
|
|
1589
|
+
const stoppedError = input.unresponsiveForMs === void 0 ? void 0 : sandboxRuntimeUnresponsiveError(input.unresponsiveForMs);
|
|
1590
|
+
const ended = async (resource2, status, unsaved2 = false) => {
|
|
1591
|
+
if (!lease) return void 0;
|
|
1592
|
+
const endedAt = status === "destroyed" ? hostLastProvenAliveAt(lease) : void 0;
|
|
1593
|
+
const updated = await options.leaseStore.update(workspaceId, resource2.epoch, {
|
|
1594
|
+
status,
|
|
1595
|
+
...status === "destroyed" ? { lastError: sandboxHostGoneError(`${resource2.sandboxId} is gone`) } : stoppedError ? { lastError: stoppedError } : {},
|
|
1596
|
+
...endedAt === void 0 ? {} : { endedAt }
|
|
1597
|
+
});
|
|
1598
|
+
if (!updated) return void 0;
|
|
1599
|
+
if (status === "destroyed") lifecycle.hostGone(lease, endedAt);
|
|
1600
|
+
else lifecycle.stopped(lease, cause, unsaved2 ? { unsaved: true } : {});
|
|
1601
|
+
return { ok: true, status, ...unsaved2 ? { unsaved: true } : {} };
|
|
1602
|
+
};
|
|
1603
|
+
if (target.status !== "ready") {
|
|
1604
|
+
const held = lease?.status === "acquiring" ? leaseResource(lease) : void 0;
|
|
1605
|
+
if (!lease || !held) return { ok: false, reason: target.reason };
|
|
1606
|
+
return await ended(held, await hostStopped(stopHost, held)) ?? { ok: false, reason: "runtime_lease_changed" };
|
|
1607
|
+
}
|
|
1608
|
+
const persistence2 = lease?.persistence;
|
|
1609
|
+
let unsaved = false;
|
|
1610
|
+
if (input.runtime && persistence2 && persistence2.capture !== "none") {
|
|
1611
|
+
try {
|
|
1612
|
+
const captured = await capture(this, workspaceId, { runtime: input.runtime, policy: "drain", idleBefore: input.idleBefore }, stoppedError ? { stoppedError } : {});
|
|
1613
|
+
const checkpoint2 = captured.checkpoint.providerReference;
|
|
1614
|
+
if (persistence2.captureSource === "preserved") await stopHost?.({ ...target, checkpoint: checkpoint2 });
|
|
1615
|
+
if (lease) lifecycle.stopped(lease, cause, captured.checkpoint.metadata.unfrozen ? { unfrozen: true } : {});
|
|
1616
|
+
return stopped;
|
|
1617
|
+
} catch (error) {
|
|
1618
|
+
if (input.idleBefore !== void 0) throw error;
|
|
1619
|
+
console.warn("A stopping sandbox was not captured; it stops on its last checkpoint", {
|
|
1620
|
+
workspaceId,
|
|
1621
|
+
epoch: target.epoch,
|
|
1622
|
+
error: error instanceof Error ? error.message : String(error)
|
|
1623
|
+
});
|
|
1624
|
+
const current = await options.leaseStore.get(workspaceId);
|
|
1625
|
+
if (current?.status === "stopped") return stopped;
|
|
1626
|
+
if (current?.epoch !== target.epoch || current.status !== "ready") return { ok: false, reason: "runtime_lease_changed" };
|
|
1627
|
+
unsaved = true;
|
|
1628
|
+
}
|
|
524
1629
|
}
|
|
525
|
-
|
|
526
|
-
|
|
1630
|
+
const checkpoint = lease?.checkpoint?.providerReference;
|
|
1631
|
+
const resource = { ...target, ...checkpoint ? { checkpoint } : {} };
|
|
1632
|
+
return await ended(resource, await hostStopped(stopHost, resource), unsaved) ?? { ok: false, reason: "runtime_lease_changed" };
|
|
527
1633
|
});
|
|
528
1634
|
},
|
|
1635
|
+
async endIfHostDown(workspaceId, epoch) {
|
|
1636
|
+
const lease = await options.leaseStore.get(workspaceId);
|
|
1637
|
+
if (!lease || lease.epoch !== epoch) return void 0;
|
|
1638
|
+
const target = await leaseTarget(lease);
|
|
1639
|
+
if (target.status !== "ready") return void 0;
|
|
1640
|
+
const unansweredAt = now();
|
|
1641
|
+
const state = options.driver.inspect ? await options.driver.inspect(target) : void 0;
|
|
1642
|
+
if (state === void 0 || state === "serving") {
|
|
1643
|
+
await options.leaseStore.update(workspaceId, epoch, { lastHealthFailureAt: unansweredAt, ...state ? { lastHeartbeatAt: unansweredAt } : {} }, "ready");
|
|
1644
|
+
return void 0;
|
|
1645
|
+
}
|
|
1646
|
+
const ended = state === "gone" ? await endGoneHostLease(workspaceId, lease, `${target.sandboxId} is gone`) : await endStoppedHostLease(workspaceId, lease);
|
|
1647
|
+
return ended ? state : void 0;
|
|
1648
|
+
},
|
|
529
1649
|
async destroy(workspaceId) {
|
|
530
|
-
return await
|
|
1650
|
+
return await mutationLifecycle(workspaceId, "destroy", async () => {
|
|
531
1651
|
const lease = await options.leaseStore.get(workspaceId);
|
|
532
1652
|
if (lease?.status === "destroyed") return { ok: true, status: "destroyed" };
|
|
533
|
-
|
|
534
|
-
|
|
1653
|
+
if (!lease) return { ok: false, reason: "runtime_lease_missing" };
|
|
1654
|
+
const target = leaseResource(lease);
|
|
1655
|
+
if (!target) return { ok: false, reason: "runtime_lease_resource_missing" };
|
|
535
1656
|
await options.driver.destroy?.(target);
|
|
536
|
-
await options.leaseStore.update(workspaceId, target.epoch, { status: "destroyed" });
|
|
1657
|
+
const updated = await options.leaseStore.update(workspaceId, target.epoch, { status: "destroyed" });
|
|
1658
|
+
if (!updated) return { ok: false, reason: "runtime_lease_changed" };
|
|
1659
|
+
lifecycle.destroyed(lease);
|
|
1660
|
+
if (lease.checkpoint && lease.persistence?.capture !== "same-resource") {
|
|
1661
|
+
await discardSnapshot(options.driver.deleteSnapshot, target, lease.checkpoint.providerReference);
|
|
1662
|
+
}
|
|
537
1663
|
return { ok: true, status: "destroyed" };
|
|
538
1664
|
});
|
|
539
1665
|
},
|
|
1666
|
+
startFresh(workspaceId) {
|
|
1667
|
+
return mutationLifecycle(workspaceId, "startFresh", () => startSandboxFresh({
|
|
1668
|
+
workspaceId,
|
|
1669
|
+
driver: options.driver,
|
|
1670
|
+
leaseStore: options.leaseStore,
|
|
1671
|
+
lifecycle,
|
|
1672
|
+
resource: leaseResource,
|
|
1673
|
+
now: now(),
|
|
1674
|
+
maxRetryCount
|
|
1675
|
+
}));
|
|
1676
|
+
},
|
|
540
1677
|
async release(workspaceId) {
|
|
541
1678
|
const lease = await options.leaseStore.get(workspaceId);
|
|
542
1679
|
if (!lease) return { released: false };
|
|
543
1680
|
await options.leaseStore.release(workspaceId);
|
|
544
1681
|
return { released: true };
|
|
545
1682
|
},
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
kept: [],
|
|
551
|
-
skipped: [],
|
|
552
|
-
failed: [],
|
|
553
|
-
listingUnsupported: true,
|
|
554
|
-
driver: options.driver.id
|
|
555
|
-
};
|
|
556
|
-
}
|
|
557
|
-
let listed;
|
|
558
|
-
try {
|
|
559
|
-
listed = await options.driver.list();
|
|
560
|
-
} catch (err) {
|
|
561
|
-
if (!isSandboxListingUnsupported(err)) throw err;
|
|
562
|
-
return {
|
|
563
|
-
destroyed: [],
|
|
564
|
-
kept: [],
|
|
565
|
-
skipped: [],
|
|
566
|
-
failed: [],
|
|
567
|
-
listingUnsupported: true,
|
|
568
|
-
driver: options.driver.id
|
|
569
|
-
};
|
|
570
|
-
}
|
|
571
|
-
const leases = new Map((await options.leaseStore.list()).map((lease) => [lease.workspaceId, lease]));
|
|
572
|
-
const result = {
|
|
573
|
-
destroyed: [],
|
|
574
|
-
kept: [],
|
|
575
|
-
skipped: [],
|
|
576
|
-
failed: []
|
|
577
|
-
};
|
|
578
|
-
const appLabel = options.appLabel ?? DEFAULT_APP_LABEL;
|
|
579
|
-
for (const target of listed) {
|
|
580
|
-
if (target.labels?.app !== appLabel) {
|
|
581
|
-
result.skipped.push({ target, reason: "unmanaged_app_label" });
|
|
582
|
-
continue;
|
|
583
|
-
}
|
|
584
|
-
const workspaceId = target.labels.workspaceId;
|
|
585
|
-
const epoch = target.labels.epoch;
|
|
586
|
-
if (!workspaceId || !epoch) {
|
|
587
|
-
result.skipped.push({ target, reason: "missing_runtime_labels" });
|
|
588
|
-
continue;
|
|
589
|
-
}
|
|
590
|
-
const lease = leases.get(workspaceId);
|
|
591
|
-
if (lease?.status === "ready" && String(lease.epoch) === epoch && lease.hostId === target.hostId && lease.sandboxId === target.sandboxId) {
|
|
592
|
-
result.kept.push(target);
|
|
593
|
-
continue;
|
|
594
|
-
}
|
|
595
|
-
if (lease?.status === "acquiring" && String(lease.epoch) === epoch) {
|
|
596
|
-
result.kept.push(target);
|
|
597
|
-
continue;
|
|
598
|
-
}
|
|
599
|
-
if (!options.driver.destroy) {
|
|
600
|
-
result.skipped.push({ target, reason: "destroy_unsupported" });
|
|
601
|
-
continue;
|
|
602
|
-
}
|
|
603
|
-
try {
|
|
604
|
-
await options.driver.destroy(target);
|
|
605
|
-
result.destroyed.push(target);
|
|
606
|
-
} catch (err) {
|
|
607
|
-
result.failed.push({ target, error: err instanceof Error ? err.message : String(err) });
|
|
608
|
-
}
|
|
609
|
-
}
|
|
610
|
-
return result;
|
|
611
|
-
},
|
|
612
|
-
list: () => options.leaseStore.list()
|
|
1683
|
+
garbageCollect: () => sweepSandboxes({ driver: options.driver, leaseStore: options.leaseStore, appLabel: options.appLabel ?? DEFAULT_APP_LABEL }),
|
|
1684
|
+
list: () => options.leaseStore.list(),
|
|
1685
|
+
recordStartPhases: (workspaceId, input) => starts.report(workspaceId, input),
|
|
1686
|
+
markStartPhase: (workspaceId, input) => starts.markAfter(workspaceId, input)
|
|
613
1687
|
};
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
if (!current) return { ok: false, reason: "runtime_lease_missing" };
|
|
617
|
-
const expectedEpoch = input.epoch ?? current.epoch;
|
|
618
|
-
if (current.epoch !== expectedEpoch) return { ok: false, reason: "runtime_lease_epoch_mismatch" };
|
|
619
|
-
const timestamp = input.now ?? now();
|
|
620
|
-
const status = input.status === "unhealthy" || input.ok === false ? "unavailable" : input.ok ? "ready" : current.status;
|
|
621
|
-
const updated = await options.leaseStore.update(workspaceId, expectedEpoch, {
|
|
622
|
-
status,
|
|
623
|
-
...input.sandboxId ? { sandboxId: input.sandboxId } : {},
|
|
624
|
-
...input.url ? { url: input.url } : {},
|
|
625
|
-
...input.hostId ? { hostId: input.hostId } : input.sandboxId ? { hostId: input.sandboxId } : {},
|
|
626
|
-
...input.driverResourceId ? { driverResourceId: input.driverResourceId } : input.sandboxId ? { driverResourceId: input.sandboxId } : {},
|
|
627
|
-
lastHeartbeatAt: timestamp,
|
|
628
|
-
...input.active ? { lastActivityAt: timestamp } : {},
|
|
629
|
-
...input.ok ? { lastError: null } : {}
|
|
630
|
-
});
|
|
631
|
-
if (!updated) return { ok: false, reason: "runtime_lease_epoch_mismatch" };
|
|
632
|
-
return { ok: true, status: updated.status };
|
|
1688
|
+
function runtimeSnapshot(workspaceId, snapshot) {
|
|
1689
|
+
return applySandboxRuntimeSnapshot({ leaseStore: options.leaseStore, workspaceId, snapshot, now });
|
|
633
1690
|
}
|
|
634
1691
|
}
|
|
635
1692
|
export {
|
|
636
1693
|
DEFAULT_WORKSPACE_RUNTIME_PORT,
|
|
637
1694
|
SANDBOX_BASELINE_EGRESS_HOSTS,
|
|
1695
|
+
SANDBOX_IMAGE_LABEL,
|
|
638
1696
|
SANDBOX_MODEL_PROVIDER_HOSTS,
|
|
639
1697
|
SANDBOX_PACKAGE_REGISTRY_HOSTS,
|
|
1698
|
+
SANDBOX_RETRY_CAP_EXCEEDED,
|
|
1699
|
+
SANDBOX_RUNTIME_UNHEALTHY,
|
|
1700
|
+
SANDBOX_START_WINDOWS,
|
|
1701
|
+
SandboxCheckpointUnusableError,
|
|
1702
|
+
SandboxHostGoneError,
|
|
1703
|
+
SandboxProviderKeyRefusedError,
|
|
1704
|
+
SandboxProviderRateLimitedError,
|
|
1705
|
+
SandboxRuntimeBootError,
|
|
1706
|
+
SandboxStartRefusedError,
|
|
1707
|
+
applySandboxLeaseFailure,
|
|
640
1708
|
applySandboxLeasePatch,
|
|
1709
|
+
applySandboxProvisionedTarget,
|
|
1710
|
+
applySandboxRuntimeSnapshot,
|
|
1711
|
+
boatBilling,
|
|
1712
|
+
brokeredPlaceholderEnv,
|
|
1713
|
+
brokeredSecretPlaceholder,
|
|
641
1714
|
captureSandboxCheckpoint,
|
|
1715
|
+
cloudflareBilling,
|
|
642
1716
|
createSandboxManager,
|
|
1717
|
+
discardSnapshot,
|
|
643
1718
|
hostedSandboxNetworkPolicy,
|
|
1719
|
+
isSandboxCheckpointUnusable,
|
|
1720
|
+
isSandboxCheckpointUnusableCause,
|
|
1721
|
+
isSandboxHostGone,
|
|
644
1722
|
isSandboxListingUnsupported,
|
|
1723
|
+
isSandboxProviderKeyRefused,
|
|
1724
|
+
isSandboxProviderRateLimited,
|
|
1725
|
+
isSandboxRuntimeBootFailure,
|
|
1726
|
+
isSandboxStartRefusal,
|
|
1727
|
+
liftSandboxRetryWait,
|
|
645
1728
|
restoreSandboxCheckpoint,
|
|
1729
|
+
runtimeBootDiagnostic,
|
|
1730
|
+
sandboxCheckpointUnusable,
|
|
1731
|
+
sandboxCheckpointUnusableCauses,
|
|
1732
|
+
sandboxCheckpointUnusableError,
|
|
646
1733
|
sandboxEgressDisposition,
|
|
1734
|
+
sandboxHostGoneError,
|
|
1735
|
+
sandboxLeaseEndedUnderStart,
|
|
1736
|
+
sandboxLeaseFailure,
|
|
1737
|
+
sandboxLeaseHeldHost,
|
|
1738
|
+
sandboxMachineCostUsd,
|
|
1739
|
+
sandboxMachineRate,
|
|
1740
|
+
sandboxNetworkPolicyWithBrokeredHosts,
|
|
1741
|
+
sandboxProviderFailedError,
|
|
1742
|
+
sandboxProviderKeyRefusedError,
|
|
1743
|
+
sandboxProviderRateLimit,
|
|
1744
|
+
sandboxProviderRateLimitedError,
|
|
1745
|
+
sandboxRuntimeBootFailedError,
|
|
1746
|
+
sandboxRuntimeBootFailure,
|
|
1747
|
+
sandboxRuntimeUnresponsiveError,
|
|
647
1748
|
sandboxSourceHost,
|
|
648
|
-
|
|
1749
|
+
sandboxStartFailureReason,
|
|
1750
|
+
sandboxStartLimitRetryMs,
|
|
1751
|
+
sandboxStopReason,
|
|
1752
|
+
validateSandboxPersistenceCapabilities,
|
|
1753
|
+
workspaceRuntimeBootReason
|
|
649
1754
|
};
|