@claxedo/sandbox-manager 0.9.0 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +233 -26
- package/dist/checkpoint-manager.d.ts +29 -4
- package/dist/checkpoint-manager.mjs +133 -24
- package/dist/command.d.ts +9 -2
- package/dist/command.mjs +8 -8
- package/dist/contract.d.ts +687 -0
- package/dist/driver-billing.d.ts +29 -0
- package/dist/driver-catalog.d.ts +42 -7
- package/dist/driver-catalog.mjs +115 -161
- package/dist/drivers/boat-client.d.ts +95 -0
- package/dist/drivers/boat-named-snapshots.d.ts +31 -0
- package/dist/drivers/boat-template.d.ts +49 -0
- package/dist/drivers/boat-template.mjs +1196 -0
- package/dist/drivers/boat.d.ts +64 -0
- package/dist/drivers/boat.mjs +990 -0
- package/dist/drivers/cloudflare.d.ts +3 -3
- package/dist/drivers/cloudflare.mjs +208 -147
- package/dist/drivers/docker.d.ts +4 -3
- package/dist/drivers/docker.mjs +154 -126
- package/dist/drivers/local-brokering.d.ts +14 -0
- package/dist/drivers/local-brokering.mjs +1029 -0
- package/dist/drivers/modal.d.ts +2 -2
- package/dist/drivers/modal.mjs +121 -111
- package/dist/drivers/transient-error.d.ts +2 -0
- package/dist/drivers/vercel.d.ts +3 -2
- package/dist/drivers/vercel.mjs +151 -115
- package/dist/egress-policy.d.ts +134 -0
- package/dist/egress-warnings.d.ts +22 -0
- package/dist/garbage-collect.d.ts +14 -0
- package/dist/hosted-network-policy.d.ts +9 -2
- package/dist/hosted-network-policy.mjs +218 -1
- package/dist/image-name.d.ts +12 -2
- package/dist/image-name.mjs +14 -5
- package/dist/image.d.ts +1 -20
- package/dist/image.mjs +11 -88
- package/dist/index.d.ts +8 -578
- package/dist/index.mjs +1379 -320
- package/dist/lease-failure.d.ts +173 -0
- package/dist/lease-lifecycle.d.ts +65 -0
- package/dist/lease-retry-wait.d.ts +8 -0
- package/dist/lease-start.d.ts +21 -0
- package/dist/lease-types.d.ts +4 -3
- package/dist/manager.d.ts +2 -0
- package/dist/runtime-env.d.ts +4 -2
- package/dist/runtime-env.mjs +37 -2
- package/dist/runtime-snapshot.d.ts +49 -0
- package/dist/runtime-version.d.ts +9 -0
- package/dist/runtime-version.mjs +2 -0
- package/dist/service-url-exposure.d.ts +0 -1
- package/dist/service-url-exposure.mjs +0 -9
- package/dist/start-fresh.d.ts +17 -0
- package/dist/start-telemetry.d.ts +37 -0
- package/dist/stores/memory.d.ts +1 -1
- package/dist/stores/memory.mjs +63 -52
- package/package.json +26 -42
- package/dist/daytona-allow-list.d.ts +0 -16
- package/dist/daytona-allow-list.mjs +0 -11
- package/dist/drivers/box.d.ts +0 -52
- package/dist/drivers/box.mjs +0 -574
- package/dist/drivers/daytona.d.ts +0 -163
- package/dist/drivers/daytona.mjs +0 -683
- package/dist/drivers/exe.d.ts +0 -26
- package/dist/drivers/exe.mjs +0 -492
- package/dist/drivers/fetch-bridge.d.ts +0 -10
- package/dist/drivers/fetch-bridge.mjs +0 -126
- package/dist/lease-policy.d.ts +0 -52
- package/dist/lease-policy.mjs +0 -290
package/dist/index.mjs
CHANGED
|
@@ -1,21 +1,649 @@
|
|
|
1
1
|
// src/constants.ts
|
|
2
2
|
var DEFAULT_WORKSPACE_RUNTIME_PORT = 2593;
|
|
3
3
|
|
|
4
|
+
// src/contract.ts
|
|
5
|
+
function validateSandboxPersistenceCapabilities(input) {
|
|
6
|
+
if (input.resume === "replacement-restore" && input.capture === "none") {
|
|
7
|
+
return { valid: false, reason: "replacement restore requires a capture source" };
|
|
8
|
+
}
|
|
9
|
+
if (input.capture === "none" && input.captureSource !== "not-applicable") {
|
|
10
|
+
return { valid: false, reason: "capture source behavior requires capture support" };
|
|
11
|
+
}
|
|
12
|
+
if (input.capture !== "none" && input.captureSource === "not-applicable") {
|
|
13
|
+
return { valid: false, reason: "capture support requires source behavior" };
|
|
14
|
+
}
|
|
15
|
+
if (input.capture === "same-resource" && input.restoreMount !== "same-resource") {
|
|
16
|
+
return { valid: false, reason: "same-resource capture requires same-resource restore" };
|
|
17
|
+
}
|
|
18
|
+
return { valid: true };
|
|
19
|
+
}
|
|
20
|
+
function sandboxLeaseHeldHost(lease) {
|
|
21
|
+
return lease?.sandboxId !== void 0 && (lease.status === "stopped" || lease.status === "unavailable") ? lease : void 0;
|
|
22
|
+
}
|
|
23
|
+
function applySandboxLeasePatch(current, patch, updatedAt) {
|
|
24
|
+
return {
|
|
25
|
+
...current,
|
|
26
|
+
...patch.status === void 0 ? {} : { status: patch.status },
|
|
27
|
+
...patch.retryCount === void 0 ? {} : { retryCount: patch.retryCount },
|
|
28
|
+
...patch.lastHeartbeatAt === void 0 ? {} : { lastHeartbeatAt: patch.lastHeartbeatAt },
|
|
29
|
+
...patch.lastActivityAt === void 0 ? {} : { lastActivityAt: patch.lastActivityAt },
|
|
30
|
+
...patch.lastHealthFailureAt === void 0 ? {} : { lastHealthFailureAt: patch.lastHealthFailureAt },
|
|
31
|
+
...patch.nextRetryAt === void 0 ? {} : { nextRetryAt: patch.nextRetryAt ?? void 0 },
|
|
32
|
+
...patch.lastError === void 0 ? {} : { lastError: patch.lastError ?? void 0 },
|
|
33
|
+
...patch.checkpoint === void 0 ? {} : { checkpoint: patch.checkpoint ?? void 0 },
|
|
34
|
+
...patch.restore === void 0 ? {} : { restore: patch.restore ?? void 0 },
|
|
35
|
+
...patch.start === void 0 ? {} : { start: patch.start },
|
|
36
|
+
updatedAt
|
|
37
|
+
};
|
|
38
|
+
}
|
|
39
|
+
function applySandboxLeaseFailure(current, error, nextRetryAt, failedAt) {
|
|
40
|
+
return {
|
|
41
|
+
...current,
|
|
42
|
+
status: "unavailable",
|
|
43
|
+
retryCount: current.retryCount + 1,
|
|
44
|
+
lastError: error,
|
|
45
|
+
nextRetryAt,
|
|
46
|
+
lastHealthFailureAt: failedAt,
|
|
47
|
+
updatedAt: failedAt,
|
|
48
|
+
...current.start ? { start: { ...current.start, failures: [...current.start.failures ?? [], { epoch: current.epoch, failedAt, error }] } } : {}
|
|
49
|
+
};
|
|
50
|
+
}
|
|
51
|
+
function applySandboxProvisionedTarget(current, target, updatedAt) {
|
|
52
|
+
const sameTarget = current.status === "ready" && current.url === target.url && current.sandboxId === target.sandboxId && current.hostId === target.hostId && current.driverResourceId === target.driverResourceId;
|
|
53
|
+
return {
|
|
54
|
+
...current,
|
|
55
|
+
status: target.url === void 0 ? "acquiring" : "ready",
|
|
56
|
+
routingId: sameTarget && current.routingId ? current.routingId : crypto.randomUUID(),
|
|
57
|
+
sandboxId: target.sandboxId,
|
|
58
|
+
url: target.url,
|
|
59
|
+
hostId: target.hostId,
|
|
60
|
+
driverResourceId: target.driverResourceId,
|
|
61
|
+
labels: target.labels,
|
|
62
|
+
...target.persistence === void 0 ? {} : { persistence: target.persistence },
|
|
63
|
+
retryCount: 0,
|
|
64
|
+
nextRetryAt: void 0,
|
|
65
|
+
lastError: void 0,
|
|
66
|
+
updatedAt
|
|
67
|
+
};
|
|
68
|
+
}
|
|
69
|
+
var SANDBOX_IMAGE_LABEL = "image";
|
|
70
|
+
function isSandboxListingUnsupported(err) {
|
|
71
|
+
return !!err && typeof err === "object" && err.listingUnsupported === true;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
// src/egress-policy.ts
|
|
75
|
+
function sandboxNetworkPolicyWithBrokeredHosts(policy, secrets) {
|
|
76
|
+
if (policy.mode !== "restricted" || !secrets?.length) return policy;
|
|
77
|
+
const hosts = [.../* @__PURE__ */ new Set([
|
|
78
|
+
...policy.hosts ?? [],
|
|
79
|
+
...secrets.flatMap((secret) => secret.hosts)
|
|
80
|
+
])].toSorted();
|
|
81
|
+
return {
|
|
82
|
+
...policy,
|
|
83
|
+
...hosts.length ? { hosts } : {}
|
|
84
|
+
};
|
|
85
|
+
}
|
|
86
|
+
function sandboxEgressDisposition(control, net) {
|
|
87
|
+
if (!net || net.mode === "allow-all") return { action: "enforce" };
|
|
88
|
+
if (control === "none") return { action: "withhold", reason: "sandbox_egress_uncontained" };
|
|
89
|
+
const hosts = net.hosts?.length ?? 0;
|
|
90
|
+
const cidrs = net.cidrs?.length ?? 0;
|
|
91
|
+
if (hosts === 0 && cidrs === 0) return { action: "enforce" };
|
|
92
|
+
if (hosts > 0) return { action: "enforce" };
|
|
93
|
+
return control === "hosts-and-cidrs" ? { action: "enforce" } : { action: "refuse", reason: "sandbox_egress_policy_unenforceable" };
|
|
94
|
+
}
|
|
95
|
+
function brokeredSecretPlaceholder(name) {
|
|
96
|
+
return `claxedo-broker:${name}`;
|
|
97
|
+
}
|
|
98
|
+
function brokeredPlaceholderEnv(secrets) {
|
|
99
|
+
return Object.fromEntries((secrets ?? []).map((secret) => [secret.name, brokeredSecretPlaceholder(secret.name)]));
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
// src/driver-billing.ts
|
|
103
|
+
var HOUR_SECONDS = 3600;
|
|
104
|
+
var boatBilling = {
|
|
105
|
+
source: { url: "https://docs.boat.dev/pricing", readOn: "2026-10-12" },
|
|
106
|
+
meter: "Boat bills machine time per second while a sandbox runs, at an hourly rate per size; a stopped sandbox costs nothing.",
|
|
107
|
+
rates: {
|
|
108
|
+
small: { size: "small", vcpu: 2, memoryGb: 4, usdPerHour: 0.018 },
|
|
109
|
+
default: { size: "default", vcpu: 4, memoryGb: 8, usdPerHour: 0.036 },
|
|
110
|
+
large: { size: "large", vcpu: 8, memoryGb: 16, usdPerHour: 0.072 }
|
|
111
|
+
}
|
|
112
|
+
};
|
|
113
|
+
var CLOUDFLARE_USD_PER_GIB_SECOND_MEMORY = 25e-7;
|
|
114
|
+
var CLOUDFLARE_USD_PER_GB_SECOND_DISK = 7e-8;
|
|
115
|
+
var CLOUDFLARE_USD_PER_VCPU_SECOND_ACTIVE = 2e-5;
|
|
116
|
+
var CLOUDFLARE_STANDARD_1 = { size: "standard-1", vcpu: 0.5, memoryGb: 4, diskGb: 8 };
|
|
117
|
+
var cloudflareBilling = {
|
|
118
|
+
source: { url: "https://developers.cloudflare.com/containers/pricing/", readOn: "2026-10-12" },
|
|
119
|
+
meter: "Cloudflare bills a container for every 10 ms it runs: memory at $0.0000025 per GiB-second and disk at $0.00000007 per GB-second for the whole run, plus CPU at $0.00002 per vCPU-second of active use.",
|
|
120
|
+
rates: {
|
|
121
|
+
default: {
|
|
122
|
+
size: CLOUDFLARE_STANDARD_1.size,
|
|
123
|
+
vcpu: CLOUDFLARE_STANDARD_1.vcpu,
|
|
124
|
+
memoryGb: CLOUDFLARE_STANDARD_1.memoryGb,
|
|
125
|
+
usdPerHour: (CLOUDFLARE_STANDARD_1.memoryGb * CLOUDFLARE_USD_PER_GIB_SECOND_MEMORY + CLOUDFLARE_STANDARD_1.diskGb * CLOUDFLARE_USD_PER_GB_SECOND_DISK) * HOUR_SECONDS
|
|
126
|
+
}
|
|
127
|
+
},
|
|
128
|
+
excludes: `active CPU, billed at $${CLOUDFLARE_USD_PER_VCPU_SECOND_ACTIVE} per vCPU-second and not measured by Claxedo, and the monthly allowance a Workers Paid plan includes`
|
|
129
|
+
};
|
|
130
|
+
function sandboxMachineRate(billing, machineClass) {
|
|
131
|
+
if (!billing) return void 0;
|
|
132
|
+
return billing.rates[machineClass ?? "default"] ?? (Object.keys(billing.rates).length === 1 ? billing.rates.default : void 0);
|
|
133
|
+
}
|
|
134
|
+
function sandboxMachineCostUsd(rate, activeMs) {
|
|
135
|
+
return rate.usdPerHour * activeMs / (HOUR_SECONDS * 1e3);
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
// ../sandbox-contract/dist/index.mjs
|
|
139
|
+
function ipv4Number(part) {
|
|
140
|
+
let value;
|
|
141
|
+
if (/^0[xX][0-9a-fA-F]+$/.test(part))
|
|
142
|
+
value = Number.parseInt(part.slice(2), 16);
|
|
143
|
+
else if (/^0[0-7]+$/.test(part))
|
|
144
|
+
value = Number.parseInt(part.slice(1), 8);
|
|
145
|
+
else if (/^\d+$/.test(part))
|
|
146
|
+
value = Number.parseInt(part, 10);
|
|
147
|
+
else
|
|
148
|
+
return;
|
|
149
|
+
return Number.isSafeInteger(value) ? value : void 0;
|
|
150
|
+
}
|
|
151
|
+
function parseIpv4(input) {
|
|
152
|
+
const parts = input.split(".");
|
|
153
|
+
if (parts.length > 1 && parts[parts.length - 1] === "")
|
|
154
|
+
parts.pop();
|
|
155
|
+
if (parts.length < 1 || parts.length > 4)
|
|
156
|
+
return;
|
|
157
|
+
const numbers = [];
|
|
158
|
+
for (const part of parts) {
|
|
159
|
+
const value = ipv4Number(part);
|
|
160
|
+
if (value === void 0)
|
|
161
|
+
return;
|
|
162
|
+
numbers.push(value);
|
|
163
|
+
}
|
|
164
|
+
const last = numbers[numbers.length - 1];
|
|
165
|
+
if (last >= 256 ** (5 - numbers.length))
|
|
166
|
+
return;
|
|
167
|
+
for (const value of numbers.slice(0, -1))
|
|
168
|
+
if (value > 255)
|
|
169
|
+
return;
|
|
170
|
+
let address = last;
|
|
171
|
+
for (let i = 0; i < numbers.length - 1; i++)
|
|
172
|
+
address += numbers[i] * 256 ** (3 - i);
|
|
173
|
+
return address >>> 0;
|
|
174
|
+
}
|
|
175
|
+
function hextet(part) {
|
|
176
|
+
return /^[0-9a-fA-F]{1,4}$/.test(part) ? Number.parseInt(part, 16) : void 0;
|
|
177
|
+
}
|
|
178
|
+
function embeddedQuad(input) {
|
|
179
|
+
const parts = input.split(".");
|
|
180
|
+
if (parts.length !== 4)
|
|
181
|
+
return;
|
|
182
|
+
const octets = [];
|
|
183
|
+
for (const part of parts) {
|
|
184
|
+
if (!/^\d{1,3}$/.test(part))
|
|
185
|
+
return;
|
|
186
|
+
const value = Number.parseInt(part, 10);
|
|
187
|
+
if (value > 255)
|
|
188
|
+
return;
|
|
189
|
+
octets.push(value);
|
|
190
|
+
}
|
|
191
|
+
return octets;
|
|
192
|
+
}
|
|
193
|
+
function parseIpv6(input) {
|
|
194
|
+
let body = input;
|
|
195
|
+
let tail;
|
|
196
|
+
const lastColon = body.lastIndexOf(":");
|
|
197
|
+
if (lastColon >= 0 && body.includes(".")) {
|
|
198
|
+
tail = embeddedQuad(body.slice(lastColon + 1));
|
|
199
|
+
if (!tail)
|
|
200
|
+
return;
|
|
201
|
+
body = body.slice(0, lastColon + 1) + "0:0";
|
|
202
|
+
}
|
|
203
|
+
if ((body.match(/::/g) ?? []).length > 1)
|
|
204
|
+
return;
|
|
205
|
+
let halves;
|
|
206
|
+
if (body.includes("::")) {
|
|
207
|
+
const [left = "", right = ""] = body.split("::");
|
|
208
|
+
const leftParts = left === "" ? [] : left.split(":");
|
|
209
|
+
const rightParts = right === "" ? [] : right.split(":");
|
|
210
|
+
const missing = 8 - leftParts.length - rightParts.length;
|
|
211
|
+
if (missing < 1)
|
|
212
|
+
return;
|
|
213
|
+
halves = [...leftParts, ...Array.from({ length: missing }, () => "0"), ...rightParts];
|
|
214
|
+
} else {
|
|
215
|
+
halves = body.split(":");
|
|
216
|
+
if (halves.length !== 8)
|
|
217
|
+
return;
|
|
218
|
+
}
|
|
219
|
+
if (halves.length !== 8)
|
|
220
|
+
return;
|
|
221
|
+
const segments = [];
|
|
222
|
+
for (const part of halves) {
|
|
223
|
+
const value = hextet(part);
|
|
224
|
+
if (value === void 0)
|
|
225
|
+
return;
|
|
226
|
+
segments.push(value);
|
|
227
|
+
}
|
|
228
|
+
if (tail) {
|
|
229
|
+
segments[6] = tail[0] * 256 + tail[1];
|
|
230
|
+
segments[7] = tail[2] * 256 + tail[3];
|
|
231
|
+
}
|
|
232
|
+
return segments;
|
|
233
|
+
}
|
|
234
|
+
function parseIpAddress(input) {
|
|
235
|
+
if (!input)
|
|
236
|
+
return;
|
|
237
|
+
let host = input.trim().toLowerCase();
|
|
238
|
+
if (host.startsWith("[") && host.endsWith("]") && host.includes(":"))
|
|
239
|
+
host = host.slice(1, -1);
|
|
240
|
+
if (host.includes(":")) {
|
|
241
|
+
const segments = parseIpv6(host);
|
|
242
|
+
return segments ? { kind: "v6", segments } : void 0;
|
|
243
|
+
}
|
|
244
|
+
const value = parseIpv4(host);
|
|
245
|
+
return value === void 0 ? void 0 : { kind: "v4", value };
|
|
246
|
+
}
|
|
247
|
+
var V4_PRIVATE = [
|
|
248
|
+
[0, 8],
|
|
249
|
+
[167772160, 8],
|
|
250
|
+
[1681915904, 10],
|
|
251
|
+
[2130706432, 8],
|
|
252
|
+
[2851995648, 16],
|
|
253
|
+
[2886729728, 12],
|
|
254
|
+
[3221225472, 24],
|
|
255
|
+
[3221225984, 24],
|
|
256
|
+
[3223307264, 24],
|
|
257
|
+
[3224682752, 24],
|
|
258
|
+
[3227017984, 24],
|
|
259
|
+
[3232235520, 16],
|
|
260
|
+
[3232706560, 24],
|
|
261
|
+
[3323068416, 15],
|
|
262
|
+
[3325256704, 24],
|
|
263
|
+
[3405803776, 24],
|
|
264
|
+
[3758096384, 4],
|
|
265
|
+
[4026531840, 4]
|
|
266
|
+
];
|
|
267
|
+
function inCidr(value, base, bits) {
|
|
268
|
+
return (value ^ base) >>> 32 - bits === 0;
|
|
269
|
+
}
|
|
270
|
+
function isPrivateIpv4(value) {
|
|
271
|
+
return V4_PRIVATE.some(([base, bits]) => inCidr(value, base, bits));
|
|
272
|
+
}
|
|
273
|
+
function embeddedIpv4(segments) {
|
|
274
|
+
const [a, b, c, d, e, f, g, h] = segments;
|
|
275
|
+
if (a === 0 && b === 0 && c === 0 && d === 0 && e === 0) {
|
|
276
|
+
if (f === 65535 || f === 0)
|
|
277
|
+
return g * 65536 + h >>> 0;
|
|
278
|
+
}
|
|
279
|
+
if (a === 100 && b === 65435 && c === 0 && d === 0 && e === 0 && f === 0)
|
|
280
|
+
return g * 65536 + h >>> 0;
|
|
281
|
+
if (a === 8194)
|
|
282
|
+
return b * 65536 + c >>> 0;
|
|
283
|
+
if (a === 8193 && b === 0)
|
|
284
|
+
return (g ^ 65535) * 65536 + (h ^ 65535) >>> 0;
|
|
285
|
+
return;
|
|
286
|
+
}
|
|
287
|
+
function isUnspecified(segments) {
|
|
288
|
+
return segments.every((segment) => segment === 0);
|
|
289
|
+
}
|
|
290
|
+
function isIpv6Loopback(segments) {
|
|
291
|
+
return segments.slice(0, 7).every((segment) => segment === 0) && segments[7] === 1;
|
|
292
|
+
}
|
|
293
|
+
function isPrivateIpAddress(ip) {
|
|
294
|
+
if (ip.kind === "v4")
|
|
295
|
+
return isPrivateIpv4(ip.value);
|
|
296
|
+
const segments = ip.segments;
|
|
297
|
+
if (isUnspecified(segments) || isIpv6Loopback(segments))
|
|
298
|
+
return true;
|
|
299
|
+
const first = segments[0];
|
|
300
|
+
if ((first & 65280) === 65280)
|
|
301
|
+
return true;
|
|
302
|
+
if ((first & 65024) === 64512)
|
|
303
|
+
return true;
|
|
304
|
+
if ((first & 65472) === 65152)
|
|
305
|
+
return true;
|
|
306
|
+
if ((first & 65472) === 65216)
|
|
307
|
+
return true;
|
|
308
|
+
if (first === 256 && segments[1] === 0 && segments[2] === 0 && segments[3] === 0)
|
|
309
|
+
return true;
|
|
310
|
+
if (first === 8193 && segments[1] === 3512)
|
|
311
|
+
return true;
|
|
312
|
+
const embedded = embeddedIpv4(segments);
|
|
313
|
+
if (embedded !== void 0)
|
|
314
|
+
return isPrivateIpv4(embedded);
|
|
315
|
+
return false;
|
|
316
|
+
}
|
|
317
|
+
var sandboxCheckpointUnusableCauses = ["expired", "runtime_outdated", "runtime_modified", "not_restorable"];
|
|
318
|
+
function isSandboxCheckpointUnusableCause(value) {
|
|
319
|
+
return sandboxCheckpointUnusableCauses.some((cause) => cause === value);
|
|
320
|
+
}
|
|
321
|
+
var sandboxDriverIds = ["modal", "vercel", "cloudflare", "boat", "docker"];
|
|
322
|
+
var sandboxDriverLabels = {
|
|
323
|
+
modal: "Modal",
|
|
324
|
+
vercel: "Vercel",
|
|
325
|
+
cloudflare: "Cloudflare",
|
|
326
|
+
boat: "Boat",
|
|
327
|
+
docker: "Docker"
|
|
328
|
+
};
|
|
329
|
+
function isSandboxDriverID(input) {
|
|
330
|
+
return !!input && sandboxDriverIds.includes(input);
|
|
331
|
+
}
|
|
332
|
+
function safeRepoUrl(input) {
|
|
333
|
+
try {
|
|
334
|
+
const url = new URL(input);
|
|
335
|
+
return url.protocol === "https:" || url.protocol === "http:" || url.protocol === "ssh:" ? input : void 0;
|
|
336
|
+
} catch {
|
|
337
|
+
return /^[\w.-]+@[\w.-]+:[\w./-]+$/.test(input) ? input : void 0;
|
|
338
|
+
}
|
|
339
|
+
}
|
|
340
|
+
function repoUrlHost(input) {
|
|
341
|
+
if (!safeRepoUrl(input))
|
|
342
|
+
return;
|
|
343
|
+
try {
|
|
344
|
+
const host = new URL(input).hostname;
|
|
345
|
+
return host || void 0;
|
|
346
|
+
} catch {
|
|
347
|
+
return /^[\w.-]+@([\w.-]+):[\w./-]+$/.exec(input)?.[1]?.toLowerCase();
|
|
348
|
+
}
|
|
349
|
+
}
|
|
350
|
+
function bareRepoHost(host) {
|
|
351
|
+
const bare = host.trim().toLowerCase();
|
|
352
|
+
return bare.startsWith("[") && bare.endsWith("]") && bare.includes(":") ? bare.slice(1, -1) : bare;
|
|
353
|
+
}
|
|
354
|
+
function loopbackRepoName(bare) {
|
|
355
|
+
return bare === "localhost" || bare.endsWith(".localhost");
|
|
356
|
+
}
|
|
357
|
+
function publicRepoHost(repoUrl) {
|
|
358
|
+
const host = repoUrlHost(repoUrl);
|
|
359
|
+
if (!host)
|
|
360
|
+
return;
|
|
361
|
+
const bare = bareRepoHost(host);
|
|
362
|
+
if (loopbackRepoName(bare))
|
|
363
|
+
return;
|
|
364
|
+
const ip = parseIpAddress(bare);
|
|
365
|
+
if (ip && isPrivateIpAddress(ip))
|
|
366
|
+
return;
|
|
367
|
+
return host;
|
|
368
|
+
}
|
|
369
|
+
|
|
370
|
+
// src/lease-failure.ts
|
|
371
|
+
function isSandboxRuntimeBootFailure(err) {
|
|
372
|
+
return !!err && typeof err === "object" && err.runtimeBootFailed === true;
|
|
373
|
+
}
|
|
374
|
+
var SandboxRuntimeBootError = class extends Error {
|
|
375
|
+
runtimeBootFailed = true;
|
|
376
|
+
constructor(reason) {
|
|
377
|
+
super(reason);
|
|
378
|
+
this.name = "SandboxRuntimeBootError";
|
|
379
|
+
}
|
|
380
|
+
};
|
|
381
|
+
function isSandboxCheckpointUnusable(err) {
|
|
382
|
+
return !!err && typeof err === "object" && isSandboxCheckpointUnusableCause(err.checkpointUnusable);
|
|
383
|
+
}
|
|
384
|
+
var SandboxCheckpointUnusableError = class extends Error {
|
|
385
|
+
constructor(checkpointUnusable, reason) {
|
|
386
|
+
super(reason);
|
|
387
|
+
this.checkpointUnusable = checkpointUnusable;
|
|
388
|
+
this.name = "SandboxCheckpointUnusableError";
|
|
389
|
+
}
|
|
390
|
+
};
|
|
391
|
+
function isSandboxHostGone(err) {
|
|
392
|
+
return !!err && typeof err === "object" && err.hostGone === true;
|
|
393
|
+
}
|
|
394
|
+
var SandboxHostGoneError = class extends Error {
|
|
395
|
+
hostGone = true;
|
|
396
|
+
constructor(reason) {
|
|
397
|
+
super(reason);
|
|
398
|
+
this.name = "SandboxHostGoneError";
|
|
399
|
+
}
|
|
400
|
+
};
|
|
401
|
+
function isSandboxProviderKeyRefused(err) {
|
|
402
|
+
return !!err && typeof err === "object" && err.providerKeyRefused === true;
|
|
403
|
+
}
|
|
404
|
+
var SandboxProviderKeyRefusedError = class extends Error {
|
|
405
|
+
providerKeyRefused = true;
|
|
406
|
+
constructor(reason) {
|
|
407
|
+
super(reason);
|
|
408
|
+
this.name = "SandboxProviderKeyRefusedError";
|
|
409
|
+
}
|
|
410
|
+
};
|
|
411
|
+
function isSandboxStartRefusal(err) {
|
|
412
|
+
return !!err && typeof err === "object" && err.startRefused === true;
|
|
413
|
+
}
|
|
414
|
+
var SandboxStartRefusedError = class extends Error {
|
|
415
|
+
startRefused = true;
|
|
416
|
+
constructor(reason) {
|
|
417
|
+
super(reason);
|
|
418
|
+
this.name = "SandboxStartRefusedError";
|
|
419
|
+
}
|
|
420
|
+
};
|
|
421
|
+
function sandboxDriverLabel(driver) {
|
|
422
|
+
return isSandboxDriverID(driver) ? sandboxDriverLabels[driver] : driver;
|
|
423
|
+
}
|
|
424
|
+
var PROVIDER_KEY_REFUSED = "provider_key_refused:";
|
|
425
|
+
var PROVIDER_FAILED = "provider_failed:";
|
|
426
|
+
function sandboxProviderKeyRefusedError(driver, message) {
|
|
427
|
+
return `${PROVIDER_KEY_REFUSED}${driver}: ${message}`;
|
|
428
|
+
}
|
|
429
|
+
function sandboxProviderFailedError(driver, message) {
|
|
430
|
+
return `${PROVIDER_FAILED}${driver}: ${message}`;
|
|
431
|
+
}
|
|
432
|
+
function codedDriver(error, prefix) {
|
|
433
|
+
if (!error.startsWith(prefix)) return void 0;
|
|
434
|
+
const rest = error.slice(prefix.length);
|
|
435
|
+
const end = rest.indexOf(": ");
|
|
436
|
+
return end < 0 ? rest : rest.slice(0, end);
|
|
437
|
+
}
|
|
438
|
+
var SANDBOX_START_WINDOWS = ["minute", "hour", "day"];
|
|
439
|
+
function isSandboxProviderRateLimited(err) {
|
|
440
|
+
const limit = !!err && typeof err === "object" ? err.providerRateLimited : void 0;
|
|
441
|
+
return !!limit && typeof limit === "object" && SANDBOX_START_WINDOWS.some((window) => window === limit.window);
|
|
442
|
+
}
|
|
443
|
+
var SandboxProviderRateLimitedError = class extends Error {
|
|
444
|
+
constructor(providerRateLimited, reason) {
|
|
445
|
+
super(reason);
|
|
446
|
+
this.providerRateLimited = providerRateLimited;
|
|
447
|
+
this.name = "SandboxProviderRateLimitedError";
|
|
448
|
+
}
|
|
449
|
+
};
|
|
450
|
+
function sandboxStartLimitRetryMs(window) {
|
|
451
|
+
return window === "minute" ? 6e4 : window === "hour" ? 36e5 : void 0;
|
|
452
|
+
}
|
|
453
|
+
var PROVIDER_RATE_LIMITED = "provider_rate_limited:";
|
|
454
|
+
var START_LIMIT_NAMES = { minute: "per-minute", hour: "hourly", day: "daily" };
|
|
455
|
+
function sandboxProviderRateLimitedError(driver, limit) {
|
|
456
|
+
const used = `${sandboxDriverLabel(driver)}'s ${START_LIMIT_NAMES[limit.window]} start limit (${limit.limit}) is used up`;
|
|
457
|
+
return `${PROVIDER_RATE_LIMITED}${limit.window}: ${limit.window === "day" ? `${used}; try again later.` : `${used}.`}`;
|
|
458
|
+
}
|
|
459
|
+
function sandboxProviderRateLimit(error) {
|
|
460
|
+
if (!error?.startsWith(PROVIDER_RATE_LIMITED)) return void 0;
|
|
461
|
+
const window = error.slice(PROVIDER_RATE_LIMITED.length).split(":", 1)[0];
|
|
462
|
+
return SANDBOX_START_WINDOWS.find((name) => name === window);
|
|
463
|
+
}
|
|
464
|
+
function providerRateLimitedReason(error) {
|
|
465
|
+
return sandboxProviderRateLimit(error) === void 0 ? void 0 : error.slice(error.indexOf(": ") + 2);
|
|
466
|
+
}
|
|
467
|
+
var RUNTIME_BOOT_FAILED = "runtime_boot_failed: ";
|
|
468
|
+
var CHECKPOINT_UNUSABLE = "checkpoint_unusable:";
|
|
469
|
+
var RUNTIME_HOST_GONE = "runtime_host_gone: ";
|
|
470
|
+
function sandboxRuntimeBootFailedError(reason) {
|
|
471
|
+
return `${RUNTIME_BOOT_FAILED}${reason}`;
|
|
472
|
+
}
|
|
473
|
+
function sandboxCheckpointUnusableError(cause, reason) {
|
|
474
|
+
return `${CHECKPOINT_UNUSABLE}${cause}: ${reason}`;
|
|
475
|
+
}
|
|
476
|
+
function sandboxCheckpointUnusable(error) {
|
|
477
|
+
if (!error?.startsWith(CHECKPOINT_UNUSABLE)) return void 0;
|
|
478
|
+
const rest = error.slice(CHECKPOINT_UNUSABLE.length);
|
|
479
|
+
const end = rest.indexOf(": ");
|
|
480
|
+
const cause = end < 0 ? void 0 : rest.slice(0, end);
|
|
481
|
+
return isSandboxCheckpointUnusableCause(cause) ? { cause, message: rest.slice(end + 2) } : void 0;
|
|
482
|
+
}
|
|
483
|
+
function sandboxHostGoneError(reason) {
|
|
484
|
+
return `${RUNTIME_HOST_GONE}${reason}`;
|
|
485
|
+
}
|
|
486
|
+
var RUNTIME_UNRESPONSIVE = "runtime_unresponsive:";
|
|
487
|
+
function sandboxRuntimeUnresponsiveError(forMs) {
|
|
488
|
+
return `${RUNTIME_UNRESPONSIVE}${Math.max(1, Math.round(forMs / 6e4))}`;
|
|
489
|
+
}
|
|
490
|
+
function sandboxStopReason(error) {
|
|
491
|
+
if (!error?.startsWith(RUNTIME_UNRESPONSIVE)) return void 0;
|
|
492
|
+
const minutes = Number(error.slice(RUNTIME_UNRESPONSIVE.length));
|
|
493
|
+
return `The workspace's runtime stopped answering; it was stopped after ${minutes} ${minutes === 1 ? "minute" : "minutes"}.`;
|
|
494
|
+
}
|
|
495
|
+
var WORKSPACE_RUNTIME_BOOT_FAILED_LINE = "workspace_runtime_boot_failed: ";
|
|
496
|
+
function workspaceRuntimeBootReason(diagnostic) {
|
|
497
|
+
const at = diagnostic.lastIndexOf(WORKSPACE_RUNTIME_BOOT_FAILED_LINE);
|
|
498
|
+
if (at < 0) return void 0;
|
|
499
|
+
const [reason = ""] = diagnostic.slice(at + WORKSPACE_RUNTIME_BOOT_FAILED_LINE.length).split(/\n\s+at /);
|
|
500
|
+
return reason.trim().replace(/^Error: /, "") || void 0;
|
|
501
|
+
}
|
|
502
|
+
function runtimeBootDiagnostic(log, maxChars) {
|
|
503
|
+
const trimmed = log.trim();
|
|
504
|
+
const reason = workspaceRuntimeBootReason(trimmed);
|
|
505
|
+
if (reason === void 0) return trimmed.slice(-maxChars);
|
|
506
|
+
return `${WORKSPACE_RUNTIME_BOOT_FAILED_LINE}${reason}`.slice(0, maxChars);
|
|
507
|
+
}
|
|
508
|
+
function sandboxRuntimeBootFailure(error) {
|
|
509
|
+
const unusable = sandboxCheckpointUnusable(error);
|
|
510
|
+
if (unusable) return unusable.message;
|
|
511
|
+
if (!error?.startsWith(RUNTIME_BOOT_FAILED)) return void 0;
|
|
512
|
+
const diagnostic = error.slice(RUNTIME_BOOT_FAILED.length);
|
|
513
|
+
return workspaceRuntimeBootReason(diagnostic) ?? diagnostic;
|
|
514
|
+
}
|
|
515
|
+
var SANDBOX_RUNTIME_UNHEALTHY = "runtime_unhealthy";
|
|
516
|
+
var SANDBOX_RETRY_CAP_EXCEEDED = "runtime_retry_cap_exceeded";
|
|
517
|
+
var CODED_REASONS = /* @__PURE__ */ new Map([
|
|
518
|
+
[SANDBOX_RUNTIME_UNHEALTHY, "The workspace machine stopped answering."],
|
|
519
|
+
[SANDBOX_RETRY_CAP_EXCEEDED, "Several tries in a row failed; try again later."]
|
|
520
|
+
]);
|
|
521
|
+
function firstLine(text) {
|
|
522
|
+
return text.split("\n", 1)[0]?.trim() ?? text;
|
|
523
|
+
}
|
|
524
|
+
function sandboxStartFailureReason(error) {
|
|
525
|
+
const boot = sandboxRuntimeBootFailure(error);
|
|
526
|
+
if (boot !== void 0) return firstLine(boot);
|
|
527
|
+
const limited = providerRateLimitedReason(error);
|
|
528
|
+
if (limited !== void 0) return limited;
|
|
529
|
+
const keyRefused = codedDriver(error, PROVIDER_KEY_REFUSED);
|
|
530
|
+
if (keyRefused !== void 0) return `${sandboxDriverLabel(keyRefused)} refused the sandbox provider key; fix it in Settings \u2192 Sandbox providers.`;
|
|
531
|
+
const failed = codedDriver(error, PROVIDER_FAILED);
|
|
532
|
+
if (failed !== void 0) return `${sandboxDriverLabel(failed)} returned an error.`;
|
|
533
|
+
return CODED_REASONS.get(error) ?? firstLine(error);
|
|
534
|
+
}
|
|
535
|
+
var LEASE_ENDED_ERRORS = /* @__PURE__ */ new Set(["runtime_lease_changed", "runtime_lease_stopped", "runtime_lease_destroyed"]);
|
|
536
|
+
function sandboxLeaseEndedUnderStart(error) {
|
|
537
|
+
return error !== void 0 && LEASE_ENDED_ERRORS.has(error);
|
|
538
|
+
}
|
|
539
|
+
function sandboxLeaseFailure(lease, input) {
|
|
540
|
+
if (lease.status !== "unavailable" || !lease.lastError) return void 0;
|
|
541
|
+
const unusable = sandboxCheckpointUnusable(lease.lastError);
|
|
542
|
+
if (unusable) return { kind: "checkpoint_unusable", cause: unusable.cause, message: firstLine(unusable.message), retrying: false };
|
|
543
|
+
const kind = sandboxRuntimeBootFailure(lease.lastError) !== void 0 ? "boot" : lease.lastError === SANDBOX_RUNTIME_UNHEALTHY ? "unhealthy" : "provider";
|
|
544
|
+
const retrying = kind !== "boot" && lease.retryCount < input.maxRetryCount && lease.nextRetryAt !== void 0 && lease.nextRetryAt > input.now;
|
|
545
|
+
return { kind, message: sandboxStartFailureReason(lease.lastError), retrying };
|
|
546
|
+
}
|
|
547
|
+
|
|
548
|
+
// src/lease-retry-wait.ts
|
|
549
|
+
async function liftSandboxRetryWait(leaseStore, workspaceId) {
|
|
550
|
+
const lease = await leaseStore.get(workspaceId);
|
|
551
|
+
if (lease?.status !== "unavailable" || lease.retryCount === 0 && lease.nextRetryAt === void 0) return false;
|
|
552
|
+
return !!await leaseStore.update(workspaceId, lease.epoch, { retryCount: 0, nextRetryAt: null }, "unavailable");
|
|
553
|
+
}
|
|
554
|
+
|
|
555
|
+
// ../claxedo-helpers/dist/api-error.mjs
|
|
556
|
+
var PUBLIC_API_ERRORS = {
|
|
557
|
+
repository_provider_unauthorized: { status: 502, message: "Repository provider refused authorization" },
|
|
558
|
+
repository_provider_unavailable: { status: 502, message: "Repository provider is unavailable" },
|
|
559
|
+
host_tunnel_timeout: { status: 503, retryable: true, message: "Host tunnel timed out" },
|
|
560
|
+
host_tunnel_response_body_too_large: { status: 413, message: "Host tunnel response body exceeds the relay limit" },
|
|
561
|
+
host_tunnel_unavailable: { status: 503, retryable: true, message: "Host tunnel is unavailable" },
|
|
562
|
+
workspace_checkpoint_unavailable: { status: 503, retryable: true, message: "Workspace checkpoint is unavailable" },
|
|
563
|
+
workspace_checkpoint_conflict: { status: 409, message: "Workspace checkpoint conflicted" },
|
|
564
|
+
invalid_input: { family: "access", status: 400, message: "The request names an invalid value" },
|
|
565
|
+
organization_policy_denied: { family: "access", status: 403, message: "Organization creation is disabled for this deployment" },
|
|
566
|
+
org_admin_required: { family: "access", status: 403, message: "Organization administrator authority is required" },
|
|
567
|
+
org_owner_required: { family: "access", status: 403, message: "Only an organization owner may grant, change or remove the owner role" },
|
|
568
|
+
org_owner_protected: { family: "access", status: 409, message: "The organization's founding owner cannot be removed or demoted" },
|
|
569
|
+
team_member_org_membership_required: { family: "access", status: 403, message: "The team member must belong to the team organization" },
|
|
570
|
+
project_member_org_membership_required: { family: "access", status: 403, message: "The project member must belong to the project organization" },
|
|
571
|
+
org_membership_required: { family: "access", status: 403, message: "Organization membership is required" },
|
|
572
|
+
project_admin_required: { family: "access", status: 403, message: "Project administrator authority is required" },
|
|
573
|
+
team_not_allowed_on_personal_org: { family: "access", status: 400, message: "Personal organizations cannot contain teams" },
|
|
574
|
+
team_member_target_required: { family: "access", status: 400, message: "Exactly one team member target is required" },
|
|
575
|
+
org_invitation_invalid: { family: "access", status: 409, message: "Invitation is invalid or unavailable" },
|
|
576
|
+
org_invitation_pending: { family: "access", status: 409, message: "An invitation to this address is already pending" },
|
|
577
|
+
org_invitation_rate_limited: { family: "access", status: 429, message: "Organization invitation limit exceeded" },
|
|
578
|
+
org_invitation_email_mismatch: { family: "access", status: 403, message: "Sign in with the invited verified email address" },
|
|
579
|
+
org_invitation_delivery_unavailable: { family: "access", status: 503, message: "Invitation delivery is unavailable" },
|
|
580
|
+
team_not_found: { family: "access", status: 404, message: "Team not found" },
|
|
581
|
+
team_member_not_found: { family: "access", status: 404, message: "Team member not found" },
|
|
582
|
+
org_member_not_found: { family: "access", status: 404, message: "Organization member not found" },
|
|
583
|
+
project_not_found: { family: "access", status: 404, message: "Project not found" },
|
|
584
|
+
project_member_not_found: { family: "access", status: 404, message: "Project member not found" },
|
|
585
|
+
project_member_owner_immutable: { family: "access", status: 409, message: "The project owner's access cannot be changed" },
|
|
586
|
+
resource_conflict: { family: "access", status: 409, message: "Organization or team authority changed concurrently" },
|
|
587
|
+
session_not_found: { family: "session_share", status: 404, message: "This session is not on the control plane, so it cannot be shared from People yet." },
|
|
588
|
+
session_share_admin_required: { family: "session_share", status: 403, message: "Only the person who started this session can manage its People." },
|
|
589
|
+
session_share_target_required: { family: "session_share", status: 400, message: "A session is shared with exactly one person" },
|
|
590
|
+
session_share_target_not_found: { family: "session_share", status: 404, message: "That person was not found in this organization." },
|
|
591
|
+
session_share_level_invalid: { family: "session_share", status: 400, message: "A share level is either follow or send." },
|
|
592
|
+
workspace_backing_conflict: { status: 409, message: "Workspace is provisioned in the cloud and cannot be placed on a machine" },
|
|
593
|
+
invalid_session_list_cursor: { status: 400, message: "Session list cursor does not match this query" },
|
|
594
|
+
BAD_REQUEST: { status: 400, message: "Invalid request" },
|
|
595
|
+
INTERNAL_SERVER_ERROR: { status: 500, message: "Internal error" },
|
|
596
|
+
CONTROL_PLANE_UNAVAILABLE: { status: 503, message: "Control Plane services are not configured" },
|
|
597
|
+
credential_verification_unsupported: { status: 400, message: "Credential provider does not support verification" },
|
|
598
|
+
credential_shape_invalid: { status: 400, message: "Credential secret has an unsupported shape" },
|
|
599
|
+
credential_provider_unavailable: { status: 502, retryable: true, message: "Credential provider request failed" },
|
|
600
|
+
credential_verification_failed: { status: 502, message: "Credential provider verification failed" },
|
|
601
|
+
credential_redirect_denied: { status: 502, message: "Sandbox provider redirects are not allowed" },
|
|
602
|
+
credential_endpoint_invalid: { status: 400, message: "Invalid credential provider endpoint" },
|
|
603
|
+
host_attestation_denied: { status: 403, message: "host_attestation_denied" },
|
|
604
|
+
signature_replayed: { status: 409, message: "signature_replayed" },
|
|
605
|
+
host_enrollment_not_found: { status: 404, message: "host_enrollment_not_found" },
|
|
606
|
+
enrollment_revoked: { status: 403, message: "enrollment_revoked" },
|
|
607
|
+
enrollment_paused: { status: 403, message: "enrollment_paused" },
|
|
608
|
+
enrollment_owner_ineligible: { status: 403, message: "enrollment_owner_ineligible" },
|
|
609
|
+
enrollment_key_version_mismatch: { status: 403, message: "enrollment_key_version_mismatch" },
|
|
610
|
+
invitation_invalid: { status: 403, message: "invitation_invalid" },
|
|
611
|
+
invitation_expired: { status: 410, message: "invitation_expired" },
|
|
612
|
+
invitation_revoked: { status: 410, message: "invitation_revoked" },
|
|
613
|
+
invitation_redeemed: { status: 409, message: "invitation_redeemed" },
|
|
614
|
+
invitation_host_conflict: { status: 409, message: "invitation_host_conflict" },
|
|
615
|
+
enrollment_generation_superseded: { status: 409, message: "enrollment_generation_superseded" },
|
|
616
|
+
host_assignment_outside_scope: { status: 400, message: "host_assignment_outside_scope" },
|
|
617
|
+
host_sealing_key_undeclared: { status: 409, message: "host_sealing_key_undeclared" },
|
|
618
|
+
host_provider_config_revision_stale: { status: 409, message: "host_provider_config_revision_stale" },
|
|
619
|
+
identity_conflict: { status: 409, message: "identity_conflict" },
|
|
620
|
+
registration_transition_denied: { status: 403, message: "registration_transition_denied" },
|
|
621
|
+
session_reservation_spent: { status: 409, message: "This session's create was undone; reserve it again to retry" },
|
|
622
|
+
actor_authorization_denied: { status: 403, message: "actor_authorization_denied" }
|
|
623
|
+
};
|
|
624
|
+
function publicApiErrorShape(code) {
|
|
625
|
+
const entry = PUBLIC_API_ERRORS[code];
|
|
626
|
+
return { status: entry.status, retryable: entry.retryable === true };
|
|
627
|
+
}
|
|
628
|
+
function publicApiFailure(code, message = PUBLIC_API_ERRORS[code].message) {
|
|
629
|
+
return Object.assign(new Error(message), { code, ...publicApiErrorShape(code) });
|
|
630
|
+
}
|
|
631
|
+
|
|
4
632
|
// src/checkpoint-manager.ts
|
|
5
633
|
async function captureSandboxCheckpoint(input) {
|
|
6
634
|
const lease = await requireLease(input.leaseStore, input.workspaceId);
|
|
7
|
-
if (lease.status !== "ready") throw
|
|
635
|
+
if (lease.status !== "ready") throw publicApiFailure("workspace_checkpoint_conflict", "workspace_checkpoint_lease_not_ready");
|
|
8
636
|
const persistence = lease.persistence;
|
|
9
|
-
if (!persistence || persistence.capture === "none") throw
|
|
637
|
+
if (!persistence || persistence.capture === "none") throw publicApiFailure("workspace_checkpoint_conflict", "workspace_checkpoint_unsupported");
|
|
10
638
|
const target = await input.target(input.workspaceId);
|
|
11
|
-
if (target.status !== "ready") throw
|
|
12
|
-
|
|
639
|
+
if (target.status !== "ready") throw publicApiFailure("workspace_checkpoint_unavailable", target.reason);
|
|
640
|
+
const discard = persistence.capture === "same-resource" ? void 0 : input.deleteSnapshot;
|
|
13
641
|
let capturedSource = false;
|
|
642
|
+
let result;
|
|
14
643
|
try {
|
|
15
|
-
await input.request.
|
|
16
|
-
await input.
|
|
17
|
-
|
|
18
|
-
if (!captured.ok) throw new Error(captured.reason);
|
|
644
|
+
const frozen = await freezeForCapture(input.request, input.endsLease !== void 0, input.workspaceId);
|
|
645
|
+
const captured = persistence.capture === "same-resource" ? { ok: true, snapshotId: target.driverResourceId ?? target.sandboxId } : await input.snapshot(input.workspaceId, (await input.leaseStore.get(input.workspaceId))?.checkpoint?.providerReference);
|
|
646
|
+
if (!captured.ok) throw publicApiFailure("workspace_checkpoint_conflict", captured.reason);
|
|
19
647
|
capturedSource = true;
|
|
20
648
|
const capturedAt = (input.now ?? Date.now)();
|
|
21
649
|
const checkpoint = {
|
|
@@ -26,28 +654,59 @@ async function captureSandboxCheckpoint(input) {
|
|
|
26
654
|
metadata: {
|
|
27
655
|
scope: persistence.capture,
|
|
28
656
|
sourceBehavior: requireCaptureSource(persistence.captureSource),
|
|
29
|
-
|
|
30
|
-
|
|
657
|
+
restoreMount: requireRestoreMount(persistence.restoreMount),
|
|
658
|
+
...frozen ? {} : { unfrozen: true }
|
|
31
659
|
}
|
|
32
660
|
};
|
|
661
|
+
const replaced = (await input.leaseStore.get(input.workspaceId))?.checkpoint;
|
|
33
662
|
const updated = await input.leaseStore.update(input.workspaceId, lease.epoch, {
|
|
34
663
|
checkpoint,
|
|
35
664
|
restore: null,
|
|
36
|
-
...
|
|
665
|
+
...input.endsLease?.stoppedError ? { lastError: input.endsLease.stoppedError } : {},
|
|
666
|
+
...input.endsLease || persistence.captureSource === "stopped" || persistence.captureSource === "deleted" ? { status: "stopped" } : {}
|
|
37
667
|
});
|
|
38
|
-
if (!updated)
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
668
|
+
if (!updated) {
|
|
669
|
+
await discardSnapshot(discard, target, captured.snapshotId);
|
|
670
|
+
throw publicApiFailure("workspace_checkpoint_conflict", "workspace_checkpoint_epoch_fenced");
|
|
671
|
+
}
|
|
672
|
+
if (replaced && replaced.providerReference !== captured.snapshotId) {
|
|
673
|
+
await discardSnapshot(discard, target, replaced.providerReference);
|
|
43
674
|
}
|
|
675
|
+
result = { status: "ready", lease: updated, checkpoint };
|
|
676
|
+
} catch (error) {
|
|
677
|
+
if (!capturedSource || persistence.captureSource === "preserved") await thawAfterFailure(input.request.runtime, input.workspaceId);
|
|
678
|
+
throw error;
|
|
679
|
+
}
|
|
680
|
+
if (!input.endsLease && persistence.captureSource === "preserved") await input.request.runtime.resume();
|
|
681
|
+
return result;
|
|
682
|
+
}
|
|
683
|
+
async function freezeForCapture(request, endsLease, workspaceId) {
|
|
684
|
+
try {
|
|
685
|
+
await request.runtime.freeze(request.policy ?? "drain", { idleBefore: request.idleBefore });
|
|
686
|
+
await request.runtime.flush();
|
|
687
|
+
await request.runtime.scrub();
|
|
688
|
+
return true;
|
|
689
|
+
} catch (error) {
|
|
690
|
+
if (!endsLease || request.idleBefore !== void 0) throw error;
|
|
691
|
+
console.warn("A stopping sandbox's runtime did not freeze; its files are captured as they are", { workspaceId, error: String(error) });
|
|
692
|
+
return false;
|
|
44
693
|
}
|
|
45
694
|
}
|
|
695
|
+
async function thawAfterFailure(runtime, workspaceId) {
|
|
696
|
+
await runtime.resume().catch((error) => {
|
|
697
|
+
console.warn("A runtime was not thawed after its capture failed", { workspaceId, error: String(error) });
|
|
698
|
+
});
|
|
699
|
+
}
|
|
700
|
+
async function discardSnapshot(discard, target, snapshotId) {
|
|
701
|
+
await discard?.(target, snapshotId).catch((error) => {
|
|
702
|
+
console.warn("A superseded sandbox snapshot could not be deleted", { workspaceId: target.workspaceId, snapshotId, error: String(error) });
|
|
703
|
+
});
|
|
704
|
+
}
|
|
46
705
|
async function restoreSandboxCheckpoint(input) {
|
|
47
706
|
const lease = await requireLease(input.leaseStore, input.workspaceId);
|
|
48
707
|
const checkpoint = lease.checkpoint;
|
|
49
708
|
if (!checkpoint || input.request.checkpointId && checkpoint.id !== input.request.checkpointId) {
|
|
50
|
-
throw
|
|
709
|
+
throw publicApiFailure("workspace_checkpoint_conflict", "workspace_checkpoint_not_found");
|
|
51
710
|
}
|
|
52
711
|
if (lease.restore?.state === "ready" && lease.restore.checkpointId === checkpoint.id) {
|
|
53
712
|
return { status: "ready", lease, checkpoint };
|
|
@@ -61,7 +720,7 @@ async function restoreSandboxCheckpoint(input) {
|
|
|
61
720
|
startedAt: (input.now ?? Date.now)()
|
|
62
721
|
};
|
|
63
722
|
const staged = lease.status === "ready" && lease.epoch === checkpoint.sourceEpoch ? await input.leaseStore.update(input.workspaceId, lease.epoch, { status: "stopped", restore: restoring }) : await input.leaseStore.update(input.workspaceId, lease.epoch, { restore: restoring });
|
|
64
|
-
if (!staged) throw
|
|
723
|
+
if (!staged) throw publicApiFailure("workspace_checkpoint_conflict", "workspace_restore_epoch_fenced");
|
|
65
724
|
const result = await input.ensure(input.workspaceId, {
|
|
66
725
|
homeRegion: staged.homeRegion,
|
|
67
726
|
...input.request.ensure,
|
|
@@ -84,7 +743,7 @@ async function restoreSandboxCheckpoint(input) {
|
|
|
84
743
|
error: result.error ?? "workspace_restore_unavailable"
|
|
85
744
|
}
|
|
86
745
|
});
|
|
87
|
-
throw
|
|
746
|
+
throw publicApiFailure("workspace_checkpoint_unavailable", result.error ?? "workspace_restore_unavailable");
|
|
88
747
|
}
|
|
89
748
|
await input.request.runtime.reconcile({ epoch: result.epoch, checkpointId: checkpoint.id });
|
|
90
749
|
const ready = await input.leaseStore.update(input.workspaceId, result.epoch, {
|
|
@@ -97,21 +756,21 @@ async function restoreSandboxCheckpoint(input) {
|
|
|
97
756
|
completedAt: (input.now ?? Date.now)()
|
|
98
757
|
}
|
|
99
758
|
});
|
|
100
|
-
if (!ready) throw
|
|
759
|
+
if (!ready) throw publicApiFailure("workspace_checkpoint_conflict", "workspace_restore_epoch_fenced");
|
|
101
760
|
return { status: "ready", lease: ready, checkpoint };
|
|
102
761
|
}
|
|
103
762
|
async function requireLease(store, workspaceId) {
|
|
104
763
|
const lease = await store.get(workspaceId);
|
|
105
|
-
if (!lease) throw
|
|
764
|
+
if (!lease) throw publicApiFailure("workspace_checkpoint_conflict", "workspace_checkpoint_lease_missing");
|
|
106
765
|
return lease;
|
|
107
766
|
}
|
|
108
767
|
function requireCaptureSource(value) {
|
|
109
768
|
if (value === "preserved" || value === "stopped" || value === "deleted") return value;
|
|
110
|
-
throw
|
|
769
|
+
throw publicApiFailure("workspace_checkpoint_conflict", "workspace_checkpoint_capabilities_invalid");
|
|
111
770
|
}
|
|
112
771
|
function requireRestoreMount(value) {
|
|
113
|
-
if (value === "same-resource" || value === "
|
|
114
|
-
throw
|
|
772
|
+
if (value === "same-resource" || value === "new-resource") return value;
|
|
773
|
+
throw publicApiFailure("workspace_checkpoint_conflict", "workspace_checkpoint_capabilities_invalid");
|
|
115
774
|
}
|
|
116
775
|
|
|
117
776
|
// src/hosted-network-policy.ts
|
|
@@ -151,7 +810,7 @@ function hostOf(value) {
|
|
|
151
810
|
}
|
|
152
811
|
function sandboxSourceHost(source) {
|
|
153
812
|
if (!source || source.kind !== "git") return void 0;
|
|
154
|
-
return
|
|
813
|
+
return publicRepoHost(source.repoUrl);
|
|
155
814
|
}
|
|
156
815
|
function hostedSandboxNetworkPolicy(input) {
|
|
157
816
|
const hosts = /* @__PURE__ */ new Set();
|
|
@@ -169,87 +828,319 @@ function hostedSandboxNetworkPolicy(input) {
|
|
|
169
828
|
return { mode: "restricted", hosts: [...hosts] };
|
|
170
829
|
}
|
|
171
830
|
|
|
172
|
-
// src/
|
|
173
|
-
function
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
if (
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
return { valid: false, reason: "capture support requires source behavior" };
|
|
182
|
-
}
|
|
183
|
-
if (input.capture === "same-resource" && input.restoreMount !== "same-resource") {
|
|
184
|
-
return { valid: false, reason: "same-resource capture requires same-resource restore" };
|
|
831
|
+
// src/runtime-snapshot.ts
|
|
832
|
+
async function applySandboxRuntimeSnapshot(input) {
|
|
833
|
+
const { leaseStore, workspaceId, snapshot } = input;
|
|
834
|
+
const liveness = input.liveness ?? { kind: "demote" };
|
|
835
|
+
const current = await leaseStore.get(workspaceId);
|
|
836
|
+
if (!current) return { ok: false, reason: "runtime_lease_missing" };
|
|
837
|
+
if (current.epoch !== snapshot.epoch) return { ok: false, reason: "runtime_lease_epoch_mismatch" };
|
|
838
|
+
if (!current.sandboxId || !current.url || !current.hostId) {
|
|
839
|
+
return { ok: false, reason: "runtime_lease_not_provisioned" };
|
|
185
840
|
}
|
|
186
|
-
if (
|
|
187
|
-
return {
|
|
841
|
+
if (current.status === "stopped" || current.status === "destroyed") {
|
|
842
|
+
return { ok: false, reason: "runtime_lease_not_serving" };
|
|
188
843
|
}
|
|
189
|
-
|
|
844
|
+
const timestamp = snapshot.now ?? (input.now ?? Date.now)();
|
|
845
|
+
const retry = liveness.kind === "retry-budget";
|
|
846
|
+
const updated = await leaseStore.update(workspaceId, snapshot.epoch, {
|
|
847
|
+
status: snapshot.ok ? "ready" : "unavailable",
|
|
848
|
+
lastHeartbeatAt: timestamp,
|
|
849
|
+
...snapshot.active ? { lastActivityAt: timestamp } : {},
|
|
850
|
+
...snapshot.ok ? { lastError: null } : {},
|
|
851
|
+
...retry && snapshot.ok ? { retryCount: 0, nextRetryAt: null } : {},
|
|
852
|
+
...retry && !snapshot.ok ? {
|
|
853
|
+
retryCount: current.retryCount + 1,
|
|
854
|
+
nextRetryAt: liveness.nextRetryAt({ retryCount: current.retryCount + 1, now: timestamp }) ?? null,
|
|
855
|
+
lastError: SANDBOX_RUNTIME_UNHEALTHY
|
|
856
|
+
} : {}
|
|
857
|
+
}, current.status);
|
|
858
|
+
if (!updated) return { ok: false, reason: await refusedReason(leaseStore, workspaceId, snapshot.epoch) };
|
|
859
|
+
return { ok: true, status: updated.status };
|
|
190
860
|
}
|
|
191
|
-
function
|
|
192
|
-
|
|
193
|
-
if (
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
if (hosts === 0 && cidrs === 0) return { action: "enforce" };
|
|
197
|
-
if (hosts > 0) return { action: "enforce" };
|
|
198
|
-
return control === "hosts-and-cidrs" ? { action: "enforce" } : { action: "refuse", reason: "sandbox_egress_policy_unenforceable" };
|
|
861
|
+
async function refusedReason(leaseStore, workspaceId, epoch) {
|
|
862
|
+
const current = await leaseStore.get(workspaceId);
|
|
863
|
+
if (!current) return "runtime_lease_missing";
|
|
864
|
+
if (current.epoch !== epoch) return "runtime_lease_epoch_mismatch";
|
|
865
|
+
return "runtime_lease_not_serving";
|
|
199
866
|
}
|
|
867
|
+
|
|
868
|
+
// src/egress-warnings.ts
|
|
200
869
|
function egressUnenforcedMessage(input) {
|
|
201
870
|
const scope = input.workspaceId ? `workspace ${input.workspaceId}` : "every sandbox it provisions";
|
|
202
871
|
const withheld = input.requested ? ` The requested allowlist (${(input.requested.hosts ?? []).length} host(s), ${(input.requested.cidrs ?? []).length} cidr(s)) was withheld, not applied.` : "";
|
|
203
|
-
return `[sandbox-manager] SANDBOX EGRESS IS UNRESTRICTED: driver "${input.driver}" declares egressControl: "none", so ${scope} can reach ANY host on the internet, including attacker-controlled buckets \u2014 agent-authored code inside the sandbox has an unmonitored exfiltration path.${withheld} Select a driver that can enforce egress (
|
|
872
|
+
return `[sandbox-manager] SANDBOX EGRESS IS UNRESTRICTED: driver "${input.driver}" declares egressControl: "none", so ${scope} can reach ANY host on the internet, including attacker-controlled buckets \u2014 agent-authored code inside the sandbox has an unmonitored exfiltration path.${withheld} Select a driver that can enforce egress (vercel) to close it. See public-docs/sandbox-egress.md.`;
|
|
204
873
|
}
|
|
205
874
|
var compositionEgressWarnings = /* @__PURE__ */ new Set();
|
|
875
|
+
var ensureEgressWarnings = /* @__PURE__ */ new Set();
|
|
206
876
|
function defaultEgressUnenforcedSink(event) {
|
|
207
877
|
console.warn(event.message);
|
|
208
878
|
}
|
|
209
|
-
function
|
|
210
|
-
|
|
879
|
+
function createEgressWarnings(input) {
|
|
880
|
+
const { id, egressControl } = input.driver;
|
|
881
|
+
const sink = input.sink ?? defaultEgressUnenforcedSink;
|
|
882
|
+
const key = `${id}|${egressControl}`;
|
|
883
|
+
if (egressControl === "none" && !compositionEgressWarnings.has(key)) {
|
|
884
|
+
compositionEgressWarnings.add(key);
|
|
885
|
+
sink({
|
|
886
|
+
phase: "composition",
|
|
887
|
+
reason: "sandbox_egress_uncontained",
|
|
888
|
+
driver: id,
|
|
889
|
+
egressControl,
|
|
890
|
+
message: egressUnenforcedMessage({ phase: "composition", driver: id })
|
|
891
|
+
});
|
|
892
|
+
}
|
|
893
|
+
return {
|
|
894
|
+
withheld(start) {
|
|
895
|
+
if (ensureEgressWarnings.has(key)) return;
|
|
896
|
+
ensureEgressWarnings.add(key);
|
|
897
|
+
sink({
|
|
898
|
+
phase: "ensure",
|
|
899
|
+
reason: "sandbox_egress_uncontained",
|
|
900
|
+
driver: id,
|
|
901
|
+
egressControl,
|
|
902
|
+
message: egressUnenforcedMessage({ phase: "ensure", driver: id, workspaceId: start.workspaceId, requested: start.requested }),
|
|
903
|
+
workspaceId: start.workspaceId,
|
|
904
|
+
...start.labels ? { labels: start.labels } : {},
|
|
905
|
+
requested: start.requested
|
|
906
|
+
});
|
|
907
|
+
}
|
|
908
|
+
};
|
|
211
909
|
}
|
|
212
|
-
|
|
213
|
-
|
|
910
|
+
|
|
911
|
+
// src/garbage-collect.ts
|
|
912
|
+
async function sweepSandboxes(input) {
|
|
913
|
+
const { driver, leaseStore, appLabel } = input;
|
|
914
|
+
const blind = { destroyed: [], kept: [], skipped: [], failed: [], listingUnsupported: true, driver: driver.id };
|
|
915
|
+
if (!driver.list) return blind;
|
|
916
|
+
let listed;
|
|
917
|
+
try {
|
|
918
|
+
listed = await driver.list();
|
|
919
|
+
} catch (err) {
|
|
920
|
+
if (!isSandboxListingUnsupported(err)) throw err;
|
|
921
|
+
return blind;
|
|
922
|
+
}
|
|
923
|
+
const leases = new Map((await leaseStore.list()).map((lease) => [lease.workspaceId, lease]));
|
|
924
|
+
const result = { destroyed: [], kept: [], skipped: [], failed: [] };
|
|
925
|
+
for (const target of listed) {
|
|
926
|
+
if (target.labels?.app !== appLabel) {
|
|
927
|
+
result.skipped.push({ target, reason: "unmanaged_app_label" });
|
|
928
|
+
continue;
|
|
929
|
+
}
|
|
930
|
+
const workspaceId = target.labels.workspaceId;
|
|
931
|
+
const epoch = target.labels.epoch;
|
|
932
|
+
if (!workspaceId || !epoch) {
|
|
933
|
+
result.skipped.push({ target, reason: "missing_runtime_labels" });
|
|
934
|
+
continue;
|
|
935
|
+
}
|
|
936
|
+
const lease = leases.get(workspaceId);
|
|
937
|
+
const leaseOwnsResource = lease !== void 0 && lease.status !== "destroyed" && (lease.sandboxId === target.sandboxId || lease.driverResourceId !== void 0 && lease.driverResourceId === target.driverResourceId);
|
|
938
|
+
if (leaseOwnsResource) {
|
|
939
|
+
result.kept.push(target);
|
|
940
|
+
continue;
|
|
941
|
+
}
|
|
942
|
+
if (lease?.status === "acquiring" && String(lease.epoch) === epoch) {
|
|
943
|
+
result.kept.push(target);
|
|
944
|
+
continue;
|
|
945
|
+
}
|
|
946
|
+
if (!driver.destroy) {
|
|
947
|
+
result.skipped.push({ target, reason: "destroy_unsupported" });
|
|
948
|
+
continue;
|
|
949
|
+
}
|
|
950
|
+
try {
|
|
951
|
+
await driver.destroy(target);
|
|
952
|
+
result.destroyed.push(target);
|
|
953
|
+
} catch (err) {
|
|
954
|
+
result.failed.push({ target, error: err instanceof Error ? err.message : String(err) });
|
|
955
|
+
}
|
|
956
|
+
}
|
|
957
|
+
return result;
|
|
214
958
|
}
|
|
215
|
-
|
|
959
|
+
|
|
960
|
+
// src/start-telemetry.ts
|
|
961
|
+
function createSandboxStartRecorder(options) {
|
|
962
|
+
function emit(lease, progress, timing, extra) {
|
|
963
|
+
try {
|
|
964
|
+
options.onStartPhase?.({
|
|
965
|
+
...timing,
|
|
966
|
+
sinceStartMs: Math.max(0, options.now() - progress.requestedAt),
|
|
967
|
+
workspaceId: lease.workspaceId,
|
|
968
|
+
epoch: lease.epoch,
|
|
969
|
+
driver: options.driver,
|
|
970
|
+
homeRegion: lease.homeRegion,
|
|
971
|
+
bootMode: progress.bootMode,
|
|
972
|
+
labels: extra.labels,
|
|
973
|
+
...extra.repoSizeBytes === void 0 ? {} : { repoSizeBytes: extra.repoSizeBytes }
|
|
974
|
+
});
|
|
975
|
+
} catch {
|
|
976
|
+
}
|
|
977
|
+
}
|
|
978
|
+
async function append(lease, progress, timings, extra) {
|
|
979
|
+
const fresh = timings.filter((timing, index) => !progress.phases.includes(timing.phase) && timings.findIndex((other) => other.phase === timing.phase) === index);
|
|
980
|
+
if (!fresh.length) return void 0;
|
|
981
|
+
const next = {
|
|
982
|
+
...progress,
|
|
983
|
+
markedAt: extra.markedAt ?? progress.markedAt,
|
|
984
|
+
phases: [...progress.phases, ...fresh.map((timing) => timing.phase)]
|
|
985
|
+
};
|
|
986
|
+
if (!await options.leaseStore.update(lease.workspaceId, lease.epoch, { start: next })) return void 0;
|
|
987
|
+
for (const timing of fresh) emit(lease, next, timing, extra);
|
|
988
|
+
return next;
|
|
989
|
+
}
|
|
990
|
+
async function started(workspaceId, epoch) {
|
|
991
|
+
const lease = await options.leaseStore.get(workspaceId);
|
|
992
|
+
return lease?.epoch === epoch && lease.start ? { lease, progress: lease.start } : void 0;
|
|
993
|
+
}
|
|
216
994
|
return {
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
995
|
+
/**
|
|
996
|
+
* The phases one provision call observes, each timed and written into the
|
|
997
|
+
* lease as it ends, so a poller reading the lease mid-boot sees how far the
|
|
998
|
+
* start has come. `opened` writes the epoch's `lease_decision` before the
|
|
999
|
+
* driver runs, so a stop that lands mid-boot finds the boot it ends on the
|
|
1000
|
+
* row. A serving lease is not starting. An epoch acquired with a start it
|
|
1001
|
+
* continues has that start and no phase yet; one acquired without begins
|
|
1002
|
+
* its own here.
|
|
1003
|
+
*/
|
|
1004
|
+
observe(lease, input) {
|
|
1005
|
+
if (lease.status === "ready") return void 0;
|
|
1006
|
+
let progress = lease.start ?? { requestedAt: input.enteredAt, startedAt: input.enteredAt, bootMode: input.bootMode, markedAt: input.enteredAt, phases: [] };
|
|
1007
|
+
const ended = [];
|
|
1008
|
+
let markedAt = progress.markedAt;
|
|
1009
|
+
const mark = (phase) => {
|
|
1010
|
+
if (progress.phases.includes(phase) || ended.some((timing) => timing.phase === phase)) return;
|
|
1011
|
+
const at = options.now();
|
|
1012
|
+
ended.push({ phase, durationMs: Math.max(0, at - markedAt) });
|
|
1013
|
+
markedAt = at;
|
|
1014
|
+
};
|
|
1015
|
+
const write = async () => {
|
|
1016
|
+
const written = await append(lease, progress, ended.splice(0), { labels: input.labels, markedAt }).catch(() => void 0);
|
|
1017
|
+
if (written) progress = written;
|
|
1018
|
+
};
|
|
1019
|
+
if (!progress.phases.length) mark("lease_decision");
|
|
1020
|
+
return {
|
|
1021
|
+
opened: write,
|
|
1022
|
+
end: async (phase) => {
|
|
1023
|
+
mark(phase);
|
|
1024
|
+
await write();
|
|
1025
|
+
}
|
|
1026
|
+
};
|
|
1027
|
+
},
|
|
1028
|
+
async report(workspaceId, input) {
|
|
1029
|
+
const start = await started(workspaceId, input.epoch);
|
|
1030
|
+
if (!start) return;
|
|
1031
|
+
await append(start.lease, start.progress, input.phases, {
|
|
1032
|
+
labels: start.lease.labels ?? {},
|
|
1033
|
+
...input.repoSizeBytes === void 0 ? {} : { repoSizeBytes: input.repoSizeBytes }
|
|
1034
|
+
});
|
|
1035
|
+
},
|
|
1036
|
+
async markAfter(workspaceId, input) {
|
|
1037
|
+
const start = await started(workspaceId, input.epoch);
|
|
1038
|
+
if (!start || input.notBefore < start.progress.startedAt) return;
|
|
1039
|
+
const at = options.now();
|
|
1040
|
+
await append(start.lease, start.progress, [{ phase: input.phase, durationMs: Math.max(0, at - start.progress.markedAt) }], {
|
|
1041
|
+
labels: start.lease.labels ?? {},
|
|
1042
|
+
markedAt: at
|
|
1043
|
+
});
|
|
1044
|
+
}
|
|
233
1045
|
};
|
|
234
1046
|
}
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
1047
|
+
|
|
1048
|
+
// src/lease-lifecycle.ts
|
|
1049
|
+
function createLifecycleReporter(sink, now) {
|
|
1050
|
+
const base = (lease) => ({ workspaceId: lease.workspaceId, epoch: lease.epoch, driver: lease.driver, labels: lease.labels ?? {} });
|
|
1051
|
+
const active = (lease, endedAt = now()) => lease.status === "ready" && lease.start ? { activeMs: Math.max(0, endedAt - lease.start.startedAt) } : {};
|
|
1052
|
+
const report = (event) => {
|
|
1053
|
+
try {
|
|
1054
|
+
sink?.(event);
|
|
1055
|
+
} catch {
|
|
1056
|
+
}
|
|
1057
|
+
};
|
|
241
1058
|
return {
|
|
242
|
-
...
|
|
243
|
-
|
|
1059
|
+
stopped: (lease, cause, outcome = {}) => report({ ...base(lease), kind: "stopped", cause, ...active(lease), ...outcome }),
|
|
1060
|
+
/** `endedAt` is when the host was last proven serving: a stop the provider made on its own is learned after the fact. */
|
|
1061
|
+
hostStopped: (lease, endedAt) => report({ ...base(lease), kind: "stopped", cause: "provider", ...active(lease, endedAt) }),
|
|
1062
|
+
destroyed: (lease) => report({ ...base(lease), kind: "destroyed", hostGone: false, ...active(lease) }),
|
|
1063
|
+
/** `endedAt` is when the host was last proven serving: the provider's loss is learned after the fact. */
|
|
1064
|
+
hostGone: (lease, endedAt) => report({ ...base(lease), kind: "destroyed", hostGone: true, ...active(lease, endedAt) }),
|
|
1065
|
+
failed: (lease, failure) => report({ ...base(lease), kind: "failed", ...failure }),
|
|
1066
|
+
startedFresh: (lease, checkpointUnusable) => report({ ...base(lease), kind: "startedFresh", ...checkpointUnusable ? { checkpointUnusable } : {} })
|
|
244
1067
|
};
|
|
245
1068
|
}
|
|
246
|
-
|
|
247
|
-
|
|
1069
|
+
|
|
1070
|
+
// src/lease-start.ts
|
|
1071
|
+
function leaseBootMode(lease, driver) {
|
|
1072
|
+
const restoring = Boolean(lease.checkpoint) && lease.status !== "ready" && driver.metadata.persistence.resume === "replacement-restore";
|
|
1073
|
+
if (restoring) return "restore";
|
|
1074
|
+
if (lease.sandboxId && lease.url && driver.resumeHost) return "resume";
|
|
1075
|
+
return "cold-start";
|
|
1076
|
+
}
|
|
1077
|
+
function continuedStart(lease, input) {
|
|
1078
|
+
if (!lease?.start) return void 0;
|
|
1079
|
+
const failure = sandboxLeaseFailure(lease, input);
|
|
1080
|
+
const continued = lease.status === "acquiring" || failure?.kind === "provider" && lease.nextRetryAt !== void 0 && lease.retryCount < input.maxRetryCount;
|
|
1081
|
+
if (!continued) return void 0;
|
|
1082
|
+
const { requestedAt, bootMode, failures } = lease.start;
|
|
1083
|
+
return { requestedAt, bootMode, startedAt: input.now, markedAt: input.now, phases: [], ...failures ? { failures } : {} };
|
|
248
1084
|
}
|
|
1085
|
+
|
|
1086
|
+
// src/start-fresh.ts
|
|
1087
|
+
function startsFreshFromImage(persistence) {
|
|
1088
|
+
return persistence.capture === "filesystem" && persistence.restoreMount === "new-resource";
|
|
1089
|
+
}
|
|
1090
|
+
async function startSandboxFresh(input) {
|
|
1091
|
+
if (!startsFreshFromImage(input.driver.metadata.persistence)) return { ok: false, reason: "start_fresh_unsupported" };
|
|
1092
|
+
const lease = await input.leaseStore.get(input.workspaceId);
|
|
1093
|
+
if (!lease) return { ok: false, reason: "runtime_lease_missing" };
|
|
1094
|
+
const failure = sandboxLeaseFailure(lease, { now: input.now, maxRetryCount: input.maxRetryCount });
|
|
1095
|
+
if (lease.status === "ready" || lease.status === "acquiring" || failure?.retrying) return { ok: false, reason: "workspace_running" };
|
|
1096
|
+
const status = lease.status === "destroyed" ? "destroyed" : "stopped";
|
|
1097
|
+
const updated = await input.leaseStore.update(input.workspaceId, lease.epoch, {
|
|
1098
|
+
status,
|
|
1099
|
+
checkpoint: null,
|
|
1100
|
+
restore: null,
|
|
1101
|
+
lastError: null,
|
|
1102
|
+
nextRetryAt: null,
|
|
1103
|
+
retryCount: 0
|
|
1104
|
+
}, lease.status);
|
|
1105
|
+
if (!updated) return { ok: false, reason: "runtime_lease_changed" };
|
|
1106
|
+
const resource = input.resource(lease);
|
|
1107
|
+
if (lease.checkpoint && resource) await discardSnapshot(input.driver.deleteSnapshot, resource, lease.checkpoint.providerReference);
|
|
1108
|
+
input.lifecycle.startedFresh(lease, failure?.kind === "checkpoint_unusable" ? failure.cause : void 0);
|
|
1109
|
+
return { ok: true, status };
|
|
1110
|
+
}
|
|
1111
|
+
|
|
1112
|
+
// src/runtime-env.ts
|
|
1113
|
+
var WORKSPACE_RUNTIME_IDENTITY_ENV_KEYS = [
|
|
1114
|
+
"WORKSPACE_RUNTIME_WORKSPACE_ID",
|
|
1115
|
+
"WORKSPACE_RUNTIME_HOST_ID",
|
|
1116
|
+
"WORKSPACE_RUNTIME_RELAY_WORKSPACE_IDS"
|
|
1117
|
+
];
|
|
1118
|
+
function workspaceRuntimeIdentityEnvConflicts(env) {
|
|
1119
|
+
if (!env) return [];
|
|
1120
|
+
return WORKSPACE_RUNTIME_IDENTITY_ENV_KEYS.filter((key) => env[key] !== void 0);
|
|
1121
|
+
}
|
|
1122
|
+
var WORKSPACE_RUNTIME_PUBLIC_ENV_KEYS = /* @__PURE__ */ new Set([
|
|
1123
|
+
...WORKSPACE_RUNTIME_IDENTITY_ENV_KEYS,
|
|
1124
|
+
"WORKSPACE_RUNTIME_DIRECTORY",
|
|
1125
|
+
"WORKSPACE_RUNTIME_PORT",
|
|
1126
|
+
"WORKSPACE_RUNTIME_HOST",
|
|
1127
|
+
"WORKSPACE_RUNTIME_SOURCE_KIND",
|
|
1128
|
+
"WORKSPACE_RUNTIME_GIT_REPO_URL",
|
|
1129
|
+
"WORKSPACE_RUNTIME_GIT_BRANCH",
|
|
1130
|
+
"WORKSPACE_RUNTIME_NATIVE_HARNESS",
|
|
1131
|
+
"WORKSPACE_RUNTIME_RELAY_JWKS_URL",
|
|
1132
|
+
"WORKSPACE_RUNTIME_MANAGEMENT_JWKS_URL",
|
|
1133
|
+
"WORKSPACE_RUNTIME_SESSION_AUTHORITY_URL"
|
|
1134
|
+
]);
|
|
1135
|
+
|
|
1136
|
+
// src/manager.ts
|
|
249
1137
|
var DEFAULT_APP_LABEL = "claxedo";
|
|
250
1138
|
function sandboxId(driver, workspaceId) {
|
|
251
1139
|
return `${driver}-${workspaceId}`;
|
|
252
1140
|
}
|
|
1141
|
+
function leaseHostId(driver, workspaceId, lease, managerInput) {
|
|
1142
|
+
return managerInput?.hostId ?? lease.hostId ?? sandboxId(driver.id, workspaceId);
|
|
1143
|
+
}
|
|
253
1144
|
function ensureHostInput(input) {
|
|
254
1145
|
const egress = sandboxEgressDisposition(input.driver.metadata.egressControl, input.managerInput?.net);
|
|
255
1146
|
const labels = {
|
|
@@ -261,7 +1152,7 @@ function ensureHostInput(input) {
|
|
|
261
1152
|
};
|
|
262
1153
|
return {
|
|
263
1154
|
workspaceId: input.workspaceId,
|
|
264
|
-
hostId: input.
|
|
1155
|
+
hostId: leaseHostId(input.driver, input.workspaceId, input.lease, input.managerInput),
|
|
265
1156
|
homeRegion: input.homeRegion,
|
|
266
1157
|
epoch: input.lease.epoch,
|
|
267
1158
|
labels,
|
|
@@ -269,13 +1160,14 @@ function ensureHostInput(input) {
|
|
|
269
1160
|
workspaceRoot: input.managerInput?.workspaceRoot ?? "/workspace",
|
|
270
1161
|
runtimeCwd: input.managerInput?.runtimeCwd,
|
|
271
1162
|
workspaceRuntimePort: input.managerInput?.workspaceRuntimePort ?? DEFAULT_WORKSPACE_RUNTIME_PORT,
|
|
272
|
-
env: input.managerInput?.env
|
|
1163
|
+
env: { ...input.managerInput?.env, ...input.leaseEnv },
|
|
273
1164
|
// Brokered secrets ride their own channel — NEVER merged into labels or env.
|
|
274
1165
|
// Presence of the key, not its length, is the signal: `[]` is the caller
|
|
275
1166
|
// withdrawing every brokered secret, and dropping it would reach the driver
|
|
276
1167
|
// as "preserve what you have".
|
|
277
1168
|
...input.managerInput?.secrets ? { secrets: input.managerInput.secrets } : {},
|
|
278
1169
|
source: input.managerInput?.source,
|
|
1170
|
+
...input.lease.machineClass ?? input.managerInput?.machineClass ? { machineClass: input.lease.machineClass ?? input.managerInput?.machineClass } : {},
|
|
279
1171
|
exposure: input.managerInput?.exposure ?? (input.driver.metadata.targetAccess === "loopback" ? { kind: "loopback" } : { kind: "relay" }),
|
|
280
1172
|
// Withheld, never downgraded: `undefined` here is the same shape a caller
|
|
281
1173
|
// that asked for no containment produces, so the throwing drivers take
|
|
@@ -294,38 +1186,18 @@ function createSandboxManager(options) {
|
|
|
294
1186
|
const retryDelayMs = options.retryDelayMs ?? ((retryCount) => Math.min(6e4, 1e3 * 2 ** Math.max(0, retryCount - 1)));
|
|
295
1187
|
const maxRetryCount = options.maxRetryCount ?? Number.POSITIVE_INFINITY;
|
|
296
1188
|
const retryCapCooldownMs = options.retryCapCooldownMs ?? 10 * 6e4;
|
|
297
|
-
const
|
|
298
|
-
|
|
1189
|
+
const egressWarnings = createEgressWarnings({
|
|
1190
|
+
driver: { id: options.driver.id, egressControl: options.driver.metadata.egressControl },
|
|
1191
|
+
...options.onEgressUnenforced ? { sink: options.onEgressUnenforced } : {}
|
|
1192
|
+
});
|
|
299
1193
|
const lifecycleOperations = /* @__PURE__ */ new Map();
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
phase: "ensure",
|
|
308
|
-
driver: options.driver.id,
|
|
309
|
-
workspaceId: input.workspaceId,
|
|
310
|
-
requested: input.requested
|
|
311
|
-
}),
|
|
312
|
-
workspaceId: input.workspaceId,
|
|
313
|
-
requested: input.requested
|
|
314
|
-
});
|
|
315
|
-
}
|
|
316
|
-
if (egressControl === "none") {
|
|
317
|
-
const key = `${options.driver.id}|${egressControl}`;
|
|
318
|
-
if (!compositionEgressWarnings.has(key)) {
|
|
319
|
-
compositionEgressWarnings.add(key);
|
|
320
|
-
onEgressUnenforced({
|
|
321
|
-
phase: "composition",
|
|
322
|
-
reason: "sandbox_egress_uncontained",
|
|
323
|
-
driver: options.driver.id,
|
|
324
|
-
egressControl,
|
|
325
|
-
message: egressUnenforcedMessage({ phase: "composition", driver: options.driver.id })
|
|
326
|
-
});
|
|
327
|
-
}
|
|
328
|
-
}
|
|
1194
|
+
const lifecycle = createLifecycleReporter(options.onLifecycle, now);
|
|
1195
|
+
const starts = createSandboxStartRecorder({
|
|
1196
|
+
leaseStore: options.leaseStore,
|
|
1197
|
+
driver: options.driver.id,
|
|
1198
|
+
now,
|
|
1199
|
+
...options.onStartPhase ? { onStartPhase: options.onStartPhase } : {}
|
|
1200
|
+
});
|
|
329
1201
|
function lifecycleQueue(current) {
|
|
330
1202
|
return current?.promise.catch(() => void 0) ?? Promise.resolve();
|
|
331
1203
|
}
|
|
@@ -347,43 +1219,64 @@ function createSandboxManager(options) {
|
|
|
347
1219
|
const operation = lifecycleQueue(current).then(run);
|
|
348
1220
|
return lifecycleSettle(workspaceId, { kind, promise: operation }, operation);
|
|
349
1221
|
}
|
|
350
|
-
|
|
351
|
-
if (
|
|
352
|
-
return { status: "unavailable", reason: "runtime_lease_not_ready" };
|
|
353
|
-
}
|
|
1222
|
+
function leaseResource(lease) {
|
|
1223
|
+
if (!lease.sandboxId || !lease.hostId) return void 0;
|
|
354
1224
|
return {
|
|
355
|
-
status: "ready",
|
|
356
1225
|
sandboxId: lease.sandboxId,
|
|
357
1226
|
url: lease.url,
|
|
358
1227
|
workspaceId: lease.workspaceId,
|
|
359
1228
|
hostId: lease.hostId,
|
|
360
1229
|
driverResourceId: lease.driverResourceId,
|
|
361
1230
|
labels: lease.labels,
|
|
362
|
-
driver: {
|
|
363
|
-
id: lease.driver,
|
|
364
|
-
resourceId: lease.driverResourceId ?? lease.sandboxId
|
|
365
|
-
},
|
|
1231
|
+
driver: { id: lease.driver, resourceId: lease.driverResourceId ?? lease.sandboxId },
|
|
366
1232
|
epoch: lease.epoch,
|
|
367
1233
|
homeRegion: lease.homeRegion
|
|
368
1234
|
};
|
|
369
1235
|
}
|
|
370
|
-
function
|
|
371
|
-
const
|
|
372
|
-
if (
|
|
373
|
-
|
|
374
|
-
|
|
1236
|
+
async function leaseTarget(lease) {
|
|
1237
|
+
const resource = leaseResource(lease);
|
|
1238
|
+
if (lease.status !== "ready" || !resource || !resource.url || !lease.routingId) {
|
|
1239
|
+
const failure = sandboxLeaseFailure(lease, { now: now(), maxRetryCount });
|
|
1240
|
+
const stopReason = lease.status === "stopped" ? sandboxStopReason(lease.lastError) : void 0;
|
|
1241
|
+
return {
|
|
1242
|
+
status: "unavailable",
|
|
1243
|
+
reason: "runtime_lease_not_ready",
|
|
1244
|
+
leaseStatus: lease.status,
|
|
1245
|
+
...lease.nextRetryAt !== void 0 ? { retryAfterMs: Math.max(0, lease.nextRetryAt - now()) } : {},
|
|
1246
|
+
...failure ? { failure } : {},
|
|
1247
|
+
...stopReason ? { stopReason } : {},
|
|
1248
|
+
...lease.start && (lease.status === "acquiring" || lease.status === "unavailable") ? { start: { ...lease.start, epoch: lease.epoch } } : {},
|
|
1249
|
+
...lease.status === "acquiring" && lease.hostId ? { booting: { epoch: lease.epoch, hostId: lease.hostId } } : {}
|
|
1250
|
+
};
|
|
1251
|
+
}
|
|
1252
|
+
const booted = resource.labels?.[SANDBOX_IMAGE_LABEL];
|
|
1253
|
+
const imageOutdated = options.driver.image !== void 0 && booted !== void 0 && booted !== options.driver.image;
|
|
1254
|
+
return {
|
|
1255
|
+
...resource,
|
|
1256
|
+
status: "ready",
|
|
1257
|
+
url: resource.url,
|
|
1258
|
+
routingId: lease.routingId,
|
|
1259
|
+
...imageOutdated ? { imageOutdated: true } : {},
|
|
1260
|
+
...lease.start ? { start: { ...lease.start, epoch: lease.epoch } } : {}
|
|
1261
|
+
};
|
|
375
1262
|
}
|
|
376
|
-
async function
|
|
1263
|
+
async function provisionLease(workspaceId, lease, homeRegion, managerInput, enteredAt) {
|
|
377
1264
|
try {
|
|
1265
|
+
const leaseEnv = await managerInput.leaseEnv?.({ epoch: lease.epoch, hostId: leaseHostId(options.driver, workspaceId, lease, managerInput) });
|
|
1266
|
+
const identityConflicts = workspaceRuntimeIdentityEnvConflicts(leaseEnv);
|
|
1267
|
+
if (identityConflicts.length) {
|
|
1268
|
+
return { status: "unavailable", error: `sandbox env cannot set runtime identity: ${identityConflicts.join(", ")}`, epoch: lease.epoch, homeRegion };
|
|
1269
|
+
}
|
|
378
1270
|
const ensure = ensureHostInput({
|
|
379
1271
|
driver: options.driver,
|
|
380
1272
|
workspaceId,
|
|
381
1273
|
homeRegion,
|
|
382
1274
|
lease,
|
|
383
1275
|
managerInput,
|
|
1276
|
+
...leaseEnv ? { leaseEnv } : {},
|
|
384
1277
|
appLabel: options.appLabel ?? DEFAULT_APP_LABEL
|
|
385
1278
|
});
|
|
386
|
-
if (ensure.secrets?.length && options.driver.metadata.secretBrokering
|
|
1279
|
+
if (ensure.secrets?.length && options.driver.metadata.secretBrokering !== "native") {
|
|
387
1280
|
return {
|
|
388
1281
|
status: "unavailable",
|
|
389
1282
|
error: "secret_brokering_unsupported",
|
|
@@ -392,12 +1285,27 @@ function createSandboxManager(options) {
|
|
|
392
1285
|
};
|
|
393
1286
|
}
|
|
394
1287
|
const resuming = Boolean(lease.sandboxId && ensure.bootSource?.kind === "default" && options.driver.resumeHost);
|
|
1288
|
+
const bootMode = resuming && lease.url ? "resume" : ensure.bootSource?.kind === "driver-snapshot" ? "restore" : "cold-start";
|
|
1289
|
+
const start = starts.observe(lease, { bootMode, enteredAt, labels: ensure.labels });
|
|
1290
|
+
await start?.opened();
|
|
1291
|
+
ensure.onResource = async (resource) => {
|
|
1292
|
+
const recorded = await options.leaseStore.recordTarget(workspaceId, lease.epoch, {
|
|
1293
|
+
...resource,
|
|
1294
|
+
labels: { ...resource.labels, ...ensure.labels },
|
|
1295
|
+
persistence: options.driver.metadata.persistence
|
|
1296
|
+
});
|
|
1297
|
+
if (!recorded) throw new Error("runtime_lease_changed");
|
|
1298
|
+
await start?.end("provider_ready");
|
|
1299
|
+
};
|
|
1300
|
+
ensure.onMachineReady = async () => start?.end("machine_ready");
|
|
1301
|
+
ensure.onImageReady = async () => start?.end("image_ready");
|
|
395
1302
|
const target = resuming ? await options.driver.resumeHost({ lease, ensure }) : await options.driver.ensureHost(ensure);
|
|
396
1303
|
if ("provisioning" in target) {
|
|
397
|
-
await options.leaseStore.update(workspaceId, lease.epoch, {
|
|
1304
|
+
const updated2 = await options.leaseStore.update(workspaceId, lease.epoch, {
|
|
398
1305
|
status: "acquiring",
|
|
399
1306
|
nextRetryAt: now() + target.retryAfterMs
|
|
400
|
-
});
|
|
1307
|
+
}, lease.status);
|
|
1308
|
+
if (!updated2) return { status: "unavailable", error: "runtime_lease_changed", epoch: lease.epoch, homeRegion };
|
|
401
1309
|
return {
|
|
402
1310
|
status: "provisioning",
|
|
403
1311
|
retryAfterMs: target.retryAfterMs,
|
|
@@ -406,23 +1314,27 @@ function createSandboxManager(options) {
|
|
|
406
1314
|
// Honest progress for the connect UI: which of the three boot paths
|
|
407
1315
|
// this cycle is on. Derived from the same inputs that CHOSE the path
|
|
408
1316
|
// above, so it cannot drift from what actually ran.
|
|
409
|
-
bootMode
|
|
1317
|
+
bootMode
|
|
410
1318
|
};
|
|
411
1319
|
}
|
|
412
|
-
const updated = await options.leaseStore.
|
|
413
|
-
status: "ready",
|
|
1320
|
+
const updated = await options.leaseStore.recordTarget(workspaceId, lease.epoch, {
|
|
414
1321
|
sandboxId: target.sandboxId,
|
|
415
1322
|
url: target.url,
|
|
416
1323
|
hostId: target.hostId,
|
|
417
1324
|
driverResourceId: target.driverResourceId,
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
1325
|
+
// The placement labels win over anything the driver echoed back: `app`,
|
|
1326
|
+
// `workspaceId` and `epoch` are what garbage collection reads to decide
|
|
1327
|
+
// whether a live sandbox belongs to this deployment, and a driver that
|
|
1328
|
+
// returned its own label set would otherwise decide that answer.
|
|
1329
|
+
labels: { ...target.labels, ...ensure.labels },
|
|
422
1330
|
persistence: options.driver.metadata.persistence
|
|
423
1331
|
});
|
|
424
|
-
|
|
425
|
-
|
|
1332
|
+
if (!updated) return { status: "unavailable", error: "runtime_lease_changed", epoch: lease.epoch, homeRegion };
|
|
1333
|
+
const resolved = await leaseTarget(updated);
|
|
1334
|
+
if (resolved.status === "ready") {
|
|
1335
|
+
await start?.end("runtime_ready");
|
|
1336
|
+
return resolved;
|
|
1337
|
+
}
|
|
426
1338
|
return {
|
|
427
1339
|
status: "unavailable",
|
|
428
1340
|
error: resolved.reason,
|
|
@@ -430,93 +1342,211 @@ function createSandboxManager(options) {
|
|
|
430
1342
|
homeRegion
|
|
431
1343
|
};
|
|
432
1344
|
} catch (err) {
|
|
433
|
-
const
|
|
434
|
-
if (lease.
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
1345
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
1346
|
+
if (isSandboxHostGone(err) && lease.sandboxId) {
|
|
1347
|
+
return hostGone(workspaceId, lease, managerInput, message, enteredAt);
|
|
1348
|
+
}
|
|
1349
|
+
const checkpointUnusable = isSandboxCheckpointUnusable(err) ? err.checkpointUnusable : void 0;
|
|
1350
|
+
const bootFailed = checkpointUnusable !== void 0 || isSandboxRuntimeBootFailure(err);
|
|
1351
|
+
const startLimit = !bootFailed && isSandboxProviderRateLimited(err) ? err.providerRateLimited : void 0;
|
|
1352
|
+
const error = checkpointUnusable ? sandboxCheckpointUnusableError(checkpointUnusable, message) : bootFailed ? sandboxRuntimeBootFailedError(message) : startLimit ? sandboxProviderRateLimitedError(options.driver.id, startLimit) : isSandboxProviderKeyRefused(err) ? sandboxProviderKeyRefusedError(options.driver.id, message) : isSandboxStartRefusal(err) || sandboxLeaseEndedUnderStart(message) ? message : sandboxProviderFailedError(options.driver.id, message);
|
|
1353
|
+
if (lease.status === "ready" && !bootFailed) {
|
|
1354
|
+
const updated = await options.leaseStore.update(workspaceId, lease.epoch, { lastError: error }, "ready");
|
|
1355
|
+
if (updated) {
|
|
1356
|
+
const resolved = await leaseTarget(updated);
|
|
1357
|
+
if (resolved.status === "ready") return { ...resolved, stale: true };
|
|
1358
|
+
}
|
|
438
1359
|
}
|
|
439
1360
|
const nextRetryCount = lease.retryCount + 1;
|
|
440
|
-
const
|
|
441
|
-
const
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
1361
|
+
const startLimitRetryMs = startLimit ? sandboxStartLimitRetryMs(startLimit.window) : void 0;
|
|
1362
|
+
const dayLimited = startLimit !== void 0 && startLimitRetryMs === void 0;
|
|
1363
|
+
const retryCapped = dayLimited || nextRetryCount >= maxRetryCount;
|
|
1364
|
+
const hostLeftRunning = bootFailed || retryCapped ? !await stopHeldHost(workspaceId, lease.epoch) : false;
|
|
1365
|
+
lifecycle.failed(lease, {
|
|
1366
|
+
bootFailed,
|
|
1367
|
+
retrying: !bootFailed && !retryCapped,
|
|
1368
|
+
...startLimit ? { startLimit: startLimit.window } : {},
|
|
1369
|
+
...hostLeftRunning ? { hostLeftRunning: true } : {},
|
|
1370
|
+
...checkpointUnusable ? { checkpointUnusable } : {}
|
|
1371
|
+
});
|
|
1372
|
+
const backoffMs = retryCapped ? retryCapCooldownMs : retryDelayMs(nextRetryCount);
|
|
1373
|
+
const retryInMs = dayLimited ? void 0 : Math.max(backoffMs, startLimitRetryMs ?? 0);
|
|
1374
|
+
const failed = await options.leaseStore.recordFailure(workspaceId, lease.epoch, error, retryInMs === void 0 ? void 0 : now() + retryInMs);
|
|
447
1375
|
return {
|
|
448
1376
|
status: "unavailable",
|
|
449
|
-
retryAfterMs: failed?.nextRetryAt ? Math.max(0, failed.nextRetryAt - now()) : retryAfterMs,
|
|
1377
|
+
...dayLimited ? {} : { retryAfterMs: failed?.nextRetryAt ? Math.max(0, failed.nextRetryAt - now()) : retryAfterMs },
|
|
450
1378
|
error,
|
|
451
1379
|
epoch: lease.epoch,
|
|
452
|
-
homeRegion
|
|
1380
|
+
homeRegion,
|
|
1381
|
+
...retryCapped ? { exhausted: true } : {}
|
|
1382
|
+
};
|
|
1383
|
+
}
|
|
1384
|
+
}
|
|
1385
|
+
async function hostGone(workspaceId, lease, managerInput, message, enteredAt) {
|
|
1386
|
+
if (!await endGoneHostLease(workspaceId, lease, message)) return { status: "unavailable", error: "runtime_lease_changed", epoch: lease.epoch, homeRegion: lease.homeRegion };
|
|
1387
|
+
const admitted = await admit(workspaceId, managerInput);
|
|
1388
|
+
if (!("lease" in admitted)) return admitted;
|
|
1389
|
+
return provisionLease(workspaceId, admitted.lease, admitted.lease.homeRegion, managerInput, enteredAt);
|
|
1390
|
+
}
|
|
1391
|
+
async function endGoneHostLease(workspaceId, lease, message) {
|
|
1392
|
+
const endedAt = hostLastProvenAliveAt(lease);
|
|
1393
|
+
const ended = await options.leaseStore.update(workspaceId, lease.epoch, {
|
|
1394
|
+
status: "destroyed",
|
|
1395
|
+
lastError: sandboxHostGoneError(message),
|
|
1396
|
+
...endedAt === void 0 ? {} : { endedAt }
|
|
1397
|
+
});
|
|
1398
|
+
if (!ended) return false;
|
|
1399
|
+
lifecycle.hostGone(lease, endedAt);
|
|
1400
|
+
return true;
|
|
1401
|
+
}
|
|
1402
|
+
async function endStoppedHostLease(workspaceId, lease) {
|
|
1403
|
+
const endedAt = hostLastProvenAliveAt(lease);
|
|
1404
|
+
const ended = await options.leaseStore.update(workspaceId, lease.epoch, { status: "stopped", ...endedAt === void 0 ? {} : { endedAt } });
|
|
1405
|
+
if (!ended) return false;
|
|
1406
|
+
lifecycle.hostStopped(lease, endedAt);
|
|
1407
|
+
return true;
|
|
1408
|
+
}
|
|
1409
|
+
function hostLastProvenAliveAt(lease) {
|
|
1410
|
+
if (lease.status !== "ready" || !lease.start) return void 0;
|
|
1411
|
+
return Math.max(lease.start.markedAt, lease.lastHeartbeatAt ?? 0, lease.lastActivityAt ?? 0);
|
|
1412
|
+
}
|
|
1413
|
+
async function hostStopped(stopHost, resource) {
|
|
1414
|
+
try {
|
|
1415
|
+
await stopHost?.(resource);
|
|
1416
|
+
return "stopped";
|
|
1417
|
+
} catch (err) {
|
|
1418
|
+
if (!isSandboxHostGone(err)) throw err;
|
|
1419
|
+
return "destroyed";
|
|
1420
|
+
}
|
|
1421
|
+
}
|
|
1422
|
+
async function stopHeldHost(workspaceId, epoch) {
|
|
1423
|
+
const current = await options.leaseStore.get(workspaceId);
|
|
1424
|
+
const resource = current?.epoch === epoch ? leaseResource(current) : void 0;
|
|
1425
|
+
const stopHost = options.driver.suspend ?? options.driver.stop;
|
|
1426
|
+
if (!current || !resource || !stopHost) return true;
|
|
1427
|
+
try {
|
|
1428
|
+
await stopHost({ ...resource, ...current.checkpoint ? { checkpoint: current.checkpoint.providerReference } : {} });
|
|
1429
|
+
return true;
|
|
1430
|
+
} catch (stopError) {
|
|
1431
|
+
console.warn("A sandbox host was not stopped after its start failed", {
|
|
1432
|
+
workspaceId,
|
|
1433
|
+
epoch,
|
|
1434
|
+
sandboxId: resource.sandboxId,
|
|
1435
|
+
error: stopError instanceof Error ? stopError.message : String(stopError)
|
|
1436
|
+
});
|
|
1437
|
+
return false;
|
|
1438
|
+
}
|
|
1439
|
+
}
|
|
1440
|
+
function capture(manager, workspaceId, request, endsLease) {
|
|
1441
|
+
return captureSandboxCheckpoint({
|
|
1442
|
+
workspaceId,
|
|
1443
|
+
request,
|
|
1444
|
+
leaseStore: options.leaseStore,
|
|
1445
|
+
target: (id) => manager.target(id),
|
|
1446
|
+
snapshot: (id, committed) => manager.snapshot(id, committed),
|
|
1447
|
+
...options.driver.deleteSnapshot ? { deleteSnapshot: options.driver.deleteSnapshot } : {},
|
|
1448
|
+
...endsLease ? { endsLease } : {},
|
|
1449
|
+
now
|
|
1450
|
+
});
|
|
1451
|
+
}
|
|
1452
|
+
function wanted(lease, afterMs = retryAfterMs) {
|
|
1453
|
+
return { status: "provisioning", retryAfterMs: afterMs, epoch: lease.epoch, homeRegion: lease.homeRegion, bootMode: leaseBootMode(lease, options.driver) };
|
|
1454
|
+
}
|
|
1455
|
+
function waiting(existing) {
|
|
1456
|
+
if (!existing) return void 0;
|
|
1457
|
+
if (existing.nextRetryAt && existing.nextRetryAt > now()) {
|
|
1458
|
+
if (existing.status === "acquiring") return wanted(existing, existing.nextRetryAt - now());
|
|
1459
|
+
return {
|
|
1460
|
+
status: "unavailable",
|
|
1461
|
+
retryAfterMs: existing.nextRetryAt - now(),
|
|
1462
|
+
error: existing.lastError,
|
|
1463
|
+
epoch: existing.epoch,
|
|
1464
|
+
homeRegion: existing.homeRegion,
|
|
1465
|
+
...existing.retryCount >= maxRetryCount ? { exhausted: true } : {}
|
|
453
1466
|
};
|
|
454
1467
|
}
|
|
1468
|
+
if (existing.status === "unavailable" && existing.retryCount >= maxRetryCount && existing.nextRetryAt === void 0 && sandboxProviderRateLimit(existing.lastError) !== "day") {
|
|
1469
|
+
return {
|
|
1470
|
+
status: "unavailable",
|
|
1471
|
+
error: existing.lastError ?? SANDBOX_RETRY_CAP_EXCEEDED,
|
|
1472
|
+
epoch: existing.epoch,
|
|
1473
|
+
homeRegion: existing.homeRegion
|
|
1474
|
+
};
|
|
1475
|
+
}
|
|
1476
|
+
return void 0;
|
|
1477
|
+
}
|
|
1478
|
+
function continues(lease) {
|
|
1479
|
+
if (lease.status === "ready") return true;
|
|
1480
|
+
if (lease.status !== "acquiring") return false;
|
|
1481
|
+
const stale = now() - lease.updatedAt >= staleAfterMs;
|
|
1482
|
+
return lease.sandboxId ? lease.nextRetryAt !== void 0 || stale : lease.nextRetryAt !== void 0 && !stale;
|
|
1483
|
+
}
|
|
1484
|
+
async function admit(workspaceId, input) {
|
|
1485
|
+
const egress = sandboxEgressDisposition(options.driver.metadata.egressControl, input.net);
|
|
1486
|
+
if (egress.action === "withhold" && input.net) {
|
|
1487
|
+
egressWarnings.withheld({ workspaceId, ...input.labels ? { labels: input.labels } : {}, requested: input.net });
|
|
1488
|
+
}
|
|
1489
|
+
if (egress.action === "refuse") {
|
|
1490
|
+
return { status: "unavailable", error: egress.reason, homeRegion: input.homeRegion };
|
|
1491
|
+
}
|
|
1492
|
+
const identityConflicts = workspaceRuntimeIdentityEnvConflicts(input.env);
|
|
1493
|
+
if (identityConflicts.length) {
|
|
1494
|
+
return {
|
|
1495
|
+
status: "unavailable",
|
|
1496
|
+
error: `sandbox env cannot set runtime identity: ${identityConflicts.join(", ")}`,
|
|
1497
|
+
homeRegion: input.homeRegion
|
|
1498
|
+
};
|
|
1499
|
+
}
|
|
1500
|
+
if (input.machineClass && !options.driver.metadata.machineClasses?.includes(input.machineClass)) {
|
|
1501
|
+
return { status: "unavailable", error: `sandbox_machine_class_unsupported: ${options.driver.id} has no ${input.machineClass} machines`, homeRegion: input.homeRegion };
|
|
1502
|
+
}
|
|
1503
|
+
const existing = await options.leaseStore.get(workspaceId);
|
|
1504
|
+
const wait = waiting(existing);
|
|
1505
|
+
if (wait) return wait;
|
|
1506
|
+
if (existing && continues(existing)) return { lease: existing };
|
|
1507
|
+
const start = continuedStart(existing, { now: now(), maxRetryCount });
|
|
1508
|
+
const acquired = await options.leaseStore.acquire(workspaceId, {
|
|
1509
|
+
homeRegion: input.homeRegion,
|
|
1510
|
+
driver: options.driver.id,
|
|
1511
|
+
staleAfterMs,
|
|
1512
|
+
now: now(),
|
|
1513
|
+
...input.labels ? { labels: input.labels } : {},
|
|
1514
|
+
...input.machineClass ? { machineClass: input.machineClass } : {},
|
|
1515
|
+
...start ? { start } : {}
|
|
1516
|
+
});
|
|
1517
|
+
if (!acquired.acquired) return wanted(acquired.lease, acquired.retryAfterMs);
|
|
1518
|
+
if (!existing) input.onLeaseOpened?.();
|
|
1519
|
+
return { lease: acquired.lease };
|
|
455
1520
|
}
|
|
456
1521
|
return {
|
|
457
1522
|
async ensure(workspaceId, input) {
|
|
458
|
-
const
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
homeRegion: existing.homeRegion,
|
|
473
|
-
bootMode: leaseBootMode(existing)
|
|
474
|
-
};
|
|
475
|
-
}
|
|
476
|
-
return {
|
|
477
|
-
status: "unavailable",
|
|
478
|
-
retryAfterMs: existing.nextRetryAt - now(),
|
|
479
|
-
error: existing.lastError,
|
|
480
|
-
epoch: existing.epoch,
|
|
481
|
-
homeRegion: existing.homeRegion
|
|
482
|
-
};
|
|
483
|
-
}
|
|
484
|
-
if (existing?.status === "unavailable" && existing.retryCount >= maxRetryCount && existing.nextRetryAt === void 0) {
|
|
485
|
-
return {
|
|
486
|
-
status: "unavailable",
|
|
487
|
-
error: existing.lastError ?? "runtime_retry_cap_exceeded",
|
|
488
|
-
epoch: existing.epoch,
|
|
489
|
-
homeRegion: existing.homeRegion
|
|
490
|
-
};
|
|
491
|
-
}
|
|
492
|
-
if (existing?.status === "ready") {
|
|
493
|
-
return provision(workspaceId, existing, existing.homeRegion, input);
|
|
1523
|
+
const enteredAt = now();
|
|
1524
|
+
const admitted = await admit(workspaceId, input);
|
|
1525
|
+
if (!("lease" in admitted)) return admitted;
|
|
1526
|
+
return provisionLease(workspaceId, admitted.lease, admitted.lease.homeRegion, input, enteredAt);
|
|
1527
|
+
},
|
|
1528
|
+
async acquire(workspaceId, input) {
|
|
1529
|
+
const admitted = await admit(workspaceId, input);
|
|
1530
|
+
return "lease" in admitted ? wanted(admitted.lease) : admitted;
|
|
1531
|
+
},
|
|
1532
|
+
async provision(workspaceId, epoch, input) {
|
|
1533
|
+
const enteredAt = now();
|
|
1534
|
+
const lease = await options.leaseStore.get(workspaceId);
|
|
1535
|
+
if (!lease || lease.epoch !== epoch) {
|
|
1536
|
+
return { status: "unavailable", error: "runtime_lease_changed", epoch, homeRegion: input.homeRegion };
|
|
494
1537
|
}
|
|
495
|
-
if (
|
|
496
|
-
return
|
|
1538
|
+
if (lease.status === "stopped" || lease.status === "destroyed") {
|
|
1539
|
+
return { status: "unavailable", error: `runtime_lease_${lease.status}`, epoch, homeRegion: lease.homeRegion };
|
|
497
1540
|
}
|
|
498
|
-
const
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
staleAfterMs,
|
|
502
|
-
now: now()
|
|
503
|
-
});
|
|
504
|
-
if (!acquired.acquired) {
|
|
505
|
-
return {
|
|
506
|
-
status: "provisioning",
|
|
507
|
-
retryAfterMs: acquired.retryAfterMs,
|
|
508
|
-
epoch: acquired.lease.epoch,
|
|
509
|
-
homeRegion: acquired.lease.homeRegion,
|
|
510
|
-
bootMode: leaseBootMode(acquired.lease)
|
|
511
|
-
};
|
|
512
|
-
}
|
|
513
|
-
return provision(workspaceId, acquired.lease, input.homeRegion, input);
|
|
1541
|
+
const admitted = lease.status === "unavailable" ? await admit(workspaceId, input) : waiting(lease) ?? { lease };
|
|
1542
|
+
if (!("lease" in admitted)) return admitted;
|
|
1543
|
+
return provisionLease(workspaceId, admitted.lease, admitted.lease.homeRegion, input, enteredAt);
|
|
514
1544
|
},
|
|
515
1545
|
async register(workspaceId, input) {
|
|
516
|
-
return await
|
|
1546
|
+
return await runtimeSnapshot(workspaceId, input);
|
|
517
1547
|
},
|
|
518
1548
|
async heartbeat(workspaceId, input) {
|
|
519
|
-
return await
|
|
1549
|
+
return await runtimeSnapshot(workspaceId, input);
|
|
520
1550
|
},
|
|
521
1551
|
async target(workspaceId) {
|
|
522
1552
|
const lease = await options.leaseStore.get(workspaceId);
|
|
@@ -529,21 +1559,14 @@ function createSandboxManager(options) {
|
|
|
529
1559
|
await options.driver.touch?.(target);
|
|
530
1560
|
return { touched: true, status: "ready" };
|
|
531
1561
|
},
|
|
532
|
-
async snapshot(workspaceId) {
|
|
1562
|
+
async snapshot(workspaceId, committed) {
|
|
533
1563
|
const target = await this.target(workspaceId);
|
|
534
1564
|
if (target.status !== "ready") return { ok: false, reason: target.reason };
|
|
535
1565
|
if (!options.driver.snapshot) return { ok: false, reason: "snapshot_unsupported" };
|
|
536
|
-
return { ok: true, ...await options.driver.snapshot(target) };
|
|
1566
|
+
return { ok: true, ...await options.driver.snapshot(target, committed) };
|
|
537
1567
|
},
|
|
538
1568
|
async checkpoint(workspaceId, input) {
|
|
539
|
-
return await checkpointLifecycle(workspaceId, "checkpoint", () =>
|
|
540
|
-
workspaceId,
|
|
541
|
-
request: input,
|
|
542
|
-
leaseStore: options.leaseStore,
|
|
543
|
-
target: (id) => this.target(id),
|
|
544
|
-
snapshot: (id) => this.snapshot(id),
|
|
545
|
-
now
|
|
546
|
-
}));
|
|
1569
|
+
return await checkpointLifecycle(workspaceId, "checkpoint", () => capture(this, workspaceId, input));
|
|
547
1570
|
},
|
|
548
1571
|
async restore(workspaceId, input) {
|
|
549
1572
|
return await checkpointLifecycle(workspaceId, "restore", () => restoreSandboxCheckpoint({
|
|
@@ -554,142 +1577,178 @@ function createSandboxManager(options) {
|
|
|
554
1577
|
now
|
|
555
1578
|
}));
|
|
556
1579
|
},
|
|
557
|
-
async stop(workspaceId) {
|
|
1580
|
+
async stop(workspaceId, input = {}) {
|
|
558
1581
|
return await mutationLifecycle(workspaceId, "stop", async () => {
|
|
559
1582
|
const lease = await options.leaseStore.get(workspaceId);
|
|
560
|
-
if (lease?.
|
|
1583
|
+
if (input.expectedEpoch !== void 0 && lease?.epoch !== input.expectedEpoch) return { ok: false, reason: "runtime_lease_changed" };
|
|
1584
|
+
const stopped = { ok: true, status: "stopped" };
|
|
1585
|
+
if (lease?.status === "stopped") return stopped;
|
|
561
1586
|
const target = await this.target(workspaceId);
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
1587
|
+
const stopHost = input.hostStopsItself ? void 0 : options.driver.suspend ?? options.driver.stop;
|
|
1588
|
+
const cause = input.idleBefore !== void 0 ? "idle" : input.unresponsiveForMs !== void 0 ? "unresponsive" : "explicit";
|
|
1589
|
+
const stoppedError = input.unresponsiveForMs === void 0 ? void 0 : sandboxRuntimeUnresponsiveError(input.unresponsiveForMs);
|
|
1590
|
+
const ended = async (resource2, status, unsaved2 = false) => {
|
|
1591
|
+
if (!lease) return void 0;
|
|
1592
|
+
const endedAt = status === "destroyed" ? hostLastProvenAliveAt(lease) : void 0;
|
|
1593
|
+
const updated = await options.leaseStore.update(workspaceId, resource2.epoch, {
|
|
1594
|
+
status,
|
|
1595
|
+
...status === "destroyed" ? { lastError: sandboxHostGoneError(`${resource2.sandboxId} is gone`) } : stoppedError ? { lastError: stoppedError } : {},
|
|
1596
|
+
...endedAt === void 0 ? {} : { endedAt }
|
|
1597
|
+
});
|
|
1598
|
+
if (!updated) return void 0;
|
|
1599
|
+
if (status === "destroyed") lifecycle.hostGone(lease, endedAt);
|
|
1600
|
+
else lifecycle.stopped(lease, cause, unsaved2 ? { unsaved: true } : {});
|
|
1601
|
+
return { ok: true, status, ...unsaved2 ? { unsaved: true } : {} };
|
|
1602
|
+
};
|
|
1603
|
+
if (target.status !== "ready") {
|
|
1604
|
+
const held = lease?.status === "acquiring" ? leaseResource(lease) : void 0;
|
|
1605
|
+
if (!lease || !held) return { ok: false, reason: target.reason };
|
|
1606
|
+
return await ended(held, await hostStopped(stopHost, held)) ?? { ok: false, reason: "runtime_lease_changed" };
|
|
1607
|
+
}
|
|
1608
|
+
const persistence2 = lease?.persistence;
|
|
1609
|
+
let unsaved = false;
|
|
1610
|
+
if (input.runtime && persistence2 && persistence2.capture !== "none") {
|
|
1611
|
+
try {
|
|
1612
|
+
const captured = await capture(this, workspaceId, { runtime: input.runtime, policy: "drain", idleBefore: input.idleBefore }, stoppedError ? { stoppedError } : {});
|
|
1613
|
+
const checkpoint2 = captured.checkpoint.providerReference;
|
|
1614
|
+
if (persistence2.captureSource === "preserved") await stopHost?.({ ...target, checkpoint: checkpoint2 });
|
|
1615
|
+
if (lease) lifecycle.stopped(lease, cause, captured.checkpoint.metadata.unfrozen ? { unfrozen: true } : {});
|
|
1616
|
+
return stopped;
|
|
1617
|
+
} catch (error) {
|
|
1618
|
+
if (input.idleBefore !== void 0) throw error;
|
|
1619
|
+
console.warn("A stopping sandbox was not captured; it stops on its last checkpoint", {
|
|
1620
|
+
workspaceId,
|
|
1621
|
+
epoch: target.epoch,
|
|
1622
|
+
error: error instanceof Error ? error.message : String(error)
|
|
1623
|
+
});
|
|
1624
|
+
const current = await options.leaseStore.get(workspaceId);
|
|
1625
|
+
if (current?.status === "stopped") return stopped;
|
|
1626
|
+
if (current?.epoch !== target.epoch || current.status !== "ready") return { ok: false, reason: "runtime_lease_changed" };
|
|
1627
|
+
unsaved = true;
|
|
1628
|
+
}
|
|
567
1629
|
}
|
|
568
|
-
|
|
569
|
-
|
|
1630
|
+
const checkpoint = lease?.checkpoint?.providerReference;
|
|
1631
|
+
const resource = { ...target, ...checkpoint ? { checkpoint } : {} };
|
|
1632
|
+
return await ended(resource, await hostStopped(stopHost, resource), unsaved) ?? { ok: false, reason: "runtime_lease_changed" };
|
|
570
1633
|
});
|
|
571
1634
|
},
|
|
1635
|
+
async endIfHostDown(workspaceId, epoch) {
|
|
1636
|
+
const lease = await options.leaseStore.get(workspaceId);
|
|
1637
|
+
if (!lease || lease.epoch !== epoch) return void 0;
|
|
1638
|
+
const target = await leaseTarget(lease);
|
|
1639
|
+
if (target.status !== "ready") return void 0;
|
|
1640
|
+
const unansweredAt = now();
|
|
1641
|
+
const state = options.driver.inspect ? await options.driver.inspect(target) : void 0;
|
|
1642
|
+
if (state === void 0 || state === "serving") {
|
|
1643
|
+
await options.leaseStore.update(workspaceId, epoch, { lastHealthFailureAt: unansweredAt, ...state ? { lastHeartbeatAt: unansweredAt } : {} }, "ready");
|
|
1644
|
+
return void 0;
|
|
1645
|
+
}
|
|
1646
|
+
const ended = state === "gone" ? await endGoneHostLease(workspaceId, lease, `${target.sandboxId} is gone`) : await endStoppedHostLease(workspaceId, lease);
|
|
1647
|
+
return ended ? state : void 0;
|
|
1648
|
+
},
|
|
572
1649
|
async destroy(workspaceId) {
|
|
573
1650
|
return await mutationLifecycle(workspaceId, "destroy", async () => {
|
|
574
1651
|
const lease = await options.leaseStore.get(workspaceId);
|
|
575
1652
|
if (lease?.status === "destroyed") return { ok: true, status: "destroyed" };
|
|
576
|
-
|
|
577
|
-
|
|
1653
|
+
if (!lease) return { ok: false, reason: "runtime_lease_missing" };
|
|
1654
|
+
const target = leaseResource(lease);
|
|
1655
|
+
if (!target) return { ok: false, reason: "runtime_lease_resource_missing" };
|
|
578
1656
|
await options.driver.destroy?.(target);
|
|
579
|
-
await options.leaseStore.update(workspaceId, target.epoch, { status: "destroyed" });
|
|
1657
|
+
const updated = await options.leaseStore.update(workspaceId, target.epoch, { status: "destroyed" });
|
|
1658
|
+
if (!updated) return { ok: false, reason: "runtime_lease_changed" };
|
|
1659
|
+
lifecycle.destroyed(lease);
|
|
1660
|
+
if (lease.checkpoint && lease.persistence?.capture !== "same-resource") {
|
|
1661
|
+
await discardSnapshot(options.driver.deleteSnapshot, target, lease.checkpoint.providerReference);
|
|
1662
|
+
}
|
|
580
1663
|
return { ok: true, status: "destroyed" };
|
|
581
1664
|
});
|
|
582
1665
|
},
|
|
1666
|
+
startFresh(workspaceId) {
|
|
1667
|
+
return mutationLifecycle(workspaceId, "startFresh", () => startSandboxFresh({
|
|
1668
|
+
workspaceId,
|
|
1669
|
+
driver: options.driver,
|
|
1670
|
+
leaseStore: options.leaseStore,
|
|
1671
|
+
lifecycle,
|
|
1672
|
+
resource: leaseResource,
|
|
1673
|
+
now: now(),
|
|
1674
|
+
maxRetryCount
|
|
1675
|
+
}));
|
|
1676
|
+
},
|
|
583
1677
|
async release(workspaceId) {
|
|
584
1678
|
const lease = await options.leaseStore.get(workspaceId);
|
|
585
1679
|
if (!lease) return { released: false };
|
|
586
1680
|
await options.leaseStore.release(workspaceId);
|
|
587
1681
|
return { released: true };
|
|
588
1682
|
},
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
kept: [],
|
|
594
|
-
skipped: [],
|
|
595
|
-
failed: [],
|
|
596
|
-
listingUnsupported: true,
|
|
597
|
-
driver: options.driver.id
|
|
598
|
-
};
|
|
599
|
-
}
|
|
600
|
-
let listed;
|
|
601
|
-
try {
|
|
602
|
-
listed = await options.driver.list();
|
|
603
|
-
} catch (err) {
|
|
604
|
-
if (!isSandboxListingUnsupported(err)) throw err;
|
|
605
|
-
return {
|
|
606
|
-
destroyed: [],
|
|
607
|
-
kept: [],
|
|
608
|
-
skipped: [],
|
|
609
|
-
failed: [],
|
|
610
|
-
listingUnsupported: true,
|
|
611
|
-
driver: options.driver.id
|
|
612
|
-
};
|
|
613
|
-
}
|
|
614
|
-
const leases = new Map((await options.leaseStore.list()).map((lease) => [lease.workspaceId, lease]));
|
|
615
|
-
const result = {
|
|
616
|
-
destroyed: [],
|
|
617
|
-
kept: [],
|
|
618
|
-
skipped: [],
|
|
619
|
-
failed: []
|
|
620
|
-
};
|
|
621
|
-
const appLabel = options.appLabel ?? DEFAULT_APP_LABEL;
|
|
622
|
-
for (const target of listed) {
|
|
623
|
-
if (target.labels?.app !== appLabel) {
|
|
624
|
-
result.skipped.push({ target, reason: "unmanaged_app_label" });
|
|
625
|
-
continue;
|
|
626
|
-
}
|
|
627
|
-
const workspaceId = target.labels.workspaceId;
|
|
628
|
-
const epoch = target.labels.epoch;
|
|
629
|
-
if (!workspaceId || !epoch) {
|
|
630
|
-
result.skipped.push({ target, reason: "missing_runtime_labels" });
|
|
631
|
-
continue;
|
|
632
|
-
}
|
|
633
|
-
const lease = leases.get(workspaceId);
|
|
634
|
-
if (lease?.status === "ready" && String(lease.epoch) === epoch && lease.hostId === target.hostId && lease.sandboxId === target.sandboxId) {
|
|
635
|
-
result.kept.push(target);
|
|
636
|
-
continue;
|
|
637
|
-
}
|
|
638
|
-
if (lease?.status === "acquiring" && String(lease.epoch) === epoch) {
|
|
639
|
-
result.kept.push(target);
|
|
640
|
-
continue;
|
|
641
|
-
}
|
|
642
|
-
if (!options.driver.destroy) {
|
|
643
|
-
result.skipped.push({ target, reason: "destroy_unsupported" });
|
|
644
|
-
continue;
|
|
645
|
-
}
|
|
646
|
-
try {
|
|
647
|
-
await options.driver.destroy(target);
|
|
648
|
-
result.destroyed.push(target);
|
|
649
|
-
} catch (err) {
|
|
650
|
-
result.failed.push({ target, error: err instanceof Error ? err.message : String(err) });
|
|
651
|
-
}
|
|
652
|
-
}
|
|
653
|
-
return result;
|
|
654
|
-
},
|
|
655
|
-
list: () => options.leaseStore.list()
|
|
1683
|
+
garbageCollect: () => sweepSandboxes({ driver: options.driver, leaseStore: options.leaseStore, appLabel: options.appLabel ?? DEFAULT_APP_LABEL }),
|
|
1684
|
+
list: () => options.leaseStore.list(),
|
|
1685
|
+
recordStartPhases: (workspaceId, input) => starts.report(workspaceId, input),
|
|
1686
|
+
markStartPhase: (workspaceId, input) => starts.markAfter(workspaceId, input)
|
|
656
1687
|
};
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
if (!current) return { ok: false, reason: "runtime_lease_missing" };
|
|
660
|
-
const expectedEpoch = input.epoch ?? current.epoch;
|
|
661
|
-
if (current.epoch !== expectedEpoch) return { ok: false, reason: "runtime_lease_epoch_mismatch" };
|
|
662
|
-
const timestamp = input.now ?? now();
|
|
663
|
-
const status = input.status === "unhealthy" || !input.ok ? "unavailable" : input.ok ? "ready" : current.status;
|
|
664
|
-
const updated = await options.leaseStore.update(workspaceId, expectedEpoch, {
|
|
665
|
-
status,
|
|
666
|
-
...input.sandboxId ? { sandboxId: input.sandboxId } : {},
|
|
667
|
-
...input.url ? { url: input.url } : {},
|
|
668
|
-
...input.hostId ? { hostId: input.hostId } : input.sandboxId ? { hostId: input.sandboxId } : {},
|
|
669
|
-
...input.driverResourceId ? { driverResourceId: input.driverResourceId } : input.sandboxId ? { driverResourceId: input.sandboxId } : {},
|
|
670
|
-
lastHeartbeatAt: timestamp,
|
|
671
|
-
...input.active ? { lastActivityAt: timestamp } : {},
|
|
672
|
-
...input.ok ? { lastError: null } : {}
|
|
673
|
-
});
|
|
674
|
-
if (!updated) return { ok: false, reason: "runtime_lease_epoch_mismatch" };
|
|
675
|
-
return { ok: true, status: updated.status };
|
|
1688
|
+
function runtimeSnapshot(workspaceId, snapshot) {
|
|
1689
|
+
return applySandboxRuntimeSnapshot({ leaseStore: options.leaseStore, workspaceId, snapshot, now });
|
|
676
1690
|
}
|
|
677
1691
|
}
|
|
678
1692
|
export {
|
|
679
1693
|
DEFAULT_WORKSPACE_RUNTIME_PORT,
|
|
680
1694
|
SANDBOX_BASELINE_EGRESS_HOSTS,
|
|
1695
|
+
SANDBOX_IMAGE_LABEL,
|
|
681
1696
|
SANDBOX_MODEL_PROVIDER_HOSTS,
|
|
682
1697
|
SANDBOX_PACKAGE_REGISTRY_HOSTS,
|
|
1698
|
+
SANDBOX_RETRY_CAP_EXCEEDED,
|
|
1699
|
+
SANDBOX_RUNTIME_UNHEALTHY,
|
|
1700
|
+
SANDBOX_START_WINDOWS,
|
|
1701
|
+
SandboxCheckpointUnusableError,
|
|
1702
|
+
SandboxHostGoneError,
|
|
1703
|
+
SandboxProviderKeyRefusedError,
|
|
1704
|
+
SandboxProviderRateLimitedError,
|
|
1705
|
+
SandboxRuntimeBootError,
|
|
1706
|
+
SandboxStartRefusedError,
|
|
1707
|
+
applySandboxLeaseFailure,
|
|
683
1708
|
applySandboxLeasePatch,
|
|
1709
|
+
applySandboxProvisionedTarget,
|
|
1710
|
+
applySandboxRuntimeSnapshot,
|
|
1711
|
+
boatBilling,
|
|
684
1712
|
brokeredPlaceholderEnv,
|
|
685
1713
|
brokeredSecretPlaceholder,
|
|
686
1714
|
captureSandboxCheckpoint,
|
|
1715
|
+
cloudflareBilling,
|
|
687
1716
|
createSandboxManager,
|
|
1717
|
+
discardSnapshot,
|
|
688
1718
|
hostedSandboxNetworkPolicy,
|
|
1719
|
+
isSandboxCheckpointUnusable,
|
|
1720
|
+
isSandboxCheckpointUnusableCause,
|
|
1721
|
+
isSandboxHostGone,
|
|
689
1722
|
isSandboxListingUnsupported,
|
|
1723
|
+
isSandboxProviderKeyRefused,
|
|
1724
|
+
isSandboxProviderRateLimited,
|
|
1725
|
+
isSandboxRuntimeBootFailure,
|
|
1726
|
+
isSandboxStartRefusal,
|
|
1727
|
+
liftSandboxRetryWait,
|
|
690
1728
|
restoreSandboxCheckpoint,
|
|
1729
|
+
runtimeBootDiagnostic,
|
|
1730
|
+
sandboxCheckpointUnusable,
|
|
1731
|
+
sandboxCheckpointUnusableCauses,
|
|
1732
|
+
sandboxCheckpointUnusableError,
|
|
691
1733
|
sandboxEgressDisposition,
|
|
1734
|
+
sandboxHostGoneError,
|
|
1735
|
+
sandboxLeaseEndedUnderStart,
|
|
1736
|
+
sandboxLeaseFailure,
|
|
1737
|
+
sandboxLeaseHeldHost,
|
|
1738
|
+
sandboxMachineCostUsd,
|
|
1739
|
+
sandboxMachineRate,
|
|
692
1740
|
sandboxNetworkPolicyWithBrokeredHosts,
|
|
1741
|
+
sandboxProviderFailedError,
|
|
1742
|
+
sandboxProviderKeyRefusedError,
|
|
1743
|
+
sandboxProviderRateLimit,
|
|
1744
|
+
sandboxProviderRateLimitedError,
|
|
1745
|
+
sandboxRuntimeBootFailedError,
|
|
1746
|
+
sandboxRuntimeBootFailure,
|
|
1747
|
+
sandboxRuntimeUnresponsiveError,
|
|
693
1748
|
sandboxSourceHost,
|
|
694
|
-
|
|
1749
|
+
sandboxStartFailureReason,
|
|
1750
|
+
sandboxStartLimitRetryMs,
|
|
1751
|
+
sandboxStopReason,
|
|
1752
|
+
validateSandboxPersistenceCapabilities,
|
|
1753
|
+
workspaceRuntimeBootReason
|
|
695
1754
|
};
|