@moinulmoin/eden-runtime-cloudflare 0.9.0 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/eden-eve-host-worker.mjs +18836 -16582
- package/dist/eve-host-runtime.d.ts +36 -7
- package/dist/eve-host-runtime.js +754 -77
- package/dist/eve-host.d.ts +45 -5
- package/dist/eve-host.js +56 -2
- package/dist/index.d.ts +2 -2
- package/dist/index.js +1 -1
- package/package.json +2 -2
package/dist/eve-host-runtime.js
CHANGED
|
@@ -3,6 +3,85 @@ import { WorldCore } from "@moinulmoin/eden-world-cloudflare/core";
|
|
|
3
3
|
import { EVE_HOST_DEFAULTS, EVE_HOST_SLEEP_SNAPSHOT_MARGIN_MS, EVE_SCHEDULE_WAKE_LEAD_MS, parseEveSleepAfterMs, EveHostError, assertNonEmpty, assertStableOrigin, createEveReadinessGate, createTrustedEveRequest, eveScheduleCronFiresWithin, readProtectedRuntimeVariables, WORKER_NAME_PATTERN, isEveWorkflowInternalRoute, } from "./eve-host.js";
|
|
4
4
|
const EVE_CLOUDFLARE_CONTAINERS_CA_PATH = "/etc/cloudflare/certs/cloudflare-containers-ca.crt";
|
|
5
5
|
const EVE_HOST_STALE_WORKER_REASON = "stale Worker version";
|
|
6
|
+
function deploymentImages(env) {
|
|
7
|
+
const raw = env.EDEN_EVE_DEPLOYMENTS;
|
|
8
|
+
return typeof raw === "string" ? JSON.parse(raw) : {};
|
|
9
|
+
}
|
|
10
|
+
function deploymentInstance(env, deploymentId, base = env.EVE_CONTAINER_INSTANCE_NAME) {
|
|
11
|
+
if (typeof base !== "string")
|
|
12
|
+
throw new Error("The Container instance identity is unavailable.");
|
|
13
|
+
// Existing runs used this identity before affinity was introduced.
|
|
14
|
+
return env.EDEN_EVE_DEPLOYMENTS === undefined || deploymentId === "eden-cloudflare" ? base : `${base}:${deploymentId}`;
|
|
15
|
+
}
|
|
16
|
+
function deploymentRequest(request, deploymentId) {
|
|
17
|
+
const headers = new Headers(request.headers);
|
|
18
|
+
headers.set("x-eden-eve-deployment-id", deploymentId);
|
|
19
|
+
return new Request(request, { headers });
|
|
20
|
+
}
|
|
21
|
+
async function sandboxSessionDirectory(sessionId) {
|
|
22
|
+
// Eve 0.91.0 and 1.0.0-rc.3 just-bash both use provider identity version 2.
|
|
23
|
+
// Sorting these keys produces ordinary compact JSON. The adjacent braces
|
|
24
|
+
// in Eve's template literal close the interpolation and object separately.
|
|
25
|
+
const identity = JSON.stringify({ sessionId, version: 2 });
|
|
26
|
+
const digest = await crypto.subtle.digest("SHA-256", new TextEncoder().encode(identity));
|
|
27
|
+
return Array.from(new Uint8Array(digest), (byte) => byte.toString(16).padStart(2, "0")).join("").slice(0, 24);
|
|
28
|
+
}
|
|
29
|
+
const EVE_HOST_IMPORT_SANDBOXES = String.raw `
|
|
30
|
+
const fs = require("node:fs/promises");
|
|
31
|
+
const path = require("node:path");
|
|
32
|
+
const crypto = require("node:crypto");
|
|
33
|
+
const { createReadStream } = require("node:fs");
|
|
34
|
+
async function digest(file) {
|
|
35
|
+
const hash = crypto.createHash("sha256");
|
|
36
|
+
for await (const chunk of createReadStream(file)) hash.update(chunk);
|
|
37
|
+
return hash.digest("hex");
|
|
38
|
+
}
|
|
39
|
+
async function equal(a, b) {
|
|
40
|
+
const [x, y] = await Promise.all([fs.lstat(a), fs.lstat(b)]);
|
|
41
|
+
if (x.isSymbolicLink() || y.isSymbolicLink()) {
|
|
42
|
+
return x.isSymbolicLink() && y.isSymbolicLink() && await fs.readlink(a) === await fs.readlink(b);
|
|
43
|
+
}
|
|
44
|
+
if (x.isDirectory() && y.isDirectory()) {
|
|
45
|
+
const [left, right] = await Promise.all([fs.readdir(a), fs.readdir(b)]);
|
|
46
|
+
left.sort(); right.sort();
|
|
47
|
+
if (left.length !== right.length || left.some((name, i) => name !== right[i])) return false;
|
|
48
|
+
for (const name of left) if (!await equal(path.join(a, name), path.join(b, name))) return false;
|
|
49
|
+
return true;
|
|
50
|
+
}
|
|
51
|
+
return x.isFile() && y.isFile() && x.size === y.size && await digest(a) === await digest(b);
|
|
52
|
+
}
|
|
53
|
+
async function publish(from, to) {
|
|
54
|
+
// Both paths are on the destination workspace filesystem. Linux performs
|
|
55
|
+
// one atomic directory publication; an existing destination is never replaced.
|
|
56
|
+
const script = "import ctypes,os,sys;libc=ctypes.CDLL(None,use_errno=True);result=libc.renameat2(-100,os.fsencode(sys.argv[1]),-100,os.fsencode(sys.argv[2]),1);err=ctypes.get_errno();sys.exit(0) if result==0 else (_ for _ in ()).throw(OSError(err,os.strerror(err)))";
|
|
57
|
+
const { spawnSync } = require("node:child_process");
|
|
58
|
+
const result = spawnSync("python3", ["-c", script, from, to], { encoding: "utf8" });
|
|
59
|
+
if (result.error) throw result.error;
|
|
60
|
+
if (result.status !== 0) {
|
|
61
|
+
// A racing destination creation is a conflict unless its complete tree
|
|
62
|
+
// is identical. Never merge into or overwrite a live session directory.
|
|
63
|
+
if (!await equal(from, to)) throw new Error(result.stderr || "Atomic sandbox publication failed");
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
(async () => {
|
|
67
|
+
const [stage, root] = process.argv.slice(1);
|
|
68
|
+
const source = path.join(stage, "sessions");
|
|
69
|
+
await fs.mkdir(path.join(root, "sessions"), { recursive: true });
|
|
70
|
+
const pending = [];
|
|
71
|
+
for (const name of await fs.readdir(source)) {
|
|
72
|
+
const from = path.join(source, name);
|
|
73
|
+
const to = path.join(root, "sessions", name);
|
|
74
|
+
let exists = true;
|
|
75
|
+
try { await fs.lstat(to); } catch (error) { if (error.code !== "ENOENT") throw error; exists = false; }
|
|
76
|
+
if (exists) {
|
|
77
|
+
if (!await equal(from, to)) throw new Error("Sandbox import conflicts with an existing target session: " + name);
|
|
78
|
+
} else pending.push([from, to]);
|
|
79
|
+
}
|
|
80
|
+
// Validate every conflict before publishing. Never replace a directory
|
|
81
|
+
// that may belong to a turn already running on the target.
|
|
82
|
+
for (const [from, to] of pending) await publish(from, to);
|
|
83
|
+
})().catch(error => { console.error(error.message); process.exitCode = 1; });
|
|
84
|
+
`;
|
|
6
85
|
/**
|
|
7
86
|
* Cloudflare's durable_object containers expose `/dev/kvm`, which makes Eve
|
|
8
87
|
* versions before 0.83 pick microsandbox at runtime, although Eden's isolated
|
|
@@ -53,56 +132,98 @@ function isMissingContainerSnapshot(error) {
|
|
|
53
132
|
/Snapshot ".*" was not found/u.test(error.message);
|
|
54
133
|
}
|
|
55
134
|
/** The deployment's single SQLite-backed Workflow World. */
|
|
56
|
-
export class EdenWorldDurableObject {
|
|
135
|
+
export class EdenWorldDurableObject extends DurableObject {
|
|
57
136
|
core;
|
|
58
137
|
constructor(ctx, env) {
|
|
138
|
+
super(ctx, env);
|
|
59
139
|
this.core = new WorldCore({
|
|
140
|
+
...(env.EDEN_EVE_DEPLOYMENT_ID === undefined ? {} : { currentDeploymentId: env.EDEN_EVE_DEPLOYMENT_ID }),
|
|
60
141
|
sql: {
|
|
61
142
|
exec: ctx.storage.sql.exec.bind(ctx.storage.sql),
|
|
62
143
|
transactionSync: ctx.storage.transactionSync.bind(ctx.storage),
|
|
63
144
|
},
|
|
145
|
+
waitUntil: (promise) => ctx.waitUntil(promise),
|
|
64
146
|
scheduleAlarm: async (atMs) => {
|
|
65
147
|
if (atMs === null)
|
|
66
148
|
await ctx.storage.deleteAlarm();
|
|
67
149
|
else
|
|
68
150
|
await ctx.storage.setAlarm(atMs);
|
|
69
151
|
},
|
|
152
|
+
retireDeployment: async (deploymentId) => {
|
|
153
|
+
if (deploymentId === env.EDEN_EVE_DEPLOYMENT_ID)
|
|
154
|
+
throw new Error("Cannot retire the current ingress deployment.");
|
|
155
|
+
if (!env.EVE_CONTAINER)
|
|
156
|
+
throw new Error("The Container namespace is unavailable for retirement.");
|
|
157
|
+
if (env.EDEN_EVE_DEPLOYMENTS === undefined)
|
|
158
|
+
throw new Error("Cannot retire a deployment without deployment-aware Container routing.");
|
|
159
|
+
const namespace = env.EVE_CONTAINER;
|
|
160
|
+
await namespace.getByName(deploymentInstance(env, deploymentId)).retire(deploymentId);
|
|
161
|
+
},
|
|
70
162
|
delivery: {
|
|
71
163
|
deliver: async (message) => {
|
|
72
164
|
const origin = env.EVE_PUBLIC_ORIGIN;
|
|
73
|
-
const
|
|
74
|
-
|
|
165
|
+
const deploymentId = env.EDEN_EVE_DEPLOYMENTS === undefined ? env.EDEN_EVE_DEPLOYMENT_ID : message.deploymentId ?? "eden-cloudflare";
|
|
166
|
+
let sandboxTransferCompleted = false;
|
|
167
|
+
const settleUnstarted = async () => {
|
|
168
|
+
const world = env.EDEN_WORLD;
|
|
169
|
+
await world.getByName("world").deliverySettled(message.messageId, message.attempt);
|
|
170
|
+
};
|
|
171
|
+
if (typeof origin !== "string") {
|
|
172
|
+
await settleUnstarted();
|
|
75
173
|
throw new Error("The World queue Container identity is unavailable.");
|
|
76
174
|
}
|
|
77
175
|
const namespace = env.EVE_CONTAINER;
|
|
78
|
-
const
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
"
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
}
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
176
|
+
const target = namespace.getByName(deploymentInstance(env, deploymentId));
|
|
177
|
+
if (message.sourceDeploymentId && message.sourceDeploymentId !== deploymentId && message.sandboxSessionId) {
|
|
178
|
+
try {
|
|
179
|
+
const result = await target.importSandboxSessions(message.sourceDeploymentId, deploymentId, message.messageId, message.sandboxSessionId);
|
|
180
|
+
if (result === "source-export-failed") {
|
|
181
|
+
await settleUnstarted();
|
|
182
|
+
return { ok: false, sandboxTransferFailed: true };
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
catch (error) {
|
|
186
|
+
await settleUnstarted();
|
|
187
|
+
throw error;
|
|
188
|
+
}
|
|
189
|
+
sandboxTransferCompleted = true;
|
|
190
|
+
}
|
|
93
191
|
try {
|
|
94
|
-
|
|
192
|
+
const response = await target.loopbackFetch(deploymentRequest(new Request(new URL(`/.well-known/workflow/v1/${message.path}`, origin), {
|
|
193
|
+
method: "POST",
|
|
194
|
+
headers: {
|
|
195
|
+
...message.headers,
|
|
196
|
+
"content-type": "application/json",
|
|
197
|
+
"x-vqs-queue-name": message.queueName,
|
|
198
|
+
"x-vqs-message-id": message.messageId,
|
|
199
|
+
"x-vqs-message-attempt": String(message.attempt),
|
|
200
|
+
},
|
|
201
|
+
body: message.body,
|
|
202
|
+
}), deploymentId), { messageId: message.messageId, attempt: message.attempt });
|
|
203
|
+
// Body transport errors must remain retryable; an HTTP status alone
|
|
204
|
+
// is not an acknowledgement of a streamed Container response.
|
|
205
|
+
const body = await response.text();
|
|
206
|
+
let result = null;
|
|
207
|
+
try {
|
|
208
|
+
result = JSON.parse(body);
|
|
209
|
+
}
|
|
210
|
+
catch {
|
|
211
|
+
// World-local also accepts successful non-JSON responses.
|
|
212
|
+
}
|
|
213
|
+
const timeout = result?.timeoutSeconds;
|
|
214
|
+
return {
|
|
215
|
+
ok: response.ok && timeout === undefined,
|
|
216
|
+
sandboxTransferCompleted,
|
|
217
|
+
...(typeof timeout === "number" && timeout >= 0
|
|
218
|
+
? { retryAfterMs: timeout * 1_000 }
|
|
219
|
+
: {}),
|
|
220
|
+
};
|
|
95
221
|
}
|
|
96
|
-
catch {
|
|
97
|
-
|
|
222
|
+
catch (error) {
|
|
223
|
+
if (!sandboxTransferCompleted)
|
|
224
|
+
throw error;
|
|
225
|
+
return { ok: false, sandboxTransferCompleted: true };
|
|
98
226
|
}
|
|
99
|
-
const timeout = result?.timeoutSeconds;
|
|
100
|
-
return {
|
|
101
|
-
ok: response.ok && timeout === undefined,
|
|
102
|
-
...(typeof timeout === "number" && timeout >= 0
|
|
103
|
-
? { retryAfterMs: timeout * 1_000 }
|
|
104
|
-
: {}),
|
|
105
|
-
};
|
|
106
227
|
},
|
|
107
228
|
},
|
|
108
229
|
});
|
|
@@ -116,6 +237,17 @@ export class EdenWorldDurableObject {
|
|
|
116
237
|
headers: { "content-type": "application/cbor" },
|
|
117
238
|
});
|
|
118
239
|
}
|
|
240
|
+
deploymentBusy(deploymentId) {
|
|
241
|
+
const activity = this.core.deploymentActivity(deploymentId);
|
|
242
|
+
return activity.activeDeliveries > 0 || activity.unfinishedTransfers > 0;
|
|
243
|
+
}
|
|
244
|
+
deliverySettled(messageId, attempt) {
|
|
245
|
+
this.core.deliverySettled(messageId, attempt);
|
|
246
|
+
}
|
|
247
|
+
async controlFetch(request) {
|
|
248
|
+
const result = await this.core.handleControlRpc(new Uint8Array(await request.arrayBuffer()));
|
|
249
|
+
return new Response(result, { headers: { "content-type": "application/cbor" } });
|
|
250
|
+
}
|
|
119
251
|
async alarm() {
|
|
120
252
|
await this.core.runAlarm();
|
|
121
253
|
}
|
|
@@ -128,13 +260,15 @@ export class EdenWorldDurableObject {
|
|
|
128
260
|
* URLs stay correct; the DO registers an outbound-HTTPS intercept for that
|
|
129
261
|
* hostname aimed at `EveHostLoopback`, and this function routes the request:
|
|
130
262
|
* non-public hosts egress plainly, `/__eden/world/rpc` reaches the World
|
|
131
|
-
* Durable Object, and
|
|
132
|
-
* internal port
|
|
133
|
-
* unauthenticated queue
|
|
263
|
+
* Durable Object, and local World queue deliveries re-enter the container
|
|
264
|
+
* over its internal port. Cloudflare World queue delivery uses direct DO RPC,
|
|
265
|
+
* so its unauthenticated outbound queue requests remain forbidden.
|
|
134
266
|
*/
|
|
135
|
-
export async function routeEveOutboundRequest(request, env) {
|
|
267
|
+
export async function routeEveOutboundRequest(request, env, sourceDeploymentId, expectedWorldToken) {
|
|
136
268
|
const publicOrigin = env.EVE_PUBLIC_ORIGIN;
|
|
137
|
-
const
|
|
269
|
+
const deploymentId = sourceDeploymentId ?? "eden-cloudflare";
|
|
270
|
+
const instanceName = typeof deploymentId === "string"
|
|
271
|
+
? deploymentInstance(env, deploymentId) : undefined;
|
|
138
272
|
if (typeof publicOrigin !== "string" ||
|
|
139
273
|
typeof instanceName !== "string") {
|
|
140
274
|
throw new EveHostError("HOST_ORIGIN_UNAVAILABLE", "The container loopback identity is unavailable to the outbound handler.");
|
|
@@ -146,16 +280,32 @@ export async function routeEveOutboundRequest(request, env) {
|
|
|
146
280
|
return fetch(request);
|
|
147
281
|
}
|
|
148
282
|
if (new URL(request.url).pathname === "/__eden/world/rpc") {
|
|
283
|
+
const route = deploymentImages(env)[deploymentId];
|
|
284
|
+
const authRequired = route?.worldRpcAuth === true ||
|
|
285
|
+
(env.EDEN_EVE_DEPLOYMENTS === undefined && env.EDEN_EVE_WORLD_RPC_AUTH === true);
|
|
286
|
+
if (authRequired) {
|
|
287
|
+
const supplied = request.headers.get("x-eden-world-token") ?? "";
|
|
288
|
+
const encoder = new TextEncoder();
|
|
289
|
+
const expected = encoder.encode(expectedWorldToken ?? "");
|
|
290
|
+
const actual = encoder.encode(supplied);
|
|
291
|
+
const subtle = crypto.subtle;
|
|
292
|
+
if (expected.length === 0 || actual.length !== expected.length || !subtle.timingSafeEqual(actual, expected)) {
|
|
293
|
+
return new Response(null, { status: 403 });
|
|
294
|
+
}
|
|
295
|
+
}
|
|
149
296
|
const world = env.EDEN_WORLD;
|
|
150
297
|
if (world === undefined)
|
|
151
298
|
return new Response(null, { status: 404 });
|
|
152
299
|
return world.get(world.idFromName("world")).fetch(request);
|
|
153
300
|
}
|
|
301
|
+
if (env.EDEN_EVE_WORLD_CLOUDFLARE && isEveWorkflowInternalRoute(new URL(request.url).pathname)) {
|
|
302
|
+
return new Response(null, { status: 403 });
|
|
303
|
+
}
|
|
154
304
|
const namespace = env.EVE_CONTAINER;
|
|
155
305
|
if (namespace === undefined) {
|
|
156
306
|
throw new EveHostError("HOST_READINESS_UNPROVEN", "The configured Container binding is unavailable to the outbound handler.");
|
|
157
307
|
}
|
|
158
|
-
return namespace.getByName(instanceName).loopbackFetch(request);
|
|
308
|
+
return namespace.getByName(instanceName).loopbackFetch(deploymentRequest(request, deploymentId));
|
|
159
309
|
}
|
|
160
310
|
/**
|
|
161
311
|
* This module is the only Eve-host surface that depends on provider runtime
|
|
@@ -171,7 +321,8 @@ export async function routeEveOutboundRequest(request, env) {
|
|
|
171
321
|
*/
|
|
172
322
|
export class EveHostLoopback extends WorkerEntrypoint {
|
|
173
323
|
async fetch(request) {
|
|
174
|
-
|
|
324
|
+
const props = this.ctx.props;
|
|
325
|
+
return routeEveOutboundRequest(request, this.env, props.deploymentId, props.expectedWorldToken);
|
|
175
326
|
}
|
|
176
327
|
}
|
|
177
328
|
/**
|
|
@@ -204,6 +355,136 @@ export class EveHostDurableContainer extends DurableObject {
|
|
|
204
355
|
envVars;
|
|
205
356
|
sleepAfterMs;
|
|
206
357
|
readiness;
|
|
358
|
+
deploymentId;
|
|
359
|
+
imports = new Map();
|
|
360
|
+
importQueue = Promise.resolve();
|
|
361
|
+
activeExports = 0;
|
|
362
|
+
activeDeliveries = 0;
|
|
363
|
+
activeForwarding = 0;
|
|
364
|
+
schedulerRoleReconcile;
|
|
365
|
+
retainedFences;
|
|
366
|
+
deliveryClaims = new Map();
|
|
367
|
+
async identifyDeployment(requested) {
|
|
368
|
+
const affinity = this.env.EDEN_EVE_DEPLOYMENTS !== undefined;
|
|
369
|
+
let stored = affinity ? this.deploymentId ?? await this.ctx.storage.get("eden.eve.host.deployment-id") : undefined;
|
|
370
|
+
const namespace = this.env.EVE_CONTAINER;
|
|
371
|
+
const legacy = affinity && this.ctx.id.equals(namespace.idFromName(this.env.EVE_CONTAINER_INSTANCE_NAME));
|
|
372
|
+
if (legacy && stored !== undefined && stored !== "eden-cloudflare" && deploymentImages(this.env)[stored] === undefined)
|
|
373
|
+
stored = undefined;
|
|
374
|
+
const named = affinity && !legacy
|
|
375
|
+
? Object.keys(deploymentImages(this.env)).find((id) => this.ctx.id.equals(namespace.idFromName(deploymentInstance(this.env, id))))
|
|
376
|
+
: undefined;
|
|
377
|
+
const owner = affinity ? (legacy ? "eden-cloudflare" : named) : this.env.EDEN_EVE_DEPLOYMENT_ID;
|
|
378
|
+
const id = affinity ? requested ?? stored ?? owner : owner;
|
|
379
|
+
if (affinity && (owner === undefined || id !== owner)) {
|
|
380
|
+
throw new Error("The Container deployment identity does not match its instance name.");
|
|
381
|
+
}
|
|
382
|
+
if (!id || (stored !== undefined && stored !== id)) {
|
|
383
|
+
throw new Error("The Container deployment identity does not match its persisted owner.");
|
|
384
|
+
}
|
|
385
|
+
const record = deploymentImages(this.env)[id];
|
|
386
|
+
if (stored === undefined)
|
|
387
|
+
await this.ctx.storage.put("eden.eve.host.deployment-id", id);
|
|
388
|
+
this.deploymentId = id;
|
|
389
|
+
this.envVars.EDEN_EVE_DEPLOYMENT_ID = id;
|
|
390
|
+
this.envVars.EDEN_EVE_RELEASE_ID = record?.releaseId ?? this.env.EDEN_EVE_RELEASE_ID ?? id;
|
|
391
|
+
this.envVars.EVE_SCHEDULES_DISABLED = id === this.env.EDEN_EVE_DEPLOYMENT_ID ? "false" : "true";
|
|
392
|
+
this.envVars.EDEN_EVE_GENERATION_ID = record?.generationId ?? this.env.EDEN_EVE_GENERATION_ID;
|
|
393
|
+
if (record?.runtimeRevisionHandle !== undefined) {
|
|
394
|
+
this.envVars.EDEN_EVE_RUNTIME_REVISION = record.runtimeRevisionHandle;
|
|
395
|
+
}
|
|
396
|
+
else {
|
|
397
|
+
delete this.envVars.EDEN_EVE_RUNTIME_REVISION;
|
|
398
|
+
}
|
|
399
|
+
return id;
|
|
400
|
+
}
|
|
401
|
+
async fenceRetainedSchedules() {
|
|
402
|
+
if (this.deploymentId !== this.env.EDEN_EVE_DEPLOYMENT_ID)
|
|
403
|
+
return;
|
|
404
|
+
this.retainedFences ??= (async () => {
|
|
405
|
+
const namespace = this.env.EVE_CONTAINER;
|
|
406
|
+
for (const id of Object.keys(deploymentImages(this.env))) {
|
|
407
|
+
if (id === this.deploymentId)
|
|
408
|
+
continue;
|
|
409
|
+
const request = new Request(new URL("/__eden/internal/fence-schedules", this.env.EVE_PUBLIC_ORIGIN), {
|
|
410
|
+
headers: {
|
|
411
|
+
"x-eden-eve-deployment-id": id,
|
|
412
|
+
"x-eden-eve-release-id": this.env.EDEN_EVE_RELEASE_ID ?? this.env.EDEN_EVE_DEPLOYMENT_ID,
|
|
413
|
+
},
|
|
414
|
+
});
|
|
415
|
+
let response;
|
|
416
|
+
try {
|
|
417
|
+
response = await namespace.getByName(deploymentInstance(this.env, id)).fetch(request);
|
|
418
|
+
}
|
|
419
|
+
catch (error) {
|
|
420
|
+
if (!(error instanceof Error) || !error.message.includes(EVE_HOST_STALE_WORKER_REASON))
|
|
421
|
+
throw error;
|
|
422
|
+
response = await namespace.getByName(deploymentInstance(this.env, id)).fetch(request);
|
|
423
|
+
}
|
|
424
|
+
if (!response.ok)
|
|
425
|
+
throw new Error(`Cannot fence schedules on retained deployment ${id}.`);
|
|
426
|
+
await response.body?.cancel();
|
|
427
|
+
}
|
|
428
|
+
})().catch((error) => { this.retainedFences = undefined; throw error; });
|
|
429
|
+
await this.retainedFences;
|
|
430
|
+
}
|
|
431
|
+
async deploymentBusy(deploymentId) {
|
|
432
|
+
if (this.activeDeliveries > 0 || this.activeExports > 0 || this.imports.size > 0)
|
|
433
|
+
return true;
|
|
434
|
+
if (!this.env.EDEN_EVE_WORLD_CLOUDFLARE)
|
|
435
|
+
return false;
|
|
436
|
+
const world = this.env.EDEN_WORLD;
|
|
437
|
+
return world.getByName("world").deploymentBusy(deploymentId);
|
|
438
|
+
}
|
|
439
|
+
async reconcileSchedulerRole() {
|
|
440
|
+
const runtime = this.containerRuntime();
|
|
441
|
+
if (this.deploymentId !== this.env.EDEN_EVE_DEPLOYMENT_ID || !runtime?.running)
|
|
442
|
+
return;
|
|
443
|
+
if (await this.ctx.storage.get("eden.eve.host.started-schedules-disabled") !== true)
|
|
444
|
+
return;
|
|
445
|
+
this.schedulerRoleReconcile ??= (async () => {
|
|
446
|
+
await this.ctx.storage.put("eden.eve.host.scheduler-fence-pending", true);
|
|
447
|
+
const canRestart = async () => {
|
|
448
|
+
if (this.activeForwarding > 0 || this.activeExports > 0 || this.imports.size > 0)
|
|
449
|
+
return false;
|
|
450
|
+
if (!this.env.EDEN_EVE_WORLD_CLOUDFLARE)
|
|
451
|
+
return true;
|
|
452
|
+
const world = this.env.EDEN_WORLD;
|
|
453
|
+
return !await world.getByName("world").deploymentBusy(this.deploymentId);
|
|
454
|
+
};
|
|
455
|
+
if (!await canRestart()) {
|
|
456
|
+
throw new EveHostError("HOST_READINESS_UNPROVEN", "Scheduler promotion awaits active retained deployment work.");
|
|
457
|
+
}
|
|
458
|
+
const snapshot = await runtime.snapshotContainer({ name: "eden-workspace" });
|
|
459
|
+
const image = (await runtime.inspect().catch(() => null))?.image ||
|
|
460
|
+
await this.ctx.storage.get(EVE_HOST_CONTAINER_STARTED_IMAGE_KEY) || this.expectedImage(runtime);
|
|
461
|
+
if (!image)
|
|
462
|
+
throw new Error("Cannot identify the scheduler promotion snapshot image.");
|
|
463
|
+
await this.ctx.storage.put(EVE_HOST_CONTAINER_SNAPSHOT_KEY, { id: snapshot.id, image, at: Date.now() });
|
|
464
|
+
if (!await canRestart()) {
|
|
465
|
+
throw new EveHostError("HOST_READINESS_UNPROVEN", "Scheduler promotion awaits active retained deployment work.");
|
|
466
|
+
}
|
|
467
|
+
await runtime.destroy(new Error("scheduler-role-promotion"));
|
|
468
|
+
const stoppedAt = Date.now();
|
|
469
|
+
while (runtime.running && Date.now() - stoppedAt < 10_000) {
|
|
470
|
+
await new Promise((resolve) => setTimeout(resolve, 25));
|
|
471
|
+
}
|
|
472
|
+
if (runtime.running) {
|
|
473
|
+
throw new EveHostError("HOST_READINESS_UNPROVEN", "The previous scheduler process has not stopped.");
|
|
474
|
+
}
|
|
475
|
+
this.recordContainerStopped();
|
|
476
|
+
})().finally(() => { this.schedulerRoleReconcile = undefined; });
|
|
477
|
+
await this.schedulerRoleReconcile;
|
|
478
|
+
}
|
|
479
|
+
expectedImage(runtime) {
|
|
480
|
+
const id = this.deploymentId;
|
|
481
|
+
if (id === undefined)
|
|
482
|
+
throw new Error("Container deployment identity has not been resolved.");
|
|
483
|
+
const record = deploymentImages(this.env)[id];
|
|
484
|
+
if (this.env.EDEN_EVE_DEPLOYMENTS !== undefined && record === undefined && id !== "eden-cloudflare")
|
|
485
|
+
throw new Error(`Deployment ${id} has no retained Container image.`);
|
|
486
|
+
return runtime.images[record?.image ?? "eve"];
|
|
487
|
+
}
|
|
207
488
|
constructor(ctx, env) {
|
|
208
489
|
super(ctx, env);
|
|
209
490
|
const publicOrigin = env.EVE_PUBLIC_ORIGIN;
|
|
@@ -261,6 +542,10 @@ export class EveHostDurableContainer extends DurableObject {
|
|
|
261
542
|
// DO restarts lose the inactivity timeout; the monitor for this
|
|
262
543
|
// container generation is also gone, so both are re-attached here.
|
|
263
544
|
this.ctx.blockConcurrencyWhile(async () => {
|
|
545
|
+
if (env.EDEN_EVE_DEPLOYMENTS !== undefined) {
|
|
546
|
+
await this.identifyDeployment();
|
|
547
|
+
await this.registerLoopbackIntercept(runtime);
|
|
548
|
+
}
|
|
264
549
|
await runtime.setInactivityTimeout(this.inactivityTimeoutMs);
|
|
265
550
|
this.attachMonitor(runtime);
|
|
266
551
|
});
|
|
@@ -278,10 +563,15 @@ export class EveHostDurableContainer extends DurableObject {
|
|
|
278
563
|
if (this.monitorAttached)
|
|
279
564
|
return;
|
|
280
565
|
this.monitorAttached = true;
|
|
566
|
+
const generation = this.readinessEpoch;
|
|
281
567
|
runtime.monitor().then(() => {
|
|
568
|
+
if (generation !== this.readinessEpoch)
|
|
569
|
+
return;
|
|
282
570
|
this.monitorAttached = false;
|
|
283
571
|
this.recordContainerStopped();
|
|
284
572
|
}, (error) => {
|
|
573
|
+
if (generation !== this.readinessEpoch)
|
|
574
|
+
return;
|
|
285
575
|
this.monitorAttached = false;
|
|
286
576
|
if (this.startedFromSnapshot && isMissingContainerSnapshot(error)) {
|
|
287
577
|
// Cloudflare deletes a snapshot 30 days after it was created or
|
|
@@ -298,6 +588,14 @@ export class EveHostDurableContainer extends DurableObject {
|
|
|
298
588
|
}
|
|
299
589
|
recordContainerStopped() {
|
|
300
590
|
this.readinessEpoch += 1;
|
|
591
|
+
this.monitorAttached = false;
|
|
592
|
+
for (const [claim, pending] of this.deliveryClaims) {
|
|
593
|
+
if (pending.generation !== this.readinessEpoch - 1)
|
|
594
|
+
continue;
|
|
595
|
+
this.deliveryClaims.delete(claim);
|
|
596
|
+
pending.fail(new Error("The Container generation stopped before its queue delivery completed."));
|
|
597
|
+
this.ctx.waitUntil(this.settleQueueDelivery(claim));
|
|
598
|
+
}
|
|
301
599
|
this.readinessEvidence = undefined;
|
|
302
600
|
this.imageVerified = false;
|
|
303
601
|
this.pendingCarryRestore = false;
|
|
@@ -310,7 +608,10 @@ export class EveHostDurableContainer extends DurableObject {
|
|
|
310
608
|
const entryFactory = exportsMap?.["EveHostLoopback"];
|
|
311
609
|
if (typeof entryFactory !== "function")
|
|
312
610
|
return;
|
|
313
|
-
const entrypoint = entryFactory({ props: {
|
|
611
|
+
const entrypoint = entryFactory({ props: {
|
|
612
|
+
deploymentId: await this.identifyDeployment(),
|
|
613
|
+
expectedWorldToken: await this.ctx.storage.get("eden.eve.host.world-token") ?? "",
|
|
614
|
+
} });
|
|
314
615
|
await runtime.interceptOutboundHttps(new URL(this.env.EVE_PUBLIC_ORIGIN ?? "").hostname, entrypoint);
|
|
315
616
|
}
|
|
316
617
|
/**
|
|
@@ -321,7 +622,8 @@ export class EveHostDurableContainer extends DurableObject {
|
|
|
321
622
|
async touchActivity() {
|
|
322
623
|
const now = Date.now();
|
|
323
624
|
await this.ctx.storage.put(EVE_HOST_CONTAINER_ACTIVITY_KEY, now);
|
|
324
|
-
await this.ctx.storage.
|
|
625
|
+
const fencing = await this.ctx.storage.get("eden.eve.host.scheduler-fence-pending") === true;
|
|
626
|
+
await this.ctx.storage.setAlarm(fencing ? now : now + this.sleepAfterMs);
|
|
325
627
|
}
|
|
326
628
|
/**
|
|
327
629
|
* Starts the container when it is not running. A stored workspace snapshot
|
|
@@ -337,7 +639,7 @@ export class EveHostDurableContainer extends DurableObject {
|
|
|
337
639
|
}
|
|
338
640
|
if (runtime.running)
|
|
339
641
|
return;
|
|
340
|
-
const expectedImage = runtime
|
|
642
|
+
const expectedImage = this.expectedImage(runtime);
|
|
341
643
|
if (expectedImage === undefined) {
|
|
342
644
|
throw new EveHostError("HOST_READINESS_UNPROVEN", "The configured Container image is unavailable to this Worker version.");
|
|
343
645
|
}
|
|
@@ -346,6 +648,18 @@ export class EveHostDurableContainer extends DurableObject {
|
|
|
346
648
|
storedSnapshot.image === expectedImage
|
|
347
649
|
? storedSnapshot.id
|
|
348
650
|
: undefined;
|
|
651
|
+
// Persist the fresh-start restore obligation before the process exists:
|
|
652
|
+
// a DO replacement must not mistake a partially restored process for ready.
|
|
653
|
+
if (snapshotId === undefined) {
|
|
654
|
+
await this.ctx.storage.put("eden.eve.host.restore-pending", globalThis.crypto.randomUUID());
|
|
655
|
+
}
|
|
656
|
+
else {
|
|
657
|
+
await this.ctx.storage.delete("eden.eve.host.restore-pending");
|
|
658
|
+
}
|
|
659
|
+
await this.ctx.storage.put("eden.eve.host.started-schedules-disabled", this.envVars.EVE_SCHEDULES_DISABLED === "true");
|
|
660
|
+
const worldToken = globalThis.crypto.randomUUID();
|
|
661
|
+
await this.ctx.storage.put("eden.eve.host.world-token", worldToken);
|
|
662
|
+
this.envVars.EDEN_WORLD_TOKEN = worldToken;
|
|
349
663
|
runtime.start({
|
|
350
664
|
...(snapshotId === undefined
|
|
351
665
|
? { image: expectedImage }
|
|
@@ -355,12 +669,19 @@ export class EveHostDurableContainer extends DurableObject {
|
|
|
355
669
|
env: this.envVars,
|
|
356
670
|
entrypoint: EVE_HOST_ENTRYPOINT,
|
|
357
671
|
});
|
|
672
|
+
console.log("eden.container.start", JSON.stringify({
|
|
673
|
+
deploymentId: this.deploymentId, image: expectedImage,
|
|
674
|
+
source: snapshotId === undefined ? "image" : "snapshot",
|
|
675
|
+
}));
|
|
358
676
|
// Set before any await: a failure below makes the readiness retry see a
|
|
359
677
|
// running container and skip this method, so the restore must already be
|
|
360
678
|
// pending. A start from the configured image (an update) has no sandbox
|
|
361
679
|
// sessions yet; a same-image snapshot wake already contains them.
|
|
362
680
|
this.pendingCarryRestore = snapshotId === undefined;
|
|
363
681
|
this.startedFromSnapshot = snapshotId !== undefined;
|
|
682
|
+
// A fresh process receives the retained scheduler fence even when its
|
|
683
|
+
// filesystem came from a snapshot of the formerly-current deployment.
|
|
684
|
+
await this.ctx.storage.delete("eden.eve.host.scheduler-fence-pending");
|
|
364
685
|
// Both start sources run `expectedImage` (a snapshot is only restored
|
|
365
686
|
// from that image).
|
|
366
687
|
await this.ctx.storage.put(EVE_HOST_CONTAINER_STARTED_IMAGE_KEY, expectedImage);
|
|
@@ -383,7 +704,7 @@ export class EveHostDurableContainer extends DurableObject {
|
|
|
383
704
|
const runtime = this.containerRuntime();
|
|
384
705
|
if (runtime === undefined || !runtime.running)
|
|
385
706
|
return;
|
|
386
|
-
const expectedImage = runtime
|
|
707
|
+
const expectedImage = this.expectedImage(runtime);
|
|
387
708
|
let info;
|
|
388
709
|
try {
|
|
389
710
|
info = await runtime.inspect();
|
|
@@ -422,12 +743,12 @@ export class EveHostDurableContainer extends DurableObject {
|
|
|
422
743
|
* image. One carry runs at a time on this Durable Object; a failure never
|
|
423
744
|
* blocks the destroy that follows and leaves the previous carry intact.
|
|
424
745
|
*/
|
|
425
|
-
carrySandboxSessionsOut(runtime) {
|
|
426
|
-
const run = this.carryOutQueue.then(() => this.carrySandboxSessionsOutNow(runtime));
|
|
746
|
+
carrySandboxSessionsOut(runtime, required = false) {
|
|
747
|
+
const run = this.carryOutQueue.then(() => this.carrySandboxSessionsOutNow(runtime, required));
|
|
427
748
|
this.carryOutQueue = run.then(() => undefined, () => undefined);
|
|
428
749
|
return run;
|
|
429
750
|
}
|
|
430
|
-
async carrySandboxSessionsOutNow(runtime) {
|
|
751
|
+
async carrySandboxSessionsOutNow(runtime, required = false) {
|
|
431
752
|
const previous = await this.ctx.storage.get(EVE_HOST_SANDBOX_CARRY_KEY);
|
|
432
753
|
const carryId = globalThis.crypto.randomUUID();
|
|
433
754
|
const chunkPrefix = `${EVE_HOST_SANDBOX_CARRY_CHUNK_PREFIX}.${carryId}.`;
|
|
@@ -501,6 +822,8 @@ export class EveHostDurableContainer extends DurableObject {
|
|
|
501
822
|
await reader.cancel().catch(() => undefined);
|
|
502
823
|
if (overflowed) {
|
|
503
824
|
await discardWrittenChunks();
|
|
825
|
+
if (required)
|
|
826
|
+
throw new Error("The imported sandbox carry exceeds the archive limit.");
|
|
504
827
|
console.error(`The Eve sandbox session carry exceeded ${EVE_HOST_CARRY_LIMITS.maxBytes} compressed bytes; keeping the previous carry.`);
|
|
505
828
|
return;
|
|
506
829
|
}
|
|
@@ -519,6 +842,8 @@ export class EveHostDurableContainer extends DurableObject {
|
|
|
519
842
|
}
|
|
520
843
|
if (exitCode !== 0) {
|
|
521
844
|
await discardWrittenChunks();
|
|
845
|
+
if (required)
|
|
846
|
+
throw new Error(`The imported sandbox carry failed with exit code ${exitCode}.`);
|
|
522
847
|
console.error(`The Eve sandbox session carry failed with exit code ${exitCode}; keeping the previous carry.`);
|
|
523
848
|
return;
|
|
524
849
|
}
|
|
@@ -545,10 +870,22 @@ export class EveHostDurableContainer extends DurableObject {
|
|
|
545
870
|
* the same promise, and no request is forwarded before it settles.
|
|
546
871
|
*/
|
|
547
872
|
async restoreCarriedSandboxSessions() {
|
|
548
|
-
|
|
873
|
+
const restoreGeneration = await this.ctx.storage.get("eden.eve.host.restore-pending");
|
|
874
|
+
if (!this.pendingCarryRestore && restoreGeneration === undefined)
|
|
549
875
|
return;
|
|
550
|
-
|
|
551
|
-
|
|
876
|
+
const required = await this.ctx.storage.get("eden.eve.host.has-imported-sandboxes") === true;
|
|
877
|
+
this.carryRestorePromise ??= this.carrySandboxSessionsIn(required);
|
|
878
|
+
try {
|
|
879
|
+
await this.carryRestorePromise;
|
|
880
|
+
}
|
|
881
|
+
catch (error) {
|
|
882
|
+
this.carryRestorePromise = undefined;
|
|
883
|
+
throw error;
|
|
884
|
+
}
|
|
885
|
+
if (restoreGeneration !== undefined &&
|
|
886
|
+
await this.ctx.storage.get("eden.eve.host.restore-pending") === restoreGeneration) {
|
|
887
|
+
await this.ctx.storage.delete("eden.eve.host.restore-pending");
|
|
888
|
+
}
|
|
552
889
|
this.pendingCarryRestore = false;
|
|
553
890
|
}
|
|
554
891
|
/**
|
|
@@ -558,11 +895,14 @@ export class EveHostDurableContainer extends DurableObject {
|
|
|
558
895
|
* failure is logged and swallowed: the agent still serves, only the
|
|
559
896
|
* carried files are missing.
|
|
560
897
|
*/
|
|
561
|
-
async carrySandboxSessionsIn() {
|
|
898
|
+
async carrySandboxSessionsIn(required = false) {
|
|
562
899
|
try {
|
|
563
900
|
const carry = await this.ctx.storage.get(EVE_HOST_SANDBOX_CARRY_KEY);
|
|
564
|
-
if (carry === undefined)
|
|
901
|
+
if (carry === undefined) {
|
|
902
|
+
if (required)
|
|
903
|
+
throw new Error("The imported sandbox carry is missing.");
|
|
565
904
|
return;
|
|
905
|
+
}
|
|
566
906
|
let nextChunk = 0;
|
|
567
907
|
const archive = new ReadableStream({
|
|
568
908
|
pull: async (controller) => {
|
|
@@ -584,10 +924,15 @@ export class EveHostDurableContainer extends DurableObject {
|
|
|
584
924
|
if (runtime === undefined) {
|
|
585
925
|
throw new Error("the Container runtime is unavailable");
|
|
586
926
|
}
|
|
927
|
+
const restoreGeneration = await this.ctx.storage.get("eden.eve.host.restore-pending");
|
|
928
|
+
const stageId = await sandboxSessionDirectory(restoreGeneration ?? carry.id);
|
|
929
|
+
const stage = `${EVE_HOST_SANDBOX_CARRY_SOURCE}/.eden-restores/${stageId}`;
|
|
587
930
|
const exec = await runtime.exec([
|
|
588
931
|
"sh",
|
|
589
932
|
"-c",
|
|
590
|
-
|
|
933
|
+
required
|
|
934
|
+
? `rm -rf ${stage} && mkdir -p ${stage} && tar -xzf - -C ${stage}`
|
|
935
|
+
: `mkdir -p ${EVE_HOST_SANDBOX_CARRY_SOURCE} && tar -xzf - -C ${EVE_HOST_SANDBOX_CARRY_SOURCE}`,
|
|
591
936
|
], { stdin: archive, stderr: "pipe" });
|
|
592
937
|
let exitCode = -1;
|
|
593
938
|
try {
|
|
@@ -596,15 +941,30 @@ export class EveHostDurableContainer extends DurableObject {
|
|
|
596
941
|
catch {
|
|
597
942
|
exitCode = -1;
|
|
598
943
|
}
|
|
599
|
-
if (exitCode !== 0)
|
|
600
|
-
|
|
944
|
+
if (exitCode !== 0)
|
|
945
|
+
throw new Error(`The sandbox restore failed with exit code ${exitCode}.`);
|
|
946
|
+
if (required) {
|
|
947
|
+
const publish = await runtime.exec(["node", "-e", EVE_HOST_IMPORT_SANDBOXES, stage, EVE_HOST_SANDBOX_CARRY_SOURCE], { stdout: "ignore", stderr: "pipe" });
|
|
948
|
+
const error = publish.stderr === null ? "" : await new Response(publish.stderr).text();
|
|
949
|
+
if (await publish.exitCode !== 0)
|
|
950
|
+
throw new Error(`The imported sandbox restore publication failed: ${error}`);
|
|
951
|
+
const cleanup = await runtime.exec(["rm", "-rf", stage], { stdout: "ignore", stderr: "ignore" });
|
|
952
|
+
await cleanup.exitCode;
|
|
601
953
|
}
|
|
954
|
+
await this.ctx.storage.delete("eden.eve.host.sandbox-restore-failed");
|
|
602
955
|
}
|
|
603
956
|
catch (error) {
|
|
957
|
+
await this.ctx.storage.put("eden.eve.host.sandbox-restore-failed", true);
|
|
958
|
+
if (required)
|
|
959
|
+
throw error;
|
|
604
960
|
console.error("The Eve sandbox session restore failed; the agent serves without the carried files:", error instanceof Error ? error.message : "unknown");
|
|
605
961
|
}
|
|
606
962
|
}
|
|
607
|
-
async ensureEveReady(signal = new AbortController().signal) {
|
|
963
|
+
async ensureEveReady(signal = new AbortController().signal, requireSchedulerRole = true) {
|
|
964
|
+
await this.identifyDeployment();
|
|
965
|
+
await this.fenceRetainedSchedules();
|
|
966
|
+
if (requireSchedulerRole)
|
|
967
|
+
await this.reconcileSchedulerRole();
|
|
608
968
|
await this.reconcileRunningImage();
|
|
609
969
|
if (this.readinessEvidence !== undefined) {
|
|
610
970
|
return this.readinessEvidence;
|
|
@@ -626,13 +986,18 @@ export class EveHostDurableContainer extends DurableObject {
|
|
|
626
986
|
}
|
|
627
987
|
async fetch(request) {
|
|
628
988
|
const requestDeploymentId = request.headers.get("x-eden-eve-deployment-id") ?? undefined;
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
// A live container/monitor can pin the DO to an older Worker version.
|
|
632
|
-
// Reset the DO, not its container or storage; the retry's current code
|
|
633
|
-
// reconciles the image and carries sandbox sessions through the update.
|
|
989
|
+
const requestedRelease = request.headers.get("x-eden-eve-release-id") ?? requestDeploymentId;
|
|
990
|
+
if (requestedRelease !== undefined && requestedRelease !== (this.env.EDEN_EVE_RELEASE_ID ?? this.env.EDEN_EVE_DEPLOYMENT_ID)) {
|
|
634
991
|
this.ctx.abort(EVE_HOST_STALE_WORKER_REASON);
|
|
635
992
|
}
|
|
993
|
+
await this.identifyDeployment(requestDeploymentId);
|
|
994
|
+
if (new URL(request.url).pathname === "/__eden/internal/fence-schedules") {
|
|
995
|
+
if (this.deploymentId !== this.env.EDEN_EVE_DEPLOYMENT_ID) {
|
|
996
|
+
await this.ctx.storage.put("eden.eve.host.scheduler-fence-pending", true);
|
|
997
|
+
await this.ctx.storage.setAlarm(Date.now());
|
|
998
|
+
}
|
|
999
|
+
return new Response(null, { status: 204 });
|
|
1000
|
+
}
|
|
636
1001
|
const response = await this.forwardToEve(request);
|
|
637
1002
|
// Reporting the verified start back lets deploy health-gate promotion on
|
|
638
1003
|
// the served container actually running this generation's image: the
|
|
@@ -640,11 +1005,11 @@ export class EveHostDurableContainer extends DurableObject {
|
|
|
640
1005
|
// Worker version's configured image and the request names this
|
|
641
1006
|
// deployment.
|
|
642
1007
|
if (requestDeploymentId !== undefined &&
|
|
643
|
-
requestDeploymentId === this.
|
|
1008
|
+
requestDeploymentId === this.deploymentId &&
|
|
644
1009
|
this.readinessEvidence !== undefined &&
|
|
645
1010
|
this.imageVerified) {
|
|
646
1011
|
const headers = new Headers(response.headers);
|
|
647
|
-
headers.set("x-eden-eve-started-deployment", requestDeploymentId);
|
|
1012
|
+
headers.set("x-eden-eve-started-deployment", this.env.EDEN_EVE_RELEASE_ID ?? requestDeploymentId);
|
|
648
1013
|
return new Response(response.body, {
|
|
649
1014
|
status: response.status,
|
|
650
1015
|
statusText: response.statusText,
|
|
@@ -653,25 +1018,305 @@ export class EveHostDurableContainer extends DurableObject {
|
|
|
653
1018
|
}
|
|
654
1019
|
return response;
|
|
655
1020
|
}
|
|
1021
|
+
async retire(deploymentId) {
|
|
1022
|
+
// World routes the fenced target to its named DO. Retirement must not
|
|
1023
|
+
// restore or trust a stored owner left by an earlier retirement attempt.
|
|
1024
|
+
if (await this.deploymentBusy(deploymentId)) {
|
|
1025
|
+
throw new Error("Cannot retire a deployment with active deliveries or transfers.");
|
|
1026
|
+
}
|
|
1027
|
+
const runtime = this.containerRuntime();
|
|
1028
|
+
if (runtime?.running)
|
|
1029
|
+
await runtime.destroy(new Error("deployment-retired"));
|
|
1030
|
+
this.recordContainerStopped();
|
|
1031
|
+
await this.ctx.storage.deleteAlarm();
|
|
1032
|
+
await this.ctx.storage.deleteAll();
|
|
1033
|
+
}
|
|
1034
|
+
/** Read-only export: never replace the source carry or stop its process. */
|
|
1035
|
+
async exportSandboxSessions(deploymentId, sessionId) {
|
|
1036
|
+
this.activeExports += 1;
|
|
1037
|
+
let released = false;
|
|
1038
|
+
const release = () => {
|
|
1039
|
+
if (!released)
|
|
1040
|
+
this.activeExports -= 1;
|
|
1041
|
+
released = true;
|
|
1042
|
+
};
|
|
1043
|
+
try {
|
|
1044
|
+
await this.identifyDeployment(deploymentId);
|
|
1045
|
+
await this.ensureEveReady(undefined, false);
|
|
1046
|
+
if (await this.ctx.storage.get("eden.eve.host.sandbox-restore-failed") === true) {
|
|
1047
|
+
throw new Error("Cannot export sandbox sessions after a failed carry restore.");
|
|
1048
|
+
}
|
|
1049
|
+
await this.touchActivity();
|
|
1050
|
+
const directory = await sandboxSessionDirectory(sessionId);
|
|
1051
|
+
const runtime = this.containerRuntime();
|
|
1052
|
+
const probe = await runtime.exec(["sh", "-c", `if [ -d ${EVE_HOST_SANDBOX_CARRY_SOURCE}/sessions/${directory} ]; then exit 0; elif [ ! -e ${EVE_HOST_SANDBOX_CARRY_SOURCE}/sessions/${directory} ]; then exit 3; else exit 2; fi`], { stdout: "ignore", stderr: "ignore" });
|
|
1053
|
+
const existence = await probe.exitCode;
|
|
1054
|
+
if (existence === 3) {
|
|
1055
|
+
release();
|
|
1056
|
+
return new Response(new Uint8Array(), { headers: { "x-eden-sandbox-directory-exists": "false" } });
|
|
1057
|
+
}
|
|
1058
|
+
if (existence !== 0)
|
|
1059
|
+
throw new Error("Cannot establish source sandbox directory existence.");
|
|
1060
|
+
const exec = await runtime.exec([
|
|
1061
|
+
"sh", "-c",
|
|
1062
|
+
`cd ${EVE_HOST_SANDBOX_CARRY_SOURCE} && tar -czf - sessions/${directory}`,
|
|
1063
|
+
], { stdout: "pipe", stderr: "ignore" });
|
|
1064
|
+
if (exec.stdout === null)
|
|
1065
|
+
throw new Error("Sandbox export stdout is unavailable.");
|
|
1066
|
+
const reader = exec.stdout.getReader();
|
|
1067
|
+
let bytes = 0;
|
|
1068
|
+
const archive = new ReadableStream({
|
|
1069
|
+
pull: async (controller) => {
|
|
1070
|
+
try {
|
|
1071
|
+
const result = await reader.read();
|
|
1072
|
+
if (result.done) {
|
|
1073
|
+
if (await exec.exitCode !== 0)
|
|
1074
|
+
throw new Error("The source sandbox archive failed.");
|
|
1075
|
+
release();
|
|
1076
|
+
controller.close();
|
|
1077
|
+
return;
|
|
1078
|
+
}
|
|
1079
|
+
bytes += result.value.byteLength;
|
|
1080
|
+
if (bytes > EVE_HOST_CARRY_LIMITS.maxBytes)
|
|
1081
|
+
throw new Error("The source sandbox archive exceeds the carry limit.");
|
|
1082
|
+
controller.enqueue(result.value);
|
|
1083
|
+
}
|
|
1084
|
+
catch (error) {
|
|
1085
|
+
release();
|
|
1086
|
+
await reader.cancel().catch(() => undefined);
|
|
1087
|
+
controller.error(error);
|
|
1088
|
+
}
|
|
1089
|
+
},
|
|
1090
|
+
cancel: async () => { release(); await reader.cancel(); },
|
|
1091
|
+
});
|
|
1092
|
+
return new Response(archive, { headers: { "content-type": "application/gzip", "x-eden-sandbox-directory-exists": "true" } });
|
|
1093
|
+
}
|
|
1094
|
+
catch (error) {
|
|
1095
|
+
release();
|
|
1096
|
+
throw error;
|
|
1097
|
+
}
|
|
1098
|
+
}
|
|
1099
|
+
/** Retry-idempotent imports finish before the migrated queue is dispatched. */
|
|
1100
|
+
async importSandboxSessions(sourceDeploymentId, deploymentId, transferId, sessionId) {
|
|
1101
|
+
await this.identifyDeployment(deploymentId);
|
|
1102
|
+
const key = `eden.eve.host.sandbox-import.v2.${sourceDeploymentId}.${sessionId}.${transferId}`;
|
|
1103
|
+
if (await this.ctx.storage.get(key)) {
|
|
1104
|
+
await this.ensureEveReady(undefined, false);
|
|
1105
|
+
return;
|
|
1106
|
+
}
|
|
1107
|
+
const existing = this.imports.get(key);
|
|
1108
|
+
if (existing !== undefined)
|
|
1109
|
+
return existing;
|
|
1110
|
+
const run = this.importQueue.then(async () => {
|
|
1111
|
+
await this.ensureEveReady(undefined, false);
|
|
1112
|
+
if (await this.ctx.storage.get(key))
|
|
1113
|
+
return;
|
|
1114
|
+
await this.touchActivity();
|
|
1115
|
+
const namespace = this.env.EVE_CONTAINER;
|
|
1116
|
+
let response;
|
|
1117
|
+
try {
|
|
1118
|
+
response = await namespace.getByName(deploymentInstance(this.env, sourceDeploymentId))
|
|
1119
|
+
.exportSandboxSessions(sourceDeploymentId, sessionId);
|
|
1120
|
+
}
|
|
1121
|
+
catch {
|
|
1122
|
+
return "source-export-failed";
|
|
1123
|
+
}
|
|
1124
|
+
if (!response.ok || response.body === null)
|
|
1125
|
+
return "source-export-failed";
|
|
1126
|
+
const noDirectory = response.headers.get("x-eden-sandbox-directory-exists") === "false";
|
|
1127
|
+
const stageId = await sandboxSessionDirectory(JSON.stringify([sourceDeploymentId, sessionId, transferId]));
|
|
1128
|
+
const stage = `${EVE_HOST_SANDBOX_CARRY_SOURCE}/.eden-imports/${stageId}`;
|
|
1129
|
+
const runtime = this.containerRuntime();
|
|
1130
|
+
let sourceExportFailed = false;
|
|
1131
|
+
const reader = response.body.getReader();
|
|
1132
|
+
const archive = new ReadableStream({
|
|
1133
|
+
pull: async (controller) => {
|
|
1134
|
+
try {
|
|
1135
|
+
const result = await reader.read();
|
|
1136
|
+
if (result.done)
|
|
1137
|
+
controller.close();
|
|
1138
|
+
else
|
|
1139
|
+
controller.enqueue(result.value);
|
|
1140
|
+
}
|
|
1141
|
+
catch (error) {
|
|
1142
|
+
sourceExportFailed = true;
|
|
1143
|
+
controller.error(error);
|
|
1144
|
+
}
|
|
1145
|
+
},
|
|
1146
|
+
cancel: (reason) => reader.cancel(reason),
|
|
1147
|
+
});
|
|
1148
|
+
try {
|
|
1149
|
+
const extract = await runtime.exec([
|
|
1150
|
+
"sh", "-c",
|
|
1151
|
+
`rm -rf ${stage} && mkdir -p ${stage} && cat > ${stage}/archive.tgz && if [ -s ${stage}/archive.tgz ]; then tar -xzf ${stage}/archive.tgz -C ${stage}; else ${noDirectory ? `mkdir -p ${stage}/sessions` : "exit 1"}; fi`,
|
|
1152
|
+
], { stdin: archive, stdout: "ignore", stderr: "ignore" });
|
|
1153
|
+
const exitCode = await extract.exitCode;
|
|
1154
|
+
if (sourceExportFailed)
|
|
1155
|
+
return "source-export-failed";
|
|
1156
|
+
if (exitCode !== 0)
|
|
1157
|
+
throw new Error(`Sandbox archive extraction failed; staging retained at ${stage}.`);
|
|
1158
|
+
}
|
|
1159
|
+
catch (error) {
|
|
1160
|
+
if (sourceExportFailed)
|
|
1161
|
+
return "source-export-failed";
|
|
1162
|
+
throw error;
|
|
1163
|
+
}
|
|
1164
|
+
const publish = await runtime.exec([
|
|
1165
|
+
"node", "-e", EVE_HOST_IMPORT_SANDBOXES, stage, EVE_HOST_SANDBOX_CARRY_SOURCE,
|
|
1166
|
+
], { stdout: "ignore", stderr: "pipe" });
|
|
1167
|
+
const errorText = publish.stderr === null ? "" : await new Response(publish.stderr).text();
|
|
1168
|
+
if (await publish.exitCode !== 0)
|
|
1169
|
+
throw new Error(`Sandbox import refused: ${errorText.trim()}; staging retained at ${stage}.`);
|
|
1170
|
+
// Persist imports before acknowledging them, including crash recovery.
|
|
1171
|
+
// An older whole-container snapshot must not hide the refreshed carry.
|
|
1172
|
+
await this.ctx.storage.delete(EVE_HOST_CONTAINER_SNAPSHOT_KEY);
|
|
1173
|
+
await this.carrySandboxSessionsOut(runtime, true);
|
|
1174
|
+
await this.ctx.storage.put("eden.eve.host.has-imported-sandboxes", true);
|
|
1175
|
+
await this.ctx.storage.put(key, true);
|
|
1176
|
+
const cleanup = await runtime.exec(["rm", "-rf", stage], { stdout: "ignore", stderr: "ignore" });
|
|
1177
|
+
await cleanup.exitCode;
|
|
1178
|
+
});
|
|
1179
|
+
this.importQueue = run.then(() => undefined, () => undefined);
|
|
1180
|
+
this.imports.set(key, run);
|
|
1181
|
+
try {
|
|
1182
|
+
return await run;
|
|
1183
|
+
}
|
|
1184
|
+
finally {
|
|
1185
|
+
this.imports.delete(key);
|
|
1186
|
+
}
|
|
1187
|
+
}
|
|
656
1188
|
/**
|
|
657
1189
|
* Queue deliveries and loopback traffic enter here: same readiness and
|
|
658
1190
|
* forwarding as `fetch` without the deployment-header echo rules (the
|
|
659
1191
|
* loopback path is not a deployment-health signal).
|
|
660
1192
|
*/
|
|
661
|
-
async loopbackFetch(request) {
|
|
662
|
-
|
|
1193
|
+
async loopbackFetch(request, claim) {
|
|
1194
|
+
await this.identifyDeployment(request.headers.get("x-eden-eve-deployment-id") ?? undefined);
|
|
1195
|
+
return this.forwardToEve(request, claim);
|
|
663
1196
|
}
|
|
664
|
-
async
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
const
|
|
668
|
-
|
|
669
|
-
|
|
1197
|
+
async settleQueueDelivery(claim) {
|
|
1198
|
+
if (!this.env.EDEN_EVE_WORLD_CLOUDFLARE || claim === undefined)
|
|
1199
|
+
return;
|
|
1200
|
+
const { messageId, attempt } = claim;
|
|
1201
|
+
const world = this.env.EDEN_WORLD;
|
|
1202
|
+
// Resolve the current World actor rather than relying on the dispatcher
|
|
1203
|
+
// instance that initiated this potentially long-lived HTTP request.
|
|
1204
|
+
await world.getByName("world").deliverySettled(messageId, attempt);
|
|
1205
|
+
}
|
|
1206
|
+
async forwardToEve(request, claim) {
|
|
1207
|
+
this.activeDeliveries += 1;
|
|
1208
|
+
let released = false;
|
|
1209
|
+
let forwarding = false;
|
|
1210
|
+
let forwardingGeneration;
|
|
1211
|
+
const release = () => {
|
|
1212
|
+
if (released)
|
|
1213
|
+
return;
|
|
1214
|
+
released = true;
|
|
1215
|
+
if (claim !== undefined)
|
|
1216
|
+
this.deliveryClaims.delete(claim);
|
|
1217
|
+
this.activeDeliveries -= 1;
|
|
1218
|
+
if (forwarding)
|
|
1219
|
+
this.activeForwarding -= 1;
|
|
1220
|
+
if (forwardingGeneration === undefined || forwardingGeneration === this.readinessEpoch) {
|
|
1221
|
+
this.ctx.waitUntil(this.touchActivity());
|
|
1222
|
+
}
|
|
1223
|
+
};
|
|
1224
|
+
try {
|
|
1225
|
+
// Already-pinned work must be able to drain a scheduler-disabled
|
|
1226
|
+
// promoted instance; public readiness waits for its safe role restart.
|
|
1227
|
+
const workflowDelivery = /^\/\.well-known\/workflow\/v1\/(?:flow|step)(?:\/|$)/u.test(new URL(request.url).pathname);
|
|
1228
|
+
await this.ensureEveReady(request.signal, !workflowDelivery);
|
|
1229
|
+
await this.touchActivity();
|
|
1230
|
+
const runtime = this.containerRuntime();
|
|
1231
|
+
if (runtime === undefined)
|
|
1232
|
+
throw new Error("The Container runtime is unavailable.");
|
|
1233
|
+
const url = new URL(request.url);
|
|
1234
|
+
let failGeneration;
|
|
1235
|
+
const generationStopped = new Promise((_resolve, reject) => {
|
|
1236
|
+
failGeneration = reject;
|
|
1237
|
+
});
|
|
1238
|
+
forwardingGeneration = this.readinessEpoch;
|
|
1239
|
+
if (claim !== undefined) {
|
|
1240
|
+
this.deliveryClaims.set(claim, {
|
|
1241
|
+
generation: forwardingGeneration,
|
|
1242
|
+
fail: (error) => { release(); failGeneration(error); },
|
|
1243
|
+
});
|
|
1244
|
+
}
|
|
1245
|
+
forwarding = true;
|
|
1246
|
+
this.activeForwarding += 1;
|
|
1247
|
+
const upstreamInit = {
|
|
1248
|
+
method: request.method,
|
|
1249
|
+
headers: request.headers,
|
|
1250
|
+
body: request.body,
|
|
1251
|
+
redirect: request.redirect,
|
|
1252
|
+
duplex: "half",
|
|
1253
|
+
// A dispatcher replacement must not cancel the actual queue handler.
|
|
1254
|
+
...(workflowDelivery ? {} : { signal: request.signal }),
|
|
1255
|
+
};
|
|
1256
|
+
const response = await Promise.race([runtime.getTcpPort(EVE_HOST_DEFAULTS.internalPort).fetch(new Request(`http://container${url.pathname}${url.search}`, upstreamInit)), generationStopped]);
|
|
1257
|
+
if (response.body === null) {
|
|
1258
|
+
await this.settleQueueDelivery(claim);
|
|
1259
|
+
release();
|
|
1260
|
+
return response;
|
|
1261
|
+
}
|
|
1262
|
+
const reader = response.body.getReader();
|
|
1263
|
+
let downstreamCancelled = false;
|
|
1264
|
+
const body = new ReadableStream({
|
|
1265
|
+
pull: async (controller) => {
|
|
1266
|
+
try {
|
|
1267
|
+
const result = await Promise.race([reader.read(), generationStopped]);
|
|
1268
|
+
if (result.done) {
|
|
1269
|
+
await this.settleQueueDelivery(claim);
|
|
1270
|
+
release();
|
|
1271
|
+
if (!downstreamCancelled)
|
|
1272
|
+
controller.close();
|
|
1273
|
+
}
|
|
1274
|
+
else if (!downstreamCancelled)
|
|
1275
|
+
controller.enqueue(result.value);
|
|
1276
|
+
}
|
|
1277
|
+
catch (error) {
|
|
1278
|
+
if (!downstreamCancelled) {
|
|
1279
|
+
release();
|
|
1280
|
+
controller.error(error);
|
|
1281
|
+
}
|
|
1282
|
+
}
|
|
1283
|
+
},
|
|
1284
|
+
cancel: async (reason) => {
|
|
1285
|
+
if (!workflowDelivery) {
|
|
1286
|
+
try {
|
|
1287
|
+
await reader.cancel(reason);
|
|
1288
|
+
}
|
|
1289
|
+
finally {
|
|
1290
|
+
release();
|
|
1291
|
+
}
|
|
1292
|
+
return;
|
|
1293
|
+
}
|
|
1294
|
+
downstreamCancelled = true;
|
|
1295
|
+
// Keep observing the upstream invocation independently of its old
|
|
1296
|
+
// dispatcher consumer. Only actual EOF permits exact settlement.
|
|
1297
|
+
this.ctx.waitUntil((async () => {
|
|
1298
|
+
try {
|
|
1299
|
+
while (!(await Promise.race([reader.read(), generationStopped])).done) { /* Discard the response body. */ }
|
|
1300
|
+
await this.settleQueueDelivery(claim);
|
|
1301
|
+
}
|
|
1302
|
+
finally {
|
|
1303
|
+
release();
|
|
1304
|
+
}
|
|
1305
|
+
})());
|
|
1306
|
+
},
|
|
1307
|
+
});
|
|
1308
|
+
return new Response(body, { status: response.status, statusText: response.statusText, headers: response.headers });
|
|
1309
|
+
}
|
|
1310
|
+
catch (error) {
|
|
1311
|
+
try {
|
|
1312
|
+
if (!forwarding)
|
|
1313
|
+
await this.settleQueueDelivery(claim);
|
|
1314
|
+
}
|
|
1315
|
+
finally {
|
|
1316
|
+
release();
|
|
1317
|
+
}
|
|
1318
|
+
throw error;
|
|
670
1319
|
}
|
|
671
|
-
const url = new URL(request.url);
|
|
672
|
-
return runtime
|
|
673
|
-
.getTcpPort(EVE_HOST_DEFAULTS.internalPort)
|
|
674
|
-
.fetch(new Request(`http://container${url.pathname}${url.search}`, request));
|
|
675
1320
|
}
|
|
676
1321
|
/**
|
|
677
1322
|
* The idle-sleep path: once the container has been quiet for `sleepAfter`,
|
|
@@ -681,10 +1326,16 @@ export class EveHostDurableContainer extends DurableObject {
|
|
|
681
1326
|
* for the running container stops.
|
|
682
1327
|
*/
|
|
683
1328
|
async alarm() {
|
|
1329
|
+
const deploymentId = await this.identifyDeployment();
|
|
684
1330
|
const runtime = this.containerRuntime();
|
|
685
1331
|
const lastActivity = (await this.ctx.storage.get(EVE_HOST_CONTAINER_ACTIVITY_KEY)) ?? 0;
|
|
686
1332
|
const now = Date.now();
|
|
687
|
-
if (
|
|
1333
|
+
if (await this.deploymentBusy(deploymentId)) {
|
|
1334
|
+
await this.ctx.storage.setAlarm(now + this.sleepAfterMs);
|
|
1335
|
+
return;
|
|
1336
|
+
}
|
|
1337
|
+
const fencePending = await this.ctx.storage.get("eden.eve.host.scheduler-fence-pending") === true;
|
|
1338
|
+
if (!fencePending && now - lastActivity < this.sleepAfterMs) {
|
|
688
1339
|
await this.ctx.storage.setAlarm(lastActivity + this.sleepAfterMs);
|
|
689
1340
|
return;
|
|
690
1341
|
}
|
|
@@ -705,7 +1356,7 @@ export class EveHostDurableContainer extends DurableObject {
|
|
|
705
1356
|
name: "eden-workspace",
|
|
706
1357
|
});
|
|
707
1358
|
const image = (await runtime.inspect().catch(() => null))?.image ??
|
|
708
|
-
runtime
|
|
1359
|
+
this.expectedImage(runtime) ??
|
|
709
1360
|
"";
|
|
710
1361
|
const record = {
|
|
711
1362
|
id: snapshot.id,
|
|
@@ -718,15 +1369,21 @@ export class EveHostDurableContainer extends DurableObject {
|
|
|
718
1369
|
// request that arrived meanwhile must not have its container destroyed
|
|
719
1370
|
// underneath it. Keep running and re-arm sleep from that activity.
|
|
720
1371
|
const activityAfterSnapshot = (await this.ctx.storage.get(EVE_HOST_CONTAINER_ACTIVITY_KEY)) ?? 0;
|
|
721
|
-
if (activityAfterSnapshot !== lastActivity) {
|
|
722
|
-
await this.ctx.storage.
|
|
1372
|
+
if (activityAfterSnapshot !== lastActivity || await this.deploymentBusy(deploymentId)) {
|
|
1373
|
+
const fencing = await this.ctx.storage.get("eden.eve.host.scheduler-fence-pending") === true;
|
|
1374
|
+
await this.ctx.storage.setAlarm(fencing ? Date.now() : activityAfterSnapshot + this.sleepAfterMs);
|
|
723
1375
|
return;
|
|
724
1376
|
}
|
|
725
1377
|
await runtime.destroy(new Error("idle-snapshot-done"));
|
|
726
1378
|
this.recordContainerStopped();
|
|
1379
|
+
await this.ctx.storage.delete("eden.eve.host.scheduler-fence-pending");
|
|
727
1380
|
}
|
|
728
1381
|
catch (error) {
|
|
729
1382
|
const message = error instanceof Error ? error.message : "unknown";
|
|
1383
|
+
if (await this.deploymentBusy(deploymentId)) {
|
|
1384
|
+
await this.touchActivity();
|
|
1385
|
+
return;
|
|
1386
|
+
}
|
|
730
1387
|
if (attempts + 1 >= EVE_HOST_SNAPSHOT_MAX_ATTEMPTS) {
|
|
731
1388
|
console.error(`The Eve workspace snapshot failed ${attempts + 1} times; destroying the container without it: ${message}`);
|
|
732
1389
|
await this.ctx.storage.put(EVE_HOST_CONTAINER_SNAPSHOT_ATTEMPTS_KEY, 0);
|
|
@@ -752,12 +1409,13 @@ export function createEveHostWorker(options) {
|
|
|
752
1409
|
if (namespace === undefined) {
|
|
753
1410
|
throw new EveHostError("HOST_READINESS_UNPROVEN", "The configured Container binding is unavailable.");
|
|
754
1411
|
}
|
|
755
|
-
return namespace.getByName(options.stableContainerInstanceName);
|
|
1412
|
+
return namespace.getByName(deploymentInstance(env, options.deploymentId, options.stableContainerInstanceName));
|
|
756
1413
|
};
|
|
757
1414
|
const forwardToContainer = async (request, env) => {
|
|
758
1415
|
const metadata = {
|
|
759
1416
|
publicOrigin: options.publicOrigin,
|
|
760
1417
|
deploymentId: options.deploymentId,
|
|
1418
|
+
releaseId: options.releaseId ?? options.deploymentId,
|
|
761
1419
|
generationId: options.generationId,
|
|
762
1420
|
correlationId: globalThis.crypto.randomUUID(),
|
|
763
1421
|
...(options.runtimeRevisionHandle === undefined
|
|
@@ -793,10 +1451,28 @@ export function createEveHostWorker(options) {
|
|
|
793
1451
|
};
|
|
794
1452
|
return {
|
|
795
1453
|
async fetch(request, env) {
|
|
1454
|
+
if (new URL(request.url).pathname === "/__eden/world/rpc") {
|
|
1455
|
+
if (request.method !== "POST")
|
|
1456
|
+
return new Response(null, { status: 405 });
|
|
1457
|
+
const token = env.EDEN_EVE_CONTROL_TOKEN;
|
|
1458
|
+
const authorization = request.headers.get("authorization") ?? "";
|
|
1459
|
+
if (typeof token !== "string" || token.length === 0)
|
|
1460
|
+
return new Response(null, { status: 404 });
|
|
1461
|
+
// Hash both values to fixed lengths before comparing every byte.
|
|
1462
|
+
const digest = async (value) => new Uint8Array(await crypto.subtle.digest("SHA-256", new TextEncoder().encode(value)));
|
|
1463
|
+
const [expected, supplied] = await Promise.all([digest(`Bearer ${token}`), digest(authorization)]);
|
|
1464
|
+
let mismatch = 0;
|
|
1465
|
+
for (let index = 0; index < expected.length; index += 1)
|
|
1466
|
+
mismatch |= expected[index] ^ supplied[index];
|
|
1467
|
+
if (mismatch !== 0)
|
|
1468
|
+
return new Response(null, { status: 401 });
|
|
1469
|
+
const world = env.EDEN_WORLD;
|
|
1470
|
+
return world === undefined ? new Response(null, { status: 404 }) : world.getByName("world").controlFetch(request);
|
|
1471
|
+
}
|
|
796
1472
|
// Queue deliveries to these endpoints arrive through the container's
|
|
797
1473
|
// intercepted outbound loopback instead, so any request that reaches
|
|
798
1474
|
// the public Worker fetch path is external and unauthenticated.
|
|
799
|
-
if (isEveWorkflowInternalRoute(new URL(request.url).pathname)) {
|
|
1475
|
+
if (isEveWorkflowInternalRoute(new URL(request.url).pathname) || new URL(request.url).pathname.startsWith("/__eden/internal/")) {
|
|
800
1476
|
return new Response(null, { status: 404 });
|
|
801
1477
|
}
|
|
802
1478
|
const response = await forwardToContainer(request, env);
|
|
@@ -804,8 +1480,9 @@ export function createEveHostWorker(options) {
|
|
|
804
1480
|
headers.set("x-eden-eve-worker-name", options.workerName);
|
|
805
1481
|
headers.set("x-eden-eve-public-origin", options.publicOrigin);
|
|
806
1482
|
headers.set("x-eden-eve-deployment-id", options.deploymentId);
|
|
1483
|
+
headers.set("x-eden-eve-release-id", options.releaseId ?? options.deploymentId);
|
|
807
1484
|
headers.set("x-eden-eve-generation-id", options.generationId);
|
|
808
|
-
headers.set("x-eden-eve-container-instance", options.stableContainerInstanceName);
|
|
1485
|
+
headers.set("x-eden-eve-container-instance", deploymentInstance(env, options.deploymentId, options.stableContainerInstanceName));
|
|
809
1486
|
if (options.runtimeRevisionHandle !== undefined) {
|
|
810
1487
|
headers.set("x-eden-eve-runtime-revision", options.runtimeRevisionHandle);
|
|
811
1488
|
}
|