@moinulmoin/eden-runtime-cloudflare 0.9.0 → 0.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -3,6 +3,85 @@ import { WorldCore } from "@moinulmoin/eden-world-cloudflare/core";
3
3
  import { EVE_HOST_DEFAULTS, EVE_HOST_SLEEP_SNAPSHOT_MARGIN_MS, EVE_SCHEDULE_WAKE_LEAD_MS, parseEveSleepAfterMs, EveHostError, assertNonEmpty, assertStableOrigin, createEveReadinessGate, createTrustedEveRequest, eveScheduleCronFiresWithin, readProtectedRuntimeVariables, WORKER_NAME_PATTERN, isEveWorkflowInternalRoute, } from "./eve-host.js";
4
4
  const EVE_CLOUDFLARE_CONTAINERS_CA_PATH = "/etc/cloudflare/certs/cloudflare-containers-ca.crt";
5
5
  const EVE_HOST_STALE_WORKER_REASON = "stale Worker version";
6
+ function deploymentImages(env) {
7
+ const raw = env.EDEN_EVE_DEPLOYMENTS;
8
+ return typeof raw === "string" ? JSON.parse(raw) : {};
9
+ }
10
+ function deploymentInstance(env, deploymentId, base = env.EVE_CONTAINER_INSTANCE_NAME) {
11
+ if (typeof base !== "string")
12
+ throw new Error("The Container instance identity is unavailable.");
13
+ // Existing runs used this identity before affinity was introduced.
14
+ return env.EDEN_EVE_DEPLOYMENTS === undefined || deploymentId === "eden-cloudflare" ? base : `${base}:${deploymentId}`;
15
+ }
16
+ function deploymentRequest(request, deploymentId) {
17
+ const headers = new Headers(request.headers);
18
+ headers.set("x-eden-eve-deployment-id", deploymentId);
19
+ return new Request(request, { headers });
20
+ }
21
+ async function sandboxSessionDirectory(sessionId) {
22
+ // Eve 0.91.0 and 1.0.0-rc.3 just-bash both use provider identity version 2.
23
+ // Sorting these keys produces ordinary compact JSON. The adjacent braces
24
+ // in Eve's template literal close the interpolation and object separately.
25
+ const identity = JSON.stringify({ sessionId, version: 2 });
26
+ const digest = await crypto.subtle.digest("SHA-256", new TextEncoder().encode(identity));
27
+ return Array.from(new Uint8Array(digest), (byte) => byte.toString(16).padStart(2, "0")).join("").slice(0, 24);
28
+ }
29
+ const EVE_HOST_IMPORT_SANDBOXES = String.raw `
30
+ const fs = require("node:fs/promises");
31
+ const path = require("node:path");
32
+ const crypto = require("node:crypto");
33
+ const { createReadStream } = require("node:fs");
34
+ async function digest(file) {
35
+ const hash = crypto.createHash("sha256");
36
+ for await (const chunk of createReadStream(file)) hash.update(chunk);
37
+ return hash.digest("hex");
38
+ }
39
+ async function equal(a, b) {
40
+ const [x, y] = await Promise.all([fs.lstat(a), fs.lstat(b)]);
41
+ if (x.isSymbolicLink() || y.isSymbolicLink()) {
42
+ return x.isSymbolicLink() && y.isSymbolicLink() && await fs.readlink(a) === await fs.readlink(b);
43
+ }
44
+ if (x.isDirectory() && y.isDirectory()) {
45
+ const [left, right] = await Promise.all([fs.readdir(a), fs.readdir(b)]);
46
+ left.sort(); right.sort();
47
+ if (left.length !== right.length || left.some((name, i) => name !== right[i])) return false;
48
+ for (const name of left) if (!await equal(path.join(a, name), path.join(b, name))) return false;
49
+ return true;
50
+ }
51
+ return x.isFile() && y.isFile() && x.size === y.size && await digest(a) === await digest(b);
52
+ }
53
+ async function publish(from, to) {
54
+ // Both paths are on the destination workspace filesystem. Linux performs
55
+ // one atomic directory publication; an existing destination is never replaced.
56
+ const script = "import ctypes,os,sys;libc=ctypes.CDLL(None,use_errno=True);result=libc.renameat2(-100,os.fsencode(sys.argv[1]),-100,os.fsencode(sys.argv[2]),1);err=ctypes.get_errno();sys.exit(0) if result==0 else (_ for _ in ()).throw(OSError(err,os.strerror(err)))";
57
+ const { spawnSync } = require("node:child_process");
58
+ const result = spawnSync("python3", ["-c", script, from, to], { encoding: "utf8" });
59
+ if (result.error) throw result.error;
60
+ if (result.status !== 0) {
61
+ // A racing destination creation is a conflict unless its complete tree
62
+ // is identical. Never merge into or overwrite a live session directory.
63
+ if (!await equal(from, to)) throw new Error(result.stderr || "Atomic sandbox publication failed");
64
+ }
65
+ }
66
+ (async () => {
67
+ const [stage, root] = process.argv.slice(1);
68
+ const source = path.join(stage, "sessions");
69
+ await fs.mkdir(path.join(root, "sessions"), { recursive: true });
70
+ const pending = [];
71
+ for (const name of await fs.readdir(source)) {
72
+ const from = path.join(source, name);
73
+ const to = path.join(root, "sessions", name);
74
+ let exists = true;
75
+ try { await fs.lstat(to); } catch (error) { if (error.code !== "ENOENT") throw error; exists = false; }
76
+ if (exists) {
77
+ if (!await equal(from, to)) throw new Error("Sandbox import conflicts with an existing target session: " + name);
78
+ } else pending.push([from, to]);
79
+ }
80
+ // Validate every conflict before publishing. Never replace a directory
81
+ // that may belong to a turn already running on the target.
82
+ for (const [from, to] of pending) await publish(from, to);
83
+ })().catch(error => { console.error(error.message); process.exitCode = 1; });
84
+ `;
6
85
  /**
7
86
  * Cloudflare's durable_object containers expose `/dev/kvm`, which makes Eve
8
87
  * versions before 0.83 pick microsandbox at runtime, although Eden's isolated
@@ -53,56 +132,98 @@ function isMissingContainerSnapshot(error) {
53
132
  /Snapshot ".*" was not found/u.test(error.message);
54
133
  }
55
134
  /** The deployment's single SQLite-backed Workflow World. */
56
- export class EdenWorldDurableObject {
135
+ export class EdenWorldDurableObject extends DurableObject {
57
136
  core;
58
137
  constructor(ctx, env) {
138
+ super(ctx, env);
59
139
  this.core = new WorldCore({
140
+ ...(env.EDEN_EVE_DEPLOYMENT_ID === undefined ? {} : { currentDeploymentId: env.EDEN_EVE_DEPLOYMENT_ID }),
60
141
  sql: {
61
142
  exec: ctx.storage.sql.exec.bind(ctx.storage.sql),
62
143
  transactionSync: ctx.storage.transactionSync.bind(ctx.storage),
63
144
  },
145
+ waitUntil: (promise) => ctx.waitUntil(promise),
64
146
  scheduleAlarm: async (atMs) => {
65
147
  if (atMs === null)
66
148
  await ctx.storage.deleteAlarm();
67
149
  else
68
150
  await ctx.storage.setAlarm(atMs);
69
151
  },
152
+ retireDeployment: async (deploymentId) => {
153
+ if (deploymentId === env.EDEN_EVE_DEPLOYMENT_ID)
154
+ throw new Error("Cannot retire the current ingress deployment.");
155
+ if (!env.EVE_CONTAINER)
156
+ throw new Error("The Container namespace is unavailable for retirement.");
157
+ if (env.EDEN_EVE_DEPLOYMENTS === undefined)
158
+ throw new Error("Cannot retire a deployment without deployment-aware Container routing.");
159
+ const namespace = env.EVE_CONTAINER;
160
+ await namespace.getByName(deploymentInstance(env, deploymentId)).retire(deploymentId);
161
+ },
70
162
  delivery: {
71
163
  deliver: async (message) => {
72
164
  const origin = env.EVE_PUBLIC_ORIGIN;
73
- const instance = env.EVE_CONTAINER_INSTANCE_NAME;
74
- if (typeof origin !== "string" || typeof instance !== "string") {
165
+ const deploymentId = env.EDEN_EVE_DEPLOYMENTS === undefined ? env.EDEN_EVE_DEPLOYMENT_ID : message.deploymentId ?? "eden-cloudflare";
166
+ let sandboxTransferCompleted = false;
167
+ const settleUnstarted = async () => {
168
+ const world = env.EDEN_WORLD;
169
+ await world.getByName("world").deliverySettled(message.messageId, message.attempt);
170
+ };
171
+ if (typeof origin !== "string") {
172
+ await settleUnstarted();
75
173
  throw new Error("The World queue Container identity is unavailable.");
76
174
  }
77
175
  const namespace = env.EVE_CONTAINER;
78
- const response = await namespace.getByName(instance).loopbackFetch(new Request(new URL(`/.well-known/workflow/v1/${message.path}`, origin), {
79
- method: "POST",
80
- headers: {
81
- ...message.headers,
82
- "content-type": "application/json",
83
- "x-vqs-queue-name": message.queueName,
84
- "x-vqs-message-id": message.messageId,
85
- "x-vqs-message-attempt": String(message.attempt),
86
- },
87
- body: message.body,
88
- }));
89
- // Body transport errors must remain retryable; an HTTP status alone
90
- // is not an acknowledgement of a streamed Container response.
91
- const body = await response.text();
92
- let result = null;
176
+ const target = namespace.getByName(deploymentInstance(env, deploymentId));
177
+ if (message.sourceDeploymentId && message.sourceDeploymentId !== deploymentId && message.sandboxSessionId) {
178
+ try {
179
+ const result = await target.importSandboxSessions(message.sourceDeploymentId, deploymentId, message.messageId, message.sandboxSessionId);
180
+ if (result === "source-export-failed") {
181
+ await settleUnstarted();
182
+ return { ok: false, sandboxTransferFailed: true };
183
+ }
184
+ }
185
+ catch (error) {
186
+ await settleUnstarted();
187
+ throw error;
188
+ }
189
+ sandboxTransferCompleted = true;
190
+ }
93
191
  try {
94
- result = JSON.parse(body);
192
+ const response = await target.loopbackFetch(deploymentRequest(new Request(new URL(`/.well-known/workflow/v1/${message.path}`, origin), {
193
+ method: "POST",
194
+ headers: {
195
+ ...message.headers,
196
+ "content-type": "application/json",
197
+ "x-vqs-queue-name": message.queueName,
198
+ "x-vqs-message-id": message.messageId,
199
+ "x-vqs-message-attempt": String(message.attempt),
200
+ },
201
+ body: message.body,
202
+ }), deploymentId), { messageId: message.messageId, attempt: message.attempt });
203
+ // Body transport errors must remain retryable; an HTTP status alone
204
+ // is not an acknowledgement of a streamed Container response.
205
+ const body = await response.text();
206
+ let result = null;
207
+ try {
208
+ result = JSON.parse(body);
209
+ }
210
+ catch {
211
+ // World-local also accepts successful non-JSON responses.
212
+ }
213
+ const timeout = result?.timeoutSeconds;
214
+ return {
215
+ ok: response.ok && timeout === undefined,
216
+ sandboxTransferCompleted,
217
+ ...(typeof timeout === "number" && timeout >= 0
218
+ ? { retryAfterMs: timeout * 1_000 }
219
+ : {}),
220
+ };
95
221
  }
96
- catch {
97
- // World-local also accepts successful non-JSON responses.
222
+ catch (error) {
223
+ if (!sandboxTransferCompleted)
224
+ throw error;
225
+ return { ok: false, sandboxTransferCompleted: true };
98
226
  }
99
- const timeout = result?.timeoutSeconds;
100
- return {
101
- ok: response.ok && timeout === undefined,
102
- ...(typeof timeout === "number" && timeout >= 0
103
- ? { retryAfterMs: timeout * 1_000 }
104
- : {}),
105
- };
106
227
  },
107
228
  },
108
229
  });
@@ -116,6 +237,17 @@ export class EdenWorldDurableObject {
116
237
  headers: { "content-type": "application/cbor" },
117
238
  });
118
239
  }
240
+ deploymentBusy(deploymentId) {
241
+ const activity = this.core.deploymentActivity(deploymentId);
242
+ return activity.activeDeliveries > 0 || activity.unfinishedTransfers > 0;
243
+ }
244
+ deliverySettled(messageId, attempt) {
245
+ this.core.deliverySettled(messageId, attempt);
246
+ }
247
+ async controlFetch(request) {
248
+ const result = await this.core.handleControlRpc(new Uint8Array(await request.arrayBuffer()));
249
+ return new Response(result, { headers: { "content-type": "application/cbor" } });
250
+ }
119
251
  async alarm() {
120
252
  await this.core.runAlarm();
121
253
  }
@@ -128,13 +260,15 @@ export class EdenWorldDurableObject {
128
260
  * URLs stay correct; the DO registers an outbound-HTTPS intercept for that
129
261
  * hostname aimed at `EveHostLoopback`, and this function routes the request:
130
262
  * non-public hosts egress plainly, `/__eden/world/rpc` reaches the World
131
- * Durable Object, and everything else re-enters the container over the
132
- * internal port so the public Worker path can keep refusing the
133
- * unauthenticated queue endpoints.
263
+ * Durable Object, and local World queue deliveries re-enter the container
264
+ * over its internal port. Cloudflare World queue delivery uses direct DO RPC,
265
+ * so its unauthenticated outbound queue requests remain forbidden.
134
266
  */
135
- export async function routeEveOutboundRequest(request, env) {
267
+ export async function routeEveOutboundRequest(request, env, sourceDeploymentId, expectedWorldToken) {
136
268
  const publicOrigin = env.EVE_PUBLIC_ORIGIN;
137
- const instanceName = env.EVE_CONTAINER_INSTANCE_NAME;
269
+ const deploymentId = sourceDeploymentId ?? "eden-cloudflare";
270
+ const instanceName = typeof deploymentId === "string"
271
+ ? deploymentInstance(env, deploymentId) : undefined;
138
272
  if (typeof publicOrigin !== "string" ||
139
273
  typeof instanceName !== "string") {
140
274
  throw new EveHostError("HOST_ORIGIN_UNAVAILABLE", "The container loopback identity is unavailable to the outbound handler.");
@@ -146,16 +280,32 @@ export async function routeEveOutboundRequest(request, env) {
146
280
  return fetch(request);
147
281
  }
148
282
  if (new URL(request.url).pathname === "/__eden/world/rpc") {
283
+ const route = deploymentImages(env)[deploymentId];
284
+ const authRequired = route?.worldRpcAuth === true ||
285
+ (env.EDEN_EVE_DEPLOYMENTS === undefined && env.EDEN_EVE_WORLD_RPC_AUTH === true);
286
+ if (authRequired) {
287
+ const supplied = request.headers.get("x-eden-world-token") ?? "";
288
+ const encoder = new TextEncoder();
289
+ const expected = encoder.encode(expectedWorldToken ?? "");
290
+ const actual = encoder.encode(supplied);
291
+ const subtle = crypto.subtle;
292
+ if (expected.length === 0 || actual.length !== expected.length || !subtle.timingSafeEqual(actual, expected)) {
293
+ return new Response(null, { status: 403 });
294
+ }
295
+ }
149
296
  const world = env.EDEN_WORLD;
150
297
  if (world === undefined)
151
298
  return new Response(null, { status: 404 });
152
299
  return world.get(world.idFromName("world")).fetch(request);
153
300
  }
301
+ if (env.EDEN_EVE_WORLD_CLOUDFLARE && isEveWorkflowInternalRoute(new URL(request.url).pathname)) {
302
+ return new Response(null, { status: 403 });
303
+ }
154
304
  const namespace = env.EVE_CONTAINER;
155
305
  if (namespace === undefined) {
156
306
  throw new EveHostError("HOST_READINESS_UNPROVEN", "The configured Container binding is unavailable to the outbound handler.");
157
307
  }
158
- return namespace.getByName(instanceName).loopbackFetch(request);
308
+ return namespace.getByName(instanceName).loopbackFetch(deploymentRequest(request, deploymentId));
159
309
  }
160
310
  /**
161
311
  * This module is the only Eve-host surface that depends on provider runtime
@@ -171,7 +321,8 @@ export async function routeEveOutboundRequest(request, env) {
171
321
  */
172
322
  export class EveHostLoopback extends WorkerEntrypoint {
173
323
  async fetch(request) {
174
- return routeEveOutboundRequest(request, this.env);
324
+ const props = this.ctx.props;
325
+ return routeEveOutboundRequest(request, this.env, props.deploymentId, props.expectedWorldToken);
175
326
  }
176
327
  }
177
328
  /**
@@ -204,6 +355,136 @@ export class EveHostDurableContainer extends DurableObject {
204
355
  envVars;
205
356
  sleepAfterMs;
206
357
  readiness;
358
+ deploymentId;
359
+ imports = new Map();
360
+ importQueue = Promise.resolve();
361
+ activeExports = 0;
362
+ activeDeliveries = 0;
363
+ activeForwarding = 0;
364
+ schedulerRoleReconcile;
365
+ retainedFences;
366
+ deliveryClaims = new Map();
367
+ async identifyDeployment(requested) {
368
+ const affinity = this.env.EDEN_EVE_DEPLOYMENTS !== undefined;
369
+ let stored = affinity ? this.deploymentId ?? await this.ctx.storage.get("eden.eve.host.deployment-id") : undefined;
370
+ const namespace = this.env.EVE_CONTAINER;
371
+ const legacy = affinity && this.ctx.id.equals(namespace.idFromName(this.env.EVE_CONTAINER_INSTANCE_NAME));
372
+ if (legacy && stored !== undefined && stored !== "eden-cloudflare" && deploymentImages(this.env)[stored] === undefined)
373
+ stored = undefined;
374
+ const named = affinity && !legacy
375
+ ? Object.keys(deploymentImages(this.env)).find((id) => this.ctx.id.equals(namespace.idFromName(deploymentInstance(this.env, id))))
376
+ : undefined;
377
+ const owner = affinity ? (legacy ? "eden-cloudflare" : named) : this.env.EDEN_EVE_DEPLOYMENT_ID;
378
+ const id = affinity ? requested ?? stored ?? owner : owner;
379
+ if (affinity && (owner === undefined || id !== owner)) {
380
+ throw new Error("The Container deployment identity does not match its instance name.");
381
+ }
382
+ if (!id || (stored !== undefined && stored !== id)) {
383
+ throw new Error("The Container deployment identity does not match its persisted owner.");
384
+ }
385
+ const record = deploymentImages(this.env)[id];
386
+ if (stored === undefined)
387
+ await this.ctx.storage.put("eden.eve.host.deployment-id", id);
388
+ this.deploymentId = id;
389
+ this.envVars.EDEN_EVE_DEPLOYMENT_ID = id;
390
+ this.envVars.EDEN_EVE_RELEASE_ID = record?.releaseId ?? this.env.EDEN_EVE_RELEASE_ID ?? id;
391
+ this.envVars.EVE_SCHEDULES_DISABLED = id === this.env.EDEN_EVE_DEPLOYMENT_ID ? "false" : "true";
392
+ this.envVars.EDEN_EVE_GENERATION_ID = record?.generationId ?? this.env.EDEN_EVE_GENERATION_ID;
393
+ if (record?.runtimeRevisionHandle !== undefined) {
394
+ this.envVars.EDEN_EVE_RUNTIME_REVISION = record.runtimeRevisionHandle;
395
+ }
396
+ else {
397
+ delete this.envVars.EDEN_EVE_RUNTIME_REVISION;
398
+ }
399
+ return id;
400
+ }
401
+ async fenceRetainedSchedules() {
402
+ if (this.deploymentId !== this.env.EDEN_EVE_DEPLOYMENT_ID)
403
+ return;
404
+ this.retainedFences ??= (async () => {
405
+ const namespace = this.env.EVE_CONTAINER;
406
+ for (const id of Object.keys(deploymentImages(this.env))) {
407
+ if (id === this.deploymentId)
408
+ continue;
409
+ const request = new Request(new URL("/__eden/internal/fence-schedules", this.env.EVE_PUBLIC_ORIGIN), {
410
+ headers: {
411
+ "x-eden-eve-deployment-id": id,
412
+ "x-eden-eve-release-id": this.env.EDEN_EVE_RELEASE_ID ?? this.env.EDEN_EVE_DEPLOYMENT_ID,
413
+ },
414
+ });
415
+ let response;
416
+ try {
417
+ response = await namespace.getByName(deploymentInstance(this.env, id)).fetch(request);
418
+ }
419
+ catch (error) {
420
+ if (!(error instanceof Error) || !error.message.includes(EVE_HOST_STALE_WORKER_REASON))
421
+ throw error;
422
+ response = await namespace.getByName(deploymentInstance(this.env, id)).fetch(request);
423
+ }
424
+ if (!response.ok)
425
+ throw new Error(`Cannot fence schedules on retained deployment ${id}.`);
426
+ await response.body?.cancel();
427
+ }
428
+ })().catch((error) => { this.retainedFences = undefined; throw error; });
429
+ await this.retainedFences;
430
+ }
431
+ async deploymentBusy(deploymentId) {
432
+ if (this.activeDeliveries > 0 || this.activeExports > 0 || this.imports.size > 0)
433
+ return true;
434
+ if (!this.env.EDEN_EVE_WORLD_CLOUDFLARE)
435
+ return false;
436
+ const world = this.env.EDEN_WORLD;
437
+ return world.getByName("world").deploymentBusy(deploymentId);
438
+ }
439
+ async reconcileSchedulerRole() {
440
+ const runtime = this.containerRuntime();
441
+ if (this.deploymentId !== this.env.EDEN_EVE_DEPLOYMENT_ID || !runtime?.running)
442
+ return;
443
+ if (await this.ctx.storage.get("eden.eve.host.started-schedules-disabled") !== true)
444
+ return;
445
+ this.schedulerRoleReconcile ??= (async () => {
446
+ await this.ctx.storage.put("eden.eve.host.scheduler-fence-pending", true);
447
+ const canRestart = async () => {
448
+ if (this.activeForwarding > 0 || this.activeExports > 0 || this.imports.size > 0)
449
+ return false;
450
+ if (!this.env.EDEN_EVE_WORLD_CLOUDFLARE)
451
+ return true;
452
+ const world = this.env.EDEN_WORLD;
453
+ return !await world.getByName("world").deploymentBusy(this.deploymentId);
454
+ };
455
+ if (!await canRestart()) {
456
+ throw new EveHostError("HOST_READINESS_UNPROVEN", "Scheduler promotion awaits active retained deployment work.");
457
+ }
458
+ const snapshot = await runtime.snapshotContainer({ name: "eden-workspace" });
459
+ const image = (await runtime.inspect().catch(() => null))?.image ||
460
+ await this.ctx.storage.get(EVE_HOST_CONTAINER_STARTED_IMAGE_KEY) || this.expectedImage(runtime);
461
+ if (!image)
462
+ throw new Error("Cannot identify the scheduler promotion snapshot image.");
463
+ await this.ctx.storage.put(EVE_HOST_CONTAINER_SNAPSHOT_KEY, { id: snapshot.id, image, at: Date.now() });
464
+ if (!await canRestart()) {
465
+ throw new EveHostError("HOST_READINESS_UNPROVEN", "Scheduler promotion awaits active retained deployment work.");
466
+ }
467
+ await runtime.destroy(new Error("scheduler-role-promotion"));
468
+ const stoppedAt = Date.now();
469
+ while (runtime.running && Date.now() - stoppedAt < 10_000) {
470
+ await new Promise((resolve) => setTimeout(resolve, 25));
471
+ }
472
+ if (runtime.running) {
473
+ throw new EveHostError("HOST_READINESS_UNPROVEN", "The previous scheduler process has not stopped.");
474
+ }
475
+ this.recordContainerStopped();
476
+ })().finally(() => { this.schedulerRoleReconcile = undefined; });
477
+ await this.schedulerRoleReconcile;
478
+ }
479
+ expectedImage(runtime) {
480
+ const id = this.deploymentId;
481
+ if (id === undefined)
482
+ throw new Error("Container deployment identity has not been resolved.");
483
+ const record = deploymentImages(this.env)[id];
484
+ if (this.env.EDEN_EVE_DEPLOYMENTS !== undefined && record === undefined && id !== "eden-cloudflare")
485
+ throw new Error(`Deployment ${id} has no retained Container image.`);
486
+ return runtime.images[record?.image ?? "eve"];
487
+ }
207
488
  constructor(ctx, env) {
208
489
  super(ctx, env);
209
490
  const publicOrigin = env.EVE_PUBLIC_ORIGIN;
@@ -261,6 +542,10 @@ export class EveHostDurableContainer extends DurableObject {
261
542
  // DO restarts lose the inactivity timeout; the monitor for this
262
543
  // container generation is also gone, so both are re-attached here.
263
544
  this.ctx.blockConcurrencyWhile(async () => {
545
+ if (env.EDEN_EVE_DEPLOYMENTS !== undefined) {
546
+ await this.identifyDeployment();
547
+ await this.registerLoopbackIntercept(runtime);
548
+ }
264
549
  await runtime.setInactivityTimeout(this.inactivityTimeoutMs);
265
550
  this.attachMonitor(runtime);
266
551
  });
@@ -278,10 +563,15 @@ export class EveHostDurableContainer extends DurableObject {
278
563
  if (this.monitorAttached)
279
564
  return;
280
565
  this.monitorAttached = true;
566
+ const generation = this.readinessEpoch;
281
567
  runtime.monitor().then(() => {
568
+ if (generation !== this.readinessEpoch)
569
+ return;
282
570
  this.monitorAttached = false;
283
571
  this.recordContainerStopped();
284
572
  }, (error) => {
573
+ if (generation !== this.readinessEpoch)
574
+ return;
285
575
  this.monitorAttached = false;
286
576
  if (this.startedFromSnapshot && isMissingContainerSnapshot(error)) {
287
577
  // Cloudflare deletes a snapshot 30 days after it was created or
@@ -298,6 +588,14 @@ export class EveHostDurableContainer extends DurableObject {
298
588
  }
299
589
  recordContainerStopped() {
300
590
  this.readinessEpoch += 1;
591
+ this.monitorAttached = false;
592
+ for (const [claim, pending] of this.deliveryClaims) {
593
+ if (pending.generation !== this.readinessEpoch - 1)
594
+ continue;
595
+ this.deliveryClaims.delete(claim);
596
+ pending.fail(new Error("The Container generation stopped before its queue delivery completed."));
597
+ this.ctx.waitUntil(this.settleQueueDelivery(claim));
598
+ }
301
599
  this.readinessEvidence = undefined;
302
600
  this.imageVerified = false;
303
601
  this.pendingCarryRestore = false;
@@ -310,7 +608,10 @@ export class EveHostDurableContainer extends DurableObject {
310
608
  const entryFactory = exportsMap?.["EveHostLoopback"];
311
609
  if (typeof entryFactory !== "function")
312
610
  return;
313
- const entrypoint = entryFactory({ props: {} });
611
+ const entrypoint = entryFactory({ props: {
612
+ deploymentId: await this.identifyDeployment(),
613
+ expectedWorldToken: await this.ctx.storage.get("eden.eve.host.world-token") ?? "",
614
+ } });
314
615
  await runtime.interceptOutboundHttps(new URL(this.env.EVE_PUBLIC_ORIGIN ?? "").hostname, entrypoint);
315
616
  }
316
617
  /**
@@ -321,7 +622,8 @@ export class EveHostDurableContainer extends DurableObject {
321
622
  async touchActivity() {
322
623
  const now = Date.now();
323
624
  await this.ctx.storage.put(EVE_HOST_CONTAINER_ACTIVITY_KEY, now);
324
- await this.ctx.storage.setAlarm(now + this.sleepAfterMs);
625
+ const fencing = await this.ctx.storage.get("eden.eve.host.scheduler-fence-pending") === true;
626
+ await this.ctx.storage.setAlarm(fencing ? now : now + this.sleepAfterMs);
325
627
  }
326
628
  /**
327
629
  * Starts the container when it is not running. A stored workspace snapshot
@@ -337,7 +639,7 @@ export class EveHostDurableContainer extends DurableObject {
337
639
  }
338
640
  if (runtime.running)
339
641
  return;
340
- const expectedImage = runtime.images["eve"];
642
+ const expectedImage = this.expectedImage(runtime);
341
643
  if (expectedImage === undefined) {
342
644
  throw new EveHostError("HOST_READINESS_UNPROVEN", "The configured Container image is unavailable to this Worker version.");
343
645
  }
@@ -346,6 +648,18 @@ export class EveHostDurableContainer extends DurableObject {
346
648
  storedSnapshot.image === expectedImage
347
649
  ? storedSnapshot.id
348
650
  : undefined;
651
+ // Persist the fresh-start restore obligation before the process exists:
652
+ // a DO replacement must not mistake a partially restored process for ready.
653
+ if (snapshotId === undefined) {
654
+ await this.ctx.storage.put("eden.eve.host.restore-pending", globalThis.crypto.randomUUID());
655
+ }
656
+ else {
657
+ await this.ctx.storage.delete("eden.eve.host.restore-pending");
658
+ }
659
+ await this.ctx.storage.put("eden.eve.host.started-schedules-disabled", this.envVars.EVE_SCHEDULES_DISABLED === "true");
660
+ const worldToken = globalThis.crypto.randomUUID();
661
+ await this.ctx.storage.put("eden.eve.host.world-token", worldToken);
662
+ this.envVars.EDEN_WORLD_TOKEN = worldToken;
349
663
  runtime.start({
350
664
  ...(snapshotId === undefined
351
665
  ? { image: expectedImage }
@@ -355,12 +669,19 @@ export class EveHostDurableContainer extends DurableObject {
355
669
  env: this.envVars,
356
670
  entrypoint: EVE_HOST_ENTRYPOINT,
357
671
  });
672
+ console.log("eden.container.start", JSON.stringify({
673
+ deploymentId: this.deploymentId, image: expectedImage,
674
+ source: snapshotId === undefined ? "image" : "snapshot",
675
+ }));
358
676
  // Set before any await: a failure below makes the readiness retry see a
359
677
  // running container and skip this method, so the restore must already be
360
678
  // pending. A start from the configured image (an update) has no sandbox
361
679
  // sessions yet; a same-image snapshot wake already contains them.
362
680
  this.pendingCarryRestore = snapshotId === undefined;
363
681
  this.startedFromSnapshot = snapshotId !== undefined;
682
+ // A fresh process receives the retained scheduler fence even when its
683
+ // filesystem came from a snapshot of the formerly-current deployment.
684
+ await this.ctx.storage.delete("eden.eve.host.scheduler-fence-pending");
364
685
  // Both start sources run `expectedImage` (a snapshot is only restored
365
686
  // from that image).
366
687
  await this.ctx.storage.put(EVE_HOST_CONTAINER_STARTED_IMAGE_KEY, expectedImage);
@@ -383,7 +704,7 @@ export class EveHostDurableContainer extends DurableObject {
383
704
  const runtime = this.containerRuntime();
384
705
  if (runtime === undefined || !runtime.running)
385
706
  return;
386
- const expectedImage = runtime.images["eve"];
707
+ const expectedImage = this.expectedImage(runtime);
387
708
  let info;
388
709
  try {
389
710
  info = await runtime.inspect();
@@ -422,12 +743,12 @@ export class EveHostDurableContainer extends DurableObject {
422
743
  * image. One carry runs at a time on this Durable Object; a failure never
423
744
  * blocks the destroy that follows and leaves the previous carry intact.
424
745
  */
425
- carrySandboxSessionsOut(runtime) {
426
- const run = this.carryOutQueue.then(() => this.carrySandboxSessionsOutNow(runtime));
746
+ carrySandboxSessionsOut(runtime, required = false) {
747
+ const run = this.carryOutQueue.then(() => this.carrySandboxSessionsOutNow(runtime, required));
427
748
  this.carryOutQueue = run.then(() => undefined, () => undefined);
428
749
  return run;
429
750
  }
430
- async carrySandboxSessionsOutNow(runtime) {
751
+ async carrySandboxSessionsOutNow(runtime, required = false) {
431
752
  const previous = await this.ctx.storage.get(EVE_HOST_SANDBOX_CARRY_KEY);
432
753
  const carryId = globalThis.crypto.randomUUID();
433
754
  const chunkPrefix = `${EVE_HOST_SANDBOX_CARRY_CHUNK_PREFIX}.${carryId}.`;
@@ -501,6 +822,8 @@ export class EveHostDurableContainer extends DurableObject {
501
822
  await reader.cancel().catch(() => undefined);
502
823
  if (overflowed) {
503
824
  await discardWrittenChunks();
825
+ if (required)
826
+ throw new Error("The imported sandbox carry exceeds the archive limit.");
504
827
  console.error(`The Eve sandbox session carry exceeded ${EVE_HOST_CARRY_LIMITS.maxBytes} compressed bytes; keeping the previous carry.`);
505
828
  return;
506
829
  }
@@ -519,6 +842,8 @@ export class EveHostDurableContainer extends DurableObject {
519
842
  }
520
843
  if (exitCode !== 0) {
521
844
  await discardWrittenChunks();
845
+ if (required)
846
+ throw new Error(`The imported sandbox carry failed with exit code ${exitCode}.`);
522
847
  console.error(`The Eve sandbox session carry failed with exit code ${exitCode}; keeping the previous carry.`);
523
848
  return;
524
849
  }
@@ -545,10 +870,22 @@ export class EveHostDurableContainer extends DurableObject {
545
870
  * the same promise, and no request is forwarded before it settles.
546
871
  */
547
872
  async restoreCarriedSandboxSessions() {
548
- if (!this.pendingCarryRestore)
873
+ const restoreGeneration = await this.ctx.storage.get("eden.eve.host.restore-pending");
874
+ if (!this.pendingCarryRestore && restoreGeneration === undefined)
549
875
  return;
550
- this.carryRestorePromise ??= this.carrySandboxSessionsIn();
551
- await this.carryRestorePromise;
876
+ const required = await this.ctx.storage.get("eden.eve.host.has-imported-sandboxes") === true;
877
+ this.carryRestorePromise ??= this.carrySandboxSessionsIn(required);
878
+ try {
879
+ await this.carryRestorePromise;
880
+ }
881
+ catch (error) {
882
+ this.carryRestorePromise = undefined;
883
+ throw error;
884
+ }
885
+ if (restoreGeneration !== undefined &&
886
+ await this.ctx.storage.get("eden.eve.host.restore-pending") === restoreGeneration) {
887
+ await this.ctx.storage.delete("eden.eve.host.restore-pending");
888
+ }
552
889
  this.pendingCarryRestore = false;
553
890
  }
554
891
  /**
@@ -558,11 +895,14 @@ export class EveHostDurableContainer extends DurableObject {
558
895
  * failure is logged and swallowed: the agent still serves, only the
559
896
  * carried files are missing.
560
897
  */
561
- async carrySandboxSessionsIn() {
898
+ async carrySandboxSessionsIn(required = false) {
562
899
  try {
563
900
  const carry = await this.ctx.storage.get(EVE_HOST_SANDBOX_CARRY_KEY);
564
- if (carry === undefined)
901
+ if (carry === undefined) {
902
+ if (required)
903
+ throw new Error("The imported sandbox carry is missing.");
565
904
  return;
905
+ }
566
906
  let nextChunk = 0;
567
907
  const archive = new ReadableStream({
568
908
  pull: async (controller) => {
@@ -584,10 +924,15 @@ export class EveHostDurableContainer extends DurableObject {
584
924
  if (runtime === undefined) {
585
925
  throw new Error("the Container runtime is unavailable");
586
926
  }
927
+ const restoreGeneration = await this.ctx.storage.get("eden.eve.host.restore-pending");
928
+ const stageId = await sandboxSessionDirectory(restoreGeneration ?? carry.id);
929
+ const stage = `${EVE_HOST_SANDBOX_CARRY_SOURCE}/.eden-restores/${stageId}`;
587
930
  const exec = await runtime.exec([
588
931
  "sh",
589
932
  "-c",
590
- `mkdir -p ${EVE_HOST_SANDBOX_CARRY_SOURCE} && tar -xzf - -C ${EVE_HOST_SANDBOX_CARRY_SOURCE}`,
933
+ required
934
+ ? `rm -rf ${stage} && mkdir -p ${stage} && tar -xzf - -C ${stage}`
935
+ : `mkdir -p ${EVE_HOST_SANDBOX_CARRY_SOURCE} && tar -xzf - -C ${EVE_HOST_SANDBOX_CARRY_SOURCE}`,
591
936
  ], { stdin: archive, stderr: "pipe" });
592
937
  let exitCode = -1;
593
938
  try {
@@ -596,15 +941,30 @@ export class EveHostDurableContainer extends DurableObject {
596
941
  catch {
597
942
  exitCode = -1;
598
943
  }
599
- if (exitCode !== 0) {
600
- console.error(`The Eve sandbox session restore failed with exit code ${exitCode}; the agent serves without the carried files.`);
944
+ if (exitCode !== 0)
945
+ throw new Error(`The sandbox restore failed with exit code ${exitCode}.`);
946
+ if (required) {
947
+ const publish = await runtime.exec(["node", "-e", EVE_HOST_IMPORT_SANDBOXES, stage, EVE_HOST_SANDBOX_CARRY_SOURCE], { stdout: "ignore", stderr: "pipe" });
948
+ const error = publish.stderr === null ? "" : await new Response(publish.stderr).text();
949
+ if (await publish.exitCode !== 0)
950
+ throw new Error(`The imported sandbox restore publication failed: ${error}`);
951
+ const cleanup = await runtime.exec(["rm", "-rf", stage], { stdout: "ignore", stderr: "ignore" });
952
+ await cleanup.exitCode;
601
953
  }
954
+ await this.ctx.storage.delete("eden.eve.host.sandbox-restore-failed");
602
955
  }
603
956
  catch (error) {
957
+ await this.ctx.storage.put("eden.eve.host.sandbox-restore-failed", true);
958
+ if (required)
959
+ throw error;
604
960
  console.error("The Eve sandbox session restore failed; the agent serves without the carried files:", error instanceof Error ? error.message : "unknown");
605
961
  }
606
962
  }
607
- async ensureEveReady(signal = new AbortController().signal) {
963
+ async ensureEveReady(signal = new AbortController().signal, requireSchedulerRole = true) {
964
+ await this.identifyDeployment();
965
+ await this.fenceRetainedSchedules();
966
+ if (requireSchedulerRole)
967
+ await this.reconcileSchedulerRole();
608
968
  await this.reconcileRunningImage();
609
969
  if (this.readinessEvidence !== undefined) {
610
970
  return this.readinessEvidence;
@@ -626,13 +986,18 @@ export class EveHostDurableContainer extends DurableObject {
626
986
  }
627
987
  async fetch(request) {
628
988
  const requestDeploymentId = request.headers.get("x-eden-eve-deployment-id") ?? undefined;
629
- if (requestDeploymentId !== undefined &&
630
- requestDeploymentId !== this.env.EDEN_EVE_DEPLOYMENT_ID) {
631
- // A live container/monitor can pin the DO to an older Worker version.
632
- // Reset the DO, not its container or storage; the retry's current code
633
- // reconciles the image and carries sandbox sessions through the update.
989
+ const requestedRelease = request.headers.get("x-eden-eve-release-id") ?? requestDeploymentId;
990
+ if (requestedRelease !== undefined && requestedRelease !== (this.env.EDEN_EVE_RELEASE_ID ?? this.env.EDEN_EVE_DEPLOYMENT_ID)) {
634
991
  this.ctx.abort(EVE_HOST_STALE_WORKER_REASON);
635
992
  }
993
+ await this.identifyDeployment(requestDeploymentId);
994
+ if (new URL(request.url).pathname === "/__eden/internal/fence-schedules") {
995
+ if (this.deploymentId !== this.env.EDEN_EVE_DEPLOYMENT_ID) {
996
+ await this.ctx.storage.put("eden.eve.host.scheduler-fence-pending", true);
997
+ await this.ctx.storage.setAlarm(Date.now());
998
+ }
999
+ return new Response(null, { status: 204 });
1000
+ }
636
1001
  const response = await this.forwardToEve(request);
637
1002
  // Reporting the verified start back lets deploy health-gate promotion on
638
1003
  // the served container actually running this generation's image: the
@@ -640,11 +1005,11 @@ export class EveHostDurableContainer extends DurableObject {
640
1005
  // Worker version's configured image and the request names this
641
1006
  // deployment.
642
1007
  if (requestDeploymentId !== undefined &&
643
- requestDeploymentId === this.env.EDEN_EVE_DEPLOYMENT_ID &&
1008
+ requestDeploymentId === this.deploymentId &&
644
1009
  this.readinessEvidence !== undefined &&
645
1010
  this.imageVerified) {
646
1011
  const headers = new Headers(response.headers);
647
- headers.set("x-eden-eve-started-deployment", requestDeploymentId);
1012
+ headers.set("x-eden-eve-started-deployment", this.env.EDEN_EVE_RELEASE_ID ?? requestDeploymentId);
648
1013
  return new Response(response.body, {
649
1014
  status: response.status,
650
1015
  statusText: response.statusText,
@@ -653,25 +1018,305 @@ export class EveHostDurableContainer extends DurableObject {
653
1018
  }
654
1019
  return response;
655
1020
  }
1021
+ async retire(deploymentId) {
1022
+ // World routes the fenced target to its named DO. Retirement must not
1023
+ // restore or trust a stored owner left by an earlier retirement attempt.
1024
+ if (await this.deploymentBusy(deploymentId)) {
1025
+ throw new Error("Cannot retire a deployment with active deliveries or transfers.");
1026
+ }
1027
+ const runtime = this.containerRuntime();
1028
+ if (runtime?.running)
1029
+ await runtime.destroy(new Error("deployment-retired"));
1030
+ this.recordContainerStopped();
1031
+ await this.ctx.storage.deleteAlarm();
1032
+ await this.ctx.storage.deleteAll();
1033
+ }
1034
+ /** Read-only export: never replace the source carry or stop its process. */
1035
+ async exportSandboxSessions(deploymentId, sessionId) {
1036
+ this.activeExports += 1;
1037
+ let released = false;
1038
+ const release = () => {
1039
+ if (!released)
1040
+ this.activeExports -= 1;
1041
+ released = true;
1042
+ };
1043
+ try {
1044
+ await this.identifyDeployment(deploymentId);
1045
+ await this.ensureEveReady(undefined, false);
1046
+ if (await this.ctx.storage.get("eden.eve.host.sandbox-restore-failed") === true) {
1047
+ throw new Error("Cannot export sandbox sessions after a failed carry restore.");
1048
+ }
1049
+ await this.touchActivity();
1050
+ const directory = await sandboxSessionDirectory(sessionId);
1051
+ const runtime = this.containerRuntime();
1052
+ const probe = await runtime.exec(["sh", "-c", `if [ -d ${EVE_HOST_SANDBOX_CARRY_SOURCE}/sessions/${directory} ]; then exit 0; elif [ ! -e ${EVE_HOST_SANDBOX_CARRY_SOURCE}/sessions/${directory} ]; then exit 3; else exit 2; fi`], { stdout: "ignore", stderr: "ignore" });
1053
+ const existence = await probe.exitCode;
1054
+ if (existence === 3) {
1055
+ release();
1056
+ return new Response(new Uint8Array(), { headers: { "x-eden-sandbox-directory-exists": "false" } });
1057
+ }
1058
+ if (existence !== 0)
1059
+ throw new Error("Cannot establish source sandbox directory existence.");
1060
+ const exec = await runtime.exec([
1061
+ "sh", "-c",
1062
+ `cd ${EVE_HOST_SANDBOX_CARRY_SOURCE} && tar -czf - sessions/${directory}`,
1063
+ ], { stdout: "pipe", stderr: "ignore" });
1064
+ if (exec.stdout === null)
1065
+ throw new Error("Sandbox export stdout is unavailable.");
1066
+ const reader = exec.stdout.getReader();
1067
+ let bytes = 0;
1068
+ const archive = new ReadableStream({
1069
+ pull: async (controller) => {
1070
+ try {
1071
+ const result = await reader.read();
1072
+ if (result.done) {
1073
+ if (await exec.exitCode !== 0)
1074
+ throw new Error("The source sandbox archive failed.");
1075
+ release();
1076
+ controller.close();
1077
+ return;
1078
+ }
1079
+ bytes += result.value.byteLength;
1080
+ if (bytes > EVE_HOST_CARRY_LIMITS.maxBytes)
1081
+ throw new Error("The source sandbox archive exceeds the carry limit.");
1082
+ controller.enqueue(result.value);
1083
+ }
1084
+ catch (error) {
1085
+ release();
1086
+ await reader.cancel().catch(() => undefined);
1087
+ controller.error(error);
1088
+ }
1089
+ },
1090
+ cancel: async () => { release(); await reader.cancel(); },
1091
+ });
1092
+ return new Response(archive, { headers: { "content-type": "application/gzip", "x-eden-sandbox-directory-exists": "true" } });
1093
+ }
1094
+ catch (error) {
1095
+ release();
1096
+ throw error;
1097
+ }
1098
+ }
1099
+ /** Retry-idempotent imports finish before the migrated queue is dispatched. */
1100
+ async importSandboxSessions(sourceDeploymentId, deploymentId, transferId, sessionId) {
1101
+ await this.identifyDeployment(deploymentId);
1102
+ const key = `eden.eve.host.sandbox-import.v2.${sourceDeploymentId}.${sessionId}.${transferId}`;
1103
+ if (await this.ctx.storage.get(key)) {
1104
+ await this.ensureEveReady(undefined, false);
1105
+ return;
1106
+ }
1107
+ const existing = this.imports.get(key);
1108
+ if (existing !== undefined)
1109
+ return existing;
1110
+ const run = this.importQueue.then(async () => {
1111
+ await this.ensureEveReady(undefined, false);
1112
+ if (await this.ctx.storage.get(key))
1113
+ return;
1114
+ await this.touchActivity();
1115
+ const namespace = this.env.EVE_CONTAINER;
1116
+ let response;
1117
+ try {
1118
+ response = await namespace.getByName(deploymentInstance(this.env, sourceDeploymentId))
1119
+ .exportSandboxSessions(sourceDeploymentId, sessionId);
1120
+ }
1121
+ catch {
1122
+ return "source-export-failed";
1123
+ }
1124
+ if (!response.ok || response.body === null)
1125
+ return "source-export-failed";
1126
+ const noDirectory = response.headers.get("x-eden-sandbox-directory-exists") === "false";
1127
+ const stageId = await sandboxSessionDirectory(JSON.stringify([sourceDeploymentId, sessionId, transferId]));
1128
+ const stage = `${EVE_HOST_SANDBOX_CARRY_SOURCE}/.eden-imports/${stageId}`;
1129
+ const runtime = this.containerRuntime();
1130
+ let sourceExportFailed = false;
1131
+ const reader = response.body.getReader();
1132
+ const archive = new ReadableStream({
1133
+ pull: async (controller) => {
1134
+ try {
1135
+ const result = await reader.read();
1136
+ if (result.done)
1137
+ controller.close();
1138
+ else
1139
+ controller.enqueue(result.value);
1140
+ }
1141
+ catch (error) {
1142
+ sourceExportFailed = true;
1143
+ controller.error(error);
1144
+ }
1145
+ },
1146
+ cancel: (reason) => reader.cancel(reason),
1147
+ });
1148
+ try {
1149
+ const extract = await runtime.exec([
1150
+ "sh", "-c",
1151
+ `rm -rf ${stage} && mkdir -p ${stage} && cat > ${stage}/archive.tgz && if [ -s ${stage}/archive.tgz ]; then tar -xzf ${stage}/archive.tgz -C ${stage}; else ${noDirectory ? `mkdir -p ${stage}/sessions` : "exit 1"}; fi`,
1152
+ ], { stdin: archive, stdout: "ignore", stderr: "ignore" });
1153
+ const exitCode = await extract.exitCode;
1154
+ if (sourceExportFailed)
1155
+ return "source-export-failed";
1156
+ if (exitCode !== 0)
1157
+ throw new Error(`Sandbox archive extraction failed; staging retained at ${stage}.`);
1158
+ }
1159
+ catch (error) {
1160
+ if (sourceExportFailed)
1161
+ return "source-export-failed";
1162
+ throw error;
1163
+ }
1164
+ const publish = await runtime.exec([
1165
+ "node", "-e", EVE_HOST_IMPORT_SANDBOXES, stage, EVE_HOST_SANDBOX_CARRY_SOURCE,
1166
+ ], { stdout: "ignore", stderr: "pipe" });
1167
+ const errorText = publish.stderr === null ? "" : await new Response(publish.stderr).text();
1168
+ if (await publish.exitCode !== 0)
1169
+ throw new Error(`Sandbox import refused: ${errorText.trim()}; staging retained at ${stage}.`);
1170
+ // Persist imports before acknowledging them, including crash recovery.
1171
+ // An older whole-container snapshot must not hide the refreshed carry.
1172
+ await this.ctx.storage.delete(EVE_HOST_CONTAINER_SNAPSHOT_KEY);
1173
+ await this.carrySandboxSessionsOut(runtime, true);
1174
+ await this.ctx.storage.put("eden.eve.host.has-imported-sandboxes", true);
1175
+ await this.ctx.storage.put(key, true);
1176
+ const cleanup = await runtime.exec(["rm", "-rf", stage], { stdout: "ignore", stderr: "ignore" });
1177
+ await cleanup.exitCode;
1178
+ });
1179
+ this.importQueue = run.then(() => undefined, () => undefined);
1180
+ this.imports.set(key, run);
1181
+ try {
1182
+ return await run;
1183
+ }
1184
+ finally {
1185
+ this.imports.delete(key);
1186
+ }
1187
+ }
656
1188
  /**
657
1189
  * Queue deliveries and loopback traffic enter here: same readiness and
658
1190
  * forwarding as `fetch` without the deployment-header echo rules (the
659
1191
  * loopback path is not a deployment-health signal).
660
1192
  */
661
- async loopbackFetch(request) {
662
- return this.forwardToEve(request);
1193
+ async loopbackFetch(request, claim) {
1194
+ await this.identifyDeployment(request.headers.get("x-eden-eve-deployment-id") ?? undefined);
1195
+ return this.forwardToEve(request, claim);
663
1196
  }
664
- async forwardToEve(request) {
665
- await this.ensureEveReady(request.signal);
666
- await this.touchActivity();
667
- const runtime = this.containerRuntime();
668
- if (runtime === undefined) {
669
- throw new EveHostError("HOST_READINESS_UNPROVEN", "The Container runtime is unavailable on this Durable Object.");
1197
+ async settleQueueDelivery(claim) {
1198
+ if (!this.env.EDEN_EVE_WORLD_CLOUDFLARE || claim === undefined)
1199
+ return;
1200
+ const { messageId, attempt } = claim;
1201
+ const world = this.env.EDEN_WORLD;
1202
+ // Resolve the current World actor rather than relying on the dispatcher
1203
+ // instance that initiated this potentially long-lived HTTP request.
1204
+ await world.getByName("world").deliverySettled(messageId, attempt);
1205
+ }
1206
+ async forwardToEve(request, claim) {
1207
+ this.activeDeliveries += 1;
1208
+ let released = false;
1209
+ let forwarding = false;
1210
+ let forwardingGeneration;
1211
+ const release = () => {
1212
+ if (released)
1213
+ return;
1214
+ released = true;
1215
+ if (claim !== undefined)
1216
+ this.deliveryClaims.delete(claim);
1217
+ this.activeDeliveries -= 1;
1218
+ if (forwarding)
1219
+ this.activeForwarding -= 1;
1220
+ if (forwardingGeneration === undefined || forwardingGeneration === this.readinessEpoch) {
1221
+ this.ctx.waitUntil(this.touchActivity());
1222
+ }
1223
+ };
1224
+ try {
1225
+ // Already-pinned work must be able to drain a scheduler-disabled
1226
+ // promoted instance; public readiness waits for its safe role restart.
1227
+ const workflowDelivery = /^\/\.well-known\/workflow\/v1\/(?:flow|step)(?:\/|$)/u.test(new URL(request.url).pathname);
1228
+ await this.ensureEveReady(request.signal, !workflowDelivery);
1229
+ await this.touchActivity();
1230
+ const runtime = this.containerRuntime();
1231
+ if (runtime === undefined)
1232
+ throw new Error("The Container runtime is unavailable.");
1233
+ const url = new URL(request.url);
1234
+ let failGeneration;
1235
+ const generationStopped = new Promise((_resolve, reject) => {
1236
+ failGeneration = reject;
1237
+ });
1238
+ forwardingGeneration = this.readinessEpoch;
1239
+ if (claim !== undefined) {
1240
+ this.deliveryClaims.set(claim, {
1241
+ generation: forwardingGeneration,
1242
+ fail: (error) => { release(); failGeneration(error); },
1243
+ });
1244
+ }
1245
+ forwarding = true;
1246
+ this.activeForwarding += 1;
1247
+ const upstreamInit = {
1248
+ method: request.method,
1249
+ headers: request.headers,
1250
+ body: request.body,
1251
+ redirect: request.redirect,
1252
+ duplex: "half",
1253
+ // A dispatcher replacement must not cancel the actual queue handler.
1254
+ ...(workflowDelivery ? {} : { signal: request.signal }),
1255
+ };
1256
+ const response = await Promise.race([runtime.getTcpPort(EVE_HOST_DEFAULTS.internalPort).fetch(new Request(`http://container${url.pathname}${url.search}`, upstreamInit)), generationStopped]);
1257
+ if (response.body === null) {
1258
+ await this.settleQueueDelivery(claim);
1259
+ release();
1260
+ return response;
1261
+ }
1262
+ const reader = response.body.getReader();
1263
+ let downstreamCancelled = false;
1264
+ const body = new ReadableStream({
1265
+ pull: async (controller) => {
1266
+ try {
1267
+ const result = await Promise.race([reader.read(), generationStopped]);
1268
+ if (result.done) {
1269
+ await this.settleQueueDelivery(claim);
1270
+ release();
1271
+ if (!downstreamCancelled)
1272
+ controller.close();
1273
+ }
1274
+ else if (!downstreamCancelled)
1275
+ controller.enqueue(result.value);
1276
+ }
1277
+ catch (error) {
1278
+ if (!downstreamCancelled) {
1279
+ release();
1280
+ controller.error(error);
1281
+ }
1282
+ }
1283
+ },
1284
+ cancel: async (reason) => {
1285
+ if (!workflowDelivery) {
1286
+ try {
1287
+ await reader.cancel(reason);
1288
+ }
1289
+ finally {
1290
+ release();
1291
+ }
1292
+ return;
1293
+ }
1294
+ downstreamCancelled = true;
1295
+ // Keep observing the upstream invocation independently of its old
1296
+ // dispatcher consumer. Only actual EOF permits exact settlement.
1297
+ this.ctx.waitUntil((async () => {
1298
+ try {
1299
+ while (!(await Promise.race([reader.read(), generationStopped])).done) { /* Discard the response body. */ }
1300
+ await this.settleQueueDelivery(claim);
1301
+ }
1302
+ finally {
1303
+ release();
1304
+ }
1305
+ })());
1306
+ },
1307
+ });
1308
+ return new Response(body, { status: response.status, statusText: response.statusText, headers: response.headers });
1309
+ }
1310
+ catch (error) {
1311
+ try {
1312
+ if (!forwarding)
1313
+ await this.settleQueueDelivery(claim);
1314
+ }
1315
+ finally {
1316
+ release();
1317
+ }
1318
+ throw error;
670
1319
  }
671
- const url = new URL(request.url);
672
- return runtime
673
- .getTcpPort(EVE_HOST_DEFAULTS.internalPort)
674
- .fetch(new Request(`http://container${url.pathname}${url.search}`, request));
675
1320
  }
676
1321
  /**
677
1322
  * The idle-sleep path: once the container has been quiet for `sleepAfter`,
@@ -681,10 +1326,16 @@ export class EveHostDurableContainer extends DurableObject {
681
1326
  * for the running container stops.
682
1327
  */
683
1328
  async alarm() {
1329
+ const deploymentId = await this.identifyDeployment();
684
1330
  const runtime = this.containerRuntime();
685
1331
  const lastActivity = (await this.ctx.storage.get(EVE_HOST_CONTAINER_ACTIVITY_KEY)) ?? 0;
686
1332
  const now = Date.now();
687
- if (now - lastActivity < this.sleepAfterMs) {
1333
+ if (await this.deploymentBusy(deploymentId)) {
1334
+ await this.ctx.storage.setAlarm(now + this.sleepAfterMs);
1335
+ return;
1336
+ }
1337
+ const fencePending = await this.ctx.storage.get("eden.eve.host.scheduler-fence-pending") === true;
1338
+ if (!fencePending && now - lastActivity < this.sleepAfterMs) {
688
1339
  await this.ctx.storage.setAlarm(lastActivity + this.sleepAfterMs);
689
1340
  return;
690
1341
  }
@@ -705,7 +1356,7 @@ export class EveHostDurableContainer extends DurableObject {
705
1356
  name: "eden-workspace",
706
1357
  });
707
1358
  const image = (await runtime.inspect().catch(() => null))?.image ??
708
- runtime.images["eve"] ??
1359
+ this.expectedImage(runtime) ??
709
1360
  "";
710
1361
  const record = {
711
1362
  id: snapshot.id,
@@ -718,15 +1369,21 @@ export class EveHostDurableContainer extends DurableObject {
718
1369
  // request that arrived meanwhile must not have its container destroyed
719
1370
  // underneath it. Keep running and re-arm sleep from that activity.
720
1371
  const activityAfterSnapshot = (await this.ctx.storage.get(EVE_HOST_CONTAINER_ACTIVITY_KEY)) ?? 0;
721
- if (activityAfterSnapshot !== lastActivity) {
722
- await this.ctx.storage.setAlarm(activityAfterSnapshot + this.sleepAfterMs);
1372
+ if (activityAfterSnapshot !== lastActivity || await this.deploymentBusy(deploymentId)) {
1373
+ const fencing = await this.ctx.storage.get("eden.eve.host.scheduler-fence-pending") === true;
1374
+ await this.ctx.storage.setAlarm(fencing ? Date.now() : activityAfterSnapshot + this.sleepAfterMs);
723
1375
  return;
724
1376
  }
725
1377
  await runtime.destroy(new Error("idle-snapshot-done"));
726
1378
  this.recordContainerStopped();
1379
+ await this.ctx.storage.delete("eden.eve.host.scheduler-fence-pending");
727
1380
  }
728
1381
  catch (error) {
729
1382
  const message = error instanceof Error ? error.message : "unknown";
1383
+ if (await this.deploymentBusy(deploymentId)) {
1384
+ await this.touchActivity();
1385
+ return;
1386
+ }
730
1387
  if (attempts + 1 >= EVE_HOST_SNAPSHOT_MAX_ATTEMPTS) {
731
1388
  console.error(`The Eve workspace snapshot failed ${attempts + 1} times; destroying the container without it: ${message}`);
732
1389
  await this.ctx.storage.put(EVE_HOST_CONTAINER_SNAPSHOT_ATTEMPTS_KEY, 0);
@@ -752,12 +1409,13 @@ export function createEveHostWorker(options) {
752
1409
  if (namespace === undefined) {
753
1410
  throw new EveHostError("HOST_READINESS_UNPROVEN", "The configured Container binding is unavailable.");
754
1411
  }
755
- return namespace.getByName(options.stableContainerInstanceName);
1412
+ return namespace.getByName(deploymentInstance(env, options.deploymentId, options.stableContainerInstanceName));
756
1413
  };
757
1414
  const forwardToContainer = async (request, env) => {
758
1415
  const metadata = {
759
1416
  publicOrigin: options.publicOrigin,
760
1417
  deploymentId: options.deploymentId,
1418
+ releaseId: options.releaseId ?? options.deploymentId,
761
1419
  generationId: options.generationId,
762
1420
  correlationId: globalThis.crypto.randomUUID(),
763
1421
  ...(options.runtimeRevisionHandle === undefined
@@ -793,10 +1451,28 @@ export function createEveHostWorker(options) {
793
1451
  };
794
1452
  return {
795
1453
  async fetch(request, env) {
1454
+ if (new URL(request.url).pathname === "/__eden/world/rpc") {
1455
+ if (request.method !== "POST")
1456
+ return new Response(null, { status: 405 });
1457
+ const token = env.EDEN_EVE_CONTROL_TOKEN;
1458
+ const authorization = request.headers.get("authorization") ?? "";
1459
+ if (typeof token !== "string" || token.length === 0)
1460
+ return new Response(null, { status: 404 });
1461
+ // Hash both values to fixed lengths before comparing every byte.
1462
+ const digest = async (value) => new Uint8Array(await crypto.subtle.digest("SHA-256", new TextEncoder().encode(value)));
1463
+ const [expected, supplied] = await Promise.all([digest(`Bearer ${token}`), digest(authorization)]);
1464
+ let mismatch = 0;
1465
+ for (let index = 0; index < expected.length; index += 1)
1466
+ mismatch |= expected[index] ^ supplied[index];
1467
+ if (mismatch !== 0)
1468
+ return new Response(null, { status: 401 });
1469
+ const world = env.EDEN_WORLD;
1470
+ return world === undefined ? new Response(null, { status: 404 }) : world.getByName("world").controlFetch(request);
1471
+ }
796
1472
  // Queue deliveries to these endpoints arrive through the container's
797
1473
  // intercepted outbound loopback instead, so any request that reaches
798
1474
  // the public Worker fetch path is external and unauthenticated.
799
- if (isEveWorkflowInternalRoute(new URL(request.url).pathname)) {
1475
+ if (isEveWorkflowInternalRoute(new URL(request.url).pathname) || new URL(request.url).pathname.startsWith("/__eden/internal/")) {
800
1476
  return new Response(null, { status: 404 });
801
1477
  }
802
1478
  const response = await forwardToContainer(request, env);
@@ -804,8 +1480,9 @@ export function createEveHostWorker(options) {
804
1480
  headers.set("x-eden-eve-worker-name", options.workerName);
805
1481
  headers.set("x-eden-eve-public-origin", options.publicOrigin);
806
1482
  headers.set("x-eden-eve-deployment-id", options.deploymentId);
1483
+ headers.set("x-eden-eve-release-id", options.releaseId ?? options.deploymentId);
807
1484
  headers.set("x-eden-eve-generation-id", options.generationId);
808
- headers.set("x-eden-eve-container-instance", options.stableContainerInstanceName);
1485
+ headers.set("x-eden-eve-container-instance", deploymentInstance(env, options.deploymentId, options.stableContainerInstanceName));
809
1486
  if (options.runtimeRevisionHandle !== undefined) {
810
1487
  headers.set("x-eden-eve-runtime-revision", options.runtimeRevisionHandle);
811
1488
  }