@moinulmoin/eden-runtime-cloudflare 0.8.0 → 0.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -3,13 +3,91 @@ import { WorldCore } from "@moinulmoin/eden-world-cloudflare/core";
3
3
  import { EVE_HOST_DEFAULTS, EVE_HOST_SLEEP_SNAPSHOT_MARGIN_MS, EVE_SCHEDULE_WAKE_LEAD_MS, parseEveSleepAfterMs, EveHostError, assertNonEmpty, assertStableOrigin, createEveReadinessGate, createTrustedEveRequest, eveScheduleCronFiresWithin, readProtectedRuntimeVariables, WORKER_NAME_PATTERN, isEveWorkflowInternalRoute, } from "./eve-host.js";
4
4
  const EVE_CLOUDFLARE_CONTAINERS_CA_PATH = "/etc/cloudflare/certs/cloudflare-containers-ca.crt";
5
5
  const EVE_HOST_STALE_WORKER_REASON = "stale Worker version";
6
+ function deploymentImages(env) {
7
+ const raw = env.EDEN_EVE_DEPLOYMENTS;
8
+ return typeof raw === "string" ? JSON.parse(raw) : {};
9
+ }
10
+ function deploymentInstance(env, deploymentId, base = env.EVE_CONTAINER_INSTANCE_NAME) {
11
+ if (typeof base !== "string")
12
+ throw new Error("The Container instance identity is unavailable.");
13
+ // Existing runs used this identity before affinity was introduced.
14
+ return env.EDEN_EVE_DEPLOYMENTS === undefined || deploymentId === "eden-cloudflare" ? base : `${base}:${deploymentId}`;
15
+ }
16
+ function deploymentRequest(request, deploymentId) {
17
+ const headers = new Headers(request.headers);
18
+ headers.set("x-eden-eve-deployment-id", deploymentId);
19
+ return new Request(request, { headers });
20
+ }
21
+ async function sandboxSessionDirectory(sessionId) {
22
+ // Eve 0.91.0 and 1.0.0-rc.3 just-bash both use provider identity version 2.
23
+ // Sorting these keys produces ordinary compact JSON. The adjacent braces
24
+ // in Eve's template literal close the interpolation and object separately.
25
+ const identity = JSON.stringify({ sessionId, version: 2 });
26
+ const digest = await crypto.subtle.digest("SHA-256", new TextEncoder().encode(identity));
27
+ return Array.from(new Uint8Array(digest), (byte) => byte.toString(16).padStart(2, "0")).join("").slice(0, 24);
28
+ }
29
+ const EVE_HOST_IMPORT_SANDBOXES = String.raw `
30
+ const fs = require("node:fs/promises");
31
+ const path = require("node:path");
32
+ const crypto = require("node:crypto");
33
+ const { createReadStream } = require("node:fs");
34
+ async function digest(file) {
35
+ const hash = crypto.createHash("sha256");
36
+ for await (const chunk of createReadStream(file)) hash.update(chunk);
37
+ return hash.digest("hex");
38
+ }
39
+ async function equal(a, b) {
40
+ const [x, y] = await Promise.all([fs.lstat(a), fs.lstat(b)]);
41
+ if (x.isSymbolicLink() || y.isSymbolicLink()) {
42
+ return x.isSymbolicLink() && y.isSymbolicLink() && await fs.readlink(a) === await fs.readlink(b);
43
+ }
44
+ if (x.isDirectory() && y.isDirectory()) {
45
+ const [left, right] = await Promise.all([fs.readdir(a), fs.readdir(b)]);
46
+ left.sort(); right.sort();
47
+ if (left.length !== right.length || left.some((name, i) => name !== right[i])) return false;
48
+ for (const name of left) if (!await equal(path.join(a, name), path.join(b, name))) return false;
49
+ return true;
50
+ }
51
+ return x.isFile() && y.isFile() && x.size === y.size && await digest(a) === await digest(b);
52
+ }
53
+ async function publish(from, to) {
54
+ // Both paths are on the destination workspace filesystem. Linux performs
55
+ // one atomic directory publication; an existing destination is never replaced.
56
+ const script = "import ctypes,os,sys;libc=ctypes.CDLL(None,use_errno=True);result=libc.renameat2(-100,os.fsencode(sys.argv[1]),-100,os.fsencode(sys.argv[2]),1);err=ctypes.get_errno();sys.exit(0) if result==0 else (_ for _ in ()).throw(OSError(err,os.strerror(err)))";
57
+ const { spawnSync } = require("node:child_process");
58
+ const result = spawnSync("python3", ["-c", script, from, to], { encoding: "utf8" });
59
+ if (result.error) throw result.error;
60
+ if (result.status !== 0) {
61
+ // A racing destination creation is a conflict unless its complete tree
62
+ // is identical. Never merge into or overwrite a live session directory.
63
+ if (!await equal(from, to)) throw new Error(result.stderr || "Atomic sandbox publication failed");
64
+ }
65
+ }
66
+ (async () => {
67
+ const [stage, root] = process.argv.slice(1);
68
+ const source = path.join(stage, "sessions");
69
+ await fs.mkdir(path.join(root, "sessions"), { recursive: true });
70
+ const pending = [];
71
+ for (const name of await fs.readdir(source)) {
72
+ const from = path.join(source, name);
73
+ const to = path.join(root, "sessions", name);
74
+ let exists = true;
75
+ try { await fs.lstat(to); } catch (error) { if (error.code !== "ENOENT") throw error; exists = false; }
76
+ if (exists) {
77
+ if (!await equal(from, to)) throw new Error("Sandbox import conflicts with an existing target session: " + name);
78
+ } else pending.push([from, to]);
79
+ }
80
+ // Validate every conflict before publishing. Never replace a directory
81
+ // that may belong to a turn already running on the target.
82
+ for (const [from, to] of pending) await publish(from, to);
83
+ })().catch(error => { console.error(error.message); process.exitCode = 1; });
84
+ `;
6
85
  /**
7
- * Cloudflare's durable_object containers expose `/dev/kvm`, which makes Eve's
8
- * `DefaultSandbox` pick microsandbox at runtime, while Eden's isolated builder
9
- * (no KVM, no Docker daemon) prepared the just-bash template — every sandbox
10
- * tool then fails with a provider mismatch. Hiding the device before Eve
11
- * starts keeps runtime selection equal to the build's. Same Eve start
12
- * command as the image entrypoint.
86
+ * Cloudflare's durable_object containers expose `/dev/kvm`, which makes Eve
87
+ * versions before 0.83 pick microsandbox at runtime, although Eden's isolated
88
+ * builder prepared just-bash. Hide it to keep those versions' provider
89
+ * selection aligned. Eve 0.83 removed microsandbox and records its provider
90
+ * during build; the device removal remains for older supported Eve releases.
13
91
  */
14
92
  const EVE_HOST_ENTRYPOINT = Object.freeze([
15
93
  "/bin/sh",
@@ -54,56 +132,98 @@ function isMissingContainerSnapshot(error) {
54
132
  /Snapshot ".*" was not found/u.test(error.message);
55
133
  }
56
134
  /** The deployment's single SQLite-backed Workflow World. */
57
- export class EdenWorldDurableObject {
135
+ export class EdenWorldDurableObject extends DurableObject {
58
136
  core;
59
137
  constructor(ctx, env) {
138
+ super(ctx, env);
60
139
  this.core = new WorldCore({
140
+ ...(env.EDEN_EVE_DEPLOYMENT_ID === undefined ? {} : { currentDeploymentId: env.EDEN_EVE_DEPLOYMENT_ID }),
61
141
  sql: {
62
142
  exec: ctx.storage.sql.exec.bind(ctx.storage.sql),
63
143
  transactionSync: ctx.storage.transactionSync.bind(ctx.storage),
64
144
  },
145
+ waitUntil: (promise) => ctx.waitUntil(promise),
65
146
  scheduleAlarm: async (atMs) => {
66
147
  if (atMs === null)
67
148
  await ctx.storage.deleteAlarm();
68
149
  else
69
150
  await ctx.storage.setAlarm(atMs);
70
151
  },
152
+ retireDeployment: async (deploymentId) => {
153
+ if (deploymentId === env.EDEN_EVE_DEPLOYMENT_ID)
154
+ throw new Error("Cannot retire the current ingress deployment.");
155
+ if (!env.EVE_CONTAINER)
156
+ throw new Error("The Container namespace is unavailable for retirement.");
157
+ if (env.EDEN_EVE_DEPLOYMENTS === undefined)
158
+ throw new Error("Cannot retire a deployment without deployment-aware Container routing.");
159
+ const namespace = env.EVE_CONTAINER;
160
+ await namespace.getByName(deploymentInstance(env, deploymentId)).retire(deploymentId);
161
+ },
71
162
  delivery: {
72
163
  deliver: async (message) => {
73
164
  const origin = env.EVE_PUBLIC_ORIGIN;
74
- const instance = env.EVE_CONTAINER_INSTANCE_NAME;
75
- if (typeof origin !== "string" || typeof instance !== "string") {
165
+ const deploymentId = env.EDEN_EVE_DEPLOYMENTS === undefined ? env.EDEN_EVE_DEPLOYMENT_ID : message.deploymentId ?? "eden-cloudflare";
166
+ let sandboxTransferCompleted = false;
167
+ const settleUnstarted = async () => {
168
+ const world = env.EDEN_WORLD;
169
+ await world.getByName("world").deliverySettled(message.messageId, message.attempt);
170
+ };
171
+ if (typeof origin !== "string") {
172
+ await settleUnstarted();
76
173
  throw new Error("The World queue Container identity is unavailable.");
77
174
  }
78
175
  const namespace = env.EVE_CONTAINER;
79
- const response = await namespace.getByName(instance).loopbackFetch(new Request(new URL(`/.well-known/workflow/v1/${message.path}`, origin), {
80
- method: "POST",
81
- headers: {
82
- ...message.headers,
83
- "content-type": "application/json",
84
- "x-vqs-queue-name": message.queueName,
85
- "x-vqs-message-id": message.messageId,
86
- "x-vqs-message-attempt": String(message.attempt),
87
- },
88
- body: message.body,
89
- }));
90
- // Body transport errors must remain retryable; an HTTP status alone
91
- // is not an acknowledgement of a streamed Container response.
92
- const body = await response.text();
93
- let result = null;
176
+ const target = namespace.getByName(deploymentInstance(env, deploymentId));
177
+ if (message.sourceDeploymentId && message.sourceDeploymentId !== deploymentId && message.sandboxSessionId) {
178
+ try {
179
+ const result = await target.importSandboxSessions(message.sourceDeploymentId, deploymentId, message.messageId, message.sandboxSessionId);
180
+ if (result === "source-export-failed") {
181
+ await settleUnstarted();
182
+ return { ok: false, sandboxTransferFailed: true };
183
+ }
184
+ }
185
+ catch (error) {
186
+ await settleUnstarted();
187
+ throw error;
188
+ }
189
+ sandboxTransferCompleted = true;
190
+ }
94
191
  try {
95
- result = JSON.parse(body);
192
+ const response = await target.loopbackFetch(deploymentRequest(new Request(new URL(`/.well-known/workflow/v1/${message.path}`, origin), {
193
+ method: "POST",
194
+ headers: {
195
+ ...message.headers,
196
+ "content-type": "application/json",
197
+ "x-vqs-queue-name": message.queueName,
198
+ "x-vqs-message-id": message.messageId,
199
+ "x-vqs-message-attempt": String(message.attempt),
200
+ },
201
+ body: message.body,
202
+ }), deploymentId), { messageId: message.messageId, attempt: message.attempt });
203
+ // Body transport errors must remain retryable; an HTTP status alone
204
+ // is not an acknowledgement of a streamed Container response.
205
+ const body = await response.text();
206
+ let result = null;
207
+ try {
208
+ result = JSON.parse(body);
209
+ }
210
+ catch {
211
+ // World-local also accepts successful non-JSON responses.
212
+ }
213
+ const timeout = result?.timeoutSeconds;
214
+ return {
215
+ ok: response.ok && timeout === undefined,
216
+ sandboxTransferCompleted,
217
+ ...(typeof timeout === "number" && timeout >= 0
218
+ ? { retryAfterMs: timeout * 1_000 }
219
+ : {}),
220
+ };
96
221
  }
97
- catch {
98
- // World-local also accepts successful non-JSON responses.
222
+ catch (error) {
223
+ if (!sandboxTransferCompleted)
224
+ throw error;
225
+ return { ok: false, sandboxTransferCompleted: true };
99
226
  }
100
- const timeout = result?.timeoutSeconds;
101
- return {
102
- ok: response.ok && timeout === undefined,
103
- ...(typeof timeout === "number" && timeout >= 0
104
- ? { retryAfterMs: timeout * 1_000 }
105
- : {}),
106
- };
107
227
  },
108
228
  },
109
229
  });
@@ -117,6 +237,17 @@ export class EdenWorldDurableObject {
117
237
  headers: { "content-type": "application/cbor" },
118
238
  });
119
239
  }
240
+ deploymentBusy(deploymentId) {
241
+ const activity = this.core.deploymentActivity(deploymentId);
242
+ return activity.activeDeliveries > 0 || activity.unfinishedTransfers > 0;
243
+ }
244
+ deliverySettled(messageId, attempt) {
245
+ this.core.deliverySettled(messageId, attempt);
246
+ }
247
+ async controlFetch(request) {
248
+ const result = await this.core.handleControlRpc(new Uint8Array(await request.arrayBuffer()));
249
+ return new Response(result, { headers: { "content-type": "application/cbor" } });
250
+ }
120
251
  async alarm() {
121
252
  await this.core.runAlarm();
122
253
  }
@@ -129,13 +260,15 @@ export class EdenWorldDurableObject {
129
260
  * URLs stay correct; the DO registers an outbound-HTTPS intercept for that
130
261
  * hostname aimed at `EveHostLoopback`, and this function routes the request:
131
262
  * non-public hosts egress plainly, `/__eden/world/rpc` reaches the World
132
- * Durable Object, and everything else re-enters the container over the
133
- * internal port so the public Worker path can keep refusing the
134
- * unauthenticated queue endpoints.
263
+ * Durable Object, and local World queue deliveries re-enter the container
264
+ * over its internal port. Cloudflare World queue delivery uses direct DO RPC,
265
+ * so its unauthenticated outbound queue requests remain forbidden.
135
266
  */
136
- export async function routeEveOutboundRequest(request, env) {
267
+ export async function routeEveOutboundRequest(request, env, sourceDeploymentId, expectedWorldToken) {
137
268
  const publicOrigin = env.EVE_PUBLIC_ORIGIN;
138
- const instanceName = env.EVE_CONTAINER_INSTANCE_NAME;
269
+ const deploymentId = sourceDeploymentId ?? "eden-cloudflare";
270
+ const instanceName = typeof deploymentId === "string"
271
+ ? deploymentInstance(env, deploymentId) : undefined;
139
272
  if (typeof publicOrigin !== "string" ||
140
273
  typeof instanceName !== "string") {
141
274
  throw new EveHostError("HOST_ORIGIN_UNAVAILABLE", "The container loopback identity is unavailable to the outbound handler.");
@@ -147,16 +280,32 @@ export async function routeEveOutboundRequest(request, env) {
147
280
  return fetch(request);
148
281
  }
149
282
  if (new URL(request.url).pathname === "/__eden/world/rpc") {
283
+ const route = deploymentImages(env)[deploymentId];
284
+ const authRequired = route?.worldRpcAuth === true ||
285
+ (env.EDEN_EVE_DEPLOYMENTS === undefined && env.EDEN_EVE_WORLD_RPC_AUTH === true);
286
+ if (authRequired) {
287
+ const supplied = request.headers.get("x-eden-world-token") ?? "";
288
+ const encoder = new TextEncoder();
289
+ const expected = encoder.encode(expectedWorldToken ?? "");
290
+ const actual = encoder.encode(supplied);
291
+ const subtle = crypto.subtle;
292
+ if (expected.length === 0 || actual.length !== expected.length || !subtle.timingSafeEqual(actual, expected)) {
293
+ return new Response(null, { status: 403 });
294
+ }
295
+ }
150
296
  const world = env.EDEN_WORLD;
151
297
  if (world === undefined)
152
298
  return new Response(null, { status: 404 });
153
299
  return world.get(world.idFromName("world")).fetch(request);
154
300
  }
301
+ if (env.EDEN_EVE_WORLD_CLOUDFLARE && isEveWorkflowInternalRoute(new URL(request.url).pathname)) {
302
+ return new Response(null, { status: 403 });
303
+ }
155
304
  const namespace = env.EVE_CONTAINER;
156
305
  if (namespace === undefined) {
157
306
  throw new EveHostError("HOST_READINESS_UNPROVEN", "The configured Container binding is unavailable to the outbound handler.");
158
307
  }
159
- return namespace.getByName(instanceName).loopbackFetch(request);
308
+ return namespace.getByName(instanceName).loopbackFetch(deploymentRequest(request, deploymentId));
160
309
  }
161
310
  /**
162
311
  * This module is the only Eve-host surface that depends on provider runtime
@@ -172,7 +321,8 @@ export async function routeEveOutboundRequest(request, env) {
172
321
  */
173
322
  export class EveHostLoopback extends WorkerEntrypoint {
174
323
  async fetch(request) {
175
- return routeEveOutboundRequest(request, this.env);
324
+ const props = this.ctx.props;
325
+ return routeEveOutboundRequest(request, this.env, props.deploymentId, props.expectedWorldToken);
176
326
  }
177
327
  }
178
328
  /**
@@ -205,6 +355,136 @@ export class EveHostDurableContainer extends DurableObject {
205
355
  envVars;
206
356
  sleepAfterMs;
207
357
  readiness;
358
+ deploymentId;
359
+ imports = new Map();
360
+ importQueue = Promise.resolve();
361
+ activeExports = 0;
362
+ activeDeliveries = 0;
363
+ activeForwarding = 0;
364
+ schedulerRoleReconcile;
365
+ retainedFences;
366
+ deliveryClaims = new Map();
367
+ async identifyDeployment(requested) {
368
+ const affinity = this.env.EDEN_EVE_DEPLOYMENTS !== undefined;
369
+ let stored = affinity ? this.deploymentId ?? await this.ctx.storage.get("eden.eve.host.deployment-id") : undefined;
370
+ const namespace = this.env.EVE_CONTAINER;
371
+ const legacy = affinity && this.ctx.id.equals(namespace.idFromName(this.env.EVE_CONTAINER_INSTANCE_NAME));
372
+ if (legacy && stored !== undefined && stored !== "eden-cloudflare" && deploymentImages(this.env)[stored] === undefined)
373
+ stored = undefined;
374
+ const named = affinity && !legacy
375
+ ? Object.keys(deploymentImages(this.env)).find((id) => this.ctx.id.equals(namespace.idFromName(deploymentInstance(this.env, id))))
376
+ : undefined;
377
+ const owner = affinity ? (legacy ? "eden-cloudflare" : named) : this.env.EDEN_EVE_DEPLOYMENT_ID;
378
+ const id = affinity ? requested ?? stored ?? owner : owner;
379
+ if (affinity && (owner === undefined || id !== owner)) {
380
+ throw new Error("The Container deployment identity does not match its instance name.");
381
+ }
382
+ if (!id || (stored !== undefined && stored !== id)) {
383
+ throw new Error("The Container deployment identity does not match its persisted owner.");
384
+ }
385
+ const record = deploymentImages(this.env)[id];
386
+ if (stored === undefined)
387
+ await this.ctx.storage.put("eden.eve.host.deployment-id", id);
388
+ this.deploymentId = id;
389
+ this.envVars.EDEN_EVE_DEPLOYMENT_ID = id;
390
+ this.envVars.EDEN_EVE_RELEASE_ID = record?.releaseId ?? this.env.EDEN_EVE_RELEASE_ID ?? id;
391
+ this.envVars.EVE_SCHEDULES_DISABLED = id === this.env.EDEN_EVE_DEPLOYMENT_ID ? "false" : "true";
392
+ this.envVars.EDEN_EVE_GENERATION_ID = record?.generationId ?? this.env.EDEN_EVE_GENERATION_ID;
393
+ if (record?.runtimeRevisionHandle !== undefined) {
394
+ this.envVars.EDEN_EVE_RUNTIME_REVISION = record.runtimeRevisionHandle;
395
+ }
396
+ else {
397
+ delete this.envVars.EDEN_EVE_RUNTIME_REVISION;
398
+ }
399
+ return id;
400
+ }
401
+ async fenceRetainedSchedules() {
402
+ if (this.deploymentId !== this.env.EDEN_EVE_DEPLOYMENT_ID)
403
+ return;
404
+ this.retainedFences ??= (async () => {
405
+ const namespace = this.env.EVE_CONTAINER;
406
+ for (const id of Object.keys(deploymentImages(this.env))) {
407
+ if (id === this.deploymentId)
408
+ continue;
409
+ const request = new Request(new URL("/__eden/internal/fence-schedules", this.env.EVE_PUBLIC_ORIGIN), {
410
+ headers: {
411
+ "x-eden-eve-deployment-id": id,
412
+ "x-eden-eve-release-id": this.env.EDEN_EVE_RELEASE_ID ?? this.env.EDEN_EVE_DEPLOYMENT_ID,
413
+ },
414
+ });
415
+ let response;
416
+ try {
417
+ response = await namespace.getByName(deploymentInstance(this.env, id)).fetch(request);
418
+ }
419
+ catch (error) {
420
+ if (!(error instanceof Error) || !error.message.includes(EVE_HOST_STALE_WORKER_REASON))
421
+ throw error;
422
+ response = await namespace.getByName(deploymentInstance(this.env, id)).fetch(request);
423
+ }
424
+ if (!response.ok)
425
+ throw new Error(`Cannot fence schedules on retained deployment ${id}.`);
426
+ await response.body?.cancel();
427
+ }
428
+ })().catch((error) => { this.retainedFences = undefined; throw error; });
429
+ await this.retainedFences;
430
+ }
431
+ async deploymentBusy(deploymentId) {
432
+ if (this.activeDeliveries > 0 || this.activeExports > 0 || this.imports.size > 0)
433
+ return true;
434
+ if (!this.env.EDEN_EVE_WORLD_CLOUDFLARE)
435
+ return false;
436
+ const world = this.env.EDEN_WORLD;
437
+ return world.getByName("world").deploymentBusy(deploymentId);
438
+ }
439
+ async reconcileSchedulerRole() {
440
+ const runtime = this.containerRuntime();
441
+ if (this.deploymentId !== this.env.EDEN_EVE_DEPLOYMENT_ID || !runtime?.running)
442
+ return;
443
+ if (await this.ctx.storage.get("eden.eve.host.started-schedules-disabled") !== true)
444
+ return;
445
+ this.schedulerRoleReconcile ??= (async () => {
446
+ await this.ctx.storage.put("eden.eve.host.scheduler-fence-pending", true);
447
+ const canRestart = async () => {
448
+ if (this.activeForwarding > 0 || this.activeExports > 0 || this.imports.size > 0)
449
+ return false;
450
+ if (!this.env.EDEN_EVE_WORLD_CLOUDFLARE)
451
+ return true;
452
+ const world = this.env.EDEN_WORLD;
453
+ return !await world.getByName("world").deploymentBusy(this.deploymentId);
454
+ };
455
+ if (!await canRestart()) {
456
+ throw new EveHostError("HOST_READINESS_UNPROVEN", "Scheduler promotion awaits active retained deployment work.");
457
+ }
458
+ const snapshot = await runtime.snapshotContainer({ name: "eden-workspace" });
459
+ const image = (await runtime.inspect().catch(() => null))?.image ||
460
+ await this.ctx.storage.get(EVE_HOST_CONTAINER_STARTED_IMAGE_KEY) || this.expectedImage(runtime);
461
+ if (!image)
462
+ throw new Error("Cannot identify the scheduler promotion snapshot image.");
463
+ await this.ctx.storage.put(EVE_HOST_CONTAINER_SNAPSHOT_KEY, { id: snapshot.id, image, at: Date.now() });
464
+ if (!await canRestart()) {
465
+ throw new EveHostError("HOST_READINESS_UNPROVEN", "Scheduler promotion awaits active retained deployment work.");
466
+ }
467
+ await runtime.destroy(new Error("scheduler-role-promotion"));
468
+ const stoppedAt = Date.now();
469
+ while (runtime.running && Date.now() - stoppedAt < 10_000) {
470
+ await new Promise((resolve) => setTimeout(resolve, 25));
471
+ }
472
+ if (runtime.running) {
473
+ throw new EveHostError("HOST_READINESS_UNPROVEN", "The previous scheduler process has not stopped.");
474
+ }
475
+ this.recordContainerStopped();
476
+ })().finally(() => { this.schedulerRoleReconcile = undefined; });
477
+ await this.schedulerRoleReconcile;
478
+ }
479
+ expectedImage(runtime) {
480
+ const id = this.deploymentId;
481
+ if (id === undefined)
482
+ throw new Error("Container deployment identity has not been resolved.");
483
+ const record = deploymentImages(this.env)[id];
484
+ if (this.env.EDEN_EVE_DEPLOYMENTS !== undefined && record === undefined && id !== "eden-cloudflare")
485
+ throw new Error(`Deployment ${id} has no retained Container image.`);
486
+ return runtime.images[record?.image ?? "eve"];
487
+ }
208
488
  constructor(ctx, env) {
209
489
  super(ctx, env);
210
490
  const publicOrigin = env.EVE_PUBLIC_ORIGIN;
@@ -262,6 +542,10 @@ export class EveHostDurableContainer extends DurableObject {
262
542
  // DO restarts lose the inactivity timeout; the monitor for this
263
543
  // container generation is also gone, so both are re-attached here.
264
544
  this.ctx.blockConcurrencyWhile(async () => {
545
+ if (env.EDEN_EVE_DEPLOYMENTS !== undefined) {
546
+ await this.identifyDeployment();
547
+ await this.registerLoopbackIntercept(runtime);
548
+ }
265
549
  await runtime.setInactivityTimeout(this.inactivityTimeoutMs);
266
550
  this.attachMonitor(runtime);
267
551
  });
@@ -279,10 +563,15 @@ export class EveHostDurableContainer extends DurableObject {
279
563
  if (this.monitorAttached)
280
564
  return;
281
565
  this.monitorAttached = true;
566
+ const generation = this.readinessEpoch;
282
567
  runtime.monitor().then(() => {
568
+ if (generation !== this.readinessEpoch)
569
+ return;
283
570
  this.monitorAttached = false;
284
571
  this.recordContainerStopped();
285
572
  }, (error) => {
573
+ if (generation !== this.readinessEpoch)
574
+ return;
286
575
  this.monitorAttached = false;
287
576
  if (this.startedFromSnapshot && isMissingContainerSnapshot(error)) {
288
577
  // Cloudflare deletes a snapshot 30 days after it was created or
@@ -299,6 +588,14 @@ export class EveHostDurableContainer extends DurableObject {
299
588
  }
300
589
  recordContainerStopped() {
301
590
  this.readinessEpoch += 1;
591
+ this.monitorAttached = false;
592
+ for (const [claim, pending] of this.deliveryClaims) {
593
+ if (pending.generation !== this.readinessEpoch - 1)
594
+ continue;
595
+ this.deliveryClaims.delete(claim);
596
+ pending.fail(new Error("The Container generation stopped before its queue delivery completed."));
597
+ this.ctx.waitUntil(this.settleQueueDelivery(claim));
598
+ }
302
599
  this.readinessEvidence = undefined;
303
600
  this.imageVerified = false;
304
601
  this.pendingCarryRestore = false;
@@ -311,7 +608,10 @@ export class EveHostDurableContainer extends DurableObject {
311
608
  const entryFactory = exportsMap?.["EveHostLoopback"];
312
609
  if (typeof entryFactory !== "function")
313
610
  return;
314
- const entrypoint = entryFactory({ props: {} });
611
+ const entrypoint = entryFactory({ props: {
612
+ deploymentId: await this.identifyDeployment(),
613
+ expectedWorldToken: await this.ctx.storage.get("eden.eve.host.world-token") ?? "",
614
+ } });
315
615
  await runtime.interceptOutboundHttps(new URL(this.env.EVE_PUBLIC_ORIGIN ?? "").hostname, entrypoint);
316
616
  }
317
617
  /**
@@ -322,7 +622,8 @@ export class EveHostDurableContainer extends DurableObject {
322
622
  async touchActivity() {
323
623
  const now = Date.now();
324
624
  await this.ctx.storage.put(EVE_HOST_CONTAINER_ACTIVITY_KEY, now);
325
- await this.ctx.storage.setAlarm(now + this.sleepAfterMs);
625
+ const fencing = await this.ctx.storage.get("eden.eve.host.scheduler-fence-pending") === true;
626
+ await this.ctx.storage.setAlarm(fencing ? now : now + this.sleepAfterMs);
326
627
  }
327
628
  /**
328
629
  * Starts the container when it is not running. A stored workspace snapshot
@@ -338,7 +639,7 @@ export class EveHostDurableContainer extends DurableObject {
338
639
  }
339
640
  if (runtime.running)
340
641
  return;
341
- const expectedImage = runtime.images["eve"];
642
+ const expectedImage = this.expectedImage(runtime);
342
643
  if (expectedImage === undefined) {
343
644
  throw new EveHostError("HOST_READINESS_UNPROVEN", "The configured Container image is unavailable to this Worker version.");
344
645
  }
@@ -347,6 +648,18 @@ export class EveHostDurableContainer extends DurableObject {
347
648
  storedSnapshot.image === expectedImage
348
649
  ? storedSnapshot.id
349
650
  : undefined;
651
+ // Persist the fresh-start restore obligation before the process exists:
652
+ // a DO replacement must not mistake a partially restored process for ready.
653
+ if (snapshotId === undefined) {
654
+ await this.ctx.storage.put("eden.eve.host.restore-pending", globalThis.crypto.randomUUID());
655
+ }
656
+ else {
657
+ await this.ctx.storage.delete("eden.eve.host.restore-pending");
658
+ }
659
+ await this.ctx.storage.put("eden.eve.host.started-schedules-disabled", this.envVars.EVE_SCHEDULES_DISABLED === "true");
660
+ const worldToken = globalThis.crypto.randomUUID();
661
+ await this.ctx.storage.put("eden.eve.host.world-token", worldToken);
662
+ this.envVars.EDEN_WORLD_TOKEN = worldToken;
350
663
  runtime.start({
351
664
  ...(snapshotId === undefined
352
665
  ? { image: expectedImage }
@@ -356,12 +669,19 @@ export class EveHostDurableContainer extends DurableObject {
356
669
  env: this.envVars,
357
670
  entrypoint: EVE_HOST_ENTRYPOINT,
358
671
  });
672
+ console.log("eden.container.start", JSON.stringify({
673
+ deploymentId: this.deploymentId, image: expectedImage,
674
+ source: snapshotId === undefined ? "image" : "snapshot",
675
+ }));
359
676
  // Set before any await: a failure below makes the readiness retry see a
360
677
  // running container and skip this method, so the restore must already be
361
678
  // pending. A start from the configured image (an update) has no sandbox
362
679
  // sessions yet; a same-image snapshot wake already contains them.
363
680
  this.pendingCarryRestore = snapshotId === undefined;
364
681
  this.startedFromSnapshot = snapshotId !== undefined;
682
+ // A fresh process receives the retained scheduler fence even when its
683
+ // filesystem came from a snapshot of the formerly-current deployment.
684
+ await this.ctx.storage.delete("eden.eve.host.scheduler-fence-pending");
365
685
  // Both start sources run `expectedImage` (a snapshot is only restored
366
686
  // from that image).
367
687
  await this.ctx.storage.put(EVE_HOST_CONTAINER_STARTED_IMAGE_KEY, expectedImage);
@@ -384,7 +704,7 @@ export class EveHostDurableContainer extends DurableObject {
384
704
  const runtime = this.containerRuntime();
385
705
  if (runtime === undefined || !runtime.running)
386
706
  return;
387
- const expectedImage = runtime.images["eve"];
707
+ const expectedImage = this.expectedImage(runtime);
388
708
  let info;
389
709
  try {
390
710
  info = await runtime.inspect();
@@ -423,12 +743,12 @@ export class EveHostDurableContainer extends DurableObject {
423
743
  * image. One carry runs at a time on this Durable Object; a failure never
424
744
  * blocks the destroy that follows and leaves the previous carry intact.
425
745
  */
426
- carrySandboxSessionsOut(runtime) {
427
- const run = this.carryOutQueue.then(() => this.carrySandboxSessionsOutNow(runtime));
746
+ carrySandboxSessionsOut(runtime, required = false) {
747
+ const run = this.carryOutQueue.then(() => this.carrySandboxSessionsOutNow(runtime, required));
428
748
  this.carryOutQueue = run.then(() => undefined, () => undefined);
429
749
  return run;
430
750
  }
431
- async carrySandboxSessionsOutNow(runtime) {
751
+ async carrySandboxSessionsOutNow(runtime, required = false) {
432
752
  const previous = await this.ctx.storage.get(EVE_HOST_SANDBOX_CARRY_KEY);
433
753
  const carryId = globalThis.crypto.randomUUID();
434
754
  const chunkPrefix = `${EVE_HOST_SANDBOX_CARRY_CHUNK_PREFIX}.${carryId}.`;
@@ -502,6 +822,8 @@ export class EveHostDurableContainer extends DurableObject {
502
822
  await reader.cancel().catch(() => undefined);
503
823
  if (overflowed) {
504
824
  await discardWrittenChunks();
825
+ if (required)
826
+ throw new Error("The imported sandbox carry exceeds the archive limit.");
505
827
  console.error(`The Eve sandbox session carry exceeded ${EVE_HOST_CARRY_LIMITS.maxBytes} compressed bytes; keeping the previous carry.`);
506
828
  return;
507
829
  }
@@ -520,6 +842,8 @@ export class EveHostDurableContainer extends DurableObject {
520
842
  }
521
843
  if (exitCode !== 0) {
522
844
  await discardWrittenChunks();
845
+ if (required)
846
+ throw new Error(`The imported sandbox carry failed with exit code ${exitCode}.`);
523
847
  console.error(`The Eve sandbox session carry failed with exit code ${exitCode}; keeping the previous carry.`);
524
848
  return;
525
849
  }
@@ -546,10 +870,22 @@ export class EveHostDurableContainer extends DurableObject {
546
870
  * the same promise, and no request is forwarded before it settles.
547
871
  */
548
872
  async restoreCarriedSandboxSessions() {
549
- if (!this.pendingCarryRestore)
873
+ const restoreGeneration = await this.ctx.storage.get("eden.eve.host.restore-pending");
874
+ if (!this.pendingCarryRestore && restoreGeneration === undefined)
550
875
  return;
551
- this.carryRestorePromise ??= this.carrySandboxSessionsIn();
552
- await this.carryRestorePromise;
876
+ const required = await this.ctx.storage.get("eden.eve.host.has-imported-sandboxes") === true;
877
+ this.carryRestorePromise ??= this.carrySandboxSessionsIn(required);
878
+ try {
879
+ await this.carryRestorePromise;
880
+ }
881
+ catch (error) {
882
+ this.carryRestorePromise = undefined;
883
+ throw error;
884
+ }
885
+ if (restoreGeneration !== undefined &&
886
+ await this.ctx.storage.get("eden.eve.host.restore-pending") === restoreGeneration) {
887
+ await this.ctx.storage.delete("eden.eve.host.restore-pending");
888
+ }
553
889
  this.pendingCarryRestore = false;
554
890
  }
555
891
  /**
@@ -559,11 +895,14 @@ export class EveHostDurableContainer extends DurableObject {
559
895
  * failure is logged and swallowed: the agent still serves, only the
560
896
  * carried files are missing.
561
897
  */
562
- async carrySandboxSessionsIn() {
898
+ async carrySandboxSessionsIn(required = false) {
563
899
  try {
564
900
  const carry = await this.ctx.storage.get(EVE_HOST_SANDBOX_CARRY_KEY);
565
- if (carry === undefined)
901
+ if (carry === undefined) {
902
+ if (required)
903
+ throw new Error("The imported sandbox carry is missing.");
566
904
  return;
905
+ }
567
906
  let nextChunk = 0;
568
907
  const archive = new ReadableStream({
569
908
  pull: async (controller) => {
@@ -585,10 +924,15 @@ export class EveHostDurableContainer extends DurableObject {
585
924
  if (runtime === undefined) {
586
925
  throw new Error("the Container runtime is unavailable");
587
926
  }
927
+ const restoreGeneration = await this.ctx.storage.get("eden.eve.host.restore-pending");
928
+ const stageId = await sandboxSessionDirectory(restoreGeneration ?? carry.id);
929
+ const stage = `${EVE_HOST_SANDBOX_CARRY_SOURCE}/.eden-restores/${stageId}`;
588
930
  const exec = await runtime.exec([
589
931
  "sh",
590
932
  "-c",
591
- `mkdir -p ${EVE_HOST_SANDBOX_CARRY_SOURCE} && tar -xzf - -C ${EVE_HOST_SANDBOX_CARRY_SOURCE}`,
933
+ required
934
+ ? `rm -rf ${stage} && mkdir -p ${stage} && tar -xzf - -C ${stage}`
935
+ : `mkdir -p ${EVE_HOST_SANDBOX_CARRY_SOURCE} && tar -xzf - -C ${EVE_HOST_SANDBOX_CARRY_SOURCE}`,
592
936
  ], { stdin: archive, stderr: "pipe" });
593
937
  let exitCode = -1;
594
938
  try {
@@ -597,15 +941,30 @@ export class EveHostDurableContainer extends DurableObject {
597
941
  catch {
598
942
  exitCode = -1;
599
943
  }
600
- if (exitCode !== 0) {
601
- console.error(`The Eve sandbox session restore failed with exit code ${exitCode}; the agent serves without the carried files.`);
944
+ if (exitCode !== 0)
945
+ throw new Error(`The sandbox restore failed with exit code ${exitCode}.`);
946
+ if (required) {
947
+ const publish = await runtime.exec(["node", "-e", EVE_HOST_IMPORT_SANDBOXES, stage, EVE_HOST_SANDBOX_CARRY_SOURCE], { stdout: "ignore", stderr: "pipe" });
948
+ const error = publish.stderr === null ? "" : await new Response(publish.stderr).text();
949
+ if (await publish.exitCode !== 0)
950
+ throw new Error(`The imported sandbox restore publication failed: ${error}`);
951
+ const cleanup = await runtime.exec(["rm", "-rf", stage], { stdout: "ignore", stderr: "ignore" });
952
+ await cleanup.exitCode;
602
953
  }
954
+ await this.ctx.storage.delete("eden.eve.host.sandbox-restore-failed");
603
955
  }
604
956
  catch (error) {
957
+ await this.ctx.storage.put("eden.eve.host.sandbox-restore-failed", true);
958
+ if (required)
959
+ throw error;
605
960
  console.error("The Eve sandbox session restore failed; the agent serves without the carried files:", error instanceof Error ? error.message : "unknown");
606
961
  }
607
962
  }
608
- async ensureEveReady(signal = new AbortController().signal) {
963
+ async ensureEveReady(signal = new AbortController().signal, requireSchedulerRole = true) {
964
+ await this.identifyDeployment();
965
+ await this.fenceRetainedSchedules();
966
+ if (requireSchedulerRole)
967
+ await this.reconcileSchedulerRole();
609
968
  await this.reconcileRunningImage();
610
969
  if (this.readinessEvidence !== undefined) {
611
970
  return this.readinessEvidence;
@@ -627,13 +986,18 @@ export class EveHostDurableContainer extends DurableObject {
627
986
  }
628
987
  async fetch(request) {
629
988
  const requestDeploymentId = request.headers.get("x-eden-eve-deployment-id") ?? undefined;
630
- if (requestDeploymentId !== undefined &&
631
- requestDeploymentId !== this.env.EDEN_EVE_DEPLOYMENT_ID) {
632
- // A live container/monitor can pin the DO to an older Worker version.
633
- // Reset the DO, not its container or storage; the retry's current code
634
- // reconciles the image and carries sandbox sessions through the update.
989
+ const requestedRelease = request.headers.get("x-eden-eve-release-id") ?? requestDeploymentId;
990
+ if (requestedRelease !== undefined && requestedRelease !== (this.env.EDEN_EVE_RELEASE_ID ?? this.env.EDEN_EVE_DEPLOYMENT_ID)) {
635
991
  this.ctx.abort(EVE_HOST_STALE_WORKER_REASON);
636
992
  }
993
+ await this.identifyDeployment(requestDeploymentId);
994
+ if (new URL(request.url).pathname === "/__eden/internal/fence-schedules") {
995
+ if (this.deploymentId !== this.env.EDEN_EVE_DEPLOYMENT_ID) {
996
+ await this.ctx.storage.put("eden.eve.host.scheduler-fence-pending", true);
997
+ await this.ctx.storage.setAlarm(Date.now());
998
+ }
999
+ return new Response(null, { status: 204 });
1000
+ }
637
1001
  const response = await this.forwardToEve(request);
638
1002
  // Reporting the verified start back lets deploy health-gate promotion on
639
1003
  // the served container actually running this generation's image: the
@@ -641,11 +1005,11 @@ export class EveHostDurableContainer extends DurableObject {
641
1005
  // Worker version's configured image and the request names this
642
1006
  // deployment.
643
1007
  if (requestDeploymentId !== undefined &&
644
- requestDeploymentId === this.env.EDEN_EVE_DEPLOYMENT_ID &&
1008
+ requestDeploymentId === this.deploymentId &&
645
1009
  this.readinessEvidence !== undefined &&
646
1010
  this.imageVerified) {
647
1011
  const headers = new Headers(response.headers);
648
- headers.set("x-eden-eve-started-deployment", requestDeploymentId);
1012
+ headers.set("x-eden-eve-started-deployment", this.env.EDEN_EVE_RELEASE_ID ?? requestDeploymentId);
649
1013
  return new Response(response.body, {
650
1014
  status: response.status,
651
1015
  statusText: response.statusText,
@@ -654,25 +1018,305 @@ export class EveHostDurableContainer extends DurableObject {
654
1018
  }
655
1019
  return response;
656
1020
  }
1021
+ async retire(deploymentId) {
1022
+ // World routes the fenced target to its named DO. Retirement must not
1023
+ // restore or trust a stored owner left by an earlier retirement attempt.
1024
+ if (await this.deploymentBusy(deploymentId)) {
1025
+ throw new Error("Cannot retire a deployment with active deliveries or transfers.");
1026
+ }
1027
+ const runtime = this.containerRuntime();
1028
+ if (runtime?.running)
1029
+ await runtime.destroy(new Error("deployment-retired"));
1030
+ this.recordContainerStopped();
1031
+ await this.ctx.storage.deleteAlarm();
1032
+ await this.ctx.storage.deleteAll();
1033
+ }
1034
+ /** Read-only export: never replace the source carry or stop its process. */
1035
+ async exportSandboxSessions(deploymentId, sessionId) {
1036
+ this.activeExports += 1;
1037
+ let released = false;
1038
+ const release = () => {
1039
+ if (!released)
1040
+ this.activeExports -= 1;
1041
+ released = true;
1042
+ };
1043
+ try {
1044
+ await this.identifyDeployment(deploymentId);
1045
+ await this.ensureEveReady(undefined, false);
1046
+ if (await this.ctx.storage.get("eden.eve.host.sandbox-restore-failed") === true) {
1047
+ throw new Error("Cannot export sandbox sessions after a failed carry restore.");
1048
+ }
1049
+ await this.touchActivity();
1050
+ const directory = await sandboxSessionDirectory(sessionId);
1051
+ const runtime = this.containerRuntime();
1052
+ const probe = await runtime.exec(["sh", "-c", `if [ -d ${EVE_HOST_SANDBOX_CARRY_SOURCE}/sessions/${directory} ]; then exit 0; elif [ ! -e ${EVE_HOST_SANDBOX_CARRY_SOURCE}/sessions/${directory} ]; then exit 3; else exit 2; fi`], { stdout: "ignore", stderr: "ignore" });
1053
+ const existence = await probe.exitCode;
1054
+ if (existence === 3) {
1055
+ release();
1056
+ return new Response(new Uint8Array(), { headers: { "x-eden-sandbox-directory-exists": "false" } });
1057
+ }
1058
+ if (existence !== 0)
1059
+ throw new Error("Cannot establish source sandbox directory existence.");
1060
+ const exec = await runtime.exec([
1061
+ "sh", "-c",
1062
+ `cd ${EVE_HOST_SANDBOX_CARRY_SOURCE} && tar -czf - sessions/${directory}`,
1063
+ ], { stdout: "pipe", stderr: "ignore" });
1064
+ if (exec.stdout === null)
1065
+ throw new Error("Sandbox export stdout is unavailable.");
1066
+ const reader = exec.stdout.getReader();
1067
+ let bytes = 0;
1068
+ const archive = new ReadableStream({
1069
+ pull: async (controller) => {
1070
+ try {
1071
+ const result = await reader.read();
1072
+ if (result.done) {
1073
+ if (await exec.exitCode !== 0)
1074
+ throw new Error("The source sandbox archive failed.");
1075
+ release();
1076
+ controller.close();
1077
+ return;
1078
+ }
1079
+ bytes += result.value.byteLength;
1080
+ if (bytes > EVE_HOST_CARRY_LIMITS.maxBytes)
1081
+ throw new Error("The source sandbox archive exceeds the carry limit.");
1082
+ controller.enqueue(result.value);
1083
+ }
1084
+ catch (error) {
1085
+ release();
1086
+ await reader.cancel().catch(() => undefined);
1087
+ controller.error(error);
1088
+ }
1089
+ },
1090
+ cancel: async () => { release(); await reader.cancel(); },
1091
+ });
1092
+ return new Response(archive, { headers: { "content-type": "application/gzip", "x-eden-sandbox-directory-exists": "true" } });
1093
+ }
1094
+ catch (error) {
1095
+ release();
1096
+ throw error;
1097
+ }
1098
+ }
1099
+ /** Retry-idempotent imports finish before the migrated queue is dispatched. */
1100
+ async importSandboxSessions(sourceDeploymentId, deploymentId, transferId, sessionId) {
1101
+ await this.identifyDeployment(deploymentId);
1102
+ const key = `eden.eve.host.sandbox-import.v2.${sourceDeploymentId}.${sessionId}.${transferId}`;
1103
+ if (await this.ctx.storage.get(key)) {
1104
+ await this.ensureEveReady(undefined, false);
1105
+ return;
1106
+ }
1107
+ const existing = this.imports.get(key);
1108
+ if (existing !== undefined)
1109
+ return existing;
1110
+ const run = this.importQueue.then(async () => {
1111
+ await this.ensureEveReady(undefined, false);
1112
+ if (await this.ctx.storage.get(key))
1113
+ return;
1114
+ await this.touchActivity();
1115
+ const namespace = this.env.EVE_CONTAINER;
1116
+ let response;
1117
+ try {
1118
+ response = await namespace.getByName(deploymentInstance(this.env, sourceDeploymentId))
1119
+ .exportSandboxSessions(sourceDeploymentId, sessionId);
1120
+ }
1121
+ catch {
1122
+ return "source-export-failed";
1123
+ }
1124
+ if (!response.ok || response.body === null)
1125
+ return "source-export-failed";
1126
+ const noDirectory = response.headers.get("x-eden-sandbox-directory-exists") === "false";
1127
+ const stageId = await sandboxSessionDirectory(JSON.stringify([sourceDeploymentId, sessionId, transferId]));
1128
+ const stage = `${EVE_HOST_SANDBOX_CARRY_SOURCE}/.eden-imports/${stageId}`;
1129
+ const runtime = this.containerRuntime();
1130
+ let sourceExportFailed = false;
1131
+ const reader = response.body.getReader();
1132
+ const archive = new ReadableStream({
1133
+ pull: async (controller) => {
1134
+ try {
1135
+ const result = await reader.read();
1136
+ if (result.done)
1137
+ controller.close();
1138
+ else
1139
+ controller.enqueue(result.value);
1140
+ }
1141
+ catch (error) {
1142
+ sourceExportFailed = true;
1143
+ controller.error(error);
1144
+ }
1145
+ },
1146
+ cancel: (reason) => reader.cancel(reason),
1147
+ });
1148
+ try {
1149
+ const extract = await runtime.exec([
1150
+ "sh", "-c",
1151
+ `rm -rf ${stage} && mkdir -p ${stage} && cat > ${stage}/archive.tgz && if [ -s ${stage}/archive.tgz ]; then tar -xzf ${stage}/archive.tgz -C ${stage}; else ${noDirectory ? `mkdir -p ${stage}/sessions` : "exit 1"}; fi`,
1152
+ ], { stdin: archive, stdout: "ignore", stderr: "ignore" });
1153
+ const exitCode = await extract.exitCode;
1154
+ if (sourceExportFailed)
1155
+ return "source-export-failed";
1156
+ if (exitCode !== 0)
1157
+ throw new Error(`Sandbox archive extraction failed; staging retained at ${stage}.`);
1158
+ }
1159
+ catch (error) {
1160
+ if (sourceExportFailed)
1161
+ return "source-export-failed";
1162
+ throw error;
1163
+ }
1164
+ const publish = await runtime.exec([
1165
+ "node", "-e", EVE_HOST_IMPORT_SANDBOXES, stage, EVE_HOST_SANDBOX_CARRY_SOURCE,
1166
+ ], { stdout: "ignore", stderr: "pipe" });
1167
+ const errorText = publish.stderr === null ? "" : await new Response(publish.stderr).text();
1168
+ if (await publish.exitCode !== 0)
1169
+ throw new Error(`Sandbox import refused: ${errorText.trim()}; staging retained at ${stage}.`);
1170
+ // Persist imports before acknowledging them, including crash recovery.
1171
+ // An older whole-container snapshot must not hide the refreshed carry.
1172
+ await this.ctx.storage.delete(EVE_HOST_CONTAINER_SNAPSHOT_KEY);
1173
+ await this.carrySandboxSessionsOut(runtime, true);
1174
+ await this.ctx.storage.put("eden.eve.host.has-imported-sandboxes", true);
1175
+ await this.ctx.storage.put(key, true);
1176
+ const cleanup = await runtime.exec(["rm", "-rf", stage], { stdout: "ignore", stderr: "ignore" });
1177
+ await cleanup.exitCode;
1178
+ });
1179
+ this.importQueue = run.then(() => undefined, () => undefined);
1180
+ this.imports.set(key, run);
1181
+ try {
1182
+ return await run;
1183
+ }
1184
+ finally {
1185
+ this.imports.delete(key);
1186
+ }
1187
+ }
657
1188
  /**
658
1189
  * Queue deliveries and loopback traffic enter here: same readiness and
659
1190
  * forwarding as `fetch` without the deployment-header echo rules (the
660
1191
  * loopback path is not a deployment-health signal).
661
1192
  */
662
- async loopbackFetch(request) {
663
- return this.forwardToEve(request);
1193
+ async loopbackFetch(request, claim) {
1194
+ await this.identifyDeployment(request.headers.get("x-eden-eve-deployment-id") ?? undefined);
1195
+ return this.forwardToEve(request, claim);
664
1196
  }
665
- async forwardToEve(request) {
666
- await this.ensureEveReady(request.signal);
667
- await this.touchActivity();
668
- const runtime = this.containerRuntime();
669
- if (runtime === undefined) {
670
- throw new EveHostError("HOST_READINESS_UNPROVEN", "The Container runtime is unavailable on this Durable Object.");
1197
+ async settleQueueDelivery(claim) {
1198
+ if (!this.env.EDEN_EVE_WORLD_CLOUDFLARE || claim === undefined)
1199
+ return;
1200
+ const { messageId, attempt } = claim;
1201
+ const world = this.env.EDEN_WORLD;
1202
+ // Resolve the current World actor rather than relying on the dispatcher
1203
+ // instance that initiated this potentially long-lived HTTP request.
1204
+ await world.getByName("world").deliverySettled(messageId, attempt);
1205
+ }
1206
+ async forwardToEve(request, claim) {
1207
+ this.activeDeliveries += 1;
1208
+ let released = false;
1209
+ let forwarding = false;
1210
+ let forwardingGeneration;
1211
+ const release = () => {
1212
+ if (released)
1213
+ return;
1214
+ released = true;
1215
+ if (claim !== undefined)
1216
+ this.deliveryClaims.delete(claim);
1217
+ this.activeDeliveries -= 1;
1218
+ if (forwarding)
1219
+ this.activeForwarding -= 1;
1220
+ if (forwardingGeneration === undefined || forwardingGeneration === this.readinessEpoch) {
1221
+ this.ctx.waitUntil(this.touchActivity());
1222
+ }
1223
+ };
1224
+ try {
1225
+ // Already-pinned work must be able to drain a scheduler-disabled
1226
+ // promoted instance; public readiness waits for its safe role restart.
1227
+ const workflowDelivery = /^\/\.well-known\/workflow\/v1\/(?:flow|step)(?:\/|$)/u.test(new URL(request.url).pathname);
1228
+ await this.ensureEveReady(request.signal, !workflowDelivery);
1229
+ await this.touchActivity();
1230
+ const runtime = this.containerRuntime();
1231
+ if (runtime === undefined)
1232
+ throw new Error("The Container runtime is unavailable.");
1233
+ const url = new URL(request.url);
1234
+ let failGeneration;
1235
+ const generationStopped = new Promise((_resolve, reject) => {
1236
+ failGeneration = reject;
1237
+ });
1238
+ forwardingGeneration = this.readinessEpoch;
1239
+ if (claim !== undefined) {
1240
+ this.deliveryClaims.set(claim, {
1241
+ generation: forwardingGeneration,
1242
+ fail: (error) => { release(); failGeneration(error); },
1243
+ });
1244
+ }
1245
+ forwarding = true;
1246
+ this.activeForwarding += 1;
1247
+ const upstreamInit = {
1248
+ method: request.method,
1249
+ headers: request.headers,
1250
+ body: request.body,
1251
+ redirect: request.redirect,
1252
+ duplex: "half",
1253
+ // A dispatcher replacement must not cancel the actual queue handler.
1254
+ ...(workflowDelivery ? {} : { signal: request.signal }),
1255
+ };
1256
+ const response = await Promise.race([runtime.getTcpPort(EVE_HOST_DEFAULTS.internalPort).fetch(new Request(`http://container${url.pathname}${url.search}`, upstreamInit)), generationStopped]);
1257
+ if (response.body === null) {
1258
+ await this.settleQueueDelivery(claim);
1259
+ release();
1260
+ return response;
1261
+ }
1262
+ const reader = response.body.getReader();
1263
+ let downstreamCancelled = false;
1264
+ const body = new ReadableStream({
1265
+ pull: async (controller) => {
1266
+ try {
1267
+ const result = await Promise.race([reader.read(), generationStopped]);
1268
+ if (result.done) {
1269
+ await this.settleQueueDelivery(claim);
1270
+ release();
1271
+ if (!downstreamCancelled)
1272
+ controller.close();
1273
+ }
1274
+ else if (!downstreamCancelled)
1275
+ controller.enqueue(result.value);
1276
+ }
1277
+ catch (error) {
1278
+ if (!downstreamCancelled) {
1279
+ release();
1280
+ controller.error(error);
1281
+ }
1282
+ }
1283
+ },
1284
+ cancel: async (reason) => {
1285
+ if (!workflowDelivery) {
1286
+ try {
1287
+ await reader.cancel(reason);
1288
+ }
1289
+ finally {
1290
+ release();
1291
+ }
1292
+ return;
1293
+ }
1294
+ downstreamCancelled = true;
1295
+ // Keep observing the upstream invocation independently of its old
1296
+ // dispatcher consumer. Only actual EOF permits exact settlement.
1297
+ this.ctx.waitUntil((async () => {
1298
+ try {
1299
+ while (!(await Promise.race([reader.read(), generationStopped])).done) { /* Discard the response body. */ }
1300
+ await this.settleQueueDelivery(claim);
1301
+ }
1302
+ finally {
1303
+ release();
1304
+ }
1305
+ })());
1306
+ },
1307
+ });
1308
+ return new Response(body, { status: response.status, statusText: response.statusText, headers: response.headers });
1309
+ }
1310
+ catch (error) {
1311
+ try {
1312
+ if (!forwarding)
1313
+ await this.settleQueueDelivery(claim);
1314
+ }
1315
+ finally {
1316
+ release();
1317
+ }
1318
+ throw error;
671
1319
  }
672
- const url = new URL(request.url);
673
- return runtime
674
- .getTcpPort(EVE_HOST_DEFAULTS.internalPort)
675
- .fetch(new Request(`http://container${url.pathname}${url.search}`, request));
676
1320
  }
677
1321
  /**
678
1322
  * The idle-sleep path: once the container has been quiet for `sleepAfter`,
@@ -682,10 +1326,16 @@ export class EveHostDurableContainer extends DurableObject {
682
1326
  * for the running container stops.
683
1327
  */
684
1328
  async alarm() {
1329
+ const deploymentId = await this.identifyDeployment();
685
1330
  const runtime = this.containerRuntime();
686
1331
  const lastActivity = (await this.ctx.storage.get(EVE_HOST_CONTAINER_ACTIVITY_KEY)) ?? 0;
687
1332
  const now = Date.now();
688
- if (now - lastActivity < this.sleepAfterMs) {
1333
+ if (await this.deploymentBusy(deploymentId)) {
1334
+ await this.ctx.storage.setAlarm(now + this.sleepAfterMs);
1335
+ return;
1336
+ }
1337
+ const fencePending = await this.ctx.storage.get("eden.eve.host.scheduler-fence-pending") === true;
1338
+ if (!fencePending && now - lastActivity < this.sleepAfterMs) {
689
1339
  await this.ctx.storage.setAlarm(lastActivity + this.sleepAfterMs);
690
1340
  return;
691
1341
  }
@@ -706,7 +1356,7 @@ export class EveHostDurableContainer extends DurableObject {
706
1356
  name: "eden-workspace",
707
1357
  });
708
1358
  const image = (await runtime.inspect().catch(() => null))?.image ??
709
- runtime.images["eve"] ??
1359
+ this.expectedImage(runtime) ??
710
1360
  "";
711
1361
  const record = {
712
1362
  id: snapshot.id,
@@ -719,15 +1369,21 @@ export class EveHostDurableContainer extends DurableObject {
719
1369
  // request that arrived meanwhile must not have its container destroyed
720
1370
  // underneath it. Keep running and re-arm sleep from that activity.
721
1371
  const activityAfterSnapshot = (await this.ctx.storage.get(EVE_HOST_CONTAINER_ACTIVITY_KEY)) ?? 0;
722
- if (activityAfterSnapshot !== lastActivity) {
723
- await this.ctx.storage.setAlarm(activityAfterSnapshot + this.sleepAfterMs);
1372
+ if (activityAfterSnapshot !== lastActivity || await this.deploymentBusy(deploymentId)) {
1373
+ const fencing = await this.ctx.storage.get("eden.eve.host.scheduler-fence-pending") === true;
1374
+ await this.ctx.storage.setAlarm(fencing ? Date.now() : activityAfterSnapshot + this.sleepAfterMs);
724
1375
  return;
725
1376
  }
726
1377
  await runtime.destroy(new Error("idle-snapshot-done"));
727
1378
  this.recordContainerStopped();
1379
+ await this.ctx.storage.delete("eden.eve.host.scheduler-fence-pending");
728
1380
  }
729
1381
  catch (error) {
730
1382
  const message = error instanceof Error ? error.message : "unknown";
1383
+ if (await this.deploymentBusy(deploymentId)) {
1384
+ await this.touchActivity();
1385
+ return;
1386
+ }
731
1387
  if (attempts + 1 >= EVE_HOST_SNAPSHOT_MAX_ATTEMPTS) {
732
1388
  console.error(`The Eve workspace snapshot failed ${attempts + 1} times; destroying the container without it: ${message}`);
733
1389
  await this.ctx.storage.put(EVE_HOST_CONTAINER_SNAPSHOT_ATTEMPTS_KEY, 0);
@@ -753,12 +1409,13 @@ export function createEveHostWorker(options) {
753
1409
  if (namespace === undefined) {
754
1410
  throw new EveHostError("HOST_READINESS_UNPROVEN", "The configured Container binding is unavailable.");
755
1411
  }
756
- return namespace.getByName(options.stableContainerInstanceName);
1412
+ return namespace.getByName(deploymentInstance(env, options.deploymentId, options.stableContainerInstanceName));
757
1413
  };
758
1414
  const forwardToContainer = async (request, env) => {
759
1415
  const metadata = {
760
1416
  publicOrigin: options.publicOrigin,
761
1417
  deploymentId: options.deploymentId,
1418
+ releaseId: options.releaseId ?? options.deploymentId,
762
1419
  generationId: options.generationId,
763
1420
  correlationId: globalThis.crypto.randomUUID(),
764
1421
  ...(options.runtimeRevisionHandle === undefined
@@ -794,10 +1451,28 @@ export function createEveHostWorker(options) {
794
1451
  };
795
1452
  return {
796
1453
  async fetch(request, env) {
1454
+ if (new URL(request.url).pathname === "/__eden/world/rpc") {
1455
+ if (request.method !== "POST")
1456
+ return new Response(null, { status: 405 });
1457
+ const token = env.EDEN_EVE_CONTROL_TOKEN;
1458
+ const authorization = request.headers.get("authorization") ?? "";
1459
+ if (typeof token !== "string" || token.length === 0)
1460
+ return new Response(null, { status: 404 });
1461
+ // Hash both values to fixed lengths before comparing every byte.
1462
+ const digest = async (value) => new Uint8Array(await crypto.subtle.digest("SHA-256", new TextEncoder().encode(value)));
1463
+ const [expected, supplied] = await Promise.all([digest(`Bearer ${token}`), digest(authorization)]);
1464
+ let mismatch = 0;
1465
+ for (let index = 0; index < expected.length; index += 1)
1466
+ mismatch |= expected[index] ^ supplied[index];
1467
+ if (mismatch !== 0)
1468
+ return new Response(null, { status: 401 });
1469
+ const world = env.EDEN_WORLD;
1470
+ return world === undefined ? new Response(null, { status: 404 }) : world.getByName("world").controlFetch(request);
1471
+ }
797
1472
  // Queue deliveries to these endpoints arrive through the container's
798
1473
  // intercepted outbound loopback instead, so any request that reaches
799
1474
  // the public Worker fetch path is external and unauthenticated.
800
- if (isEveWorkflowInternalRoute(new URL(request.url).pathname)) {
1475
+ if (isEveWorkflowInternalRoute(new URL(request.url).pathname) || new URL(request.url).pathname.startsWith("/__eden/internal/")) {
801
1476
  return new Response(null, { status: 404 });
802
1477
  }
803
1478
  const response = await forwardToContainer(request, env);
@@ -805,8 +1480,9 @@ export function createEveHostWorker(options) {
805
1480
  headers.set("x-eden-eve-worker-name", options.workerName);
806
1481
  headers.set("x-eden-eve-public-origin", options.publicOrigin);
807
1482
  headers.set("x-eden-eve-deployment-id", options.deploymentId);
1483
+ headers.set("x-eden-eve-release-id", options.releaseId ?? options.deploymentId);
808
1484
  headers.set("x-eden-eve-generation-id", options.generationId);
809
- headers.set("x-eden-eve-container-instance", options.stableContainerInstanceName);
1485
+ headers.set("x-eden-eve-container-instance", deploymentInstance(env, options.deploymentId, options.stableContainerInstanceName));
810
1486
  if (options.runtimeRevisionHandle !== undefined) {
811
1487
  headers.set("x-eden-eve-runtime-revision", options.runtimeRevisionHandle);
812
1488
  }