@moinulmoin/eden-runtime-cloudflare 0.6.0 → 0.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/eden-eve-host-worker.mjs +339 -130
- package/dist/eve-host-runtime.d.ts +49 -11
- package/dist/eve-host-runtime.js +346 -99
- package/dist/eve-host.d.ts +16 -26
- package/dist/eve-host.js +44 -70
- package/dist/index.d.ts +2 -2
- package/dist/index.js +1 -1
- package/package.json +2 -2
package/dist/eve-host-runtime.js
CHANGED
|
@@ -1,33 +1,52 @@
|
|
|
1
1
|
import { DurableObject, WorkerEntrypoint } from "cloudflare:workers";
|
|
2
2
|
import { WorldCore } from "@moinulmoin/eden-world-cloudflare/core";
|
|
3
|
-
import { EVE_HOST_DEFAULTS, EVE_SCHEDULE_WAKE_LEAD_MS, EveHostError, assertNonEmpty, assertStableOrigin,
|
|
3
|
+
import { EVE_HOST_DEFAULTS, EVE_HOST_SLEEP_SNAPSHOT_MARGIN_MS, EVE_SCHEDULE_WAKE_LEAD_MS, parseEveSleepAfterMs, EveHostError, assertNonEmpty, assertStableOrigin, createEveReadinessGate, createTrustedEveRequest, eveScheduleCronFiresWithin, readProtectedRuntimeVariables, WORKER_NAME_PATTERN, isEveWorkflowInternalRoute, } from "./eve-host.js";
|
|
4
4
|
const EVE_CLOUDFLARE_CONTAINERS_CA_PATH = "/etc/cloudflare/certs/cloudflare-containers-ca.crt";
|
|
5
|
+
const EVE_HOST_STALE_WORKER_REASON = "stale Worker version";
|
|
6
|
+
/**
|
|
7
|
+
* Cloudflare's durable_object containers expose `/dev/kvm`, which makes Eve's
|
|
8
|
+
* `DefaultSandbox` pick microsandbox at runtime, while Eden's isolated builder
|
|
9
|
+
* (no KVM, no Docker daemon) prepared the just-bash template — every sandbox
|
|
10
|
+
* tool then fails with a provider mismatch. Hiding the device before Eve
|
|
11
|
+
* starts keeps runtime selection equal to the build's. Same Eve start
|
|
12
|
+
* command as the image entrypoint.
|
|
13
|
+
*/
|
|
14
|
+
const EVE_HOST_ENTRYPOINT = Object.freeze([
|
|
15
|
+
"/bin/sh",
|
|
16
|
+
"-c",
|
|
17
|
+
"rm -f /dev/kvm; cd /workspace && exec ./node_modules/.bin/eve start --host 0.0.0.0 --port 8080",
|
|
18
|
+
]);
|
|
5
19
|
/** Durable Object storage keys owned by `EveHostDurableContainer`. */
|
|
6
20
|
const EVE_HOST_CONTAINER_ACTIVITY_KEY = "eden.eve.host.last-activity-at";
|
|
7
21
|
const EVE_HOST_CONTAINER_SNAPSHOT_KEY = "eden.eve.host.workspace-snapshot";
|
|
8
22
|
const EVE_HOST_CONTAINER_SNAPSHOT_ATTEMPTS_KEY = "eden.eve.host.snapshot-attempts";
|
|
23
|
+
/**
|
|
24
|
+
* The image the current container was started with. `inspect()` reports an
|
|
25
|
+
* empty image for a container restored from a snapshot (and while one is
|
|
26
|
+
* starting), so update detection falls back to this record.
|
|
27
|
+
*/
|
|
28
|
+
const EVE_HOST_CONTAINER_STARTED_IMAGE_KEY = "eden.eve.host.started-image";
|
|
29
|
+
/**
|
|
30
|
+
* Sandbox-session carry keys. The meta record names the active carry; its
|
|
31
|
+
* archive streams into `<chunkPrefix>.<carryId>.<index>` values so no single
|
|
32
|
+
* key+value pair approaches the SQLite-backed 2 MiB storage limit and the
|
|
33
|
+
* whole archive never sits in isolate memory (128 MB limit).
|
|
34
|
+
*/
|
|
35
|
+
const EVE_HOST_SANDBOX_CARRY_KEY = "eden.eve.host.sandbox-carry";
|
|
36
|
+
const EVE_HOST_SANDBOX_CARRY_CHUNK_PREFIX = "eden.eve.host.sandbox-carry";
|
|
37
|
+
/** The only `/workspace` subtree carried across an image update. */
|
|
38
|
+
const EVE_HOST_SANDBOX_CARRY_SOURCE = "/workspace/.eve/sandbox-cache/just-bash";
|
|
39
|
+
/**
|
|
40
|
+
* Carry archive limits. `chunkBytes` stays safely under the per-key limit;
|
|
41
|
+
* `maxBytes` caps the compressed archive (tests shrink both).
|
|
42
|
+
*/
|
|
43
|
+
export const EVE_HOST_CARRY_LIMITS = {
|
|
44
|
+
chunkBytes: 1024 * 1024,
|
|
45
|
+
maxBytes: 1024 * 1024 * 1024,
|
|
46
|
+
};
|
|
9
47
|
/** Snapshot attempts the alarm retries before destroying anyway so a stuck
|
|
10
48
|
* snapshot path never keeps billing alive forever. */
|
|
11
49
|
const EVE_HOST_SNAPSHOT_MAX_ATTEMPTS = 3;
|
|
12
|
-
/** Parses the `sleepAfter` duration spellings wrangler accepts ("300ms",
|
|
13
|
-
* "30s", "15m", "24h", "7d", or a bare number of seconds). */
|
|
14
|
-
function sleepAfterDurationMs(value) {
|
|
15
|
-
const match = /^([0-9]+)(ms|s|m|h|d)?$/u.exec(value.trim());
|
|
16
|
-
if (match === null)
|
|
17
|
-
return undefined;
|
|
18
|
-
const amount = Number.parseInt(match[1] ?? "", 10);
|
|
19
|
-
const unit = match[2] ?? "s";
|
|
20
|
-
const factor = unit === "ms"
|
|
21
|
-
? 1
|
|
22
|
-
: unit === "s"
|
|
23
|
-
? 1_000
|
|
24
|
-
: unit === "m"
|
|
25
|
-
? 60_000
|
|
26
|
-
: unit === "h"
|
|
27
|
-
? 3_600_000
|
|
28
|
-
: 86_400_000;
|
|
29
|
-
return amount * factor;
|
|
30
|
-
}
|
|
31
50
|
/** The deployment's single SQLite-backed Workflow World. */
|
|
32
51
|
export class EdenWorldDurableObject {
|
|
33
52
|
core;
|
|
@@ -46,13 +65,11 @@ export class EdenWorldDurableObject {
|
|
|
46
65
|
delivery: {
|
|
47
66
|
deliver: async (message) => {
|
|
48
67
|
const origin = env.EVE_PUBLIC_ORIGIN;
|
|
49
|
-
const binding = env.EVE_CONTAINER_BINDING_NAME;
|
|
50
68
|
const instance = env.EVE_CONTAINER_INSTANCE_NAME;
|
|
51
|
-
if (typeof origin !== "string" || typeof
|
|
52
|
-
typeof instance !== "string") {
|
|
69
|
+
if (typeof origin !== "string" || typeof instance !== "string") {
|
|
53
70
|
throw new Error("The World queue Container identity is unavailable.");
|
|
54
71
|
}
|
|
55
|
-
const namespace = env
|
|
72
|
+
const namespace = env.EVE_CONTAINER;
|
|
56
73
|
const response = await namespace.getByName(instance).loopbackFetch(new Request(new URL(`/.well-known/workflow/v1/${message.path}`, origin), {
|
|
57
74
|
method: "POST",
|
|
58
75
|
headers: {
|
|
@@ -112,15 +129,12 @@ export class EdenWorldDurableObject {
|
|
|
112
129
|
*/
|
|
113
130
|
export async function routeEveOutboundRequest(request, env) {
|
|
114
131
|
const publicOrigin = env.EVE_PUBLIC_ORIGIN;
|
|
115
|
-
const bindingName = env.EVE_CONTAINER_BINDING_NAME;
|
|
116
132
|
const instanceName = env.EVE_CONTAINER_INSTANCE_NAME;
|
|
117
133
|
if (typeof publicOrigin !== "string" ||
|
|
118
|
-
typeof bindingName !== "string" ||
|
|
119
134
|
typeof instanceName !== "string") {
|
|
120
135
|
throw new EveHostError("HOST_ORIGIN_UNAVAILABLE", "The container loopback identity is unavailable to the outbound handler.");
|
|
121
136
|
}
|
|
122
137
|
assertStableOrigin(publicOrigin);
|
|
123
|
-
assertNonEmpty(bindingName, "Container binding name");
|
|
124
138
|
assertNonEmpty(instanceName, "Container instance name");
|
|
125
139
|
const hostname = new URL(request.url).hostname;
|
|
126
140
|
if (hostname !== new URL(publicOrigin).hostname) {
|
|
@@ -132,7 +146,7 @@ export async function routeEveOutboundRequest(request, env) {
|
|
|
132
146
|
return new Response(null, { status: 404 });
|
|
133
147
|
return world.get(world.idFromName("world")).fetch(request);
|
|
134
148
|
}
|
|
135
|
-
const namespace = env
|
|
149
|
+
const namespace = env.EVE_CONTAINER;
|
|
136
150
|
if (namespace === undefined) {
|
|
137
151
|
throw new EveHostError("HOST_READINESS_UNPROVEN", "The configured Container binding is unavailable to the outbound handler.");
|
|
138
152
|
}
|
|
@@ -163,8 +177,6 @@ export class EveHostLoopback extends WorkerEntrypoint {
|
|
|
163
177
|
* running image predates the configured one.
|
|
164
178
|
*/
|
|
165
179
|
export class EveHostDurableContainer extends DurableObject {
|
|
166
|
-
lifecycle = createEveHostLifecycleObserver();
|
|
167
|
-
readinessStarted = false;
|
|
168
180
|
/** Evidence is cached only while the container it was proven on keeps running. */
|
|
169
181
|
readinessEvidence;
|
|
170
182
|
/** True once the running container's image was proven to equal `images.eve`. */
|
|
@@ -176,6 +188,12 @@ export class EveHostDurableContainer extends DurableObject {
|
|
|
176
188
|
*/
|
|
177
189
|
readinessEpoch = 0;
|
|
178
190
|
monitorAttached = false;
|
|
191
|
+
/** Serializes sandbox-session carry-outs on this Durable Object instance. */
|
|
192
|
+
carryOutQueue = Promise.resolve();
|
|
193
|
+
/** True only while the last start was fresh from `images.eve`, not a snapshot. */
|
|
194
|
+
pendingCarryRestore = false;
|
|
195
|
+
/** Shared single-flight carry restore for the current container generation. */
|
|
196
|
+
carryRestorePromise;
|
|
179
197
|
envVars;
|
|
180
198
|
sleepAfterMs;
|
|
181
199
|
readiness;
|
|
@@ -215,14 +233,7 @@ export class EveHostDurableContainer extends DurableObject {
|
|
|
215
233
|
? {}
|
|
216
234
|
: { EDEN_EVE_RUNTIME_REVISION: env.EDEN_EVE_RUNTIME_REVISION }),
|
|
217
235
|
};
|
|
218
|
-
|
|
219
|
-
this.sleepAfterMs =
|
|
220
|
-
sleepAfterOverride === undefined
|
|
221
|
-
? sleepAfterDurationMs(EVE_HOST_DEFAULTS.sleepAfter) ?? 86_400_000
|
|
222
|
-
: sleepAfterDurationMs(sleepAfterOverride) ??
|
|
223
|
-
(() => {
|
|
224
|
-
throw new EveHostError("HOST_READINESS_UNPROVEN", "The Container sleepAfter override is not a parseable duration.");
|
|
225
|
-
})();
|
|
236
|
+
this.sleepAfterMs = parseEveSleepAfterMs(env.EDEN_EVE_CONTAINER_SLEEP_AFTER ?? EVE_HOST_DEFAULTS.sleepAfter);
|
|
226
237
|
this.readiness = createEveReadinessGate({
|
|
227
238
|
start: async () => {
|
|
228
239
|
await this.ensureContainerStarted();
|
|
@@ -251,9 +262,10 @@ export class EveHostDurableContainer extends DurableObject {
|
|
|
251
262
|
containerRuntime() {
|
|
252
263
|
return this.ctx.container;
|
|
253
264
|
}
|
|
254
|
-
/** Container lifetime the platform keeps after the DO goes inactive
|
|
265
|
+
/** Container lifetime the platform keeps after the DO goes inactive: long
|
|
266
|
+
* enough for the idle alarm to snapshot first, within Cloudflare's 6 h cap. */
|
|
255
267
|
get inactivityTimeoutMs() {
|
|
256
|
-
return this.sleepAfterMs +
|
|
268
|
+
return this.sleepAfterMs + EVE_HOST_SLEEP_SNAPSHOT_MARGIN_MS;
|
|
257
269
|
}
|
|
258
270
|
attachMonitor(runtime) {
|
|
259
271
|
if (this.monitorAttached)
|
|
@@ -261,19 +273,19 @@ export class EveHostDurableContainer extends DurableObject {
|
|
|
261
273
|
this.monitorAttached = true;
|
|
262
274
|
runtime.monitor().then(() => {
|
|
263
275
|
this.monitorAttached = false;
|
|
264
|
-
this.recordContainerStopped(
|
|
265
|
-
}, (
|
|
276
|
+
this.recordContainerStopped();
|
|
277
|
+
}, () => {
|
|
266
278
|
this.monitorAttached = false;
|
|
267
|
-
this.recordContainerStopped(
|
|
279
|
+
this.recordContainerStopped();
|
|
268
280
|
});
|
|
269
281
|
}
|
|
270
|
-
recordContainerStopped(
|
|
282
|
+
recordContainerStopped() {
|
|
271
283
|
this.readinessEpoch += 1;
|
|
272
|
-
this.readinessStarted = false;
|
|
273
284
|
this.readinessEvidence = undefined;
|
|
274
285
|
this.imageVerified = false;
|
|
286
|
+
this.pendingCarryRestore = false;
|
|
287
|
+
this.carryRestorePromise = undefined;
|
|
275
288
|
this.readiness.reset();
|
|
276
|
-
this.lifecycle.record("stopped", reason);
|
|
277
289
|
}
|
|
278
290
|
/** Routes the container's own requests to its public origin back inside. */
|
|
279
291
|
async registerLoopbackIntercept(runtime) {
|
|
@@ -299,7 +311,7 @@ export class EveHostDurableContainer extends DurableObject {
|
|
|
299
311
|
* is restored only when it was captured from the exact image this Worker
|
|
300
312
|
* version configures — restoring an older snapshot silently runs the
|
|
301
313
|
* previous image — so after an image update the container starts fresh and
|
|
302
|
-
*
|
|
314
|
+
* the readiness path restores the carried sandbox sessions instead.
|
|
303
315
|
*/
|
|
304
316
|
async ensureContainerStarted() {
|
|
305
317
|
const runtime = this.containerRuntime();
|
|
@@ -313,28 +325,39 @@ export class EveHostDurableContainer extends DurableObject {
|
|
|
313
325
|
throw new EveHostError("HOST_READINESS_UNPROVEN", "The configured Container image is unavailable to this Worker version.");
|
|
314
326
|
}
|
|
315
327
|
const storedSnapshot = await this.ctx.storage.get(EVE_HOST_CONTAINER_SNAPSHOT_KEY);
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
328
|
+
const snapshotId = storedSnapshot !== undefined &&
|
|
329
|
+
storedSnapshot.image === expectedImage
|
|
330
|
+
? storedSnapshot.id
|
|
331
|
+
: undefined;
|
|
319
332
|
runtime.start({
|
|
320
|
-
...(
|
|
321
|
-
? {
|
|
322
|
-
: {
|
|
333
|
+
...(snapshotId === undefined
|
|
334
|
+
? { image: expectedImage }
|
|
335
|
+
: { containerSnapshot: { id: snapshotId } }),
|
|
323
336
|
instance: EVE_HOST_DEFAULTS.instance,
|
|
324
337
|
enableInternet: true,
|
|
325
338
|
env: this.envVars,
|
|
339
|
+
entrypoint: EVE_HOST_ENTRYPOINT,
|
|
326
340
|
});
|
|
341
|
+
// Set before any await: a failure below makes the readiness retry see a
|
|
342
|
+
// running container and skip this method, so the restore must already be
|
|
343
|
+
// pending. A start from the configured image (an update) has no sandbox
|
|
344
|
+
// sessions yet; a same-image snapshot wake already contains them.
|
|
345
|
+
this.pendingCarryRestore = snapshotId === undefined;
|
|
346
|
+
// Both start sources run `expectedImage` (a snapshot is only restored
|
|
347
|
+
// from that image).
|
|
348
|
+
await this.ctx.storage.put(EVE_HOST_CONTAINER_STARTED_IMAGE_KEY, expectedImage);
|
|
327
349
|
// Intercepts do not survive a container start and late registration lags;
|
|
328
350
|
// register inside the same block before readiness polling.
|
|
329
351
|
await this.registerLoopbackIntercept(runtime);
|
|
330
352
|
this.attachMonitor(runtime);
|
|
331
353
|
await runtime.setInactivityTimeout(this.inactivityTimeoutMs);
|
|
332
|
-
this.lifecycle.record("started");
|
|
333
354
|
}
|
|
334
355
|
/**
|
|
335
|
-
* Replaces the running container when
|
|
336
|
-
*
|
|
337
|
-
*
|
|
356
|
+
* Replaces the running container when it still serves a pre-update image:
|
|
357
|
+
* this Worker version's `images.eve` is selected at `start()`, so an
|
|
358
|
+
* in-place update must stop the stale instance itself. `inspect()` reports
|
|
359
|
+
* an empty image for a snapshot-restored container, so the image recorded
|
|
360
|
+
* at start stands in for it.
|
|
338
361
|
*/
|
|
339
362
|
async reconcileRunningImage() {
|
|
340
363
|
if (this.imageVerified)
|
|
@@ -350,51 +373,246 @@ export class EveHostDurableContainer extends DurableObject {
|
|
|
350
373
|
catch {
|
|
351
374
|
info = null;
|
|
352
375
|
}
|
|
353
|
-
|
|
376
|
+
const runningImage = info?.image !== undefined && info.image !== ""
|
|
377
|
+
? info.image
|
|
378
|
+
: await this.ctx.storage.get(EVE_HOST_CONTAINER_STARTED_IMAGE_KEY);
|
|
379
|
+
if (runningImage === expectedImage) {
|
|
354
380
|
this.imageVerified = true;
|
|
355
381
|
return;
|
|
356
382
|
}
|
|
357
|
-
this.
|
|
358
|
-
|
|
383
|
+
this.recordContainerStopped();
|
|
384
|
+
// The stale container's conversation sandboxes must survive the update:
|
|
385
|
+
// stream them into Durable Object storage before the destroy makes the
|
|
386
|
+
// container unreachable. A failed carry still lets the update proceed.
|
|
387
|
+
try {
|
|
388
|
+
await this.carrySandboxSessionsOut(runtime);
|
|
389
|
+
}
|
|
390
|
+
catch (error) {
|
|
391
|
+
console.error("The Eve sandbox session carry failed before the image update:", error instanceof Error ? error.message : "unknown");
|
|
392
|
+
}
|
|
359
393
|
await runtime.destroy();
|
|
360
394
|
const stoppedAt = Date.now();
|
|
361
395
|
while (runtime.running && Date.now() - stoppedAt < 10_000) {
|
|
362
396
|
await new Promise((resolve) => setTimeout(resolve, 25));
|
|
363
397
|
}
|
|
364
398
|
}
|
|
399
|
+
/**
|
|
400
|
+
* Streams `/workspace/.eve/sandbox-cache/just-bash/sessions` — every
|
|
401
|
+
* conversation's sandbox files and metadata — into Durable Object storage,
|
|
402
|
+
* so a fresh container started by an image update can restore them. Not
|
|
403
|
+
* carried: templates and the rest of `/workspace`, which come from the new
|
|
404
|
+
* image. One carry runs at a time on this Durable Object; a failure never
|
|
405
|
+
* blocks the destroy that follows and leaves the previous carry intact.
|
|
406
|
+
*/
|
|
407
|
+
carrySandboxSessionsOut(runtime) {
|
|
408
|
+
const run = this.carryOutQueue.then(() => this.carrySandboxSessionsOutNow(runtime));
|
|
409
|
+
this.carryOutQueue = run.then(() => undefined, () => undefined);
|
|
410
|
+
return run;
|
|
411
|
+
}
|
|
412
|
+
async carrySandboxSessionsOutNow(runtime) {
|
|
413
|
+
const previous = await this.ctx.storage.get(EVE_HOST_SANDBOX_CARRY_KEY);
|
|
414
|
+
const carryId = globalThis.crypto.randomUUID();
|
|
415
|
+
const chunkPrefix = `${EVE_HOST_SANDBOX_CARRY_CHUNK_PREFIX}.${carryId}.`;
|
|
416
|
+
const writtenChunkKeys = [];
|
|
417
|
+
const discardWrittenChunks = async () => {
|
|
418
|
+
for (const key of writtenChunkKeys)
|
|
419
|
+
await this.ctx.storage.delete(key);
|
|
420
|
+
};
|
|
421
|
+
const exec = await runtime.exec([
|
|
422
|
+
"sh",
|
|
423
|
+
"-c",
|
|
424
|
+
`cd ${EVE_HOST_SANDBOX_CARRY_SOURCE} && [ -d sessions ] && tar -czf - sessions || true`,
|
|
425
|
+
], { stdout: "pipe", stderr: "pipe" });
|
|
426
|
+
const buffered = [];
|
|
427
|
+
let bufferedBytes = 0;
|
|
428
|
+
let chunks = 0;
|
|
429
|
+
let bytes = 0;
|
|
430
|
+
const flushOnce = async () => {
|
|
431
|
+
if (bufferedBytes === 0)
|
|
432
|
+
return;
|
|
433
|
+
const size = Math.min(bufferedBytes, EVE_HOST_CARRY_LIMITS.chunkBytes);
|
|
434
|
+
const value = new Uint8Array(size);
|
|
435
|
+
let offset = 0;
|
|
436
|
+
while (offset < size) {
|
|
437
|
+
const part = buffered[0];
|
|
438
|
+
if (part === undefined)
|
|
439
|
+
break;
|
|
440
|
+
const take = Math.min(part.byteLength, size - offset);
|
|
441
|
+
value.set(take === part.byteLength ? part : part.subarray(0, take), offset);
|
|
442
|
+
offset += take;
|
|
443
|
+
if (take === part.byteLength)
|
|
444
|
+
buffered.shift();
|
|
445
|
+
else
|
|
446
|
+
buffered[0] = part.subarray(take);
|
|
447
|
+
}
|
|
448
|
+
bufferedBytes -= size;
|
|
449
|
+
const key = `${chunkPrefix}${chunks}`;
|
|
450
|
+
writtenChunkKeys.push(key);
|
|
451
|
+
await this.ctx.storage.put(key, value);
|
|
452
|
+
chunks += 1;
|
|
453
|
+
bytes += value.byteLength;
|
|
454
|
+
};
|
|
455
|
+
const flushBufferedChunks = async () => {
|
|
456
|
+
while (bufferedBytes >= EVE_HOST_CARRY_LIMITS.chunkBytes) {
|
|
457
|
+
await flushOnce();
|
|
458
|
+
}
|
|
459
|
+
};
|
|
460
|
+
try {
|
|
461
|
+
const stdout = exec.stdout;
|
|
462
|
+
if (stdout === null)
|
|
463
|
+
throw new Error("the exec stdout was not piped");
|
|
464
|
+
const reader = stdout.getReader();
|
|
465
|
+
let overflowed = false;
|
|
466
|
+
for (;;) {
|
|
467
|
+
const { done, value } = await reader.read();
|
|
468
|
+
if (done)
|
|
469
|
+
break;
|
|
470
|
+
if (value === undefined)
|
|
471
|
+
continue;
|
|
472
|
+
buffered.push(value);
|
|
473
|
+
bufferedBytes += value.byteLength;
|
|
474
|
+
if (bytes + bufferedBytes > EVE_HOST_CARRY_LIMITS.maxBytes) {
|
|
475
|
+
overflowed = true;
|
|
476
|
+
break;
|
|
477
|
+
}
|
|
478
|
+
await flushBufferedChunks();
|
|
479
|
+
}
|
|
480
|
+
await reader.cancel().catch(() => undefined);
|
|
481
|
+
if (overflowed) {
|
|
482
|
+
await discardWrittenChunks();
|
|
483
|
+
console.error(`The Eve sandbox session carry exceeded ${EVE_HOST_CARRY_LIMITS.maxBytes} compressed bytes; keeping the previous carry.`);
|
|
484
|
+
return;
|
|
485
|
+
}
|
|
486
|
+
await flushOnce();
|
|
487
|
+
}
|
|
488
|
+
catch (error) {
|
|
489
|
+
await discardWrittenChunks();
|
|
490
|
+
throw error;
|
|
491
|
+
}
|
|
492
|
+
let exitCode = -1;
|
|
493
|
+
try {
|
|
494
|
+
exitCode = await exec.exitCode;
|
|
495
|
+
}
|
|
496
|
+
catch {
|
|
497
|
+
exitCode = -1;
|
|
498
|
+
}
|
|
499
|
+
if (exitCode !== 0) {
|
|
500
|
+
await discardWrittenChunks();
|
|
501
|
+
console.error(`The Eve sandbox session carry failed with exit code ${exitCode}; keeping the previous carry.`);
|
|
502
|
+
return;
|
|
503
|
+
}
|
|
504
|
+
if (bytes === 0) {
|
|
505
|
+
// No sessions directory in this container (nothing to carry); keep any
|
|
506
|
+
// previous carry as the restore source.
|
|
507
|
+
return;
|
|
508
|
+
}
|
|
509
|
+
await this.ctx.storage.put(EVE_HOST_SANDBOX_CARRY_KEY, {
|
|
510
|
+
id: carryId,
|
|
511
|
+
chunks,
|
|
512
|
+
bytes,
|
|
513
|
+
at: Date.now(),
|
|
514
|
+
});
|
|
515
|
+
if (previous !== undefined) {
|
|
516
|
+
for (let index = 0; index < previous.chunks; index += 1) {
|
|
517
|
+
await this.ctx.storage.delete(`${EVE_HOST_SANDBOX_CARRY_CHUNK_PREFIX}.${previous.id}.${index}`);
|
|
518
|
+
}
|
|
519
|
+
}
|
|
520
|
+
}
|
|
521
|
+
/**
|
|
522
|
+
* Single restore per fresh container generation, shared by every
|
|
523
|
+
* concurrent readiness caller: the first caller starts it, everyone awaits
|
|
524
|
+
* the same promise, and no request is forwarded before it settles.
|
|
525
|
+
*/
|
|
526
|
+
async restoreCarriedSandboxSessions() {
|
|
527
|
+
if (!this.pendingCarryRestore)
|
|
528
|
+
return;
|
|
529
|
+
this.carryRestorePromise ??= this.carrySandboxSessionsIn();
|
|
530
|
+
await this.carryRestorePromise;
|
|
531
|
+
this.pendingCarryRestore = false;
|
|
532
|
+
}
|
|
533
|
+
/**
|
|
534
|
+
* Extracts the carried sandbox sessions into a container that started
|
|
535
|
+
* fresh from `images.eve`. The archive streams chunk by chunk from Durable
|
|
536
|
+
* Object storage through exec stdin, so it never sits in isolate memory. A
|
|
537
|
+
* failure is logged and swallowed: the agent still serves, only the
|
|
538
|
+
* carried files are missing.
|
|
539
|
+
*/
|
|
540
|
+
async carrySandboxSessionsIn() {
|
|
541
|
+
try {
|
|
542
|
+
const carry = await this.ctx.storage.get(EVE_HOST_SANDBOX_CARRY_KEY);
|
|
543
|
+
if (carry === undefined)
|
|
544
|
+
return;
|
|
545
|
+
let nextChunk = 0;
|
|
546
|
+
const archive = new ReadableStream({
|
|
547
|
+
pull: async (controller) => {
|
|
548
|
+
if (nextChunk >= carry.chunks) {
|
|
549
|
+
controller.close();
|
|
550
|
+
return;
|
|
551
|
+
}
|
|
552
|
+
const chunk = await this.ctx.storage.get(`${EVE_HOST_SANDBOX_CARRY_CHUNK_PREFIX}.${carry.id}.${nextChunk}`);
|
|
553
|
+
const index = nextChunk;
|
|
554
|
+
nextChunk += 1;
|
|
555
|
+
if (chunk === undefined) {
|
|
556
|
+
controller.error(new Error(`The sandbox carry chunk ${index} is missing.`));
|
|
557
|
+
return;
|
|
558
|
+
}
|
|
559
|
+
controller.enqueue(chunk);
|
|
560
|
+
},
|
|
561
|
+
});
|
|
562
|
+
const runtime = this.containerRuntime();
|
|
563
|
+
if (runtime === undefined) {
|
|
564
|
+
throw new Error("the Container runtime is unavailable");
|
|
565
|
+
}
|
|
566
|
+
const exec = await runtime.exec([
|
|
567
|
+
"sh",
|
|
568
|
+
"-c",
|
|
569
|
+
`mkdir -p ${EVE_HOST_SANDBOX_CARRY_SOURCE} && tar -xzf - -C ${EVE_HOST_SANDBOX_CARRY_SOURCE}`,
|
|
570
|
+
], { stdin: archive, stderr: "pipe" });
|
|
571
|
+
let exitCode = -1;
|
|
572
|
+
try {
|
|
573
|
+
exitCode = await exec.exitCode;
|
|
574
|
+
}
|
|
575
|
+
catch {
|
|
576
|
+
exitCode = -1;
|
|
577
|
+
}
|
|
578
|
+
if (exitCode !== 0) {
|
|
579
|
+
console.error(`The Eve sandbox session restore failed with exit code ${exitCode}; the agent serves without the carried files.`);
|
|
580
|
+
}
|
|
581
|
+
}
|
|
582
|
+
catch (error) {
|
|
583
|
+
console.error("The Eve sandbox session restore failed; the agent serves without the carried files:", error instanceof Error ? error.message : "unknown");
|
|
584
|
+
}
|
|
585
|
+
}
|
|
365
586
|
async ensureEveReady(signal = new AbortController().signal) {
|
|
366
587
|
await this.reconcileRunningImage();
|
|
367
588
|
if (this.readinessEvidence !== undefined) {
|
|
368
589
|
return this.readinessEvidence;
|
|
369
590
|
}
|
|
370
|
-
if (!this.readinessStarted) {
|
|
371
|
-
this.readinessStarted = true;
|
|
372
|
-
this.lifecycle.record("start_requested");
|
|
373
|
-
}
|
|
374
591
|
const epoch = this.readinessEpoch;
|
|
375
592
|
const evidence = await this.readiness(signal);
|
|
376
593
|
if (epoch === this.readinessEpoch) {
|
|
594
|
+
// Restore carried sandbox sessions while still inside the single-flight
|
|
595
|
+
// readiness path, before evidence is cached: every fetch and loopback
|
|
596
|
+
// request waits on this evidence, so nothing reaches Eve first.
|
|
597
|
+
await this.restoreCarriedSandboxSessions();
|
|
377
598
|
this.readinessEvidence = evidence;
|
|
378
599
|
// Readiness under this Worker version proves the running container
|
|
379
600
|
// serves `images.eve`: it was either verified by inspect() or freshly
|
|
380
601
|
// started from the configured image above.
|
|
381
602
|
this.imageVerified = true;
|
|
382
603
|
}
|
|
383
|
-
this.lifecycle.record("health_ready", evidence.healthStatus);
|
|
384
604
|
return evidence;
|
|
385
605
|
}
|
|
386
606
|
async fetch(request) {
|
|
387
607
|
const requestDeploymentId = request.headers.get("x-eden-eve-deployment-id") ?? undefined;
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
608
|
+
if (requestDeploymentId !== undefined &&
|
|
609
|
+
requestDeploymentId !== this.env.EDEN_EVE_DEPLOYMENT_ID) {
|
|
610
|
+
// A live container/monitor can pin the DO to an older Worker version.
|
|
611
|
+
// Reset the DO, not its container or storage; the retry's current code
|
|
612
|
+
// reconciles the image and carries sandbox sessions through the update.
|
|
613
|
+
this.ctx.abort(EVE_HOST_STALE_WORKER_REASON);
|
|
393
614
|
}
|
|
394
|
-
const
|
|
395
|
-
const response = await runtime
|
|
396
|
-
.getTcpPort(EVE_HOST_DEFAULTS.internalPort)
|
|
397
|
-
.fetch(new Request(`http://container${url.pathname}${url.search}`, request));
|
|
615
|
+
const response = await this.forwardToEve(request);
|
|
398
616
|
// Reporting the verified start back lets deploy health-gate promotion on
|
|
399
617
|
// the served container actually running this generation's image: the
|
|
400
618
|
// header echoes only when the running container's image equals this
|
|
@@ -420,6 +638,9 @@ export class EveHostDurableContainer extends DurableObject {
|
|
|
420
638
|
* loopback path is not a deployment-health signal).
|
|
421
639
|
*/
|
|
422
640
|
async loopbackFetch(request) {
|
|
641
|
+
return this.forwardToEve(request);
|
|
642
|
+
}
|
|
643
|
+
async forwardToEve(request) {
|
|
423
644
|
await this.ensureEveReady(request.signal);
|
|
424
645
|
await this.touchActivity();
|
|
425
646
|
const runtime = this.containerRuntime();
|
|
@@ -448,6 +669,15 @@ export class EveHostDurableContainer extends DurableObject {
|
|
|
448
669
|
}
|
|
449
670
|
if (runtime === undefined || !runtime.running)
|
|
450
671
|
return;
|
|
672
|
+
// The snapshot restores same-image wakes; the carry additionally lets an
|
|
673
|
+
// update that lands while asleep restore the conversation sandboxes. A
|
|
674
|
+
// failed carry must not degrade the snapshot path, so it is only logged.
|
|
675
|
+
try {
|
|
676
|
+
await this.carrySandboxSessionsOut(runtime);
|
|
677
|
+
}
|
|
678
|
+
catch (error) {
|
|
679
|
+
console.error("The Eve sandbox session carry failed during idle sleep:", error instanceof Error ? error.message : "unknown");
|
|
680
|
+
}
|
|
451
681
|
const attempts = (await this.ctx.storage.get(EVE_HOST_CONTAINER_SNAPSHOT_ATTEMPTS_KEY)) ?? 0;
|
|
452
682
|
try {
|
|
453
683
|
const snapshot = await runtime.snapshotContainer({
|
|
@@ -472,7 +702,7 @@ export class EveHostDurableContainer extends DurableObject {
|
|
|
472
702
|
return;
|
|
473
703
|
}
|
|
474
704
|
await runtime.destroy(new Error("idle-snapshot-done"));
|
|
475
|
-
this.recordContainerStopped(
|
|
705
|
+
this.recordContainerStopped();
|
|
476
706
|
}
|
|
477
707
|
catch (error) {
|
|
478
708
|
const message = error instanceof Error ? error.message : "unknown";
|
|
@@ -480,7 +710,7 @@ export class EveHostDurableContainer extends DurableObject {
|
|
|
480
710
|
console.error(`The Eve workspace snapshot failed ${attempts + 1} times; destroying the container without it: ${message}`);
|
|
481
711
|
await this.ctx.storage.put(EVE_HOST_CONTAINER_SNAPSHOT_ATTEMPTS_KEY, 0);
|
|
482
712
|
await runtime.destroy(new Error("idle-snapshot-abandoned"));
|
|
483
|
-
this.recordContainerStopped(
|
|
713
|
+
this.recordContainerStopped();
|
|
484
714
|
return;
|
|
485
715
|
}
|
|
486
716
|
console.error(`The Eve workspace snapshot failed (attempt ${attempts + 1}): ${message}`);
|
|
@@ -495,17 +725,51 @@ export function createEveHostWorker(options) {
|
|
|
495
725
|
}
|
|
496
726
|
assertStableOrigin(options.publicOrigin, options.workerName);
|
|
497
727
|
assertNonEmpty(options.stableContainerInstanceName, "Container instance name");
|
|
498
|
-
if (!IDENTIFIER_PATTERN.test(options.containerBindingName)) {
|
|
499
|
-
throw new EveHostError("HOST_READINESS_UNPROVEN", "The Container binding name is not valid.");
|
|
500
|
-
}
|
|
501
728
|
const schedules = options.schedules ?? [];
|
|
502
729
|
const resolveContainer = (env) => {
|
|
503
|
-
const namespace = env
|
|
730
|
+
const namespace = env.EVE_CONTAINER;
|
|
504
731
|
if (namespace === undefined) {
|
|
505
732
|
throw new EveHostError("HOST_READINESS_UNPROVEN", "The configured Container binding is unavailable.");
|
|
506
733
|
}
|
|
507
734
|
return namespace.getByName(options.stableContainerInstanceName);
|
|
508
735
|
};
|
|
736
|
+
const forwardToContainer = async (request, env) => {
|
|
737
|
+
const metadata = {
|
|
738
|
+
publicOrigin: options.publicOrigin,
|
|
739
|
+
deploymentId: options.deploymentId,
|
|
740
|
+
generationId: options.generationId,
|
|
741
|
+
correlationId: globalThis.crypto.randomUUID(),
|
|
742
|
+
...(options.runtimeRevisionHandle === undefined
|
|
743
|
+
? {}
|
|
744
|
+
: { runtimeRevisionHandle: options.runtimeRevisionHandle }),
|
|
745
|
+
};
|
|
746
|
+
const forwarded = createTrustedEveRequest(request, metadata);
|
|
747
|
+
// Synchronize the DO version before sending an upload. Cloning a body
|
|
748
|
+
// would retain an unread tee of the entire upload in isolate memory.
|
|
749
|
+
const bodyless = forwarded.body === null
|
|
750
|
+
? forwarded
|
|
751
|
+
: createTrustedEveRequest(new Request(new URL(EVE_HOST_DEFAULTS.healthPath, options.publicOrigin), { signal: forwarded.signal }), metadata);
|
|
752
|
+
let container = resolveContainer(env);
|
|
753
|
+
let response;
|
|
754
|
+
try {
|
|
755
|
+
response = await container.fetch(bodyless);
|
|
756
|
+
}
|
|
757
|
+
catch (error) {
|
|
758
|
+
if (!(error instanceof Error) ||
|
|
759
|
+
!error.message.includes(EVE_HOST_STALE_WORKER_REASON)) {
|
|
760
|
+
throw error;
|
|
761
|
+
}
|
|
762
|
+
// Only the pre-forwarding stale abort is safe to retry. A failed stub
|
|
763
|
+
// can remain broken, so resolve it again and retry the bodyless request once.
|
|
764
|
+
container = resolveContainer(env);
|
|
765
|
+
response = await container.fetch(bodyless);
|
|
766
|
+
}
|
|
767
|
+
if (bodyless === forwarded)
|
|
768
|
+
return response;
|
|
769
|
+
await response.body?.cancel();
|
|
770
|
+
// Stream the original body exactly once, outside the retry path.
|
|
771
|
+
return container.fetch(forwarded);
|
|
772
|
+
};
|
|
509
773
|
return {
|
|
510
774
|
async fetch(request, env) {
|
|
511
775
|
// Queue deliveries to these endpoints arrive through the container's
|
|
@@ -514,17 +778,7 @@ export function createEveHostWorker(options) {
|
|
|
514
778
|
if (isEveWorkflowInternalRoute(new URL(request.url).pathname)) {
|
|
515
779
|
return new Response(null, { status: 404 });
|
|
516
780
|
}
|
|
517
|
-
const
|
|
518
|
-
const forwarded = createTrustedEveRequest(request, {
|
|
519
|
-
publicOrigin: options.publicOrigin,
|
|
520
|
-
deploymentId: options.deploymentId,
|
|
521
|
-
generationId: options.generationId,
|
|
522
|
-
correlationId: globalThis.crypto.randomUUID(),
|
|
523
|
-
...(options.runtimeRevisionHandle === undefined
|
|
524
|
-
? {}
|
|
525
|
-
: { runtimeRevisionHandle: options.runtimeRevisionHandle }),
|
|
526
|
-
});
|
|
527
|
-
const response = await container.fetch(forwarded);
|
|
781
|
+
const response = await forwardToContainer(request, env);
|
|
528
782
|
const headers = new Headers(response.headers);
|
|
529
783
|
headers.set("x-eden-eve-worker-name", options.workerName);
|
|
530
784
|
headers.set("x-eden-eve-public-origin", options.publicOrigin);
|
|
@@ -549,14 +803,7 @@ export function createEveHostWorker(options) {
|
|
|
549
803
|
return;
|
|
550
804
|
// Wake only: the in-process Nitro runner inside the container
|
|
551
805
|
// fires the tick itself, so the schedule runs exactly once.
|
|
552
|
-
ctx.waitUntil(
|
|
553
|
-
.fetch(new Request(new URL(EVE_HOST_DEFAULTS.healthPath, options.publicOrigin), {
|
|
554
|
-
headers: {
|
|
555
|
-
// Stamp the Worker's own deployment so a stale Durable
|
|
556
|
-
// Object never falls back to its own environment.
|
|
557
|
-
"x-eden-eve-deployment-id": options.deploymentId,
|
|
558
|
-
},
|
|
559
|
-
}))
|
|
806
|
+
ctx.waitUntil(forwardToContainer(new Request(new URL(EVE_HOST_DEFAULTS.healthPath, options.publicOrigin)), env)
|
|
560
807
|
.catch((error) => {
|
|
561
808
|
console.error("The scheduled wake request failed:", error instanceof Error ? error.message : "unknown");
|
|
562
809
|
}));
|