@moinulmoin/eden-runtime-cloudflare 0.6.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/eden-eve-host-worker.mjs +305 -120
- package/dist/eve-host-runtime.d.ts +49 -11
- package/dist/eve-host-runtime.js +301 -88
- package/dist/eve-host.d.ts +16 -26
- package/dist/eve-host.js +44 -70
- package/dist/index.d.ts +2 -2
- package/dist/index.js +1 -1
- package/package.json +2 -2
package/dist/eve-host-runtime.js
CHANGED
|
@@ -1,33 +1,51 @@
|
|
|
1
1
|
import { DurableObject, WorkerEntrypoint } from "cloudflare:workers";
|
|
2
2
|
import { WorldCore } from "@moinulmoin/eden-world-cloudflare/core";
|
|
3
|
-
import { EVE_HOST_DEFAULTS, EVE_SCHEDULE_WAKE_LEAD_MS, EveHostError, assertNonEmpty, assertStableOrigin,
|
|
3
|
+
import { EVE_HOST_DEFAULTS, EVE_HOST_SLEEP_SNAPSHOT_MARGIN_MS, EVE_SCHEDULE_WAKE_LEAD_MS, parseEveSleepAfterMs, EveHostError, assertNonEmpty, assertStableOrigin, createEveReadinessGate, createTrustedEveRequest, eveScheduleCronFiresWithin, readProtectedRuntimeVariables, WORKER_NAME_PATTERN, isEveWorkflowInternalRoute, } from "./eve-host.js";
|
|
4
4
|
const EVE_CLOUDFLARE_CONTAINERS_CA_PATH = "/etc/cloudflare/certs/cloudflare-containers-ca.crt";
|
|
5
|
+
/**
|
|
6
|
+
* Cloudflare's durable_object containers expose `/dev/kvm`, which makes Eve's
|
|
7
|
+
* `DefaultSandbox` pick microsandbox at runtime, while Eden's isolated builder
|
|
8
|
+
* (no KVM, no Docker daemon) prepared the just-bash template — every sandbox
|
|
9
|
+
* tool then fails with a provider mismatch. Hiding the device before Eve
|
|
10
|
+
* starts keeps runtime selection equal to the build's. Same Eve start
|
|
11
|
+
* command as the image entrypoint.
|
|
12
|
+
*/
|
|
13
|
+
const EVE_HOST_ENTRYPOINT = Object.freeze([
|
|
14
|
+
"/bin/sh",
|
|
15
|
+
"-c",
|
|
16
|
+
"rm -f /dev/kvm; cd /workspace && exec ./node_modules/.bin/eve start --host 0.0.0.0 --port 8080",
|
|
17
|
+
]);
|
|
5
18
|
/** Durable Object storage keys owned by `EveHostDurableContainer`. */
|
|
6
19
|
const EVE_HOST_CONTAINER_ACTIVITY_KEY = "eden.eve.host.last-activity-at";
|
|
7
20
|
const EVE_HOST_CONTAINER_SNAPSHOT_KEY = "eden.eve.host.workspace-snapshot";
|
|
8
21
|
const EVE_HOST_CONTAINER_SNAPSHOT_ATTEMPTS_KEY = "eden.eve.host.snapshot-attempts";
|
|
22
|
+
/**
|
|
23
|
+
* The image the current container was started with. `inspect()` reports an
|
|
24
|
+
* empty image for a container restored from a snapshot (and while one is
|
|
25
|
+
* starting), so update detection falls back to this record.
|
|
26
|
+
*/
|
|
27
|
+
const EVE_HOST_CONTAINER_STARTED_IMAGE_KEY = "eden.eve.host.started-image";
|
|
28
|
+
/**
|
|
29
|
+
* Sandbox-session carry keys. The meta record names the active carry; its
|
|
30
|
+
* archive streams into `<chunkPrefix>.<carryId>.<index>` values so no single
|
|
31
|
+
* key+value pair approaches the SQLite-backed 2 MiB storage limit and the
|
|
32
|
+
* whole archive never sits in isolate memory (128 MB limit).
|
|
33
|
+
*/
|
|
34
|
+
const EVE_HOST_SANDBOX_CARRY_KEY = "eden.eve.host.sandbox-carry";
|
|
35
|
+
const EVE_HOST_SANDBOX_CARRY_CHUNK_PREFIX = "eden.eve.host.sandbox-carry";
|
|
36
|
+
/** The only `/workspace` subtree carried across an image update. */
|
|
37
|
+
const EVE_HOST_SANDBOX_CARRY_SOURCE = "/workspace/.eve/sandbox-cache/just-bash";
|
|
38
|
+
/**
|
|
39
|
+
* Carry archive limits. `chunkBytes` stays safely under the per-key limit;
|
|
40
|
+
* `maxBytes` caps the compressed archive (tests shrink both).
|
|
41
|
+
*/
|
|
42
|
+
export const EVE_HOST_CARRY_LIMITS = {
|
|
43
|
+
chunkBytes: 1024 * 1024,
|
|
44
|
+
maxBytes: 1024 * 1024 * 1024,
|
|
45
|
+
};
|
|
9
46
|
/** Snapshot attempts the alarm retries before destroying anyway so a stuck
|
|
10
47
|
* snapshot path never keeps billing alive forever. */
|
|
11
48
|
const EVE_HOST_SNAPSHOT_MAX_ATTEMPTS = 3;
|
|
12
|
-
/** Parses the `sleepAfter` duration spellings wrangler accepts ("300ms",
|
|
13
|
-
* "30s", "15m", "24h", "7d", or a bare number of seconds). */
|
|
14
|
-
function sleepAfterDurationMs(value) {
|
|
15
|
-
const match = /^([0-9]+)(ms|s|m|h|d)?$/u.exec(value.trim());
|
|
16
|
-
if (match === null)
|
|
17
|
-
return undefined;
|
|
18
|
-
const amount = Number.parseInt(match[1] ?? "", 10);
|
|
19
|
-
const unit = match[2] ?? "s";
|
|
20
|
-
const factor = unit === "ms"
|
|
21
|
-
? 1
|
|
22
|
-
: unit === "s"
|
|
23
|
-
? 1_000
|
|
24
|
-
: unit === "m"
|
|
25
|
-
? 60_000
|
|
26
|
-
: unit === "h"
|
|
27
|
-
? 3_600_000
|
|
28
|
-
: 86_400_000;
|
|
29
|
-
return amount * factor;
|
|
30
|
-
}
|
|
31
49
|
/** The deployment's single SQLite-backed Workflow World. */
|
|
32
50
|
export class EdenWorldDurableObject {
|
|
33
51
|
core;
|
|
@@ -46,13 +64,11 @@ export class EdenWorldDurableObject {
|
|
|
46
64
|
delivery: {
|
|
47
65
|
deliver: async (message) => {
|
|
48
66
|
const origin = env.EVE_PUBLIC_ORIGIN;
|
|
49
|
-
const binding = env.EVE_CONTAINER_BINDING_NAME;
|
|
50
67
|
const instance = env.EVE_CONTAINER_INSTANCE_NAME;
|
|
51
|
-
if (typeof origin !== "string" || typeof
|
|
52
|
-
typeof instance !== "string") {
|
|
68
|
+
if (typeof origin !== "string" || typeof instance !== "string") {
|
|
53
69
|
throw new Error("The World queue Container identity is unavailable.");
|
|
54
70
|
}
|
|
55
|
-
const namespace = env
|
|
71
|
+
const namespace = env.EVE_CONTAINER;
|
|
56
72
|
const response = await namespace.getByName(instance).loopbackFetch(new Request(new URL(`/.well-known/workflow/v1/${message.path}`, origin), {
|
|
57
73
|
method: "POST",
|
|
58
74
|
headers: {
|
|
@@ -112,15 +128,12 @@ export class EdenWorldDurableObject {
|
|
|
112
128
|
*/
|
|
113
129
|
export async function routeEveOutboundRequest(request, env) {
|
|
114
130
|
const publicOrigin = env.EVE_PUBLIC_ORIGIN;
|
|
115
|
-
const bindingName = env.EVE_CONTAINER_BINDING_NAME;
|
|
116
131
|
const instanceName = env.EVE_CONTAINER_INSTANCE_NAME;
|
|
117
132
|
if (typeof publicOrigin !== "string" ||
|
|
118
|
-
typeof bindingName !== "string" ||
|
|
119
133
|
typeof instanceName !== "string") {
|
|
120
134
|
throw new EveHostError("HOST_ORIGIN_UNAVAILABLE", "The container loopback identity is unavailable to the outbound handler.");
|
|
121
135
|
}
|
|
122
136
|
assertStableOrigin(publicOrigin);
|
|
123
|
-
assertNonEmpty(bindingName, "Container binding name");
|
|
124
137
|
assertNonEmpty(instanceName, "Container instance name");
|
|
125
138
|
const hostname = new URL(request.url).hostname;
|
|
126
139
|
if (hostname !== new URL(publicOrigin).hostname) {
|
|
@@ -132,7 +145,7 @@ export async function routeEveOutboundRequest(request, env) {
|
|
|
132
145
|
return new Response(null, { status: 404 });
|
|
133
146
|
return world.get(world.idFromName("world")).fetch(request);
|
|
134
147
|
}
|
|
135
|
-
const namespace = env
|
|
148
|
+
const namespace = env.EVE_CONTAINER;
|
|
136
149
|
if (namespace === undefined) {
|
|
137
150
|
throw new EveHostError("HOST_READINESS_UNPROVEN", "The configured Container binding is unavailable to the outbound handler.");
|
|
138
151
|
}
|
|
@@ -163,8 +176,6 @@ export class EveHostLoopback extends WorkerEntrypoint {
|
|
|
163
176
|
* running image predates the configured one.
|
|
164
177
|
*/
|
|
165
178
|
export class EveHostDurableContainer extends DurableObject {
|
|
166
|
-
lifecycle = createEveHostLifecycleObserver();
|
|
167
|
-
readinessStarted = false;
|
|
168
179
|
/** Evidence is cached only while the container it was proven on keeps running. */
|
|
169
180
|
readinessEvidence;
|
|
170
181
|
/** True once the running container's image was proven to equal `images.eve`. */
|
|
@@ -176,6 +187,12 @@ export class EveHostDurableContainer extends DurableObject {
|
|
|
176
187
|
*/
|
|
177
188
|
readinessEpoch = 0;
|
|
178
189
|
monitorAttached = false;
|
|
190
|
+
/** Serializes sandbox-session carry-outs on this Durable Object instance. */
|
|
191
|
+
carryOutQueue = Promise.resolve();
|
|
192
|
+
/** True only while the last start was fresh from `images.eve`, not a snapshot. */
|
|
193
|
+
pendingCarryRestore = false;
|
|
194
|
+
/** Shared single-flight carry restore for the current container generation. */
|
|
195
|
+
carryRestorePromise;
|
|
179
196
|
envVars;
|
|
180
197
|
sleepAfterMs;
|
|
181
198
|
readiness;
|
|
@@ -215,14 +232,7 @@ export class EveHostDurableContainer extends DurableObject {
|
|
|
215
232
|
? {}
|
|
216
233
|
: { EDEN_EVE_RUNTIME_REVISION: env.EDEN_EVE_RUNTIME_REVISION }),
|
|
217
234
|
};
|
|
218
|
-
|
|
219
|
-
this.sleepAfterMs =
|
|
220
|
-
sleepAfterOverride === undefined
|
|
221
|
-
? sleepAfterDurationMs(EVE_HOST_DEFAULTS.sleepAfter) ?? 86_400_000
|
|
222
|
-
: sleepAfterDurationMs(sleepAfterOverride) ??
|
|
223
|
-
(() => {
|
|
224
|
-
throw new EveHostError("HOST_READINESS_UNPROVEN", "The Container sleepAfter override is not a parseable duration.");
|
|
225
|
-
})();
|
|
235
|
+
this.sleepAfterMs = parseEveSleepAfterMs(env.EDEN_EVE_CONTAINER_SLEEP_AFTER ?? EVE_HOST_DEFAULTS.sleepAfter);
|
|
226
236
|
this.readiness = createEveReadinessGate({
|
|
227
237
|
start: async () => {
|
|
228
238
|
await this.ensureContainerStarted();
|
|
@@ -251,9 +261,10 @@ export class EveHostDurableContainer extends DurableObject {
|
|
|
251
261
|
containerRuntime() {
|
|
252
262
|
return this.ctx.container;
|
|
253
263
|
}
|
|
254
|
-
/** Container lifetime the platform keeps after the DO goes inactive
|
|
264
|
+
/** Container lifetime the platform keeps after the DO goes inactive: long
|
|
265
|
+
* enough for the idle alarm to snapshot first, within Cloudflare's 6 h cap. */
|
|
255
266
|
get inactivityTimeoutMs() {
|
|
256
|
-
return this.sleepAfterMs +
|
|
267
|
+
return this.sleepAfterMs + EVE_HOST_SLEEP_SNAPSHOT_MARGIN_MS;
|
|
257
268
|
}
|
|
258
269
|
attachMonitor(runtime) {
|
|
259
270
|
if (this.monitorAttached)
|
|
@@ -261,19 +272,19 @@ export class EveHostDurableContainer extends DurableObject {
|
|
|
261
272
|
this.monitorAttached = true;
|
|
262
273
|
runtime.monitor().then(() => {
|
|
263
274
|
this.monitorAttached = false;
|
|
264
|
-
this.recordContainerStopped(
|
|
265
|
-
}, (
|
|
275
|
+
this.recordContainerStopped();
|
|
276
|
+
}, () => {
|
|
266
277
|
this.monitorAttached = false;
|
|
267
|
-
this.recordContainerStopped(
|
|
278
|
+
this.recordContainerStopped();
|
|
268
279
|
});
|
|
269
280
|
}
|
|
270
|
-
recordContainerStopped(
|
|
281
|
+
recordContainerStopped() {
|
|
271
282
|
this.readinessEpoch += 1;
|
|
272
|
-
this.readinessStarted = false;
|
|
273
283
|
this.readinessEvidence = undefined;
|
|
274
284
|
this.imageVerified = false;
|
|
285
|
+
this.pendingCarryRestore = false;
|
|
286
|
+
this.carryRestorePromise = undefined;
|
|
275
287
|
this.readiness.reset();
|
|
276
|
-
this.lifecycle.record("stopped", reason);
|
|
277
288
|
}
|
|
278
289
|
/** Routes the container's own requests to its public origin back inside. */
|
|
279
290
|
async registerLoopbackIntercept(runtime) {
|
|
@@ -299,7 +310,7 @@ export class EveHostDurableContainer extends DurableObject {
|
|
|
299
310
|
* is restored only when it was captured from the exact image this Worker
|
|
300
311
|
* version configures — restoring an older snapshot silently runs the
|
|
301
312
|
* previous image — so after an image update the container starts fresh and
|
|
302
|
-
*
|
|
313
|
+
* the readiness path restores the carried sandbox sessions instead.
|
|
303
314
|
*/
|
|
304
315
|
async ensureContainerStarted() {
|
|
305
316
|
const runtime = this.containerRuntime();
|
|
@@ -313,28 +324,39 @@ export class EveHostDurableContainer extends DurableObject {
|
|
|
313
324
|
throw new EveHostError("HOST_READINESS_UNPROVEN", "The configured Container image is unavailable to this Worker version.");
|
|
314
325
|
}
|
|
315
326
|
const storedSnapshot = await this.ctx.storage.get(EVE_HOST_CONTAINER_SNAPSHOT_KEY);
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
327
|
+
const snapshotId = storedSnapshot !== undefined &&
|
|
328
|
+
storedSnapshot.image === expectedImage
|
|
329
|
+
? storedSnapshot.id
|
|
330
|
+
: undefined;
|
|
319
331
|
runtime.start({
|
|
320
|
-
...(
|
|
321
|
-
? {
|
|
322
|
-
: {
|
|
332
|
+
...(snapshotId === undefined
|
|
333
|
+
? { image: expectedImage }
|
|
334
|
+
: { containerSnapshot: { id: snapshotId } }),
|
|
323
335
|
instance: EVE_HOST_DEFAULTS.instance,
|
|
324
336
|
enableInternet: true,
|
|
325
337
|
env: this.envVars,
|
|
338
|
+
entrypoint: EVE_HOST_ENTRYPOINT,
|
|
326
339
|
});
|
|
340
|
+
// Set before any await: a failure below makes the readiness retry see a
|
|
341
|
+
// running container and skip this method, so the restore must already be
|
|
342
|
+
// pending. A start from the configured image (an update) has no sandbox
|
|
343
|
+
// sessions yet; a same-image snapshot wake already contains them.
|
|
344
|
+
this.pendingCarryRestore = snapshotId === undefined;
|
|
345
|
+
// Both start sources run `expectedImage` (a snapshot is only restored
|
|
346
|
+
// from that image).
|
|
347
|
+
await this.ctx.storage.put(EVE_HOST_CONTAINER_STARTED_IMAGE_KEY, expectedImage);
|
|
327
348
|
// Intercepts do not survive a container start and late registration lags;
|
|
328
349
|
// register inside the same block before readiness polling.
|
|
329
350
|
await this.registerLoopbackIntercept(runtime);
|
|
330
351
|
this.attachMonitor(runtime);
|
|
331
352
|
await runtime.setInactivityTimeout(this.inactivityTimeoutMs);
|
|
332
|
-
this.lifecycle.record("started");
|
|
333
353
|
}
|
|
334
354
|
/**
|
|
335
|
-
* Replaces the running container when
|
|
336
|
-
*
|
|
337
|
-
*
|
|
355
|
+
* Replaces the running container when it still serves a pre-update image:
|
|
356
|
+
* this Worker version's `images.eve` is selected at `start()`, so an
|
|
357
|
+
* in-place update must stop the stale instance itself. `inspect()` reports
|
|
358
|
+
* an empty image for a snapshot-restored container, so the image recorded
|
|
359
|
+
* at start stands in for it.
|
|
338
360
|
*/
|
|
339
361
|
async reconcileRunningImage() {
|
|
340
362
|
if (this.imageVerified)
|
|
@@ -350,51 +372,239 @@ export class EveHostDurableContainer extends DurableObject {
|
|
|
350
372
|
catch {
|
|
351
373
|
info = null;
|
|
352
374
|
}
|
|
353
|
-
|
|
375
|
+
const runningImage = info?.image !== undefined && info.image !== ""
|
|
376
|
+
? info.image
|
|
377
|
+
: await this.ctx.storage.get(EVE_HOST_CONTAINER_STARTED_IMAGE_KEY);
|
|
378
|
+
if (runningImage === expectedImage) {
|
|
354
379
|
this.imageVerified = true;
|
|
355
380
|
return;
|
|
356
381
|
}
|
|
357
|
-
this.
|
|
358
|
-
|
|
382
|
+
this.recordContainerStopped();
|
|
383
|
+
// The stale container's conversation sandboxes must survive the update:
|
|
384
|
+
// stream them into Durable Object storage before the destroy makes the
|
|
385
|
+
// container unreachable. A failed carry still lets the update proceed.
|
|
386
|
+
try {
|
|
387
|
+
await this.carrySandboxSessionsOut(runtime);
|
|
388
|
+
}
|
|
389
|
+
catch (error) {
|
|
390
|
+
console.error("The Eve sandbox session carry failed before the image update:", error instanceof Error ? error.message : "unknown");
|
|
391
|
+
}
|
|
359
392
|
await runtime.destroy();
|
|
360
393
|
const stoppedAt = Date.now();
|
|
361
394
|
while (runtime.running && Date.now() - stoppedAt < 10_000) {
|
|
362
395
|
await new Promise((resolve) => setTimeout(resolve, 25));
|
|
363
396
|
}
|
|
364
397
|
}
|
|
398
|
+
/**
|
|
399
|
+
* Streams `/workspace/.eve/sandbox-cache/just-bash/sessions` — every
|
|
400
|
+
* conversation's sandbox files and metadata — into Durable Object storage,
|
|
401
|
+
* so a fresh container started by an image update can restore them. Not
|
|
402
|
+
* carried: templates and the rest of `/workspace`, which come from the new
|
|
403
|
+
* image. One carry runs at a time on this Durable Object; a failure never
|
|
404
|
+
* blocks the destroy that follows and leaves the previous carry intact.
|
|
405
|
+
*/
|
|
406
|
+
carrySandboxSessionsOut(runtime) {
|
|
407
|
+
const run = this.carryOutQueue.then(() => this.carrySandboxSessionsOutNow(runtime));
|
|
408
|
+
this.carryOutQueue = run.then(() => undefined, () => undefined);
|
|
409
|
+
return run;
|
|
410
|
+
}
|
|
411
|
+
async carrySandboxSessionsOutNow(runtime) {
|
|
412
|
+
const previous = await this.ctx.storage.get(EVE_HOST_SANDBOX_CARRY_KEY);
|
|
413
|
+
const carryId = globalThis.crypto.randomUUID();
|
|
414
|
+
const chunkPrefix = `${EVE_HOST_SANDBOX_CARRY_CHUNK_PREFIX}.${carryId}.`;
|
|
415
|
+
const writtenChunkKeys = [];
|
|
416
|
+
const discardWrittenChunks = async () => {
|
|
417
|
+
for (const key of writtenChunkKeys)
|
|
418
|
+
await this.ctx.storage.delete(key);
|
|
419
|
+
};
|
|
420
|
+
const exec = await runtime.exec([
|
|
421
|
+
"sh",
|
|
422
|
+
"-c",
|
|
423
|
+
`cd ${EVE_HOST_SANDBOX_CARRY_SOURCE} && [ -d sessions ] && tar -czf - sessions || true`,
|
|
424
|
+
], { stdout: "pipe", stderr: "pipe" });
|
|
425
|
+
const buffered = [];
|
|
426
|
+
let bufferedBytes = 0;
|
|
427
|
+
let chunks = 0;
|
|
428
|
+
let bytes = 0;
|
|
429
|
+
const flushOnce = async () => {
|
|
430
|
+
if (bufferedBytes === 0)
|
|
431
|
+
return;
|
|
432
|
+
const size = Math.min(bufferedBytes, EVE_HOST_CARRY_LIMITS.chunkBytes);
|
|
433
|
+
const value = new Uint8Array(size);
|
|
434
|
+
let offset = 0;
|
|
435
|
+
while (offset < size) {
|
|
436
|
+
const part = buffered[0];
|
|
437
|
+
if (part === undefined)
|
|
438
|
+
break;
|
|
439
|
+
const take = Math.min(part.byteLength, size - offset);
|
|
440
|
+
value.set(take === part.byteLength ? part : part.subarray(0, take), offset);
|
|
441
|
+
offset += take;
|
|
442
|
+
if (take === part.byteLength)
|
|
443
|
+
buffered.shift();
|
|
444
|
+
else
|
|
445
|
+
buffered[0] = part.subarray(take);
|
|
446
|
+
}
|
|
447
|
+
bufferedBytes -= size;
|
|
448
|
+
const key = `${chunkPrefix}${chunks}`;
|
|
449
|
+
writtenChunkKeys.push(key);
|
|
450
|
+
await this.ctx.storage.put(key, value);
|
|
451
|
+
chunks += 1;
|
|
452
|
+
bytes += value.byteLength;
|
|
453
|
+
};
|
|
454
|
+
const flushBufferedChunks = async () => {
|
|
455
|
+
while (bufferedBytes >= EVE_HOST_CARRY_LIMITS.chunkBytes) {
|
|
456
|
+
await flushOnce();
|
|
457
|
+
}
|
|
458
|
+
};
|
|
459
|
+
try {
|
|
460
|
+
const stdout = exec.stdout;
|
|
461
|
+
if (stdout === null)
|
|
462
|
+
throw new Error("the exec stdout was not piped");
|
|
463
|
+
const reader = stdout.getReader();
|
|
464
|
+
let overflowed = false;
|
|
465
|
+
for (;;) {
|
|
466
|
+
const { done, value } = await reader.read();
|
|
467
|
+
if (done)
|
|
468
|
+
break;
|
|
469
|
+
if (value === undefined)
|
|
470
|
+
continue;
|
|
471
|
+
buffered.push(value);
|
|
472
|
+
bufferedBytes += value.byteLength;
|
|
473
|
+
if (bytes + bufferedBytes > EVE_HOST_CARRY_LIMITS.maxBytes) {
|
|
474
|
+
overflowed = true;
|
|
475
|
+
break;
|
|
476
|
+
}
|
|
477
|
+
await flushBufferedChunks();
|
|
478
|
+
}
|
|
479
|
+
await reader.cancel().catch(() => undefined);
|
|
480
|
+
if (overflowed) {
|
|
481
|
+
await discardWrittenChunks();
|
|
482
|
+
console.error(`The Eve sandbox session carry exceeded ${EVE_HOST_CARRY_LIMITS.maxBytes} compressed bytes; keeping the previous carry.`);
|
|
483
|
+
return;
|
|
484
|
+
}
|
|
485
|
+
await flushOnce();
|
|
486
|
+
}
|
|
487
|
+
catch (error) {
|
|
488
|
+
await discardWrittenChunks();
|
|
489
|
+
throw error;
|
|
490
|
+
}
|
|
491
|
+
let exitCode = -1;
|
|
492
|
+
try {
|
|
493
|
+
exitCode = await exec.exitCode;
|
|
494
|
+
}
|
|
495
|
+
catch {
|
|
496
|
+
exitCode = -1;
|
|
497
|
+
}
|
|
498
|
+
if (exitCode !== 0) {
|
|
499
|
+
await discardWrittenChunks();
|
|
500
|
+
console.error(`The Eve sandbox session carry failed with exit code ${exitCode}; keeping the previous carry.`);
|
|
501
|
+
return;
|
|
502
|
+
}
|
|
503
|
+
if (bytes === 0) {
|
|
504
|
+
// No sessions directory in this container (nothing to carry); keep any
|
|
505
|
+
// previous carry as the restore source.
|
|
506
|
+
return;
|
|
507
|
+
}
|
|
508
|
+
await this.ctx.storage.put(EVE_HOST_SANDBOX_CARRY_KEY, {
|
|
509
|
+
id: carryId,
|
|
510
|
+
chunks,
|
|
511
|
+
bytes,
|
|
512
|
+
at: Date.now(),
|
|
513
|
+
});
|
|
514
|
+
if (previous !== undefined) {
|
|
515
|
+
for (let index = 0; index < previous.chunks; index += 1) {
|
|
516
|
+
await this.ctx.storage.delete(`${EVE_HOST_SANDBOX_CARRY_CHUNK_PREFIX}.${previous.id}.${index}`);
|
|
517
|
+
}
|
|
518
|
+
}
|
|
519
|
+
}
|
|
520
|
+
/**
|
|
521
|
+
* Single restore per fresh container generation, shared by every
|
|
522
|
+
* concurrent readiness caller: the first caller starts it, everyone awaits
|
|
523
|
+
* the same promise, and no request is forwarded before it settles.
|
|
524
|
+
*/
|
|
525
|
+
async restoreCarriedSandboxSessions() {
|
|
526
|
+
if (!this.pendingCarryRestore)
|
|
527
|
+
return;
|
|
528
|
+
this.carryRestorePromise ??= this.carrySandboxSessionsIn();
|
|
529
|
+
await this.carryRestorePromise;
|
|
530
|
+
this.pendingCarryRestore = false;
|
|
531
|
+
}
|
|
532
|
+
/**
|
|
533
|
+
* Extracts the carried sandbox sessions into a container that started
|
|
534
|
+
* fresh from `images.eve`. The archive streams chunk by chunk from Durable
|
|
535
|
+
* Object storage through exec stdin, so it never sits in isolate memory. A
|
|
536
|
+
* failure is logged and swallowed: the agent still serves, only the
|
|
537
|
+
* carried files are missing.
|
|
538
|
+
*/
|
|
539
|
+
async carrySandboxSessionsIn() {
|
|
540
|
+
try {
|
|
541
|
+
const carry = await this.ctx.storage.get(EVE_HOST_SANDBOX_CARRY_KEY);
|
|
542
|
+
if (carry === undefined)
|
|
543
|
+
return;
|
|
544
|
+
let nextChunk = 0;
|
|
545
|
+
const archive = new ReadableStream({
|
|
546
|
+
pull: async (controller) => {
|
|
547
|
+
if (nextChunk >= carry.chunks) {
|
|
548
|
+
controller.close();
|
|
549
|
+
return;
|
|
550
|
+
}
|
|
551
|
+
const chunk = await this.ctx.storage.get(`${EVE_HOST_SANDBOX_CARRY_CHUNK_PREFIX}.${carry.id}.${nextChunk}`);
|
|
552
|
+
const index = nextChunk;
|
|
553
|
+
nextChunk += 1;
|
|
554
|
+
if (chunk === undefined) {
|
|
555
|
+
controller.error(new Error(`The sandbox carry chunk ${index} is missing.`));
|
|
556
|
+
return;
|
|
557
|
+
}
|
|
558
|
+
controller.enqueue(chunk);
|
|
559
|
+
},
|
|
560
|
+
});
|
|
561
|
+
const runtime = this.containerRuntime();
|
|
562
|
+
if (runtime === undefined) {
|
|
563
|
+
throw new Error("the Container runtime is unavailable");
|
|
564
|
+
}
|
|
565
|
+
const exec = await runtime.exec([
|
|
566
|
+
"sh",
|
|
567
|
+
"-c",
|
|
568
|
+
`mkdir -p ${EVE_HOST_SANDBOX_CARRY_SOURCE} && tar -xzf - -C ${EVE_HOST_SANDBOX_CARRY_SOURCE}`,
|
|
569
|
+
], { stdin: archive, stderr: "pipe" });
|
|
570
|
+
let exitCode = -1;
|
|
571
|
+
try {
|
|
572
|
+
exitCode = await exec.exitCode;
|
|
573
|
+
}
|
|
574
|
+
catch {
|
|
575
|
+
exitCode = -1;
|
|
576
|
+
}
|
|
577
|
+
if (exitCode !== 0) {
|
|
578
|
+
console.error(`The Eve sandbox session restore failed with exit code ${exitCode}; the agent serves without the carried files.`);
|
|
579
|
+
}
|
|
580
|
+
}
|
|
581
|
+
catch (error) {
|
|
582
|
+
console.error("The Eve sandbox session restore failed; the agent serves without the carried files:", error instanceof Error ? error.message : "unknown");
|
|
583
|
+
}
|
|
584
|
+
}
|
|
365
585
|
async ensureEveReady(signal = new AbortController().signal) {
|
|
366
586
|
await this.reconcileRunningImage();
|
|
367
587
|
if (this.readinessEvidence !== undefined) {
|
|
368
588
|
return this.readinessEvidence;
|
|
369
589
|
}
|
|
370
|
-
if (!this.readinessStarted) {
|
|
371
|
-
this.readinessStarted = true;
|
|
372
|
-
this.lifecycle.record("start_requested");
|
|
373
|
-
}
|
|
374
590
|
const epoch = this.readinessEpoch;
|
|
375
591
|
const evidence = await this.readiness(signal);
|
|
376
592
|
if (epoch === this.readinessEpoch) {
|
|
593
|
+
// Restore carried sandbox sessions while still inside the single-flight
|
|
594
|
+
// readiness path, before evidence is cached: every fetch and loopback
|
|
595
|
+
// request waits on this evidence, so nothing reaches Eve first.
|
|
596
|
+
await this.restoreCarriedSandboxSessions();
|
|
377
597
|
this.readinessEvidence = evidence;
|
|
378
598
|
// Readiness under this Worker version proves the running container
|
|
379
599
|
// serves `images.eve`: it was either verified by inspect() or freshly
|
|
380
600
|
// started from the configured image above.
|
|
381
601
|
this.imageVerified = true;
|
|
382
602
|
}
|
|
383
|
-
this.lifecycle.record("health_ready", evidence.healthStatus);
|
|
384
603
|
return evidence;
|
|
385
604
|
}
|
|
386
605
|
async fetch(request) {
|
|
387
606
|
const requestDeploymentId = request.headers.get("x-eden-eve-deployment-id") ?? undefined;
|
|
388
|
-
await this.
|
|
389
|
-
await this.touchActivity();
|
|
390
|
-
const runtime = this.containerRuntime();
|
|
391
|
-
if (runtime === undefined) {
|
|
392
|
-
throw new EveHostError("HOST_READINESS_UNPROVEN", "The Container runtime is unavailable on this Durable Object.");
|
|
393
|
-
}
|
|
394
|
-
const url = new URL(request.url);
|
|
395
|
-
const response = await runtime
|
|
396
|
-
.getTcpPort(EVE_HOST_DEFAULTS.internalPort)
|
|
397
|
-
.fetch(new Request(`http://container${url.pathname}${url.search}`, request));
|
|
607
|
+
const response = await this.forwardToEve(request);
|
|
398
608
|
// Reporting the verified start back lets deploy health-gate promotion on
|
|
399
609
|
// the served container actually running this generation's image: the
|
|
400
610
|
// header echoes only when the running container's image equals this
|
|
@@ -420,6 +630,9 @@ export class EveHostDurableContainer extends DurableObject {
|
|
|
420
630
|
* loopback path is not a deployment-health signal).
|
|
421
631
|
*/
|
|
422
632
|
async loopbackFetch(request) {
|
|
633
|
+
return this.forwardToEve(request);
|
|
634
|
+
}
|
|
635
|
+
async forwardToEve(request) {
|
|
423
636
|
await this.ensureEveReady(request.signal);
|
|
424
637
|
await this.touchActivity();
|
|
425
638
|
const runtime = this.containerRuntime();
|
|
@@ -448,6 +661,15 @@ export class EveHostDurableContainer extends DurableObject {
|
|
|
448
661
|
}
|
|
449
662
|
if (runtime === undefined || !runtime.running)
|
|
450
663
|
return;
|
|
664
|
+
// The snapshot restores same-image wakes; the carry additionally lets an
|
|
665
|
+
// update that lands while asleep restore the conversation sandboxes. A
|
|
666
|
+
// failed carry must not degrade the snapshot path, so it is only logged.
|
|
667
|
+
try {
|
|
668
|
+
await this.carrySandboxSessionsOut(runtime);
|
|
669
|
+
}
|
|
670
|
+
catch (error) {
|
|
671
|
+
console.error("The Eve sandbox session carry failed during idle sleep:", error instanceof Error ? error.message : "unknown");
|
|
672
|
+
}
|
|
451
673
|
const attempts = (await this.ctx.storage.get(EVE_HOST_CONTAINER_SNAPSHOT_ATTEMPTS_KEY)) ?? 0;
|
|
452
674
|
try {
|
|
453
675
|
const snapshot = await runtime.snapshotContainer({
|
|
@@ -472,7 +694,7 @@ export class EveHostDurableContainer extends DurableObject {
|
|
|
472
694
|
return;
|
|
473
695
|
}
|
|
474
696
|
await runtime.destroy(new Error("idle-snapshot-done"));
|
|
475
|
-
this.recordContainerStopped(
|
|
697
|
+
this.recordContainerStopped();
|
|
476
698
|
}
|
|
477
699
|
catch (error) {
|
|
478
700
|
const message = error instanceof Error ? error.message : "unknown";
|
|
@@ -480,7 +702,7 @@ export class EveHostDurableContainer extends DurableObject {
|
|
|
480
702
|
console.error(`The Eve workspace snapshot failed ${attempts + 1} times; destroying the container without it: ${message}`);
|
|
481
703
|
await this.ctx.storage.put(EVE_HOST_CONTAINER_SNAPSHOT_ATTEMPTS_KEY, 0);
|
|
482
704
|
await runtime.destroy(new Error("idle-snapshot-abandoned"));
|
|
483
|
-
this.recordContainerStopped(
|
|
705
|
+
this.recordContainerStopped();
|
|
484
706
|
return;
|
|
485
707
|
}
|
|
486
708
|
console.error(`The Eve workspace snapshot failed (attempt ${attempts + 1}): ${message}`);
|
|
@@ -495,12 +717,9 @@ export function createEveHostWorker(options) {
|
|
|
495
717
|
}
|
|
496
718
|
assertStableOrigin(options.publicOrigin, options.workerName);
|
|
497
719
|
assertNonEmpty(options.stableContainerInstanceName, "Container instance name");
|
|
498
|
-
if (!IDENTIFIER_PATTERN.test(options.containerBindingName)) {
|
|
499
|
-
throw new EveHostError("HOST_READINESS_UNPROVEN", "The Container binding name is not valid.");
|
|
500
|
-
}
|
|
501
720
|
const schedules = options.schedules ?? [];
|
|
502
721
|
const resolveContainer = (env) => {
|
|
503
|
-
const namespace = env
|
|
722
|
+
const namespace = env.EVE_CONTAINER;
|
|
504
723
|
if (namespace === undefined) {
|
|
505
724
|
throw new EveHostError("HOST_READINESS_UNPROVEN", "The configured Container binding is unavailable.");
|
|
506
725
|
}
|
|
@@ -550,13 +769,7 @@ export function createEveHostWorker(options) {
|
|
|
550
769
|
// Wake only: the in-process Nitro runner inside the container
|
|
551
770
|
// fires the tick itself, so the schedule runs exactly once.
|
|
552
771
|
ctx.waitUntil(resolveContainer(env)
|
|
553
|
-
.fetch(new Request(new URL(EVE_HOST_DEFAULTS.healthPath, options.publicOrigin)
|
|
554
|
-
headers: {
|
|
555
|
-
// Stamp the Worker's own deployment so a stale Durable
|
|
556
|
-
// Object never falls back to its own environment.
|
|
557
|
-
"x-eden-eve-deployment-id": options.deploymentId,
|
|
558
|
-
},
|
|
559
|
-
}))
|
|
772
|
+
.fetch(new Request(new URL(EVE_HOST_DEFAULTS.healthPath, options.publicOrigin)))
|
|
560
773
|
.catch((error) => {
|
|
561
774
|
console.error("The scheduled wake request failed:", error instanceof Error ? error.message : "unknown");
|
|
562
775
|
}));
|
package/dist/eve-host.d.ts
CHANGED
|
@@ -3,8 +3,21 @@ export declare const EVE_HOST_DEFAULTS: {
|
|
|
3
3
|
readonly healthPath: "/eve/v1/health";
|
|
4
4
|
readonly internalPort: 8080;
|
|
5
5
|
readonly instance: "standard-1";
|
|
6
|
-
readonly sleepAfter: "
|
|
6
|
+
readonly sleepAfter: "1h";
|
|
7
7
|
};
|
|
8
|
+
/**
|
|
9
|
+
* Cloudflare stops a container at most 6 hours after its Durable Object goes
|
|
10
|
+
* inactive (`setInactivityTimeout` rejects longer values). The idle-sleep
|
|
11
|
+
* alarm must fire before that so it can snapshot first, and the inactivity
|
|
12
|
+
* timeout is `sleepAfter` plus this margin.
|
|
13
|
+
*/
|
|
14
|
+
export declare const EVE_HOST_SLEEP_SNAPSHOT_MARGIN_MS: number;
|
|
15
|
+
export declare const EVE_HOST_MAX_SLEEP_AFTER_MS: number;
|
|
16
|
+
/**
|
|
17
|
+
* Parses a `sleepAfter` duration ("300ms", "30s", "15m", "1h", or a bare
|
|
18
|
+
* number of seconds) and enforces Cloudflare's inactivity ceiling.
|
|
19
|
+
*/
|
|
20
|
+
export declare function parseEveSleepAfterMs(value: string): number;
|
|
8
21
|
export declare const EVE_SCHEDULE_WAKE_TRIGGER_CRON = "* * * * *";
|
|
9
22
|
/**
|
|
10
23
|
* How far ahead of a tick the every-minute wake trigger starts the Container.
|
|
@@ -50,10 +63,10 @@ export declare function parseEveScheduleCron(cron: string): ParsedEveCron | unde
|
|
|
50
63
|
export declare function eveScheduleCronFiresWithin(cron: string, nowMs: number, leadMs?: number): boolean;
|
|
51
64
|
/** Queue and World RPC routes are private; webhooks and manifests stay forwarded. */
|
|
52
65
|
export declare function isEveWorkflowInternalRoute(pathname: string): boolean;
|
|
53
|
-
export declare const EVE_HOST_OWNED_HEADERS: readonly ["forwarded", "x-
|
|
66
|
+
export declare const EVE_HOST_OWNED_HEADERS: readonly ["forwarded", "x-real-ip", "true-client-ip", "x-original-host", "x-original-proto", "x-envoy-original-host", "x-envoy-original-proto", "x-proxy-host", "x-host", "x-eve-public-origin", "x-workflow-local-base-url"];
|
|
54
67
|
export declare const WORKER_NAME_PATTERN: RegExp;
|
|
55
68
|
export declare const IDENTIFIER_PATTERN: RegExp;
|
|
56
|
-
export type EveHostErrorCode = "HOST_ORIGIN_UNAVAILABLE" | "HOST_REQUEST_ABORTED" | "HOST_READINESS_UNPROVEN"
|
|
69
|
+
export type EveHostErrorCode = "HOST_ORIGIN_UNAVAILABLE" | "HOST_REQUEST_ABORTED" | "HOST_READINESS_UNPROVEN";
|
|
57
70
|
export declare class EveHostError extends Error {
|
|
58
71
|
readonly code: EveHostErrorCode;
|
|
59
72
|
constructor(code: EveHostErrorCode, message: string);
|
|
@@ -66,8 +79,6 @@ export declare function resolveStableWorkersDevOrigin(request: StableWorkersDevO
|
|
|
66
79
|
export interface EveHostIdentity {
|
|
67
80
|
readonly workerName: string;
|
|
68
81
|
readonly containerApplicationName: string;
|
|
69
|
-
readonly containerClassName: string;
|
|
70
|
-
readonly containerBindingName: string;
|
|
71
82
|
readonly stableContainerInstanceName: string;
|
|
72
83
|
readonly deploymentId: string;
|
|
73
84
|
readonly generationId: string;
|
|
@@ -122,7 +133,6 @@ export interface EveHostWranglerConfig {
|
|
|
122
133
|
readonly vars: {
|
|
123
134
|
readonly EVE_PUBLIC_ORIGIN: string;
|
|
124
135
|
readonly EVE_CONTAINER_INSTANCE_NAME: string;
|
|
125
|
-
readonly EVE_CONTAINER_BINDING_NAME: string;
|
|
126
136
|
readonly EDEN_EVE_DEPLOYMENT_ID: string;
|
|
127
137
|
readonly EDEN_EVE_GENERATION_ID: string;
|
|
128
138
|
readonly EVE_RUNTIME_VARIABLE_NAMES: readonly string[];
|
|
@@ -181,31 +191,11 @@ export interface EveHostForwardingMetadata {
|
|
|
181
191
|
readonly runtimeRevisionHandle?: string;
|
|
182
192
|
}
|
|
183
193
|
export declare function createTrustedEveRequest(request: Request, metadata: EveHostForwardingMetadata): Request;
|
|
184
|
-
export interface EveContainerTransport {
|
|
185
|
-
readonly containerFetch: (request: Request) => Promise<Response>;
|
|
186
|
-
readonly fetch: (request: Request) => Promise<Response>;
|
|
187
|
-
}
|
|
188
|
-
export interface EveHostProxyOptions extends EveHostForwardingMetadata {
|
|
189
|
-
readonly transport: EveContainerTransport;
|
|
190
|
-
readonly ensureReady: (signal: AbortSignal) => Promise<void>;
|
|
191
|
-
}
|
|
192
|
-
export declare function createEveHostProxy(options: EveHostProxyOptions): (request: Request) => Promise<Response>;
|
|
193
|
-
export interface EveHostLifecycleEvent {
|
|
194
|
-
readonly type: "start_requested" | "started" | "health_ready" | "stopped" | "errored" | "image_mismatch" | "replaced";
|
|
195
|
-
readonly at: number;
|
|
196
|
-
readonly safeStatus?: string;
|
|
197
|
-
}
|
|
198
|
-
export interface EveHostLifecycleObserver {
|
|
199
|
-
readonly events: readonly EveHostLifecycleEvent[];
|
|
200
|
-
record(type: EveHostLifecycleEvent["type"], safeStatus?: string): EveHostLifecycleEvent;
|
|
201
|
-
}
|
|
202
|
-
export declare function createEveHostLifecycleObserver(now?: () => number): EveHostLifecycleObserver;
|
|
203
194
|
export interface EveHostContainerEnvironment {
|
|
204
195
|
readonly [name: string]: unknown;
|
|
205
196
|
readonly EVE_RUNTIME_VARIABLE_NAMES?: readonly string[];
|
|
206
197
|
readonly EVE_PUBLIC_ORIGIN?: string;
|
|
207
198
|
readonly EVE_CONTAINER_INSTANCE_NAME?: string;
|
|
208
|
-
readonly EVE_CONTAINER_BINDING_NAME?: string;
|
|
209
199
|
readonly EDEN_EVE_DEPLOYMENT_ID?: string;
|
|
210
200
|
readonly EDEN_EVE_GENERATION_ID?: string;
|
|
211
201
|
readonly EDEN_EVE_RUNTIME_REVISION?: string;
|