@moinulmoin/eden-runtime-cloudflare 0.5.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +0 -9
- package/dist/eden-eve-host-worker.mjs +381 -1743
- package/dist/eve-host-runtime.d.ts +75 -53
- package/dist/eve-host-runtime.js +299 -186
- package/dist/eve-host.d.ts +24 -26
- package/dist/eve-host.js +108 -99
- package/package.json +3 -4
package/dist/eve-host-runtime.js
CHANGED
|
@@ -1,8 +1,33 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { DurableObject, WorkerEntrypoint } from "cloudflare:workers";
|
|
2
2
|
import { WorldCore } from "@moinulmoin/eden-world-cloudflare/core";
|
|
3
3
|
import { EVE_HOST_DEFAULTS, EVE_SCHEDULE_WAKE_LEAD_MS, EveHostError, assertNonEmpty, assertStableOrigin, createEveHostLifecycleObserver, createEveReadinessGate, createTrustedEveRequest, eveScheduleCronFiresWithin, IDENTIFIER_PATTERN, readProtectedRuntimeVariables, WORKER_NAME_PATTERN, isEveWorkflowInternalRoute, } from "./eve-host.js";
|
|
4
|
-
export { ContainerProxy };
|
|
5
4
|
const EVE_CLOUDFLARE_CONTAINERS_CA_PATH = "/etc/cloudflare/certs/cloudflare-containers-ca.crt";
|
|
5
|
+
/** Durable Object storage keys owned by `EveHostDurableContainer`. */
|
|
6
|
+
const EVE_HOST_CONTAINER_ACTIVITY_KEY = "eden.eve.host.last-activity-at";
|
|
7
|
+
const EVE_HOST_CONTAINER_SNAPSHOT_KEY = "eden.eve.host.workspace-snapshot";
|
|
8
|
+
const EVE_HOST_CONTAINER_SNAPSHOT_ATTEMPTS_KEY = "eden.eve.host.snapshot-attempts";
|
|
9
|
+
/** Snapshot attempts the alarm retries before destroying anyway so a stuck
|
|
10
|
+
* snapshot path never keeps billing alive forever. */
|
|
11
|
+
const EVE_HOST_SNAPSHOT_MAX_ATTEMPTS = 3;
|
|
12
|
+
/** Parses the `sleepAfter` duration spellings wrangler accepts ("300ms",
|
|
13
|
+
* "30s", "15m", "24h", "7d", or a bare number of seconds). */
|
|
14
|
+
function sleepAfterDurationMs(value) {
|
|
15
|
+
const match = /^([0-9]+)(ms|s|m|h|d)?$/u.exec(value.trim());
|
|
16
|
+
if (match === null)
|
|
17
|
+
return undefined;
|
|
18
|
+
const amount = Number.parseInt(match[1] ?? "", 10);
|
|
19
|
+
const unit = match[2] ?? "s";
|
|
20
|
+
const factor = unit === "ms"
|
|
21
|
+
? 1
|
|
22
|
+
: unit === "s"
|
|
23
|
+
? 1_000
|
|
24
|
+
: unit === "m"
|
|
25
|
+
? 60_000
|
|
26
|
+
: unit === "h"
|
|
27
|
+
? 3_600_000
|
|
28
|
+
: 86_400_000;
|
|
29
|
+
return amount * factor;
|
|
30
|
+
}
|
|
6
31
|
/** The deployment's single SQLite-backed Workflow World. */
|
|
7
32
|
export class EdenWorldDurableObject {
|
|
8
33
|
core;
|
|
@@ -28,7 +53,7 @@ export class EdenWorldDurableObject {
|
|
|
28
53
|
throw new Error("The World queue Container identity is unavailable.");
|
|
29
54
|
}
|
|
30
55
|
const namespace = env[binding];
|
|
31
|
-
const response = await namespace.getByName(instance).
|
|
56
|
+
const response = await namespace.getByName(instance).loopbackFetch(new Request(new URL(`/.well-known/workflow/v1/${message.path}`, origin), {
|
|
32
57
|
method: "POST",
|
|
33
58
|
headers: {
|
|
34
59
|
...message.headers,
|
|
@@ -38,7 +63,7 @@ export class EdenWorldDurableObject {
|
|
|
38
63
|
"x-vqs-message-attempt": String(message.attempt),
|
|
39
64
|
},
|
|
40
65
|
body: message.body,
|
|
41
|
-
})
|
|
66
|
+
}));
|
|
42
67
|
// Body transport errors must remain retryable; an HTTP status alone
|
|
43
68
|
// is not an acknowledgement of a streamed Container response.
|
|
44
69
|
const body = await response.text();
|
|
@@ -74,13 +99,16 @@ export class EdenWorldDurableObject {
|
|
|
74
99
|
}
|
|
75
100
|
}
|
|
76
101
|
/**
|
|
77
|
-
* Delivers the container's own HTTPS requests to its public workers.dev
|
|
78
|
-
* back into the
|
|
79
|
-
*
|
|
80
|
-
*
|
|
81
|
-
*
|
|
82
|
-
*
|
|
83
|
-
*
|
|
102
|
+
* Delivers the container's own HTTPS requests to its public workers.dev
|
|
103
|
+
* origin back into the deployment without crossing the public edge. The
|
|
104
|
+
* Workflow local World posts queue deliveries to `WORKFLOW_LOCAL_BASE_URL`,
|
|
105
|
+
* which must remain the public origin so Eve's externally visible callback
|
|
106
|
+
* URLs stay correct; the DO registers an outbound-HTTPS intercept for that
|
|
107
|
+
* hostname aimed at `EveHostLoopback`, and this function routes the request:
|
|
108
|
+
* non-public hosts egress plainly, `/__eden/world/rpc` reaches the World
|
|
109
|
+
* Durable Object, and everything else re-enters the container over the
|
|
110
|
+
* internal port so the public Worker path can keep refusing the
|
|
111
|
+
* unauthenticated queue endpoints.
|
|
84
112
|
*/
|
|
85
113
|
export async function routeEveOutboundRequest(request, env) {
|
|
86
114
|
const publicOrigin = env.EVE_PUBLIC_ORIGIN;
|
|
@@ -108,44 +136,48 @@ export async function routeEveOutboundRequest(request, env) {
|
|
|
108
136
|
if (namespace === undefined) {
|
|
109
137
|
throw new EveHostError("HOST_READINESS_UNPROVEN", "The configured Container binding is unavailable to the outbound handler.");
|
|
110
138
|
}
|
|
111
|
-
return namespace
|
|
112
|
-
.getByName(instanceName)
|
|
113
|
-
.containerFetch(request, EVE_HOST_DEFAULTS.internalPort);
|
|
139
|
+
return namespace.getByName(instanceName).loopbackFetch(request);
|
|
114
140
|
}
|
|
115
141
|
/**
|
|
116
|
-
*
|
|
117
|
-
*
|
|
118
|
-
*
|
|
119
|
-
* `
|
|
120
|
-
|
|
142
|
+
* This module is the only Eve-host surface that depends on provider runtime
|
|
143
|
+
* packages. It is bundled into the deployed Worker directory instead of the
|
|
144
|
+
* package's public declaration surface, so plain-Node consumers never load
|
|
145
|
+
* `cloudflare:workers`-dependent code.
|
|
146
|
+
*/
|
|
147
|
+
/**
|
|
148
|
+
* The WorkerEntrypoint the container's intercepted outbound HTTPS traffic is
|
|
149
|
+
* delivered to (registered by `EveHostDurableContainer` right after every
|
|
150
|
+
* `ctx.container.start()`). Props carry the DO instance name because
|
|
151
|
+
* entrypoint props cannot hold a Fetcher.
|
|
121
152
|
*/
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
const EVE_HOST_CONTAINER_RESTART_ATTEMPT_KEY = "eden.eve.host.restart-attempted-at";
|
|
127
|
-
/** Compares image references by their generation-scoped repository name so
|
|
128
|
-
* digest and tag spellings of the same pushed image match. */
|
|
129
|
-
function containerRepositoryName(image) {
|
|
130
|
-
return (image.split("@")[0] ?? "").split(":")[0] ?? "";
|
|
153
|
+
export class EveHostLoopback extends WorkerEntrypoint {
|
|
154
|
+
async fetch(request) {
|
|
155
|
+
return routeEveOutboundRequest(request, this.env);
|
|
156
|
+
}
|
|
131
157
|
}
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
158
|
+
/**
|
|
159
|
+
* The Eve host Durable Object. It drives `ctx.container` directly: the
|
|
160
|
+
* image is selected at `start()` from this Worker version's `images.eve`
|
|
161
|
+
* entry, the writable filesystem is preserved across sleep via a container
|
|
162
|
+
* snapshot, and an update restarts the instance when `inspect()` shows the
|
|
163
|
+
* running image predates the configured one.
|
|
164
|
+
*/
|
|
165
|
+
export class EveHostDurableContainer extends DurableObject {
|
|
139
166
|
lifecycle = createEveHostLifecycleObserver();
|
|
140
167
|
readinessStarted = false;
|
|
141
|
-
/** Evidence is cached only
|
|
168
|
+
/** Evidence is cached only while the container it was proven on keeps running. */
|
|
142
169
|
readinessEvidence;
|
|
170
|
+
/** True once the running container's image was proven to equal `images.eve`. */
|
|
171
|
+
imageVerified = false;
|
|
143
172
|
/**
|
|
144
173
|
* Bumped whenever the Container stops or is reset. Readiness work captures
|
|
145
|
-
* the epoch it started under and may only
|
|
146
|
-
*
|
|
174
|
+
* the epoch it started under and may only cache evidence if no stop
|
|
175
|
+
* happened in between.
|
|
147
176
|
*/
|
|
148
177
|
readinessEpoch = 0;
|
|
178
|
+
monitorAttached = false;
|
|
179
|
+
envVars;
|
|
180
|
+
sleepAfterMs;
|
|
149
181
|
readiness;
|
|
150
182
|
constructor(ctx, env) {
|
|
151
183
|
super(ctx, env);
|
|
@@ -184,132 +216,156 @@ export class EveHostContainer extends Container {
|
|
|
184
216
|
: { EDEN_EVE_RUNTIME_REVISION: env.EDEN_EVE_RUNTIME_REVISION }),
|
|
185
217
|
};
|
|
186
218
|
const sleepAfterOverride = env.EDEN_EVE_CONTAINER_SLEEP_AFTER;
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
219
|
+
this.sleepAfterMs =
|
|
220
|
+
sleepAfterOverride === undefined
|
|
221
|
+
? sleepAfterDurationMs(EVE_HOST_DEFAULTS.sleepAfter) ?? 86_400_000
|
|
222
|
+
: sleepAfterDurationMs(sleepAfterOverride) ??
|
|
223
|
+
(() => {
|
|
224
|
+
throw new EveHostError("HOST_READINESS_UNPROVEN", "The Container sleepAfter override is not a parseable duration.");
|
|
225
|
+
})();
|
|
193
226
|
this.readiness = createEveReadinessGate({
|
|
194
|
-
|
|
195
|
-
|
|
227
|
+
start: async () => {
|
|
228
|
+
await this.ensureContainerStarted();
|
|
229
|
+
},
|
|
230
|
+
healthFetch: (request) => {
|
|
231
|
+
const container = this.containerRuntime();
|
|
232
|
+
if (container === undefined) {
|
|
233
|
+
throw new EveHostError("HOST_READINESS_UNPROVEN", "The Container runtime is unavailable.");
|
|
234
|
+
}
|
|
235
|
+
const url = new URL(request.url);
|
|
236
|
+
return container
|
|
237
|
+
.getTcpPort(EVE_HOST_DEFAULTS.internalPort)
|
|
238
|
+
.fetch(new Request(`http://container${url.pathname}${url.search}`, request));
|
|
239
|
+
},
|
|
196
240
|
});
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
* DO-managed Container instances are excluded from the application's fleet
|
|
206
|
-
* rollout — the image is only re-selected on the next
|
|
207
|
-
* `ctx.container.start()` — so an update must stop the instance itself.
|
|
208
|
-
* A missing marker means the running image cannot be trusted, so the
|
|
209
|
-
* container is restarted too (first update from runtimes that predate the
|
|
210
|
-
* marker). Restarts are rate-limited because Cloudflare activates a new
|
|
211
|
-
* Worker before the image rollout lands; starting in that window would
|
|
212
|
-
* re-launch the *old* image, so the deployment marker is only recorded
|
|
213
|
-
* after `inspect()` proves the running image is the expected one.
|
|
214
|
-
*/
|
|
215
|
-
async reconcileStartedDeployment(requestDeploymentId, requestImage) {
|
|
216
|
-
// Requests without the Worker's deployment header (scheduled wakes on old
|
|
217
|
-
// Worker code, probes) cannot arbitrate drift; the Durable Object's own
|
|
218
|
-
// environment may itself be stale, so never fall back to it here.
|
|
219
|
-
if (requestDeploymentId === undefined)
|
|
220
|
-
return;
|
|
221
|
-
const storage = this.ctx?.storage;
|
|
222
|
-
if (storage === undefined ||
|
|
223
|
-
typeof storage.get !== "function" ||
|
|
224
|
-
typeof storage.put !== "function") {
|
|
225
|
-
return;
|
|
226
|
-
}
|
|
227
|
-
const lastStarted = await storage.get(EVE_HOST_CONTAINER_DEPLOYMENT_KEY);
|
|
228
|
-
if (lastStarted === requestDeploymentId)
|
|
229
|
-
return;
|
|
230
|
-
const runtime = this.ctx?.container;
|
|
231
|
-
if (runtime?.running !== true)
|
|
232
|
-
return;
|
|
233
|
-
if (requestImage !== undefined &&
|
|
234
|
-
(await this.containerServesImage(runtime, requestImage)) === true) {
|
|
235
|
-
// The running instance already provably serves this deployment's image
|
|
236
|
-
// (e.g. the rollout replaced it); safe to mark it as started.
|
|
237
|
-
await storage.put(EVE_HOST_CONTAINER_DEPLOYMENT_KEY, requestDeploymentId);
|
|
238
|
-
return;
|
|
241
|
+
const runtime = this.containerRuntime();
|
|
242
|
+
if (runtime?.running === true) {
|
|
243
|
+
// DO restarts lose the inactivity timeout; the monitor for this
|
|
244
|
+
// container generation is also gone, so both are re-attached here.
|
|
245
|
+
this.ctx.blockConcurrencyWhile(async () => {
|
|
246
|
+
await runtime.setInactivityTimeout(this.inactivityTimeoutMs);
|
|
247
|
+
this.attachMonitor(runtime);
|
|
248
|
+
});
|
|
239
249
|
}
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
250
|
+
}
|
|
251
|
+
containerRuntime() {
|
|
252
|
+
return this.ctx.container;
|
|
253
|
+
}
|
|
254
|
+
/** Container lifetime the platform keeps after the DO goes inactive. */
|
|
255
|
+
get inactivityTimeoutMs() {
|
|
256
|
+
return this.sleepAfterMs + 15 * 60_000;
|
|
257
|
+
}
|
|
258
|
+
attachMonitor(runtime) {
|
|
259
|
+
if (this.monitorAttached)
|
|
243
260
|
return;
|
|
244
|
-
|
|
245
|
-
|
|
261
|
+
this.monitorAttached = true;
|
|
262
|
+
runtime.monitor().then(() => {
|
|
263
|
+
this.monitorAttached = false;
|
|
264
|
+
this.recordContainerStopped("exit");
|
|
265
|
+
}, (error) => {
|
|
266
|
+
this.monitorAttached = false;
|
|
267
|
+
this.recordContainerStopped(error instanceof Error ? error.name : "runtime_signal");
|
|
268
|
+
});
|
|
269
|
+
}
|
|
270
|
+
recordContainerStopped(reason) {
|
|
246
271
|
this.readinessEpoch += 1;
|
|
247
272
|
this.readinessStarted = false;
|
|
248
273
|
this.readinessEvidence = undefined;
|
|
274
|
+
this.imageVerified = false;
|
|
249
275
|
this.readiness.reset();
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
276
|
+
this.lifecycle.record("stopped", reason);
|
|
277
|
+
}
|
|
278
|
+
/** Routes the container's own requests to its public origin back inside. */
|
|
279
|
+
async registerLoopbackIntercept(runtime) {
|
|
280
|
+
const exportsMap = this.ctx.exports;
|
|
281
|
+
const entryFactory = exportsMap?.["EveHostLoopback"];
|
|
282
|
+
if (typeof entryFactory !== "function")
|
|
283
|
+
return;
|
|
284
|
+
const entrypoint = entryFactory({ props: {} });
|
|
285
|
+
await runtime.interceptOutboundHttps(new URL(this.env.EVE_PUBLIC_ORIGIN ?? "").hostname, entrypoint);
|
|
286
|
+
}
|
|
287
|
+
/**
|
|
288
|
+
* Records request activity and arms the sleep alarm at
|
|
289
|
+
* `lastActivityAt + sleepAfter`. The alarm — not the inactivity timeout —
|
|
290
|
+
* is what snapshots the writable filesystem before the container stops.
|
|
291
|
+
*/
|
|
292
|
+
async touchActivity() {
|
|
293
|
+
const now = Date.now();
|
|
294
|
+
await this.ctx.storage.put(EVE_HOST_CONTAINER_ACTIVITY_KEY, now);
|
|
295
|
+
await this.ctx.storage.setAlarm(now + this.sleepAfterMs);
|
|
296
|
+
}
|
|
297
|
+
/**
|
|
298
|
+
* Starts the container when it is not running. A stored workspace snapshot
|
|
299
|
+
* is restored only when it was captured from the exact image this Worker
|
|
300
|
+
* version configures — restoring an older snapshot silently runs the
|
|
301
|
+
* previous image — so after an image update the container starts fresh and
|
|
302
|
+
* workspace files intentionally do not carry across.
|
|
303
|
+
*/
|
|
304
|
+
async ensureContainerStarted() {
|
|
305
|
+
const runtime = this.containerRuntime();
|
|
306
|
+
if (runtime === undefined) {
|
|
307
|
+
throw new EveHostError("HOST_READINESS_UNPROVEN", "The Container runtime is unavailable on this Durable Object.");
|
|
254
308
|
}
|
|
309
|
+
if (runtime.running)
|
|
310
|
+
return;
|
|
311
|
+
const expectedImage = runtime.images["eve"];
|
|
312
|
+
if (expectedImage === undefined) {
|
|
313
|
+
throw new EveHostError("HOST_READINESS_UNPROVEN", "The configured Container image is unavailable to this Worker version.");
|
|
314
|
+
}
|
|
315
|
+
const storedSnapshot = await this.ctx.storage.get(EVE_HOST_CONTAINER_SNAPSHOT_KEY);
|
|
316
|
+
if (this.lifecycle.events.some((event) => event.type === "started")) {
|
|
317
|
+
this.lifecycle.record("replaced");
|
|
318
|
+
}
|
|
319
|
+
runtime.start({
|
|
320
|
+
...(storedSnapshot !== undefined && storedSnapshot.image === expectedImage
|
|
321
|
+
? { containerSnapshot: { id: storedSnapshot.id } }
|
|
322
|
+
: { image: expectedImage }),
|
|
323
|
+
instance: EVE_HOST_DEFAULTS.instance,
|
|
324
|
+
enableInternet: true,
|
|
325
|
+
env: this.envVars,
|
|
326
|
+
});
|
|
327
|
+
// Intercepts do not survive a container start and late registration lags;
|
|
328
|
+
// register inside the same block before readiness polling.
|
|
329
|
+
await this.registerLoopbackIntercept(runtime);
|
|
330
|
+
this.attachMonitor(runtime);
|
|
331
|
+
await runtime.setInactivityTimeout(this.inactivityTimeoutMs);
|
|
332
|
+
this.lifecycle.record("started");
|
|
255
333
|
}
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
334
|
+
/**
|
|
335
|
+
* Replaces the running container when `inspect()` proves it still serves a
|
|
336
|
+
* pre-update image: this Worker version's `images.eve` is selected at
|
|
337
|
+
* `start()`, so an in-place update must stop the stale instance itself.
|
|
338
|
+
*/
|
|
339
|
+
async reconcileRunningImage() {
|
|
340
|
+
if (this.imageVerified)
|
|
341
|
+
return;
|
|
342
|
+
const runtime = this.containerRuntime();
|
|
343
|
+
if (runtime === undefined || !runtime.running)
|
|
344
|
+
return;
|
|
345
|
+
const expectedImage = runtime.images["eve"];
|
|
260
346
|
let info;
|
|
261
347
|
try {
|
|
262
|
-
info = await inspect
|
|
348
|
+
info = await runtime.inspect();
|
|
263
349
|
}
|
|
264
350
|
catch {
|
|
265
|
-
|
|
351
|
+
info = null;
|
|
266
352
|
}
|
|
267
|
-
if (info?.image ===
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
containerRepositoryName(expectedImage);
|
|
271
|
-
}
|
|
272
|
-
/**
|
|
273
|
-
* Records the started deployment only when the running Container provably
|
|
274
|
-
* serves this deployment's image (`inspect()` matches the forwarded
|
|
275
|
-
* `x-eden-eve-image`). `false` or unknown (no `inspect`, a throwing call,
|
|
276
|
-
* or no image field) never records — a start inside the
|
|
277
|
-
* Worker-active-before-image-rollout window must not pin the old image.
|
|
278
|
-
* When the Worker stamps no image, the marker is recorded unverified,
|
|
279
|
-
* matching pre-drift behavior.
|
|
280
|
-
*/
|
|
281
|
-
async recordStartedDeployment(requestDeploymentId, requestImage) {
|
|
282
|
-
if (requestDeploymentId === undefined)
|
|
283
|
-
return true;
|
|
284
|
-
const storage = this.ctx?.storage;
|
|
285
|
-
if (storage === undefined ||
|
|
286
|
-
typeof storage.get !== "function" ||
|
|
287
|
-
typeof storage.put !== "function") {
|
|
288
|
-
return true;
|
|
353
|
+
if (info?.image === expectedImage) {
|
|
354
|
+
this.imageVerified = true;
|
|
355
|
+
return;
|
|
289
356
|
}
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
}
|
|
297
|
-
const verified = await this.containerServesImage(runtime, requestImage);
|
|
298
|
-
if (verified !== true) {
|
|
299
|
-
this.lifecycle.record("image_mismatch");
|
|
300
|
-
return false;
|
|
301
|
-
}
|
|
357
|
+
this.lifecycle.record("image_mismatch");
|
|
358
|
+
this.recordContainerStopped("replaced");
|
|
359
|
+
await runtime.destroy();
|
|
360
|
+
const stoppedAt = Date.now();
|
|
361
|
+
while (runtime.running && Date.now() - stoppedAt < 10_000) {
|
|
362
|
+
await new Promise((resolve) => setTimeout(resolve, 25));
|
|
302
363
|
}
|
|
303
|
-
await storage.put(EVE_HOST_CONTAINER_DEPLOYMENT_KEY, requestDeploymentId);
|
|
304
|
-
return true;
|
|
305
364
|
}
|
|
306
|
-
async ensureEveReady(signal = new AbortController().signal
|
|
307
|
-
await this.
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
if (this.readinessEvidence !== undefined &&
|
|
311
|
-
this.readinessEvidence.deploymentId === requestDeploymentId) {
|
|
312
|
-
return this.readinessEvidence.evidence;
|
|
365
|
+
async ensureEveReady(signal = new AbortController().signal) {
|
|
366
|
+
await this.reconcileRunningImage();
|
|
367
|
+
if (this.readinessEvidence !== undefined) {
|
|
368
|
+
return this.readinessEvidence;
|
|
313
369
|
}
|
|
314
370
|
if (!this.readinessStarted) {
|
|
315
371
|
this.readinessStarted = true;
|
|
@@ -317,26 +373,37 @@ export class EveHostContainer extends Container {
|
|
|
317
373
|
}
|
|
318
374
|
const epoch = this.readinessEpoch;
|
|
319
375
|
const evidence = await this.readiness(signal);
|
|
320
|
-
if (epoch === this.readinessEpoch
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
};
|
|
376
|
+
if (epoch === this.readinessEpoch) {
|
|
377
|
+
this.readinessEvidence = evidence;
|
|
378
|
+
// Readiness under this Worker version proves the running container
|
|
379
|
+
// serves `images.eve`: it was either verified by inspect() or freshly
|
|
380
|
+
// started from the configured image above.
|
|
381
|
+
this.imageVerified = true;
|
|
327
382
|
}
|
|
328
383
|
this.lifecycle.record("health_ready", evidence.healthStatus);
|
|
329
384
|
return evidence;
|
|
330
385
|
}
|
|
331
386
|
async fetch(request) {
|
|
332
387
|
const requestDeploymentId = request.headers.get("x-eden-eve-deployment-id") ?? undefined;
|
|
333
|
-
await this.ensureEveReady(request.signal
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
388
|
+
await this.ensureEveReady(request.signal);
|
|
389
|
+
await this.touchActivity();
|
|
390
|
+
const runtime = this.containerRuntime();
|
|
391
|
+
if (runtime === undefined) {
|
|
392
|
+
throw new EveHostError("HOST_READINESS_UNPROVEN", "The Container runtime is unavailable on this Durable Object.");
|
|
393
|
+
}
|
|
394
|
+
const url = new URL(request.url);
|
|
395
|
+
const response = await runtime
|
|
396
|
+
.getTcpPort(EVE_HOST_DEFAULTS.internalPort)
|
|
397
|
+
.fetch(new Request(`http://container${url.pathname}${url.search}`, request));
|
|
398
|
+
// Reporting the verified start back lets deploy health-gate promotion on
|
|
399
|
+
// the served container actually running this generation's image: the
|
|
400
|
+
// header echoes only when the running container's image equals this
|
|
401
|
+
// Worker version's configured image and the request names this
|
|
402
|
+
// deployment.
|
|
403
|
+
if (requestDeploymentId !== undefined &&
|
|
404
|
+
requestDeploymentId === this.env.EDEN_EVE_DEPLOYMENT_ID &&
|
|
405
|
+
this.readinessEvidence !== undefined &&
|
|
406
|
+
this.imageVerified) {
|
|
340
407
|
const headers = new Headers(response.headers);
|
|
341
408
|
headers.set("x-eden-eve-started-deployment", requestDeploymentId);
|
|
342
409
|
return new Response(response.body, {
|
|
@@ -347,26 +414,79 @@ export class EveHostContainer extends Container {
|
|
|
347
414
|
}
|
|
348
415
|
return response;
|
|
349
416
|
}
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
417
|
+
/**
|
|
418
|
+
* Queue deliveries and loopback traffic enter here: same readiness and
|
|
419
|
+
* forwarding as `fetch` without the deployment-header echo rules (the
|
|
420
|
+
* loopback path is not a deployment-health signal).
|
|
421
|
+
*/
|
|
422
|
+
async loopbackFetch(request) {
|
|
423
|
+
await this.ensureEveReady(request.signal);
|
|
424
|
+
await this.touchActivity();
|
|
425
|
+
const runtime = this.containerRuntime();
|
|
426
|
+
if (runtime === undefined) {
|
|
427
|
+
throw new EveHostError("HOST_READINESS_UNPROVEN", "The Container runtime is unavailable on this Durable Object.");
|
|
353
428
|
}
|
|
354
|
-
|
|
429
|
+
const url = new URL(request.url);
|
|
430
|
+
return runtime
|
|
431
|
+
.getTcpPort(EVE_HOST_DEFAULTS.internalPort)
|
|
432
|
+
.fetch(new Request(`http://container${url.pathname}${url.search}`, request));
|
|
355
433
|
}
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
this.
|
|
365
|
-
this.
|
|
366
|
-
|
|
367
|
-
this.
|
|
368
|
-
|
|
369
|
-
|
|
434
|
+
/**
|
|
435
|
+
* The idle-sleep path: once the container has been quiet for `sleepAfter`,
|
|
436
|
+
* snapshot the writable filesystem (so `/workspace` survives the sleep)
|
|
437
|
+
* and destroy the instance. A snapshot failure retries up to
|
|
438
|
+
* `EVE_HOST_SNAPSHOT_MAX_ATTEMPTS` times, then destroys anyway so billing
|
|
439
|
+
* for the running container stops.
|
|
440
|
+
*/
|
|
441
|
+
async alarm() {
|
|
442
|
+
const runtime = this.containerRuntime();
|
|
443
|
+
const lastActivity = (await this.ctx.storage.get(EVE_HOST_CONTAINER_ACTIVITY_KEY)) ?? 0;
|
|
444
|
+
const now = Date.now();
|
|
445
|
+
if (now - lastActivity < this.sleepAfterMs) {
|
|
446
|
+
await this.ctx.storage.setAlarm(lastActivity + this.sleepAfterMs);
|
|
447
|
+
return;
|
|
448
|
+
}
|
|
449
|
+
if (runtime === undefined || !runtime.running)
|
|
450
|
+
return;
|
|
451
|
+
const attempts = (await this.ctx.storage.get(EVE_HOST_CONTAINER_SNAPSHOT_ATTEMPTS_KEY)) ?? 0;
|
|
452
|
+
try {
|
|
453
|
+
const snapshot = await runtime.snapshotContainer({
|
|
454
|
+
name: "eden-workspace",
|
|
455
|
+
});
|
|
456
|
+
const image = (await runtime.inspect().catch(() => null))?.image ??
|
|
457
|
+
runtime.images["eve"] ??
|
|
458
|
+
"";
|
|
459
|
+
const record = {
|
|
460
|
+
id: snapshot.id,
|
|
461
|
+
image,
|
|
462
|
+
at: now,
|
|
463
|
+
};
|
|
464
|
+
await this.ctx.storage.put(EVE_HOST_CONTAINER_SNAPSHOT_KEY, record);
|
|
465
|
+
await this.ctx.storage.put(EVE_HOST_CONTAINER_SNAPSHOT_ATTEMPTS_KEY, 0);
|
|
466
|
+
// The snapshot takes seconds and requests may interleave with it; a
|
|
467
|
+
// request that arrived meanwhile must not have its container destroyed
|
|
468
|
+
// underneath it. Keep running and re-arm sleep from that activity.
|
|
469
|
+
const activityAfterSnapshot = (await this.ctx.storage.get(EVE_HOST_CONTAINER_ACTIVITY_KEY)) ?? 0;
|
|
470
|
+
if (activityAfterSnapshot !== lastActivity) {
|
|
471
|
+
await this.ctx.storage.setAlarm(activityAfterSnapshot + this.sleepAfterMs);
|
|
472
|
+
return;
|
|
473
|
+
}
|
|
474
|
+
await runtime.destroy(new Error("idle-snapshot-done"));
|
|
475
|
+
this.recordContainerStopped("snapshot:idle");
|
|
476
|
+
}
|
|
477
|
+
catch (error) {
|
|
478
|
+
const message = error instanceof Error ? error.message : "unknown";
|
|
479
|
+
if (attempts + 1 >= EVE_HOST_SNAPSHOT_MAX_ATTEMPTS) {
|
|
480
|
+
console.error(`The Eve workspace snapshot failed ${attempts + 1} times; destroying the container without it: ${message}`);
|
|
481
|
+
await this.ctx.storage.put(EVE_HOST_CONTAINER_SNAPSHOT_ATTEMPTS_KEY, 0);
|
|
482
|
+
await runtime.destroy(new Error("idle-snapshot-abandoned"));
|
|
483
|
+
this.recordContainerStopped("snapshot:abandoned");
|
|
484
|
+
return;
|
|
485
|
+
}
|
|
486
|
+
console.error(`The Eve workspace snapshot failed (attempt ${attempts + 1}): ${message}`);
|
|
487
|
+
await this.ctx.storage.put(EVE_HOST_CONTAINER_SNAPSHOT_ATTEMPTS_KEY, attempts + 1);
|
|
488
|
+
await this.ctx.storage.setAlarm(now + 60_000);
|
|
489
|
+
}
|
|
370
490
|
}
|
|
371
491
|
}
|
|
372
492
|
export function createEveHostWorker(options) {
|
|
@@ -400,9 +520,6 @@ export function createEveHostWorker(options) {
|
|
|
400
520
|
deploymentId: options.deploymentId,
|
|
401
521
|
generationId: options.generationId,
|
|
402
522
|
correlationId: globalThis.crypto.randomUUID(),
|
|
403
|
-
...(options.containerImage === undefined
|
|
404
|
-
? {}
|
|
405
|
-
: { containerImage: options.containerImage }),
|
|
406
523
|
...(options.runtimeRevisionHandle === undefined
|
|
407
524
|
? {}
|
|
408
525
|
: { runtimeRevisionHandle: options.runtimeRevisionHandle }),
|
|
@@ -436,12 +553,8 @@ export function createEveHostWorker(options) {
|
|
|
436
553
|
.fetch(new Request(new URL(EVE_HOST_DEFAULTS.healthPath, options.publicOrigin), {
|
|
437
554
|
headers: {
|
|
438
555
|
// Stamp the Worker's own deployment so a stale Durable
|
|
439
|
-
// Object never falls back to its own environment
|
|
440
|
-
// deciding whether to restart the running Container.
|
|
556
|
+
// Object never falls back to its own environment.
|
|
441
557
|
"x-eden-eve-deployment-id": options.deploymentId,
|
|
442
|
-
...(options.containerImage === undefined
|
|
443
|
-
? {}
|
|
444
|
-
: { "x-eden-eve-image": options.containerImage }),
|
|
445
558
|
},
|
|
446
559
|
}))
|
|
447
560
|
.catch((error) => {
|