@evident-ai/runner-cdk 0.1.1-dev.da70cd4 → 3.4.1-dev.1856549
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +76 -19
- package/dist/controller-lambda/handler.js +50523 -0
- package/dist/evident-scale-to-zero-construct.js +4 -5
- package/dist/index.d.ts +4 -0
- package/dist/index.js +14 -1
- package/dist/microvm/constants.d.ts +7 -0
- package/dist/microvm/constants.js +34 -0
- package/dist/microvm/construct.d.ts +87 -0
- package/dist/microvm/construct.js +253 -0
- package/dist/microvm/controller/doorbell.d.ts +73 -0
- package/dist/microvm/controller/doorbell.js +107 -0
- package/dist/microvm/controller/handle-doorbell.d.ts +27 -0
- package/dist/microvm/controller/handle-doorbell.js +480 -0
- package/dist/microvm/controller/microvm-client.d.ts +75 -0
- package/dist/microvm/controller/microvm-client.js +7 -0
- package/dist/microvm/controller/shape-catalogue.d.ts +64 -0
- package/dist/microvm/controller/shape-catalogue.js +108 -0
- package/dist/microvm/controller/throttle-retry.d.ts +11 -0
- package/dist/microvm/controller/throttle-retry.js +27 -0
- package/dist/microvm/image/stage-context.d.ts +33 -0
- package/dist/microvm/image/stage-context.js +148 -0
- package/dist/microvm/shapes.d.ts +72 -0
- package/dist/microvm/shapes.js +93 -0
- package/dist/microvm-image-context/Dockerfile +227 -0
- package/dist/microvm-image-context/hook-server.js +286 -0
- package/dist/microvm-image-context/hooks/common.sh +1418 -0
- package/dist/microvm-image-context/hooks/resume +79 -0
- package/dist/microvm-image-context/hooks/run +117 -0
- package/dist/microvm-image-context/hooks/suspend +19 -0
- package/dist/microvm-image-context/hooks/terminate +34 -0
- package/dist/waker/construct.js +1 -1
- package/package.json +15 -7
|
@@ -0,0 +1,480 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.handleDoorbell = handleDoorbell;
|
|
4
|
+
const node_crypto_1 = require("node:crypto");
|
|
5
|
+
const sdk_1 = require("@evident/sdk");
|
|
6
|
+
const constants_1 = require("../constants");
|
|
7
|
+
const doorbell_1 = require("./doorbell");
|
|
8
|
+
const throttle_retry_1 = require("./throttle-retry");
|
|
9
|
+
const REASON = {
|
|
10
|
+
invalidSignature: 'invalid_signature',
|
|
11
|
+
noMicrovmId: 'no_microvm_id',
|
|
12
|
+
microvmNotFound: 'microvm_not_found',
|
|
13
|
+
alreadyRunning: 'already_running',
|
|
14
|
+
alreadyStarting: 'already_starting',
|
|
15
|
+
terminated: 'terminated_or_terminating',
|
|
16
|
+
imageVersionOutdated: 'image_version_outdated',
|
|
17
|
+
resumeFailedFallingBackToRun: 'resume_failed_falling_back_to_run',
|
|
18
|
+
suspendingPollExhausted: 'suspending_poll_exhausted',
|
|
19
|
+
suspendWithoutMicrovmId: 'suspend_without_microvm_id',
|
|
20
|
+
suspendAlreadyGone: 'suspend_already_gone',
|
|
21
|
+
getStateFailed: 'get_state_failed',
|
|
22
|
+
runFailed: 'run_failed',
|
|
23
|
+
runAlreadyStartedForToken: 'run_already_started_for_token',
|
|
24
|
+
suspendFailed: 'suspend_failed',
|
|
25
|
+
suspended: 'suspended',
|
|
26
|
+
resumed: 'resumed',
|
|
27
|
+
started: 'started',
|
|
28
|
+
shapesDescribed: 'shapes_described',
|
|
29
|
+
unknownShape: 'unknown_shape',
|
|
30
|
+
};
|
|
31
|
+
/** The log vocabulary the runbook greps for; `none`/`rejected` read as `noop`/`reject`. */
|
|
32
|
+
const LOG_ACTION = {
|
|
33
|
+
none: 'noop',
|
|
34
|
+
run: 'run',
|
|
35
|
+
resume: 'resume',
|
|
36
|
+
suspend: 'suspend',
|
|
37
|
+
rejected: 'reject',
|
|
38
|
+
describe: 'describe',
|
|
39
|
+
};
|
|
40
|
+
function decide({ statusCode, action, reason, state, runnerId, microvmId, startedAt, shapes, polled, }) {
|
|
41
|
+
const line = `[doorbell] state=${state ?? 'none'} action=${LOG_ACTION[action]} ` +
|
|
42
|
+
`microvm_id=${microvmId ?? '-'} runner_id=${runnerId ?? '-'} reason=${reason}` +
|
|
43
|
+
(polled === undefined ? '' : ` polled=${polled}`);
|
|
44
|
+
if (statusCode >= 500) {
|
|
45
|
+
console.error(line);
|
|
46
|
+
}
|
|
47
|
+
else if (statusCode >= 400) {
|
|
48
|
+
console.warn(line);
|
|
49
|
+
}
|
|
50
|
+
else {
|
|
51
|
+
console.log(line);
|
|
52
|
+
}
|
|
53
|
+
return {
|
|
54
|
+
statusCode,
|
|
55
|
+
body: JSON.stringify({
|
|
56
|
+
reason,
|
|
57
|
+
...(microvmId === undefined ? {} : { microvm_id: microvmId }),
|
|
58
|
+
...(startedAt === undefined ? {} : { microvm_started_at: startedAt.toISOString() }),
|
|
59
|
+
...(shapes === undefined ? {} : { shapes }),
|
|
60
|
+
...(polled === true ? { polled: true } : {}),
|
|
61
|
+
}),
|
|
62
|
+
action: { kind: action },
|
|
63
|
+
};
|
|
64
|
+
}
|
|
65
|
+
/**
|
|
66
|
+
* `RunMicrovm`'s idempotency key. Keyed on `runner_id` plus a "generation"
|
|
67
|
+
* marker: the dead VM's `microvm_id` on the recovery arms (`doorbell.microvmId`
|
|
68
|
+
* is defined — TERMINATED/TERMINATING/not_found and the resume-failed
|
|
69
|
+
* fallback), else `occurred_at` on the cold arm (no `microvm_id` yet). The id
|
|
70
|
+
* advances every VM generation — `mvm-A` dies, the token keyed on it starts
|
|
71
|
+
* `mvm-B`; once Evident learns `mvm-B`, the NEXT doorbell for that runner is
|
|
72
|
+
* keyed on `mvm-B` instead — so the token self-advances, and the only case
|
|
73
|
+
* that ever replays is the one we want collapsed: two concurrent wakes racing
|
|
74
|
+
* to replace the SAME dead id (#940). The cold arm keeps `occurred_at`: with
|
|
75
|
+
* no id yet, keying on `runner_id` alone would be stable for the runner's
|
|
76
|
+
* whole life, so an unrelated cold wake later in that runner's life would
|
|
77
|
+
* replay a stale cached response and start nothing.
|
|
78
|
+
*
|
|
79
|
+
* `RunMicrovm`'s idempotency is parameter-sensitive (measured, #895): a
|
|
80
|
+
* repeated token with a different `runHookPayload` is rejected
|
|
81
|
+
* (`ValidationException`, not the `ConflictException` the docs advertise).
|
|
82
|
+
* Evident mints a fresh runner key per delivery attempt, so a retry reuses
|
|
83
|
+
* this token but carries a different `runHookPayload` — exactly that
|
|
84
|
+
* rejection. `isClientTokenParameterMismatch` maps it to a 200 replay rather
|
|
85
|
+
* than a 500 (#938): AWS already accepted the token and started a VM under
|
|
86
|
+
* it, but the API gives idempotency, not lookup, so that response cannot
|
|
87
|
+
* carry a `microvm_id`. See the README's "occurred_at is required..."
|
|
88
|
+
* section.
|
|
89
|
+
*
|
|
90
|
+
* Deliberately excludes `shape` (D4): including it would mean a sender bug
|
|
91
|
+
* that retried the same event with a different shape starts TWO concurrent
|
|
92
|
+
* 8-hour VMs instead of deduping to one.
|
|
93
|
+
*/
|
|
94
|
+
function clientTokenFor(doorbell) {
|
|
95
|
+
const generation = doorbell.microvmId ?? doorbell.occurredAt;
|
|
96
|
+
return (0, node_crypto_1.createHash)('sha256').update(`${doorbell.runnerId}:${generation}`).digest('hex');
|
|
97
|
+
}
|
|
98
|
+
/**
|
|
99
|
+
* MicroVM image versions are dotted decimals (`48.0`, `50.0`), so they must NOT
|
|
100
|
+
* be compared as strings — `'9.0' > '10.0'` lexically. Returns the numeric
|
|
101
|
+
* components, or `undefined` for anything not exclusively digits and dots,
|
|
102
|
+
* which the caller must treat as "cannot tell".
|
|
103
|
+
*
|
|
104
|
+
* Deliberately private to the controller rather than imported from
|
|
105
|
+
* `@evident-ai/lambda-microvm-cdk`, which exports the same logic for pruning:
|
|
106
|
+
* this file is bundled into the controller Lambda, and importing that package
|
|
107
|
+
* would pull `aws-cdk-lib` into the bundle. Same reasoning as the duplicated
|
|
108
|
+
* `package-independence.test.ts` guards — a small copy beats a dependency that
|
|
109
|
+
* drags a whole CDK library across a runtime boundary.
|
|
110
|
+
*/
|
|
111
|
+
function parseImageVersion(value) {
|
|
112
|
+
const parts = value.trim().split('.');
|
|
113
|
+
if (parts.some((part) => !/^\d+$/.test(part))) {
|
|
114
|
+
return undefined;
|
|
115
|
+
}
|
|
116
|
+
const numbers = parts.map(Number);
|
|
117
|
+
return numbers.every(Number.isSafeInteger) ? numbers : undefined;
|
|
118
|
+
}
|
|
119
|
+
/** Negative when `a` orders before `b`, positive when after, 0 when equal. */
|
|
120
|
+
function compareImageVersions(a, b) {
|
|
121
|
+
for (let i = 0; i < Math.max(a.length, b.length); i++) {
|
|
122
|
+
const mine = a[i] ?? 0;
|
|
123
|
+
const theirs = b[i] ?? 0;
|
|
124
|
+
if (mine !== theirs) {
|
|
125
|
+
return mine - theirs;
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
return 0;
|
|
129
|
+
}
|
|
130
|
+
/**
|
|
131
|
+
* Whether the SUSPENDED VM must be thrown away and recreated because a newer
|
|
132
|
+
* image has been published since it booted — a resume would otherwise keep
|
|
133
|
+
* that VM on its old baked-in hooks and repo checkout for up to 8 hours.
|
|
134
|
+
*
|
|
135
|
+
* FAIL-SAFE, one direction only: recreating destroys the VM's filesystem
|
|
136
|
+
* (only the credentials and `opencode.db` in the object store survive), so
|
|
137
|
+
* ONLY provably-contrary evidence — a running version we parsed, a latest
|
|
138
|
+
* version we parsed, and the first strictly below the second — returns true.
|
|
139
|
+
* A throwing/denied lookup, an absent version on either side, and anything
|
|
140
|
+
* unparseable all warn and return false, i.e. resume exactly as before
|
|
141
|
+
* (`development-workflow.mdc`: never fail a gate on absent evidence).
|
|
142
|
+
*/
|
|
143
|
+
async function shouldRecreateForNewerImage(doorbell, shape, runningVersion, microvm) {
|
|
144
|
+
if (runningVersion === undefined) {
|
|
145
|
+
console.warn(`[doorbell] GetMicrovm reported no imageVersion for runner ${doorbell.runnerId}; resuming`);
|
|
146
|
+
return false;
|
|
147
|
+
}
|
|
148
|
+
const running = parseImageVersion(runningVersion);
|
|
149
|
+
if (running === undefined) {
|
|
150
|
+
console.warn(`[doorbell] unparseable running imageVersion "${runningVersion}" for runner ` +
|
|
151
|
+
`${doorbell.runnerId}; resuming`);
|
|
152
|
+
return false;
|
|
153
|
+
}
|
|
154
|
+
let latestVersion;
|
|
155
|
+
try {
|
|
156
|
+
// Not wrapped in `withThrottleRetry`, unlike every mutation: this runs on
|
|
157
|
+
// a path with a hard 8 s deadline and its failure mode is already the safe
|
|
158
|
+
// one, so spending the backoff budget to defend a resume buys nothing.
|
|
159
|
+
latestVersion = await microvm.latestImageVersion(shape.imageArn);
|
|
160
|
+
}
|
|
161
|
+
catch (error) {
|
|
162
|
+
console.warn(`[doorbell] GetMicrovmImage failed for runner ${doorbell.runnerId}; resuming on the ` +
|
|
163
|
+
`running image ${runningVersion}: ${error instanceof Error ? error.message : String(error)}`);
|
|
164
|
+
return false;
|
|
165
|
+
}
|
|
166
|
+
const latest = latestVersion === undefined ? undefined : parseImageVersion(latestVersion);
|
|
167
|
+
if (latest === undefined) {
|
|
168
|
+
console.warn(`[doorbell] no comparable latest image version (${latestVersion ?? 'none'}) for runner ` +
|
|
169
|
+
`${doorbell.runnerId}; resuming on the running image ${runningVersion}`);
|
|
170
|
+
return false;
|
|
171
|
+
}
|
|
172
|
+
// Strictly below, not merely different: a running version ABOVE the latest
|
|
173
|
+
// active one is not contrary evidence — the version it booted from has since
|
|
174
|
+
// been deactivated, and recreating would not roll it forward. Resume.
|
|
175
|
+
return compareImageVersions(running, latest) < 0;
|
|
176
|
+
}
|
|
177
|
+
async function handleDoorbell({ rawBody, signatureHeader, doorbellSecret, shapes, microvm, sleep, random, }) {
|
|
178
|
+
if (!(0, sdk_1.verifyEvidentSignature)(rawBody, signatureHeader, doorbellSecret)) {
|
|
179
|
+
return decide({
|
|
180
|
+
statusCode: 401,
|
|
181
|
+
action: 'rejected',
|
|
182
|
+
reason: REASON.invalidSignature,
|
|
183
|
+
});
|
|
184
|
+
}
|
|
185
|
+
const parsed = (0, doorbell_1.parseDoorbell)(rawBody);
|
|
186
|
+
if (!parsed.ok) {
|
|
187
|
+
return decide({
|
|
188
|
+
statusCode: 400,
|
|
189
|
+
action: 'rejected',
|
|
190
|
+
reason: parsed.reason,
|
|
191
|
+
});
|
|
192
|
+
}
|
|
193
|
+
const doorbell = parsed.doorbell;
|
|
194
|
+
const timing = { sleep, random };
|
|
195
|
+
if (doorbell.type === doorbell_1.SHAPES_EVENT_TYPE) {
|
|
196
|
+
// A pure read of deploy-time metadata — see doorbell.ts's header comment
|
|
197
|
+
// (D2) for why replay is deliberately undefended here.
|
|
198
|
+
return decide({
|
|
199
|
+
statusCode: 200,
|
|
200
|
+
action: 'describe',
|
|
201
|
+
reason: REASON.shapesDescribed,
|
|
202
|
+
shapes: shapes.advertise(),
|
|
203
|
+
});
|
|
204
|
+
}
|
|
205
|
+
// Resolve the requested shape immediately after parseDoorbell/the describe
|
|
206
|
+
// arm — BEFORE the suspend_requested branch and the whole state machine
|
|
207
|
+
// (D5). Resolving lazily inside runMicrovm would return 200 and silently
|
|
208
|
+
// ignore a removed shape on the RUNNING/PENDING no-op path and the
|
|
209
|
+
// successful-resume path, since runMicrovm is never reached on either. A
|
|
210
|
+
// suspend doorbell has no `shape` field, so nothing here runs for it.
|
|
211
|
+
if (doorbell.type !== 'runner.suspend_requested') {
|
|
212
|
+
const resolvedShape = shapes.resolve(doorbell.shape);
|
|
213
|
+
if (resolvedShape === undefined) {
|
|
214
|
+
// 400, not 404/409: a redelivery of the identical body cannot fix this.
|
|
215
|
+
// Never echo the requested shape name — the closed-constant rule
|
|
216
|
+
// (doorbell.ts's DOORBELL_REJECTION) applies to every reason here too,
|
|
217
|
+
// and Evident persists response_body.
|
|
218
|
+
return decide({
|
|
219
|
+
statusCode: 400,
|
|
220
|
+
action: 'rejected',
|
|
221
|
+
reason: REASON.unknownShape,
|
|
222
|
+
runnerId: doorbell.runnerId,
|
|
223
|
+
microvmId: doorbell.microvmId,
|
|
224
|
+
});
|
|
225
|
+
}
|
|
226
|
+
return handleWakeDoorbell(doorbell, resolvedShape, microvm, timing);
|
|
227
|
+
}
|
|
228
|
+
return suspendMicrovm(doorbell, microvm, timing);
|
|
229
|
+
}
|
|
230
|
+
async function handleWakeDoorbell(doorbell, shape, microvm, timing) {
|
|
231
|
+
if (doorbell.microvmId === undefined) {
|
|
232
|
+
return runMicrovm(doorbell, shape, microvm, timing, false, REASON.noMicrovmId);
|
|
233
|
+
}
|
|
234
|
+
let described;
|
|
235
|
+
let polled = false;
|
|
236
|
+
try {
|
|
237
|
+
described = await microvm.describe(doorbell.microvmId);
|
|
238
|
+
if (described.state === 'SUSPENDING') {
|
|
239
|
+
described = await pollWhileSuspending(doorbell.microvmId, microvm, timing.sleep);
|
|
240
|
+
polled = true;
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
catch (error) {
|
|
244
|
+
console.error(`[doorbell] GetMicrovm failed for runner ${doorbell.runnerId}: ` +
|
|
245
|
+
`${error instanceof Error ? error.message : String(error)}`);
|
|
246
|
+
// `polled` is omitted here, not passed as `false`: it is only assigned
|
|
247
|
+
// `true` after `pollWhileSuspending` returns, so a throw mid-poll would
|
|
248
|
+
// make `false` a fabricated claim that no poll was ever attempted.
|
|
249
|
+
return decide({
|
|
250
|
+
statusCode: 500,
|
|
251
|
+
action: 'none',
|
|
252
|
+
reason: REASON.getStateFailed,
|
|
253
|
+
runnerId: doorbell.runnerId,
|
|
254
|
+
microvmId: doorbell.microvmId,
|
|
255
|
+
});
|
|
256
|
+
}
|
|
257
|
+
const state = described.state;
|
|
258
|
+
if (state === 'SUSPENDING') {
|
|
259
|
+
// Retryable: Evident redelivers, by which time the suspend has settled.
|
|
260
|
+
// This arm is only reachable after a poll (the initial describe would
|
|
261
|
+
// not still be SUSPENDING), so `polled` is always true here.
|
|
262
|
+
return decide({
|
|
263
|
+
statusCode: 409,
|
|
264
|
+
action: 'none',
|
|
265
|
+
reason: REASON.suspendingPollExhausted,
|
|
266
|
+
state,
|
|
267
|
+
runnerId: doorbell.runnerId,
|
|
268
|
+
microvmId: doorbell.microvmId,
|
|
269
|
+
polled,
|
|
270
|
+
});
|
|
271
|
+
}
|
|
272
|
+
switch (state) {
|
|
273
|
+
case 'RUNNING':
|
|
274
|
+
case 'PENDING':
|
|
275
|
+
// Not a silent no-op: the doorbell fired because Evident believes the
|
|
276
|
+
// tunnel is down, so a live VM here is worth seeing in the logs.
|
|
277
|
+
return decide({
|
|
278
|
+
statusCode: 200,
|
|
279
|
+
action: 'none',
|
|
280
|
+
reason: state === 'RUNNING' ? REASON.alreadyRunning : REASON.alreadyStarting,
|
|
281
|
+
state,
|
|
282
|
+
runnerId: doorbell.runnerId,
|
|
283
|
+
microvmId: doorbell.microvmId,
|
|
284
|
+
polled,
|
|
285
|
+
startedAt: described.startedAt,
|
|
286
|
+
});
|
|
287
|
+
case 'SUSPENDED':
|
|
288
|
+
// A newer image has been published since this VM booted: resuming would
|
|
289
|
+
// keep it on the old baked-in hooks and repo checkout for up to 8 h, so
|
|
290
|
+
// take the same recreate path the arm below takes — which, passing no
|
|
291
|
+
// `imageVersion`, boots the latest (handler.ts's `run`).
|
|
292
|
+
if (await shouldRecreateForNewerImage(doorbell, shape, described.imageVersion, microvm)) {
|
|
293
|
+
return runMicrovm(doorbell, shape, microvm, timing, polled, REASON.imageVersionOutdated, state);
|
|
294
|
+
}
|
|
295
|
+
return resumeMicrovm(doorbell, doorbell.microvmId, shape, microvm, timing, polled, described.startedAt);
|
|
296
|
+
default:
|
|
297
|
+
// TERMINATED, TERMINATING, not_found — gone or going, so a fresh VM either
|
|
298
|
+
// way. This is what self-heals a stale `microvm_id`.
|
|
299
|
+
return runMicrovm(doorbell, shape, microvm, timing, polled, state === 'not_found' ? REASON.microvmNotFound : REASON.terminated, state);
|
|
300
|
+
}
|
|
301
|
+
}
|
|
302
|
+
/**
|
|
303
|
+
* Phase 0 measured suspend at ~405 ms, so resolving the race in-handler beats
|
|
304
|
+
* bouncing it through a Cloudflare Queues retry (backoff, a failed delivery, a
|
|
305
|
+
* stuck conversation). Returns the first non-`SUSPENDING` state it reads.
|
|
306
|
+
*/
|
|
307
|
+
async function pollWhileSuspending(microvmId, microvm, sleep) {
|
|
308
|
+
let described = { state: 'SUSPENDING' };
|
|
309
|
+
for (let read = 1; read < constants_1.SUSPENDING_POLL_ATTEMPTS && described.state === 'SUSPENDING'; read++) {
|
|
310
|
+
await sleep(constants_1.SUSPENDING_POLL_INTERVAL_MS);
|
|
311
|
+
described = await microvm.describe(microvmId);
|
|
312
|
+
}
|
|
313
|
+
return described;
|
|
314
|
+
}
|
|
315
|
+
/**
|
|
316
|
+
* True only when AWS rejected a repeated `clientToken` for carrying a
|
|
317
|
+
* different `runHookPayload` (measured, #895 / README's "occurred_at is
|
|
318
|
+
* required..." section) — the case #938 exists to fix: the VM this token
|
|
319
|
+
* names has actually started, so the retry that hit this must not 500 into a
|
|
320
|
+
* DLQ. Both the exception name AND both message substrings are required:
|
|
321
|
+
* `name` alone is far too wide (a genuine `RunMicrovm` failure sharing the
|
|
322
|
+
* name would then get a false 200, and the queue would stop retrying it —
|
|
323
|
+
* the runner never wakes, silently). Any wording this doesn't recognise falls
|
|
324
|
+
* through to today's 500, which is the safe direction to fail in.
|
|
325
|
+
*/
|
|
326
|
+
function isClientTokenParameterMismatch(error) {
|
|
327
|
+
const e = error;
|
|
328
|
+
if (e?.name !== 'ValidationException' && e?.name !== 'ConflictException') {
|
|
329
|
+
return false;
|
|
330
|
+
}
|
|
331
|
+
const message = typeof e.message === 'string' ? e.message.toLowerCase() : '';
|
|
332
|
+
return message.includes('clienttoken') && message.includes('different request parameters');
|
|
333
|
+
}
|
|
334
|
+
async function runMicrovm(doorbell, shape, microvm, timing, polled, reason, state) {
|
|
335
|
+
let started;
|
|
336
|
+
try {
|
|
337
|
+
started = await (0, throttle_retry_1.withThrottleRetry)(() => microvm.run({
|
|
338
|
+
imageIdentifier: shape.imageArn,
|
|
339
|
+
runHookPayload: doorbell.runHookPayload,
|
|
340
|
+
clientToken: clientTokenFor(doorbell),
|
|
341
|
+
}), timing);
|
|
342
|
+
}
|
|
343
|
+
catch (error) {
|
|
344
|
+
if (isClientTokenParameterMismatch(error)) {
|
|
345
|
+
// AWS already accepted this clientToken and started a VM under it; we
|
|
346
|
+
// just can't name it (RunMicrovm's idempotency gives no lookup) — see
|
|
347
|
+
// clientTokenFor's docstring. Log the complete AWS message verbatim, not
|
|
348
|
+
// just the runner id: the predicate above is matched against an
|
|
349
|
+
// untrusted, AWS-owned string, so if AWS ever rewords it we need to
|
|
350
|
+
// learn that from a log line, not from a runner that silently never
|
|
351
|
+
// wakes.
|
|
352
|
+
const name = error instanceof Error ? error.name : 'unknown';
|
|
353
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
354
|
+
console.warn(`[doorbell] RunMicrovm clientToken already used for runner ${doorbell.runnerId}: ` +
|
|
355
|
+
`${name}: ${message}`);
|
|
356
|
+
// No `microvmId` (see the comment above), and therefore deliberately no
|
|
357
|
+
// `startedAt` either: `decide`'s invariant is that `microvm_started_at`
|
|
358
|
+
// only ever appears beside `microvm_id`. A timestamp with no id to pair
|
|
359
|
+
// it with is exactly the stale pairing this feature exists to prevent
|
|
360
|
+
// — Evident would have no VM to attach the expiry deadline to.
|
|
361
|
+
return decide({
|
|
362
|
+
statusCode: 200,
|
|
363
|
+
action: 'run',
|
|
364
|
+
reason: REASON.runAlreadyStartedForToken,
|
|
365
|
+
state,
|
|
366
|
+
runnerId: doorbell.runnerId,
|
|
367
|
+
polled,
|
|
368
|
+
});
|
|
369
|
+
}
|
|
370
|
+
console.error(`[doorbell] RunMicrovm failed for runner ${doorbell.runnerId}: ` +
|
|
371
|
+
`${error instanceof Error ? error.message : String(error)}`);
|
|
372
|
+
return decide({
|
|
373
|
+
statusCode: 500,
|
|
374
|
+
action: 'run',
|
|
375
|
+
reason: REASON.runFailed,
|
|
376
|
+
state,
|
|
377
|
+
runnerId: doorbell.runnerId,
|
|
378
|
+
polled,
|
|
379
|
+
});
|
|
380
|
+
}
|
|
381
|
+
return decide({
|
|
382
|
+
statusCode: 200,
|
|
383
|
+
action: 'run',
|
|
384
|
+
reason,
|
|
385
|
+
state,
|
|
386
|
+
runnerId: doorbell.runnerId,
|
|
387
|
+
microvmId: started.microvmId,
|
|
388
|
+
polled,
|
|
389
|
+
startedAt: started.startedAt,
|
|
390
|
+
});
|
|
391
|
+
}
|
|
392
|
+
async function resumeMicrovm(doorbell, microvmId, shape, microvm, timing, polled,
|
|
393
|
+
// Threaded in from the `describe()` result that led here — `ResumeMicrovm`
|
|
394
|
+
// itself returns no `startedAt` (its response is empty), so this arm has no
|
|
395
|
+
// timestamp of its own to report.
|
|
396
|
+
startedAt) {
|
|
397
|
+
try {
|
|
398
|
+
await (0, throttle_retry_1.withThrottleRetry)(() => microvm.resume(microvmId), timing);
|
|
399
|
+
}
|
|
400
|
+
catch (error) {
|
|
401
|
+
console.error(`[doorbell] ResumeMicrovm failed for runner ${doorbell.runnerId}: ` +
|
|
402
|
+
`${error instanceof Error ? error.message : String(error)}`);
|
|
403
|
+
return runMicrovm(doorbell, shape, microvm, timing, polled, REASON.resumeFailedFallingBackToRun, 'SUSPENDED');
|
|
404
|
+
}
|
|
405
|
+
return decide({
|
|
406
|
+
statusCode: 200,
|
|
407
|
+
action: 'resume',
|
|
408
|
+
reason: REASON.resumed,
|
|
409
|
+
state: 'SUSPENDED',
|
|
410
|
+
runnerId: doorbell.runnerId,
|
|
411
|
+
microvmId,
|
|
412
|
+
polled,
|
|
413
|
+
startedAt,
|
|
414
|
+
});
|
|
415
|
+
}
|
|
416
|
+
async function suspendMicrovm(doorbell, microvm, timing) {
|
|
417
|
+
const microvmId = doorbell.microvmId;
|
|
418
|
+
if (microvmId === undefined) {
|
|
419
|
+
// Nothing to suspend, and a redelivery would not change that — 200 so the
|
|
420
|
+
// doorbell is not retried, but loudly enough to notice a mis-wired sender.
|
|
421
|
+
return decide({
|
|
422
|
+
statusCode: 200,
|
|
423
|
+
action: 'none',
|
|
424
|
+
reason: REASON.suspendWithoutMicrovmId,
|
|
425
|
+
runnerId: doorbell.runnerId,
|
|
426
|
+
});
|
|
427
|
+
}
|
|
428
|
+
let state;
|
|
429
|
+
try {
|
|
430
|
+
// Only the state matters here: a suspend never consults the image version.
|
|
431
|
+
({ state } = await microvm.describe(microvmId));
|
|
432
|
+
}
|
|
433
|
+
catch (error) {
|
|
434
|
+
console.error(`[doorbell] GetMicrovm failed for runner ${doorbell.runnerId}: ` +
|
|
435
|
+
`${error instanceof Error ? error.message : String(error)}`);
|
|
436
|
+
return decide({
|
|
437
|
+
statusCode: 500,
|
|
438
|
+
action: 'none',
|
|
439
|
+
reason: REASON.getStateFailed,
|
|
440
|
+
runnerId: doorbell.runnerId,
|
|
441
|
+
microvmId,
|
|
442
|
+
});
|
|
443
|
+
}
|
|
444
|
+
if (state === 'TERMINATED' || state === 'TERMINATING' || state === 'not_found') {
|
|
445
|
+
// The desired end state already holds — the VM is gone or going, so
|
|
446
|
+
// SuspendMicrovm must never be called here (that is what turned a
|
|
447
|
+
// no-op into a blanket 500-into-DLQ, #1182).
|
|
448
|
+
return decide({
|
|
449
|
+
statusCode: 200,
|
|
450
|
+
action: 'none',
|
|
451
|
+
reason: REASON.suspendAlreadyGone,
|
|
452
|
+
state,
|
|
453
|
+
runnerId: doorbell.runnerId,
|
|
454
|
+
microvmId,
|
|
455
|
+
});
|
|
456
|
+
}
|
|
457
|
+
try {
|
|
458
|
+
await (0, throttle_retry_1.withThrottleRetry)(() => microvm.suspend(microvmId), timing);
|
|
459
|
+
}
|
|
460
|
+
catch (error) {
|
|
461
|
+
console.error(`[doorbell] SuspendMicrovm failed for runner ${doorbell.runnerId}: ` +
|
|
462
|
+
`${error instanceof Error ? error.message : String(error)}`);
|
|
463
|
+
return decide({
|
|
464
|
+
statusCode: 500,
|
|
465
|
+
action: 'suspend',
|
|
466
|
+
reason: REASON.suspendFailed,
|
|
467
|
+
state,
|
|
468
|
+
runnerId: doorbell.runnerId,
|
|
469
|
+
microvmId,
|
|
470
|
+
});
|
|
471
|
+
}
|
|
472
|
+
return decide({
|
|
473
|
+
statusCode: 200,
|
|
474
|
+
action: 'suspend',
|
|
475
|
+
reason: REASON.suspended,
|
|
476
|
+
state,
|
|
477
|
+
runnerId: doorbell.runnerId,
|
|
478
|
+
microvmId,
|
|
479
|
+
});
|
|
480
|
+
}
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The MicroVM lifecycle surface the decision core depends on — one method per AWS
|
|
3
|
+
* operation, no SDK types, so tests drive it with a hand-rolled fake and the
|
|
4
|
+
* runtime shell is the only place that knows about `@aws-sdk/client-lambda-microvms`.
|
|
5
|
+
*/
|
|
6
|
+
/** Mirrors the SDK's `MicrovmState`; all six values, not just the three §4 names. */
|
|
7
|
+
export type MicrovmLifecycleState = 'PENDING' | 'RUNNING' | 'SUSPENDED' | 'SUSPENDING' | 'TERMINATED' | 'TERMINATING';
|
|
8
|
+
/** What one `GetMicrovm` tells the decision core. */
|
|
9
|
+
export type MicrovmDescription = {
|
|
10
|
+
state: MicrovmLifecycleState | 'not_found';
|
|
11
|
+
/**
|
|
12
|
+
* The image version this VM is running, absent when `not_found` or when
|
|
13
|
+
* `GetMicrovm` returned none. Its absence must never be read as "up to
|
|
14
|
+
* date" — see `handle-doorbell.ts`'s `shouldRecreateForNewerImage`.
|
|
15
|
+
*/
|
|
16
|
+
imageVersion?: string;
|
|
17
|
+
/**
|
|
18
|
+
* When this VM instance started, straight from `GetMicrovm`'s own
|
|
19
|
+
* `startedAt` — absent when `not_found` or when `GetMicrovm` returned none.
|
|
20
|
+
* Evident derives this VM's AWS-imposed 8 h expiry deadline from this
|
|
21
|
+
* value, so its absence must ALWAYS be read as UNKNOWN, never as "it just
|
|
22
|
+
* started" — a fabricated timestamp is worse than none. Never synthesize
|
|
23
|
+
* one (e.g. `new Date()`) here or in any caller.
|
|
24
|
+
*/
|
|
25
|
+
startedAt?: Date;
|
|
26
|
+
};
|
|
27
|
+
export interface MicrovmClient {
|
|
28
|
+
describe(microvmId: string): Promise<MicrovmDescription>;
|
|
29
|
+
/**
|
|
30
|
+
* The image's latest ACTIVE version — the one a `run()` below boots, since
|
|
31
|
+
* it passes no `imageVersion`. `undefined` when the image reports none.
|
|
32
|
+
* Throws whatever the API threw; callers must treat both that and
|
|
33
|
+
* `undefined` as "unknown", never as "unchanged".
|
|
34
|
+
*/
|
|
35
|
+
latestImageVersion(imageIdentifier: string): Promise<string | undefined>;
|
|
36
|
+
/**
|
|
37
|
+
* `imageIdentifier` is chosen by the decision core from the shape catalogue
|
|
38
|
+
* (a doorbell asks for a shape by name; the core resolves it to an image
|
|
39
|
+
* ARN before calling here). `clientToken` is still derived by the decision
|
|
40
|
+
* core (from `runner_id:occurred_at`) and deliberately excludes the shape —
|
|
41
|
+
* it is idempotency-critical, so it is unit-tested where it is built.
|
|
42
|
+
*
|
|
43
|
+
* No `idlePolicy` field (AC5, #732) — deliberately omitted, not an oversight.
|
|
44
|
+
* `maxIdleDurationSeconds` measures INBOUND endpoint traffic only; our runner
|
|
45
|
+
* is outbound-only (it dials the Cloudflare relay), so AWS sees an idle
|
|
46
|
+
* endpoint no matter how busy the agent is — Phase 0 measured a VM with
|
|
47
|
+
* `maxIdleDurationSeconds=60` and zero inbound traffic suspended after 71 s
|
|
48
|
+
* while its guest process was alive and working
|
|
49
|
+
* (docs/spikes/lambda-microvms-phase0/README.md). AWS's own Claude-agents
|
|
50
|
+
* guidance recommending `maxIdleDurationSeconds: 120` targets VMs reached
|
|
51
|
+
* inbound and does not apply here. Idle is instead decided IN the VM by
|
|
52
|
+
* `evident run --idle-timeout` (runner/docker-images/microvm/hooks/common.sh's
|
|
53
|
+
* `IDLE_TIMEOUT_SECONDS`), whose clean exit drives this
|
|
54
|
+
* client's own `suspend()` via the `runner.suspend_requested` doorbell.
|
|
55
|
+
*/
|
|
56
|
+
run(input: {
|
|
57
|
+
imageIdentifier: string;
|
|
58
|
+
runHookPayload: string;
|
|
59
|
+
clientToken: string;
|
|
60
|
+
}): Promise<{
|
|
61
|
+
microvmId: string;
|
|
62
|
+
/**
|
|
63
|
+
* When AWS reports this VM instance started, straight from
|
|
64
|
+
* `RunMicrovm`'s own `startedAt` (verified: `RunMicrovmResponse` returns
|
|
65
|
+
* it — `@aws-sdk/client-lambda-microvms` `models_0.d.ts`). Absent means
|
|
66
|
+
* AWS's own response omitted it, and must be read as UNKNOWN, never as
|
|
67
|
+
* "it just started" — Evident derives this VM's AWS-imposed 8 h expiry
|
|
68
|
+
* deadline from this value, so a fabricated timestamp is worse than
|
|
69
|
+
* none. Never synthesize one (e.g. `new Date()`) here or in any caller.
|
|
70
|
+
*/
|
|
71
|
+
startedAt?: Date;
|
|
72
|
+
}>;
|
|
73
|
+
resume(microvmId: string): Promise<void>;
|
|
74
|
+
suspend(microvmId: string): Promise<void>;
|
|
75
|
+
}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* The MicroVM lifecycle surface the decision core depends on — one method per AWS
|
|
4
|
+
* operation, no SDK types, so tests drive it with a hand-rolled fake and the
|
|
5
|
+
* runtime shell is the only place that knows about `@aws-sdk/client-lambda-microvms`.
|
|
6
|
+
*/
|
|
7
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Parses the `MICROVM_SHAPES` environment variable (a JSON array, written by
|
|
3
|
+
* CloudFormation from `infrastructure/evident-microvm/src/shapes.ts` at
|
|
4
|
+
* deploy time — see D6/D7 in the plan) into a `ShapeCatalogue` the pure
|
|
5
|
+
* decision core can query, without that core ever touching `process.env`
|
|
6
|
+
* itself.
|
|
7
|
+
*
|
|
8
|
+
* A malformed catalogue means a broken deploy: `parseShapeCatalogue` throws
|
|
9
|
+
* once, loudly, at cold start rather than handing back a half-parsed
|
|
10
|
+
* catalogue that degrades per-request.
|
|
11
|
+
*
|
|
12
|
+
* `resolve()` never falls back to the default shape for a *named* request —
|
|
13
|
+
* only the absence of a name (`undefined`) means "give me the default". An
|
|
14
|
+
* unrecognised name returns `undefined` so the caller can reject loudly
|
|
15
|
+
* (AC-4) instead of silently launching the default shape a doorbell did not
|
|
16
|
+
* ask for.
|
|
17
|
+
*/
|
|
18
|
+
export interface ControllerShape {
|
|
19
|
+
readonly name: string;
|
|
20
|
+
readonly title: string;
|
|
21
|
+
readonly description: string;
|
|
22
|
+
readonly memoryMiB: number;
|
|
23
|
+
readonly imageArn: string;
|
|
24
|
+
readonly isDefault: boolean;
|
|
25
|
+
}
|
|
26
|
+
/**
|
|
27
|
+
* The wire projection of a shape for Evident's future picker. Deliberately
|
|
28
|
+
* omits `imageArn`: it is an account-internal identifier Evident never needs
|
|
29
|
+
* (it asks for a shape by name; the controller resolves the ARN) and has no
|
|
30
|
+
* business crossing the account boundary.
|
|
31
|
+
*/
|
|
32
|
+
export interface AdvertisedShape {
|
|
33
|
+
readonly name: string;
|
|
34
|
+
readonly title: string;
|
|
35
|
+
readonly description: string;
|
|
36
|
+
readonly memory_mib: number;
|
|
37
|
+
readonly is_default: boolean;
|
|
38
|
+
}
|
|
39
|
+
export interface ShapeCatalogue {
|
|
40
|
+
/**
|
|
41
|
+
* `undefined` (no `shape` field on the doorbell) resolves to the default
|
|
42
|
+
* shape. A defined-but-unrecognised name resolves to `undefined` — the
|
|
43
|
+
* caller must treat that as a rejection, never as "use the default".
|
|
44
|
+
*/
|
|
45
|
+
resolve(name: string | undefined): ControllerShape | undefined;
|
|
46
|
+
/**
|
|
47
|
+
* Returns every shape in catalogue order — the order the operator wrote
|
|
48
|
+
* them in `shapes.ts`, with the default first (enforced there). Evident's
|
|
49
|
+
* future picker gets a deliberate ordering rather than an accidental one.
|
|
50
|
+
*/
|
|
51
|
+
advertise(): AdvertisedShape[];
|
|
52
|
+
}
|
|
53
|
+
/**
|
|
54
|
+
* Parses and validates the `MICROVM_SHAPES` catalogue. Mirrors
|
|
55
|
+
* `validateShapes`'s (`src/shapes.ts`) message vocabulary so an operator
|
|
56
|
+
* reading a controller cold-start failure recognises it as the same rule
|
|
57
|
+
* they already met at synth time.
|
|
58
|
+
*
|
|
59
|
+
* Throws on: invalid JSON, a non-array, an empty array, an entry missing a
|
|
60
|
+
* non-blank `name`/`title`/`description`/`image_arn`, a non-number
|
|
61
|
+
* `memory_mib`, a non-boolean `is_default`, a duplicate `name`, or
|
|
62
|
+
* zero/multiple `is_default: true` entries.
|
|
63
|
+
*/
|
|
64
|
+
export declare function parseShapeCatalogue(raw: string): ShapeCatalogue;
|