@irtio/cli 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundle.d.ts +9 -1
- package/dist/bundle.js +3 -1
- package/dist/chunk-BPE452KF.js +180 -0
- package/dist/{chunk-J3G6AUJY.js → chunk-I37DLT7K.js} +1 -1
- package/dist/{chunk-ZWCLCYCS.js → chunk-KRQUAEN2.js} +35 -6
- package/dist/{chunk-UYN5PWLT.js → chunk-NVUKSP5U.js} +3 -1
- package/dist/chunk-TQU6345E.js +36 -0
- package/dist/{deploy-4W6AYYEW.js → deploy-6OM3UYNL.js} +93 -15
- package/dist/{dev-BFWFWJ4J.js → dev-LOSTB4CD.js} +1095 -114
- package/dist/index.js +46 -10
- package/dist/init.d.ts +3 -0
- package/dist/init.js +236 -8
- package/dist/keys-KEKO3EJ6.js +174 -0
- package/dist/{login-US2YT5ZB.js → login-OV2EFNTJ.js} +26 -5
- package/dist/{logs-6GLBO3TP.js → logs-5OUDPAIX.js} +11 -3
- package/dist/{migrate-FQIDF747.js → migrate-3R6VRW4J.js} +7 -5
- package/dist/rollback-CLQVYFHW.js +122 -0
- package/dist/rooms-OJ3JLYHD.js +199 -0
- package/dist/simulate.d.ts +42 -1
- package/dist/simulate.js +129 -6
- package/dist/static-deploy-2LB6H54Y.js +199 -0
- package/dist/{whoami-5Q32SXFX.js → whoami-S73O6KJF.js} +2 -2
- package/package.json +9 -7
- package/dist/rooms-JXT3PRHN.js +0 -102
|
@@ -1,14 +1,29 @@
|
|
|
1
|
+
import {
|
|
2
|
+
clientImportsRoom
|
|
3
|
+
} from "./chunk-TQU6345E.js";
|
|
1
4
|
import {
|
|
2
5
|
BundleError,
|
|
3
6
|
bundleRoom
|
|
4
|
-
} from "./chunk-
|
|
7
|
+
} from "./chunk-KRQUAEN2.js";
|
|
8
|
+
import {
|
|
9
|
+
DEFAULT_SAVE_RETAIN,
|
|
10
|
+
DiskStore,
|
|
11
|
+
KV_ERRORS,
|
|
12
|
+
PLAYER_ISSUER_RE,
|
|
13
|
+
PRE_MIGRATION_SAVE_ID,
|
|
14
|
+
listSaves,
|
|
15
|
+
mintSaveId,
|
|
16
|
+
pruneSaves,
|
|
17
|
+
saveKey
|
|
18
|
+
} from "./chunk-BPE452KF.js";
|
|
5
19
|
|
|
6
20
|
// src/dev.ts
|
|
7
21
|
import { existsSync } from "fs";
|
|
8
22
|
import { watch } from "fs";
|
|
9
|
-
import { mkdir
|
|
10
|
-
import
|
|
11
|
-
import
|
|
23
|
+
import { mkdir, readFile } from "fs/promises";
|
|
24
|
+
import { createRequire } from "module";
|
|
25
|
+
import * as path from "path";
|
|
26
|
+
import { fileURLToPath, pathToFileURL as pathToFileURL3 } from "url";
|
|
12
27
|
|
|
13
28
|
// ../supervisor/src/contract.ts
|
|
14
29
|
var DEFAULT_LIMITS = {
|
|
@@ -24,7 +39,8 @@ var DEFAULT_LIMITS = {
|
|
|
24
39
|
workerReadyTimeoutMs: 1e4,
|
|
25
40
|
relayIdleMs: 3e4,
|
|
26
41
|
relayReconnectGraceMs: 3e4,
|
|
27
|
-
relayMaxClients: 64
|
|
42
|
+
relayMaxClients: 64,
|
|
43
|
+
saveRetain: DEFAULT_SAVE_RETAIN
|
|
28
44
|
};
|
|
29
45
|
|
|
30
46
|
// ../supervisor/src/resilience.ts
|
|
@@ -65,75 +81,13 @@ async function putSnapshotWithRetry(store, key, bytes, metrics, log) {
|
|
|
65
81
|
return false;
|
|
66
82
|
}
|
|
67
83
|
|
|
68
|
-
// ../store/dist/index.js
|
|
69
|
-
import { mkdir, readFile, readdir, rename, rm, writeFile } from "fs/promises";
|
|
70
|
-
import * as path from "path";
|
|
71
|
-
var DiskStore = class {
|
|
72
|
-
constructor(dir) {
|
|
73
|
-
this.dir = dir;
|
|
74
|
-
}
|
|
75
|
-
dir;
|
|
76
|
-
fileFor(key) {
|
|
77
|
-
if (!/^[A-Za-z0-9_.@/-]+$/.test(key) || key.includes("..")) {
|
|
78
|
-
throw new Error(`invalid object store key ${JSON.stringify(key)}`);
|
|
79
|
-
}
|
|
80
|
-
return path.join(this.dir, `${key}.snap`);
|
|
81
|
-
}
|
|
82
|
-
async put(key, bytes) {
|
|
83
|
-
const file = this.fileFor(key);
|
|
84
|
-
await mkdir(path.dirname(file), { recursive: true });
|
|
85
|
-
const tmp = `${file}.${process.pid}.${Math.random().toString(36).slice(2)}.tmp`;
|
|
86
|
-
await writeFile(tmp, bytes);
|
|
87
|
-
for (let attempt = 0; ; attempt++) {
|
|
88
|
-
try {
|
|
89
|
-
await rename(tmp, file);
|
|
90
|
-
return;
|
|
91
|
-
} catch (err) {
|
|
92
|
-
const code = err.code;
|
|
93
|
-
if ((code === "EPERM" || code === "EBUSY") && attempt < 10) {
|
|
94
|
-
await new Promise((resolve2) => setTimeout(resolve2, 5 * (attempt + 1)));
|
|
95
|
-
continue;
|
|
96
|
-
}
|
|
97
|
-
throw err;
|
|
98
|
-
}
|
|
99
|
-
}
|
|
100
|
-
}
|
|
101
|
-
async get(key) {
|
|
102
|
-
try {
|
|
103
|
-
return new Uint8Array(await readFile(this.fileFor(key)));
|
|
104
|
-
} catch (err) {
|
|
105
|
-
if (err.code === "ENOENT") return void 0;
|
|
106
|
-
throw err;
|
|
107
|
-
}
|
|
108
|
-
}
|
|
109
|
-
async delete(key) {
|
|
110
|
-
await rm(this.fileFor(key), { force: true });
|
|
111
|
-
}
|
|
112
|
-
async list(prefix) {
|
|
113
|
-
const out = [];
|
|
114
|
-
const walk = async (dir, rel) => {
|
|
115
|
-
let entries;
|
|
116
|
-
try {
|
|
117
|
-
entries = await readdir(dir, { withFileTypes: true });
|
|
118
|
-
} catch (err) {
|
|
119
|
-
if (err.code === "ENOENT") return;
|
|
120
|
-
throw err;
|
|
121
|
-
}
|
|
122
|
-
for (const e of entries) {
|
|
123
|
-
const r = rel ? `${rel}/${e.name}` : e.name;
|
|
124
|
-
if (e.isDirectory()) await walk(path.join(dir, e.name), r);
|
|
125
|
-
else if (e.name.endsWith(".snap")) out.push(r.slice(0, -".snap".length));
|
|
126
|
-
}
|
|
127
|
-
};
|
|
128
|
-
await walk(this.dir, "");
|
|
129
|
-
return out.filter((k) => k.startsWith(prefix)).sort();
|
|
130
|
-
}
|
|
131
|
-
};
|
|
132
|
-
|
|
133
84
|
// ../supervisor/src/server.ts
|
|
134
85
|
import { execFile } from "child_process";
|
|
135
86
|
import { randomBytes, timingSafeEqual as timingSafeEqual2 } from "crypto";
|
|
87
|
+
import { mkdtemp, writeFile } from "fs/promises";
|
|
136
88
|
import { createServer } from "http";
|
|
89
|
+
import { tmpdir } from "os";
|
|
90
|
+
import * as nodePath from "path";
|
|
137
91
|
import { pathToFileURL as pathToFileURL2 } from "url";
|
|
138
92
|
import {
|
|
139
93
|
ErrorCode as ErrorCode2,
|
|
@@ -158,6 +112,147 @@ import { bytesEqual } from "@irtio/schema";
|
|
|
158
112
|
import { isRoomDefinition } from "@irtio/server";
|
|
159
113
|
import { WebSocketServer } from "ws";
|
|
160
114
|
|
|
115
|
+
// ../supervisor/src/alarms.ts
|
|
116
|
+
var MAX_TIMER_MS = 2 ** 31 - 1;
|
|
117
|
+
var systemAlarmClock = {
|
|
118
|
+
now: () => Date.now(),
|
|
119
|
+
setTimeout: (fn, ms) => {
|
|
120
|
+
const t = setTimeout(fn, ms);
|
|
121
|
+
t.unref?.();
|
|
122
|
+
return t;
|
|
123
|
+
},
|
|
124
|
+
clearTimeout: (h) => clearTimeout(h)
|
|
125
|
+
};
|
|
126
|
+
var AlarmSet = class {
|
|
127
|
+
constructor(clock = systemAlarmClock) {
|
|
128
|
+
this.clock = clock;
|
|
129
|
+
}
|
|
130
|
+
clock;
|
|
131
|
+
/** `name` -> wall-clock ms the alarm is due at. Wall clock, not monotonic: it has to mean the
|
|
132
|
+
* same thing to a control plane on another machine and across a tenant restart. */
|
|
133
|
+
due = /* @__PURE__ */ new Map();
|
|
134
|
+
timer;
|
|
135
|
+
/** Called when one or more alarms come due, with the names in deterministic (name) order. */
|
|
136
|
+
deliver;
|
|
137
|
+
/**
|
|
138
|
+
* Where due alarms go. Attached for the room's whole life, **including while it is
|
|
139
|
+
* hibernated** — that is what makes the hibernated case work at all: the timer fires, delivery
|
|
140
|
+
* wakes the room, and the alarm lands in a running room a moment later. The worker going away
|
|
141
|
+
* is not a reason to stop counting down; only the room being gone is.
|
|
142
|
+
*/
|
|
143
|
+
attach(deliver) {
|
|
144
|
+
this.deliver = deliver;
|
|
145
|
+
this.rearm();
|
|
146
|
+
}
|
|
147
|
+
/** The room is closed for good. The timer goes; the map is no longer anyone's business. */
|
|
148
|
+
detach() {
|
|
149
|
+
this.deliver = void 0;
|
|
150
|
+
this.clearTimer();
|
|
151
|
+
}
|
|
152
|
+
/** `atMs === undefined` cancels. Arming a name that is already armed replaces its due time. */
|
|
153
|
+
set(name, atMs) {
|
|
154
|
+
if (atMs === void 0) this.due.delete(name);
|
|
155
|
+
else this.due.set(name, atMs);
|
|
156
|
+
this.rearm();
|
|
157
|
+
this.onChange?.();
|
|
158
|
+
}
|
|
159
|
+
get size() {
|
|
160
|
+
return this.due.size;
|
|
161
|
+
}
|
|
162
|
+
/** Called whenever the armed set changes, so the supervisor can persist it. */
|
|
163
|
+
onChange;
|
|
164
|
+
/** Replaces the armed set wholesale — how a room start re-arms from the persisted sidecar. */
|
|
165
|
+
load(entries) {
|
|
166
|
+
this.due.clear();
|
|
167
|
+
for (const e of entries) this.due.set(e.name, e.dueAt);
|
|
168
|
+
this.rearm();
|
|
169
|
+
}
|
|
170
|
+
/** Snapshot for `/admin/rooms` and for tests. */
|
|
171
|
+
entries() {
|
|
172
|
+
return [...this.due.entries()].map(([name, dueAt]) => ({ name, dueAt })).sort((a, b) => a.dueAt - b.dueAt || (a.name < b.name ? -1 : 1));
|
|
173
|
+
}
|
|
174
|
+
/** The earliest due time, or `undefined` when nothing is armed. What control is told. */
|
|
175
|
+
get earliestDueAt() {
|
|
176
|
+
let best;
|
|
177
|
+
for (const at of this.due.values()) if (best === void 0 || at < best) best = at;
|
|
178
|
+
return best;
|
|
179
|
+
}
|
|
180
|
+
/**
|
|
181
|
+
* Fires everything due at or before `now`, in **name order** so a room with two alarms coming
|
|
182
|
+
* due in the same instant behaves identically on every machine and every replay (D34). Each is
|
|
183
|
+
* removed *before* it is delivered, so a handler that re-arms its own name arms the next one
|
|
184
|
+
* rather than having this pass immediately cancel it back out.
|
|
185
|
+
*/
|
|
186
|
+
fireDue(now = this.clock.now()) {
|
|
187
|
+
const names = [...this.due.entries()].filter(([, at]) => at <= now).map(([name]) => name).sort();
|
|
188
|
+
if (names.length === 0) return names;
|
|
189
|
+
for (const name of names) this.due.delete(name);
|
|
190
|
+
this.deliver?.(names);
|
|
191
|
+
this.rearm();
|
|
192
|
+
this.onChange?.();
|
|
193
|
+
return names;
|
|
194
|
+
}
|
|
195
|
+
clearTimer() {
|
|
196
|
+
if (this.timer !== void 0) {
|
|
197
|
+
this.clock.clearTimeout(this.timer);
|
|
198
|
+
this.timer = void 0;
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
/**
|
|
202
|
+
* One timer, for the earliest alarm. Recomputed after every change rather than incrementally
|
|
203
|
+
* maintained: an alarm set is a handful of entries, and the incremental version is where the
|
|
204
|
+
* stale-timer bugs live.
|
|
205
|
+
*/
|
|
206
|
+
rearm() {
|
|
207
|
+
this.clearTimer();
|
|
208
|
+
if (!this.deliver) return;
|
|
209
|
+
const earliest = this.earliestDueAt;
|
|
210
|
+
if (earliest === void 0) return;
|
|
211
|
+
const delay = Math.min(MAX_TIMER_MS, Math.max(0, earliest - this.clock.now()));
|
|
212
|
+
this.timer = this.clock.setTimeout(() => {
|
|
213
|
+
this.timer = void 0;
|
|
214
|
+
if (this.fireDue().length === 0) this.rearm();
|
|
215
|
+
}, delay);
|
|
216
|
+
}
|
|
217
|
+
};
|
|
218
|
+
function alarmsKey(liveKey) {
|
|
219
|
+
return `${liveKey}/alarms`;
|
|
220
|
+
}
|
|
221
|
+
function roomIdOfAlarmsKey(projectPrefix, key) {
|
|
222
|
+
if (!key.startsWith(projectPrefix)) return void 0;
|
|
223
|
+
const rest = key.slice(projectPrefix.length);
|
|
224
|
+
if (!rest.endsWith("/alarms")) return void 0;
|
|
225
|
+
const liveKey = rest.slice(0, -"/alarms".length);
|
|
226
|
+
if (liveKey === "" || liveKey.includes("/")) return void 0;
|
|
227
|
+
const at = liveKey.lastIndexOf("@v");
|
|
228
|
+
if (at === -1) return liveKey;
|
|
229
|
+
const version = liveKey.slice(at + 2);
|
|
230
|
+
if (!/^[0-9]+$/.test(version)) return liveKey;
|
|
231
|
+
const room = liveKey.slice(0, at);
|
|
232
|
+
return room === "" ? void 0 : room;
|
|
233
|
+
}
|
|
234
|
+
function encodeAlarms(entries) {
|
|
235
|
+
return new TextEncoder().encode(JSON.stringify({ v: 1, alarms: entries }));
|
|
236
|
+
}
|
|
237
|
+
function decodeAlarms(bytes) {
|
|
238
|
+
let parsed;
|
|
239
|
+
try {
|
|
240
|
+
parsed = JSON.parse(new TextDecoder().decode(bytes));
|
|
241
|
+
} catch {
|
|
242
|
+
return [];
|
|
243
|
+
}
|
|
244
|
+
const list = parsed?.alarms;
|
|
245
|
+
if (!Array.isArray(list)) return [];
|
|
246
|
+
const out = [];
|
|
247
|
+
for (const entry of list) {
|
|
248
|
+
const e = entry;
|
|
249
|
+
if (typeof e.name !== "string" || e.name === "") continue;
|
|
250
|
+
if (typeof e.dueAt !== "number" || !Number.isFinite(e.dueAt)) continue;
|
|
251
|
+
out.push({ name: e.name, dueAt: e.dueAt });
|
|
252
|
+
}
|
|
253
|
+
return out;
|
|
254
|
+
}
|
|
255
|
+
|
|
161
256
|
// ../supervisor/src/auth.ts
|
|
162
257
|
import { createHmac, timingSafeEqual } from "crypto";
|
|
163
258
|
var TokenBucket = class {
|
|
@@ -208,6 +303,80 @@ function originAllowed(origins, allowNoOrigin, origin) {
|
|
|
208
303
|
if (origins.includes("*")) return true;
|
|
209
304
|
return origins.includes(origin);
|
|
210
305
|
}
|
|
306
|
+
var JWT_CLOCK_SKEW_MS = 6e4;
|
|
307
|
+
var JWT_SUB_RE = /^[\x21-\x7e]{1,128}$/;
|
|
308
|
+
function verifyJwt(issuers, token, projectId, now = Date.now()) {
|
|
309
|
+
const malformed = (reason) => ({
|
|
310
|
+
ok: false,
|
|
311
|
+
code: "E_TOKEN_MALFORMED",
|
|
312
|
+
reason
|
|
313
|
+
});
|
|
314
|
+
const parts = token.split(".");
|
|
315
|
+
if (parts.length !== 3) return malformed("not three dot-separated segments");
|
|
316
|
+
const [headerB64, payloadB64, signatureB64] = parts;
|
|
317
|
+
let header2;
|
|
318
|
+
let claims;
|
|
319
|
+
try {
|
|
320
|
+
header2 = JSON.parse(Buffer.from(headerB64, "base64url").toString("utf8"));
|
|
321
|
+
claims = JSON.parse(Buffer.from(payloadB64, "base64url").toString("utf8"));
|
|
322
|
+
} catch {
|
|
323
|
+
return malformed("header or payload is not base64url JSON");
|
|
324
|
+
}
|
|
325
|
+
if (typeof claims.sub !== "string" || !JWT_SUB_RE.test(claims.sub)) {
|
|
326
|
+
return malformed("sub is required: 1-128 printable ASCII characters, no spaces");
|
|
327
|
+
}
|
|
328
|
+
if (typeof claims.iss !== "string" || !PLAYER_ISSUER_RE.test(claims.iss)) {
|
|
329
|
+
return malformed("iss is required: 1-64 of a-z 0-9 . _ - (never a colon)");
|
|
330
|
+
}
|
|
331
|
+
if (typeof claims.aud !== "string") {
|
|
332
|
+
return malformed("aud (the project id) is required");
|
|
333
|
+
}
|
|
334
|
+
if (typeof claims.exp !== "number" || !Number.isFinite(claims.exp)) {
|
|
335
|
+
return malformed("exp (unix seconds) is required");
|
|
336
|
+
}
|
|
337
|
+
if (claims.aud !== projectId) {
|
|
338
|
+
return { ok: false, code: "E_TOKEN_WRONG_PROJECT", reason: "aud is another project" };
|
|
339
|
+
}
|
|
340
|
+
const accepted = issuers.filter((i) => i.issuer === claims.iss);
|
|
341
|
+
if (accepted.length === 0) {
|
|
342
|
+
return { ok: false, code: "E_TOKEN_BAD_ISSUER", reason: String(claims.iss) };
|
|
343
|
+
}
|
|
344
|
+
if (!accepted.some((i) => i.alg === header2.alg)) {
|
|
345
|
+
return { ok: false, code: "E_TOKEN_BAD_ALG", reason: String(header2.alg) };
|
|
346
|
+
}
|
|
347
|
+
const signedPart = `${headerB64}.${payloadB64}`;
|
|
348
|
+
const provided = Buffer.from(signatureB64, "base64url");
|
|
349
|
+
let signed = false;
|
|
350
|
+
for (const entry of accepted) {
|
|
351
|
+
if (entry.alg !== "HS256") continue;
|
|
352
|
+
const expected = createHmac("sha256", entry.secret).update(signedPart).digest();
|
|
353
|
+
if (provided.length === expected.length && timingSafeEqual(provided, expected)) {
|
|
354
|
+
signed = true;
|
|
355
|
+
break;
|
|
356
|
+
}
|
|
357
|
+
}
|
|
358
|
+
if (!signed) return { ok: false, code: "E_TOKEN_INVALID", reason: "signature mismatch" };
|
|
359
|
+
if (claims.exp * 1e3 + JWT_CLOCK_SKEW_MS <= now) {
|
|
360
|
+
return { ok: false, code: "E_TOKEN_EXPIRED", reason: "exp is in the past" };
|
|
361
|
+
}
|
|
362
|
+
if (typeof claims.iat === "number" && Number.isFinite(claims.iat) && claims.iat * 1e3 - JWT_CLOCK_SKEW_MS > now) {
|
|
363
|
+
return { ok: false, code: "E_TOKEN_EXPIRED", reason: "iat is in the future" };
|
|
364
|
+
}
|
|
365
|
+
if (claims.roomId !== void 0 && typeof claims.roomId !== "string") {
|
|
366
|
+
return malformed("roomId claim must be a string");
|
|
367
|
+
}
|
|
368
|
+
if (claims.role !== void 0 && typeof claims.role !== "string") {
|
|
369
|
+
return malformed("role claim must be a string");
|
|
370
|
+
}
|
|
371
|
+
return {
|
|
372
|
+
ok: true,
|
|
373
|
+
iss: claims.iss,
|
|
374
|
+
sub: claims.sub,
|
|
375
|
+
playerId: `${claims.iss}:${claims.sub}`,
|
|
376
|
+
...claims.role !== void 0 ? { role: claims.role } : {},
|
|
377
|
+
...claims.roomId !== void 0 ? { roomId: claims.roomId } : {}
|
|
378
|
+
};
|
|
379
|
+
}
|
|
211
380
|
function mac(secret, payload) {
|
|
212
381
|
return createHmac("sha256", secret).update(payload).digest("base64url");
|
|
213
382
|
}
|
|
@@ -505,7 +674,135 @@ async function wakeRoom(sup, room) {
|
|
|
505
674
|
});
|
|
506
675
|
await room.transition;
|
|
507
676
|
}
|
|
677
|
+
async function migrateRoom(sup, room, toVersion) {
|
|
678
|
+
const left = (reason) => ({
|
|
679
|
+
roomId: room.id,
|
|
680
|
+
outcome: "left",
|
|
681
|
+
reason
|
|
682
|
+
});
|
|
683
|
+
const failed = (reason) => ({
|
|
684
|
+
roomId: room.id,
|
|
685
|
+
outcome: "failed",
|
|
686
|
+
reason
|
|
687
|
+
});
|
|
688
|
+
if (room.relay) return left("relay room; nothing to migrate");
|
|
689
|
+
await room.transition.catch(() => {
|
|
690
|
+
});
|
|
691
|
+
if (room.state === "closed") return left("room is closed");
|
|
692
|
+
if (room.version >= toVersion) return left(`already on v${room.version}`);
|
|
693
|
+
if (room.state !== "running" && room.state !== "hibernated") {
|
|
694
|
+
return left(`room is ${room.state}`);
|
|
695
|
+
}
|
|
696
|
+
const fromVersion = room.version;
|
|
697
|
+
const oldKey = sup.storeKey(room);
|
|
698
|
+
let chain;
|
|
699
|
+
try {
|
|
700
|
+
chain = sup.migrationChain(fromVersion, toVersion);
|
|
701
|
+
} catch (err) {
|
|
702
|
+
return failed(err instanceof Error ? err.message : String(err));
|
|
703
|
+
}
|
|
704
|
+
let outcome = failed("migration did not complete");
|
|
705
|
+
const startedAt = performance.now();
|
|
706
|
+
const work = (async () => {
|
|
707
|
+
let bytes;
|
|
708
|
+
const worker = room.worker;
|
|
709
|
+
if (room.state === "running" && worker) {
|
|
710
|
+
room.state = "hibernating";
|
|
711
|
+
bytes = await worker.serialize();
|
|
712
|
+
if (!bytes) {
|
|
713
|
+
room.state = "running";
|
|
714
|
+
outcome = failed("the worker produced no snapshot; room left running");
|
|
715
|
+
return;
|
|
716
|
+
}
|
|
717
|
+
const log = (level, ...args) => sup.roomLog(room, level, ...args);
|
|
718
|
+
if (!await putSnapshotWithRetry(sup.store, oldKey, bytes, sup.processMetrics, log)) {
|
|
719
|
+
room.state = "running";
|
|
720
|
+
outcome = failed("could not persist the pre-migration snapshot; room left running");
|
|
721
|
+
return;
|
|
722
|
+
}
|
|
723
|
+
if (!await putSnapshotWithRetry(
|
|
724
|
+
sup.store,
|
|
725
|
+
saveKey(oldKey, PRE_MIGRATION_SAVE_ID),
|
|
726
|
+
bytes,
|
|
727
|
+
sup.processMetrics,
|
|
728
|
+
log
|
|
729
|
+
)) {
|
|
730
|
+
room.state = "running";
|
|
731
|
+
outcome = failed("could not retain the pre-migration save; room left running");
|
|
732
|
+
return;
|
|
733
|
+
}
|
|
734
|
+
worker.post({ t: "stop" });
|
|
735
|
+
await worker.terminate();
|
|
736
|
+
room.worker = void 0;
|
|
737
|
+
sup.clearRoomTimer(room);
|
|
738
|
+
} else {
|
|
739
|
+
bytes = await sup.store.get(oldKey) ?? void 0;
|
|
740
|
+
if (!bytes) {
|
|
741
|
+
outcome = failed("hibernated room has no snapshot to migrate");
|
|
742
|
+
return;
|
|
743
|
+
}
|
|
744
|
+
const ok = await putSnapshotWithRetry(
|
|
745
|
+
sup.store,
|
|
746
|
+
saveKey(oldKey, PRE_MIGRATION_SAVE_ID),
|
|
747
|
+
bytes,
|
|
748
|
+
sup.processMetrics,
|
|
749
|
+
(level, ...args) => sup.roomLog(room, level, ...args)
|
|
750
|
+
);
|
|
751
|
+
if (!ok) {
|
|
752
|
+
outcome = failed("could not retain the pre-migration save; room left hibernated");
|
|
753
|
+
return;
|
|
754
|
+
}
|
|
755
|
+
}
|
|
756
|
+
room.state = "waking";
|
|
757
|
+
room.version = toVersion;
|
|
758
|
+
const started = await sup.startWorker(room, bytes, chain);
|
|
759
|
+
if (!started) {
|
|
760
|
+
room.version = fromVersion;
|
|
761
|
+
const revived = await sup.startWorker(room, bytes);
|
|
762
|
+
if (!revived) {
|
|
763
|
+
room.worker = void 0;
|
|
764
|
+
room.state = "hibernated";
|
|
765
|
+
outcome = failed(
|
|
766
|
+
`migration to v${toVersion} failed and the v${fromVersion} worker would not restart; room left hibernated with its state intact`
|
|
767
|
+
);
|
|
768
|
+
return;
|
|
769
|
+
}
|
|
770
|
+
await sup.rejoinAll(room);
|
|
771
|
+
for (const frame of room.wakeQueue.splice(0)) sup.deliverQueued(room, frame);
|
|
772
|
+
room.state = "running";
|
|
773
|
+
outcome = failed(`migration to v${toVersion} failed; room restarted on v${fromVersion}`);
|
|
774
|
+
return;
|
|
775
|
+
}
|
|
776
|
+
sup.failOutdatedSessions(room, toVersion);
|
|
777
|
+
await sup.rejoinAll(room);
|
|
778
|
+
for (const clientId of room.pendingLeaves) {
|
|
779
|
+
room.worker?.post({ t: "leave", clientId, reason: "timeout" });
|
|
780
|
+
}
|
|
781
|
+
room.pendingLeaves.clear();
|
|
782
|
+
const queued = room.wakeQueue.splice(0);
|
|
783
|
+
for (const frame of queued) sup.deliverQueued(room, frame);
|
|
784
|
+
if (room.state === "waking") room.state = "running";
|
|
785
|
+
room.metrics.wakes++;
|
|
786
|
+
const ms = performance.now() - startedAt;
|
|
787
|
+
room.metrics.lastWakeMs = ms;
|
|
788
|
+
if (ms > room.metrics.maxWakeMs) room.metrics.maxWakeMs = ms;
|
|
789
|
+
sup.roomLog(room, "info", `migrated v${fromVersion} \u2192 v${toVersion}, gap ${ms.toFixed(1)} ms`);
|
|
790
|
+
outcome = { roomId: room.id, outcome: "migrated" };
|
|
791
|
+
})();
|
|
792
|
+
room.transition = work.catch((err) => {
|
|
793
|
+
sup.roomLog(room, "error", "migrate failed", err);
|
|
794
|
+
outcome = failed(err instanceof Error ? err.message : String(err));
|
|
795
|
+
if (room.state === "waking" || room.state === "hibernating") {
|
|
796
|
+
room.version = fromVersion;
|
|
797
|
+
room.state = room.worker ? "running" : "hibernated";
|
|
798
|
+
}
|
|
799
|
+
});
|
|
800
|
+
await room.transition;
|
|
801
|
+
return outcome;
|
|
802
|
+
}
|
|
508
803
|
async function flushAll(sup) {
|
|
804
|
+
sup.quiesceAlarms();
|
|
805
|
+
let allOk = true;
|
|
509
806
|
for (const room of sup.registry.values()) {
|
|
510
807
|
await room.transition.catch(() => {
|
|
511
808
|
});
|
|
@@ -518,29 +815,35 @@ async function flushAll(sup) {
|
|
|
518
815
|
continue;
|
|
519
816
|
}
|
|
520
817
|
const worker = room.worker;
|
|
521
|
-
if (worker) {
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
)
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
818
|
+
if (!worker) {
|
|
819
|
+
sup.clearRoomTimer(room);
|
|
820
|
+
room.state = "hibernated";
|
|
821
|
+
continue;
|
|
822
|
+
}
|
|
823
|
+
try {
|
|
824
|
+
const bytes = await worker.serialize();
|
|
825
|
+
if (bytes) {
|
|
826
|
+
const ok = await putSnapshotWithRetry(
|
|
827
|
+
sup.store,
|
|
828
|
+
sup.storeKey(room),
|
|
829
|
+
bytes,
|
|
830
|
+
sup.processMetrics,
|
|
831
|
+
(level, ...args) => sup.roomLog(room, level, ...args)
|
|
832
|
+
);
|
|
833
|
+
if (!ok) throw new Error("snapshot flush abandoned after retries");
|
|
535
834
|
}
|
|
536
835
|
worker.post({ t: "stop" });
|
|
537
836
|
await worker.terminate();
|
|
538
837
|
room.worker = void 0;
|
|
539
838
|
room.metrics.hibernations++;
|
|
839
|
+
sup.clearRoomTimer(room);
|
|
840
|
+
room.state = "hibernated";
|
|
841
|
+
} catch (err) {
|
|
842
|
+
sup.roomLog(room, "error", "flush: snapshot failed, leaving the room running", err);
|
|
843
|
+
allOk = false;
|
|
540
844
|
}
|
|
541
|
-
sup.clearRoomTimer(room);
|
|
542
|
-
room.state = "hibernated";
|
|
543
845
|
}
|
|
846
|
+
return allOk;
|
|
544
847
|
}
|
|
545
848
|
|
|
546
849
|
// ../supervisor/src/metrics.ts
|
|
@@ -629,6 +932,18 @@ var RoomRecord = class {
|
|
|
629
932
|
* before a deploy keep theirs until they idle (per-room drain, plan §3.3).
|
|
630
933
|
*/
|
|
631
934
|
version = 0;
|
|
935
|
+
/**
|
|
936
|
+
* D26: this room's durable alarms. Deliberately on the *record*, not on the worker: the record
|
|
937
|
+
* outlives every hibernation, which is exactly the lifetime an alarm needs. Nothing here is
|
|
938
|
+
* ever serialized into the hibernation blob.
|
|
939
|
+
*/
|
|
940
|
+
alarms = new AlarmSet();
|
|
941
|
+
/**
|
|
942
|
+
* D24: a save generation to start this room from instead of its live snapshot, set from the
|
|
943
|
+
* placement instruction and consumed by the next start. One shot: a restore is an event, not a
|
|
944
|
+
* standing property of the room.
|
|
945
|
+
*/
|
|
946
|
+
pendingRestoreSaveId;
|
|
632
947
|
/** Serialises room-level async work (start / wake / restart / hibernate). */
|
|
633
948
|
transition = Promise.resolve();
|
|
634
949
|
/** Periodic crash-safety snapshot (event-mode bundle rooms) or the relay idle timer. */
|
|
@@ -653,6 +968,7 @@ var RoomRecord = class {
|
|
|
653
968
|
this.logs.push(level, args);
|
|
654
969
|
}
|
|
655
970
|
info() {
|
|
971
|
+
const dueAlarmAt = this.alarms.earliestDueAt;
|
|
656
972
|
return {
|
|
657
973
|
id: this.id,
|
|
658
974
|
state: this.state,
|
|
@@ -660,7 +976,9 @@ var RoomRecord = class {
|
|
|
660
976
|
tick: this.lastTick,
|
|
661
977
|
clients: [...this.clients.values()].map((s) => s.info()),
|
|
662
978
|
metrics: { ...this.metrics, workerElu: this.worker?.elu ?? 0 },
|
|
663
|
-
createdAt: this.createdAt
|
|
979
|
+
createdAt: this.createdAt,
|
|
980
|
+
...dueAlarmAt !== void 0 ? { dueAlarmAt } : {},
|
|
981
|
+
...this.alarms.size > 0 ? { alarms: this.alarms.size } : {}
|
|
664
982
|
};
|
|
665
983
|
}
|
|
666
984
|
};
|
|
@@ -716,6 +1034,14 @@ var Session = class {
|
|
|
716
1034
|
room;
|
|
717
1035
|
role = "";
|
|
718
1036
|
name = "";
|
|
1037
|
+
/** Week 13 (D27): the verified JWT subject, when this session authenticated with one. Flows
|
|
1038
|
+
* to `JoinOptions.playerId` and therefore `ctx.playerId`; undefined for key joins, whose
|
|
1039
|
+
* playerId stays the client id exactly as before. */
|
|
1040
|
+
playerId;
|
|
1041
|
+
/** The schema hash the client presented at HELLO (week 13, D28): a migrate must close a
|
|
1042
|
+
* session whose schema is older than the room's new version by name (`E_SCHEMA_MISMATCH`)
|
|
1043
|
+
* rather than send it a resync WELCOME it cannot decode. */
|
|
1044
|
+
schemaHash8;
|
|
719
1045
|
/** Set when the session was rejected/kicked: no grace window on close. */
|
|
720
1046
|
fatal = false;
|
|
721
1047
|
/** Slow consumers are closed but keep their grace window (a resume gets a fresh snapshot). */
|
|
@@ -910,9 +1236,13 @@ var WorkerHost = class {
|
|
|
910
1236
|
});
|
|
911
1237
|
}
|
|
912
1238
|
/** Hibernation/crash-safety snapshot. `undefined` when the worker died or timed out. */
|
|
913
|
-
|
|
1239
|
+
/**
|
|
1240
|
+
* `forSave` (D24) takes the snapshot without the hibernation side effects, so a `room.save()`
|
|
1241
|
+
* leaves the room running — and, crucially, does not reject the very host call that asked.
|
|
1242
|
+
*/
|
|
1243
|
+
async serialize(timeoutMs = 1e4, forSave = false) {
|
|
914
1244
|
const msg = await this.request(
|
|
915
|
-
(reqId) => ({ t: "serialize", reqId }),
|
|
1245
|
+
(reqId) => ({ t: "serialize", reqId, ...forSave ? { forSave: true } : {} }),
|
|
916
1246
|
timeoutMs
|
|
917
1247
|
);
|
|
918
1248
|
return msg?.t === "serialized" ? msg.bytes : void 0;
|
|
@@ -949,6 +1279,14 @@ var WS_PING_MS = 2e4;
|
|
|
949
1279
|
var WS_PING_MISSES = 2;
|
|
950
1280
|
var MAX_STRIKES = 3;
|
|
951
1281
|
var JOIN_TIMEOUT_MS = 15e3;
|
|
1282
|
+
var KvUnavailable = class extends Error {
|
|
1283
|
+
code = KV_ERRORS.unavailable;
|
|
1284
|
+
name = "KvUnavailable";
|
|
1285
|
+
};
|
|
1286
|
+
function codeOf(err) {
|
|
1287
|
+
const code = err?.code;
|
|
1288
|
+
return typeof code === "string" && code.startsWith("E_") ? code : void 0;
|
|
1289
|
+
}
|
|
952
1290
|
function defaultSetSystemTime(epochMs) {
|
|
953
1291
|
if (process.platform !== "linux") return Promise.resolve(false);
|
|
954
1292
|
return new Promise((resolve2, reject) => {
|
|
@@ -979,6 +1317,15 @@ var SupervisorImpl = class {
|
|
|
979
1317
|
this.config = config;
|
|
980
1318
|
this.limits = { ...DEFAULT_LIMITS, ...config.limits };
|
|
981
1319
|
this.store = config.store;
|
|
1320
|
+
for (const [roomId, saveId] of Object.entries(config.restoreRooms ?? {})) {
|
|
1321
|
+
if (typeof saveId === "string" && saveId !== "") this.pendingRestores.set(roomId, saveId);
|
|
1322
|
+
}
|
|
1323
|
+
if (this.pendingRestores.size > 0) {
|
|
1324
|
+
this.log(
|
|
1325
|
+
"info",
|
|
1326
|
+
`placement carries ${this.pendingRestores.size} pending restore(s): ` + [...this.pendingRestores].map(([r, sv]) => `${r}<-${sv}`).join(", ")
|
|
1327
|
+
);
|
|
1328
|
+
}
|
|
982
1329
|
this.resumeSecret = config.resumeSecret ?? randomBytes(32).toString("base64url");
|
|
983
1330
|
if (config.resumeSecret === void 0) {
|
|
984
1331
|
this.log("warn", "no resumeSecret configured: resume tokens die with this process");
|
|
@@ -1050,8 +1397,20 @@ var SupervisorImpl = class {
|
|
|
1050
1397
|
/** True only once the tenant-idle flush has *completed* — `/admin/idle` reports idle from this,
|
|
1051
1398
|
* so the host agent never snapshots a VM whose rooms are still mid-flush to the store. */
|
|
1052
1399
|
idleFlushed = false;
|
|
1400
|
+
/** Consecutive tenant-idle flush attempts (§10.8) that left at least one room unflushed —
|
|
1401
|
+
* reported on `/admin/idle` so the host-agent can escalate instead of reading "not idle yet"
|
|
1402
|
+
* forever. Reset the moment a flush succeeds cleanly, or a client reconnects. */
|
|
1403
|
+
flushFailures = 0;
|
|
1053
1404
|
idleTimer;
|
|
1054
1405
|
pingTimer;
|
|
1406
|
+
/**
|
|
1407
|
+
* D24: `roomId -> saveId` placement instructions that have not been consumed yet. Populated
|
|
1408
|
+
* from `SupervisorConfig.restoreRooms` at boot — placement is the only channel into a tenant,
|
|
1409
|
+
* because the agent's admin view of one is read-only by design.
|
|
1410
|
+
*/
|
|
1411
|
+
pendingRestores = /* @__PURE__ */ new Map();
|
|
1412
|
+
/** D26: serialises the alarm-sidecar writes for one room, so the last change wins. */
|
|
1413
|
+
alarmWrites = /* @__PURE__ */ new Map();
|
|
1055
1414
|
// -------------------------------------------------------------------------
|
|
1056
1415
|
// Boot / shutdown
|
|
1057
1416
|
// -------------------------------------------------------------------------
|
|
@@ -1082,6 +1441,110 @@ var SupervisorImpl = class {
|
|
|
1082
1441
|
"info",
|
|
1083
1442
|
`supervisor listening on ${this.port} (${this.bundle ? "bundle" : "relay"} tenant)`
|
|
1084
1443
|
);
|
|
1444
|
+
await this.resumeAlarmedRooms();
|
|
1445
|
+
}
|
|
1446
|
+
/**
|
|
1447
|
+
* D26, and the piece staging caught missing: **start every room that owes an alarm**, at boot.
|
|
1448
|
+
*
|
|
1449
|
+
* Rooms are created lazily, by a joining client. That is right for every other purpose and
|
|
1450
|
+
* exactly wrong here: the whole point of the tenant-stopped case is that control places a tenant
|
|
1451
|
+
* for a room *nobody is in*. Without this, the tenant came up, created no rooms, fired nothing,
|
|
1452
|
+
* and control re-placed it every backoff window forever — which is precisely what the staging
|
|
1453
|
+
* run showed, with `dueAlarmAt` still set two and a half minutes past due and the room's
|
|
1454
|
+
* `lastSeen` frozen before the stop.
|
|
1455
|
+
*
|
|
1456
|
+
* Deliberately driven from the **store**, not from the placement instruction. Control could name
|
|
1457
|
+
* the due room — it queried it — but that only fixes the case where the tenant was placed *for*
|
|
1458
|
+
* that alarm. A tenant placed for an ordinary join, with some other room's alarm overdue, would
|
|
1459
|
+
* still never fire it. Scanning the sidecars means any tenant boot, for any reason,
|
|
1460
|
+
* re-establishes everything the project owes.
|
|
1461
|
+
*
|
|
1462
|
+
* Cost is one prefix list per boot plus one room start per armed alarm. A room whose alarm is
|
|
1463
|
+
* days out is started and hibernates again ~`idleMs` later; the timer lives on the record, which
|
|
1464
|
+
* survives that.
|
|
1465
|
+
*/
|
|
1466
|
+
async resumeAlarmedRooms() {
|
|
1467
|
+
if (!this.bundle) {
|
|
1468
|
+
this.log("info", "no bundle on this tenant \u2014 no rooms to resume for pending alarms");
|
|
1469
|
+
return;
|
|
1470
|
+
}
|
|
1471
|
+
const prefix = `${this.config.projectId}/`;
|
|
1472
|
+
let keys;
|
|
1473
|
+
try {
|
|
1474
|
+
keys = await this.store.list(prefix);
|
|
1475
|
+
} catch (err) {
|
|
1476
|
+
this.log("warn", `could not list "${prefix}" for rooms with pending alarms at boot`, err);
|
|
1477
|
+
return;
|
|
1478
|
+
}
|
|
1479
|
+
const rooms = /* @__PURE__ */ new Set();
|
|
1480
|
+
for (const key of keys) {
|
|
1481
|
+
const roomId = roomIdOfAlarmsKey(prefix, key);
|
|
1482
|
+
if (roomId !== void 0 && !this.registry.get(roomId)) rooms.add(roomId);
|
|
1483
|
+
}
|
|
1484
|
+
if (rooms.size === 0) {
|
|
1485
|
+
this.log(
|
|
1486
|
+
"info",
|
|
1487
|
+
`no rooms to resume: listed ${keys.length} key(s) under "${prefix}", none an alarm sidecar` + (keys.length > 0 ? ` (e.g. ${keys.slice(0, 3).join(", ")})` : "")
|
|
1488
|
+
);
|
|
1489
|
+
return;
|
|
1490
|
+
}
|
|
1491
|
+
this.log(
|
|
1492
|
+
"info",
|
|
1493
|
+
`resuming ${rooms.size} room(s) with pending alarms: ${[...rooms].join(", ")}`
|
|
1494
|
+
);
|
|
1495
|
+
for (const roomId of [...rooms].sort()) {
|
|
1496
|
+
try {
|
|
1497
|
+
await this.ensureRoom(roomId);
|
|
1498
|
+
} catch (err) {
|
|
1499
|
+
this.log("warn", `could not resume room ${roomId} for its pending alarm`, err);
|
|
1500
|
+
}
|
|
1501
|
+
}
|
|
1502
|
+
}
|
|
1503
|
+
/**
|
|
1504
|
+
* Everything a **VM-snapshot restore** owes the alarm clock, run from `/admin/resume` once the
|
|
1505
|
+
* guest wall clock has been corrected.
|
|
1506
|
+
*
|
|
1507
|
+
* This exists because the fleet's restart path is not a process restart. `stopTenant` snapshots
|
|
1508
|
+
* the VM and the next placement loads that snapshot with `resume_vm: true`, so the supervisor
|
|
1509
|
+
* comes back *mid-flight*: `boot()` finished minutes or hours ago and does not run again. Every
|
|
1510
|
+
* recovery step wired into boot — `resumeAlarmedRooms` included — is therefore invisible on the
|
|
1511
|
+
* only restart production actually performs. Staging showed it exactly: control placed a tenant
|
|
1512
|
+
* within 3.6 s of the alarm being due, eight times, and the tenant came up with no rooms, no
|
|
1513
|
+
* logs, and nothing armed (docs/m2-week12-report.md §4.3).
|
|
1514
|
+
*
|
|
1515
|
+
* Two halves, because a restore can be missing a room in two different ways:
|
|
1516
|
+
*
|
|
1517
|
+
* - **Rooms the snapshot carried.** Their `RoomRecord`s are restored intact, `due` map and all,
|
|
1518
|
+
* but with no timer and no delivery handler: `quiesceAlarms()` detaches both before the
|
|
1519
|
+
* tenant flushes, and only a room *start* re-attaches them. So they come back holding a
|
|
1520
|
+
* perfectly good set of alarms that nothing will ever fire. `loadAlarms` re-attaches and
|
|
1521
|
+
* re-arms, and `attach()`'s `rearm()` fires anything already overdue on the next turn.
|
|
1522
|
+
* - **Rooms it did not.** A tenant snapshotted before a room existed, or one whose record was
|
|
1523
|
+
* dropped, still owes that room's sidecar — `resumeAlarmedRooms` starts those, skipping the
|
|
1524
|
+
* ones the first half just handled.
|
|
1525
|
+
*
|
|
1526
|
+
* Order matters: re-arm what we have before listing the store, so the sweep sees a registry that
|
|
1527
|
+
* is already correct and does not start a room that only needed its clock back.
|
|
1528
|
+
*
|
|
1529
|
+
* The clock correction has to land first for any of it to be right. `AlarmSet` measures due
|
|
1530
|
+
* times against `Date.now()` — wall clock, because "due at" has to survive a process that was
|
|
1531
|
+
* not running — and a restored guest's clock is behind by however long the VM was paused (89.5 s
|
|
1532
|
+
* in the staging trace). Re-arming before the correction would compute every delay against the
|
|
1533
|
+
* stale clock and put every alarm that far into the future.
|
|
1534
|
+
*/
|
|
1535
|
+
async resumeFromVmSnapshot() {
|
|
1536
|
+
const rooms = this.registry.values().filter((r) => !r.relay && r.state !== "closed");
|
|
1537
|
+
if (rooms.length > 0) {
|
|
1538
|
+
this.log("info", `resume: re-arming alarms for ${rooms.length} restored room(s)`);
|
|
1539
|
+
for (const room of rooms) {
|
|
1540
|
+
try {
|
|
1541
|
+
await this.loadAlarms(room);
|
|
1542
|
+
} catch (err) {
|
|
1543
|
+
this.roomLog(room, "warn", "could not re-arm this room's alarms after a VM resume", err);
|
|
1544
|
+
}
|
|
1545
|
+
}
|
|
1546
|
+
}
|
|
1547
|
+
await this.resumeAlarmedRooms();
|
|
1085
1548
|
}
|
|
1086
1549
|
ready() {
|
|
1087
1550
|
return this.readyPromise;
|
|
@@ -1114,6 +1577,7 @@ var SupervisorImpl = class {
|
|
|
1114
1577
|
async close() {
|
|
1115
1578
|
if (this.closing) return;
|
|
1116
1579
|
this.closing = true;
|
|
1580
|
+
this.quiesceAlarms();
|
|
1117
1581
|
if (this.idleTimer) clearInterval(this.idleTimer);
|
|
1118
1582
|
if (this.pingTimer) clearInterval(this.pingTimer);
|
|
1119
1583
|
try {
|
|
@@ -1183,7 +1647,11 @@ var SupervisorImpl = class {
|
|
|
1183
1647
|
json({
|
|
1184
1648
|
idle: this.idleFlushed && this.sessions.size === 0,
|
|
1185
1649
|
sockets: this.sessions.size,
|
|
1186
|
-
idleForMs: this.idleSince > 0 ? Date.now() - this.idleSince : 0
|
|
1650
|
+
idleForMs: this.idleSince > 0 ? Date.now() - this.idleSince : 0,
|
|
1651
|
+
// §10.8 escalation: consecutive flush passes that left a room unflushed. The host agent
|
|
1652
|
+
// treats >=3 as worth an error-level log (once, deduped) instead of reading the tenant
|
|
1653
|
+
// as merely slow to go idle.
|
|
1654
|
+
flushFailures: this.flushFailures
|
|
1187
1655
|
});
|
|
1188
1656
|
return true;
|
|
1189
1657
|
case "/admin/rooms":
|
|
@@ -1239,6 +1707,95 @@ var SupervisorImpl = class {
|
|
|
1239
1707
|
);
|
|
1240
1708
|
}
|
|
1241
1709
|
json({ ok: true, applied, beforeMs, afterMs: Date.now() });
|
|
1710
|
+
try {
|
|
1711
|
+
await this.resumeFromVmSnapshot();
|
|
1712
|
+
} catch (err) {
|
|
1713
|
+
this.log("warn", "resume recovery failed", err);
|
|
1714
|
+
}
|
|
1715
|
+
})();
|
|
1716
|
+
});
|
|
1717
|
+
return true;
|
|
1718
|
+
}
|
|
1719
|
+
// Week 13, §2: the SECOND write on this otherwise read-only surface (`/admin/resume` is the
|
|
1720
|
+
// first, and the carve-out reasoning is the same): a deployment created while this tenant
|
|
1721
|
+
// is running has no other way in — the bundle drive is a read-only image packed at boot, so
|
|
1722
|
+
// the agent POSTs the new versions' bytes here and the supervisor registers them from
|
|
1723
|
+
// tmpfs. `strategy: 'drain'` stops there (new rooms take the new version, running rooms
|
|
1724
|
+
// drain); `'migrate'` additionally moves every live room onto it (D28, §1.2).
|
|
1725
|
+
case "/admin/deploy": {
|
|
1726
|
+
if (req.method !== "POST") {
|
|
1727
|
+
res.writeHead(405, { "content-type": "application/json" });
|
|
1728
|
+
res.end('{"error":"POST only"}');
|
|
1729
|
+
return true;
|
|
1730
|
+
}
|
|
1731
|
+
let body = "";
|
|
1732
|
+
req.on("data", (d) => {
|
|
1733
|
+
body += d.toString();
|
|
1734
|
+
});
|
|
1735
|
+
req.on("end", () => {
|
|
1736
|
+
void (async () => {
|
|
1737
|
+
let parsed;
|
|
1738
|
+
try {
|
|
1739
|
+
parsed = JSON.parse(body || "{}");
|
|
1740
|
+
} catch {
|
|
1741
|
+
res.writeHead(400, { "content-type": "application/json" });
|
|
1742
|
+
res.end('{"error":"body is not JSON"}');
|
|
1743
|
+
return;
|
|
1744
|
+
}
|
|
1745
|
+
const strategy = parsed.strategy ?? "drain";
|
|
1746
|
+
if (strategy !== "drain" && strategy !== "migrate") {
|
|
1747
|
+
res.writeHead(400, { "content-type": "application/json" });
|
|
1748
|
+
res.end(JSON.stringify({ error: `unknown strategy ${String(strategy)}` }));
|
|
1749
|
+
return;
|
|
1750
|
+
}
|
|
1751
|
+
if (!Array.isArray(parsed.deployments)) {
|
|
1752
|
+
res.writeHead(400, { "content-type": "application/json" });
|
|
1753
|
+
res.end('{"error":"body must carry a deployments array"}');
|
|
1754
|
+
return;
|
|
1755
|
+
}
|
|
1756
|
+
try {
|
|
1757
|
+
const added = [];
|
|
1758
|
+
let dir;
|
|
1759
|
+
for (const d of parsed.deployments) {
|
|
1760
|
+
if (typeof d.version !== "number" || !Number.isInteger(d.version)) {
|
|
1761
|
+
throw new Error("every deployment needs an integer version");
|
|
1762
|
+
}
|
|
1763
|
+
if (this.deployments.has(d.version)) continue;
|
|
1764
|
+
dir ??= await mkdtemp(nodePath.join(tmpdir(), "irtio-deploy-"));
|
|
1765
|
+
let bundlePath;
|
|
1766
|
+
let migrationPath;
|
|
1767
|
+
if (typeof d.bundleB64 === "string") {
|
|
1768
|
+
bundlePath = nodePath.join(dir, `v${d.version}.mjs`);
|
|
1769
|
+
await writeFile(bundlePath, Buffer.from(d.bundleB64, "base64"));
|
|
1770
|
+
}
|
|
1771
|
+
if (typeof d.migrationB64 === "string") {
|
|
1772
|
+
migrationPath = nodePath.join(dir, `m${d.version}.mjs`);
|
|
1773
|
+
await writeFile(migrationPath, Buffer.from(d.migrationB64, "base64"));
|
|
1774
|
+
}
|
|
1775
|
+
await this.addBundle({
|
|
1776
|
+
version: d.version,
|
|
1777
|
+
...typeof d.schemaJson === "string" ? { schemaJson: d.schemaJson } : {},
|
|
1778
|
+
...bundlePath !== void 0 ? { bundlePath } : {},
|
|
1779
|
+
...migrationPath !== void 0 ? { migrationPath } : {}
|
|
1780
|
+
});
|
|
1781
|
+
added.push(d.version);
|
|
1782
|
+
}
|
|
1783
|
+
const rooms = strategy === "migrate" ? await this.migrateAllRooms() : void 0;
|
|
1784
|
+
json({
|
|
1785
|
+
ok: true,
|
|
1786
|
+
deploymentVersion: this.deploymentVersion,
|
|
1787
|
+
added,
|
|
1788
|
+
...rooms !== void 0 ? { rooms } : {}
|
|
1789
|
+
});
|
|
1790
|
+
} catch (err) {
|
|
1791
|
+
this.log("error", "admin deploy failed", err);
|
|
1792
|
+
res.writeHead(500, { "content-type": "application/json" });
|
|
1793
|
+
res.end(
|
|
1794
|
+
JSON.stringify({
|
|
1795
|
+
error: err instanceof Error ? err.message : String(err)
|
|
1796
|
+
})
|
|
1797
|
+
);
|
|
1798
|
+
}
|
|
1242
1799
|
})();
|
|
1243
1800
|
});
|
|
1244
1801
|
return true;
|
|
@@ -1321,6 +1878,19 @@ var SupervisorImpl = class {
|
|
|
1321
1878
|
`deployment v${ref.version} registered; new rooms use v${this.deploymentVersion}, ${draining.length} room(s) draining on older versions`
|
|
1322
1879
|
);
|
|
1323
1880
|
}
|
|
1881
|
+
/**
|
|
1882
|
+
* Week 13 (D28): moves every live room onto the newest deployment, **one at a time** — a
|
|
1883
|
+
* tenant is one vCPU (week-12 §9.1), and a fleet of workers spawning at once on it is how the
|
|
1884
|
+
* week would find that out. Registry order; each room's outcome is reported, never thrown.
|
|
1885
|
+
*/
|
|
1886
|
+
async migrateAllRooms() {
|
|
1887
|
+
const target = this.deploymentVersion;
|
|
1888
|
+
const out = [];
|
|
1889
|
+
for (const room of [...this.registry.values()]) {
|
|
1890
|
+
out.push(await migrateRoom(this, room, target));
|
|
1891
|
+
}
|
|
1892
|
+
return out;
|
|
1893
|
+
}
|
|
1324
1894
|
/**
|
|
1325
1895
|
* The newest snapshot for a room across every version prefix, plus the version it was written
|
|
1326
1896
|
* under. Week-3 keys with no `@v` suffix read as version 0.
|
|
@@ -1386,6 +1956,7 @@ var SupervisorImpl = class {
|
|
|
1386
1956
|
this.idleSince = 0;
|
|
1387
1957
|
this.idleFired = false;
|
|
1388
1958
|
this.idleFlushed = false;
|
|
1959
|
+
this.flushFailures = 0;
|
|
1389
1960
|
ws.on("close", () => this.onSocketClose(session));
|
|
1390
1961
|
ws.on("error", () => {
|
|
1391
1962
|
});
|
|
@@ -1551,14 +2122,30 @@ var SupervisorImpl = class {
|
|
|
1551
2122
|
);
|
|
1552
2123
|
return;
|
|
1553
2124
|
}
|
|
1554
|
-
|
|
2125
|
+
const credential = hello.credential;
|
|
2126
|
+
if (credential.kind === "token" || credential.key !== this.config.projectId) {
|
|
1555
2127
|
session.fail("E_AUTH", formatError("E_AUTH"));
|
|
1556
2128
|
return;
|
|
1557
2129
|
}
|
|
1558
|
-
if (!this.helloBuckets.take(
|
|
2130
|
+
if (!this.helloBuckets.take(credential.key)) {
|
|
1559
2131
|
session.fail("E_RATE_LIMITED", formatError("E_RATE_LIMITED"));
|
|
1560
2132
|
return;
|
|
1561
2133
|
}
|
|
2134
|
+
let jwt;
|
|
2135
|
+
if (credential.kind === "jwt") {
|
|
2136
|
+
const issuers = this.config.jwtIssuers;
|
|
2137
|
+
if (issuers === void 0 || issuers.length === 0) {
|
|
2138
|
+
session.fail("E_AUTH", "this project has no JWT secret \u2014 mint one: irtio keys jwt-secret");
|
|
2139
|
+
return;
|
|
2140
|
+
}
|
|
2141
|
+
const verdict = verifyJwt(issuers, credential.token, this.config.projectId);
|
|
2142
|
+
if (!verdict.ok) {
|
|
2143
|
+
const vars = verdict.code === "E_TOKEN_MALFORMED" ? { reason: verdict.reason } : verdict.code === "E_TOKEN_BAD_ISSUER" ? { issuer: verdict.reason } : verdict.code === "E_TOKEN_BAD_ALG" ? { alg: verdict.reason } : {};
|
|
2144
|
+
session.fail(verdict.code, formatError(verdict.code, vars));
|
|
2145
|
+
return;
|
|
2146
|
+
}
|
|
2147
|
+
jwt = verdict;
|
|
2148
|
+
}
|
|
1562
2149
|
if (this.bundle) {
|
|
1563
2150
|
const known = [...this.deployments.values()].some(
|
|
1564
2151
|
(d) => d.bundle !== void 0 && bytesEqual(hello.schemaHash8, d.bundle.hash8)
|
|
@@ -1572,8 +2159,10 @@ var SupervisorImpl = class {
|
|
|
1572
2159
|
return;
|
|
1573
2160
|
}
|
|
1574
2161
|
session.state = "joining";
|
|
1575
|
-
session.role = hello.role ?? "";
|
|
2162
|
+
session.role = jwt?.role ?? hello.role ?? "";
|
|
1576
2163
|
session.name = hello.name ?? "";
|
|
2164
|
+
session.playerId = jwt?.playerId;
|
|
2165
|
+
session.schemaHash8 = hello.schemaHash8;
|
|
1577
2166
|
let resume;
|
|
1578
2167
|
if (hello.resumeToken !== void 0 && hello.resumeToken !== "") {
|
|
1579
2168
|
resume = verifyResume(this.resumeSecret, hello.resumeToken);
|
|
@@ -1599,6 +2188,10 @@ var SupervisorImpl = class {
|
|
|
1599
2188
|
} else {
|
|
1600
2189
|
roomId = newRoomCode((code) => this.registry.has(code));
|
|
1601
2190
|
}
|
|
2191
|
+
if (jwt?.roomId !== void 0 && jwt.roomId !== roomId) {
|
|
2192
|
+
session.fail("E_TOKEN_WRONG_ROOM", formatError("E_TOKEN_WRONG_ROOM"));
|
|
2193
|
+
return;
|
|
2194
|
+
}
|
|
1602
2195
|
const room = await this.ensureRoom(roomId);
|
|
1603
2196
|
if (!room) {
|
|
1604
2197
|
session.fail("E_INTERNAL", formatError("E_INTERNAL"));
|
|
@@ -1652,8 +2245,11 @@ var SupervisorImpl = class {
|
|
|
1652
2245
|
const config = this.bundle ? this.bundle.config : relayConfig(this.limits);
|
|
1653
2246
|
const room = new RoomRecord(roomId, this.bundle === void 0, config);
|
|
1654
2247
|
room.version = this.deploymentVersion;
|
|
2248
|
+
room.pendingRestoreSaveId = this.pendingRestores.get(roomId);
|
|
2249
|
+
this.pendingRestores.delete(roomId);
|
|
1655
2250
|
this.registry.set(room);
|
|
1656
|
-
const
|
|
2251
|
+
const restored = await this.loadPendingRestore(room);
|
|
2252
|
+
const found = restored ?? await this.findSnapshot(roomId);
|
|
1657
2253
|
const snapshot = found?.bytes;
|
|
1658
2254
|
if (room.relay) {
|
|
1659
2255
|
room.relayRoom = snapshot ? RelayRoom.restore(snapshot) : RelayRoom.create();
|
|
@@ -1662,6 +2258,15 @@ var SupervisorImpl = class {
|
|
|
1662
2258
|
this.armRelayIdle(room);
|
|
1663
2259
|
return room;
|
|
1664
2260
|
}
|
|
2261
|
+
if (found && found.version > room.version) {
|
|
2262
|
+
this.roomLog(
|
|
2263
|
+
room,
|
|
2264
|
+
"error",
|
|
2265
|
+
`snapshot is at v${found.version}, newer than the serving deployment v${room.version} (rolled back?); refusing to decode it with an older schema`
|
|
2266
|
+
);
|
|
2267
|
+
this.registry.delete(roomId);
|
|
2268
|
+
return void 0;
|
|
2269
|
+
}
|
|
1665
2270
|
let migrate;
|
|
1666
2271
|
if (found && found.version !== room.version) {
|
|
1667
2272
|
try {
|
|
@@ -1686,6 +2291,8 @@ var SupervisorImpl = class {
|
|
|
1686
2291
|
}
|
|
1687
2292
|
room.state = "running";
|
|
1688
2293
|
this.armSnapshotTimer(room);
|
|
2294
|
+
if (restored) await this.commitRestore(room);
|
|
2295
|
+
await this.loadAlarms(room);
|
|
1689
2296
|
return room;
|
|
1690
2297
|
})();
|
|
1691
2298
|
this.creating.set(roomId, create);
|
|
@@ -1752,6 +2359,282 @@ var SupervisorImpl = class {
|
|
|
1752
2359
|
}
|
|
1753
2360
|
return true;
|
|
1754
2361
|
}
|
|
2362
|
+
// -------------------------------------------------------------------------
|
|
2363
|
+
// Week 12: saves (D24), player KV (D25), durable alarms (D26)
|
|
2364
|
+
// -------------------------------------------------------------------------
|
|
2365
|
+
/**
|
|
2366
|
+
* Serves one `room.save()` / `room.kv.*`. Every path answers exactly once, including the
|
|
2367
|
+
* failures — a room holding a promise the host silently dropped would wedge until the
|
|
2368
|
+
* runtime's own 10 s deadline, and the room deserves the real reason before then.
|
|
2369
|
+
*/
|
|
2370
|
+
async runHostCall(room, reqId, call) {
|
|
2371
|
+
let result;
|
|
2372
|
+
try {
|
|
2373
|
+
result = await this.performHostCall(room, call);
|
|
2374
|
+
} catch (err) {
|
|
2375
|
+
result = {
|
|
2376
|
+
ok: false,
|
|
2377
|
+
code: codeOf(err) ?? "E_INTERNAL",
|
|
2378
|
+
message: err instanceof Error ? err.message : String(err)
|
|
2379
|
+
};
|
|
2380
|
+
}
|
|
2381
|
+
room.worker?.post({ t: "hostResult", reqId, result });
|
|
2382
|
+
}
|
|
2383
|
+
async performHostCall(room, call) {
|
|
2384
|
+
switch (call.kind) {
|
|
2385
|
+
case "save": {
|
|
2386
|
+
const saveId = await this.saveRoom(room);
|
|
2387
|
+
return { ok: true, value: saveId };
|
|
2388
|
+
}
|
|
2389
|
+
case "kvGet": {
|
|
2390
|
+
const value = await this.requireKv().get(call.playerId, call.key);
|
|
2391
|
+
return value === void 0 ? { ok: true } : { ok: true, value };
|
|
2392
|
+
}
|
|
2393
|
+
case "kvSet":
|
|
2394
|
+
await this.requireKv().set(call.playerId, call.key, call.value);
|
|
2395
|
+
return { ok: true };
|
|
2396
|
+
case "kvDelete":
|
|
2397
|
+
await this.requireKv().delete(call.playerId, call.key);
|
|
2398
|
+
return { ok: true };
|
|
2399
|
+
}
|
|
2400
|
+
}
|
|
2401
|
+
requireKv() {
|
|
2402
|
+
const kv = this.config.kv;
|
|
2403
|
+
if (!kv) {
|
|
2404
|
+
throw new KvUnavailable(
|
|
2405
|
+
"player kv is not configured for this tenant (no control plane behind it)"
|
|
2406
|
+
);
|
|
2407
|
+
}
|
|
2408
|
+
return kv;
|
|
2409
|
+
}
|
|
2410
|
+
/**
|
|
2411
|
+
* D24: one save generation. Serializes exactly the way hibernation does — same
|
|
2412
|
+
* `worker.serialize()`, same bytes, same `putSnapshotWithRetry` durability — writes it under
|
|
2413
|
+
* the room's `…/saves/` prefix, and prunes down to the retention limit.
|
|
2414
|
+
*
|
|
2415
|
+
* Containment, matching `hibernateRoom`'s rule: a failed save throws (so the room's promise
|
|
2416
|
+
* rejects with the reason) and touches nothing. The live hibernation key is not written on this
|
|
2417
|
+
* path at all, so a room can never be left worse off than if it had never called `save()`.
|
|
2418
|
+
*/
|
|
2419
|
+
async saveRoom(room) {
|
|
2420
|
+
const worker = room.worker;
|
|
2421
|
+
if (!worker || room.state !== "running") {
|
|
2422
|
+
throw new Error(`room.save: the room is ${room.state}, not running`);
|
|
2423
|
+
}
|
|
2424
|
+
const bytes = await worker.serialize(1e4, true);
|
|
2425
|
+
if (!bytes) throw new Error("room.save: the worker produced no snapshot");
|
|
2426
|
+
const liveKey = this.storeKey(room);
|
|
2427
|
+
const saveId = mintSaveId(Date.now(), randomBytes(2).readUInt16BE(0));
|
|
2428
|
+
const key = saveKey(liveKey, saveId);
|
|
2429
|
+
const ok = await putSnapshotWithRetry(
|
|
2430
|
+
this.store,
|
|
2431
|
+
key,
|
|
2432
|
+
bytes,
|
|
2433
|
+
this.processMetrics,
|
|
2434
|
+
(level, ...args) => this.roomLog(room, level, ...args)
|
|
2435
|
+
);
|
|
2436
|
+
if (!ok) throw new Error("room.save: the store would not take the snapshot");
|
|
2437
|
+
this.roomLog(room, "info", `saved generation ${saveId} (${bytes.length} bytes)`);
|
|
2438
|
+
try {
|
|
2439
|
+
const saves = await listSaves(this.store, liveKey);
|
|
2440
|
+
const deleted = await pruneSaves(
|
|
2441
|
+
this.store,
|
|
2442
|
+
saves,
|
|
2443
|
+
this.limits.saveRetain,
|
|
2444
|
+
(level, ...a) => this.roomLog(room, level, ...a)
|
|
2445
|
+
);
|
|
2446
|
+
if (deleted > 0) {
|
|
2447
|
+
this.roomLog(room, "info", `save retention: pruned ${deleted} old generation(s)`);
|
|
2448
|
+
}
|
|
2449
|
+
} catch (err) {
|
|
2450
|
+
this.roomLog(room, "warn", "save retention failed (the save itself is fine)", err);
|
|
2451
|
+
}
|
|
2452
|
+
return saveId;
|
|
2453
|
+
}
|
|
2454
|
+
/**
|
|
2455
|
+
* D24: the bytes a pending restore names, or `undefined` when there is no restore pending.
|
|
2456
|
+
*
|
|
2457
|
+
* A named save that has gone (pruned, or a bad id) logs a warning and returns `undefined`, so
|
|
2458
|
+
* the room starts from its live snapshot. That is deliberate and it is the single most
|
|
2459
|
+
* important decision in this path: failing the start instead would lose a live room because a
|
|
2460
|
+
* *save* went missing, which is far worse than not restoring.
|
|
2461
|
+
*/
|
|
2462
|
+
async loadPendingRestore(room) {
|
|
2463
|
+
const saveId = room.pendingRestoreSaveId;
|
|
2464
|
+
if (saveId === void 0) return void 0;
|
|
2465
|
+
const liveKey = this.storeKey(room);
|
|
2466
|
+
let bytes;
|
|
2467
|
+
try {
|
|
2468
|
+
bytes = await this.store.get(saveKey(liveKey, saveId));
|
|
2469
|
+
} catch (err) {
|
|
2470
|
+
this.roomLog(room, "error", `restore: could not read save ${saveId}`, err);
|
|
2471
|
+
}
|
|
2472
|
+
if (!bytes) {
|
|
2473
|
+
this.roomLog(
|
|
2474
|
+
room,
|
|
2475
|
+
"warn",
|
|
2476
|
+
`restore: save ${saveId} is gone; starting from the room's own snapshot instead`
|
|
2477
|
+
);
|
|
2478
|
+
room.pendingRestoreSaveId = void 0;
|
|
2479
|
+
return void 0;
|
|
2480
|
+
}
|
|
2481
|
+
this.roomLog(room, "info", `restoring from save ${saveId} (${bytes.length} bytes)`);
|
|
2482
|
+
return { bytes, version: room.version };
|
|
2483
|
+
}
|
|
2484
|
+
/**
|
|
2485
|
+
* Writes the restored state over the live key. Without this a restore would survive only until
|
|
2486
|
+
* the next wake, which would read the *old* live snapshot and quietly undo it — the kind of bug
|
|
2487
|
+
* that looks like "the restore worked and then stopped working".
|
|
2488
|
+
*/
|
|
2489
|
+
async commitRestore(room) {
|
|
2490
|
+
const worker = room.worker;
|
|
2491
|
+
if (!worker) return;
|
|
2492
|
+
const bytes = await worker.serialize(1e4, true).catch(() => void 0);
|
|
2493
|
+
if (!bytes) {
|
|
2494
|
+
this.roomLog(room, "warn", "restore: could not re-write the live snapshot");
|
|
2495
|
+
return;
|
|
2496
|
+
}
|
|
2497
|
+
const ok = await putSnapshotWithRetry(
|
|
2498
|
+
this.store,
|
|
2499
|
+
this.storeKey(room),
|
|
2500
|
+
bytes,
|
|
2501
|
+
this.processMetrics,
|
|
2502
|
+
(level, ...args) => this.roomLog(room, level, ...args)
|
|
2503
|
+
);
|
|
2504
|
+
if (!ok) this.roomLog(room, "error", "restore: the live snapshot could not be re-written");
|
|
2505
|
+
else await this.dropNewerLiveKeys(room);
|
|
2506
|
+
room.pendingRestoreSaveId = void 0;
|
|
2507
|
+
}
|
|
2508
|
+
/**
|
|
2509
|
+
* Week 13 (D28 rollback): a restore that commits under version V must not leave a live key at
|
|
2510
|
+
* a NEWER version behind — `findSnapshot` picks the newest, so the next cold start would
|
|
2511
|
+
* resurrect exactly the state the restore replaced and migrate it forward again. Before
|
|
2512
|
+
* rollback this case could not occur (a pending restore always landed on the newest version);
|
|
2513
|
+
* a rollback is precisely the case where it does. Save generations are untouched — only live
|
|
2514
|
+
* keys (and their alarm sidecars) go.
|
|
2515
|
+
*/
|
|
2516
|
+
async dropNewerLiveKeys(room) {
|
|
2517
|
+
const base = `${this.config.projectId}/${room.id}`;
|
|
2518
|
+
let keys;
|
|
2519
|
+
try {
|
|
2520
|
+
keys = await this.store.list(base);
|
|
2521
|
+
} catch {
|
|
2522
|
+
return;
|
|
2523
|
+
}
|
|
2524
|
+
for (const key of keys) {
|
|
2525
|
+
let version;
|
|
2526
|
+
if (key === base) version = 0;
|
|
2527
|
+
else if (key.startsWith(`${base}@v`)) {
|
|
2528
|
+
version = Number(key.slice(base.length + 2));
|
|
2529
|
+
if (!Number.isInteger(version)) continue;
|
|
2530
|
+
} else continue;
|
|
2531
|
+
if (version > room.version) {
|
|
2532
|
+
this.roomLog(room, "info", `restore: dropping the rolled-back live key ${key}`);
|
|
2533
|
+
await this.store.delete(key).catch(() => {
|
|
2534
|
+
});
|
|
2535
|
+
await this.store.delete(alarmsKey(key)).catch(() => {
|
|
2536
|
+
});
|
|
2537
|
+
}
|
|
2538
|
+
}
|
|
2539
|
+
}
|
|
2540
|
+
/**
|
|
2541
|
+
* D26: re-arms this room's alarms from the sidecar and starts persisting changes to it.
|
|
2542
|
+
*
|
|
2543
|
+
* Anything already due fires immediately — an alarm is "at or after", never before, and a
|
|
2544
|
+
* tenant that was stopped for an hour owes the room an hour-late alarm rather than nothing.
|
|
2545
|
+
*/
|
|
2546
|
+
async loadAlarms(room) {
|
|
2547
|
+
if (room.relay) return;
|
|
2548
|
+
let persisted = [];
|
|
2549
|
+
try {
|
|
2550
|
+
const bytes = await this.store.get(alarmsKey(this.storeKey(room)));
|
|
2551
|
+
if (bytes) persisted = decodeAlarms(bytes);
|
|
2552
|
+
} catch (err) {
|
|
2553
|
+
this.roomLog(room, "warn", "could not read this room's alarms; starting with none", err);
|
|
2554
|
+
}
|
|
2555
|
+
room.alarms.onChange = () => this.persistAlarms(room);
|
|
2556
|
+
room.alarms.attach((names) => this.deliverAlarms(room, names));
|
|
2557
|
+
if (persisted.length > 0) {
|
|
2558
|
+
room.alarms.load(persisted);
|
|
2559
|
+
const overdue = room.alarms.fireDue();
|
|
2560
|
+
if (overdue.length > 0) {
|
|
2561
|
+
this.roomLog(
|
|
2562
|
+
room,
|
|
2563
|
+
"info",
|
|
2564
|
+
`firing ${overdue.length} overdue alarm(s): ${overdue.join(", ")}`
|
|
2565
|
+
);
|
|
2566
|
+
}
|
|
2567
|
+
}
|
|
2568
|
+
}
|
|
2569
|
+
/**
|
|
2570
|
+
* Delivers due alarms into the room, **waking it first if it is asleep**. This is the whole
|
|
2571
|
+
* hibernated case: the timer kept counting because it lives on the record rather than in the
|
|
2572
|
+
* worker, and the wake is the same one a joining client would have triggered.
|
|
2573
|
+
*
|
|
2574
|
+
* The loop is not defensive padding — it is the fix for a real race that cost an hour to find.
|
|
2575
|
+
* `fireDue` removes an alarm before delivering it, so between the state check and the `await`
|
|
2576
|
+
* the room can slip into hibernation (`hibernateRoom` flips the state synchronously). A single
|
|
2577
|
+
* check-then-await therefore saw `running`, woke up to `hibernated`, and dropped an alarm that
|
|
2578
|
+
* no longer existed anywhere — a round timer that simply never fired, with a warning nobody was
|
|
2579
|
+
* watching. Re-checking after every transition covers the slip, and anything still undeliverable
|
|
2580
|
+
* is **re-armed** rather than discarded: a durable alarm that this process could not deliver is
|
|
2581
|
+
* the next placement's problem, not a lost one.
|
|
2582
|
+
*/
|
|
2583
|
+
deliverAlarms(room, names) {
|
|
2584
|
+
void (async () => {
|
|
2585
|
+
for (let attempt = 0; attempt < 4; attempt++) {
|
|
2586
|
+
if (room.state === "closed") return;
|
|
2587
|
+
if (room.state === "running" && room.worker) {
|
|
2588
|
+
for (const name of names) room.worker.post({ t: "alarm", name });
|
|
2589
|
+
return;
|
|
2590
|
+
}
|
|
2591
|
+
if (room.state === "hibernated") {
|
|
2592
|
+
await wakeRoom(this, room);
|
|
2593
|
+
continue;
|
|
2594
|
+
}
|
|
2595
|
+
await room.transition.catch(() => {
|
|
2596
|
+
});
|
|
2597
|
+
}
|
|
2598
|
+
this.roomLog(
|
|
2599
|
+
room,
|
|
2600
|
+
"warn",
|
|
2601
|
+
`alarm(s) ${names.join(", ")} came due while the room was ${room.state}; re-armed for the next start rather than dropped`
|
|
2602
|
+
);
|
|
2603
|
+
const now = Date.now();
|
|
2604
|
+
for (const name of names) room.alarms.set(name, now);
|
|
2605
|
+
})();
|
|
2606
|
+
}
|
|
2607
|
+
/**
|
|
2608
|
+
* Persists the armed set. Serialised per room so the last change wins: two alarms armed in the
|
|
2609
|
+
* same handler must not race into the sidecar in the wrong order and leave it describing a
|
|
2610
|
+
* state the room was never in.
|
|
2611
|
+
*/
|
|
2612
|
+
persistAlarms(room) {
|
|
2613
|
+
const key = alarmsKey(this.storeKey(room));
|
|
2614
|
+
const prior = this.alarmWrites.get(room.id) ?? Promise.resolve();
|
|
2615
|
+
const next = prior.catch(() => {
|
|
2616
|
+
}).then(async () => {
|
|
2617
|
+
const entries = room.alarms.entries();
|
|
2618
|
+
try {
|
|
2619
|
+
if (entries.length === 0) await this.store.delete(key);
|
|
2620
|
+
else await this.store.put(key, encodeAlarms(entries));
|
|
2621
|
+
} catch (err) {
|
|
2622
|
+
this.roomLog(room, "warn", "could not persist this room's alarms", err);
|
|
2623
|
+
}
|
|
2624
|
+
});
|
|
2625
|
+
this.alarmWrites.set(room.id, next);
|
|
2626
|
+
}
|
|
2627
|
+
/**
|
|
2628
|
+
* Stops every room's alarm clock without touching what is armed. Used on shutdown and on the
|
|
2629
|
+
* tenant-idle flush: in both, the process is about to stop being the thing that owes the room
|
|
2630
|
+
* an alarm, and the persisted sidecar is about to become the only record of it.
|
|
2631
|
+
*/
|
|
2632
|
+
quiesceAlarms() {
|
|
2633
|
+
for (const room of this.registry.values()) {
|
|
2634
|
+
room.alarms.onChange = void 0;
|
|
2635
|
+
room.alarms.detach();
|
|
2636
|
+
}
|
|
2637
|
+
}
|
|
1755
2638
|
armSnapshotTimer(room) {
|
|
1756
2639
|
this.clearRoomTimer(room);
|
|
1757
2640
|
const every = this.limits.snapshotEveryMs;
|
|
@@ -1787,12 +2670,15 @@ var SupervisorImpl = class {
|
|
|
1787
2670
|
/** A hibernated room with no sessions left has nothing in memory worth keeping (plan §3.4). */
|
|
1788
2671
|
dropRoomIfIdle(room) {
|
|
1789
2672
|
if (room.clients.size > 0) return;
|
|
2673
|
+
if (room.alarms.size > 0) return;
|
|
1790
2674
|
if (room.state === "hibernated" || room.state === "closed") this.registry.delete(room.id);
|
|
1791
2675
|
}
|
|
1792
2676
|
closeRoom(room, code, message) {
|
|
1793
2677
|
if (room.state === "closed") return;
|
|
1794
2678
|
room.state = "closed";
|
|
1795
2679
|
this.clearRoomTimer(room);
|
|
2680
|
+
room.alarms.detach();
|
|
2681
|
+
room.alarms.onChange = void 0;
|
|
1796
2682
|
for (const session of [...room.clients.values()]) {
|
|
1797
2683
|
session.fatal = true;
|
|
1798
2684
|
session.sendError(code, message, true);
|
|
@@ -1807,6 +2693,8 @@ var SupervisorImpl = class {
|
|
|
1807
2693
|
this.registry.delete(room.id);
|
|
1808
2694
|
void this.store.delete(this.storeKey(room)).catch(() => {
|
|
1809
2695
|
});
|
|
2696
|
+
void this.store.delete(alarmsKey(this.storeKey(room))).catch(() => {
|
|
2697
|
+
});
|
|
1810
2698
|
this.roomLog(room, "info", `room closed: ${message}`);
|
|
1811
2699
|
}
|
|
1812
2700
|
// -------------------------------------------------------------------------
|
|
@@ -1851,6 +2739,8 @@ var SupervisorImpl = class {
|
|
|
1851
2739
|
clientId: session.clientId,
|
|
1852
2740
|
...session.role !== "" ? { role: session.role } : {},
|
|
1853
2741
|
...session.name !== "" ? { name: session.name } : {},
|
|
2742
|
+
// D27: the verified JWT subject; absent for key joins (playerId stays the client id).
|
|
2743
|
+
...session.playerId !== void 0 ? { playerId: session.playerId } : {},
|
|
1854
2744
|
...reconnecting ? { reconnecting: true } : {}
|
|
1855
2745
|
});
|
|
1856
2746
|
});
|
|
@@ -1905,6 +2795,25 @@ var SupervisorImpl = class {
|
|
|
1905
2795
|
room.metrics.framesOut++;
|
|
1906
2796
|
}
|
|
1907
2797
|
}
|
|
2798
|
+
/**
|
|
2799
|
+
* Week 13 (D28): run between a migrate's worker start and its `rejoinAll`. A connected client
|
|
2800
|
+
* whose schema hash is not the migrated version's cannot decode the resync WELCOME that is
|
|
2801
|
+
* about to be sent — close it with `E_SCHEMA_MISMATCH` (fatal, by name) so the game can fetch
|
|
2802
|
+
* the new client, instead of feeding it bytes it will mis-decode. Clients already on the new
|
|
2803
|
+
* schema (the page reloaded after the client shipped) rejoin normally.
|
|
2804
|
+
*/
|
|
2805
|
+
failOutdatedSessions(room, version) {
|
|
2806
|
+
const hash8 = this.deployments.get(version)?.bundle?.hash8;
|
|
2807
|
+
if (!hash8) return;
|
|
2808
|
+
for (const session of room.connected()) {
|
|
2809
|
+
if (session.schemaHash8 !== void 0 && !bytesEqual(session.schemaHash8, hash8)) {
|
|
2810
|
+
session.fail(
|
|
2811
|
+
"E_SCHEMA_MISMATCH",
|
|
2812
|
+
"the room migrated to a newer deployment; reconnect with an updated client"
|
|
2813
|
+
);
|
|
2814
|
+
}
|
|
2815
|
+
}
|
|
2816
|
+
}
|
|
1908
2817
|
/** Re-joins every still-connected socket after a wake or a crash restart, with a resync WELCOME. */
|
|
1909
2818
|
async rejoinAll(room) {
|
|
1910
2819
|
for (const session of room.connected()) {
|
|
@@ -2105,6 +3014,12 @@ var SupervisorImpl = class {
|
|
|
2105
3014
|
case "sleep":
|
|
2106
3015
|
void hibernateRoom(this, room);
|
|
2107
3016
|
return;
|
|
3017
|
+
case "hostCall":
|
|
3018
|
+
void this.runHostCall(room, msg.reqId, msg.call);
|
|
3019
|
+
return;
|
|
3020
|
+
case "setAlarm":
|
|
3021
|
+
room.alarms.set(msg.name, msg.atMs);
|
|
3022
|
+
return;
|
|
2108
3023
|
case "log":
|
|
2109
3024
|
room.log(msg.level, ...msg.args);
|
|
2110
3025
|
return;
|
|
@@ -2173,13 +3088,21 @@ var SupervisorImpl = class {
|
|
|
2173
3088
|
this.idleFired = true;
|
|
2174
3089
|
void (async () => {
|
|
2175
3090
|
this.log("info", "tenant idle: flushing every room");
|
|
3091
|
+
let allOk = false;
|
|
2176
3092
|
try {
|
|
2177
|
-
await flushAll(this);
|
|
3093
|
+
allOk = await flushAll(this);
|
|
2178
3094
|
} catch (err) {
|
|
2179
3095
|
this.log("error", "tenant-idle flush failed", err);
|
|
2180
3096
|
}
|
|
2181
|
-
|
|
2182
|
-
|
|
3097
|
+
if (allOk) {
|
|
3098
|
+
this.idleFlushed = true;
|
|
3099
|
+
this.flushFailures = 0;
|
|
3100
|
+
this.config.onTenantIdle?.();
|
|
3101
|
+
} else {
|
|
3102
|
+
this.idleFired = false;
|
|
3103
|
+
this.flushFailures++;
|
|
3104
|
+
this.log("warn", `tenant idle: flush left rooms unflushed (attempt ${this.flushFailures})`);
|
|
3105
|
+
}
|
|
2183
3106
|
})();
|
|
2184
3107
|
}
|
|
2185
3108
|
// -------------------------------------------------------------------------
|
|
@@ -2469,12 +3392,12 @@ var WATCHED_EXTENSIONS = [".ts", ".js", ".mts", ".mjs", ".tsx", ".jsx"];
|
|
|
2469
3392
|
var LOG_LIMIT = 100;
|
|
2470
3393
|
function resolveEntry(cwd, room) {
|
|
2471
3394
|
if (room !== void 0) {
|
|
2472
|
-
const file =
|
|
3395
|
+
const file = path.resolve(cwd, room);
|
|
2473
3396
|
if (!existsSync(file)) throw new Error(`irtio dev: no room file at ${file}`);
|
|
2474
3397
|
return file;
|
|
2475
3398
|
}
|
|
2476
3399
|
for (const candidate of ROOM_CANDIDATES) {
|
|
2477
|
-
const file =
|
|
3400
|
+
const file = path.resolve(cwd, candidate);
|
|
2478
3401
|
if (existsSync(file)) return file;
|
|
2479
3402
|
}
|
|
2480
3403
|
throw new Error(
|
|
@@ -2484,11 +3407,11 @@ function resolveEntry(cwd, room) {
|
|
|
2484
3407
|
);
|
|
2485
3408
|
}
|
|
2486
3409
|
async function readProjectId(cwd) {
|
|
2487
|
-
const file =
|
|
3410
|
+
const file = path.join(cwd, "irtio.json");
|
|
2488
3411
|
if (!existsSync(file)) return "dev";
|
|
2489
3412
|
let parsed;
|
|
2490
3413
|
try {
|
|
2491
|
-
parsed = JSON.parse(await
|
|
3414
|
+
parsed = JSON.parse(await readFile(file, "utf8"));
|
|
2492
3415
|
} catch (err) {
|
|
2493
3416
|
throw new Error(`irtio dev: ${file} is not valid JSON: ${String(err)}`);
|
|
2494
3417
|
}
|
|
@@ -2499,6 +3422,54 @@ async function readProjectId(cwd) {
|
|
|
2499
3422
|
}
|
|
2500
3423
|
return project;
|
|
2501
3424
|
}
|
|
3425
|
+
async function readClientEntry(cwd) {
|
|
3426
|
+
const file = path.join(cwd, "irtio.json");
|
|
3427
|
+
if (!existsSync(file)) return void 0;
|
|
3428
|
+
let parsed;
|
|
3429
|
+
try {
|
|
3430
|
+
parsed = JSON.parse(await readFile(file, "utf8"));
|
|
3431
|
+
} catch (err) {
|
|
3432
|
+
throw new Error(`irtio dev: ${file} is not valid JSON: ${String(err)}`);
|
|
3433
|
+
}
|
|
3434
|
+
const client = parsed?.client;
|
|
3435
|
+
if (client === void 0) return void 0;
|
|
3436
|
+
if (typeof client !== "string" || client.length === 0) {
|
|
3437
|
+
throw new Error(`irtio dev: ${file} has a "client" that is not a non-empty string`);
|
|
3438
|
+
}
|
|
3439
|
+
return client;
|
|
3440
|
+
}
|
|
3441
|
+
async function warnIfClientImportsRoom(cwd, entry, log) {
|
|
3442
|
+
const client = await readClientEntry(cwd);
|
|
3443
|
+
if (client === void 0) return;
|
|
3444
|
+
const clientFile = path.resolve(cwd, client);
|
|
3445
|
+
if (await clientImportsRoom(clientFile, entry)) {
|
|
3446
|
+
log(
|
|
3447
|
+
pc.yellow(
|
|
3448
|
+
`irtio: WARNING: the client entry ${client} imports the room file ${path.relative(cwd, entry)} \u2014 room code must never ship to clients; move shared code (e.g. the physics world builder) into its own module both sides import`
|
|
3449
|
+
)
|
|
3450
|
+
);
|
|
3451
|
+
}
|
|
3452
|
+
}
|
|
3453
|
+
function rapierAbsolutePlugin(fromPackageDir) {
|
|
3454
|
+
let resolved;
|
|
3455
|
+
try {
|
|
3456
|
+
resolved = createRequire(path.join(fromPackageDir, "package.json")).resolve(
|
|
3457
|
+
"@dimforge/rapier3d-compat"
|
|
3458
|
+
);
|
|
3459
|
+
} catch {
|
|
3460
|
+
return void 0;
|
|
3461
|
+
}
|
|
3462
|
+
const specifier = pathToFileURL3(resolved).href;
|
|
3463
|
+
return {
|
|
3464
|
+
name: "irtio-dev-rapier-absolute",
|
|
3465
|
+
setup(build2) {
|
|
3466
|
+
build2.onResolve({ filter: /^@dimforge\/rapier3d-compat$/ }, () => ({
|
|
3467
|
+
path: specifier,
|
|
3468
|
+
external: true
|
|
3469
|
+
}));
|
|
3470
|
+
}
|
|
3471
|
+
};
|
|
3472
|
+
}
|
|
2502
3473
|
async function resolveWorkerEntry2(outDir) {
|
|
2503
3474
|
const resolver = import.meta.resolve;
|
|
2504
3475
|
if (typeof resolver === "function") {
|
|
@@ -2509,14 +3480,15 @@ async function resolveWorkerEntry2(outDir) {
|
|
|
2509
3480
|
} catch {
|
|
2510
3481
|
}
|
|
2511
3482
|
}
|
|
2512
|
-
const packagesDir =
|
|
2513
|
-
const source =
|
|
3483
|
+
const packagesDir = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..");
|
|
3484
|
+
const source = path.join(packagesDir, "runtime/src/worker/index.ts");
|
|
2514
3485
|
if (!existsSync(source)) {
|
|
2515
3486
|
throw new Error(
|
|
2516
3487
|
"irtio dev: could not resolve @irtio/runtime/worker \u2014 is @irtio/runtime installed and built?"
|
|
2517
3488
|
);
|
|
2518
3489
|
}
|
|
2519
|
-
const outfile =
|
|
3490
|
+
const outfile = path.join(outDir, "worker.mjs");
|
|
3491
|
+
const rapierPlugin = rapierAbsolutePlugin(path.join(packagesDir, "runtime"));
|
|
2520
3492
|
await esbuild.build({
|
|
2521
3493
|
entryPoints: [source],
|
|
2522
3494
|
bundle: true,
|
|
@@ -2524,11 +3496,18 @@ async function resolveWorkerEntry2(outDir) {
|
|
|
2524
3496
|
platform: "node",
|
|
2525
3497
|
outfile,
|
|
2526
3498
|
alias: {
|
|
2527
|
-
"@irtio/schema":
|
|
2528
|
-
"@irtio/server":
|
|
2529
|
-
"@irtio/protocol":
|
|
3499
|
+
"@irtio/schema": path.join(packagesDir, "schema/src/index.ts"),
|
|
3500
|
+
"@irtio/server": path.join(packagesDir, "server/src/index.ts"),
|
|
3501
|
+
"@irtio/protocol": path.join(packagesDir, "protocol/src/index.ts")
|
|
2530
3502
|
},
|
|
2531
|
-
|
|
3503
|
+
...rapierPlugin !== void 0 ? { plugins: [rapierPlugin] } : {},
|
|
3504
|
+
// Rapier stays external for the same reason it does in packages/supervisor/test/support.ts:
|
|
3505
|
+
// it is a dependency of `@irtio/runtime`, resolved once at load time. Bundling a second copy
|
|
3506
|
+
// here would give the dev worker its own WASM instance, separate from the one the room
|
|
3507
|
+
// bundle resolves via node_modules. `rapierPlugin` (above) already externalizes it at an
|
|
3508
|
+
// absolute path when `@irtio/runtime`'s own copy can be found; this string entry is the
|
|
3509
|
+
// fallback for a published install where that resolution comes for free from node_modules.
|
|
3510
|
+
external: ["node:worker_threads", "node:url", "@dimforge/rapier3d-compat"]
|
|
2532
3511
|
});
|
|
2533
3512
|
return outfile;
|
|
2534
3513
|
}
|
|
@@ -2572,13 +3551,14 @@ function send(res, status, type, body) {
|
|
|
2572
3551
|
res.end(body);
|
|
2573
3552
|
}
|
|
2574
3553
|
async function startDev(options = {}) {
|
|
2575
|
-
const cwd =
|
|
3554
|
+
const cwd = path.resolve(options.cwd ?? process.cwd());
|
|
2576
3555
|
const log = options.log ?? ((line) => console.log(line));
|
|
2577
3556
|
const entry = resolveEntry(cwd, options.room);
|
|
2578
3557
|
const projectId = await readProjectId(cwd);
|
|
2579
|
-
const outDir =
|
|
3558
|
+
const outDir = path.join(cwd, ".irtio", "dev");
|
|
2580
3559
|
const port = options.port ?? DEFAULT_PORT;
|
|
2581
|
-
await
|
|
3560
|
+
await mkdir(outDir, { recursive: true });
|
|
3561
|
+
await warnIfClientImportsRoom(cwd, entry, log);
|
|
2582
3562
|
const bundleOptions = {
|
|
2583
3563
|
entry,
|
|
2584
3564
|
outDir,
|
|
@@ -2618,13 +3598,14 @@ async function startDev(options = {}) {
|
|
|
2618
3598
|
return false;
|
|
2619
3599
|
};
|
|
2620
3600
|
const supervisor = createSupervisor({
|
|
3601
|
+
...options.limits ? { limits: options.limits } : {},
|
|
2621
3602
|
projectId,
|
|
2622
3603
|
bundlePath: bundle.file,
|
|
2623
3604
|
origins: ["*"],
|
|
2624
3605
|
port,
|
|
2625
3606
|
host: "127.0.0.1",
|
|
2626
3607
|
tenantIdleMs: 0,
|
|
2627
|
-
store: new DiskStore(
|
|
3608
|
+
store: new DiskStore(path.join(cwd, ".irtio", "snapshots")),
|
|
2628
3609
|
publicUrl: `http://localhost:${port}`,
|
|
2629
3610
|
httpHandler,
|
|
2630
3611
|
log: (level, ...args) => {
|
|
@@ -2650,7 +3631,7 @@ async function startDev(options = {}) {
|
|
|
2650
3631
|
const watcher = options.watch === false ? void 0 : startWatch();
|
|
2651
3632
|
let stopped = false;
|
|
2652
3633
|
function startWatch() {
|
|
2653
|
-
const dir =
|
|
3634
|
+
const dir = path.dirname(entry);
|
|
2654
3635
|
let timer;
|
|
2655
3636
|
let rebuilding = Promise.resolve();
|
|
2656
3637
|
let handle;
|
|
@@ -2659,7 +3640,7 @@ async function startDev(options = {}) {
|
|
|
2659
3640
|
if (filename === null) return;
|
|
2660
3641
|
const name = filename.toString();
|
|
2661
3642
|
if (name.split(/[\\/]/).some((p) => p === ".irtio" || p === "node_modules")) return;
|
|
2662
|
-
if (!WATCHED_EXTENSIONS.includes(
|
|
3643
|
+
if (!WATCHED_EXTENSIONS.includes(path.extname(name))) return;
|
|
2663
3644
|
if (timer) clearTimeout(timer);
|
|
2664
3645
|
timer = setTimeout(() => {
|
|
2665
3646
|
timer = void 0;
|