@rivetkit/engine-runner 2.0.23 → 2.0.24

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/mod.js CHANGED
@@ -10,6 +10,213 @@ function logger() {
10
10
  return LOGGER;
11
11
  }
12
12
 
13
+ // src/utils.ts
14
+ function unreachable(x) {
15
+ throw `Unreachable: ${x}`;
16
+ }
17
+ function calculateBackoff(attempt, options = {}) {
18
+ const {
19
+ initialDelay = 1e3,
20
+ maxDelay = 3e4,
21
+ multiplier = 2,
22
+ jitter = true
23
+ } = options;
24
+ let delay = Math.min(initialDelay * multiplier ** attempt, maxDelay);
25
+ if (jitter) {
26
+ delay = delay * (1 + Math.random() * 0.25);
27
+ }
28
+ return Math.floor(delay);
29
+ }
30
+ function parseWebSocketCloseReason(reason) {
31
+ var _a;
32
+ const [mainPart, rayId] = reason.split("#");
33
+ const [group, error] = mainPart.split(".");
34
+ if (!group || !error) {
35
+ (_a = logger()) == null ? void 0 : _a.warn({ msg: "failed to parse close reason", reason });
36
+ return void 0;
37
+ }
38
+ return {
39
+ group,
40
+ error,
41
+ rayId
42
+ };
43
+ }
44
+ var U16_MAX = 65535;
45
+ function wrappingLtU16(a, b) {
46
+ return a !== b && wrappingSub(b, a, U16_MAX) < U16_MAX / 2;
47
+ }
48
+ function wrappingLteU16(a, b) {
49
+ return a === b || wrappingLtU16(a, b);
50
+ }
51
+ function wrappingAddU16(a, b) {
52
+ return (a + b) % (U16_MAX + 1);
53
+ }
54
+ function wrappingSubU16(a, b) {
55
+ return wrappingSub(a, b, U16_MAX);
56
+ }
57
+ function wrappingSub(a, b, max) {
58
+ const result = a - b;
59
+ if (result < 0) {
60
+ return result + max + 1;
61
+ }
62
+ return result;
63
+ }
64
+ function arraysEqual(a, b) {
65
+ const ua = new Uint8Array(a);
66
+ const ub = new Uint8Array(b);
67
+ if (ua.length !== ub.length) return false;
68
+ for (let i = 0; i < ua.length; i++) {
69
+ if (ua[i] !== ub[i]) return false;
70
+ }
71
+ return true;
72
+ }
73
+ function promiseWithResolvers() {
74
+ let resolve;
75
+ let reject;
76
+ const promise = new Promise((res, rej) => {
77
+ resolve = res;
78
+ reject = rej;
79
+ });
80
+ return { promise, resolve, reject };
81
+ }
82
+ function idToStr(id) {
83
+ const bytes = new Uint8Array(id);
84
+ return Array.from(bytes).map((byte) => byte.toString(16).padStart(2, "0")).join("");
85
+ }
86
+
87
+ // src/actor.ts
88
+ var RunnerActor = class {
89
+ constructor(actorId, generation, config, hibernatingRequests) {
90
+ this.hibernatingRequests = hibernatingRequests;
91
+ this.actorId = actorId;
92
+ this.generation = generation;
93
+ this.config = config;
94
+ this.actorStartPromise = promiseWithResolvers();
95
+ }
96
+ actorId;
97
+ generation;
98
+ config;
99
+ pendingRequests = [];
100
+ webSockets = [];
101
+ actorStartPromise;
102
+ /**
103
+ * If restoreHibernatingRequests has been called. This is used to assert
104
+ * that the caller is implemented correctly.
105
+ **/
106
+ hibernationRestored = false;
107
+ // Pending request methods
108
+ getPendingRequest(gatewayId, requestId) {
109
+ var _a;
110
+ return (_a = this.pendingRequests.find(
111
+ (entry) => arraysEqual(entry.gatewayId, gatewayId) && arraysEqual(entry.requestId, requestId)
112
+ )) == null ? void 0 : _a.request;
113
+ }
114
+ createPendingRequest(gatewayId, requestId, clientMessageIndex) {
115
+ var _a, _b;
116
+ const exists = this.getPendingRequest(gatewayId, requestId) !== void 0;
117
+ if (exists) {
118
+ (_a = logger()) == null ? void 0 : _a.warn({
119
+ msg: "attempting to set pending request twice, replacing existing",
120
+ gatewayId: idToStr(gatewayId),
121
+ requestId: idToStr(requestId)
122
+ });
123
+ this.deletePendingRequest(gatewayId, requestId);
124
+ }
125
+ this.pendingRequests.push({
126
+ gatewayId,
127
+ requestId,
128
+ request: {
129
+ resolve: () => {
130
+ },
131
+ reject: () => {
132
+ },
133
+ actorId: this.actorId,
134
+ gatewayId,
135
+ requestId,
136
+ clientMessageIndex
137
+ }
138
+ });
139
+ (_b = logger()) == null ? void 0 : _b.debug({
140
+ msg: "added pending request",
141
+ gatewayId: idToStr(gatewayId),
142
+ requestId: idToStr(requestId),
143
+ length: this.pendingRequests.length
144
+ });
145
+ }
146
+ createPendingRequestWithStreamController(gatewayId, requestId, clientMessageIndex, streamController) {
147
+ var _a, _b;
148
+ const exists = this.getPendingRequest(gatewayId, requestId) !== void 0;
149
+ if (exists) {
150
+ (_a = logger()) == null ? void 0 : _a.warn({
151
+ msg: "attempting to set pending request twice, replacing existing",
152
+ gatewayId: idToStr(gatewayId),
153
+ requestId: idToStr(requestId)
154
+ });
155
+ this.deletePendingRequest(gatewayId, requestId);
156
+ }
157
+ this.pendingRequests.push({
158
+ gatewayId,
159
+ requestId,
160
+ request: {
161
+ resolve: () => {
162
+ },
163
+ reject: () => {
164
+ },
165
+ actorId: this.actorId,
166
+ gatewayId,
167
+ requestId,
168
+ clientMessageIndex,
169
+ streamController
170
+ }
171
+ });
172
+ (_b = logger()) == null ? void 0 : _b.debug({
173
+ msg: "added pending request with stream controller",
174
+ gatewayId: idToStr(gatewayId),
175
+ requestId: idToStr(requestId),
176
+ length: this.pendingRequests.length
177
+ });
178
+ }
179
+ deletePendingRequest(gatewayId, requestId) {
180
+ var _a;
181
+ const index = this.pendingRequests.findIndex(
182
+ (entry) => arraysEqual(entry.gatewayId, gatewayId) && arraysEqual(entry.requestId, requestId)
183
+ );
184
+ if (index !== -1) {
185
+ this.pendingRequests.splice(index, 1);
186
+ (_a = logger()) == null ? void 0 : _a.debug({
187
+ msg: "removed pending request",
188
+ gatewayId: idToStr(gatewayId),
189
+ requestId: idToStr(requestId),
190
+ length: this.pendingRequests.length
191
+ });
192
+ }
193
+ }
194
+ // WebSocket methods
195
+ getWebSocket(gatewayId, requestId) {
196
+ var _a;
197
+ return (_a = this.webSockets.find(
198
+ (entry) => arraysEqual(entry.gatewayId, gatewayId) && arraysEqual(entry.requestId, requestId)
199
+ )) == null ? void 0 : _a.ws;
200
+ }
201
+ setWebSocket(gatewayId, requestId, ws) {
202
+ var _a;
203
+ const exists = this.getWebSocket(gatewayId, requestId) !== void 0;
204
+ if (exists) {
205
+ (_a = logger()) == null ? void 0 : _a.warn({ msg: "attempting to set websocket twice" });
206
+ return;
207
+ }
208
+ this.webSockets.push({ gatewayId, requestId, ws });
209
+ }
210
+ deleteWebSocket(gatewayId, requestId) {
211
+ const index = this.webSockets.findIndex(
212
+ (entry) => arraysEqual(entry.gatewayId, gatewayId) && arraysEqual(entry.requestId, requestId)
213
+ );
214
+ if (index !== -1) {
215
+ this.webSockets.splice(index, 1);
216
+ }
217
+ }
218
+ };
219
+
13
220
  // src/stringify.ts
14
221
  function stringifyArrayBuffer(buffer) {
15
222
  return `ArrayBuffer(${buffer.byteLength})`;
@@ -21,10 +228,13 @@ function stringifyMap(map) {
21
228
  const entries = Array.from(map.entries()).map(([k, v]) => `"${k}": "${v}"`).join(", ");
22
229
  return `Map(${map.size}){${entries}}`;
23
230
  }
231
+ function stringifyMessageId(messageId) {
232
+ return `MessageId{gatewayId: ${idToStr(messageId.gatewayId)}, requestId: ${idToStr(messageId.requestId)}, messageIndex: ${messageId.messageIndex}}`;
233
+ }
24
234
  function stringifyToServerTunnelMessageKind(kind) {
25
235
  switch (kind.tag) {
26
- case "TunnelAck":
27
- return "TunnelAck";
236
+ case "DeprecatedTunnelAck":
237
+ return "DeprecatedTunnelAck";
28
238
  case "ToServerResponseStart": {
29
239
  const { status, headers, body, stream } = kind.val;
30
240
  const bodyStr = body === null ? "null" : stringifyArrayBuffer(body);
@@ -37,8 +247,8 @@ function stringifyToServerTunnelMessageKind(kind) {
37
247
  case "ToServerResponseAbort":
38
248
  return "ToServerResponseAbort";
39
249
  case "ToServerWebSocketOpen": {
40
- const { canHibernate, lastMsgIndex } = kind.val;
41
- return `ToServerWebSocketOpen{canHibernate: ${canHibernate}, lastMsgIndex: ${stringifyBigInt(lastMsgIndex)}}`;
250
+ const { canHibernate } = kind.val;
251
+ return `ToServerWebSocketOpen{canHibernate: ${canHibernate}}`;
42
252
  }
43
253
  case "ToServerWebSocketMessage": {
44
254
  const { data, binary } = kind.val;
@@ -49,17 +259,17 @@ function stringifyToServerTunnelMessageKind(kind) {
49
259
  return `ToServerWebSocketMessageAck{index: ${index}}`;
50
260
  }
51
261
  case "ToServerWebSocketClose": {
52
- const { code, reason, retry } = kind.val;
262
+ const { code, reason, hibernate } = kind.val;
53
263
  const codeStr = code === null ? "null" : code.toString();
54
264
  const reasonStr = reason === null ? "null" : `"${reason}"`;
55
- return `ToServerWebSocketClose{code: ${codeStr}, reason: ${reasonStr}, retry: ${retry}}`;
265
+ return `ToServerWebSocketClose{code: ${codeStr}, reason: ${reasonStr}, hibernate: ${hibernate}}`;
56
266
  }
57
267
  }
58
268
  }
59
269
  function stringifyToClientTunnelMessageKind(kind) {
60
270
  switch (kind.tag) {
61
- case "TunnelAck":
62
- return "TunnelAck";
271
+ case "DeprecatedTunnelAck":
272
+ return "DeprecatedTunnelAck";
63
273
  case "ToClientRequestStart": {
64
274
  const { actorId, method, path, headers, body, stream } = kind.val;
65
275
  const bodyStr = body === null ? "null" : stringifyArrayBuffer(body);
@@ -76,8 +286,8 @@ function stringifyToClientTunnelMessageKind(kind) {
76
286
  return `ToClientWebSocketOpen{actorId: "${actorId}", path: "${path}", headers: ${stringifyMap(headers)}}`;
77
287
  }
78
288
  case "ToClientWebSocketMessage": {
79
- const { index, data, binary } = kind.val;
80
- return `ToClientWebSocketMessage{index: ${index}, data: ${stringifyArrayBuffer(data)}, binary: ${binary}}`;
289
+ const { data, binary } = kind.val;
290
+ return `ToClientWebSocketMessage{data: ${stringifyArrayBuffer(data)}, binary: ${binary}}`;
81
291
  }
82
292
  case "ToClientWebSocketClose": {
83
293
  const { code, reason } = kind.val;
@@ -90,10 +300,11 @@ function stringifyToClientTunnelMessageKind(kind) {
90
300
  function stringifyCommand(command) {
91
301
  switch (command.tag) {
92
302
  case "CommandStartActor": {
93
- const { actorId, generation, config } = command.val;
303
+ const { actorId, generation, config, hibernatingRequests } = command.val;
94
304
  const keyStr = config.key === null ? "null" : `"${config.key}"`;
95
305
  const inputStr = config.input === null ? "null" : stringifyArrayBuffer(config.input);
96
- return `CommandStartActor{actorId: "${actorId}", generation: ${generation}, config: {name: "${config.name}", key: ${keyStr}, createTs: ${stringifyBigInt(config.createTs)}, input: ${inputStr}}}`;
306
+ const hibernatingRequestsStr = hibernatingRequests.length > 0 ? `[${hibernatingRequests.map((hr) => `{gatewayId: ${idToStr(hr.gatewayId)}, requestId: ${idToStr(hr.requestId)}}`).join(", ")}]` : "[]";
307
+ return `CommandStartActor{actorId: "${actorId}", generation: ${generation}, config: {name: "${config.name}", key: ${keyStr}, createTs: ${stringifyBigInt(config.createTs)}, input: ${inputStr}}, hibernatingRequests: ${hibernatingRequestsStr}}`;
97
308
  }
98
309
  case "CommandStopActor": {
99
310
  const { actorId, generation } = command.val;
@@ -132,45 +343,163 @@ function stringifyEvent(event) {
132
343
  }
133
344
  }
134
345
  }
135
-
136
- // src/tunnel.ts
137
- import { stringify as uuidstringify, v4 as uuidv4 } from "uuid";
138
-
139
- // src/utils.ts
140
- function unreachable(x) {
141
- throw `Unreachable: ${x}`;
346
+ function stringifyEventWrapper(wrapper) {
347
+ return `EventWrapper{index: ${stringifyBigInt(wrapper.index)}, inner: ${stringifyEvent(wrapper.inner)}}`;
142
348
  }
143
- function calculateBackoff(attempt, options = {}) {
144
- const {
145
- initialDelay = 1e3,
146
- maxDelay = 3e4,
147
- multiplier = 2,
148
- jitter = true
149
- } = options;
150
- let delay = Math.min(initialDelay * multiplier ** attempt, maxDelay);
151
- if (jitter) {
152
- delay = delay * (1 + Math.random() * 0.25);
349
+ function stringifyToServer(message) {
350
+ switch (message.tag) {
351
+ case "ToServerInit": {
352
+ const {
353
+ name,
354
+ version,
355
+ totalSlots,
356
+ lastCommandIdx,
357
+ prepopulateActorNames,
358
+ metadata
359
+ } = message.val;
360
+ const lastCommandIdxStr = lastCommandIdx === null ? "null" : stringifyBigInt(lastCommandIdx);
361
+ const prepopulateActorNamesStr = prepopulateActorNames === null ? "null" : `Map(${prepopulateActorNames.size})`;
362
+ const metadataStr = metadata === null ? "null" : `"${metadata}"`;
363
+ return `ToServerInit{name: "${name}", version: ${version}, totalSlots: ${totalSlots}, lastCommandIdx: ${lastCommandIdxStr}, prepopulateActorNames: ${prepopulateActorNamesStr}, metadata: ${metadataStr}}`;
364
+ }
365
+ case "ToServerEvents": {
366
+ const events = message.val;
367
+ return `ToServerEvents{count: ${events.length}, events: [${events.map((e) => stringifyEventWrapper(e)).join(", ")}]}`;
368
+ }
369
+ case "ToServerAckCommands": {
370
+ const { lastCommandIdx } = message.val;
371
+ return `ToServerAckCommands{lastCommandIdx: ${stringifyBigInt(lastCommandIdx)}}`;
372
+ }
373
+ case "ToServerStopping":
374
+ return "ToServerStopping";
375
+ case "ToServerPing": {
376
+ const { ts } = message.val;
377
+ return `ToServerPing{ts: ${stringifyBigInt(ts)}}`;
378
+ }
379
+ case "ToServerKvRequest": {
380
+ const { actorId, requestId, data } = message.val;
381
+ const dataStr = stringifyKvRequestData(data);
382
+ return `ToServerKvRequest{actorId: "${actorId}", requestId: ${requestId}, data: ${dataStr}}`;
383
+ }
384
+ case "ToServerTunnelMessage": {
385
+ const { messageId, messageKind } = message.val;
386
+ return `ToServerTunnelMessage{messageId: ${stringifyMessageId(messageId)}, messageKind: ${stringifyToServerTunnelMessageKind(messageKind)}}`;
387
+ }
153
388
  }
154
- return Math.floor(delay);
155
389
  }
156
- function parseWebSocketCloseReason(reason) {
157
- var _a;
158
- const [mainPart, rayId] = reason.split("#");
159
- const [group, error] = mainPart.split(".");
160
- if (!group || !error) {
161
- (_a = logger()) == null ? void 0 : _a.warn({ msg: "failed to parse close reason", reason });
162
- return void 0;
390
+ function stringifyToClient(message) {
391
+ switch (message.tag) {
392
+ case "ToClientInit": {
393
+ const { runnerId, lastEventIdx, metadata } = message.val;
394
+ const metadataStr = `{runnerLostThreshold: ${stringifyBigInt(metadata.runnerLostThreshold)}}`;
395
+ return `ToClientInit{runnerId: "${runnerId}", lastEventIdx: ${stringifyBigInt(lastEventIdx)}, metadata: ${metadataStr}}`;
396
+ }
397
+ case "ToClientClose":
398
+ return "ToClientClose";
399
+ case "ToClientCommands": {
400
+ const commands = message.val;
401
+ return `ToClientCommands{count: ${commands.length}, commands: [${commands.map((c) => stringifyCommandWrapper(c)).join(", ")}]}`;
402
+ }
403
+ case "ToClientAckEvents": {
404
+ const { lastEventIdx } = message.val;
405
+ return `ToClientAckEvents{lastEventIdx: ${stringifyBigInt(lastEventIdx)}}`;
406
+ }
407
+ case "ToClientKvResponse": {
408
+ const { requestId, data } = message.val;
409
+ const dataStr = stringifyKvResponseData(data);
410
+ return `ToClientKvResponse{requestId: ${requestId}, data: ${dataStr}}`;
411
+ }
412
+ case "ToClientTunnelMessage": {
413
+ const { messageId, messageKind } = message.val;
414
+ return `ToClientTunnelMessage{messageId: ${stringifyMessageId(messageId)}, messageKind: ${stringifyToClientTunnelMessageKind(messageKind)}}`;
415
+ }
416
+ }
417
+ }
418
+ function stringifyKvRequestData(data) {
419
+ switch (data.tag) {
420
+ case "KvGetRequest": {
421
+ const { keys } = data.val;
422
+ return `KvGetRequest{keys: ${keys.length}}`;
423
+ }
424
+ case "KvListRequest": {
425
+ const { query, reverse, limit } = data.val;
426
+ const reverseStr = reverse === null ? "null" : reverse.toString();
427
+ const limitStr = limit === null ? "null" : stringifyBigInt(limit);
428
+ return `KvListRequest{query: ${stringifyKvListQuery(query)}, reverse: ${reverseStr}, limit: ${limitStr}}`;
429
+ }
430
+ case "KvPutRequest": {
431
+ const { keys, values } = data.val;
432
+ return `KvPutRequest{keys: ${keys.length}, values: ${values.length}}`;
433
+ }
434
+ case "KvDeleteRequest": {
435
+ const { keys } = data.val;
436
+ return `KvDeleteRequest{keys: ${keys.length}}`;
437
+ }
438
+ case "KvDropRequest":
439
+ return "KvDropRequest";
440
+ }
441
+ }
442
+ function stringifyKvListQuery(query) {
443
+ switch (query.tag) {
444
+ case "KvListAllQuery":
445
+ return "KvListAllQuery";
446
+ case "KvListRangeQuery": {
447
+ const { start, end, exclusive } = query.val;
448
+ return `KvListRangeQuery{start: ${stringifyArrayBuffer(start)}, end: ${stringifyArrayBuffer(end)}, exclusive: ${exclusive}}`;
449
+ }
450
+ case "KvListPrefixQuery": {
451
+ const { key } = query.val;
452
+ return `KvListPrefixQuery{key: ${stringifyArrayBuffer(key)}}`;
453
+ }
454
+ }
455
+ }
456
+ function stringifyKvResponseData(data) {
457
+ switch (data.tag) {
458
+ case "KvErrorResponse": {
459
+ const { message } = data.val;
460
+ return `KvErrorResponse{message: "${message}"}`;
461
+ }
462
+ case "KvGetResponse": {
463
+ const { keys, values, metadata } = data.val;
464
+ return `KvGetResponse{keys: ${keys.length}, values: ${values.length}, metadata: ${metadata.length}}`;
465
+ }
466
+ case "KvListResponse": {
467
+ const { keys, values, metadata } = data.val;
468
+ return `KvListResponse{keys: ${keys.length}, values: ${values.length}, metadata: ${metadata.length}}`;
469
+ }
470
+ case "KvPutResponse":
471
+ return "KvPutResponse";
472
+ case "KvDeleteResponse":
473
+ return "KvDeleteResponse";
474
+ case "KvDropResponse":
475
+ return "KvDropResponse";
163
476
  }
164
- return {
165
- group,
166
- error,
167
- rayId
168
- };
169
477
  }
170
478
 
171
479
  // src/websocket-tunnel-adapter.ts
480
+ var HIBERNATABLE_SYMBOL = Symbol("hibernatable");
172
481
  var WebSocketTunnelAdapter = class {
173
- #webSocketId;
482
+ constructor(tunnel, actorId, requestId, serverMessageIndex, hibernatable, isRestoringHibernatable, request, sendCallback, closeCallback) {
483
+ this.request = request;
484
+ var _a;
485
+ this.#tunnel = tunnel;
486
+ this.#actorId = actorId;
487
+ this.#requestId = requestId;
488
+ this.#hibernatable = hibernatable;
489
+ this.#serverMessageIndex = serverMessageIndex;
490
+ this.#sendCallback = sendCallback;
491
+ this.#closeCallback = closeCallback;
492
+ if (isRestoringHibernatable) {
493
+ (_a = this.#log) == null ? void 0 : _a.debug({
494
+ msg: "setting WebSocket to OPEN state for restored connection",
495
+ actorId: this.#actorId,
496
+ requestId: this.#requestId,
497
+ hibernatable: this.#hibernatable
498
+ });
499
+ this.#readyState = 1;
500
+ }
501
+ }
502
+ // MARK: - WebSocket Compat Variables
174
503
  #readyState = 0;
175
504
  // CONNECTING
176
505
  #eventListeners = /* @__PURE__ */ new Map();
@@ -183,195 +512,143 @@ var WebSocketTunnelAdapter = class {
183
512
  #extensions = "";
184
513
  #protocol = "";
185
514
  #url = "";
515
+ // mARK: - Internal State
516
+ #tunnel;
517
+ #actorId;
518
+ #requestId;
519
+ #hibernatable;
520
+ #serverMessageIndex;
521
+ get [HIBERNATABLE_SYMBOL]() {
522
+ return this.#hibernatable;
523
+ }
524
+ /**
525
+ * Called when sending a message from this WebSocket.
526
+ *
527
+ * Used to send a tunnel message from Tunnel.
528
+ */
186
529
  #sendCallback;
530
+ /**
531
+ * Called when closing this WebSocket.
532
+ *
533
+ * Used to send a tunnel message from Tunnel
534
+ */
187
535
  #closeCallback;
188
- #canHibernate = false;
189
- // Event buffering for events fired before listeners are attached
190
- #bufferedEvents = [];
191
- constructor(webSocketId, sendCallback, closeCallback) {
192
- this.#webSocketId = webSocketId;
193
- this.#sendCallback = sendCallback;
194
- this.#closeCallback = closeCallback;
195
- }
196
- get readyState() {
197
- return this.#readyState;
536
+ get #log() {
537
+ return this.#tunnel.log;
198
538
  }
539
+ // MARK: - Lifecycle
199
540
  get bufferedAmount() {
200
541
  return this.#bufferedAmount;
201
542
  }
202
- get binaryType() {
203
- return this.#binaryType;
204
- }
205
- set binaryType(value) {
206
- if (value === "nodebuffer" || value === "arraybuffer" || value === "blob") {
207
- this.#binaryType = value;
208
- }
209
- }
210
- get extensions() {
211
- return this.#extensions;
212
- }
213
- get protocol() {
214
- return this.#protocol;
215
- }
216
- get url() {
217
- return this.#url;
218
- }
219
- /** @experimental */
220
- get canHibernate() {
221
- return this.#canHibernate;
222
- }
223
- /** @experimental */
224
- set canHibernate(value) {
225
- this.#canHibernate = value;
226
- }
227
- get onopen() {
228
- return this.#onopen;
229
- }
230
- set onopen(value) {
231
- this.#onopen = value;
232
- if (value) {
233
- this.#flushBufferedEvents("open");
234
- }
235
- }
236
- get onclose() {
237
- return this.#onclose;
238
- }
239
- set onclose(value) {
240
- this.#onclose = value;
241
- if (value) {
242
- this.#flushBufferedEvents("close");
243
- }
244
- }
245
- get onerror() {
246
- return this.#onerror;
247
- }
248
- set onerror(value) {
249
- this.#onerror = value;
250
- if (value) {
251
- this.#flushBufferedEvents("error");
252
- }
253
- }
254
- get onmessage() {
255
- return this.#onmessage;
256
- }
257
- set onmessage(value) {
258
- this.#onmessage = value;
259
- if (value) {
260
- this.#flushBufferedEvents("message");
543
+ _handleOpen(requestId) {
544
+ if (this.#readyState !== 0) {
545
+ return;
261
546
  }
547
+ this.#readyState = 1;
548
+ const event = {
549
+ type: "open",
550
+ rivetRequestId: requestId,
551
+ target: this
552
+ };
553
+ this.#fireEvent("open", event);
262
554
  }
263
- send(data) {
555
+ _handleMessage(requestId, data, serverMessageIndex, isBinary) {
556
+ var _a, _b, _c;
264
557
  if (this.#readyState !== 1) {
265
- throw new Error("WebSocket is not open");
558
+ (_a = this.#log) == null ? void 0 : _a.warn({
559
+ msg: "WebSocket message ignored - not in OPEN state",
560
+ requestId: this.#requestId,
561
+ actorId: this.#actorId,
562
+ currentReadyState: this.#readyState,
563
+ expectedReadyState: 1,
564
+ serverMessageIndex,
565
+ hibernatable: this.#hibernatable
566
+ });
567
+ return true;
568
+ }
569
+ if (this.#hibernatable) {
570
+ const previousIndex = this.#serverMessageIndex;
571
+ if (wrappingLteU16(serverMessageIndex, previousIndex)) {
572
+ (_b = this.#log) == null ? void 0 : _b.info({
573
+ msg: "received duplicate hibernating websocket message, this indicates the actor failed to ack the message index before restarting",
574
+ requestId,
575
+ actorId: this.#actorId,
576
+ previousIndex,
577
+ expectedIndex: wrappingAddU16(previousIndex, 1),
578
+ receivedIndex: serverMessageIndex
579
+ });
580
+ return true;
581
+ }
582
+ const expectedIndex = wrappingAddU16(previousIndex, 1);
583
+ if (serverMessageIndex !== expectedIndex) {
584
+ const closeReason = "ws.message_index_skip";
585
+ (_c = this.#log) == null ? void 0 : _c.warn({
586
+ msg: "hibernatable websocket message index out of sequence, closing connection",
587
+ requestId,
588
+ actorId: this.#actorId,
589
+ previousIndex,
590
+ expectedIndex,
591
+ receivedIndex: serverMessageIndex,
592
+ closeReason,
593
+ gap: wrappingSubU16(
594
+ wrappingSubU16(serverMessageIndex, previousIndex),
595
+ 1
596
+ )
597
+ });
598
+ this.close(1008, closeReason);
599
+ return true;
600
+ }
601
+ this.#serverMessageIndex = serverMessageIndex;
266
602
  }
267
- let isBinary = false;
268
603
  let messageData;
269
- if (typeof data === "string") {
270
- messageData = data;
271
- } else if (data instanceof ArrayBuffer) {
272
- isBinary = true;
273
- messageData = data;
274
- } else if (ArrayBuffer.isView(data)) {
275
- isBinary = true;
276
- const view = data;
277
- if (view.buffer instanceof SharedArrayBuffer) {
278
- const bytes = new Uint8Array(
279
- view.buffer,
280
- view.byteOffset,
281
- view.byteLength
282
- );
283
- messageData = bytes.buffer.slice(
284
- bytes.byteOffset,
285
- bytes.byteOffset + bytes.byteLength
286
- );
604
+ if (isBinary) {
605
+ if (this.#binaryType === "nodebuffer") {
606
+ messageData = Buffer.from(data);
607
+ } else if (this.#binaryType === "arraybuffer") {
608
+ if (data instanceof Uint8Array) {
609
+ messageData = data.buffer.slice(
610
+ data.byteOffset,
611
+ data.byteOffset + data.byteLength
612
+ );
613
+ } else {
614
+ messageData = data;
615
+ }
287
616
  } else {
288
- messageData = view.buffer.slice(
289
- view.byteOffset,
290
- view.byteOffset + view.byteLength
291
- );
292
- }
293
- } else if (data instanceof Blob) {
294
- throw new Error("Blob sending not implemented in tunnel adapter");
295
- } else if (typeof Buffer !== "undefined" && Buffer.isBuffer(data)) {
296
- isBinary = true;
297
- const buf = data;
298
- if (buf.buffer instanceof SharedArrayBuffer) {
299
- const bytes = new Uint8Array(
300
- buf.buffer,
301
- buf.byteOffset,
302
- buf.byteLength
303
- );
304
- messageData = bytes.buffer.slice(
305
- bytes.byteOffset,
306
- bytes.byteOffset + bytes.byteLength
307
- );
308
- } else {
309
- messageData = buf.buffer.slice(
310
- buf.byteOffset,
311
- buf.byteOffset + buf.byteLength
617
+ throw new Error(
618
+ "Blob binaryType not supported in tunnel adapter"
312
619
  );
313
620
  }
314
621
  } else {
315
- throw new Error("Invalid data type");
316
- }
317
- this.#sendCallback(messageData, isBinary);
318
- }
319
- close(code, reason) {
320
- this.closeInner(code, reason, false, true);
321
- }
322
- __closeWithRetry(code, reason) {
323
- this.closeInner(code, reason, true, true);
324
- }
325
- __closeWithoutCallback(code, reason) {
326
- this.closeInner(code, reason, false, false);
327
- }
328
- closeInner(code, reason, retry, callback) {
329
- if (this.#readyState === 2 || // CLOSING
330
- this.#readyState === 3) {
331
- return;
332
- }
333
- this.#readyState = 2;
334
- if (callback) {
335
- this.#closeCallback(code, reason, retry);
622
+ messageData = data;
336
623
  }
337
- this.#readyState = 3;
338
- const closeEvent = {
339
- wasClean: true,
340
- code: code || 1e3,
341
- reason: reason || "",
342
- type: "close",
624
+ const event = {
625
+ type: "message",
626
+ data: messageData,
627
+ rivetRequestId: requestId,
628
+ rivetMessageIndex: serverMessageIndex,
343
629
  target: this
344
630
  };
345
- this.#fireEvent("close", closeEvent);
631
+ this.#fireEvent("message", event);
632
+ return false;
346
633
  }
347
- addEventListener(type, listener, options) {
348
- if (typeof listener === "function") {
349
- let listeners = this.#eventListeners.get(type);
350
- if (!listeners) {
351
- listeners = /* @__PURE__ */ new Set();
352
- this.#eventListeners.set(type, listeners);
353
- }
354
- listeners.add(listener);
355
- this.#flushBufferedEvents(type);
356
- }
634
+ _handleClose(_requestId, code, reason) {
635
+ this.#closeInner(code, reason, true);
357
636
  }
358
- removeEventListener(type, listener, options) {
359
- if (typeof listener === "function") {
360
- const listeners = this.#eventListeners.get(type);
361
- if (listeners) {
362
- listeners.delete(listener);
363
- }
364
- }
637
+ _handleError(error) {
638
+ const event = {
639
+ type: "error",
640
+ target: this,
641
+ error
642
+ };
643
+ this.#fireEvent("error", event);
365
644
  }
366
- dispatchEvent(event) {
367
- return true;
645
+ _closeWithoutCallback(code, reason) {
646
+ this.#closeInner(code, reason, false);
368
647
  }
369
648
  #fireEvent(type, event) {
370
649
  var _a, _b, _c, _d, _e;
371
650
  const listeners = this.#eventListeners.get(type);
372
- let hasListeners = false;
373
651
  if (listeners && listeners.size > 0) {
374
- hasListeners = true;
375
652
  for (const listener of listeners) {
376
653
  try {
377
654
  listener.call(this, event);
@@ -387,7 +664,6 @@ var WebSocketTunnelAdapter = class {
387
664
  switch (type) {
388
665
  case "open":
389
666
  if (this.#onopen) {
390
- hasListeners = true;
391
667
  try {
392
668
  this.#onopen.call(this, event);
393
669
  } catch (error) {
@@ -400,7 +676,6 @@ var WebSocketTunnelAdapter = class {
400
676
  break;
401
677
  case "close":
402
678
  if (this.#onclose) {
403
- hasListeners = true;
404
679
  try {
405
680
  this.#onclose.call(this, event);
406
681
  } catch (error) {
@@ -413,7 +688,6 @@ var WebSocketTunnelAdapter = class {
413
688
  break;
414
689
  case "error":
415
690
  if (this.#onerror) {
416
- hasListeners = true;
417
691
  try {
418
692
  this.#onerror.call(this, event);
419
693
  } catch (error) {
@@ -426,7 +700,6 @@ var WebSocketTunnelAdapter = class {
426
700
  break;
427
701
  case "message":
428
702
  if (this.#onmessage) {
429
- hasListeners = true;
430
703
  try {
431
704
  this.#onmessage.call(this, event);
432
705
  } catch (error) {
@@ -438,163 +711,161 @@ var WebSocketTunnelAdapter = class {
438
711
  }
439
712
  break;
440
713
  }
441
- if (!hasListeners) {
442
- this.#bufferedEvents.push({ type, event });
443
- }
444
- }
445
- #flushBufferedEvents(type) {
446
- var _a, _b, _c, _d, _e;
447
- const eventsToFlush = this.#bufferedEvents.filter(
448
- (buffered) => buffered.type === type
449
- );
450
- this.#bufferedEvents = this.#bufferedEvents.filter(
451
- (buffered) => buffered.type !== type
452
- );
453
- for (const { event } of eventsToFlush) {
454
- const listeners = this.#eventListeners.get(type);
455
- if (listeners) {
456
- for (const listener of listeners) {
457
- try {
458
- listener.call(this, event);
459
- } catch (error) {
460
- (_a = logger()) == null ? void 0 : _a.error({
461
- msg: "error in websocket event listener",
462
- error,
463
- type
464
- });
465
- }
466
- }
467
- }
468
- switch (type) {
469
- case "open":
470
- if (this.#onopen) {
471
- try {
472
- this.#onopen.call(this, event);
473
- } catch (error) {
474
- (_b = logger()) == null ? void 0 : _b.error({
475
- msg: "error in onopen handler",
476
- error
477
- });
478
- }
479
- }
480
- break;
481
- case "close":
482
- if (this.#onclose) {
483
- try {
484
- this.#onclose.call(this, event);
485
- } catch (error) {
486
- (_c = logger()) == null ? void 0 : _c.error({
487
- msg: "error in onclose handler",
488
- error
489
- });
490
- }
491
- }
492
- break;
493
- case "error":
494
- if (this.#onerror) {
495
- try {
496
- this.#onerror.call(this, event);
497
- } catch (error) {
498
- (_d = logger()) == null ? void 0 : _d.error({
499
- msg: "error in onerror handler",
500
- error
501
- });
502
- }
503
- }
504
- break;
505
- case "message":
506
- if (this.#onmessage) {
507
- try {
508
- this.#onmessage.call(this, event);
509
- } catch (error) {
510
- (_e = logger()) == null ? void 0 : _e.error({
511
- msg: "error in onmessage handler",
512
- error
513
- });
514
- }
515
- }
516
- break;
517
- }
518
- }
519
714
  }
520
- // Internal methods called by the Tunnel class
521
- _handleOpen(requestId) {
522
- if (this.#readyState !== 0) {
715
+ #closeInner(code, reason, callback) {
716
+ if (this.#readyState === 2 || // CLOSING
717
+ this.#readyState === 3) {
523
718
  return;
524
719
  }
525
- this.#readyState = 1;
526
- const event = {
527
- type: "open",
528
- rivetRequestId: requestId,
720
+ this.#readyState = 2;
721
+ if (callback) {
722
+ this.#closeCallback(code, reason);
723
+ }
724
+ this.#readyState = 3;
725
+ const closeEvent = {
726
+ wasClean: true,
727
+ code: code || 1e3,
728
+ reason: reason || "",
729
+ type: "close",
529
730
  target: this
530
731
  };
531
- this.#fireEvent("open", event);
732
+ this.#fireEvent("close", closeEvent);
532
733
  }
533
- /// Returns false if the message was sent off.
534
- _handleMessage(requestId, data, index, isBinary) {
535
- if (this.#readyState !== 1) {
536
- return true;
734
+ // MARK: - WebSocket Compatible API
735
+ get readyState() {
736
+ return this.#readyState;
737
+ }
738
+ get binaryType() {
739
+ return this.#binaryType;
740
+ }
741
+ set binaryType(value) {
742
+ if (value === "nodebuffer" || value === "arraybuffer" || value === "blob") {
743
+ this.#binaryType = value;
744
+ }
745
+ }
746
+ get extensions() {
747
+ return this.#extensions;
748
+ }
749
+ get protocol() {
750
+ return this.#protocol;
751
+ }
752
+ get url() {
753
+ return this.#url;
754
+ }
755
+ get onopen() {
756
+ return this.#onopen;
757
+ }
758
+ set onopen(value) {
759
+ this.#onopen = value;
760
+ }
761
+ get onclose() {
762
+ return this.#onclose;
763
+ }
764
+ set onclose(value) {
765
+ this.#onclose = value;
766
+ }
767
+ get onerror() {
768
+ return this.#onerror;
769
+ }
770
+ set onerror(value) {
771
+ this.#onerror = value;
772
+ }
773
+ get onmessage() {
774
+ return this.#onmessage;
775
+ }
776
+ set onmessage(value) {
777
+ this.#onmessage = value;
778
+ }
779
+ send(data) {
780
+ if (this.#readyState === 0) {
781
+ throw new DOMException(
782
+ "WebSocket is still in CONNECTING state",
783
+ "InvalidStateError"
784
+ );
785
+ }
786
+ if (this.#readyState === 2 || this.#readyState === 3) {
787
+ return;
537
788
  }
789
+ let isBinary = false;
538
790
  let messageData;
539
- if (isBinary) {
540
- if (this.#binaryType === "nodebuffer") {
541
- messageData = Buffer.from(data);
542
- } else if (this.#binaryType === "arraybuffer") {
543
- if (data instanceof Uint8Array) {
544
- messageData = data.buffer.slice(
545
- data.byteOffset,
546
- data.byteOffset + data.byteLength
547
- );
548
- } else {
549
- messageData = data;
550
- }
791
+ if (typeof data === "string") {
792
+ messageData = data;
793
+ } else if (data instanceof ArrayBuffer) {
794
+ isBinary = true;
795
+ messageData = data;
796
+ } else if (ArrayBuffer.isView(data)) {
797
+ isBinary = true;
798
+ const view = data;
799
+ if (view.buffer instanceof SharedArrayBuffer) {
800
+ const bytes = new Uint8Array(
801
+ view.buffer,
802
+ view.byteOffset,
803
+ view.byteLength
804
+ );
805
+ messageData = bytes.buffer.slice(
806
+ bytes.byteOffset,
807
+ bytes.byteOffset + bytes.byteLength
808
+ );
551
809
  } else {
552
- throw new Error(
553
- "Blob binaryType not supported in tunnel adapter"
810
+ messageData = view.buffer.slice(
811
+ view.byteOffset,
812
+ view.byteOffset + view.byteLength
813
+ );
814
+ }
815
+ } else if (data instanceof Blob) {
816
+ throw new Error("Blob sending not implemented in tunnel adapter");
817
+ } else if (typeof Buffer !== "undefined" && Buffer.isBuffer(data)) {
818
+ isBinary = true;
819
+ const buf = data;
820
+ if (buf.buffer instanceof SharedArrayBuffer) {
821
+ const bytes = new Uint8Array(
822
+ buf.buffer,
823
+ buf.byteOffset,
824
+ buf.byteLength
825
+ );
826
+ messageData = bytes.buffer.slice(
827
+ bytes.byteOffset,
828
+ bytes.byteOffset + bytes.byteLength
829
+ );
830
+ } else {
831
+ messageData = buf.buffer.slice(
832
+ buf.byteOffset,
833
+ buf.byteOffset + buf.byteLength
554
834
  );
555
835
  }
556
836
  } else {
557
- messageData = data;
837
+ throw new Error("Invalid data type");
558
838
  }
559
- const event = {
560
- type: "message",
561
- data: messageData,
562
- rivetRequestId: requestId,
563
- rivetMessageIndex: index,
564
- target: this
565
- };
566
- this.#fireEvent("message", event);
567
- return false;
839
+ this.#sendCallback(messageData, isBinary);
568
840
  }
569
- _handleClose(requestId, code, reason) {
570
- if (this.#readyState === 3) {
571
- return;
841
+ close(code, reason) {
842
+ this.#closeInner(code, reason, true);
843
+ }
844
+ addEventListener(type, listener, options) {
845
+ if (typeof listener === "function") {
846
+ let listeners = this.#eventListeners.get(type);
847
+ if (!listeners) {
848
+ listeners = /* @__PURE__ */ new Set();
849
+ this.#eventListeners.set(type, listeners);
850
+ }
851
+ listeners.add(listener);
572
852
  }
573
- this.#readyState = 3;
574
- const event = {
575
- type: "close",
576
- wasClean: true,
577
- code: code || 1e3,
578
- reason: reason || "",
579
- rivetRequestId: requestId,
580
- target: this
581
- };
582
- this.#fireEvent("close", event);
583
853
  }
584
- _handleError(error) {
585
- const event = {
586
- type: "error",
587
- target: this,
588
- error
589
- };
590
- this.#fireEvent("error", event);
854
+ removeEventListener(type, listener, options) {
855
+ if (typeof listener === "function") {
856
+ const listeners = this.#eventListeners.get(type);
857
+ if (listeners) {
858
+ listeners.delete(listener);
859
+ }
860
+ }
861
+ }
862
+ dispatchEvent(event) {
863
+ return true;
591
864
  }
592
- // WebSocket constants for compatibility
593
865
  static CONNECTING = 0;
594
866
  static OPEN = 1;
595
867
  static CLOSING = 2;
596
868
  static CLOSED = 3;
597
- // Instance constants
598
869
  CONNECTING = 0;
599
870
  OPEN = 1;
600
871
  CLOSING = 2;
@@ -606,6 +877,7 @@ var WebSocketTunnelAdapter = class {
606
877
  pong(data, mask, cb) {
607
878
  if (cb) cb(new Error("Pong not supported in tunnel adapter"));
608
879
  }
880
+ /** @experimental */
609
881
  terminate() {
610
882
  this.#readyState = 3;
611
883
  this.#closeCallback(1006, "Abnormal Closure");
@@ -621,8 +893,6 @@ var WebSocketTunnelAdapter = class {
621
893
  };
622
894
 
623
895
  // src/tunnel.ts
624
- var GC_INTERVAL = 6e4;
625
- var MESSAGE_ACK_TIMEOUT = 5e3;
626
896
  var RunnerShutdownError = class extends Error {
627
897
  constructor() {
628
898
  super("Runner shut down");
@@ -630,13 +900,10 @@ var RunnerShutdownError = class extends Error {
630
900
  };
631
901
  var Tunnel = class {
632
902
  #runner;
633
- /** Requests over the tunnel to the actor that are in flight. */
634
- #actorPendingRequests = /* @__PURE__ */ new Map();
635
- /** WebSockets over the tunnel to the actor that are in flight. */
636
- #actorWebSockets = /* @__PURE__ */ new Map();
637
- /** Messages sent from the actor over the tunnel that have not been acked by the gateway. */
638
- #pendingTunnelMessages = /* @__PURE__ */ new Map();
639
- #gcInterval;
903
+ /** Maps request IDs to actor IDs for lookup */
904
+ #requestToActor = [];
905
+ /** Buffer for messages when not connected */
906
+ #bufferedMessages = [];
640
907
  get log() {
641
908
  return this.#runner.log;
642
909
  }
@@ -644,145 +911,367 @@ var Tunnel = class {
644
911
  this.#runner = runner;
645
912
  }
646
913
  start() {
647
- this.#startGarbageCollector();
914
+ }
915
+ resendBufferedEvents() {
916
+ var _a;
917
+ if (this.#bufferedMessages.length === 0) {
918
+ return;
919
+ }
920
+ (_a = this.log) == null ? void 0 : _a.info({
921
+ msg: "resending buffered tunnel messages",
922
+ count: this.#bufferedMessages.length
923
+ });
924
+ const messages = this.#bufferedMessages;
925
+ this.#bufferedMessages = [];
926
+ for (const { gatewayId, requestId, messageKind } of messages) {
927
+ this.#sendMessage(gatewayId, requestId, messageKind);
928
+ }
648
929
  }
649
930
  shutdown() {
650
- if (this.#gcInterval) {
651
- clearInterval(this.#gcInterval);
652
- this.#gcInterval = void 0;
931
+ for (const [_actorId, actor] of this.#runner.actors) {
932
+ for (const entry of actor.pendingRequests) {
933
+ entry.request.reject(new RunnerShutdownError());
934
+ }
935
+ actor.pendingRequests = [];
936
+ for (const entry of actor.webSockets) {
937
+ if (!entry.ws[HIBERNATABLE_SYMBOL]) {
938
+ entry.ws._closeWithoutCallback(1e3, "ws.tunnel_shutdown");
939
+ }
940
+ }
941
+ actor.webSockets = [];
942
+ }
943
+ this.#requestToActor = [];
944
+ }
945
+ async restoreHibernatingRequests(actorId, metaEntries) {
946
+ var _a, _b, _c, _d;
947
+ const actor = this.#runner.getActor(actorId);
948
+ if (!actor) {
949
+ throw new Error(
950
+ `Actor ${actorId} not found for restoring hibernating requests`
951
+ );
653
952
  }
654
- for (const [_, request] of this.#actorPendingRequests) {
655
- request.reject(new RunnerShutdownError());
953
+ if (actor.hibernationRestored) {
954
+ throw new Error(
955
+ `Actor ${actorId} already restored hibernating requests`
956
+ );
957
+ }
958
+ (_a = this.log) == null ? void 0 : _a.debug({
959
+ msg: "restoring hibernating requests",
960
+ actorId,
961
+ requests: actor.hibernatingRequests.length
962
+ });
963
+ const backgroundOperations = [];
964
+ let connectedButNotLoadedCount = 0;
965
+ let restoredCount = 0;
966
+ for (const { gatewayId, requestId } of actor.hibernatingRequests) {
967
+ const requestIdStr = idToStr(requestId);
968
+ const meta = metaEntries.find(
969
+ (entry) => arraysEqual(entry.gatewayId, gatewayId) && arraysEqual(entry.requestId, requestId)
970
+ );
971
+ if (!meta) {
972
+ (_b = this.log) == null ? void 0 : _b.warn({
973
+ msg: "closing websocket that is not persisted",
974
+ requestId: requestIdStr
975
+ });
976
+ this.#sendMessage(gatewayId, requestId, {
977
+ tag: "ToServerWebSocketClose",
978
+ val: {
979
+ code: 1e3,
980
+ reason: "ws.meta_not_found_during_restore",
981
+ hibernate: false
982
+ }
983
+ });
984
+ connectedButNotLoadedCount++;
985
+ } else {
986
+ const request = buildRequestForWebSocket(
987
+ meta.path,
988
+ meta.headers
989
+ );
990
+ const restoreOperation = this.#createWebSocket(
991
+ actorId,
992
+ gatewayId,
993
+ requestId,
994
+ requestIdStr,
995
+ meta.serverMessageIndex,
996
+ true,
997
+ true,
998
+ request,
999
+ meta.path,
1000
+ meta.headers,
1001
+ false
1002
+ ).then(() => {
1003
+ var _a2;
1004
+ const actor2 = this.#runner.getActor(actorId);
1005
+ if (actor2) {
1006
+ actor2.createPendingRequest(
1007
+ gatewayId,
1008
+ requestId,
1009
+ meta.clientMessageIndex
1010
+ );
1011
+ }
1012
+ (_a2 = this.log) == null ? void 0 : _a2.info({
1013
+ msg: "connection successfully restored",
1014
+ actorId,
1015
+ requestId: requestIdStr
1016
+ });
1017
+ }).catch((err) => {
1018
+ var _a2;
1019
+ (_a2 = this.log) == null ? void 0 : _a2.error({
1020
+ msg: "error creating websocket during restore",
1021
+ requestId: requestIdStr,
1022
+ err
1023
+ });
1024
+ this.#sendMessage(gatewayId, requestId, {
1025
+ tag: "ToServerWebSocketClose",
1026
+ val: {
1027
+ code: 1011,
1028
+ reason: "ws.restore_error",
1029
+ hibernate: false
1030
+ }
1031
+ });
1032
+ });
1033
+ backgroundOperations.push(restoreOperation);
1034
+ restoredCount++;
1035
+ }
656
1036
  }
657
- this.#actorPendingRequests.clear();
658
- for (const [_, ws] of this.#actorWebSockets) {
659
- if (!ws.canHibernate) {
660
- ws.__closeWithoutCallback(1e3, "ws.tunnel_shutdown");
1037
+ let loadedButNotConnectedCount = 0;
1038
+ for (const meta of metaEntries) {
1039
+ const requestIdStr = idToStr(meta.requestId);
1040
+ const isConnected = actor.hibernatingRequests.some(
1041
+ (req) => arraysEqual(req.gatewayId, meta.gatewayId) && arraysEqual(req.requestId, meta.requestId)
1042
+ );
1043
+ if (!isConnected) {
1044
+ (_c = this.log) == null ? void 0 : _c.warn({
1045
+ msg: "removing stale persisted websocket",
1046
+ requestId: requestIdStr
1047
+ });
1048
+ const request = buildRequestForWebSocket(
1049
+ meta.path,
1050
+ meta.headers
1051
+ );
1052
+ const cleanupOperation = this.#createWebSocket(
1053
+ actorId,
1054
+ meta.gatewayId,
1055
+ meta.requestId,
1056
+ requestIdStr,
1057
+ meta.serverMessageIndex,
1058
+ true,
1059
+ true,
1060
+ request,
1061
+ meta.path,
1062
+ meta.headers,
1063
+ true
1064
+ ).then((adapter) => {
1065
+ adapter.close(1e3, "ws.stale_metadata");
1066
+ }).catch((err) => {
1067
+ var _a2;
1068
+ (_a2 = this.log) == null ? void 0 : _a2.error({
1069
+ msg: "error creating stale websocket during restore",
1070
+ requestId: requestIdStr,
1071
+ err
1072
+ });
1073
+ });
1074
+ backgroundOperations.push(cleanupOperation);
1075
+ loadedButNotConnectedCount++;
661
1076
  }
662
1077
  }
663
- this.#actorWebSockets.clear();
1078
+ await Promise.allSettled(backgroundOperations);
1079
+ actor.hibernationRestored = true;
1080
+ (_d = this.log) == null ? void 0 : _d.info({
1081
+ msg: "restored hibernatable websockets",
1082
+ actorId,
1083
+ restoredCount,
1084
+ connectedButNotLoadedCount,
1085
+ loadedButNotConnectedCount
1086
+ });
664
1087
  }
665
- #sendMessage(requestId, messageKind) {
1088
+ /**
1089
+ * Called from WebSocketOpen message and when restoring hibernatable WebSockets.
1090
+ *
1091
+ * engineAlreadyClosed will be true if this is only being called to trigger
1092
+ * the close callback and not to send a close message to the server. This
1093
+ * is used specifically to clean up zombie WebSocket connections.
1094
+ */
1095
+ async #createWebSocket(actorId, gatewayId, requestId, requestIdStr, serverMessageIndex, isHibernatable, isRestoringHibernatable, request, path, headers, engineAlreadyClosed) {
1096
+ var _a;
1097
+ (_a = this.log) == null ? void 0 : _a.debug({
1098
+ msg: "createWebSocket creating adapter",
1099
+ actorId,
1100
+ requestIdStr,
1101
+ isHibernatable,
1102
+ path
1103
+ });
1104
+ const adapter = new WebSocketTunnelAdapter(
1105
+ this,
1106
+ actorId,
1107
+ requestIdStr,
1108
+ serverMessageIndex,
1109
+ isHibernatable,
1110
+ isRestoringHibernatable,
1111
+ request,
1112
+ (data, isBinary) => {
1113
+ const dataBuffer = typeof data === "string" ? new TextEncoder().encode(data).buffer : data;
1114
+ this.#sendMessage(gatewayId, requestId, {
1115
+ tag: "ToServerWebSocketMessage",
1116
+ val: {
1117
+ data: dataBuffer,
1118
+ binary: isBinary
1119
+ }
1120
+ });
1121
+ },
1122
+ (code, reason) => {
1123
+ if (!engineAlreadyClosed) {
1124
+ this.#sendMessage(gatewayId, requestId, {
1125
+ tag: "ToServerWebSocketClose",
1126
+ val: {
1127
+ code: code || null,
1128
+ reason: reason || null,
1129
+ hibernate: false
1130
+ }
1131
+ });
1132
+ }
1133
+ const actor2 = this.#runner.getActor(actorId);
1134
+ if (actor2) {
1135
+ actor2.deleteWebSocket(gatewayId, requestId);
1136
+ actor2.deletePendingRequest(gatewayId, requestId);
1137
+ }
1138
+ this.#removeRequestToActor(gatewayId, requestId);
1139
+ }
1140
+ );
1141
+ const actor = this.#runner.getActor(actorId);
1142
+ if (!actor) {
1143
+ throw new Error(`Actor ${actorId} not found`);
1144
+ }
1145
+ actor.setWebSocket(gatewayId, requestId, adapter);
1146
+ this.addRequestToActor(gatewayId, requestId, actorId);
1147
+ await this.#runner.config.websocket(
1148
+ this.#runner,
1149
+ actorId,
1150
+ adapter,
1151
+ gatewayId,
1152
+ requestId,
1153
+ request,
1154
+ path,
1155
+ headers,
1156
+ isHibernatable,
1157
+ isRestoringHibernatable
1158
+ );
1159
+ return adapter;
1160
+ }
1161
+ addRequestToActor(gatewayId, requestId, actorId) {
1162
+ this.#requestToActor.push({ gatewayId, requestId, actorId });
1163
+ }
1164
+ #removeRequestToActor(gatewayId, requestId) {
1165
+ const index = this.#requestToActor.findIndex(
1166
+ (entry) => arraysEqual(entry.gatewayId, gatewayId) && arraysEqual(entry.requestId, requestId)
1167
+ );
1168
+ if (index !== -1) {
1169
+ this.#requestToActor.splice(index, 1);
1170
+ }
1171
+ }
1172
+ getRequestActor(gatewayId, requestId) {
666
1173
  var _a, _b;
667
- if (!this.#runner.__webSocketReady()) {
1174
+ const entry = this.#requestToActor.find(
1175
+ (entry2) => arraysEqual(entry2.gatewayId, gatewayId) && arraysEqual(entry2.requestId, requestId)
1176
+ );
1177
+ if (!entry) {
668
1178
  (_a = this.log) == null ? void 0 : _a.warn({
669
- msg: "cannot send tunnel message, socket not connected to engine",
1179
+ msg: "missing requestToActor entry",
1180
+ requestId: idToStr(requestId)
1181
+ });
1182
+ return void 0;
1183
+ }
1184
+ const actor = this.#runner.getActor(entry.actorId);
1185
+ if (!actor) {
1186
+ (_b = this.log) == null ? void 0 : _b.warn({
1187
+ msg: "missing actor for requestToActor lookup",
1188
+ requestId: idToStr(requestId),
1189
+ actorId: entry.actorId
1190
+ });
1191
+ return void 0;
1192
+ }
1193
+ return actor;
1194
+ }
1195
+ async getAndWaitForRequestActor(gatewayId, requestId) {
1196
+ const actor = this.getRequestActor(gatewayId, requestId);
1197
+ if (!actor) return;
1198
+ await actor.actorStartPromise.promise;
1199
+ return actor;
1200
+ }
1201
+ #sendMessage(gatewayId, requestId, messageKind) {
1202
+ var _a, _b, _c, _d;
1203
+ if (!this.#runner.__webSocketReady()) {
1204
+ (_a = this.log) == null ? void 0 : _a.debug({
1205
+ msg: "buffering tunnel message, socket not connected to engine",
670
1206
  requestId: idToStr(requestId),
671
1207
  message: stringifyToServerTunnelMessageKind(messageKind)
672
1208
  });
1209
+ this.#bufferedMessages.push({ gatewayId, requestId, messageKind });
673
1210
  return;
674
1211
  }
675
- const messageId = generateUuidBuffer();
1212
+ const gatewayIdStr = idToStr(gatewayId);
676
1213
  const requestIdStr = idToStr(requestId);
677
- const messageIdStr = idToStr(messageId);
678
- this.#pendingTunnelMessages.set(messageIdStr, {
679
- sentAt: Date.now(),
680
- requestIdStr
681
- });
682
- (_b = this.log) == null ? void 0 : _b.debug({
683
- msg: "send tunnel msg",
684
- requestId: requestIdStr,
1214
+ const actor = this.getRequestActor(gatewayId, requestId);
1215
+ if (!actor) {
1216
+ (_b = this.log) == null ? void 0 : _b.warn({
1217
+ msg: "cannot send tunnel message, actor not found",
1218
+ gatewayId: gatewayIdStr,
1219
+ requestId: requestIdStr
1220
+ });
1221
+ return;
1222
+ }
1223
+ let clientMessageIndex;
1224
+ const pending = actor.getPendingRequest(gatewayId, requestId);
1225
+ if (pending) {
1226
+ clientMessageIndex = pending.clientMessageIndex;
1227
+ pending.clientMessageIndex++;
1228
+ } else {
1229
+ (_c = this.log) == null ? void 0 : _c.warn({
1230
+ msg: "missing pending request for send message, defaulting to message index 0",
1231
+ gatewayId: gatewayIdStr,
1232
+ requestId: requestIdStr
1233
+ });
1234
+ clientMessageIndex = 0;
1235
+ }
1236
+ const messageId = {
1237
+ gatewayId,
1238
+ requestId,
1239
+ messageIndex: clientMessageIndex
1240
+ };
1241
+ const messageIdStr = `${idToStr(messageId.gatewayId)}-${idToStr(messageId.requestId)}-${messageId.messageIndex}`;
1242
+ (_d = this.log) == null ? void 0 : _d.debug({
1243
+ msg: "sending tunnel msg",
685
1244
  messageId: messageIdStr,
1245
+ gatewayId: gatewayIdStr,
1246
+ requestId: requestIdStr,
1247
+ messageIndex: clientMessageIndex,
686
1248
  message: stringifyToServerTunnelMessageKind(messageKind)
687
1249
  });
688
1250
  const message = {
689
1251
  tag: "ToServerTunnelMessage",
690
1252
  val: {
691
- requestId,
692
1253
  messageId,
693
1254
  messageKind
694
1255
  }
695
1256
  };
696
1257
  this.#runner.__sendToServer(message);
697
1258
  }
698
- #sendAck(requestId, messageId) {
699
- var _a;
700
- if (!this.#runner.__webSocketReady()) {
701
- return;
702
- }
703
- const message = {
704
- tag: "ToServerTunnelMessage",
705
- val: {
706
- requestId,
707
- messageId,
708
- messageKind: { tag: "TunnelAck", val: null }
709
- }
710
- };
711
- (_a = this.log) == null ? void 0 : _a.debug({
712
- msg: "ack tunnel msg",
713
- requestId: idToStr(requestId),
714
- messageId: idToStr(messageId)
715
- });
716
- this.#runner.__sendToServer(message);
717
- }
718
- #startGarbageCollector() {
719
- if (this.#gcInterval) {
720
- clearInterval(this.#gcInterval);
721
- }
722
- this.#gcInterval = setInterval(() => {
723
- this.#gc();
724
- }, GC_INTERVAL);
725
- }
726
- #gc() {
727
- var _a;
728
- const now = Date.now();
729
- const messagesToDelete = [];
730
- for (const [messageId, pendingMessage] of this.#pendingTunnelMessages) {
731
- if (now - pendingMessage.sentAt > MESSAGE_ACK_TIMEOUT) {
732
- messagesToDelete.push(messageId);
733
- const requestIdStr = pendingMessage.requestIdStr;
734
- const pendingRequest = this.#actorPendingRequests.get(requestIdStr);
735
- if (pendingRequest) {
736
- pendingRequest.reject(
737
- new Error("Message acknowledgment timeout")
738
- );
739
- if (pendingRequest.streamController) {
740
- pendingRequest.streamController.error(
741
- new Error("Message acknowledgment timeout")
742
- );
743
- }
744
- this.#actorPendingRequests.delete(requestIdStr);
745
- }
746
- const webSocket = this.#actorWebSockets.get(requestIdStr);
747
- if (webSocket) {
748
- webSocket.__closeWithRetry(
749
- 1e3,
750
- "Message acknowledgment timeout"
751
- );
752
- this.#actorWebSockets.delete(requestIdStr);
753
- }
754
- }
755
- }
756
- if (messagesToDelete.length > 0) {
757
- (_a = this.log) == null ? void 0 : _a.warn({
758
- msg: "purging unacked tunnel messages, this indicates that the Rivet Engine is disconnected or not responding",
759
- count: messagesToDelete.length
760
- });
761
- for (const messageId of messagesToDelete) {
762
- this.#pendingTunnelMessages.delete(messageId);
763
- }
764
- }
765
- }
766
1259
  closeActiveRequests(actor) {
767
1260
  const actorId = actor.actorId;
768
- for (const requestId of actor.requests) {
769
- const pending = this.#actorPendingRequests.get(requestId);
770
- if (pending) {
771
- pending.reject(new Error(`Actor ${actorId} stopped`));
772
- this.#actorPendingRequests.delete(requestId);
1261
+ for (const entry of actor.pendingRequests) {
1262
+ entry.request.reject(new Error(`Actor ${actorId} stopped`));
1263
+ if (entry.gatewayId && entry.requestId) {
1264
+ this.#removeRequestToActor(entry.gatewayId, entry.requestId);
773
1265
  }
774
1266
  }
775
- actor.requests.clear();
776
- for (const requestIdStr of actor.webSockets) {
777
- const ws = this.#actorWebSockets.get(requestIdStr);
778
- if (ws) {
779
- ws.__closeWithRetry(1e3, "Actor stopped");
780
- this.#actorWebSockets.delete(requestIdStr);
1267
+ for (const entry of actor.webSockets) {
1268
+ const isHibernatable = entry.ws[HIBERNATABLE_SYMBOL];
1269
+ if (!isHibernatable) {
1270
+ entry.ws._closeWithoutCallback(1e3, "actor.stopped");
781
1271
  }
782
1272
  }
783
- actor.webSockets.clear();
784
1273
  }
785
- async #fetch(actorId, requestId, request) {
1274
+ async #fetch(actorId, gatewayId, requestId, request) {
786
1275
  var _a;
787
1276
  if (!this.#runner.hasActor(actorId)) {
788
1277
  (_a = this.log) == null ? void 0 : _a.warn({
@@ -797,6 +1286,7 @@ var Tunnel = class {
797
1286
  const fetchHandler = this.#runner.config.fetch(
798
1287
  this.#runner,
799
1288
  actorId,
1289
+ gatewayId,
800
1290
  requestId,
801
1291
  request
802
1292
  );
@@ -806,81 +1296,77 @@ var Tunnel = class {
806
1296
  return fetchHandler;
807
1297
  }
808
1298
  async handleTunnelMessage(message) {
809
- var _a, _b;
810
- const requestIdStr = idToStr(message.requestId);
811
- const messageIdStr = idToStr(message.messageId);
812
- (_a = this.log) == null ? void 0 : _a.debug({
813
- msg: "receive tunnel msg",
814
- requestId: requestIdStr,
815
- messageId: messageIdStr,
816
- message: stringifyToClientTunnelMessageKind(message.messageKind)
817
- });
818
- if (message.messageKind.tag === "TunnelAck") {
819
- const pending = this.#pendingTunnelMessages.get(messageIdStr);
820
- if (pending) {
821
- const didDelete = this.#pendingTunnelMessages.delete(messageIdStr);
822
- if (!didDelete) {
823
- (_b = this.log) == null ? void 0 : _b.warn({
824
- msg: "received tunnel ack for nonexistent message",
825
- requestId: requestIdStr,
826
- messageId: messageIdStr
827
- });
828
- }
829
- }
830
- } else {
831
- switch (message.messageKind.tag) {
832
- case "ToClientRequestStart":
833
- this.#sendAck(message.requestId, message.messageId);
834
- await this.#handleRequestStart(
835
- message.requestId,
836
- message.messageKind.val
837
- );
838
- break;
839
- case "ToClientRequestChunk":
840
- this.#sendAck(message.requestId, message.messageId);
841
- await this.#handleRequestChunk(
842
- message.requestId,
843
- message.messageKind.val
844
- );
845
- break;
846
- case "ToClientRequestAbort":
847
- this.#sendAck(message.requestId, message.messageId);
848
- await this.#handleRequestAbort(message.requestId);
849
- break;
850
- case "ToClientWebSocketOpen":
851
- this.#sendAck(message.requestId, message.messageId);
852
- await this.#handleWebSocketOpen(
853
- message.requestId,
854
- message.messageKind.val
855
- );
856
- break;
857
- case "ToClientWebSocketMessage": {
858
- this.#sendAck(message.requestId, message.messageId);
859
- const _unhandled = await this.#handleWebSocketMessage(
860
- message.requestId,
861
- message.messageKind.val
862
- );
863
- break;
864
- }
865
- case "ToClientWebSocketClose":
866
- this.#sendAck(message.requestId, message.messageId);
867
- await this.#handleWebSocketClose(
868
- message.requestId,
869
- message.messageKind.val
870
- );
871
- break;
872
- default:
873
- unreachable(message.messageKind);
1299
+ var _a;
1300
+ const { gatewayId, requestId, messageIndex } = message.messageId;
1301
+ const gatewayIdStr = idToStr(gatewayId);
1302
+ const requestIdStr = idToStr(requestId);
1303
+ (_a = this.log) == null ? void 0 : _a.debug({
1304
+ msg: "receive tunnel msg",
1305
+ gatewayId: gatewayIdStr,
1306
+ requestId: requestIdStr,
1307
+ messageIndex: message.messageId.messageIndex,
1308
+ message: stringifyToClientTunnelMessageKind(message.messageKind)
1309
+ });
1310
+ switch (message.messageKind.tag) {
1311
+ case "ToClientRequestStart":
1312
+ await this.#handleRequestStart(
1313
+ gatewayId,
1314
+ requestId,
1315
+ message.messageKind.val
1316
+ );
1317
+ break;
1318
+ case "ToClientRequestChunk":
1319
+ await this.#handleRequestChunk(
1320
+ gatewayId,
1321
+ requestId,
1322
+ message.messageKind.val
1323
+ );
1324
+ break;
1325
+ case "ToClientRequestAbort":
1326
+ await this.#handleRequestAbort(gatewayId, requestId);
1327
+ break;
1328
+ case "ToClientWebSocketOpen":
1329
+ await this.#handleWebSocketOpen(
1330
+ gatewayId,
1331
+ requestId,
1332
+ message.messageKind.val
1333
+ );
1334
+ break;
1335
+ case "ToClientWebSocketMessage": {
1336
+ await this.#handleWebSocketMessage(
1337
+ gatewayId,
1338
+ requestId,
1339
+ messageIndex,
1340
+ message.messageKind.val
1341
+ );
1342
+ break;
874
1343
  }
1344
+ case "ToClientWebSocketClose":
1345
+ await this.#handleWebSocketClose(
1346
+ gatewayId,
1347
+ requestId,
1348
+ message.messageKind.val
1349
+ );
1350
+ break;
1351
+ case "DeprecatedTunnelAck":
1352
+ break;
1353
+ default:
1354
+ unreachable(message.messageKind);
875
1355
  }
876
1356
  }
877
- async #handleRequestStart(requestId, req) {
878
- var _a, _b;
1357
+ async #handleRequestStart(gatewayId, requestId, req) {
1358
+ var _a, _b, _c;
879
1359
  const requestIdStr = idToStr(requestId);
880
- const actor = this.#runner.getActor(req.actorId);
881
- if (actor) {
882
- actor.requests.add(requestIdStr);
1360
+ const actor = await this.#runner.getAndWaitForActor(req.actorId);
1361
+ if (!actor) {
1362
+ (_a = this.log) == null ? void 0 : _a.warn({
1363
+ msg: "actor does not exist in handleRequestStart, request will leak",
1364
+ actorId: req.actorId,
1365
+ requestId: requestIdStr
1366
+ });
1367
+ return;
883
1368
  }
1369
+ this.addRequestToActor(gatewayId, requestId, req.actorId);
884
1370
  try {
885
1371
  const headers = new Headers();
886
1372
  for (const [key, value] of req.headers) {
@@ -894,19 +1380,22 @@ var Tunnel = class {
894
1380
  if (req.stream) {
895
1381
  const stream = new ReadableStream({
896
1382
  start: (controller) => {
897
- const existing = this.#actorPendingRequests.get(requestIdStr);
1383
+ const existing = actor.getPendingRequest(
1384
+ gatewayId,
1385
+ requestId
1386
+ );
898
1387
  if (existing) {
899
1388
  existing.streamController = controller;
900
1389
  existing.actorId = req.actorId;
1390
+ existing.gatewayId = gatewayId;
1391
+ existing.requestId = requestId;
901
1392
  } else {
902
- this.#actorPendingRequests.set(requestIdStr, {
903
- resolve: () => {
904
- },
905
- reject: () => {
906
- },
907
- streamController: controller,
908
- actorId: req.actorId
909
- });
1393
+ actor.createPendingRequestWithStreamController(
1394
+ gatewayId,
1395
+ requestId,
1396
+ 0,
1397
+ controller
1398
+ );
910
1399
  }
911
1400
  }
912
1401
  });
@@ -916,56 +1405,96 @@ var Tunnel = class {
916
1405
  });
917
1406
  const response = await this.#fetch(
918
1407
  req.actorId,
1408
+ gatewayId,
919
1409
  requestId,
920
1410
  streamingRequest
921
1411
  );
922
- await this.#sendResponse(requestId, response);
1412
+ await this.#sendResponse(
1413
+ actor.actorId,
1414
+ actor.generation,
1415
+ gatewayId,
1416
+ requestId,
1417
+ response
1418
+ );
923
1419
  } else {
1420
+ actor.createPendingRequest(gatewayId, requestId, 0);
924
1421
  const response = await this.#fetch(
925
1422
  req.actorId,
1423
+ gatewayId,
926
1424
  requestId,
927
1425
  request
928
1426
  );
929
- await this.#sendResponse(requestId, response);
1427
+ await this.#sendResponse(
1428
+ actor.actorId,
1429
+ actor.generation,
1430
+ gatewayId,
1431
+ requestId,
1432
+ response
1433
+ );
930
1434
  }
931
1435
  } catch (error) {
932
1436
  if (error instanceof RunnerShutdownError) {
933
- (_a = this.log) == null ? void 0 : _a.debug({ msg: "catught runner shutdown error" });
1437
+ (_b = this.log) == null ? void 0 : _b.debug({ msg: "catught runner shutdown error" });
934
1438
  } else {
935
- (_b = this.log) == null ? void 0 : _b.error({ msg: "error handling request", error });
1439
+ (_c = this.log) == null ? void 0 : _c.error({ msg: "error handling request", error });
936
1440
  this.#sendResponseError(
1441
+ actor.actorId,
1442
+ actor.generation,
1443
+ gatewayId,
937
1444
  requestId,
938
1445
  500,
939
1446
  "Internal Server Error"
940
1447
  );
941
1448
  }
942
1449
  } finally {
943
- const actor2 = this.#runner.getActor(req.actorId);
944
- if (actor2) {
945
- actor2.requests.delete(requestIdStr);
1450
+ if (this.#runner.hasActor(req.actorId, actor.generation)) {
1451
+ actor.deletePendingRequest(gatewayId, requestId);
1452
+ this.#removeRequestToActor(gatewayId, requestId);
946
1453
  }
947
1454
  }
948
1455
  }
949
- async #handleRequestChunk(requestId, chunk) {
950
- const requestIdStr = idToStr(requestId);
951
- const pending = this.#actorPendingRequests.get(requestIdStr);
952
- if (pending == null ? void 0 : pending.streamController) {
953
- pending.streamController.enqueue(new Uint8Array(chunk.body));
954
- if (chunk.finish) {
955
- pending.streamController.close();
956
- this.#actorPendingRequests.delete(requestIdStr);
1456
+ async #handleRequestChunk(gatewayId, requestId, chunk) {
1457
+ const actor = await this.getAndWaitForRequestActor(
1458
+ gatewayId,
1459
+ requestId
1460
+ );
1461
+ if (actor) {
1462
+ const pending = actor.getPendingRequest(gatewayId, requestId);
1463
+ if (pending == null ? void 0 : pending.streamController) {
1464
+ pending.streamController.enqueue(new Uint8Array(chunk.body));
1465
+ if (chunk.finish) {
1466
+ pending.streamController.close();
1467
+ actor.deletePendingRequest(gatewayId, requestId);
1468
+ this.#removeRequestToActor(gatewayId, requestId);
1469
+ }
957
1470
  }
958
1471
  }
959
1472
  }
960
- async #handleRequestAbort(requestId) {
961
- const requestIdStr = idToStr(requestId);
962
- const pending = this.#actorPendingRequests.get(requestIdStr);
963
- if (pending == null ? void 0 : pending.streamController) {
964
- pending.streamController.error(new Error("Request aborted"));
1473
+ async #handleRequestAbort(gatewayId, requestId) {
1474
+ const actor = await this.getAndWaitForRequestActor(
1475
+ gatewayId,
1476
+ requestId
1477
+ );
1478
+ if (actor) {
1479
+ const pending = actor.getPendingRequest(gatewayId, requestId);
1480
+ if (pending == null ? void 0 : pending.streamController) {
1481
+ pending.streamController.error(new Error("Request aborted"));
1482
+ }
1483
+ actor.deletePendingRequest(gatewayId, requestId);
1484
+ this.#removeRequestToActor(gatewayId, requestId);
965
1485
  }
966
- this.#actorPendingRequests.delete(requestIdStr);
967
1486
  }
968
- async #sendResponse(requestId, response) {
1487
+ async #sendResponse(actorId, generation, gatewayId, requestId, response) {
1488
+ var _a;
1489
+ if (!this.#runner.hasActor(actorId, generation)) {
1490
+ (_a = this.log) == null ? void 0 : _a.warn({
1491
+ msg: "actor not loaded to send response, assuming gateway has closed request",
1492
+ actorId,
1493
+ generation,
1494
+ requestId
1495
+ });
1496
+ return;
1497
+ }
969
1498
  const body = response.body ? await response.arrayBuffer() : null;
970
1499
  const headers = /* @__PURE__ */ new Map();
971
1500
  response.headers.forEach((value, key) => {
@@ -974,7 +1503,7 @@ var Tunnel = class {
974
1503
  if (body && !headers.has("content-length")) {
975
1504
  headers.set("content-length", String(body.byteLength));
976
1505
  }
977
- this.#sendMessage(requestId, {
1506
+ this.#sendMessage(gatewayId, requestId, {
978
1507
  tag: "ToServerResponseStart",
979
1508
  val: {
980
1509
  status: response.status,
@@ -984,10 +1513,20 @@ var Tunnel = class {
984
1513
  }
985
1514
  });
986
1515
  }
987
- #sendResponseError(requestId, status, message) {
1516
+ #sendResponseError(actorId, generation, gatewayId, requestId, status, message) {
1517
+ var _a;
1518
+ if (!this.#runner.hasActor(actorId, generation)) {
1519
+ (_a = this.log) == null ? void 0 : _a.warn({
1520
+ msg: "actor not loaded to send response, assuming gateway has closed request",
1521
+ actorId,
1522
+ generation,
1523
+ requestId
1524
+ });
1525
+ return;
1526
+ }
988
1527
  const headers = /* @__PURE__ */ new Map();
989
1528
  headers.set("content-type", "text/plain");
990
- this.#sendMessage(requestId, {
1529
+ this.#sendMessage(gatewayId, requestId, {
991
1530
  tag: "ToServerResponseStart",
992
1531
  val: {
993
1532
  status,
@@ -997,181 +1536,171 @@ var Tunnel = class {
997
1536
  }
998
1537
  });
999
1538
  }
1000
- async #handleWebSocketOpen(requestId, open) {
1001
- var _a, _b, _c, _d;
1539
+ async #handleWebSocketOpen(gatewayId, requestId, open) {
1540
+ var _a, _b, _c;
1002
1541
  const requestIdStr = idToStr(requestId);
1003
- const actor = this.#runner.getActor(open.actorId);
1542
+ const actor = await this.#runner.getAndWaitForActor(open.actorId);
1004
1543
  if (!actor) {
1005
1544
  (_a = this.log) == null ? void 0 : _a.warn({
1006
1545
  msg: "ignoring websocket for unknown actor",
1007
1546
  actorId: open.actorId
1008
1547
  });
1009
- this.#sendMessage(requestId, {
1548
+ this.#sendMessage(gatewayId, requestId, {
1010
1549
  tag: "ToServerWebSocketClose",
1011
1550
  val: {
1012
1551
  code: 1011,
1013
1552
  reason: "Actor not found",
1014
- retry: false
1015
- }
1016
- });
1017
- return;
1018
- }
1019
- const websocketHandler = this.#runner.config.websocket;
1020
- if (!websocketHandler) {
1021
- (_b = this.log) == null ? void 0 : _b.error({
1022
- msg: "no websocket handler configured for tunnel"
1023
- });
1024
- this.#sendMessage(requestId, {
1025
- tag: "ToServerWebSocketClose",
1026
- val: {
1027
- code: 1011,
1028
- reason: "Not Implemented",
1029
- retry: false
1553
+ hibernate: false
1030
1554
  }
1031
1555
  });
1032
1556
  return;
1033
1557
  }
1034
- const existingAdapter = this.#actorWebSockets.get(requestIdStr);
1558
+ const existingAdapter = actor.getWebSocket(gatewayId, requestId);
1035
1559
  if (existingAdapter) {
1036
- (_c = this.log) == null ? void 0 : _c.warn({
1560
+ (_b = this.log) == null ? void 0 : _b.warn({
1037
1561
  msg: "closing existing websocket for duplicate open event for the same request id",
1038
1562
  requestId: requestIdStr
1039
1563
  });
1040
- existingAdapter.__closeWithoutCallback(1e3, "ws.duplicate_open");
1041
- }
1042
- if (actor) {
1043
- actor.webSockets.add(requestIdStr);
1564
+ existingAdapter._closeWithoutCallback(1e3, "ws.duplicate_open");
1044
1565
  }
1045
1566
  try {
1046
- const adapter = new WebSocketTunnelAdapter(
1047
- requestIdStr,
1048
- (data, isBinary) => {
1049
- const dataBuffer = typeof data === "string" ? new TextEncoder().encode(data).buffer : data;
1050
- this.#sendMessage(requestId, {
1051
- tag: "ToServerWebSocketMessage",
1052
- val: {
1053
- data: dataBuffer,
1054
- binary: isBinary
1055
- }
1056
- });
1057
- },
1058
- (code, reason, retry = false) => {
1059
- this.#sendMessage(requestId, {
1060
- tag: "ToServerWebSocketClose",
1061
- val: {
1062
- code: code || null,
1063
- reason: reason || null,
1064
- retry
1065
- }
1066
- });
1067
- this.#actorWebSockets.delete(requestIdStr);
1068
- if (actor) {
1069
- actor.webSockets.delete(requestIdStr);
1070
- }
1071
- }
1567
+ const request = buildRequestForWebSocket(
1568
+ open.path,
1569
+ Object.fromEntries(open.headers)
1072
1570
  );
1073
- this.#actorWebSockets.set(requestIdStr, adapter);
1074
- const headerInit = {};
1075
- if (open.headers) {
1076
- for (const [k, v] of open.headers) {
1077
- headerInit[k] = v;
1078
- }
1079
- }
1080
- headerInit["Upgrade"] = "websocket";
1081
- headerInit["Connection"] = "Upgrade";
1082
- const request = new Request(`http://localhost${open.path}`, {
1083
- method: "GET",
1084
- headers: headerInit
1085
- });
1086
- const hibernationConfig = this.#runner.config.getActorHibernationConfig(
1571
+ const canHibernate = this.#runner.config.hibernatableWebSocket.canHibernate(
1087
1572
  actor.actorId,
1573
+ gatewayId,
1088
1574
  requestId,
1089
1575
  request
1090
1576
  );
1091
- adapter.canHibernate = hibernationConfig.enabled;
1092
- this.#sendMessage(requestId, {
1577
+ const adapter = await this.#createWebSocket(
1578
+ actor.actorId,
1579
+ gatewayId,
1580
+ requestId,
1581
+ requestIdStr,
1582
+ 0,
1583
+ canHibernate,
1584
+ false,
1585
+ request,
1586
+ open.path,
1587
+ Object.fromEntries(open.headers),
1588
+ false
1589
+ );
1590
+ actor.createPendingRequest(gatewayId, requestId, 0);
1591
+ this.#sendMessage(gatewayId, requestId, {
1093
1592
  tag: "ToServerWebSocketOpen",
1094
1593
  val: {
1095
- canHibernate: hibernationConfig.enabled,
1096
- lastMsgIndex: BigInt(hibernationConfig.lastMsgIndex ?? -1)
1594
+ canHibernate
1097
1595
  }
1098
1596
  });
1099
1597
  adapter._handleOpen(requestId);
1100
- await websocketHandler(
1101
- this.#runner,
1102
- open.actorId,
1103
- adapter,
1104
- requestId,
1105
- request
1106
- );
1107
1598
  } catch (error) {
1108
- (_d = this.log) == null ? void 0 : _d.error({ msg: "error handling websocket open", error });
1109
- this.#sendMessage(requestId, {
1599
+ (_c = this.log) == null ? void 0 : _c.error({ msg: "error handling websocket open", error });
1600
+ this.#sendMessage(gatewayId, requestId, {
1110
1601
  tag: "ToServerWebSocketClose",
1111
1602
  val: {
1112
1603
  code: 1011,
1113
1604
  reason: "Server Error",
1114
- retry: false
1605
+ hibernate: false
1115
1606
  }
1116
1607
  });
1117
- this.#actorWebSockets.delete(requestIdStr);
1118
- if (actor) {
1119
- actor.webSockets.delete(requestIdStr);
1120
- }
1608
+ actor.deleteWebSocket(gatewayId, requestId);
1609
+ actor.deletePendingRequest(gatewayId, requestId);
1610
+ this.#removeRequestToActor(gatewayId, requestId);
1121
1611
  }
1122
1612
  }
1123
- /// Returns false if the message was sent off
1124
- async #handleWebSocketMessage(requestId, msg) {
1125
- const requestIdStr = idToStr(requestId);
1126
- const adapter = this.#actorWebSockets.get(requestIdStr);
1127
- if (adapter) {
1128
- const data = msg.binary ? new Uint8Array(msg.data) : new TextDecoder().decode(new Uint8Array(msg.data));
1129
- return adapter._handleMessage(
1130
- requestId,
1131
- data,
1132
- msg.index,
1133
- msg.binary
1134
- );
1135
- } else {
1136
- return true;
1613
+ async #handleWebSocketMessage(gatewayId, requestId, serverMessageIndex, msg) {
1614
+ var _a;
1615
+ const actor = await this.getAndWaitForRequestActor(
1616
+ gatewayId,
1617
+ requestId
1618
+ );
1619
+ if (actor) {
1620
+ const adapter = actor.getWebSocket(gatewayId, requestId);
1621
+ if (adapter) {
1622
+ const data = msg.binary ? new Uint8Array(msg.data) : new TextDecoder().decode(new Uint8Array(msg.data));
1623
+ adapter._handleMessage(
1624
+ requestId,
1625
+ data,
1626
+ serverMessageIndex,
1627
+ msg.binary
1628
+ );
1629
+ return;
1630
+ }
1137
1631
  }
1632
+ (_a = this.log) == null ? void 0 : _a.warn({
1633
+ msg: "missing websocket for incoming websocket message, this may indicate the actor stopped before processing a message",
1634
+ requestId
1635
+ });
1138
1636
  }
1139
- __ackWebsocketMessage(requestId, index) {
1140
- var _a;
1637
+ sendHibernatableWebSocketMessageAck(gatewayId, requestId, clientMessageIndex) {
1638
+ var _a, _b, _c;
1639
+ const requestIdStr = idToStr(requestId);
1141
1640
  (_a = this.log) == null ? void 0 : _a.debug({
1142
1641
  msg: "ack ws msg",
1143
- requestId: idToStr(requestId),
1144
- index
1642
+ requestId: requestIdStr,
1643
+ index: clientMessageIndex
1145
1644
  });
1146
- if (index < 0 || index > 65535)
1645
+ if (clientMessageIndex < 0 || clientMessageIndex > 65535)
1147
1646
  throw new Error("invalid websocket ack index");
1148
- this.#sendMessage(requestId, {
1647
+ const actor = this.getRequestActor(gatewayId, requestId);
1648
+ if (!actor) {
1649
+ (_b = this.log) == null ? void 0 : _b.warn({
1650
+ msg: "cannot send websocket ack, actor not found",
1651
+ requestId: requestIdStr
1652
+ });
1653
+ return;
1654
+ }
1655
+ const pending = actor.getPendingRequest(gatewayId, requestId);
1656
+ if (!(pending == null ? void 0 : pending.gatewayId)) {
1657
+ (_c = this.log) == null ? void 0 : _c.warn({
1658
+ msg: "cannot send websocket ack, gatewayId not found in pending request",
1659
+ requestId: requestIdStr
1660
+ });
1661
+ return;
1662
+ }
1663
+ this.#sendMessage(pending.gatewayId, requestId, {
1149
1664
  tag: "ToServerWebSocketMessageAck",
1150
1665
  val: {
1151
- index
1666
+ index: clientMessageIndex
1152
1667
  }
1153
1668
  });
1154
1669
  }
1155
- async #handleWebSocketClose(requestId, close) {
1156
- const requestIdStr = idToStr(requestId);
1157
- const adapter = this.#actorWebSockets.get(requestIdStr);
1158
- if (adapter) {
1159
- adapter._handleClose(
1160
- requestId,
1161
- close.code || void 0,
1162
- close.reason || void 0
1163
- );
1164
- this.#actorWebSockets.delete(requestIdStr);
1670
+ async #handleWebSocketClose(gatewayId, requestId, close) {
1671
+ const actor = await this.getAndWaitForRequestActor(
1672
+ gatewayId,
1673
+ requestId
1674
+ );
1675
+ if (actor) {
1676
+ const adapter = actor.getWebSocket(gatewayId, requestId);
1677
+ if (adapter) {
1678
+ adapter._handleClose(
1679
+ requestId,
1680
+ close.code || void 0,
1681
+ close.reason || void 0
1682
+ );
1683
+ actor.deleteWebSocket(gatewayId, requestId);
1684
+ actor.deletePendingRequest(gatewayId, requestId);
1685
+ this.#removeRequestToActor(gatewayId, requestId);
1686
+ }
1165
1687
  }
1166
1688
  }
1167
1689
  };
1168
- function generateUuidBuffer() {
1169
- const buffer = new Uint8Array(16);
1170
- uuidv4(void 0, buffer);
1171
- return buffer.buffer;
1172
- }
1173
- function idToStr(id) {
1174
- return uuidstringify(new Uint8Array(id));
1690
+ function buildRequestForWebSocket(path, headers) {
1691
+ const fullHeaders = {
1692
+ ...headers,
1693
+ Upgrade: "websocket",
1694
+ Connection: "Upgrade"
1695
+ };
1696
+ if (!path.startsWith("/")) {
1697
+ throw new Error("path must start with leading slash");
1698
+ }
1699
+ const request = new Request(`http://actor${path}`, {
1700
+ method: "GET",
1701
+ headers: fullHeaders
1702
+ });
1703
+ return request;
1175
1704
  }
1176
1705
 
1177
1706
  // src/websocket.ts
@@ -1209,7 +1738,7 @@ async function importWebSocket() {
1209
1738
 
1210
1739
  // src/mod.ts
1211
1740
  var KV_EXPIRE = 3e4;
1212
- var PROTOCOL_VERSION = 2;
1741
+ var PROTOCOL_VERSION = 3;
1213
1742
  var RUNNER_PING_INTERVAL = 3e3;
1214
1743
  var EVENT_BACKLOG_WARN_THRESHOLD = 1e4;
1215
1744
  var SIGNAL_HANDLERS = [];
@@ -1219,15 +1748,15 @@ var Runner = class {
1219
1748
  return this.#config;
1220
1749
  }
1221
1750
  #actors = /* @__PURE__ */ new Map();
1222
- #actorWebSockets = /* @__PURE__ */ new Map();
1223
1751
  // WebSocket
1224
- #pegboardWebSocket;
1752
+ __pegboardWebSocket;
1225
1753
  runnerId;
1226
1754
  #lastCommandIdx = -1;
1227
1755
  #pingLoop;
1228
1756
  #nextEventIdx = 0n;
1229
1757
  #started = false;
1230
1758
  #shutdown = false;
1759
+ #shuttingDown = false;
1231
1760
  #reconnectAttempt = 0;
1232
1761
  #reconnectTimeout;
1233
1762
  // Runner lost threshold management
@@ -1239,7 +1768,7 @@ var Runner = class {
1239
1768
  // Command acknowledgment
1240
1769
  #ackInterval;
1241
1770
  // KV operations
1242
- #nextRequestId = 0;
1771
+ #nextKvRequestId = 0;
1243
1772
  #kvRequests = /* @__PURE__ */ new Map();
1244
1773
  #kvCleanupInterval;
1245
1774
  // Tunnel for HTTP/WebSocket forwarding
@@ -1270,13 +1799,6 @@ var Runner = class {
1270
1799
  }
1271
1800
  // MARK: Manage actors
1272
1801
  sleepActor(actorId, generation) {
1273
- var _a;
1274
- if (this.#shutdown) {
1275
- (_a = this.log) == null ? void 0 : _a.warn({
1276
- msg: "runner is shut down, cannot sleep actor"
1277
- });
1278
- return;
1279
- }
1280
1802
  const actor = this.getActor(actorId, generation);
1281
1803
  if (!actor) return;
1282
1804
  this.#sendActorIntent(actorId, actor.generation, "sleep");
@@ -1288,7 +1810,7 @@ var Runner = class {
1288
1810
  }
1289
1811
  async forceStopActor(actorId, generation) {
1290
1812
  var _a;
1291
- const actor = this.#removeActor(actorId, generation);
1813
+ const actor = this.getActor(actorId, generation);
1292
1814
  if (!actor) return;
1293
1815
  try {
1294
1816
  await this.#config.onActorStop(actorId, actor.generation);
@@ -1296,6 +1818,7 @@ var Runner = class {
1296
1818
  console.error(`Error in onActorStop for actor ${actorId}:`, err);
1297
1819
  }
1298
1820
  (_a = this.#tunnel) == null ? void 0 : _a.closeActiveRequests(actor);
1821
+ this.#removeActor(actorId, generation);
1299
1822
  this.#sendActorStateUpdate(actorId, actor.generation, "stopped");
1300
1823
  }
1301
1824
  #stopAllActors() {
@@ -1312,14 +1835,14 @@ var Runner = class {
1312
1835
  var _a, _b;
1313
1836
  const actor = this.#actors.get(actorId);
1314
1837
  if (!actor) {
1315
- (_a = this.log) == null ? void 0 : _a.error({
1838
+ (_a = this.log) == null ? void 0 : _a.warn({
1316
1839
  msg: "actor not found",
1317
1840
  actorId
1318
1841
  });
1319
1842
  return void 0;
1320
1843
  }
1321
1844
  if (generation !== void 0 && actor.generation !== generation) {
1322
- (_b = this.log) == null ? void 0 : _b.error({
1845
+ (_b = this.log) == null ? void 0 : _b.warn({
1323
1846
  msg: "actor generation mismatch",
1324
1847
  actorId,
1325
1848
  generation
@@ -1328,13 +1851,22 @@ var Runner = class {
1328
1851
  }
1329
1852
  return actor;
1330
1853
  }
1854
+ async getAndWaitForActor(actorId, generation) {
1855
+ const actor = this.getActor(actorId, generation);
1856
+ if (!actor) return;
1857
+ await actor.actorStartPromise.promise;
1858
+ return actor;
1859
+ }
1331
1860
  hasActor(actorId, generation) {
1332
1861
  const actor = this.#actors.get(actorId);
1333
1862
  return !!actor && (generation === void 0 || actor.generation === generation);
1334
1863
  }
1864
+ get actors() {
1865
+ return this.#actors;
1866
+ }
1335
1867
  // IMPORTANT: Make sure to call stopActiveRequests if calling #removeActor
1336
1868
  #removeActor(actorId, generation) {
1337
- var _a, _b;
1869
+ var _a, _b, _c;
1338
1870
  const actor = this.#actors.get(actorId);
1339
1871
  if (!actor) {
1340
1872
  (_a = this.log) == null ? void 0 : _a.error({
@@ -1352,6 +1884,11 @@ var Runner = class {
1352
1884
  return void 0;
1353
1885
  }
1354
1886
  this.#actors.delete(actorId);
1887
+ (_c = this.log) == null ? void 0 : _c.info({
1888
+ msg: "removed actor",
1889
+ actorId,
1890
+ actors: this.#actors.size
1891
+ });
1355
1892
  return actor;
1356
1893
  }
1357
1894
  // MARK: Start
@@ -1370,37 +1907,42 @@ var Runner = class {
1370
1907
  }
1371
1908
  if (!this.#config.noAutoShutdown) {
1372
1909
  if (!SIGNAL_HANDLERS.length) {
1373
- process.on("SIGTERM", () => {
1910
+ process.on("SIGTERM", async () => {
1374
1911
  var _a2;
1375
1912
  (_a2 = this.log) == null ? void 0 : _a2.debug("received SIGTERM");
1376
1913
  for (const handler of SIGNAL_HANDLERS) {
1377
- handler();
1914
+ await handler();
1378
1915
  }
1379
- process.exit(0);
1380
1916
  });
1381
- process.on("SIGINT", () => {
1917
+ process.on("SIGINT", async () => {
1382
1918
  var _a2;
1383
1919
  (_a2 = this.log) == null ? void 0 : _a2.debug("received SIGINT");
1384
1920
  for (const handler of SIGNAL_HANDLERS) {
1385
- handler();
1921
+ await handler();
1386
1922
  }
1387
- process.exit(0);
1388
1923
  });
1389
1924
  (_b = this.log) == null ? void 0 : _b.debug({
1390
1925
  msg: "added SIGTERM listeners"
1391
1926
  });
1392
1927
  }
1393
- SIGNAL_HANDLERS.push(() => {
1928
+ SIGNAL_HANDLERS.push(async () => {
1394
1929
  var _a2;
1395
1930
  const weak = new WeakRef(this);
1396
- (_a2 = weak.deref()) == null ? void 0 : _a2.shutdown(false, false);
1931
+ await ((_a2 = weak.deref()) == null ? void 0 : _a2.shutdown(false, false));
1397
1932
  });
1398
1933
  }
1399
1934
  }
1400
1935
  // MARK: Shutdown
1401
1936
  async shutdown(immediate, exit = false) {
1402
1937
  var _a, _b, _c, _d, _e, _f, _g, _h;
1403
- (_a = this.log) == null ? void 0 : _a.info({
1938
+ if (this.#shuttingDown) {
1939
+ (_a = this.log) == null ? void 0 : _a.debug({
1940
+ msg: "shutdown already in progress, ignoring"
1941
+ });
1942
+ return;
1943
+ }
1944
+ this.#shuttingDown = true;
1945
+ (_b = this.log) == null ? void 0 : _b.info({
1404
1946
  msg: "starting shutdown",
1405
1947
  immediate,
1406
1948
  exit
@@ -1432,27 +1974,20 @@ var Runner = class {
1432
1974
  );
1433
1975
  }
1434
1976
  this.#kvRequests.clear();
1435
- if (this.#pegboardWebSocket && this.#pegboardWebSocket.readyState === 1) {
1436
- const pegboardWebSocket = this.#pegboardWebSocket;
1977
+ if (this.__webSocketReady()) {
1978
+ const pegboardWebSocket = this.__pegboardWebSocket;
1437
1979
  if (immediate) {
1438
1980
  pegboardWebSocket.close(1e3, "pegboard.runner_shutdown");
1439
1981
  } else {
1440
1982
  try {
1441
- (_b = this.log) == null ? void 0 : _b.info({
1983
+ (_c = this.log) == null ? void 0 : _c.info({
1442
1984
  msg: "sending stopping message",
1443
1985
  readyState: pegboardWebSocket.readyState
1444
1986
  });
1445
- const encoded = protocol.encodeToServer({
1987
+ this.__sendToServer({
1446
1988
  tag: "ToServerStopping",
1447
1989
  val: null
1448
1990
  });
1449
- if (this.#pegboardWebSocket && this.#pegboardWebSocket.readyState === 1) {
1450
- this.#pegboardWebSocket.send(encoded);
1451
- } else {
1452
- (_c = this.log) == null ? void 0 : _c.error(
1453
- "WebSocket not available or not open for sending data"
1454
- );
1455
- }
1456
1991
  const closePromise = new Promise((resolve) => {
1457
1992
  if (!pegboardWebSocket)
1458
1993
  throw new Error("missing pegboardWebSocket");
@@ -1466,6 +2001,7 @@ var Runner = class {
1466
2001
  resolve();
1467
2002
  });
1468
2003
  });
2004
+ await this.#waitForActorsToStop(pegboardWebSocket);
1469
2005
  (_d = this.log) == null ? void 0 : _d.info({
1470
2006
  msg: "closing WebSocket"
1471
2007
  });
@@ -1485,15 +2021,97 @@ var Runner = class {
1485
2021
  } else {
1486
2022
  (_h = this.log) == null ? void 0 : _h.debug({
1487
2023
  msg: "no runner WebSocket to shutdown or already closed",
1488
- readyState: (_g = this.#pegboardWebSocket) == null ? void 0 : _g.readyState
2024
+ readyState: (_g = this.__pegboardWebSocket) == null ? void 0 : _g.readyState
1489
2025
  });
1490
2026
  }
1491
2027
  if (this.#tunnel) {
1492
2028
  this.#tunnel.shutdown();
1493
2029
  this.#tunnel = void 0;
1494
2030
  }
1495
- if (exit) process.exit(0);
1496
2031
  this.#config.onShutdown();
2032
+ if (exit) process.exit(0);
2033
+ }
2034
+ /**
2035
+ * Wait for all actors to stop before proceeding with shutdown.
2036
+ *
2037
+ * This method polls every 100ms to check if all actors have been stopped.
2038
+ *
2039
+ * It will resolve early if:
2040
+ * - All actors are stopped
2041
+ * - The WebSocket connection is closed
2042
+ * - The shutdown timeout is reached (120 seconds)
2043
+ */
2044
+ async #waitForActorsToStop(ws) {
2045
+ const shutdownTimeout = 12e4;
2046
+ const shutdownCheckInterval = 100;
2047
+ const progressLogInterval = 5e3;
2048
+ const shutdownStartTs = Date.now();
2049
+ let lastProgressLogTs = 0;
2050
+ return new Promise((resolve) => {
2051
+ var _a, _b;
2052
+ const checkActors = () => {
2053
+ var _a2, _b2, _c, _d;
2054
+ const now = Date.now();
2055
+ const elapsed = now - shutdownStartTs;
2056
+ const wsIsClosed = ws.readyState === 2 || ws.readyState === 3;
2057
+ if (this.#actors.size === 0) {
2058
+ (_a2 = this.log) == null ? void 0 : _a2.info({
2059
+ msg: "all actors stopped",
2060
+ elapsed
2061
+ });
2062
+ return true;
2063
+ } else if (wsIsClosed) {
2064
+ (_b2 = this.log) == null ? void 0 : _b2.warn({
2065
+ msg: "websocket closed before all actors stopped",
2066
+ remainingActors: this.#actors.size,
2067
+ elapsed
2068
+ });
2069
+ return true;
2070
+ } else if (elapsed >= shutdownTimeout) {
2071
+ (_c = this.log) == null ? void 0 : _c.warn({
2072
+ msg: "shutdown timeout reached, forcing close",
2073
+ remainingActors: this.#actors.size,
2074
+ elapsed
2075
+ });
2076
+ return true;
2077
+ } else {
2078
+ if (now - lastProgressLogTs >= progressLogInterval) {
2079
+ (_d = this.log) == null ? void 0 : _d.info({
2080
+ msg: "waiting for actors to stop",
2081
+ remainingActors: this.#actors.size,
2082
+ elapsed
2083
+ });
2084
+ lastProgressLogTs = now;
2085
+ }
2086
+ return false;
2087
+ }
2088
+ };
2089
+ if (checkActors()) {
2090
+ (_a = this.log) == null ? void 0 : _a.debug({
2091
+ msg: "actors check completed immediately"
2092
+ });
2093
+ resolve();
2094
+ return;
2095
+ }
2096
+ (_b = this.log) == null ? void 0 : _b.debug({
2097
+ msg: "starting actor wait interval",
2098
+ checkInterval: shutdownCheckInterval
2099
+ });
2100
+ const interval = setInterval(() => {
2101
+ var _a2, _b2;
2102
+ (_a2 = this.log) == null ? void 0 : _a2.debug({
2103
+ msg: "actor wait interval tick",
2104
+ actorCount: this.#actors.size
2105
+ });
2106
+ if (checkActors()) {
2107
+ (_b2 = this.log) == null ? void 0 : _b2.debug({
2108
+ msg: "actors check completed, clearing interval"
2109
+ });
2110
+ clearInterval(interval);
2111
+ resolve();
2112
+ }
2113
+ }, shutdownCheckInterval);
2114
+ });
1497
2115
  }
1498
2116
  // MARK: Networking
1499
2117
  get pegboardEndpoint() {
@@ -1512,7 +2130,7 @@ var Runner = class {
1512
2130
  protocols.push(`rivet_token.${this.config.token}`);
1513
2131
  const WS = await importWebSocket();
1514
2132
  const ws = new WS(this.pegboardUrl, protocols);
1515
- this.#pegboardWebSocket = ws;
2133
+ this.__pegboardWebSocket = ws;
1516
2134
  (_a = this.log) == null ? void 0 : _a.info({
1517
2135
  msg: "connecting",
1518
2136
  endpoint: this.pegboardEndpoint,
@@ -1564,7 +2182,6 @@ var Runner = class {
1564
2182
  tag: "ToServerInit",
1565
2183
  val: init
1566
2184
  });
1567
- this.#processUnsentKvRequests();
1568
2185
  const pingLoop = setInterval(() => {
1569
2186
  var _a3;
1570
2187
  if (ws.readyState === 1) {
@@ -1597,7 +2214,7 @@ var Runner = class {
1597
2214
  this.#ackInterval = ackLoop;
1598
2215
  });
1599
2216
  ws.addEventListener("message", async (ev) => {
1600
- var _a2, _b, _c, _d;
2217
+ var _a2, _b, _c, _d, _e, _f;
1601
2218
  let buf;
1602
2219
  if (ev.data instanceof Blob) {
1603
2220
  buf = new Uint8Array(await ev.data.arrayBuffer());
@@ -1607,19 +2224,25 @@ var Runner = class {
1607
2224
  throw new Error(`expected binary data, got ${typeof ev.data}`);
1608
2225
  }
1609
2226
  const message = protocol.decodeToClient(buf);
2227
+ (_a2 = this.log) == null ? void 0 : _a2.debug({
2228
+ msg: "received runner message",
2229
+ data: stringifyToClient(message)
2230
+ });
1610
2231
  if (message.tag === "ToClientInit") {
1611
2232
  const init = message.val;
1612
2233
  if (this.runnerId !== init.runnerId) {
1613
2234
  this.runnerId = init.runnerId;
1614
2235
  this.#eventHistory.length = 0;
1615
2236
  }
1616
- this.#runnerLostThreshold = ((_a2 = init.metadata) == null ? void 0 : _a2.runnerLostThreshold) ? Number(init.metadata.runnerLostThreshold) : void 0;
1617
- (_b = this.log) == null ? void 0 : _b.info({
2237
+ this.#runnerLostThreshold = ((_b = init.metadata) == null ? void 0 : _b.runnerLostThreshold) ? Number(init.metadata.runnerLostThreshold) : void 0;
2238
+ (_c = this.log) == null ? void 0 : _c.info({
1618
2239
  msg: "received init",
1619
2240
  lastEventIdx: init.lastEventIdx,
1620
2241
  runnerLostThreshold: this.#runnerLostThreshold
1621
2242
  });
2243
+ this.#processUnsentKvRequests();
1622
2244
  this.#resendUnacknowledgedEvents(init.lastEventIdx);
2245
+ (_d = this.#tunnel) == null ? void 0 : _d.resendBufferedEvents();
1623
2246
  this.#config.onConnected();
1624
2247
  } else if (message.tag === "ToClientCommands") {
1625
2248
  const commands = message.val;
@@ -1630,9 +2253,9 @@ var Runner = class {
1630
2253
  const kvResponse = message.val;
1631
2254
  this.#handleKvResponse(kvResponse);
1632
2255
  } else if (message.tag === "ToClientTunnelMessage") {
1633
- (_c = this.#tunnel) == null ? void 0 : _c.handleTunnelMessage(message.val);
2256
+ (_e = this.#tunnel) == null ? void 0 : _e.handleTunnelMessage(message.val);
1634
2257
  } else if (message.tag === "ToClientClose") {
1635
- (_d = this.#tunnel) == null ? void 0 : _d.shutdown();
2258
+ (_f = this.#tunnel) == null ? void 0 : _f.shutdown();
1636
2259
  ws.close(1e3, "manual closure");
1637
2260
  } else {
1638
2261
  unreachable(message);
@@ -1699,16 +2322,12 @@ var Runner = class {
1699
2322
  });
1700
2323
  }
1701
2324
  #handleCommands(commands) {
1702
- var _a, _b;
2325
+ var _a;
1703
2326
  (_a = this.log) == null ? void 0 : _a.info({
1704
2327
  msg: "received commands",
1705
2328
  commandCount: commands.length
1706
2329
  });
1707
2330
  for (const commandWrapper of commands) {
1708
- (_b = this.log) == null ? void 0 : _b.info({
1709
- msg: "received command",
1710
- command: stringifyCommandWrapper(commandWrapper)
1711
- });
1712
2331
  if (commandWrapper.inner.tag === "CommandStartActor") {
1713
2332
  this.#handleCommandStartActor(commandWrapper);
1714
2333
  } else if (commandWrapper.inner.tag === "CommandStopActor") {
@@ -1751,7 +2370,9 @@ var Runner = class {
1751
2370
  });
1752
2371
  }
1753
2372
  }
1754
- #handleCommandStartActor(commandWrapper) {
2373
+ async #handleCommandStartActor(commandWrapper) {
2374
+ var _a, _b, _c, _d;
2375
+ if (!this.#tunnel) throw new Error("missing tunnel on actor start");
1755
2376
  const startCommand = commandWrapper.inner.val;
1756
2377
  const actorId = startCommand.actorId;
1757
2378
  const generation = startCommand.generation;
@@ -1762,40 +2383,61 @@ var Runner = class {
1762
2383
  createTs: config.createTs,
1763
2384
  input: config.input ? new Uint8Array(config.input) : null
1764
2385
  };
1765
- const instance = {
2386
+ const instance = new RunnerActor(
1766
2387
  actorId,
1767
2388
  generation,
1768
- config: actorConfig,
1769
- requests: /* @__PURE__ */ new Set(),
1770
- webSockets: /* @__PURE__ */ new Set()
1771
- };
2389
+ actorConfig,
2390
+ startCommand.hibernatingRequests
2391
+ );
2392
+ const existingActor = this.#actors.get(actorId);
2393
+ if (existingActor) {
2394
+ (_a = this.log) == null ? void 0 : _a.warn({
2395
+ msg: "replacing existing actor in actors map",
2396
+ actorId,
2397
+ existingGeneration: existingActor.generation,
2398
+ newGeneration: generation,
2399
+ existingPendingRequests: existingActor.pendingRequests.length
2400
+ });
2401
+ }
1772
2402
  this.#actors.set(actorId, instance);
2403
+ for (const hr of startCommand.hibernatingRequests) {
2404
+ this.#tunnel.addRequestToActor(hr.gatewayId, hr.requestId, actorId);
2405
+ }
2406
+ (_b = this.log) == null ? void 0 : _b.info({
2407
+ msg: "created actor",
2408
+ actors: this.#actors.size,
2409
+ actorId,
2410
+ name: config.name,
2411
+ key: config.key,
2412
+ generation,
2413
+ hibernatingRequests: startCommand.hibernatingRequests.length
2414
+ });
1773
2415
  this.#sendActorStateUpdate(actorId, generation, "running");
1774
- this.#config.onActorStart(actorId, generation, actorConfig).catch((err) => {
1775
- var _a;
1776
- (_a = this.log) == null ? void 0 : _a.error({
1777
- msg: "error in onactorstart for actor",
2416
+ try {
2417
+ (_c = this.log) == null ? void 0 : _c.debug({
2418
+ msg: "calling onActorStart",
2419
+ actorId,
2420
+ generation
2421
+ });
2422
+ await this.#config.onActorStart(actorId, generation, actorConfig);
2423
+ instance.actorStartPromise.resolve();
2424
+ } catch (err) {
2425
+ (_d = this.log) == null ? void 0 : _d.error({
2426
+ msg: "error starting runner actor",
1778
2427
  actorId,
1779
2428
  err
1780
2429
  });
1781
- this.forceStopActor(actorId, generation);
1782
- });
2430
+ instance.actorStartPromise.reject(err);
2431
+ await this.forceStopActor(actorId, generation);
2432
+ }
1783
2433
  }
1784
- #handleCommandStopActor(commandWrapper) {
2434
+ async #handleCommandStopActor(commandWrapper) {
1785
2435
  const stopCommand = commandWrapper.inner.val;
1786
2436
  const actorId = stopCommand.actorId;
1787
2437
  const generation = stopCommand.generation;
1788
- this.forceStopActor(actorId, generation);
2438
+ await this.forceStopActor(actorId, generation);
1789
2439
  }
1790
2440
  #sendActorIntent(actorId, generation, intentType) {
1791
- var _a, _b;
1792
- if (this.#shutdown) {
1793
- console.trace("send actor intent", actorId, intentType);
1794
- (_a = this.log) == null ? void 0 : _a.warn({
1795
- msg: "Runner is shut down, cannot send actor intent"
1796
- });
1797
- return;
1798
- }
1799
2441
  let actorIntent;
1800
2442
  if (intentType === "sleep") {
1801
2443
  actorIntent = { tag: "ActorIntentSleep", val: null };
@@ -1821,24 +2463,12 @@ var Runner = class {
1821
2463
  }
1822
2464
  };
1823
2465
  this.#recordEvent(eventWrapper);
1824
- (_b = this.log) == null ? void 0 : _b.info({
1825
- msg: "sending event to server",
1826
- event: stringifyEvent(eventWrapper.inner),
1827
- index: eventWrapper.index.toString()
1828
- });
1829
2466
  this.__sendToServer({
1830
2467
  tag: "ToServerEvents",
1831
2468
  val: [eventWrapper]
1832
2469
  });
1833
2470
  }
1834
2471
  #sendActorStateUpdate(actorId, generation, stateType) {
1835
- var _a, _b;
1836
- if (this.#shutdown) {
1837
- (_a = this.log) == null ? void 0 : _a.warn({
1838
- msg: "Runner is shut down, cannot send actor state update"
1839
- });
1840
- return;
1841
- }
1842
2472
  let actorState;
1843
2473
  if (stateType === "running") {
1844
2474
  actorState = { tag: "ActorStateRunning", val: null };
@@ -1867,24 +2497,12 @@ var Runner = class {
1867
2497
  }
1868
2498
  };
1869
2499
  this.#recordEvent(eventWrapper);
1870
- (_b = this.log) == null ? void 0 : _b.info({
1871
- msg: "sending event to server",
1872
- event: stringifyEvent(eventWrapper.inner),
1873
- index: eventWrapper.index.toString()
1874
- });
1875
2500
  this.__sendToServer({
1876
2501
  tag: "ToServerEvents",
1877
2502
  val: [eventWrapper]
1878
2503
  });
1879
2504
  }
1880
2505
  #sendCommandAcknowledgment() {
1881
- var _a;
1882
- if (this.#shutdown) {
1883
- (_a = this.log) == null ? void 0 : _a.warn({
1884
- msg: "Runner is shut down, cannot send command acknowledgment"
1885
- });
1886
- return;
1887
- }
1888
2506
  if (this.#lastCommandIdx < 0) {
1889
2507
  return;
1890
2508
  }
@@ -2120,10 +2738,6 @@ var Runner = class {
2120
2738
  setAlarm(actorId, alarmTs, generation) {
2121
2739
  const actor = this.getActor(actorId, generation);
2122
2740
  if (!actor) return;
2123
- if (this.#shutdown) {
2124
- console.warn("Runner is shut down, cannot set alarm");
2125
- return;
2126
- }
2127
2741
  const alarmEvent = {
2128
2742
  actorId,
2129
2743
  generation: actor.generation,
@@ -2148,12 +2762,7 @@ var Runner = class {
2148
2762
  }
2149
2763
  #sendKvRequest(actorId, requestData) {
2150
2764
  return new Promise((resolve, reject) => {
2151
- if (this.#shutdown) {
2152
- reject(new Error("Runner is shut down"));
2153
- return;
2154
- }
2155
- const requestId = this.#nextRequestId++;
2156
- const isConnected = this.#pegboardWebSocket && this.#pegboardWebSocket.readyState === 1;
2765
+ const requestId = this.#nextKvRequestId++;
2157
2766
  const requestEntry = {
2158
2767
  actorId,
2159
2768
  data: requestData,
@@ -2163,7 +2772,7 @@ var Runner = class {
2163
2772
  timestamp: Date.now()
2164
2773
  };
2165
2774
  this.#kvRequests.set(requestId, requestEntry);
2166
- if (isConnected) {
2775
+ if (this.__webSocketReady()) {
2167
2776
  this.#sendSingleKvRequest(requestId);
2168
2777
  }
2169
2778
  });
@@ -2189,7 +2798,7 @@ var Runner = class {
2189
2798
  }
2190
2799
  }
2191
2800
  #processUnsentKvRequests() {
2192
- if (!this.#pegboardWebSocket || this.#pegboardWebSocket.readyState !== 1) {
2801
+ if (!this.__webSocketReady()) {
2193
2802
  return;
2194
2803
  }
2195
2804
  let processedCount = 0;
@@ -2202,29 +2811,61 @@ var Runner = class {
2202
2811
  if (processedCount > 0) {
2203
2812
  }
2204
2813
  }
2814
+ /** Asserts WebSocket exists and is ready. */
2205
2815
  __webSocketReady() {
2206
- return this.#pegboardWebSocket ? this.#pegboardWebSocket.readyState === 1 : false;
2816
+ return !!this.__pegboardWebSocket && this.__pegboardWebSocket.readyState === 1;
2207
2817
  }
2208
2818
  __sendToServer(message) {
2209
2819
  var _a, _b;
2210
- if (this.#shutdown) {
2211
- (_a = this.log) == null ? void 0 : _a.warn({
2212
- msg: "Runner is shut down, cannot send message to server"
2213
- });
2214
- return;
2215
- }
2820
+ (_a = this.log) == null ? void 0 : _a.debug({
2821
+ msg: "sending runner message",
2822
+ data: stringifyToServer(message)
2823
+ });
2216
2824
  const encoded = protocol.encodeToServer(message);
2217
- if (this.#pegboardWebSocket && this.#pegboardWebSocket.readyState === 1) {
2218
- this.#pegboardWebSocket.send(encoded);
2825
+ if (this.__webSocketReady()) {
2826
+ this.__pegboardWebSocket.send(encoded);
2219
2827
  } else {
2220
2828
  (_b = this.log) == null ? void 0 : _b.error({
2221
2829
  msg: "WebSocket not available or not open for sending data"
2222
2830
  });
2223
2831
  }
2224
2832
  }
2225
- sendWebsocketMessageAck(requestId, index) {
2226
- var _a;
2227
- (_a = this.#tunnel) == null ? void 0 : _a.__ackWebsocketMessage(requestId, index);
2833
+ sendHibernatableWebSocketMessageAck(gatewayId, requestId, index) {
2834
+ if (!this.#tunnel)
2835
+ throw new Error("missing tunnel to send message ack");
2836
+ this.#tunnel.sendHibernatableWebSocketMessageAck(
2837
+ gatewayId,
2838
+ requestId,
2839
+ index
2840
+ );
2841
+ }
2842
+ /**
2843
+ * Restores hibernatable WebSocket connections for an actor.
2844
+ *
2845
+ * This method should be called at the end of `onActorStart` after the
2846
+ * actor instance is fully initialized.
2847
+ *
2848
+ * This method will:
2849
+ * - Restore all provided hibernatable WebSocket connections
2850
+ * - Attach event listeners to the restored WebSockets
2851
+ * - Close any WebSocket connections that failed to restore
2852
+ *
2853
+ * The provided metadata list should include all hibernatable WebSockets
2854
+ * that were persisted for this actor. The gateway will automatically
2855
+ * close any connections that are not restored (i.e., not included in
2856
+ * this list).
2857
+ *
2858
+ * **Important:** This method must be called after `onActorStart` completes
2859
+ * and before marking the actor as "ready" to ensure all hibernatable
2860
+ * connections are fully restored.
2861
+ *
2862
+ * @param actorId - The ID of the actor to restore connections for
2863
+ * @param metaEntries - Array of hibernatable WebSocket metadata to restore
2864
+ */
2865
+ async restoreHibernatingRequests(actorId, metaEntries) {
2866
+ if (!this.#tunnel)
2867
+ throw new Error("missing tunnel to restore hibernating requests");
2868
+ await this.#tunnel.restoreHibernatingRequests(actorId, metaEntries);
2228
2869
  }
2229
2870
  getServerlessInitPacket() {
2230
2871
  if (!this.runnerId) return void 0;
@@ -2268,10 +2909,15 @@ var Runner = class {
2268
2909
  }, delay);
2269
2910
  }
2270
2911
  #resendUnacknowledgedEvents(lastEventIdx) {
2912
+ var _a;
2271
2913
  const eventsToResend = this.#eventHistory.filter(
2272
2914
  (event) => event.index > lastEventIdx
2273
2915
  );
2274
2916
  if (eventsToResend.length === 0) return;
2917
+ (_a = this.log) == null ? void 0 : _a.info({
2918
+ msg: "resending unacknowledged events",
2919
+ fromIndex: lastEventIdx + 1n
2920
+ });
2275
2921
  this.__sendToServer({
2276
2922
  tag: "ToServerEvents",
2277
2923
  val: eventsToResend
@@ -2298,6 +2944,8 @@ var Runner = class {
2298
2944
  }
2299
2945
  };
2300
2946
  export {
2301
- Runner
2947
+ Runner,
2948
+ RunnerActor,
2949
+ idToStr
2302
2950
  };
2303
2951
  //# sourceMappingURL=mod.js.map