@rivetkit/engine-runner 2.0.4-rc.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +203 -0
- package/dist/mod.cjs +2813 -0
- package/dist/mod.cjs.map +1 -0
- package/dist/mod.d.cts +298 -0
- package/dist/mod.d.ts +298 -0
- package/dist/mod.js +2813 -0
- package/dist/mod.js.map +1 -0
- package/package.json +43 -0
- package/src/actor.ts +210 -0
- package/src/log.ts +11 -0
- package/src/mod.ts +1862 -0
- package/src/stringify.ts +353 -0
- package/src/tunnel.ts +1169 -0
- package/src/utils.ts +175 -0
- package/src/websocket-tunnel-adapter.ts +201 -0
- package/src/websocket.ts +43 -0
package/dist/mod.cjs
ADDED
|
@@ -0,0 +1,2813 @@
|
|
|
1
|
+
"use strict";Object.defineProperty(exports, "__esModule", {value: true}); function _interopRequireWildcard(obj) { if (obj && obj.__esModule) { return obj; } else { var newObj = {}; if (obj != null) { for (var key in obj) { if (Object.prototype.hasOwnProperty.call(obj, key)) { newObj[key] = obj[key]; } } } newObj.default = obj; return newObj; } } function _nullishCoalesce(lhs, rhsFn) { if (lhs != null) { return lhs; } else { return rhsFn(); } } var _class;// src/mod.ts
|
|
2
|
+
var _enginerunnerprotocol = require('@rivetkit/engine-runner-protocol'); var protocol = _interopRequireWildcard(_enginerunnerprotocol);
|
|
3
|
+
|
|
4
|
+
// src/log.ts
|
|
5
|
+
var LOGGER;
|
|
6
|
+
function setLogger(logger2) {
|
|
7
|
+
LOGGER = logger2;
|
|
8
|
+
}
|
|
9
|
+
function logger() {
|
|
10
|
+
return LOGGER;
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
// src/utils.ts
|
|
14
|
+
function unreachable(x) {
|
|
15
|
+
throw `Unreachable: ${x}`;
|
|
16
|
+
}
|
|
17
|
+
function calculateBackoff(attempt, options = {}) {
|
|
18
|
+
const {
|
|
19
|
+
initialDelay = 1e3,
|
|
20
|
+
maxDelay = 3e4,
|
|
21
|
+
multiplier = 2,
|
|
22
|
+
jitter = true
|
|
23
|
+
} = options;
|
|
24
|
+
let delay = Math.min(initialDelay * multiplier ** attempt, maxDelay);
|
|
25
|
+
if (jitter) {
|
|
26
|
+
delay = delay * (1 + Math.random() * 0.25);
|
|
27
|
+
}
|
|
28
|
+
return Math.floor(delay);
|
|
29
|
+
}
|
|
30
|
+
function parseWebSocketCloseReason(reason) {
|
|
31
|
+
var _a;
|
|
32
|
+
const [mainPart, rayId] = reason.split("#");
|
|
33
|
+
const [group, error] = mainPart.split(".");
|
|
34
|
+
if (!group || !error) {
|
|
35
|
+
(_a = logger()) == null ? void 0 : _a.warn({ msg: "failed to parse close reason", reason });
|
|
36
|
+
return void 0;
|
|
37
|
+
}
|
|
38
|
+
return {
|
|
39
|
+
group,
|
|
40
|
+
error,
|
|
41
|
+
rayId
|
|
42
|
+
};
|
|
43
|
+
}
|
|
44
|
+
var U16_MAX = 65535;
|
|
45
|
+
function wrappingLtU16(a, b) {
|
|
46
|
+
return a !== b && wrappingSub(b, a, U16_MAX) < U16_MAX / 2;
|
|
47
|
+
}
|
|
48
|
+
function wrappingLteU16(a, b) {
|
|
49
|
+
return a === b || wrappingLtU16(a, b);
|
|
50
|
+
}
|
|
51
|
+
function wrappingAddU16(a, b) {
|
|
52
|
+
return (a + b) % (U16_MAX + 1);
|
|
53
|
+
}
|
|
54
|
+
function wrappingSubU16(a, b) {
|
|
55
|
+
return wrappingSub(a, b, U16_MAX);
|
|
56
|
+
}
|
|
57
|
+
function wrappingSub(a, b, max) {
|
|
58
|
+
const result = a - b;
|
|
59
|
+
if (result < 0) {
|
|
60
|
+
return result + max + 1;
|
|
61
|
+
}
|
|
62
|
+
return result;
|
|
63
|
+
}
|
|
64
|
+
function arraysEqual(a, b) {
|
|
65
|
+
const ua = new Uint8Array(a);
|
|
66
|
+
const ub = new Uint8Array(b);
|
|
67
|
+
if (ua.length !== ub.length) return false;
|
|
68
|
+
for (let i = 0; i < ua.length; i++) {
|
|
69
|
+
if (ua[i] !== ub[i]) return false;
|
|
70
|
+
}
|
|
71
|
+
return true;
|
|
72
|
+
}
|
|
73
|
+
function promiseWithResolvers() {
|
|
74
|
+
let resolve;
|
|
75
|
+
let reject;
|
|
76
|
+
const promise = new Promise((res, rej) => {
|
|
77
|
+
resolve = res;
|
|
78
|
+
reject = rej;
|
|
79
|
+
});
|
|
80
|
+
return { promise, resolve, reject };
|
|
81
|
+
}
|
|
82
|
+
function idToStr(id) {
|
|
83
|
+
const bytes = new Uint8Array(id);
|
|
84
|
+
return Array.from(bytes).map((byte) => byte.toString(16).padStart(2, "0")).join("");
|
|
85
|
+
}
|
|
86
|
+
function stringifyError(error) {
|
|
87
|
+
var _a;
|
|
88
|
+
if (error instanceof Error) {
|
|
89
|
+
return `${error.name}: ${error.message}${error.stack ? `
|
|
90
|
+
${error.stack}` : ""}`;
|
|
91
|
+
} else if (typeof error === "string") {
|
|
92
|
+
return error;
|
|
93
|
+
} else if (typeof error === "object" && error !== null) {
|
|
94
|
+
try {
|
|
95
|
+
return `${JSON.stringify(error)}`;
|
|
96
|
+
} catch (e2) {
|
|
97
|
+
return `[object ${((_a = error.constructor) == null ? void 0 : _a.name) || "Object"}]`;
|
|
98
|
+
}
|
|
99
|
+
} else {
|
|
100
|
+
return String(error);
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
// src/actor.ts
|
|
105
|
+
var RunnerActor = (_class = class {
|
|
106
|
+
constructor(actorId, generation, config, hibernatingRequests) {;_class.prototype.__init.call(this);_class.prototype.__init2.call(this);_class.prototype.__init3.call(this);_class.prototype.__init4.call(this);_class.prototype.__init5.call(this);_class.prototype.__init6.call(this);
|
|
107
|
+
this.hibernatingRequests = hibernatingRequests;
|
|
108
|
+
this.actorId = actorId;
|
|
109
|
+
this.generation = generation;
|
|
110
|
+
this.config = config;
|
|
111
|
+
this.actorStartPromise = promiseWithResolvers();
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
__init() {this.pendingRequests = []}
|
|
117
|
+
__init2() {this.webSockets = []}
|
|
118
|
+
|
|
119
|
+
__init3() {this.lastCommandIdx = -1n}
|
|
120
|
+
__init4() {this.nextEventIdx = 0n}
|
|
121
|
+
__init5() {this.eventHistory = []}
|
|
122
|
+
/**
|
|
123
|
+
* If restoreHibernatingRequests has been called. This is used to assert
|
|
124
|
+
* that the caller is implemented correctly.
|
|
125
|
+
**/
|
|
126
|
+
__init6() {this.hibernationRestored = false}
|
|
127
|
+
// Pending request methods
|
|
128
|
+
getPendingRequest(gatewayId, requestId) {
|
|
129
|
+
var _a;
|
|
130
|
+
return (_a = this.pendingRequests.find(
|
|
131
|
+
(entry) => arraysEqual(entry.gatewayId, gatewayId) && arraysEqual(entry.requestId, requestId)
|
|
132
|
+
)) == null ? void 0 : _a.request;
|
|
133
|
+
}
|
|
134
|
+
createPendingRequest(gatewayId, requestId, clientMessageIndex) {
|
|
135
|
+
var _a, _b;
|
|
136
|
+
const exists = this.getPendingRequest(gatewayId, requestId) !== void 0;
|
|
137
|
+
if (exists) {
|
|
138
|
+
(_a = logger()) == null ? void 0 : _a.warn({
|
|
139
|
+
msg: "attempting to set pending request twice, replacing existing",
|
|
140
|
+
gatewayId: idToStr(gatewayId),
|
|
141
|
+
requestId: idToStr(requestId)
|
|
142
|
+
});
|
|
143
|
+
this.deletePendingRequest(gatewayId, requestId);
|
|
144
|
+
}
|
|
145
|
+
this.pendingRequests.push({
|
|
146
|
+
gatewayId,
|
|
147
|
+
requestId,
|
|
148
|
+
request: {
|
|
149
|
+
resolve: () => {
|
|
150
|
+
},
|
|
151
|
+
reject: () => {
|
|
152
|
+
},
|
|
153
|
+
actorId: this.actorId,
|
|
154
|
+
gatewayId,
|
|
155
|
+
requestId,
|
|
156
|
+
clientMessageIndex
|
|
157
|
+
}
|
|
158
|
+
});
|
|
159
|
+
(_b = logger()) == null ? void 0 : _b.debug({
|
|
160
|
+
msg: "added pending request",
|
|
161
|
+
gatewayId: idToStr(gatewayId),
|
|
162
|
+
requestId: idToStr(requestId),
|
|
163
|
+
length: this.pendingRequests.length
|
|
164
|
+
});
|
|
165
|
+
}
|
|
166
|
+
createPendingRequestWithStreamController(gatewayId, requestId, clientMessageIndex, streamController) {
|
|
167
|
+
var _a, _b;
|
|
168
|
+
const exists = this.getPendingRequest(gatewayId, requestId) !== void 0;
|
|
169
|
+
if (exists) {
|
|
170
|
+
(_a = logger()) == null ? void 0 : _a.warn({
|
|
171
|
+
msg: "attempting to set pending request twice, replacing existing",
|
|
172
|
+
gatewayId: idToStr(gatewayId),
|
|
173
|
+
requestId: idToStr(requestId)
|
|
174
|
+
});
|
|
175
|
+
this.deletePendingRequest(gatewayId, requestId);
|
|
176
|
+
}
|
|
177
|
+
this.pendingRequests.push({
|
|
178
|
+
gatewayId,
|
|
179
|
+
requestId,
|
|
180
|
+
request: {
|
|
181
|
+
resolve: () => {
|
|
182
|
+
},
|
|
183
|
+
reject: () => {
|
|
184
|
+
},
|
|
185
|
+
actorId: this.actorId,
|
|
186
|
+
gatewayId,
|
|
187
|
+
requestId,
|
|
188
|
+
clientMessageIndex,
|
|
189
|
+
streamController
|
|
190
|
+
}
|
|
191
|
+
});
|
|
192
|
+
(_b = logger()) == null ? void 0 : _b.debug({
|
|
193
|
+
msg: "added pending request with stream controller",
|
|
194
|
+
gatewayId: idToStr(gatewayId),
|
|
195
|
+
requestId: idToStr(requestId),
|
|
196
|
+
length: this.pendingRequests.length
|
|
197
|
+
});
|
|
198
|
+
}
|
|
199
|
+
deletePendingRequest(gatewayId, requestId) {
|
|
200
|
+
var _a;
|
|
201
|
+
const index = this.pendingRequests.findIndex(
|
|
202
|
+
(entry) => arraysEqual(entry.gatewayId, gatewayId) && arraysEqual(entry.requestId, requestId)
|
|
203
|
+
);
|
|
204
|
+
if (index !== -1) {
|
|
205
|
+
this.pendingRequests.splice(index, 1);
|
|
206
|
+
(_a = logger()) == null ? void 0 : _a.debug({
|
|
207
|
+
msg: "removed pending request",
|
|
208
|
+
gatewayId: idToStr(gatewayId),
|
|
209
|
+
requestId: idToStr(requestId),
|
|
210
|
+
length: this.pendingRequests.length
|
|
211
|
+
});
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
// WebSocket methods
|
|
215
|
+
getWebSocket(gatewayId, requestId) {
|
|
216
|
+
var _a;
|
|
217
|
+
return (_a = this.webSockets.find(
|
|
218
|
+
(entry) => arraysEqual(entry.gatewayId, gatewayId) && arraysEqual(entry.requestId, requestId)
|
|
219
|
+
)) == null ? void 0 : _a.ws;
|
|
220
|
+
}
|
|
221
|
+
setWebSocket(gatewayId, requestId, ws) {
|
|
222
|
+
var _a;
|
|
223
|
+
const exists = this.getWebSocket(gatewayId, requestId) !== void 0;
|
|
224
|
+
if (exists) {
|
|
225
|
+
(_a = logger()) == null ? void 0 : _a.warn({ msg: "attempting to set websocket twice" });
|
|
226
|
+
return;
|
|
227
|
+
}
|
|
228
|
+
this.webSockets.push({ gatewayId, requestId, ws });
|
|
229
|
+
}
|
|
230
|
+
deleteWebSocket(gatewayId, requestId) {
|
|
231
|
+
const index = this.webSockets.findIndex(
|
|
232
|
+
(entry) => arraysEqual(entry.gatewayId, gatewayId) && arraysEqual(entry.requestId, requestId)
|
|
233
|
+
);
|
|
234
|
+
if (index !== -1) {
|
|
235
|
+
this.webSockets.splice(index, 1);
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
handleAckEvents(lastEventIdx) {
|
|
239
|
+
this.eventHistory = this.eventHistory.filter(
|
|
240
|
+
(event) => event.checkpoint.index > lastEventIdx
|
|
241
|
+
);
|
|
242
|
+
}
|
|
243
|
+
recordEvent(eventWrapper) {
|
|
244
|
+
this.eventHistory.push(eventWrapper);
|
|
245
|
+
}
|
|
246
|
+
}, _class);
|
|
247
|
+
|
|
248
|
+
// src/stringify.ts
|
|
249
|
+
function stringifyArrayBuffer(buffer) {
|
|
250
|
+
return `ArrayBuffer(${buffer.byteLength})`;
|
|
251
|
+
}
|
|
252
|
+
function stringifyBigInt(value) {
|
|
253
|
+
return `${value}n`;
|
|
254
|
+
}
|
|
255
|
+
function stringifyMap(map) {
|
|
256
|
+
const entries = Array.from(map.entries()).map(([k, v]) => `"${k}": "${v}"`).join(", ");
|
|
257
|
+
return `Map(${map.size}){${entries}}`;
|
|
258
|
+
}
|
|
259
|
+
function stringifyMessageId(messageId) {
|
|
260
|
+
return `MessageId{gatewayId: ${idToStr(messageId.gatewayId)}, requestId: ${idToStr(messageId.requestId)}, messageIndex: ${messageId.messageIndex}}`;
|
|
261
|
+
}
|
|
262
|
+
function stringifyToServerTunnelMessageKind(kind) {
|
|
263
|
+
switch (kind.tag) {
|
|
264
|
+
case "ToServerResponseStart": {
|
|
265
|
+
const { status, headers, body, stream } = kind.val;
|
|
266
|
+
const bodyStr = body === null ? "null" : stringifyArrayBuffer(body);
|
|
267
|
+
return `ToServerResponseStart{status: ${status}, headers: ${stringifyMap(headers)}, body: ${bodyStr}, stream: ${stream}}`;
|
|
268
|
+
}
|
|
269
|
+
case "ToServerResponseChunk": {
|
|
270
|
+
const { body, finish } = kind.val;
|
|
271
|
+
return `ToServerResponseChunk{body: ${stringifyArrayBuffer(body)}, finish: ${finish}}`;
|
|
272
|
+
}
|
|
273
|
+
case "ToServerResponseAbort":
|
|
274
|
+
return "ToServerResponseAbort";
|
|
275
|
+
case "ToServerWebSocketOpen": {
|
|
276
|
+
const { canHibernate } = kind.val;
|
|
277
|
+
return `ToServerWebSocketOpen{canHibernate: ${canHibernate}}`;
|
|
278
|
+
}
|
|
279
|
+
case "ToServerWebSocketMessage": {
|
|
280
|
+
const { data, binary } = kind.val;
|
|
281
|
+
return `ToServerWebSocketMessage{data: ${stringifyArrayBuffer(data)}, binary: ${binary}}`;
|
|
282
|
+
}
|
|
283
|
+
case "ToServerWebSocketMessageAck": {
|
|
284
|
+
const { index } = kind.val;
|
|
285
|
+
return `ToServerWebSocketMessageAck{index: ${index}}`;
|
|
286
|
+
}
|
|
287
|
+
case "ToServerWebSocketClose": {
|
|
288
|
+
const { code, reason, hibernate } = kind.val;
|
|
289
|
+
const codeStr = code === null ? "null" : code.toString();
|
|
290
|
+
const reasonStr = reason === null ? "null" : `"${reason}"`;
|
|
291
|
+
return `ToServerWebSocketClose{code: ${codeStr}, reason: ${reasonStr}, hibernate: ${hibernate}}`;
|
|
292
|
+
}
|
|
293
|
+
}
|
|
294
|
+
}
|
|
295
|
+
function stringifyToClientTunnelMessageKind(kind) {
|
|
296
|
+
switch (kind.tag) {
|
|
297
|
+
case "ToClientRequestStart": {
|
|
298
|
+
const { actorId, method, path, headers, body, stream } = kind.val;
|
|
299
|
+
const bodyStr = body === null ? "null" : stringifyArrayBuffer(body);
|
|
300
|
+
return `ToClientRequestStart{actorId: "${actorId}", method: "${method}", path: "${path}", headers: ${stringifyMap(headers)}, body: ${bodyStr}, stream: ${stream}}`;
|
|
301
|
+
}
|
|
302
|
+
case "ToClientRequestChunk": {
|
|
303
|
+
const { body, finish } = kind.val;
|
|
304
|
+
return `ToClientRequestChunk{body: ${stringifyArrayBuffer(body)}, finish: ${finish}}`;
|
|
305
|
+
}
|
|
306
|
+
case "ToClientRequestAbort":
|
|
307
|
+
return "ToClientRequestAbort";
|
|
308
|
+
case "ToClientWebSocketOpen": {
|
|
309
|
+
const { actorId, path, headers } = kind.val;
|
|
310
|
+
return `ToClientWebSocketOpen{actorId: "${actorId}", path: "${path}", headers: ${stringifyMap(headers)}}`;
|
|
311
|
+
}
|
|
312
|
+
case "ToClientWebSocketMessage": {
|
|
313
|
+
const { data, binary } = kind.val;
|
|
314
|
+
return `ToClientWebSocketMessage{data: ${stringifyArrayBuffer(data)}, binary: ${binary}}`;
|
|
315
|
+
}
|
|
316
|
+
case "ToClientWebSocketClose": {
|
|
317
|
+
const { code, reason } = kind.val;
|
|
318
|
+
const codeStr = code === null ? "null" : code.toString();
|
|
319
|
+
const reasonStr = reason === null ? "null" : `"${reason}"`;
|
|
320
|
+
return `ToClientWebSocketClose{code: ${codeStr}, reason: ${reasonStr}}`;
|
|
321
|
+
}
|
|
322
|
+
}
|
|
323
|
+
}
|
|
324
|
+
function stringifyCommand(command) {
|
|
325
|
+
switch (command.tag) {
|
|
326
|
+
case "CommandStartActor": {
|
|
327
|
+
const { config, hibernatingRequests } = command.val;
|
|
328
|
+
const keyStr = config.key === null ? "null" : `"${config.key}"`;
|
|
329
|
+
const inputStr = config.input === null ? "null" : stringifyArrayBuffer(config.input);
|
|
330
|
+
const hibernatingRequestsStr = hibernatingRequests.length > 0 ? `[${hibernatingRequests.map((hr) => `{gatewayId: ${idToStr(hr.gatewayId)}, requestId: ${idToStr(hr.requestId)}}`).join(", ")}]` : "[]";
|
|
331
|
+
return `CommandStartActor{config: {name: "${config.name}", key: ${keyStr}, createTs: ${stringifyBigInt(config.createTs)}, input: ${inputStr}}, hibernatingRequests: ${hibernatingRequestsStr}}`;
|
|
332
|
+
}
|
|
333
|
+
case "CommandStopActor": {
|
|
334
|
+
return `CommandStopActor`;
|
|
335
|
+
}
|
|
336
|
+
}
|
|
337
|
+
}
|
|
338
|
+
function stringifyCommandWrapper(wrapper) {
|
|
339
|
+
return `CommandWrapper{actorId: "${wrapper.checkpoint.actorId}", generation: "${wrapper.checkpoint.generation}", index: ${stringifyBigInt(wrapper.checkpoint.index)}, inner: ${stringifyCommand(wrapper.inner)}}`;
|
|
340
|
+
}
|
|
341
|
+
function stringifyEvent(event) {
|
|
342
|
+
switch (event.tag) {
|
|
343
|
+
case "EventActorIntent": {
|
|
344
|
+
const { intent } = event.val;
|
|
345
|
+
const intentStr = intent.tag === "ActorIntentSleep" ? "Sleep" : intent.tag === "ActorIntentStop" ? "Stop" : "Unknown";
|
|
346
|
+
return `EventActorIntent{intent: ${intentStr}}`;
|
|
347
|
+
}
|
|
348
|
+
case "EventActorStateUpdate": {
|
|
349
|
+
const { state } = event.val;
|
|
350
|
+
let stateStr;
|
|
351
|
+
if (state.tag === "ActorStateRunning") {
|
|
352
|
+
stateStr = "Running";
|
|
353
|
+
} else if (state.tag === "ActorStateStopped") {
|
|
354
|
+
const { code, message } = state.val;
|
|
355
|
+
const messageStr = message === null ? "null" : `"${message}"`;
|
|
356
|
+
stateStr = `Stopped{code: ${code}, message: ${messageStr}}`;
|
|
357
|
+
} else {
|
|
358
|
+
stateStr = "Unknown";
|
|
359
|
+
}
|
|
360
|
+
return `EventActorStateUpdate{state: ${stateStr}}`;
|
|
361
|
+
}
|
|
362
|
+
case "EventActorSetAlarm": {
|
|
363
|
+
const { alarmTs } = event.val;
|
|
364
|
+
const alarmTsStr = alarmTs === null ? "null" : stringifyBigInt(alarmTs);
|
|
365
|
+
return `EventActorSetAlarm{alarmTs: ${alarmTsStr}}`;
|
|
366
|
+
}
|
|
367
|
+
}
|
|
368
|
+
}
|
|
369
|
+
function stringifyEventWrapper(wrapper) {
|
|
370
|
+
return `EventWrapper{actorId: ${wrapper.checkpoint.actorId}, generation: "${wrapper.checkpoint.generation}", index: ${stringifyBigInt(wrapper.checkpoint.index)}, inner: ${stringifyEvent(wrapper.inner)}}`;
|
|
371
|
+
}
|
|
372
|
+
function stringifyToServer(message) {
|
|
373
|
+
switch (message.tag) {
|
|
374
|
+
case "ToServerInit": {
|
|
375
|
+
const {
|
|
376
|
+
name,
|
|
377
|
+
version,
|
|
378
|
+
totalSlots,
|
|
379
|
+
prepopulateActorNames,
|
|
380
|
+
metadata
|
|
381
|
+
} = message.val;
|
|
382
|
+
const prepopulateActorNamesStr = prepopulateActorNames === null ? "null" : `Map(${prepopulateActorNames.size})`;
|
|
383
|
+
const metadataStr = metadata === null ? "null" : `"${metadata}"`;
|
|
384
|
+
return `ToServerInit{name: "${name}", version: ${version}, totalSlots: ${totalSlots}, prepopulateActorNames: ${prepopulateActorNamesStr}, metadata: ${metadataStr}}`;
|
|
385
|
+
}
|
|
386
|
+
case "ToServerEvents": {
|
|
387
|
+
const events = message.val;
|
|
388
|
+
return `ToServerEvents{count: ${events.length}, events: [${events.map((e) => stringifyEventWrapper(e)).join(", ")}]}`;
|
|
389
|
+
}
|
|
390
|
+
case "ToServerAckCommands": {
|
|
391
|
+
const { lastCommandCheckpoints } = message.val;
|
|
392
|
+
const checkpointsStr = lastCommandCheckpoints.length > 0 ? `[${lastCommandCheckpoints.map((cp) => `{actorId: "${cp.actorId}", index: ${stringifyBigInt(cp.index)}}`).join(", ")}]` : "[]";
|
|
393
|
+
return `ToServerAckCommands{lastCommandCheckpoints: ${checkpointsStr}}`;
|
|
394
|
+
}
|
|
395
|
+
case "ToServerStopping":
|
|
396
|
+
return "ToServerStopping";
|
|
397
|
+
case "ToServerPong": {
|
|
398
|
+
const { ts } = message.val;
|
|
399
|
+
return `ToServerPong{ts: ${stringifyBigInt(ts)}}`;
|
|
400
|
+
}
|
|
401
|
+
case "ToServerKvRequest": {
|
|
402
|
+
const { actorId, requestId, data } = message.val;
|
|
403
|
+
const dataStr = stringifyKvRequestData(data);
|
|
404
|
+
return `ToServerKvRequest{actorId: "${actorId}", requestId: ${requestId}, data: ${dataStr}}`;
|
|
405
|
+
}
|
|
406
|
+
case "ToServerTunnelMessage": {
|
|
407
|
+
const { messageId, messageKind } = message.val;
|
|
408
|
+
return `ToServerTunnelMessage{messageId: ${stringifyMessageId(messageId)}, messageKind: ${stringifyToServerTunnelMessageKind(messageKind)}}`;
|
|
409
|
+
}
|
|
410
|
+
}
|
|
411
|
+
}
|
|
412
|
+
function stringifyToClient(message) {
|
|
413
|
+
switch (message.tag) {
|
|
414
|
+
case "ToClientInit": {
|
|
415
|
+
const { runnerId, metadata } = message.val;
|
|
416
|
+
const metadataStr = `{runnerLostThreshold: ${stringifyBigInt(metadata.runnerLostThreshold)}}`;
|
|
417
|
+
return `ToClientInit{runnerId: "${runnerId}", metadata: ${metadataStr}}`;
|
|
418
|
+
}
|
|
419
|
+
case "ToClientPing": {
|
|
420
|
+
const { ts } = message.val;
|
|
421
|
+
return `ToClientPing{ts: ${stringifyBigInt(ts)}}`;
|
|
422
|
+
}
|
|
423
|
+
case "ToClientCommands": {
|
|
424
|
+
const commands = message.val;
|
|
425
|
+
return `ToClientCommands{count: ${commands.length}, commands: [${commands.map((c) => stringifyCommandWrapper(c)).join(", ")}]}`;
|
|
426
|
+
}
|
|
427
|
+
case "ToClientAckEvents": {
|
|
428
|
+
const { lastEventCheckpoints } = message.val;
|
|
429
|
+
const checkpointsStr = lastEventCheckpoints.length > 0 ? `[${lastEventCheckpoints.map((cp) => `{actorId: "${cp.actorId}", index: ${stringifyBigInt(cp.index)}}`).join(", ")}]` : "[]";
|
|
430
|
+
return `ToClientAckEvents{lastEventCheckpoints: ${checkpointsStr}}`;
|
|
431
|
+
}
|
|
432
|
+
case "ToClientKvResponse": {
|
|
433
|
+
const { requestId, data } = message.val;
|
|
434
|
+
const dataStr = stringifyKvResponseData(data);
|
|
435
|
+
return `ToClientKvResponse{requestId: ${requestId}, data: ${dataStr}}`;
|
|
436
|
+
}
|
|
437
|
+
case "ToClientTunnelMessage": {
|
|
438
|
+
const { messageId, messageKind } = message.val;
|
|
439
|
+
return `ToClientTunnelMessage{messageId: ${stringifyMessageId(messageId)}, messageKind: ${stringifyToClientTunnelMessageKind(messageKind)}}`;
|
|
440
|
+
}
|
|
441
|
+
}
|
|
442
|
+
}
|
|
443
|
+
function stringifyKvRequestData(data) {
|
|
444
|
+
switch (data.tag) {
|
|
445
|
+
case "KvGetRequest": {
|
|
446
|
+
const { keys } = data.val;
|
|
447
|
+
return `KvGetRequest{keys: ${keys.length}}`;
|
|
448
|
+
}
|
|
449
|
+
case "KvListRequest": {
|
|
450
|
+
const { query, reverse, limit } = data.val;
|
|
451
|
+
const reverseStr = reverse === null ? "null" : reverse.toString();
|
|
452
|
+
const limitStr = limit === null ? "null" : stringifyBigInt(limit);
|
|
453
|
+
return `KvListRequest{query: ${stringifyKvListQuery(query)}, reverse: ${reverseStr}, limit: ${limitStr}}`;
|
|
454
|
+
}
|
|
455
|
+
case "KvPutRequest": {
|
|
456
|
+
const { keys, values } = data.val;
|
|
457
|
+
return `KvPutRequest{keys: ${keys.length}, values: ${values.length}}`;
|
|
458
|
+
}
|
|
459
|
+
case "KvDeleteRequest": {
|
|
460
|
+
const { keys } = data.val;
|
|
461
|
+
return `KvDeleteRequest{keys: ${keys.length}}`;
|
|
462
|
+
}
|
|
463
|
+
case "KvDropRequest":
|
|
464
|
+
return "KvDropRequest";
|
|
465
|
+
}
|
|
466
|
+
}
|
|
467
|
+
function stringifyKvListQuery(query) {
|
|
468
|
+
switch (query.tag) {
|
|
469
|
+
case "KvListAllQuery":
|
|
470
|
+
return "KvListAllQuery";
|
|
471
|
+
case "KvListRangeQuery": {
|
|
472
|
+
const { start, end, exclusive } = query.val;
|
|
473
|
+
return `KvListRangeQuery{start: ${stringifyArrayBuffer(start)}, end: ${stringifyArrayBuffer(end)}, exclusive: ${exclusive}}`;
|
|
474
|
+
}
|
|
475
|
+
case "KvListPrefixQuery": {
|
|
476
|
+
const { key } = query.val;
|
|
477
|
+
return `KvListPrefixQuery{key: ${stringifyArrayBuffer(key)}}`;
|
|
478
|
+
}
|
|
479
|
+
}
|
|
480
|
+
}
|
|
481
|
+
function stringifyKvResponseData(data) {
|
|
482
|
+
switch (data.tag) {
|
|
483
|
+
case "KvErrorResponse": {
|
|
484
|
+
const { message } = data.val;
|
|
485
|
+
return `KvErrorResponse{message: "${message}"}`;
|
|
486
|
+
}
|
|
487
|
+
case "KvGetResponse": {
|
|
488
|
+
const { keys, values, metadata } = data.val;
|
|
489
|
+
return `KvGetResponse{keys: ${keys.length}, values: ${values.length}, metadata: ${metadata.length}}`;
|
|
490
|
+
}
|
|
491
|
+
case "KvListResponse": {
|
|
492
|
+
const { keys, values, metadata } = data.val;
|
|
493
|
+
return `KvListResponse{keys: ${keys.length}, values: ${values.length}, metadata: ${metadata.length}}`;
|
|
494
|
+
}
|
|
495
|
+
case "KvPutResponse":
|
|
496
|
+
return "KvPutResponse";
|
|
497
|
+
case "KvDeleteResponse":
|
|
498
|
+
return "KvDeleteResponse";
|
|
499
|
+
case "KvDropResponse":
|
|
500
|
+
return "KvDropResponse";
|
|
501
|
+
}
|
|
502
|
+
}
|
|
503
|
+
|
|
504
|
+
// src/websocket-tunnel-adapter.ts
|
|
505
|
+
var _virtualwebsocket = require('@rivetkit/virtual-websocket');
|
|
506
|
+
var HIBERNATABLE_SYMBOL = /* @__PURE__ */ Symbol("hibernatable");
|
|
507
|
+
var WebSocketTunnelAdapter = class {
|
|
508
|
+
constructor(tunnel, actorId, requestId, serverMessageIndex, hibernatable, isRestoringHibernatable, request, sendCallback, closeCallback) {
|
|
509
|
+
this.request = request;
|
|
510
|
+
var _a;
|
|
511
|
+
this.#tunnel = tunnel;
|
|
512
|
+
this.#actorId = actorId;
|
|
513
|
+
this.#requestId = requestId;
|
|
514
|
+
this.#hibernatable = hibernatable;
|
|
515
|
+
this.#serverMessageIndex = serverMessageIndex;
|
|
516
|
+
this.#sendCallback = sendCallback;
|
|
517
|
+
this.#closeCallback = closeCallback;
|
|
518
|
+
this.#ws = new (0, _virtualwebsocket.VirtualWebSocket)({
|
|
519
|
+
getReadyState: () => this.#readyState,
|
|
520
|
+
onSend: (data) => this.#handleSend(data),
|
|
521
|
+
onClose: (code, reason) => this.#close(code, reason, true),
|
|
522
|
+
onTerminate: () => this.#terminate()
|
|
523
|
+
});
|
|
524
|
+
if (isRestoringHibernatable) {
|
|
525
|
+
(_a = this.#log) == null ? void 0 : _a.debug({
|
|
526
|
+
msg: "setting WebSocket to OPEN state for restored connection",
|
|
527
|
+
actorId: this.#actorId,
|
|
528
|
+
requestId: this.#requestId
|
|
529
|
+
});
|
|
530
|
+
this.#readyState = 1;
|
|
531
|
+
}
|
|
532
|
+
}
|
|
533
|
+
#readyState = 0;
|
|
534
|
+
#binaryType = "nodebuffer";
|
|
535
|
+
#ws;
|
|
536
|
+
#tunnel;
|
|
537
|
+
#actorId;
|
|
538
|
+
#requestId;
|
|
539
|
+
#hibernatable;
|
|
540
|
+
#serverMessageIndex;
|
|
541
|
+
#sendCallback;
|
|
542
|
+
#closeCallback;
|
|
543
|
+
get [HIBERNATABLE_SYMBOL]() {
|
|
544
|
+
return this.#hibernatable;
|
|
545
|
+
}
|
|
546
|
+
get #log() {
|
|
547
|
+
return this.#tunnel.log;
|
|
548
|
+
}
|
|
549
|
+
get websocket() {
|
|
550
|
+
return this.#ws;
|
|
551
|
+
}
|
|
552
|
+
#handleSend(data) {
|
|
553
|
+
let isBinary = false;
|
|
554
|
+
let messageData;
|
|
555
|
+
if (typeof data === "string") {
|
|
556
|
+
messageData = data;
|
|
557
|
+
} else if (data instanceof ArrayBuffer) {
|
|
558
|
+
isBinary = true;
|
|
559
|
+
messageData = data;
|
|
560
|
+
} else if (ArrayBuffer.isView(data)) {
|
|
561
|
+
isBinary = true;
|
|
562
|
+
const view = data;
|
|
563
|
+
const buffer = view.buffer instanceof SharedArrayBuffer ? new Uint8Array(view.buffer, view.byteOffset, view.byteLength).slice().buffer : view.buffer.slice(view.byteOffset, view.byteOffset + view.byteLength);
|
|
564
|
+
messageData = buffer;
|
|
565
|
+
} else {
|
|
566
|
+
throw new Error("Unsupported data type");
|
|
567
|
+
}
|
|
568
|
+
this.#sendCallback(messageData, isBinary);
|
|
569
|
+
}
|
|
570
|
+
// Called by Tunnel when WebSocket is opened
|
|
571
|
+
_handleOpen(requestId) {
|
|
572
|
+
if (this.#readyState !== 0) return;
|
|
573
|
+
this.#readyState = 1;
|
|
574
|
+
this.#ws.dispatchEvent({ type: "open", rivetRequestId: requestId, target: this.#ws });
|
|
575
|
+
}
|
|
576
|
+
// Called by Tunnel when message is received
|
|
577
|
+
_handleMessage(requestId, data, serverMessageIndex, isBinary) {
|
|
578
|
+
var _a, _b, _c;
|
|
579
|
+
if (this.#readyState !== 1) {
|
|
580
|
+
(_a = this.#log) == null ? void 0 : _a.warn({
|
|
581
|
+
msg: "WebSocket message ignored - not in OPEN state",
|
|
582
|
+
requestId: this.#requestId,
|
|
583
|
+
actorId: this.#actorId,
|
|
584
|
+
currentReadyState: this.#readyState
|
|
585
|
+
});
|
|
586
|
+
return true;
|
|
587
|
+
}
|
|
588
|
+
if (this.#hibernatable) {
|
|
589
|
+
const previousIndex = this.#serverMessageIndex;
|
|
590
|
+
if (wrappingLteU16(serverMessageIndex, previousIndex)) {
|
|
591
|
+
(_b = this.#log) == null ? void 0 : _b.info({
|
|
592
|
+
msg: "received duplicate hibernating websocket message",
|
|
593
|
+
requestId,
|
|
594
|
+
actorId: this.#actorId,
|
|
595
|
+
previousIndex,
|
|
596
|
+
receivedIndex: serverMessageIndex
|
|
597
|
+
});
|
|
598
|
+
return true;
|
|
599
|
+
}
|
|
600
|
+
const expectedIndex = wrappingAddU16(previousIndex, 1);
|
|
601
|
+
if (serverMessageIndex !== expectedIndex) {
|
|
602
|
+
const closeReason = "ws.message_index_skip";
|
|
603
|
+
(_c = this.#log) == null ? void 0 : _c.warn({
|
|
604
|
+
msg: "hibernatable websocket message index out of sequence, closing connection",
|
|
605
|
+
requestId,
|
|
606
|
+
actorId: this.#actorId,
|
|
607
|
+
previousIndex,
|
|
608
|
+
expectedIndex,
|
|
609
|
+
receivedIndex: serverMessageIndex,
|
|
610
|
+
closeReason,
|
|
611
|
+
gap: wrappingSubU16(wrappingSubU16(serverMessageIndex, previousIndex), 1)
|
|
612
|
+
});
|
|
613
|
+
this.#close(1008, closeReason, true);
|
|
614
|
+
return true;
|
|
615
|
+
}
|
|
616
|
+
this.#serverMessageIndex = serverMessageIndex;
|
|
617
|
+
}
|
|
618
|
+
let messageData = data;
|
|
619
|
+
if (isBinary && data instanceof Uint8Array) {
|
|
620
|
+
if (this.#binaryType === "nodebuffer") {
|
|
621
|
+
messageData = Buffer.from(data);
|
|
622
|
+
} else if (this.#binaryType === "arraybuffer") {
|
|
623
|
+
messageData = data.buffer.slice(data.byteOffset, data.byteOffset + data.byteLength);
|
|
624
|
+
}
|
|
625
|
+
}
|
|
626
|
+
this.#ws.dispatchEvent({
|
|
627
|
+
type: "message",
|
|
628
|
+
data: messageData,
|
|
629
|
+
rivetRequestId: requestId,
|
|
630
|
+
rivetMessageIndex: serverMessageIndex,
|
|
631
|
+
target: this.#ws
|
|
632
|
+
});
|
|
633
|
+
return false;
|
|
634
|
+
}
|
|
635
|
+
// Called by Tunnel when close is received
|
|
636
|
+
_handleClose(_requestId, code, reason) {
|
|
637
|
+
this.#close(code, reason, true);
|
|
638
|
+
}
|
|
639
|
+
// Close without sending close message to tunnel
|
|
640
|
+
_closeWithoutCallback(code, reason) {
|
|
641
|
+
this.#close(code, reason, false);
|
|
642
|
+
}
|
|
643
|
+
// Public close method (used by tunnel.ts for stale websocket cleanup)
|
|
644
|
+
close(code, reason) {
|
|
645
|
+
this.#close(code, reason, true);
|
|
646
|
+
}
|
|
647
|
+
#close(code, reason, sendCallback) {
|
|
648
|
+
if (this.#readyState >= 2) return;
|
|
649
|
+
this.#readyState = 2;
|
|
650
|
+
if (sendCallback) this.#closeCallback(code, reason);
|
|
651
|
+
this.#readyState = 3;
|
|
652
|
+
this.#ws.triggerClose(_nullishCoalesce(code, () => ( 1e3)), _nullishCoalesce(reason, () => ( "")));
|
|
653
|
+
}
|
|
654
|
+
#terminate() {
|
|
655
|
+
this.#readyState = 3;
|
|
656
|
+
this.#closeCallback(1006, "Abnormal Closure");
|
|
657
|
+
this.#ws.triggerClose(1006, "Abnormal Closure", false);
|
|
658
|
+
}
|
|
659
|
+
};
|
|
660
|
+
|
|
661
|
+
// src/tunnel.ts
|
|
662
|
+
var Tunnel = class {
|
|
663
|
+
#runner;
|
|
664
|
+
/** Maps request IDs to actor IDs for lookup */
|
|
665
|
+
#requestToActor = [];
|
|
666
|
+
/** Buffer for messages when not connected */
|
|
667
|
+
#bufferedMessages = [];
|
|
668
|
+
get log() {
|
|
669
|
+
return this.#runner.log;
|
|
670
|
+
}
|
|
671
|
+
constructor(runner) {
|
|
672
|
+
this.#runner = runner;
|
|
673
|
+
}
|
|
674
|
+
start() {
|
|
675
|
+
}
|
|
676
|
+
resendBufferedEvents() {
|
|
677
|
+
var _a;
|
|
678
|
+
if (this.#bufferedMessages.length === 0) {
|
|
679
|
+
return;
|
|
680
|
+
}
|
|
681
|
+
(_a = this.log) == null ? void 0 : _a.info({
|
|
682
|
+
msg: "resending buffered tunnel messages",
|
|
683
|
+
count: this.#bufferedMessages.length
|
|
684
|
+
});
|
|
685
|
+
const messages = this.#bufferedMessages;
|
|
686
|
+
this.#bufferedMessages = [];
|
|
687
|
+
for (const { gatewayId, requestId, messageKind } of messages) {
|
|
688
|
+
this.#sendMessage(gatewayId, requestId, messageKind);
|
|
689
|
+
}
|
|
690
|
+
}
|
|
691
|
+
shutdown() {
|
|
692
|
+
for (const [_actorId, actor] of this.#runner.actors) {
|
|
693
|
+
for (const entry of actor.pendingRequests) {
|
|
694
|
+
entry.request.reject(new RunnerShutdownError());
|
|
695
|
+
}
|
|
696
|
+
actor.pendingRequests = [];
|
|
697
|
+
for (const entry of actor.webSockets) {
|
|
698
|
+
if (!entry.ws[HIBERNATABLE_SYMBOL]) {
|
|
699
|
+
entry.ws._closeWithoutCallback(1e3, "ws.tunnel_shutdown");
|
|
700
|
+
}
|
|
701
|
+
}
|
|
702
|
+
actor.webSockets = [];
|
|
703
|
+
}
|
|
704
|
+
this.#requestToActor = [];
|
|
705
|
+
}
|
|
706
|
+
async restoreHibernatingRequests(actorId, metaEntries) {
|
|
707
|
+
var _a, _b, _c, _d;
|
|
708
|
+
const actor = this.#runner.getActor(actorId);
|
|
709
|
+
if (!actor) {
|
|
710
|
+
throw new Error(
|
|
711
|
+
`Actor ${actorId} not found for restoring hibernating requests`
|
|
712
|
+
);
|
|
713
|
+
}
|
|
714
|
+
if (actor.hibernationRestored) {
|
|
715
|
+
throw new Error(
|
|
716
|
+
`Actor ${actorId} already restored hibernating requests`
|
|
717
|
+
);
|
|
718
|
+
}
|
|
719
|
+
(_a = this.log) == null ? void 0 : _a.debug({
|
|
720
|
+
msg: "restoring hibernating requests",
|
|
721
|
+
actorId,
|
|
722
|
+
requests: actor.hibernatingRequests.length
|
|
723
|
+
});
|
|
724
|
+
const backgroundOperations = [];
|
|
725
|
+
let connectedButNotLoadedCount = 0;
|
|
726
|
+
let restoredCount = 0;
|
|
727
|
+
for (const { gatewayId, requestId } of actor.hibernatingRequests) {
|
|
728
|
+
const requestIdStr = idToStr(requestId);
|
|
729
|
+
const meta = metaEntries.find(
|
|
730
|
+
(entry) => arraysEqual(entry.gatewayId, gatewayId) && arraysEqual(entry.requestId, requestId)
|
|
731
|
+
);
|
|
732
|
+
if (!meta) {
|
|
733
|
+
(_b = this.log) == null ? void 0 : _b.warn({
|
|
734
|
+
msg: "closing websocket that is not persisted",
|
|
735
|
+
requestId: requestIdStr
|
|
736
|
+
});
|
|
737
|
+
this.#sendMessage(gatewayId, requestId, {
|
|
738
|
+
tag: "ToServerWebSocketClose",
|
|
739
|
+
val: {
|
|
740
|
+
code: 1e3,
|
|
741
|
+
reason: "ws.meta_not_found_during_restore",
|
|
742
|
+
hibernate: false
|
|
743
|
+
}
|
|
744
|
+
});
|
|
745
|
+
connectedButNotLoadedCount++;
|
|
746
|
+
} else {
|
|
747
|
+
const request = buildRequestForWebSocket(
|
|
748
|
+
meta.path,
|
|
749
|
+
meta.headers
|
|
750
|
+
);
|
|
751
|
+
const restoreOperation = this.#createWebSocket(
|
|
752
|
+
actorId,
|
|
753
|
+
gatewayId,
|
|
754
|
+
requestId,
|
|
755
|
+
requestIdStr,
|
|
756
|
+
meta.serverMessageIndex,
|
|
757
|
+
true,
|
|
758
|
+
true,
|
|
759
|
+
request,
|
|
760
|
+
meta.path,
|
|
761
|
+
meta.headers,
|
|
762
|
+
false
|
|
763
|
+
).then(() => {
|
|
764
|
+
var _a2;
|
|
765
|
+
const actor2 = this.#runner.getActor(actorId);
|
|
766
|
+
if (actor2) {
|
|
767
|
+
actor2.createPendingRequest(
|
|
768
|
+
gatewayId,
|
|
769
|
+
requestId,
|
|
770
|
+
meta.clientMessageIndex
|
|
771
|
+
);
|
|
772
|
+
}
|
|
773
|
+
(_a2 = this.log) == null ? void 0 : _a2.info({
|
|
774
|
+
msg: "connection successfully restored",
|
|
775
|
+
actorId,
|
|
776
|
+
requestId: requestIdStr
|
|
777
|
+
});
|
|
778
|
+
}).catch((err) => {
|
|
779
|
+
var _a2;
|
|
780
|
+
(_a2 = this.log) == null ? void 0 : _a2.error({
|
|
781
|
+
msg: "error creating websocket during restore",
|
|
782
|
+
requestId: requestIdStr,
|
|
783
|
+
error: stringifyError(err)
|
|
784
|
+
});
|
|
785
|
+
this.#sendMessage(gatewayId, requestId, {
|
|
786
|
+
tag: "ToServerWebSocketClose",
|
|
787
|
+
val: {
|
|
788
|
+
code: 1011,
|
|
789
|
+
reason: "ws.restore_error",
|
|
790
|
+
hibernate: false
|
|
791
|
+
}
|
|
792
|
+
});
|
|
793
|
+
});
|
|
794
|
+
backgroundOperations.push(restoreOperation);
|
|
795
|
+
restoredCount++;
|
|
796
|
+
}
|
|
797
|
+
}
|
|
798
|
+
let loadedButNotConnectedCount = 0;
|
|
799
|
+
for (const meta of metaEntries) {
|
|
800
|
+
const requestIdStr = idToStr(meta.requestId);
|
|
801
|
+
const isConnected = actor.hibernatingRequests.some(
|
|
802
|
+
(req) => arraysEqual(req.gatewayId, meta.gatewayId) && arraysEqual(req.requestId, meta.requestId)
|
|
803
|
+
);
|
|
804
|
+
if (!isConnected) {
|
|
805
|
+
(_c = this.log) == null ? void 0 : _c.warn({
|
|
806
|
+
msg: "removing stale persisted websocket",
|
|
807
|
+
requestId: requestIdStr
|
|
808
|
+
});
|
|
809
|
+
const request = buildRequestForWebSocket(
|
|
810
|
+
meta.path,
|
|
811
|
+
meta.headers
|
|
812
|
+
);
|
|
813
|
+
const cleanupOperation = this.#createWebSocket(
|
|
814
|
+
actorId,
|
|
815
|
+
meta.gatewayId,
|
|
816
|
+
meta.requestId,
|
|
817
|
+
requestIdStr,
|
|
818
|
+
meta.serverMessageIndex,
|
|
819
|
+
true,
|
|
820
|
+
true,
|
|
821
|
+
request,
|
|
822
|
+
meta.path,
|
|
823
|
+
meta.headers,
|
|
824
|
+
true
|
|
825
|
+
).then((adapter) => {
|
|
826
|
+
adapter.close(1e3, "ws.stale_metadata");
|
|
827
|
+
}).catch((err) => {
|
|
828
|
+
var _a2;
|
|
829
|
+
(_a2 = this.log) == null ? void 0 : _a2.error({
|
|
830
|
+
msg: "error creating stale websocket during restore",
|
|
831
|
+
requestId: requestIdStr,
|
|
832
|
+
error: stringifyError(err)
|
|
833
|
+
});
|
|
834
|
+
});
|
|
835
|
+
backgroundOperations.push(cleanupOperation);
|
|
836
|
+
loadedButNotConnectedCount++;
|
|
837
|
+
}
|
|
838
|
+
}
|
|
839
|
+
await Promise.allSettled(backgroundOperations);
|
|
840
|
+
actor.hibernationRestored = true;
|
|
841
|
+
(_d = this.log) == null ? void 0 : _d.info({
|
|
842
|
+
msg: "restored hibernatable websockets",
|
|
843
|
+
actorId,
|
|
844
|
+
restoredCount,
|
|
845
|
+
connectedButNotLoadedCount,
|
|
846
|
+
loadedButNotConnectedCount
|
|
847
|
+
});
|
|
848
|
+
}
|
|
849
|
+
/**
|
|
850
|
+
* Called from WebSocketOpen message and when restoring hibernatable WebSockets.
|
|
851
|
+
*
|
|
852
|
+
* engineAlreadyClosed will be true if this is only being called to trigger
|
|
853
|
+
* the close callback and not to send a close message to the server. This
|
|
854
|
+
* is used specifically to clean up zombie WebSocket connections.
|
|
855
|
+
*/
|
|
856
|
+
async #createWebSocket(actorId, gatewayId, requestId, requestIdStr, serverMessageIndex, isHibernatable, isRestoringHibernatable, request, path, headers, engineAlreadyClosed) {
|
|
857
|
+
var _a;
|
|
858
|
+
(_a = this.log) == null ? void 0 : _a.debug({
|
|
859
|
+
msg: "createWebSocket creating adapter",
|
|
860
|
+
actorId,
|
|
861
|
+
requestIdStr,
|
|
862
|
+
isHibernatable,
|
|
863
|
+
path
|
|
864
|
+
});
|
|
865
|
+
const adapter = new WebSocketTunnelAdapter(
|
|
866
|
+
this,
|
|
867
|
+
actorId,
|
|
868
|
+
requestIdStr,
|
|
869
|
+
serverMessageIndex,
|
|
870
|
+
isHibernatable,
|
|
871
|
+
isRestoringHibernatable,
|
|
872
|
+
request,
|
|
873
|
+
(data, isBinary) => {
|
|
874
|
+
const dataBuffer = typeof data === "string" ? new TextEncoder().encode(data).buffer : data;
|
|
875
|
+
this.#sendMessage(gatewayId, requestId, {
|
|
876
|
+
tag: "ToServerWebSocketMessage",
|
|
877
|
+
val: {
|
|
878
|
+
data: dataBuffer,
|
|
879
|
+
binary: isBinary
|
|
880
|
+
}
|
|
881
|
+
});
|
|
882
|
+
},
|
|
883
|
+
(code, reason) => {
|
|
884
|
+
if (!engineAlreadyClosed) {
|
|
885
|
+
this.#sendMessage(gatewayId, requestId, {
|
|
886
|
+
tag: "ToServerWebSocketClose",
|
|
887
|
+
val: {
|
|
888
|
+
code: code || null,
|
|
889
|
+
reason: reason || null,
|
|
890
|
+
hibernate: false
|
|
891
|
+
}
|
|
892
|
+
});
|
|
893
|
+
}
|
|
894
|
+
const actor2 = this.#runner.getActor(actorId);
|
|
895
|
+
if (actor2) {
|
|
896
|
+
actor2.deleteWebSocket(gatewayId, requestId);
|
|
897
|
+
actor2.deletePendingRequest(gatewayId, requestId);
|
|
898
|
+
}
|
|
899
|
+
this.#removeRequestToActor(gatewayId, requestId);
|
|
900
|
+
}
|
|
901
|
+
);
|
|
902
|
+
const actor = this.#runner.getActor(actorId);
|
|
903
|
+
if (!actor) {
|
|
904
|
+
throw new Error(`Actor ${actorId} not found`);
|
|
905
|
+
}
|
|
906
|
+
actor.setWebSocket(gatewayId, requestId, adapter);
|
|
907
|
+
this.addRequestToActor(gatewayId, requestId, actorId);
|
|
908
|
+
await this.#runner.config.websocket(
|
|
909
|
+
this.#runner,
|
|
910
|
+
actorId,
|
|
911
|
+
adapter.websocket,
|
|
912
|
+
gatewayId,
|
|
913
|
+
requestId,
|
|
914
|
+
request,
|
|
915
|
+
path,
|
|
916
|
+
headers,
|
|
917
|
+
isHibernatable,
|
|
918
|
+
isRestoringHibernatable
|
|
919
|
+
);
|
|
920
|
+
return adapter;
|
|
921
|
+
}
|
|
922
|
+
addRequestToActor(gatewayId, requestId, actorId) {
|
|
923
|
+
this.#requestToActor.push({ gatewayId, requestId, actorId });
|
|
924
|
+
}
|
|
925
|
+
#removeRequestToActor(gatewayId, requestId) {
|
|
926
|
+
const index = this.#requestToActor.findIndex(
|
|
927
|
+
(entry) => arraysEqual(entry.gatewayId, gatewayId) && arraysEqual(entry.requestId, requestId)
|
|
928
|
+
);
|
|
929
|
+
if (index !== -1) {
|
|
930
|
+
this.#requestToActor.splice(index, 1);
|
|
931
|
+
}
|
|
932
|
+
}
|
|
933
|
+
getRequestActor(gatewayId, requestId) {
|
|
934
|
+
var _a, _b;
|
|
935
|
+
const entry = this.#requestToActor.find(
|
|
936
|
+
(entry2) => arraysEqual(entry2.gatewayId, gatewayId) && arraysEqual(entry2.requestId, requestId)
|
|
937
|
+
);
|
|
938
|
+
if (!entry) {
|
|
939
|
+
(_a = this.log) == null ? void 0 : _a.warn({
|
|
940
|
+
msg: "missing requestToActor entry",
|
|
941
|
+
requestId: idToStr(requestId)
|
|
942
|
+
});
|
|
943
|
+
return void 0;
|
|
944
|
+
}
|
|
945
|
+
const actor = this.#runner.getActor(entry.actorId);
|
|
946
|
+
if (!actor) {
|
|
947
|
+
(_b = this.log) == null ? void 0 : _b.warn({
|
|
948
|
+
msg: "missing actor for requestToActor lookup",
|
|
949
|
+
requestId: idToStr(requestId),
|
|
950
|
+
actorId: entry.actorId
|
|
951
|
+
});
|
|
952
|
+
return void 0;
|
|
953
|
+
}
|
|
954
|
+
return actor;
|
|
955
|
+
}
|
|
956
|
+
async getAndWaitForRequestActor(gatewayId, requestId) {
|
|
957
|
+
const actor = this.getRequestActor(gatewayId, requestId);
|
|
958
|
+
if (!actor) return;
|
|
959
|
+
await actor.actorStartPromise.promise;
|
|
960
|
+
return actor;
|
|
961
|
+
}
|
|
962
|
+
#sendMessage(gatewayId, requestId, messageKind) {
|
|
963
|
+
var _a, _b, _c, _d;
|
|
964
|
+
if (!this.#runner.getPegboardWebSocketIfReady()) {
|
|
965
|
+
(_a = this.log) == null ? void 0 : _a.debug({
|
|
966
|
+
msg: "buffering tunnel message, socket not connected to engine",
|
|
967
|
+
requestId: idToStr(requestId),
|
|
968
|
+
message: stringifyToServerTunnelMessageKind(messageKind)
|
|
969
|
+
});
|
|
970
|
+
this.#bufferedMessages.push({ gatewayId, requestId, messageKind });
|
|
971
|
+
return;
|
|
972
|
+
}
|
|
973
|
+
const gatewayIdStr = idToStr(gatewayId);
|
|
974
|
+
const requestIdStr = idToStr(requestId);
|
|
975
|
+
const actor = this.getRequestActor(gatewayId, requestId);
|
|
976
|
+
if (!actor) {
|
|
977
|
+
(_b = this.log) == null ? void 0 : _b.warn({
|
|
978
|
+
msg: "cannot send tunnel message, actor not found",
|
|
979
|
+
gatewayId: gatewayIdStr,
|
|
980
|
+
requestId: requestIdStr
|
|
981
|
+
});
|
|
982
|
+
return;
|
|
983
|
+
}
|
|
984
|
+
let clientMessageIndex;
|
|
985
|
+
const pending = actor.getPendingRequest(gatewayId, requestId);
|
|
986
|
+
if (pending) {
|
|
987
|
+
clientMessageIndex = pending.clientMessageIndex;
|
|
988
|
+
pending.clientMessageIndex++;
|
|
989
|
+
} else {
|
|
990
|
+
(_c = this.log) == null ? void 0 : _c.warn({
|
|
991
|
+
msg: "missing pending request for send message, defaulting to message index 0",
|
|
992
|
+
gatewayId: gatewayIdStr,
|
|
993
|
+
requestId: requestIdStr
|
|
994
|
+
});
|
|
995
|
+
clientMessageIndex = 0;
|
|
996
|
+
}
|
|
997
|
+
const messageId = {
|
|
998
|
+
gatewayId,
|
|
999
|
+
requestId,
|
|
1000
|
+
messageIndex: clientMessageIndex
|
|
1001
|
+
};
|
|
1002
|
+
const messageIdStr = `${idToStr(messageId.gatewayId)}-${idToStr(messageId.requestId)}-${messageId.messageIndex}`;
|
|
1003
|
+
(_d = this.log) == null ? void 0 : _d.debug({
|
|
1004
|
+
msg: "sending tunnel msg",
|
|
1005
|
+
messageId: messageIdStr,
|
|
1006
|
+
gatewayId: gatewayIdStr,
|
|
1007
|
+
requestId: requestIdStr,
|
|
1008
|
+
messageIndex: clientMessageIndex,
|
|
1009
|
+
message: stringifyToServerTunnelMessageKind(messageKind)
|
|
1010
|
+
});
|
|
1011
|
+
const message = {
|
|
1012
|
+
tag: "ToServerTunnelMessage",
|
|
1013
|
+
val: {
|
|
1014
|
+
messageId,
|
|
1015
|
+
messageKind
|
|
1016
|
+
}
|
|
1017
|
+
};
|
|
1018
|
+
this.#runner.__sendToServer(message);
|
|
1019
|
+
}
|
|
1020
|
+
closeActiveRequests(actor) {
|
|
1021
|
+
const actorId = actor.actorId;
|
|
1022
|
+
for (const entry of actor.pendingRequests) {
|
|
1023
|
+
entry.request.reject(new Error(`Actor ${actorId} stopped`));
|
|
1024
|
+
if (entry.gatewayId && entry.requestId) {
|
|
1025
|
+
this.#removeRequestToActor(entry.gatewayId, entry.requestId);
|
|
1026
|
+
}
|
|
1027
|
+
}
|
|
1028
|
+
for (const entry of actor.webSockets) {
|
|
1029
|
+
const isHibernatable = entry.ws[HIBERNATABLE_SYMBOL];
|
|
1030
|
+
if (!isHibernatable) {
|
|
1031
|
+
entry.ws._closeWithoutCallback(1e3, "actor.stopped");
|
|
1032
|
+
}
|
|
1033
|
+
}
|
|
1034
|
+
}
|
|
1035
|
+
async #fetch(actorId, gatewayId, requestId, request) {
|
|
1036
|
+
var _a;
|
|
1037
|
+
if (!this.#runner.hasActor(actorId)) {
|
|
1038
|
+
(_a = this.log) == null ? void 0 : _a.warn({
|
|
1039
|
+
msg: "ignoring request for unknown actor",
|
|
1040
|
+
actorId
|
|
1041
|
+
});
|
|
1042
|
+
return new Response("Actor not found", {
|
|
1043
|
+
status: 503,
|
|
1044
|
+
headers: { "x-rivet-error": "runner.actor_not_found" }
|
|
1045
|
+
});
|
|
1046
|
+
}
|
|
1047
|
+
const fetchHandler = this.#runner.config.fetch(
|
|
1048
|
+
this.#runner,
|
|
1049
|
+
actorId,
|
|
1050
|
+
gatewayId,
|
|
1051
|
+
requestId,
|
|
1052
|
+
request
|
|
1053
|
+
);
|
|
1054
|
+
if (!fetchHandler) {
|
|
1055
|
+
return new Response("Not Implemented", { status: 501 });
|
|
1056
|
+
}
|
|
1057
|
+
return fetchHandler;
|
|
1058
|
+
}
|
|
1059
|
+
async handleTunnelMessage(message) {
|
|
1060
|
+
var _a;
|
|
1061
|
+
const { gatewayId, requestId, messageIndex } = message.messageId;
|
|
1062
|
+
const gatewayIdStr = idToStr(gatewayId);
|
|
1063
|
+
const requestIdStr = idToStr(requestId);
|
|
1064
|
+
(_a = this.log) == null ? void 0 : _a.debug({
|
|
1065
|
+
msg: "receive tunnel msg",
|
|
1066
|
+
gatewayId: gatewayIdStr,
|
|
1067
|
+
requestId: requestIdStr,
|
|
1068
|
+
messageIndex: message.messageId.messageIndex,
|
|
1069
|
+
message: stringifyToClientTunnelMessageKind(message.messageKind)
|
|
1070
|
+
});
|
|
1071
|
+
switch (message.messageKind.tag) {
|
|
1072
|
+
case "ToClientRequestStart":
|
|
1073
|
+
await this.#handleRequestStart(
|
|
1074
|
+
gatewayId,
|
|
1075
|
+
requestId,
|
|
1076
|
+
message.messageKind.val
|
|
1077
|
+
);
|
|
1078
|
+
break;
|
|
1079
|
+
case "ToClientRequestChunk":
|
|
1080
|
+
await this.#handleRequestChunk(
|
|
1081
|
+
gatewayId,
|
|
1082
|
+
requestId,
|
|
1083
|
+
message.messageKind.val
|
|
1084
|
+
);
|
|
1085
|
+
break;
|
|
1086
|
+
case "ToClientRequestAbort":
|
|
1087
|
+
await this.#handleRequestAbort(gatewayId, requestId);
|
|
1088
|
+
break;
|
|
1089
|
+
case "ToClientWebSocketOpen":
|
|
1090
|
+
await this.#handleWebSocketOpen(
|
|
1091
|
+
gatewayId,
|
|
1092
|
+
requestId,
|
|
1093
|
+
message.messageKind.val
|
|
1094
|
+
);
|
|
1095
|
+
break;
|
|
1096
|
+
case "ToClientWebSocketMessage": {
|
|
1097
|
+
await this.#handleWebSocketMessage(
|
|
1098
|
+
gatewayId,
|
|
1099
|
+
requestId,
|
|
1100
|
+
messageIndex,
|
|
1101
|
+
message.messageKind.val
|
|
1102
|
+
);
|
|
1103
|
+
break;
|
|
1104
|
+
}
|
|
1105
|
+
case "ToClientWebSocketClose":
|
|
1106
|
+
await this.#handleWebSocketClose(
|
|
1107
|
+
gatewayId,
|
|
1108
|
+
requestId,
|
|
1109
|
+
message.messageKind.val
|
|
1110
|
+
);
|
|
1111
|
+
break;
|
|
1112
|
+
default:
|
|
1113
|
+
unreachable(message.messageKind);
|
|
1114
|
+
}
|
|
1115
|
+
}
|
|
1116
|
+
async #handleRequestStart(gatewayId, requestId, req) {
|
|
1117
|
+
var _a, _b, _c;
|
|
1118
|
+
const requestIdStr = idToStr(requestId);
|
|
1119
|
+
const actor = await this.#runner.getAndWaitForActor(req.actorId);
|
|
1120
|
+
if (!actor) {
|
|
1121
|
+
(_a = this.log) == null ? void 0 : _a.warn({
|
|
1122
|
+
msg: "actor does not exist in handleRequestStart, request will leak",
|
|
1123
|
+
actorId: req.actorId,
|
|
1124
|
+
requestId: requestIdStr
|
|
1125
|
+
});
|
|
1126
|
+
return;
|
|
1127
|
+
}
|
|
1128
|
+
this.addRequestToActor(gatewayId, requestId, req.actorId);
|
|
1129
|
+
try {
|
|
1130
|
+
const headers = new Headers();
|
|
1131
|
+
for (const [key, value] of req.headers) {
|
|
1132
|
+
headers.append(key, value);
|
|
1133
|
+
}
|
|
1134
|
+
const request = new Request(`http://localhost${req.path}`, {
|
|
1135
|
+
method: req.method,
|
|
1136
|
+
headers,
|
|
1137
|
+
body: req.body ? new Uint8Array(req.body) : void 0
|
|
1138
|
+
});
|
|
1139
|
+
if (req.stream) {
|
|
1140
|
+
const stream = new ReadableStream({
|
|
1141
|
+
start: (controller) => {
|
|
1142
|
+
const existing = actor.getPendingRequest(
|
|
1143
|
+
gatewayId,
|
|
1144
|
+
requestId
|
|
1145
|
+
);
|
|
1146
|
+
if (existing) {
|
|
1147
|
+
existing.streamController = controller;
|
|
1148
|
+
existing.actorId = req.actorId;
|
|
1149
|
+
existing.gatewayId = gatewayId;
|
|
1150
|
+
existing.requestId = requestId;
|
|
1151
|
+
} else {
|
|
1152
|
+
actor.createPendingRequestWithStreamController(
|
|
1153
|
+
gatewayId,
|
|
1154
|
+
requestId,
|
|
1155
|
+
0,
|
|
1156
|
+
controller
|
|
1157
|
+
);
|
|
1158
|
+
}
|
|
1159
|
+
}
|
|
1160
|
+
});
|
|
1161
|
+
const streamingRequest = new Request(request, {
|
|
1162
|
+
body: stream,
|
|
1163
|
+
duplex: "half"
|
|
1164
|
+
});
|
|
1165
|
+
const response = await this.#fetch(
|
|
1166
|
+
req.actorId,
|
|
1167
|
+
gatewayId,
|
|
1168
|
+
requestId,
|
|
1169
|
+
streamingRequest
|
|
1170
|
+
);
|
|
1171
|
+
await this.#sendResponse(
|
|
1172
|
+
actor.actorId,
|
|
1173
|
+
actor.generation,
|
|
1174
|
+
gatewayId,
|
|
1175
|
+
requestId,
|
|
1176
|
+
response
|
|
1177
|
+
);
|
|
1178
|
+
} else {
|
|
1179
|
+
actor.createPendingRequest(gatewayId, requestId, 0);
|
|
1180
|
+
const response = await this.#fetch(
|
|
1181
|
+
req.actorId,
|
|
1182
|
+
gatewayId,
|
|
1183
|
+
requestId,
|
|
1184
|
+
request
|
|
1185
|
+
);
|
|
1186
|
+
await this.#sendResponse(
|
|
1187
|
+
actor.actorId,
|
|
1188
|
+
actor.generation,
|
|
1189
|
+
gatewayId,
|
|
1190
|
+
requestId,
|
|
1191
|
+
response
|
|
1192
|
+
);
|
|
1193
|
+
}
|
|
1194
|
+
} catch (error) {
|
|
1195
|
+
if (error instanceof RunnerShutdownError) {
|
|
1196
|
+
(_b = this.log) == null ? void 0 : _b.debug({ msg: "catught runner shutdown error" });
|
|
1197
|
+
} else {
|
|
1198
|
+
(_c = this.log) == null ? void 0 : _c.error({ msg: "error handling request", error });
|
|
1199
|
+
this.#sendResponseError(
|
|
1200
|
+
actor.actorId,
|
|
1201
|
+
actor.generation,
|
|
1202
|
+
gatewayId,
|
|
1203
|
+
requestId,
|
|
1204
|
+
500,
|
|
1205
|
+
"Internal Server Error"
|
|
1206
|
+
);
|
|
1207
|
+
}
|
|
1208
|
+
} finally {
|
|
1209
|
+
if (this.#runner.hasActor(req.actorId, actor.generation)) {
|
|
1210
|
+
actor.deletePendingRequest(gatewayId, requestId);
|
|
1211
|
+
this.#removeRequestToActor(gatewayId, requestId);
|
|
1212
|
+
}
|
|
1213
|
+
}
|
|
1214
|
+
}
|
|
1215
|
+
async #handleRequestChunk(gatewayId, requestId, chunk) {
|
|
1216
|
+
const actor = await this.getAndWaitForRequestActor(
|
|
1217
|
+
gatewayId,
|
|
1218
|
+
requestId
|
|
1219
|
+
);
|
|
1220
|
+
if (actor) {
|
|
1221
|
+
const pending = actor.getPendingRequest(gatewayId, requestId);
|
|
1222
|
+
if (pending == null ? void 0 : pending.streamController) {
|
|
1223
|
+
pending.streamController.enqueue(new Uint8Array(chunk.body));
|
|
1224
|
+
if (chunk.finish) {
|
|
1225
|
+
pending.streamController.close();
|
|
1226
|
+
actor.deletePendingRequest(gatewayId, requestId);
|
|
1227
|
+
this.#removeRequestToActor(gatewayId, requestId);
|
|
1228
|
+
}
|
|
1229
|
+
}
|
|
1230
|
+
}
|
|
1231
|
+
}
|
|
1232
|
+
async #handleRequestAbort(gatewayId, requestId) {
|
|
1233
|
+
const actor = await this.getAndWaitForRequestActor(
|
|
1234
|
+
gatewayId,
|
|
1235
|
+
requestId
|
|
1236
|
+
);
|
|
1237
|
+
if (actor) {
|
|
1238
|
+
const pending = actor.getPendingRequest(gatewayId, requestId);
|
|
1239
|
+
if (pending == null ? void 0 : pending.streamController) {
|
|
1240
|
+
pending.streamController.error(new Error("Request aborted"));
|
|
1241
|
+
}
|
|
1242
|
+
actor.deletePendingRequest(gatewayId, requestId);
|
|
1243
|
+
this.#removeRequestToActor(gatewayId, requestId);
|
|
1244
|
+
}
|
|
1245
|
+
}
|
|
1246
|
+
async #sendResponse(actorId, generation, gatewayId, requestId, response) {
|
|
1247
|
+
var _a;
|
|
1248
|
+
if (!this.#runner.hasActor(actorId, generation)) {
|
|
1249
|
+
(_a = this.log) == null ? void 0 : _a.warn({
|
|
1250
|
+
msg: "actor not loaded to send response, assuming gateway has closed request",
|
|
1251
|
+
actorId,
|
|
1252
|
+
generation,
|
|
1253
|
+
requestId
|
|
1254
|
+
});
|
|
1255
|
+
return;
|
|
1256
|
+
}
|
|
1257
|
+
const body = response.body ? await response.arrayBuffer() : null;
|
|
1258
|
+
const headers = /* @__PURE__ */ new Map();
|
|
1259
|
+
response.headers.forEach((value, key) => {
|
|
1260
|
+
headers.set(key, value);
|
|
1261
|
+
});
|
|
1262
|
+
if (body && !headers.has("content-length")) {
|
|
1263
|
+
headers.set("content-length", String(body.byteLength));
|
|
1264
|
+
}
|
|
1265
|
+
this.#sendMessage(gatewayId, requestId, {
|
|
1266
|
+
tag: "ToServerResponseStart",
|
|
1267
|
+
val: {
|
|
1268
|
+
status: response.status,
|
|
1269
|
+
headers,
|
|
1270
|
+
body: body || null,
|
|
1271
|
+
stream: false
|
|
1272
|
+
}
|
|
1273
|
+
});
|
|
1274
|
+
}
|
|
1275
|
+
#sendResponseError(actorId, generation, gatewayId, requestId, status, message) {
|
|
1276
|
+
var _a;
|
|
1277
|
+
if (!this.#runner.hasActor(actorId, generation)) {
|
|
1278
|
+
(_a = this.log) == null ? void 0 : _a.warn({
|
|
1279
|
+
msg: "actor not loaded to send response, assuming gateway has closed request",
|
|
1280
|
+
actorId,
|
|
1281
|
+
generation,
|
|
1282
|
+
requestId
|
|
1283
|
+
});
|
|
1284
|
+
return;
|
|
1285
|
+
}
|
|
1286
|
+
const headers = /* @__PURE__ */ new Map();
|
|
1287
|
+
headers.set("content-type", "text/plain");
|
|
1288
|
+
this.#sendMessage(gatewayId, requestId, {
|
|
1289
|
+
tag: "ToServerResponseStart",
|
|
1290
|
+
val: {
|
|
1291
|
+
status,
|
|
1292
|
+
headers,
|
|
1293
|
+
body: new TextEncoder().encode(message).buffer,
|
|
1294
|
+
stream: false
|
|
1295
|
+
}
|
|
1296
|
+
});
|
|
1297
|
+
}
|
|
1298
|
+
async #handleWebSocketOpen(gatewayId, requestId, open) {
|
|
1299
|
+
var _a, _b, _c;
|
|
1300
|
+
const requestIdStr = idToStr(requestId);
|
|
1301
|
+
const actor = await this.#runner.getAndWaitForActor(open.actorId);
|
|
1302
|
+
if (!actor) {
|
|
1303
|
+
(_a = this.log) == null ? void 0 : _a.warn({
|
|
1304
|
+
msg: "ignoring websocket for unknown actor",
|
|
1305
|
+
actorId: open.actorId
|
|
1306
|
+
});
|
|
1307
|
+
this.#sendMessage(gatewayId, requestId, {
|
|
1308
|
+
tag: "ToServerWebSocketClose",
|
|
1309
|
+
val: {
|
|
1310
|
+
code: 1011,
|
|
1311
|
+
reason: "Actor not found",
|
|
1312
|
+
hibernate: false
|
|
1313
|
+
}
|
|
1314
|
+
});
|
|
1315
|
+
return;
|
|
1316
|
+
}
|
|
1317
|
+
const existingAdapter = actor.getWebSocket(gatewayId, requestId);
|
|
1318
|
+
if (existingAdapter) {
|
|
1319
|
+
(_b = this.log) == null ? void 0 : _b.warn({
|
|
1320
|
+
msg: "closing existing websocket for duplicate open event for the same request id",
|
|
1321
|
+
requestId: requestIdStr
|
|
1322
|
+
});
|
|
1323
|
+
existingAdapter._closeWithoutCallback(1e3, "ws.duplicate_open");
|
|
1324
|
+
}
|
|
1325
|
+
try {
|
|
1326
|
+
const request = buildRequestForWebSocket(
|
|
1327
|
+
open.path,
|
|
1328
|
+
Object.fromEntries(open.headers)
|
|
1329
|
+
);
|
|
1330
|
+
const canHibernate = this.#runner.config.hibernatableWebSocket.canHibernate(
|
|
1331
|
+
actor.actorId,
|
|
1332
|
+
gatewayId,
|
|
1333
|
+
requestId,
|
|
1334
|
+
request
|
|
1335
|
+
);
|
|
1336
|
+
const adapter = await this.#createWebSocket(
|
|
1337
|
+
actor.actorId,
|
|
1338
|
+
gatewayId,
|
|
1339
|
+
requestId,
|
|
1340
|
+
requestIdStr,
|
|
1341
|
+
0,
|
|
1342
|
+
canHibernate,
|
|
1343
|
+
false,
|
|
1344
|
+
request,
|
|
1345
|
+
open.path,
|
|
1346
|
+
Object.fromEntries(open.headers),
|
|
1347
|
+
false
|
|
1348
|
+
);
|
|
1349
|
+
actor.createPendingRequest(gatewayId, requestId, 0);
|
|
1350
|
+
this.#sendMessage(gatewayId, requestId, {
|
|
1351
|
+
tag: "ToServerWebSocketOpen",
|
|
1352
|
+
val: {
|
|
1353
|
+
canHibernate
|
|
1354
|
+
}
|
|
1355
|
+
});
|
|
1356
|
+
adapter._handleOpen(requestId);
|
|
1357
|
+
} catch (error) {
|
|
1358
|
+
(_c = this.log) == null ? void 0 : _c.error({ msg: "error handling websocket open", error });
|
|
1359
|
+
this.#sendMessage(gatewayId, requestId, {
|
|
1360
|
+
tag: "ToServerWebSocketClose",
|
|
1361
|
+
val: {
|
|
1362
|
+
code: 1011,
|
|
1363
|
+
reason: "Server Error",
|
|
1364
|
+
hibernate: false
|
|
1365
|
+
}
|
|
1366
|
+
});
|
|
1367
|
+
actor.deleteWebSocket(gatewayId, requestId);
|
|
1368
|
+
actor.deletePendingRequest(gatewayId, requestId);
|
|
1369
|
+
this.#removeRequestToActor(gatewayId, requestId);
|
|
1370
|
+
}
|
|
1371
|
+
}
|
|
1372
|
+
async #handleWebSocketMessage(gatewayId, requestId, serverMessageIndex, msg) {
|
|
1373
|
+
var _a;
|
|
1374
|
+
const actor = await this.getAndWaitForRequestActor(
|
|
1375
|
+
gatewayId,
|
|
1376
|
+
requestId
|
|
1377
|
+
);
|
|
1378
|
+
if (actor) {
|
|
1379
|
+
const adapter = actor.getWebSocket(gatewayId, requestId);
|
|
1380
|
+
if (adapter) {
|
|
1381
|
+
const data = msg.binary ? new Uint8Array(msg.data) : new TextDecoder().decode(new Uint8Array(msg.data));
|
|
1382
|
+
adapter._handleMessage(
|
|
1383
|
+
requestId,
|
|
1384
|
+
data,
|
|
1385
|
+
serverMessageIndex,
|
|
1386
|
+
msg.binary
|
|
1387
|
+
);
|
|
1388
|
+
return;
|
|
1389
|
+
}
|
|
1390
|
+
}
|
|
1391
|
+
(_a = this.log) == null ? void 0 : _a.warn({
|
|
1392
|
+
msg: "missing websocket for incoming websocket message, this may indicate the actor stopped before processing a message",
|
|
1393
|
+
requestId
|
|
1394
|
+
});
|
|
1395
|
+
}
|
|
1396
|
+
sendHibernatableWebSocketMessageAck(gatewayId, requestId, clientMessageIndex) {
|
|
1397
|
+
var _a, _b, _c;
|
|
1398
|
+
const requestIdStr = idToStr(requestId);
|
|
1399
|
+
(_a = this.log) == null ? void 0 : _a.debug({
|
|
1400
|
+
msg: "ack ws msg",
|
|
1401
|
+
requestId: requestIdStr,
|
|
1402
|
+
index: clientMessageIndex
|
|
1403
|
+
});
|
|
1404
|
+
if (clientMessageIndex < 0 || clientMessageIndex > 65535)
|
|
1405
|
+
throw new Error("invalid websocket ack index");
|
|
1406
|
+
const actor = this.getRequestActor(gatewayId, requestId);
|
|
1407
|
+
if (!actor) {
|
|
1408
|
+
(_b = this.log) == null ? void 0 : _b.warn({
|
|
1409
|
+
msg: "cannot send websocket ack, actor not found",
|
|
1410
|
+
requestId: requestIdStr
|
|
1411
|
+
});
|
|
1412
|
+
return;
|
|
1413
|
+
}
|
|
1414
|
+
const pending = actor.getPendingRequest(gatewayId, requestId);
|
|
1415
|
+
if (!(pending == null ? void 0 : pending.gatewayId)) {
|
|
1416
|
+
(_c = this.log) == null ? void 0 : _c.warn({
|
|
1417
|
+
msg: "cannot send websocket ack, gatewayId not found in pending request",
|
|
1418
|
+
requestId: requestIdStr
|
|
1419
|
+
});
|
|
1420
|
+
return;
|
|
1421
|
+
}
|
|
1422
|
+
this.#sendMessage(pending.gatewayId, requestId, {
|
|
1423
|
+
tag: "ToServerWebSocketMessageAck",
|
|
1424
|
+
val: {
|
|
1425
|
+
index: clientMessageIndex
|
|
1426
|
+
}
|
|
1427
|
+
});
|
|
1428
|
+
}
|
|
1429
|
+
async #handleWebSocketClose(gatewayId, requestId, close) {
|
|
1430
|
+
const actor = await this.getAndWaitForRequestActor(
|
|
1431
|
+
gatewayId,
|
|
1432
|
+
requestId
|
|
1433
|
+
);
|
|
1434
|
+
if (actor) {
|
|
1435
|
+
const adapter = actor.getWebSocket(gatewayId, requestId);
|
|
1436
|
+
if (adapter) {
|
|
1437
|
+
adapter._handleClose(
|
|
1438
|
+
requestId,
|
|
1439
|
+
close.code || void 0,
|
|
1440
|
+
close.reason || void 0
|
|
1441
|
+
);
|
|
1442
|
+
actor.deleteWebSocket(gatewayId, requestId);
|
|
1443
|
+
actor.deletePendingRequest(gatewayId, requestId);
|
|
1444
|
+
this.#removeRequestToActor(gatewayId, requestId);
|
|
1445
|
+
}
|
|
1446
|
+
}
|
|
1447
|
+
}
|
|
1448
|
+
};
|
|
1449
|
+
function buildRequestForWebSocket(path, headers) {
|
|
1450
|
+
const fullHeaders = {
|
|
1451
|
+
...headers,
|
|
1452
|
+
Upgrade: "websocket",
|
|
1453
|
+
Connection: "Upgrade"
|
|
1454
|
+
};
|
|
1455
|
+
if (!path.startsWith("/")) {
|
|
1456
|
+
throw new Error("path must start with leading slash");
|
|
1457
|
+
}
|
|
1458
|
+
const request = new Request(`http://actor${path}`, {
|
|
1459
|
+
method: "GET",
|
|
1460
|
+
headers: fullHeaders
|
|
1461
|
+
});
|
|
1462
|
+
return request;
|
|
1463
|
+
}
|
|
1464
|
+
|
|
1465
|
+
// src/websocket.ts
|
|
1466
|
+
var webSocketPromise = null;
|
|
1467
|
+
async function importWebSocket() {
|
|
1468
|
+
if (webSocketPromise !== null) {
|
|
1469
|
+
return webSocketPromise;
|
|
1470
|
+
}
|
|
1471
|
+
webSocketPromise = (async () => {
|
|
1472
|
+
var _a, _b, _c;
|
|
1473
|
+
let _WebSocket;
|
|
1474
|
+
if (typeof WebSocket !== "undefined") {
|
|
1475
|
+
_WebSocket = WebSocket;
|
|
1476
|
+
(_a = logger()) == null ? void 0 : _a.debug({ msg: "using native websocket" });
|
|
1477
|
+
} else {
|
|
1478
|
+
try {
|
|
1479
|
+
const ws = await Promise.resolve().then(() => _interopRequireWildcard(require("ws")));
|
|
1480
|
+
_WebSocket = ws.default;
|
|
1481
|
+
(_b = logger()) == null ? void 0 : _b.debug({ msg: "using websocket from npm" });
|
|
1482
|
+
} catch (e3) {
|
|
1483
|
+
_WebSocket = class MockWebSocket {
|
|
1484
|
+
constructor() {
|
|
1485
|
+
throw new Error(
|
|
1486
|
+
'WebSocket support requires installing the "ws" peer dependency.'
|
|
1487
|
+
);
|
|
1488
|
+
}
|
|
1489
|
+
};
|
|
1490
|
+
(_c = logger()) == null ? void 0 : _c.debug({ msg: "using mock websocket" });
|
|
1491
|
+
}
|
|
1492
|
+
}
|
|
1493
|
+
return _WebSocket;
|
|
1494
|
+
})();
|
|
1495
|
+
return webSocketPromise;
|
|
1496
|
+
}
|
|
1497
|
+
|
|
1498
|
+
// src/mod.ts
|
|
1499
|
+
var KV_EXPIRE = 3e4;
|
|
1500
|
+
var PROTOCOL_VERSION = 5;
|
|
1501
|
+
var EVENT_BACKLOG_WARN_THRESHOLD = 1e4;
|
|
1502
|
+
var SIGNAL_HANDLERS = [];
|
|
1503
|
+
var RunnerShutdownError = class extends Error {
|
|
1504
|
+
constructor() {
|
|
1505
|
+
super("Runner shut down");
|
|
1506
|
+
}
|
|
1507
|
+
};
|
|
1508
|
+
var Runner = class {
|
|
1509
|
+
#config;
|
|
1510
|
+
get config() {
|
|
1511
|
+
return this.#config;
|
|
1512
|
+
}
|
|
1513
|
+
#actors = /* @__PURE__ */ new Map();
|
|
1514
|
+
// WebSocket
|
|
1515
|
+
#pegboardWebSocket;
|
|
1516
|
+
|
|
1517
|
+
#started = false;
|
|
1518
|
+
#shutdown = false;
|
|
1519
|
+
#reconnectAttempt = 0;
|
|
1520
|
+
#reconnectTimeout;
|
|
1521
|
+
// Runner lost threshold management
|
|
1522
|
+
#runnerLostThreshold;
|
|
1523
|
+
#runnerLostTimeout;
|
|
1524
|
+
// Event storage for resending
|
|
1525
|
+
#eventBacklogWarned = false;
|
|
1526
|
+
// Command acknowledgment
|
|
1527
|
+
#ackInterval;
|
|
1528
|
+
// KV operations
|
|
1529
|
+
#nextKvRequestId = 0;
|
|
1530
|
+
#kvRequests = /* @__PURE__ */ new Map();
|
|
1531
|
+
#kvCleanupInterval;
|
|
1532
|
+
// Tunnel for HTTP/WebSocket forwarding
|
|
1533
|
+
#tunnel;
|
|
1534
|
+
// Cached child logger with runner-specific attributes
|
|
1535
|
+
#logCached;
|
|
1536
|
+
get log() {
|
|
1537
|
+
if (this.#logCached) return this.#logCached;
|
|
1538
|
+
const l = logger();
|
|
1539
|
+
if (l) {
|
|
1540
|
+
if (this.runnerId) {
|
|
1541
|
+
this.#logCached = l.child({
|
|
1542
|
+
runnerId: this.runnerId
|
|
1543
|
+
});
|
|
1544
|
+
return this.#logCached;
|
|
1545
|
+
} else {
|
|
1546
|
+
return l;
|
|
1547
|
+
}
|
|
1548
|
+
}
|
|
1549
|
+
return void 0;
|
|
1550
|
+
}
|
|
1551
|
+
constructor(config) {
|
|
1552
|
+
this.#config = config;
|
|
1553
|
+
if (this.#config.logger) setLogger(this.#config.logger);
|
|
1554
|
+
this.#kvCleanupInterval = setInterval(() => {
|
|
1555
|
+
var _a;
|
|
1556
|
+
try {
|
|
1557
|
+
this.#cleanupOldKvRequests();
|
|
1558
|
+
} catch (err) {
|
|
1559
|
+
(_a = this.log) == null ? void 0 : _a.error({
|
|
1560
|
+
msg: "error cleaning up kv requests",
|
|
1561
|
+
error: stringifyError(err)
|
|
1562
|
+
});
|
|
1563
|
+
}
|
|
1564
|
+
}, 15e3);
|
|
1565
|
+
}
|
|
1566
|
+
// MARK: Manage actors
|
|
1567
|
+
sleepActor(actorId, generation) {
|
|
1568
|
+
const actor = this.getActor(actorId, generation);
|
|
1569
|
+
if (!actor) return;
|
|
1570
|
+
this.#sendActorIntent(actorId, actor.generation, "sleep");
|
|
1571
|
+
}
|
|
1572
|
+
async stopActor(actorId, generation) {
|
|
1573
|
+
const actor = this.getActor(actorId, generation);
|
|
1574
|
+
if (!actor) return;
|
|
1575
|
+
this.#sendActorIntent(actorId, actor.generation, "stop");
|
|
1576
|
+
}
|
|
1577
|
+
async forceStopActor(actorId, generation) {
|
|
1578
|
+
var _a, _b;
|
|
1579
|
+
(_a = this.log) == null ? void 0 : _a.debug({
|
|
1580
|
+
msg: "force stopping actor",
|
|
1581
|
+
actorId
|
|
1582
|
+
});
|
|
1583
|
+
const actor = this.getActor(actorId, generation);
|
|
1584
|
+
if (!actor) return;
|
|
1585
|
+
try {
|
|
1586
|
+
await this.#config.onActorStop(actorId, actor.generation);
|
|
1587
|
+
} catch (err) {
|
|
1588
|
+
console.error(`Error in onActorStop for actor ${actorId}:`, err);
|
|
1589
|
+
}
|
|
1590
|
+
(_b = this.#tunnel) == null ? void 0 : _b.closeActiveRequests(actor);
|
|
1591
|
+
this.#sendActorStateUpdate(actorId, actor.generation, "stopped");
|
|
1592
|
+
this.#removeActor(actorId, generation);
|
|
1593
|
+
}
|
|
1594
|
+
#handleLost() {
|
|
1595
|
+
var _a;
|
|
1596
|
+
(_a = this.log) == null ? void 0 : _a.info({
|
|
1597
|
+
msg: "stopping all actors due to runner lost threshold"
|
|
1598
|
+
});
|
|
1599
|
+
for (const [_, request] of this.#kvRequests.entries()) {
|
|
1600
|
+
request.reject(new RunnerShutdownError());
|
|
1601
|
+
}
|
|
1602
|
+
this.#kvRequests.clear();
|
|
1603
|
+
this.#stopAllActors();
|
|
1604
|
+
}
|
|
1605
|
+
#stopAllActors() {
|
|
1606
|
+
const actorIds = Array.from(this.#actors.keys());
|
|
1607
|
+
for (const actorId of actorIds) {
|
|
1608
|
+
this.forceStopActor(actorId).catch((err) => {
|
|
1609
|
+
var _a;
|
|
1610
|
+
(_a = this.log) == null ? void 0 : _a.error({
|
|
1611
|
+
msg: "error stopping actor",
|
|
1612
|
+
actorId,
|
|
1613
|
+
error: stringifyError(err)
|
|
1614
|
+
});
|
|
1615
|
+
});
|
|
1616
|
+
}
|
|
1617
|
+
}
|
|
1618
|
+
getActor(actorId, generation) {
|
|
1619
|
+
var _a, _b;
|
|
1620
|
+
const actor = this.#actors.get(actorId);
|
|
1621
|
+
if (!actor) {
|
|
1622
|
+
(_a = this.log) == null ? void 0 : _a.warn({
|
|
1623
|
+
msg: "actor not found",
|
|
1624
|
+
actorId
|
|
1625
|
+
});
|
|
1626
|
+
return void 0;
|
|
1627
|
+
}
|
|
1628
|
+
if (generation !== void 0 && actor.generation !== generation) {
|
|
1629
|
+
(_b = this.log) == null ? void 0 : _b.warn({
|
|
1630
|
+
msg: "actor generation mismatch",
|
|
1631
|
+
actorId,
|
|
1632
|
+
generation
|
|
1633
|
+
});
|
|
1634
|
+
return void 0;
|
|
1635
|
+
}
|
|
1636
|
+
return actor;
|
|
1637
|
+
}
|
|
1638
|
+
async getAndWaitForActor(actorId, generation) {
|
|
1639
|
+
const actor = this.getActor(actorId, generation);
|
|
1640
|
+
if (!actor) return;
|
|
1641
|
+
await actor.actorStartPromise.promise;
|
|
1642
|
+
return actor;
|
|
1643
|
+
}
|
|
1644
|
+
hasActor(actorId, generation) {
|
|
1645
|
+
const actor = this.#actors.get(actorId);
|
|
1646
|
+
return !!actor && (generation === void 0 || actor.generation === generation);
|
|
1647
|
+
}
|
|
1648
|
+
get actors() {
|
|
1649
|
+
return this.#actors;
|
|
1650
|
+
}
|
|
1651
|
+
// IMPORTANT: Make sure to call stopActiveRequests if calling #removeActor
|
|
1652
|
+
#removeActor(actorId, generation) {
|
|
1653
|
+
var _a, _b, _c;
|
|
1654
|
+
const actor = this.#actors.get(actorId);
|
|
1655
|
+
if (!actor) {
|
|
1656
|
+
(_a = this.log) == null ? void 0 : _a.error({
|
|
1657
|
+
msg: "actor not found for removal",
|
|
1658
|
+
actorId
|
|
1659
|
+
});
|
|
1660
|
+
return void 0;
|
|
1661
|
+
}
|
|
1662
|
+
if (generation !== void 0 && actor.generation !== generation) {
|
|
1663
|
+
(_b = this.log) == null ? void 0 : _b.error({
|
|
1664
|
+
msg: "actor generation mismatch",
|
|
1665
|
+
actorId,
|
|
1666
|
+
generation
|
|
1667
|
+
});
|
|
1668
|
+
return void 0;
|
|
1669
|
+
}
|
|
1670
|
+
this.#actors.delete(actorId);
|
|
1671
|
+
(_c = this.log) == null ? void 0 : _c.info({
|
|
1672
|
+
msg: "removed actor",
|
|
1673
|
+
actorId,
|
|
1674
|
+
actors: this.#actors.size
|
|
1675
|
+
});
|
|
1676
|
+
return actor;
|
|
1677
|
+
}
|
|
1678
|
+
// MARK: Start
|
|
1679
|
+
async start() {
|
|
1680
|
+
var _a, _b;
|
|
1681
|
+
if (this.#started) throw new Error("Cannot call runner.start twice");
|
|
1682
|
+
this.#started = true;
|
|
1683
|
+
(_a = this.log) == null ? void 0 : _a.info({ msg: "starting runner" });
|
|
1684
|
+
this.#tunnel = new Tunnel(this);
|
|
1685
|
+
this.#tunnel.start();
|
|
1686
|
+
try {
|
|
1687
|
+
await this.#openPegboardWebSocket();
|
|
1688
|
+
} catch (error) {
|
|
1689
|
+
this.#started = false;
|
|
1690
|
+
throw error;
|
|
1691
|
+
}
|
|
1692
|
+
if (!this.#config.noAutoShutdown) {
|
|
1693
|
+
if (!SIGNAL_HANDLERS.length) {
|
|
1694
|
+
process.on("SIGTERM", async () => {
|
|
1695
|
+
var _a2;
|
|
1696
|
+
(_a2 = this.log) == null ? void 0 : _a2.debug("received SIGTERM");
|
|
1697
|
+
for (const handler of SIGNAL_HANDLERS) {
|
|
1698
|
+
await handler();
|
|
1699
|
+
}
|
|
1700
|
+
});
|
|
1701
|
+
process.on("SIGINT", async () => {
|
|
1702
|
+
var _a2;
|
|
1703
|
+
(_a2 = this.log) == null ? void 0 : _a2.debug("received SIGINT");
|
|
1704
|
+
for (const handler of SIGNAL_HANDLERS) {
|
|
1705
|
+
await handler();
|
|
1706
|
+
}
|
|
1707
|
+
});
|
|
1708
|
+
(_b = this.log) == null ? void 0 : _b.debug({
|
|
1709
|
+
msg: "added SIGTERM listeners"
|
|
1710
|
+
});
|
|
1711
|
+
}
|
|
1712
|
+
SIGNAL_HANDLERS.push(async () => {
|
|
1713
|
+
var _a2;
|
|
1714
|
+
const weak = new WeakRef(this);
|
|
1715
|
+
await ((_a2 = weak.deref()) == null ? void 0 : _a2.shutdown(false, false));
|
|
1716
|
+
});
|
|
1717
|
+
}
|
|
1718
|
+
}
|
|
1719
|
+
// MARK: Shutdown
|
|
1720
|
+
async shutdown(immediate, exit = false) {
|
|
1721
|
+
var _a, _b, _c, _d, _e, _f, _g, _h;
|
|
1722
|
+
if (this.#shutdown) {
|
|
1723
|
+
(_a = this.log) == null ? void 0 : _a.debug({
|
|
1724
|
+
msg: "shutdown already in progress, ignoring"
|
|
1725
|
+
});
|
|
1726
|
+
return;
|
|
1727
|
+
}
|
|
1728
|
+
this.#shutdown = true;
|
|
1729
|
+
(_b = this.log) == null ? void 0 : _b.info({
|
|
1730
|
+
msg: "starting shutdown",
|
|
1731
|
+
immediate,
|
|
1732
|
+
exit
|
|
1733
|
+
});
|
|
1734
|
+
if (this.#reconnectTimeout) {
|
|
1735
|
+
clearTimeout(this.#reconnectTimeout);
|
|
1736
|
+
this.#reconnectTimeout = void 0;
|
|
1737
|
+
}
|
|
1738
|
+
if (this.#runnerLostTimeout) {
|
|
1739
|
+
clearTimeout(this.#runnerLostTimeout);
|
|
1740
|
+
this.#runnerLostTimeout = void 0;
|
|
1741
|
+
}
|
|
1742
|
+
if (this.#ackInterval) {
|
|
1743
|
+
clearInterval(this.#ackInterval);
|
|
1744
|
+
this.#ackInterval = void 0;
|
|
1745
|
+
}
|
|
1746
|
+
if (this.#kvCleanupInterval) {
|
|
1747
|
+
clearInterval(this.#kvCleanupInterval);
|
|
1748
|
+
this.#kvCleanupInterval = void 0;
|
|
1749
|
+
}
|
|
1750
|
+
for (const request of this.#kvRequests.values()) {
|
|
1751
|
+
request.reject(
|
|
1752
|
+
new Error("WebSocket connection closed during shutdown")
|
|
1753
|
+
);
|
|
1754
|
+
}
|
|
1755
|
+
this.#kvRequests.clear();
|
|
1756
|
+
const pegboardWebSocket = this.getPegboardWebSocketIfReady();
|
|
1757
|
+
if (pegboardWebSocket) {
|
|
1758
|
+
if (immediate) {
|
|
1759
|
+
pegboardWebSocket.close(1e3, "pegboard.runner_shutdown");
|
|
1760
|
+
} else {
|
|
1761
|
+
try {
|
|
1762
|
+
(_c = this.log) == null ? void 0 : _c.info({
|
|
1763
|
+
msg: "sending stopping message",
|
|
1764
|
+
readyState: pegboardWebSocket.readyState
|
|
1765
|
+
});
|
|
1766
|
+
this.__sendToServer({
|
|
1767
|
+
tag: "ToServerStopping",
|
|
1768
|
+
val: null
|
|
1769
|
+
});
|
|
1770
|
+
const closePromise = new Promise((resolve) => {
|
|
1771
|
+
if (!pegboardWebSocket)
|
|
1772
|
+
throw new Error("missing pegboardWebSocket");
|
|
1773
|
+
pegboardWebSocket.addEventListener("close", (ev) => {
|
|
1774
|
+
var _a2;
|
|
1775
|
+
(_a2 = this.log) == null ? void 0 : _a2.info({
|
|
1776
|
+
msg: "connection closed",
|
|
1777
|
+
code: ev.code,
|
|
1778
|
+
reason: ev.reason.toString()
|
|
1779
|
+
});
|
|
1780
|
+
resolve();
|
|
1781
|
+
});
|
|
1782
|
+
});
|
|
1783
|
+
await this.#waitForActorsToStop(pegboardWebSocket);
|
|
1784
|
+
(_d = this.log) == null ? void 0 : _d.info({
|
|
1785
|
+
msg: "closing WebSocket"
|
|
1786
|
+
});
|
|
1787
|
+
pegboardWebSocket.close(1e3, "pegboard.runner_shutdown");
|
|
1788
|
+
await closePromise;
|
|
1789
|
+
(_e = this.log) == null ? void 0 : _e.info({
|
|
1790
|
+
msg: "websocket shutdown completed"
|
|
1791
|
+
});
|
|
1792
|
+
} catch (error) {
|
|
1793
|
+
(_f = this.log) == null ? void 0 : _f.error({
|
|
1794
|
+
msg: "error during websocket shutdown:",
|
|
1795
|
+
error
|
|
1796
|
+
});
|
|
1797
|
+
pegboardWebSocket.close();
|
|
1798
|
+
}
|
|
1799
|
+
}
|
|
1800
|
+
} else {
|
|
1801
|
+
(_h = this.log) == null ? void 0 : _h.debug({
|
|
1802
|
+
msg: "no runner WebSocket to shutdown or already closed",
|
|
1803
|
+
readyState: (_g = this.#pegboardWebSocket) == null ? void 0 : _g.readyState
|
|
1804
|
+
});
|
|
1805
|
+
}
|
|
1806
|
+
if (this.#tunnel) {
|
|
1807
|
+
this.#tunnel.shutdown();
|
|
1808
|
+
this.#tunnel = void 0;
|
|
1809
|
+
}
|
|
1810
|
+
this.#config.onShutdown();
|
|
1811
|
+
if (exit) process.exit(0);
|
|
1812
|
+
}
|
|
1813
|
+
/**
|
|
1814
|
+
* Wait for all actors to stop before proceeding with shutdown.
|
|
1815
|
+
*
|
|
1816
|
+
* This method polls every 100ms to check if all actors have been stopped.
|
|
1817
|
+
*
|
|
1818
|
+
* It will resolve early if:
|
|
1819
|
+
* - All actors are stopped
|
|
1820
|
+
* - The WebSocket connection is closed
|
|
1821
|
+
* - The shutdown timeout is reached (120 seconds)
|
|
1822
|
+
*/
|
|
1823
|
+
async #waitForActorsToStop(ws) {
|
|
1824
|
+
const shutdownTimeout = 12e4;
|
|
1825
|
+
const shutdownCheckInterval = 100;
|
|
1826
|
+
const progressLogInterval = 5e3;
|
|
1827
|
+
const shutdownStartTs = Date.now();
|
|
1828
|
+
let lastProgressLogTs = 0;
|
|
1829
|
+
return new Promise((resolve) => {
|
|
1830
|
+
var _a, _b;
|
|
1831
|
+
const checkActors = () => {
|
|
1832
|
+
var _a2, _b2, _c, _d;
|
|
1833
|
+
const now = Date.now();
|
|
1834
|
+
const elapsed = now - shutdownStartTs;
|
|
1835
|
+
const wsIsClosed = ws.readyState === 2 || ws.readyState === 3;
|
|
1836
|
+
if (this.#actors.size === 0) {
|
|
1837
|
+
(_a2 = this.log) == null ? void 0 : _a2.info({
|
|
1838
|
+
msg: "all actors stopped",
|
|
1839
|
+
elapsed
|
|
1840
|
+
});
|
|
1841
|
+
return true;
|
|
1842
|
+
} else if (wsIsClosed) {
|
|
1843
|
+
(_b2 = this.log) == null ? void 0 : _b2.warn({
|
|
1844
|
+
msg: "websocket closed before all actors stopped",
|
|
1845
|
+
remainingActors: this.#actors.size,
|
|
1846
|
+
elapsed
|
|
1847
|
+
});
|
|
1848
|
+
return true;
|
|
1849
|
+
} else if (elapsed >= shutdownTimeout) {
|
|
1850
|
+
(_c = this.log) == null ? void 0 : _c.warn({
|
|
1851
|
+
msg: "shutdown timeout reached, forcing close",
|
|
1852
|
+
remainingActors: this.#actors.size,
|
|
1853
|
+
elapsed
|
|
1854
|
+
});
|
|
1855
|
+
return true;
|
|
1856
|
+
} else {
|
|
1857
|
+
if (now - lastProgressLogTs >= progressLogInterval) {
|
|
1858
|
+
(_d = this.log) == null ? void 0 : _d.info({
|
|
1859
|
+
msg: "waiting for actors to stop",
|
|
1860
|
+
remainingActors: this.#actors.size,
|
|
1861
|
+
elapsed
|
|
1862
|
+
});
|
|
1863
|
+
lastProgressLogTs = now;
|
|
1864
|
+
}
|
|
1865
|
+
return false;
|
|
1866
|
+
}
|
|
1867
|
+
};
|
|
1868
|
+
if (checkActors()) {
|
|
1869
|
+
(_a = this.log) == null ? void 0 : _a.debug({
|
|
1870
|
+
msg: "actors check completed immediately"
|
|
1871
|
+
});
|
|
1872
|
+
resolve();
|
|
1873
|
+
return;
|
|
1874
|
+
}
|
|
1875
|
+
(_b = this.log) == null ? void 0 : _b.debug({
|
|
1876
|
+
msg: "starting actor wait interval",
|
|
1877
|
+
checkInterval: shutdownCheckInterval
|
|
1878
|
+
});
|
|
1879
|
+
const interval = setInterval(() => {
|
|
1880
|
+
var _a2, _b2;
|
|
1881
|
+
(_a2 = this.log) == null ? void 0 : _a2.debug({
|
|
1882
|
+
msg: "actor wait interval tick",
|
|
1883
|
+
actorCount: this.#actors.size
|
|
1884
|
+
});
|
|
1885
|
+
if (checkActors()) {
|
|
1886
|
+
(_b2 = this.log) == null ? void 0 : _b2.debug({
|
|
1887
|
+
msg: "actors check completed, clearing interval"
|
|
1888
|
+
});
|
|
1889
|
+
clearInterval(interval);
|
|
1890
|
+
resolve();
|
|
1891
|
+
}
|
|
1892
|
+
}, shutdownCheckInterval);
|
|
1893
|
+
});
|
|
1894
|
+
}
|
|
1895
|
+
// MARK: Networking
|
|
1896
|
+
get pegboardEndpoint() {
|
|
1897
|
+
return this.#config.pegboardEndpoint || this.#config.endpoint;
|
|
1898
|
+
}
|
|
1899
|
+
get pegboardUrl() {
|
|
1900
|
+
const wsEndpoint = this.pegboardEndpoint.replace("http://", "ws://").replace("https://", "wss://");
|
|
1901
|
+
const baseUrl = wsEndpoint.endsWith("/") ? wsEndpoint.slice(0, -1) : wsEndpoint;
|
|
1902
|
+
return `${baseUrl}/runners/connect?protocol_version=${PROTOCOL_VERSION}&namespace=${encodeURIComponent(this.#config.namespace)}&runner_key=${encodeURIComponent(this.#config.runnerKey)}`;
|
|
1903
|
+
}
|
|
1904
|
+
// MARK: Runner protocol
|
|
1905
|
+
async #openPegboardWebSocket() {
|
|
1906
|
+
var _a, _b;
|
|
1907
|
+
const protocols = ["rivet"];
|
|
1908
|
+
if (this.config.token)
|
|
1909
|
+
protocols.push(`rivet_token.${this.config.token}`);
|
|
1910
|
+
const WS = await importWebSocket();
|
|
1911
|
+
if (this.#pegboardWebSocket && (this.#pegboardWebSocket.readyState === WS.CONNECTING || this.#pegboardWebSocket.readyState === WS.OPEN)) {
|
|
1912
|
+
(_a = this.log) == null ? void 0 : _a.error(
|
|
1913
|
+
"found duplicate pegboardWebSocket, closing previous"
|
|
1914
|
+
);
|
|
1915
|
+
this.#pegboardWebSocket.close(1e3, "duplicate_websocket");
|
|
1916
|
+
}
|
|
1917
|
+
const ws = new WS(this.pegboardUrl, protocols);
|
|
1918
|
+
this.#pegboardWebSocket = ws;
|
|
1919
|
+
(_b = this.log) == null ? void 0 : _b.info({
|
|
1920
|
+
msg: "connecting",
|
|
1921
|
+
endpoint: this.pegboardEndpoint,
|
|
1922
|
+
namespace: this.#config.namespace,
|
|
1923
|
+
runnerKey: this.#config.runnerKey,
|
|
1924
|
+
hasToken: !!this.config.token
|
|
1925
|
+
});
|
|
1926
|
+
ws.addEventListener("open", () => {
|
|
1927
|
+
var _a2, _b2;
|
|
1928
|
+
if (this.#reconnectAttempt > 0) {
|
|
1929
|
+
(_a2 = this.log) == null ? void 0 : _a2.info({
|
|
1930
|
+
msg: "runner reconnected",
|
|
1931
|
+
namespace: this.#config.namespace,
|
|
1932
|
+
runnerName: this.#config.runnerName,
|
|
1933
|
+
reconnectAttempt: this.#reconnectAttempt
|
|
1934
|
+
});
|
|
1935
|
+
} else {
|
|
1936
|
+
(_b2 = this.log) == null ? void 0 : _b2.debug({
|
|
1937
|
+
msg: "runner connected",
|
|
1938
|
+
namespace: this.#config.namespace,
|
|
1939
|
+
runnerName: this.#config.runnerName
|
|
1940
|
+
});
|
|
1941
|
+
}
|
|
1942
|
+
this.#reconnectAttempt = 0;
|
|
1943
|
+
if (this.#reconnectTimeout) {
|
|
1944
|
+
clearTimeout(this.#reconnectTimeout);
|
|
1945
|
+
this.#reconnectTimeout = void 0;
|
|
1946
|
+
}
|
|
1947
|
+
if (this.#runnerLostTimeout) {
|
|
1948
|
+
clearTimeout(this.#runnerLostTimeout);
|
|
1949
|
+
this.#runnerLostTimeout = void 0;
|
|
1950
|
+
}
|
|
1951
|
+
const init = {
|
|
1952
|
+
name: this.#config.runnerName,
|
|
1953
|
+
version: this.#config.version,
|
|
1954
|
+
totalSlots: this.#config.totalSlots,
|
|
1955
|
+
prepopulateActorNames: new Map(
|
|
1956
|
+
Object.entries(this.#config.prepopulateActorNames).map(
|
|
1957
|
+
([name, data]) => [
|
|
1958
|
+
name,
|
|
1959
|
+
{ metadata: JSON.stringify(data.metadata) }
|
|
1960
|
+
]
|
|
1961
|
+
)
|
|
1962
|
+
),
|
|
1963
|
+
metadata: JSON.stringify(this.#config.metadata)
|
|
1964
|
+
};
|
|
1965
|
+
this.__sendToServer({
|
|
1966
|
+
tag: "ToServerInit",
|
|
1967
|
+
val: init
|
|
1968
|
+
});
|
|
1969
|
+
const ackInterval = 5 * 60 * 1e3;
|
|
1970
|
+
const ackLoop = setInterval(() => {
|
|
1971
|
+
var _a3, _b3;
|
|
1972
|
+
try {
|
|
1973
|
+
if (ws.readyState === 1) {
|
|
1974
|
+
this.#sendCommandAcknowledgment();
|
|
1975
|
+
} else {
|
|
1976
|
+
clearInterval(ackLoop);
|
|
1977
|
+
(_a3 = this.log) == null ? void 0 : _a3.info({
|
|
1978
|
+
msg: "WebSocket not open, stopping ack loop"
|
|
1979
|
+
});
|
|
1980
|
+
}
|
|
1981
|
+
} catch (err) {
|
|
1982
|
+
(_b3 = this.log) == null ? void 0 : _b3.error({
|
|
1983
|
+
msg: "error in command acknowledgment loop",
|
|
1984
|
+
error: stringifyError(err)
|
|
1985
|
+
});
|
|
1986
|
+
}
|
|
1987
|
+
}, ackInterval);
|
|
1988
|
+
this.#ackInterval = ackLoop;
|
|
1989
|
+
});
|
|
1990
|
+
ws.addEventListener("message", async (ev) => {
|
|
1991
|
+
var _a2, _b2, _c, _d, _e;
|
|
1992
|
+
let buf;
|
|
1993
|
+
if (ev.data instanceof Blob) {
|
|
1994
|
+
buf = new Uint8Array(await ev.data.arrayBuffer());
|
|
1995
|
+
} else if (Buffer.isBuffer(ev.data)) {
|
|
1996
|
+
buf = new Uint8Array(ev.data);
|
|
1997
|
+
} else {
|
|
1998
|
+
throw new Error(`expected binary data, got ${typeof ev.data}`);
|
|
1999
|
+
}
|
|
2000
|
+
const message = protocol.decodeToClient(buf);
|
|
2001
|
+
(_a2 = this.log) == null ? void 0 : _a2.debug({
|
|
2002
|
+
msg: "received runner message",
|
|
2003
|
+
data: stringifyToClient(message)
|
|
2004
|
+
});
|
|
2005
|
+
if (message.tag === "ToClientInit") {
|
|
2006
|
+
const init = message.val;
|
|
2007
|
+
if (this.runnerId !== init.runnerId) {
|
|
2008
|
+
this.runnerId = init.runnerId;
|
|
2009
|
+
this.#stopAllActors();
|
|
2010
|
+
}
|
|
2011
|
+
this.#runnerLostThreshold = ((_b2 = init.metadata) == null ? void 0 : _b2.runnerLostThreshold) ? Number(init.metadata.runnerLostThreshold) : void 0;
|
|
2012
|
+
(_c = this.log) == null ? void 0 : _c.info({
|
|
2013
|
+
msg: "received init",
|
|
2014
|
+
runnerLostThreshold: this.#runnerLostThreshold
|
|
2015
|
+
});
|
|
2016
|
+
this.#processUnsentKvRequests();
|
|
2017
|
+
this.#resendUnacknowledgedEvents();
|
|
2018
|
+
(_d = this.#tunnel) == null ? void 0 : _d.resendBufferedEvents();
|
|
2019
|
+
this.#config.onConnected();
|
|
2020
|
+
} else if (message.tag === "ToClientCommands") {
|
|
2021
|
+
const commands = message.val;
|
|
2022
|
+
this.#handleCommands(commands);
|
|
2023
|
+
} else if (message.tag === "ToClientAckEvents") {
|
|
2024
|
+
this.#handleAckEvents(message.val);
|
|
2025
|
+
} else if (message.tag === "ToClientKvResponse") {
|
|
2026
|
+
const kvResponse = message.val;
|
|
2027
|
+
this.#handleKvResponse(kvResponse);
|
|
2028
|
+
} else if (message.tag === "ToClientTunnelMessage") {
|
|
2029
|
+
(_e = this.#tunnel) == null ? void 0 : _e.handleTunnelMessage(message.val).catch((err) => {
|
|
2030
|
+
var _a3;
|
|
2031
|
+
(_a3 = this.log) == null ? void 0 : _a3.error({
|
|
2032
|
+
msg: "error handling tunnel message",
|
|
2033
|
+
error: stringifyError(err)
|
|
2034
|
+
});
|
|
2035
|
+
});
|
|
2036
|
+
} else if (message.tag === "ToClientPing") {
|
|
2037
|
+
this.__sendToServer({
|
|
2038
|
+
tag: "ToServerPong",
|
|
2039
|
+
val: {
|
|
2040
|
+
ts: message.val.ts
|
|
2041
|
+
}
|
|
2042
|
+
});
|
|
2043
|
+
} else {
|
|
2044
|
+
unreachable(message);
|
|
2045
|
+
}
|
|
2046
|
+
});
|
|
2047
|
+
ws.addEventListener("error", (ev) => {
|
|
2048
|
+
var _a2, _b2;
|
|
2049
|
+
(_a2 = this.log) == null ? void 0 : _a2.error({
|
|
2050
|
+
msg: `WebSocket error: ${stringifyError(ev.error)}`
|
|
2051
|
+
});
|
|
2052
|
+
if (!this.#shutdown) {
|
|
2053
|
+
if (!this.#runnerLostTimeout && this.#runnerLostThreshold && this.#runnerLostThreshold > 0) {
|
|
2054
|
+
(_b2 = this.log) == null ? void 0 : _b2.info({
|
|
2055
|
+
msg: "starting runner lost timeout",
|
|
2056
|
+
seconds: this.#runnerLostThreshold / 1e3
|
|
2057
|
+
});
|
|
2058
|
+
this.#runnerLostTimeout = setTimeout(() => {
|
|
2059
|
+
var _a3;
|
|
2060
|
+
try {
|
|
2061
|
+
this.#handleLost();
|
|
2062
|
+
} catch (err) {
|
|
2063
|
+
(_a3 = this.log) == null ? void 0 : _a3.error({
|
|
2064
|
+
msg: "error handling runner lost",
|
|
2065
|
+
error: stringifyError(err)
|
|
2066
|
+
});
|
|
2067
|
+
}
|
|
2068
|
+
}, this.#runnerLostThreshold);
|
|
2069
|
+
}
|
|
2070
|
+
this.#scheduleReconnect();
|
|
2071
|
+
}
|
|
2072
|
+
});
|
|
2073
|
+
ws.addEventListener("close", async (ev) => {
|
|
2074
|
+
var _a2, _b2, _c, _d;
|
|
2075
|
+
if (!this.#shutdown) {
|
|
2076
|
+
const closeError = parseWebSocketCloseReason(ev.reason);
|
|
2077
|
+
if ((closeError == null ? void 0 : closeError.group) === "ws" && (closeError == null ? void 0 : closeError.error) === "eviction") {
|
|
2078
|
+
(_a2 = this.log) == null ? void 0 : _a2.info("runner websocket evicted");
|
|
2079
|
+
this.#config.onDisconnected(ev.code, ev.reason);
|
|
2080
|
+
await this.shutdown(true);
|
|
2081
|
+
} else {
|
|
2082
|
+
(_b2 = this.log) == null ? void 0 : _b2.warn({
|
|
2083
|
+
msg: "runner disconnected",
|
|
2084
|
+
code: ev.code,
|
|
2085
|
+
reason: ev.reason.toString(),
|
|
2086
|
+
closeError
|
|
2087
|
+
});
|
|
2088
|
+
this.#config.onDisconnected(ev.code, ev.reason);
|
|
2089
|
+
}
|
|
2090
|
+
if (this.#ackInterval) {
|
|
2091
|
+
clearInterval(this.#ackInterval);
|
|
2092
|
+
this.#ackInterval = void 0;
|
|
2093
|
+
}
|
|
2094
|
+
if (!this.#runnerLostTimeout && this.#runnerLostThreshold && this.#runnerLostThreshold > 0) {
|
|
2095
|
+
(_c = this.log) == null ? void 0 : _c.info({
|
|
2096
|
+
msg: "starting runner lost timeout",
|
|
2097
|
+
seconds: this.#runnerLostThreshold / 1e3
|
|
2098
|
+
});
|
|
2099
|
+
this.#runnerLostTimeout = setTimeout(() => {
|
|
2100
|
+
var _a3;
|
|
2101
|
+
try {
|
|
2102
|
+
this.#handleLost();
|
|
2103
|
+
} catch (err) {
|
|
2104
|
+
(_a3 = this.log) == null ? void 0 : _a3.error({
|
|
2105
|
+
msg: "error handling runner lost",
|
|
2106
|
+
error: stringifyError(err)
|
|
2107
|
+
});
|
|
2108
|
+
}
|
|
2109
|
+
}, this.#runnerLostThreshold);
|
|
2110
|
+
}
|
|
2111
|
+
this.#scheduleReconnect();
|
|
2112
|
+
} else {
|
|
2113
|
+
(_d = this.log) == null ? void 0 : _d.info("websocket closed");
|
|
2114
|
+
this.#config.onDisconnected(ev.code, ev.reason);
|
|
2115
|
+
}
|
|
2116
|
+
});
|
|
2117
|
+
}
|
|
2118
|
+
#handleCommands(commands) {
|
|
2119
|
+
var _a;
|
|
2120
|
+
(_a = this.log) == null ? void 0 : _a.info({
|
|
2121
|
+
msg: "received commands",
|
|
2122
|
+
commandCount: commands.length
|
|
2123
|
+
});
|
|
2124
|
+
for (const commandWrapper of commands) {
|
|
2125
|
+
if (commandWrapper.inner.tag === "CommandStartActor") {
|
|
2126
|
+
this.#handleCommandStartActor(commandWrapper).catch((err) => {
|
|
2127
|
+
var _a2;
|
|
2128
|
+
(_a2 = this.log) == null ? void 0 : _a2.error({
|
|
2129
|
+
msg: "error handling start actor command",
|
|
2130
|
+
actorId: commandWrapper.checkpoint.actorId,
|
|
2131
|
+
error: stringifyError(err)
|
|
2132
|
+
});
|
|
2133
|
+
});
|
|
2134
|
+
const actor = this.getActor(
|
|
2135
|
+
commandWrapper.checkpoint.actorId,
|
|
2136
|
+
commandWrapper.checkpoint.generation
|
|
2137
|
+
);
|
|
2138
|
+
if (actor)
|
|
2139
|
+
actor.lastCommandIdx = commandWrapper.checkpoint.index;
|
|
2140
|
+
} else if (commandWrapper.inner.tag === "CommandStopActor") {
|
|
2141
|
+
this.#handleCommandStopActor(commandWrapper).catch((err) => {
|
|
2142
|
+
var _a2;
|
|
2143
|
+
(_a2 = this.log) == null ? void 0 : _a2.error({
|
|
2144
|
+
msg: "error handling stop actor command",
|
|
2145
|
+
actorId: commandWrapper.checkpoint.actorId,
|
|
2146
|
+
error: stringifyError(err)
|
|
2147
|
+
});
|
|
2148
|
+
});
|
|
2149
|
+
} else {
|
|
2150
|
+
unreachable(commandWrapper.inner);
|
|
2151
|
+
}
|
|
2152
|
+
}
|
|
2153
|
+
}
|
|
2154
|
+
#handleAckEvents(ack) {
|
|
2155
|
+
var _a;
|
|
2156
|
+
const originalTotalEvents = Array.from(this.#actors).reduce(
|
|
2157
|
+
(s, [_, actor]) => s + actor.eventHistory.length,
|
|
2158
|
+
0
|
|
2159
|
+
);
|
|
2160
|
+
for (const [_, actor] of this.#actors) {
|
|
2161
|
+
const checkpoint = ack.lastEventCheckpoints.find(
|
|
2162
|
+
(x) => x.actorId == actor.actorId
|
|
2163
|
+
);
|
|
2164
|
+
if (checkpoint) actor.handleAckEvents(checkpoint.index);
|
|
2165
|
+
}
|
|
2166
|
+
const totalEvents = Array.from(this.#actors).reduce(
|
|
2167
|
+
(s, [_, actor]) => s + actor.eventHistory.length,
|
|
2168
|
+
0
|
|
2169
|
+
);
|
|
2170
|
+
const prunedCount = originalTotalEvents - totalEvents;
|
|
2171
|
+
if (prunedCount > 0) {
|
|
2172
|
+
(_a = this.log) == null ? void 0 : _a.info({
|
|
2173
|
+
msg: "pruned acknowledged events",
|
|
2174
|
+
prunedCount
|
|
2175
|
+
});
|
|
2176
|
+
}
|
|
2177
|
+
if (totalEvents <= EVENT_BACKLOG_WARN_THRESHOLD) {
|
|
2178
|
+
this.#eventBacklogWarned = false;
|
|
2179
|
+
}
|
|
2180
|
+
}
|
|
2181
|
+
/** Track events to send to the server in case we need to resend it on disconnect. */
|
|
2182
|
+
#recordEvent(eventWrapper) {
|
|
2183
|
+
var _a;
|
|
2184
|
+
const actor = this.getActor(eventWrapper.checkpoint.actorId);
|
|
2185
|
+
if (!actor) return;
|
|
2186
|
+
actor.recordEvent(eventWrapper);
|
|
2187
|
+
const totalEvents = Array.from(this.#actors).reduce(
|
|
2188
|
+
(s, [_, actor2]) => s + actor2.eventHistory.length,
|
|
2189
|
+
0
|
|
2190
|
+
);
|
|
2191
|
+
if (totalEvents > EVENT_BACKLOG_WARN_THRESHOLD && !this.#eventBacklogWarned) {
|
|
2192
|
+
this.#eventBacklogWarned = true;
|
|
2193
|
+
(_a = this.log) == null ? void 0 : _a.warn({
|
|
2194
|
+
msg: "unacknowledged event backlog exceeds threshold",
|
|
2195
|
+
backlogSize: totalEvents,
|
|
2196
|
+
threshold: EVENT_BACKLOG_WARN_THRESHOLD
|
|
2197
|
+
});
|
|
2198
|
+
}
|
|
2199
|
+
}
|
|
2200
|
+
async #handleCommandStartActor(commandWrapper) {
|
|
2201
|
+
var _a, _b, _c, _d;
|
|
2202
|
+
if (!this.#tunnel) throw new Error("missing tunnel on actor start");
|
|
2203
|
+
const startCommand = commandWrapper.inner.val;
|
|
2204
|
+
const actorId = commandWrapper.checkpoint.actorId;
|
|
2205
|
+
const generation = commandWrapper.checkpoint.generation;
|
|
2206
|
+
const config = startCommand.config;
|
|
2207
|
+
const actorConfig = {
|
|
2208
|
+
name: config.name,
|
|
2209
|
+
key: config.key,
|
|
2210
|
+
createTs: config.createTs,
|
|
2211
|
+
input: config.input ? new Uint8Array(config.input) : null
|
|
2212
|
+
};
|
|
2213
|
+
const instance = new RunnerActor(
|
|
2214
|
+
actorId,
|
|
2215
|
+
generation,
|
|
2216
|
+
actorConfig,
|
|
2217
|
+
startCommand.hibernatingRequests
|
|
2218
|
+
);
|
|
2219
|
+
const existingActor = this.#actors.get(actorId);
|
|
2220
|
+
if (existingActor) {
|
|
2221
|
+
(_a = this.log) == null ? void 0 : _a.warn({
|
|
2222
|
+
msg: "replacing existing actor in actors map",
|
|
2223
|
+
actorId,
|
|
2224
|
+
existingGeneration: existingActor.generation,
|
|
2225
|
+
newGeneration: generation,
|
|
2226
|
+
existingPendingRequests: existingActor.pendingRequests.length
|
|
2227
|
+
});
|
|
2228
|
+
}
|
|
2229
|
+
this.#actors.set(actorId, instance);
|
|
2230
|
+
for (const hr of startCommand.hibernatingRequests) {
|
|
2231
|
+
this.#tunnel.addRequestToActor(hr.gatewayId, hr.requestId, actorId);
|
|
2232
|
+
}
|
|
2233
|
+
(_b = this.log) == null ? void 0 : _b.info({
|
|
2234
|
+
msg: "created actor",
|
|
2235
|
+
actors: this.#actors.size,
|
|
2236
|
+
actorId,
|
|
2237
|
+
name: config.name,
|
|
2238
|
+
key: config.key,
|
|
2239
|
+
generation,
|
|
2240
|
+
hibernatingRequests: startCommand.hibernatingRequests.length
|
|
2241
|
+
});
|
|
2242
|
+
this.#sendActorStateUpdate(actorId, generation, "running");
|
|
2243
|
+
try {
|
|
2244
|
+
(_c = this.log) == null ? void 0 : _c.debug({
|
|
2245
|
+
msg: "calling onActorStart",
|
|
2246
|
+
actorId,
|
|
2247
|
+
generation
|
|
2248
|
+
});
|
|
2249
|
+
await this.#config.onActorStart(actorId, generation, actorConfig);
|
|
2250
|
+
instance.actorStartPromise.resolve();
|
|
2251
|
+
} catch (err) {
|
|
2252
|
+
(_d = this.log) == null ? void 0 : _d.error({
|
|
2253
|
+
msg: "error starting runner actor",
|
|
2254
|
+
actorId,
|
|
2255
|
+
err
|
|
2256
|
+
});
|
|
2257
|
+
instance.actorStartPromise.reject(err);
|
|
2258
|
+
await this.forceStopActor(actorId, generation);
|
|
2259
|
+
}
|
|
2260
|
+
}
|
|
2261
|
+
async #handleCommandStopActor(commandWrapper) {
|
|
2262
|
+
const stopCommand = commandWrapper.inner.val;
|
|
2263
|
+
const actorId = commandWrapper.checkpoint.actorId;
|
|
2264
|
+
const generation = commandWrapper.checkpoint.generation;
|
|
2265
|
+
await this.forceStopActor(actorId, generation);
|
|
2266
|
+
}
|
|
2267
|
+
#sendActorIntent(actorId, generation, intentType) {
|
|
2268
|
+
const actor = this.getActor(actorId, generation);
|
|
2269
|
+
if (!actor) return;
|
|
2270
|
+
let actorIntent;
|
|
2271
|
+
if (intentType === "sleep") {
|
|
2272
|
+
actorIntent = { tag: "ActorIntentSleep", val: null };
|
|
2273
|
+
} else if (intentType === "stop") {
|
|
2274
|
+
actorIntent = {
|
|
2275
|
+
tag: "ActorIntentStop",
|
|
2276
|
+
val: null
|
|
2277
|
+
};
|
|
2278
|
+
} else {
|
|
2279
|
+
unreachable(intentType);
|
|
2280
|
+
}
|
|
2281
|
+
const intentEvent = {
|
|
2282
|
+
intent: actorIntent
|
|
2283
|
+
};
|
|
2284
|
+
const eventWrapper = {
|
|
2285
|
+
checkpoint: {
|
|
2286
|
+
actorId,
|
|
2287
|
+
generation,
|
|
2288
|
+
index: actor.nextEventIdx++
|
|
2289
|
+
},
|
|
2290
|
+
inner: {
|
|
2291
|
+
tag: "EventActorIntent",
|
|
2292
|
+
val: intentEvent
|
|
2293
|
+
}
|
|
2294
|
+
};
|
|
2295
|
+
this.#recordEvent(eventWrapper);
|
|
2296
|
+
this.__sendToServer({
|
|
2297
|
+
tag: "ToServerEvents",
|
|
2298
|
+
val: [eventWrapper]
|
|
2299
|
+
});
|
|
2300
|
+
}
|
|
2301
|
+
#sendActorStateUpdate(actorId, generation, stateType) {
|
|
2302
|
+
const actor = this.getActor(actorId, generation);
|
|
2303
|
+
if (!actor) return;
|
|
2304
|
+
let actorState;
|
|
2305
|
+
if (stateType === "running") {
|
|
2306
|
+
actorState = { tag: "ActorStateRunning", val: null };
|
|
2307
|
+
} else if (stateType === "stopped") {
|
|
2308
|
+
actorState = {
|
|
2309
|
+
tag: "ActorStateStopped",
|
|
2310
|
+
val: {
|
|
2311
|
+
code: protocol.StopCode.Ok,
|
|
2312
|
+
message: null
|
|
2313
|
+
}
|
|
2314
|
+
};
|
|
2315
|
+
} else {
|
|
2316
|
+
unreachable(stateType);
|
|
2317
|
+
}
|
|
2318
|
+
const stateUpdateEvent = {
|
|
2319
|
+
state: actorState
|
|
2320
|
+
};
|
|
2321
|
+
const eventWrapper = {
|
|
2322
|
+
checkpoint: {
|
|
2323
|
+
actorId,
|
|
2324
|
+
generation,
|
|
2325
|
+
index: actor.nextEventIdx++
|
|
2326
|
+
},
|
|
2327
|
+
inner: {
|
|
2328
|
+
tag: "EventActorStateUpdate",
|
|
2329
|
+
val: stateUpdateEvent
|
|
2330
|
+
}
|
|
2331
|
+
};
|
|
2332
|
+
this.#recordEvent(eventWrapper);
|
|
2333
|
+
this.__sendToServer({
|
|
2334
|
+
tag: "ToServerEvents",
|
|
2335
|
+
val: [eventWrapper]
|
|
2336
|
+
});
|
|
2337
|
+
}
|
|
2338
|
+
#sendCommandAcknowledgment() {
|
|
2339
|
+
const lastCommandCheckpoints = [];
|
|
2340
|
+
for (const [_, actor] of this.#actors) {
|
|
2341
|
+
if (actor.lastCommandIdx < 0) {
|
|
2342
|
+
continue;
|
|
2343
|
+
}
|
|
2344
|
+
lastCommandCheckpoints.push({
|
|
2345
|
+
actorId: actor.actorId,
|
|
2346
|
+
generation: actor.generation,
|
|
2347
|
+
index: actor.lastCommandIdx
|
|
2348
|
+
});
|
|
2349
|
+
}
|
|
2350
|
+
this.__sendToServer({
|
|
2351
|
+
tag: "ToServerAckCommands",
|
|
2352
|
+
val: {
|
|
2353
|
+
lastCommandCheckpoints
|
|
2354
|
+
}
|
|
2355
|
+
});
|
|
2356
|
+
}
|
|
2357
|
+
#handleKvResponse(response) {
|
|
2358
|
+
var _a;
|
|
2359
|
+
const requestId = response.requestId;
|
|
2360
|
+
const request = this.#kvRequests.get(requestId);
|
|
2361
|
+
if (!request) {
|
|
2362
|
+
(_a = this.log) == null ? void 0 : _a.error({
|
|
2363
|
+
msg: "received kv response for unknown request id",
|
|
2364
|
+
requestId
|
|
2365
|
+
});
|
|
2366
|
+
return;
|
|
2367
|
+
}
|
|
2368
|
+
this.#kvRequests.delete(requestId);
|
|
2369
|
+
if (response.data.tag === "KvErrorResponse") {
|
|
2370
|
+
request.reject(
|
|
2371
|
+
new Error(response.data.val.message || "Unknown KV error")
|
|
2372
|
+
);
|
|
2373
|
+
} else {
|
|
2374
|
+
request.resolve(response.data.val);
|
|
2375
|
+
}
|
|
2376
|
+
}
|
|
2377
|
+
#parseGetResponseSimple(response, requestedKeys) {
|
|
2378
|
+
const responseKeys = [];
|
|
2379
|
+
const responseValues = [];
|
|
2380
|
+
for (const key of response.keys) {
|
|
2381
|
+
responseKeys.push(new Uint8Array(key));
|
|
2382
|
+
}
|
|
2383
|
+
for (const value of response.values) {
|
|
2384
|
+
responseValues.push(new Uint8Array(value));
|
|
2385
|
+
}
|
|
2386
|
+
const result = [];
|
|
2387
|
+
for (const requestedKey of requestedKeys) {
|
|
2388
|
+
let found = false;
|
|
2389
|
+
for (let i = 0; i < responseKeys.length; i++) {
|
|
2390
|
+
if (this.#keysEqual(requestedKey, responseKeys[i])) {
|
|
2391
|
+
result.push(responseValues[i]);
|
|
2392
|
+
found = true;
|
|
2393
|
+
break;
|
|
2394
|
+
}
|
|
2395
|
+
}
|
|
2396
|
+
if (!found) {
|
|
2397
|
+
result.push(null);
|
|
2398
|
+
}
|
|
2399
|
+
}
|
|
2400
|
+
return result;
|
|
2401
|
+
}
|
|
2402
|
+
#keysEqual(key1, key2) {
|
|
2403
|
+
if (key1.length !== key2.length) return false;
|
|
2404
|
+
for (let i = 0; i < key1.length; i++) {
|
|
2405
|
+
if (key1[i] !== key2[i]) return false;
|
|
2406
|
+
}
|
|
2407
|
+
return true;
|
|
2408
|
+
}
|
|
2409
|
+
//#parseGetResponse(response: protocol.KvGetResponse) {
|
|
2410
|
+
// const keys: string[] = [];
|
|
2411
|
+
// const values: Uint8Array[] = [];
|
|
2412
|
+
// const metadata: { version: Uint8Array; createTs: bigint }[] = [];
|
|
2413
|
+
//
|
|
2414
|
+
// for (const key of response.keys) {
|
|
2415
|
+
// keys.push(new TextDecoder().decode(key));
|
|
2416
|
+
// }
|
|
2417
|
+
//
|
|
2418
|
+
// for (const value of response.values) {
|
|
2419
|
+
// values.push(new Uint8Array(value));
|
|
2420
|
+
// }
|
|
2421
|
+
//
|
|
2422
|
+
// for (const meta of response.metadata) {
|
|
2423
|
+
// metadata.push({
|
|
2424
|
+
// version: new Uint8Array(meta.version),
|
|
2425
|
+
// createTs: meta.createTs,
|
|
2426
|
+
// });
|
|
2427
|
+
// }
|
|
2428
|
+
//
|
|
2429
|
+
// return { keys, values, metadata };
|
|
2430
|
+
//}
|
|
2431
|
+
#parseListResponseSimple(response) {
|
|
2432
|
+
const result = [];
|
|
2433
|
+
for (let i = 0; i < response.keys.length; i++) {
|
|
2434
|
+
const key = response.keys[i];
|
|
2435
|
+
const value = response.values[i];
|
|
2436
|
+
if (key && value) {
|
|
2437
|
+
const keyBytes = new Uint8Array(key);
|
|
2438
|
+
const valueBytes = new Uint8Array(value);
|
|
2439
|
+
result.push([keyBytes, valueBytes]);
|
|
2440
|
+
}
|
|
2441
|
+
}
|
|
2442
|
+
return result;
|
|
2443
|
+
}
|
|
2444
|
+
//#parseListResponse(response: protocol.KvListResponse) {
|
|
2445
|
+
// const keys: string[] = [];
|
|
2446
|
+
// const values: Uint8Array[] = [];
|
|
2447
|
+
// const metadata: { version: Uint8Array; createTs: bigint }[] = [];
|
|
2448
|
+
//
|
|
2449
|
+
// for (const key of response.keys) {
|
|
2450
|
+
// keys.push(new TextDecoder().decode(key));
|
|
2451
|
+
// }
|
|
2452
|
+
//
|
|
2453
|
+
// for (const value of response.values) {
|
|
2454
|
+
// values.push(new Uint8Array(value));
|
|
2455
|
+
// }
|
|
2456
|
+
//
|
|
2457
|
+
// for (const meta of response.metadata) {
|
|
2458
|
+
// metadata.push({
|
|
2459
|
+
// version: new Uint8Array(meta.version),
|
|
2460
|
+
// createTs: meta.createTs,
|
|
2461
|
+
// });
|
|
2462
|
+
// }
|
|
2463
|
+
//
|
|
2464
|
+
// return { keys, values, metadata };
|
|
2465
|
+
//}
|
|
2466
|
+
// MARK: KV Operations
|
|
2467
|
+
async kvGet(actorId, keys) {
|
|
2468
|
+
const kvKeys = keys.map(
|
|
2469
|
+
(key) => key.buffer.slice(
|
|
2470
|
+
key.byteOffset,
|
|
2471
|
+
key.byteOffset + key.byteLength
|
|
2472
|
+
)
|
|
2473
|
+
);
|
|
2474
|
+
const requestData = {
|
|
2475
|
+
tag: "KvGetRequest",
|
|
2476
|
+
val: { keys: kvKeys }
|
|
2477
|
+
};
|
|
2478
|
+
const response = await this.#sendKvRequest(actorId, requestData);
|
|
2479
|
+
return this.#parseGetResponseSimple(response, keys);
|
|
2480
|
+
}
|
|
2481
|
+
async kvListAll(actorId, options) {
|
|
2482
|
+
const requestData = {
|
|
2483
|
+
tag: "KvListRequest",
|
|
2484
|
+
val: {
|
|
2485
|
+
query: { tag: "KvListAllQuery", val: null },
|
|
2486
|
+
reverse: (options == null ? void 0 : options.reverse) || null,
|
|
2487
|
+
limit: (options == null ? void 0 : options.limit) !== void 0 ? BigInt(options.limit) : null
|
|
2488
|
+
}
|
|
2489
|
+
};
|
|
2490
|
+
const response = await this.#sendKvRequest(actorId, requestData);
|
|
2491
|
+
return this.#parseListResponseSimple(response);
|
|
2492
|
+
}
|
|
2493
|
+
async kvListRange(actorId, start, end, exclusive, options) {
|
|
2494
|
+
const startKey = start.buffer.slice(
|
|
2495
|
+
start.byteOffset,
|
|
2496
|
+
start.byteOffset + start.byteLength
|
|
2497
|
+
);
|
|
2498
|
+
const endKey = end.buffer.slice(
|
|
2499
|
+
end.byteOffset,
|
|
2500
|
+
end.byteOffset + end.byteLength
|
|
2501
|
+
);
|
|
2502
|
+
const requestData = {
|
|
2503
|
+
tag: "KvListRequest",
|
|
2504
|
+
val: {
|
|
2505
|
+
query: {
|
|
2506
|
+
tag: "KvListRangeQuery",
|
|
2507
|
+
val: {
|
|
2508
|
+
start: startKey,
|
|
2509
|
+
end: endKey,
|
|
2510
|
+
exclusive: exclusive || false
|
|
2511
|
+
}
|
|
2512
|
+
},
|
|
2513
|
+
reverse: (options == null ? void 0 : options.reverse) || null,
|
|
2514
|
+
limit: (options == null ? void 0 : options.limit) !== void 0 ? BigInt(options.limit) : null
|
|
2515
|
+
}
|
|
2516
|
+
};
|
|
2517
|
+
const response = await this.#sendKvRequest(actorId, requestData);
|
|
2518
|
+
return this.#parseListResponseSimple(response);
|
|
2519
|
+
}
|
|
2520
|
+
async kvListPrefix(actorId, prefix, options) {
|
|
2521
|
+
const prefixKey = prefix.buffer.slice(
|
|
2522
|
+
prefix.byteOffset,
|
|
2523
|
+
prefix.byteOffset + prefix.byteLength
|
|
2524
|
+
);
|
|
2525
|
+
const requestData = {
|
|
2526
|
+
tag: "KvListRequest",
|
|
2527
|
+
val: {
|
|
2528
|
+
query: {
|
|
2529
|
+
tag: "KvListPrefixQuery",
|
|
2530
|
+
val: { key: prefixKey }
|
|
2531
|
+
},
|
|
2532
|
+
reverse: (options == null ? void 0 : options.reverse) || null,
|
|
2533
|
+
limit: (options == null ? void 0 : options.limit) !== void 0 ? BigInt(options.limit) : null
|
|
2534
|
+
}
|
|
2535
|
+
};
|
|
2536
|
+
const response = await this.#sendKvRequest(actorId, requestData);
|
|
2537
|
+
return this.#parseListResponseSimple(response);
|
|
2538
|
+
}
|
|
2539
|
+
async kvPut(actorId, entries) {
|
|
2540
|
+
const keys = entries.map(
|
|
2541
|
+
([key, _value]) => key.buffer.slice(
|
|
2542
|
+
key.byteOffset,
|
|
2543
|
+
key.byteOffset + key.byteLength
|
|
2544
|
+
)
|
|
2545
|
+
);
|
|
2546
|
+
const values = entries.map(
|
|
2547
|
+
([_key, value]) => value.buffer.slice(
|
|
2548
|
+
value.byteOffset,
|
|
2549
|
+
value.byteOffset + value.byteLength
|
|
2550
|
+
)
|
|
2551
|
+
);
|
|
2552
|
+
const requestData = {
|
|
2553
|
+
tag: "KvPutRequest",
|
|
2554
|
+
val: { keys, values }
|
|
2555
|
+
};
|
|
2556
|
+
await this.#sendKvRequest(actorId, requestData);
|
|
2557
|
+
}
|
|
2558
|
+
async kvDelete(actorId, keys) {
|
|
2559
|
+
const kvKeys = keys.map(
|
|
2560
|
+
(key) => key.buffer.slice(
|
|
2561
|
+
key.byteOffset,
|
|
2562
|
+
key.byteOffset + key.byteLength
|
|
2563
|
+
)
|
|
2564
|
+
);
|
|
2565
|
+
const requestData = {
|
|
2566
|
+
tag: "KvDeleteRequest",
|
|
2567
|
+
val: { keys: kvKeys }
|
|
2568
|
+
};
|
|
2569
|
+
await this.#sendKvRequest(actorId, requestData);
|
|
2570
|
+
}
|
|
2571
|
+
async kvDrop(actorId) {
|
|
2572
|
+
const requestData = {
|
|
2573
|
+
tag: "KvDropRequest",
|
|
2574
|
+
val: null
|
|
2575
|
+
};
|
|
2576
|
+
await this.#sendKvRequest(actorId, requestData);
|
|
2577
|
+
}
|
|
2578
|
+
// MARK: Alarm Operations
|
|
2579
|
+
setAlarm(actorId, alarmTs, generation) {
|
|
2580
|
+
const actor = this.getActor(actorId, generation);
|
|
2581
|
+
if (!actor) return;
|
|
2582
|
+
const alarmEvent = {
|
|
2583
|
+
alarmTs: alarmTs !== null ? BigInt(alarmTs) : null
|
|
2584
|
+
};
|
|
2585
|
+
const eventWrapper = {
|
|
2586
|
+
checkpoint: {
|
|
2587
|
+
actorId,
|
|
2588
|
+
generation: actor.generation,
|
|
2589
|
+
index: actor.nextEventIdx++
|
|
2590
|
+
},
|
|
2591
|
+
inner: {
|
|
2592
|
+
tag: "EventActorSetAlarm",
|
|
2593
|
+
val: alarmEvent
|
|
2594
|
+
}
|
|
2595
|
+
};
|
|
2596
|
+
this.#recordEvent(eventWrapper);
|
|
2597
|
+
this.__sendToServer({
|
|
2598
|
+
tag: "ToServerEvents",
|
|
2599
|
+
val: [eventWrapper]
|
|
2600
|
+
});
|
|
2601
|
+
}
|
|
2602
|
+
clearAlarm(actorId, generation) {
|
|
2603
|
+
this.setAlarm(actorId, null, generation);
|
|
2604
|
+
}
|
|
2605
|
+
#sendKvRequest(actorId, requestData) {
|
|
2606
|
+
return new Promise((resolve, reject) => {
|
|
2607
|
+
const requestId = this.#nextKvRequestId++;
|
|
2608
|
+
const requestEntry = {
|
|
2609
|
+
actorId,
|
|
2610
|
+
data: requestData,
|
|
2611
|
+
resolve,
|
|
2612
|
+
reject,
|
|
2613
|
+
sent: false,
|
|
2614
|
+
timestamp: Date.now()
|
|
2615
|
+
};
|
|
2616
|
+
this.#kvRequests.set(requestId, requestEntry);
|
|
2617
|
+
if (this.getPegboardWebSocketIfReady()) {
|
|
2618
|
+
this.#sendSingleKvRequest(requestId);
|
|
2619
|
+
}
|
|
2620
|
+
});
|
|
2621
|
+
}
|
|
2622
|
+
#sendSingleKvRequest(requestId) {
|
|
2623
|
+
const request = this.#kvRequests.get(requestId);
|
|
2624
|
+
if (!request || request.sent) return;
|
|
2625
|
+
try {
|
|
2626
|
+
const kvRequest = {
|
|
2627
|
+
actorId: request.actorId,
|
|
2628
|
+
requestId,
|
|
2629
|
+
data: request.data
|
|
2630
|
+
};
|
|
2631
|
+
this.__sendToServer({
|
|
2632
|
+
tag: "ToServerKvRequest",
|
|
2633
|
+
val: kvRequest
|
|
2634
|
+
});
|
|
2635
|
+
request.sent = true;
|
|
2636
|
+
request.timestamp = Date.now();
|
|
2637
|
+
} catch (error) {
|
|
2638
|
+
this.#kvRequests.delete(requestId);
|
|
2639
|
+
request.reject(error);
|
|
2640
|
+
}
|
|
2641
|
+
}
|
|
2642
|
+
#processUnsentKvRequests() {
|
|
2643
|
+
if (!this.getPegboardWebSocketIfReady()) {
|
|
2644
|
+
return;
|
|
2645
|
+
}
|
|
2646
|
+
let processedCount = 0;
|
|
2647
|
+
for (const [requestId, request] of this.#kvRequests.entries()) {
|
|
2648
|
+
if (!request.sent) {
|
|
2649
|
+
this.#sendSingleKvRequest(requestId);
|
|
2650
|
+
processedCount++;
|
|
2651
|
+
}
|
|
2652
|
+
}
|
|
2653
|
+
if (processedCount > 0) {
|
|
2654
|
+
}
|
|
2655
|
+
}
|
|
2656
|
+
/** Asserts WebSocket exists and is ready. */
|
|
2657
|
+
getPegboardWebSocketIfReady() {
|
|
2658
|
+
if (!!this.#pegboardWebSocket && this.#pegboardWebSocket.readyState === 1) {
|
|
2659
|
+
return this.#pegboardWebSocket;
|
|
2660
|
+
} else {
|
|
2661
|
+
return void 0;
|
|
2662
|
+
}
|
|
2663
|
+
}
|
|
2664
|
+
__sendToServer(message) {
|
|
2665
|
+
var _a, _b;
|
|
2666
|
+
(_a = this.log) == null ? void 0 : _a.debug({
|
|
2667
|
+
msg: "sending runner message",
|
|
2668
|
+
data: stringifyToServer(message)
|
|
2669
|
+
});
|
|
2670
|
+
const encoded = protocol.encodeToServer(message);
|
|
2671
|
+
const pegboardWebSocket = this.getPegboardWebSocketIfReady();
|
|
2672
|
+
if (pegboardWebSocket) {
|
|
2673
|
+
pegboardWebSocket.send(encoded);
|
|
2674
|
+
} else {
|
|
2675
|
+
(_b = this.log) == null ? void 0 : _b.error({
|
|
2676
|
+
msg: "WebSocket not available or not open for sending data"
|
|
2677
|
+
});
|
|
2678
|
+
}
|
|
2679
|
+
}
|
|
2680
|
+
sendHibernatableWebSocketMessageAck(gatewayId, requestId, index) {
|
|
2681
|
+
if (!this.#tunnel)
|
|
2682
|
+
throw new Error("missing tunnel to send message ack");
|
|
2683
|
+
this.#tunnel.sendHibernatableWebSocketMessageAck(
|
|
2684
|
+
gatewayId,
|
|
2685
|
+
requestId,
|
|
2686
|
+
index
|
|
2687
|
+
);
|
|
2688
|
+
}
|
|
2689
|
+
/**
|
|
2690
|
+
* Restores hibernatable WebSocket connections for an actor.
|
|
2691
|
+
*
|
|
2692
|
+
* This method should be called at the end of `onActorStart` after the
|
|
2693
|
+
* actor instance is fully initialized.
|
|
2694
|
+
*
|
|
2695
|
+
* This method will:
|
|
2696
|
+
* - Restore all provided hibernatable WebSocket connections
|
|
2697
|
+
* - Attach event listeners to the restored WebSockets
|
|
2698
|
+
* - Close any WebSocket connections that failed to restore
|
|
2699
|
+
*
|
|
2700
|
+
* The provided metadata list should include all hibernatable WebSockets
|
|
2701
|
+
* that were persisted for this actor. The gateway will automatically
|
|
2702
|
+
* close any connections that are not restored (i.e., not included in
|
|
2703
|
+
* this list).
|
|
2704
|
+
*
|
|
2705
|
+
* **Important:** This method must be called after `onActorStart` completes
|
|
2706
|
+
* and before marking the actor as "ready" to ensure all hibernatable
|
|
2707
|
+
* connections are fully restored.
|
|
2708
|
+
*
|
|
2709
|
+
* @param actorId - The ID of the actor to restore connections for
|
|
2710
|
+
* @param metaEntries - Array of hibernatable WebSocket metadata to restore
|
|
2711
|
+
*/
|
|
2712
|
+
async restoreHibernatingRequests(actorId, metaEntries) {
|
|
2713
|
+
if (!this.#tunnel)
|
|
2714
|
+
throw new Error("missing tunnel to restore hibernating requests");
|
|
2715
|
+
await this.#tunnel.restoreHibernatingRequests(actorId, metaEntries);
|
|
2716
|
+
}
|
|
2717
|
+
getServerlessInitPacket() {
|
|
2718
|
+
if (!this.runnerId) return void 0;
|
|
2719
|
+
const data = protocol.encodeToServerlessServer({
|
|
2720
|
+
tag: "ToServerlessServerInit",
|
|
2721
|
+
val: {
|
|
2722
|
+
runnerId: this.runnerId,
|
|
2723
|
+
runnerProtocolVersion: PROTOCOL_VERSION
|
|
2724
|
+
}
|
|
2725
|
+
});
|
|
2726
|
+
const buffer = Buffer.alloc(data.length + 2);
|
|
2727
|
+
buffer.writeUInt16LE(PROTOCOL_VERSION, 0);
|
|
2728
|
+
Buffer.from(data).copy(buffer, 2);
|
|
2729
|
+
return buffer.toString("base64");
|
|
2730
|
+
}
|
|
2731
|
+
#scheduleReconnect() {
|
|
2732
|
+
var _a, _b, _c;
|
|
2733
|
+
if (this.#shutdown) {
|
|
2734
|
+
(_a = this.log) == null ? void 0 : _a.debug({
|
|
2735
|
+
msg: "Runner is shut down, not attempting reconnect"
|
|
2736
|
+
});
|
|
2737
|
+
return;
|
|
2738
|
+
}
|
|
2739
|
+
const delay = calculateBackoff(this.#reconnectAttempt, {
|
|
2740
|
+
initialDelay: 1e3,
|
|
2741
|
+
maxDelay: 3e4,
|
|
2742
|
+
multiplier: 2,
|
|
2743
|
+
jitter: true
|
|
2744
|
+
});
|
|
2745
|
+
(_b = this.log) == null ? void 0 : _b.debug({
|
|
2746
|
+
msg: `Scheduling reconnect attempt ${this.#reconnectAttempt + 1} in ${delay}ms`
|
|
2747
|
+
});
|
|
2748
|
+
if (this.#reconnectTimeout) {
|
|
2749
|
+
(_c = this.log) == null ? void 0 : _c.info(
|
|
2750
|
+
"clearing previous reconnect timeout in schedule reconnect"
|
|
2751
|
+
);
|
|
2752
|
+
clearTimeout(this.#reconnectTimeout);
|
|
2753
|
+
}
|
|
2754
|
+
this.#reconnectTimeout = setTimeout(() => {
|
|
2755
|
+
var _a2;
|
|
2756
|
+
if (!this.#shutdown) {
|
|
2757
|
+
this.#reconnectAttempt++;
|
|
2758
|
+
(_a2 = this.log) == null ? void 0 : _a2.debug({
|
|
2759
|
+
msg: `Attempting to reconnect (attempt ${this.#reconnectAttempt})...`
|
|
2760
|
+
});
|
|
2761
|
+
this.#openPegboardWebSocket().catch((err) => {
|
|
2762
|
+
var _a3;
|
|
2763
|
+
(_a3 = this.log) == null ? void 0 : _a3.error({
|
|
2764
|
+
msg: "error during websocket reconnection",
|
|
2765
|
+
error: stringifyError(err)
|
|
2766
|
+
});
|
|
2767
|
+
});
|
|
2768
|
+
}
|
|
2769
|
+
}, delay);
|
|
2770
|
+
}
|
|
2771
|
+
#resendUnacknowledgedEvents() {
|
|
2772
|
+
var _a;
|
|
2773
|
+
const eventsToResend = [];
|
|
2774
|
+
for (const [_, actor] of this.#actors) {
|
|
2775
|
+
eventsToResend.push(...actor.eventHistory);
|
|
2776
|
+
}
|
|
2777
|
+
if (eventsToResend.length === 0) return;
|
|
2778
|
+
(_a = this.log) == null ? void 0 : _a.info({
|
|
2779
|
+
msg: "resending unacknowledged events",
|
|
2780
|
+
count: eventsToResend.length
|
|
2781
|
+
});
|
|
2782
|
+
this.__sendToServer({
|
|
2783
|
+
tag: "ToServerEvents",
|
|
2784
|
+
val: eventsToResend
|
|
2785
|
+
});
|
|
2786
|
+
}
|
|
2787
|
+
#cleanupOldKvRequests() {
|
|
2788
|
+
const thirtySecondsAgo = Date.now() - KV_EXPIRE;
|
|
2789
|
+
const toDelete = [];
|
|
2790
|
+
for (const [requestId, request] of this.#kvRequests.entries()) {
|
|
2791
|
+
if (request.timestamp < thirtySecondsAgo) {
|
|
2792
|
+
request.reject(
|
|
2793
|
+
new Error(
|
|
2794
|
+
"KV request timed out waiting for WebSocket connection"
|
|
2795
|
+
)
|
|
2796
|
+
);
|
|
2797
|
+
toDelete.push(requestId);
|
|
2798
|
+
}
|
|
2799
|
+
}
|
|
2800
|
+
for (const requestId of toDelete) {
|
|
2801
|
+
this.#kvRequests.delete(requestId);
|
|
2802
|
+
}
|
|
2803
|
+
if (toDelete.length > 0) {
|
|
2804
|
+
}
|
|
2805
|
+
}
|
|
2806
|
+
};
|
|
2807
|
+
|
|
2808
|
+
|
|
2809
|
+
|
|
2810
|
+
|
|
2811
|
+
|
|
2812
|
+
exports.Runner = Runner; exports.RunnerActor = RunnerActor; exports.RunnerShutdownError = RunnerShutdownError; exports.idToStr = idToStr;
|
|
2813
|
+
//# sourceMappingURL=mod.cjs.map
|