@rivetkit/engine-runner 2.0.26 → 2.0.27
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +10 -10
- package/dist/mod.cjs +126 -230
- package/dist/mod.cjs.map +1 -1
- package/dist/mod.d.cts +1 -9
- package/dist/mod.d.ts +1 -9
- package/dist/mod.js +117 -221
- package/dist/mod.js.map +1 -1
- package/package.json +2 -2
- package/src/actor.ts +4 -18
- package/src/mod.ts +92 -176
- package/src/stringify.ts +26 -26
- package/src/tunnel.ts +13 -4
- package/src/utils.ts +0 -16
package/dist/mod.js
CHANGED
|
@@ -83,23 +83,6 @@ function idToStr(id) {
|
|
|
83
83
|
const bytes = new Uint8Array(id);
|
|
84
84
|
return Array.from(bytes).map((byte) => byte.toString(16).padStart(2, "0")).join("");
|
|
85
85
|
}
|
|
86
|
-
function stringifyError(error) {
|
|
87
|
-
var _a;
|
|
88
|
-
if (error instanceof Error) {
|
|
89
|
-
return `${error.name}: ${error.message}${error.stack ? `
|
|
90
|
-
${error.stack}` : ""}`;
|
|
91
|
-
} else if (typeof error === "string") {
|
|
92
|
-
return error;
|
|
93
|
-
} else if (typeof error === "object" && error !== null) {
|
|
94
|
-
try {
|
|
95
|
-
return `${JSON.stringify(error)}`;
|
|
96
|
-
} catch {
|
|
97
|
-
return `[object ${((_a = error.constructor) == null ? void 0 : _a.name) || "Object"}]`;
|
|
98
|
-
}
|
|
99
|
-
} else {
|
|
100
|
-
return String(error);
|
|
101
|
-
}
|
|
102
|
-
}
|
|
103
86
|
|
|
104
87
|
// src/actor.ts
|
|
105
88
|
var RunnerActor = class {
|
|
@@ -116,9 +99,6 @@ var RunnerActor = class {
|
|
|
116
99
|
pendingRequests = [];
|
|
117
100
|
webSockets = [];
|
|
118
101
|
actorStartPromise;
|
|
119
|
-
lastCommandIdx = -1n;
|
|
120
|
-
nextEventIdx = 0n;
|
|
121
|
-
eventHistory = [];
|
|
122
102
|
/**
|
|
123
103
|
* If restoreHibernatingRequests has been called. This is used to assert
|
|
124
104
|
* that the caller is implemented correctly.
|
|
@@ -235,14 +215,6 @@ var RunnerActor = class {
|
|
|
235
215
|
this.webSockets.splice(index, 1);
|
|
236
216
|
}
|
|
237
217
|
}
|
|
238
|
-
handleAckEvents(lastEventIdx) {
|
|
239
|
-
this.eventHistory = this.eventHistory.filter(
|
|
240
|
-
(event) => event.checkpoint.index > lastEventIdx
|
|
241
|
-
);
|
|
242
|
-
}
|
|
243
|
-
recordEvent(eventWrapper) {
|
|
244
|
-
this.eventHistory.push(eventWrapper);
|
|
245
|
-
}
|
|
246
218
|
};
|
|
247
219
|
|
|
248
220
|
// src/stringify.ts
|
|
@@ -261,6 +233,8 @@ function stringifyMessageId(messageId) {
|
|
|
261
233
|
}
|
|
262
234
|
function stringifyToServerTunnelMessageKind(kind) {
|
|
263
235
|
switch (kind.tag) {
|
|
236
|
+
case "DeprecatedTunnelAck":
|
|
237
|
+
return "DeprecatedTunnelAck";
|
|
264
238
|
case "ToServerResponseStart": {
|
|
265
239
|
const { status, headers, body, stream } = kind.val;
|
|
266
240
|
const bodyStr = body === null ? "null" : stringifyArrayBuffer(body);
|
|
@@ -294,6 +268,8 @@ function stringifyToServerTunnelMessageKind(kind) {
|
|
|
294
268
|
}
|
|
295
269
|
function stringifyToClientTunnelMessageKind(kind) {
|
|
296
270
|
switch (kind.tag) {
|
|
271
|
+
case "DeprecatedTunnelAck":
|
|
272
|
+
return "DeprecatedTunnelAck";
|
|
297
273
|
case "ToClientRequestStart": {
|
|
298
274
|
const { actorId, method, path, headers, body, stream } = kind.val;
|
|
299
275
|
const bodyStr = body === null ? "null" : stringifyArrayBuffer(body);
|
|
@@ -324,20 +300,20 @@ function stringifyToClientTunnelMessageKind(kind) {
|
|
|
324
300
|
function stringifyCommand(command) {
|
|
325
301
|
switch (command.tag) {
|
|
326
302
|
case "CommandStartActor": {
|
|
327
|
-
const { generation, config, hibernatingRequests } = command.val;
|
|
303
|
+
const { actorId, generation, config, hibernatingRequests } = command.val;
|
|
328
304
|
const keyStr = config.key === null ? "null" : `"${config.key}"`;
|
|
329
305
|
const inputStr = config.input === null ? "null" : stringifyArrayBuffer(config.input);
|
|
330
306
|
const hibernatingRequestsStr = hibernatingRequests.length > 0 ? `[${hibernatingRequests.map((hr) => `{gatewayId: ${idToStr(hr.gatewayId)}, requestId: ${idToStr(hr.requestId)}}`).join(", ")}]` : "[]";
|
|
331
|
-
return `CommandStartActor{generation: ${generation}, config: {name: "${config.name}", key: ${keyStr}, createTs: ${stringifyBigInt(config.createTs)}, input: ${inputStr}}, hibernatingRequests: ${hibernatingRequestsStr}}`;
|
|
307
|
+
return `CommandStartActor{actorId: "${actorId}", generation: ${generation}, config: {name: "${config.name}", key: ${keyStr}, createTs: ${stringifyBigInt(config.createTs)}, input: ${inputStr}}, hibernatingRequests: ${hibernatingRequestsStr}}`;
|
|
332
308
|
}
|
|
333
309
|
case "CommandStopActor": {
|
|
334
|
-
const { generation } = command.val;
|
|
335
|
-
return `CommandStopActor{generation: ${generation}}`;
|
|
310
|
+
const { actorId, generation } = command.val;
|
|
311
|
+
return `CommandStopActor{actorId: "${actorId}", generation: ${generation}}`;
|
|
336
312
|
}
|
|
337
313
|
}
|
|
338
314
|
}
|
|
339
315
|
function stringifyCommandWrapper(wrapper) {
|
|
340
|
-
return `CommandWrapper{
|
|
316
|
+
return `CommandWrapper{index: ${stringifyBigInt(wrapper.index)}, inner: ${stringifyCommand(wrapper.inner)}}`;
|
|
341
317
|
}
|
|
342
318
|
function stringifyEvent(event) {
|
|
343
319
|
switch (event.tag) {
|
|
@@ -368,7 +344,7 @@ function stringifyEvent(event) {
|
|
|
368
344
|
}
|
|
369
345
|
}
|
|
370
346
|
function stringifyEventWrapper(wrapper) {
|
|
371
|
-
return `EventWrapper{
|
|
347
|
+
return `EventWrapper{index: ${stringifyBigInt(wrapper.index)}, inner: ${stringifyEvent(wrapper.inner)}}`;
|
|
372
348
|
}
|
|
373
349
|
function stringifyToServer(message) {
|
|
374
350
|
switch (message.tag) {
|
|
@@ -377,27 +353,28 @@ function stringifyToServer(message) {
|
|
|
377
353
|
name,
|
|
378
354
|
version,
|
|
379
355
|
totalSlots,
|
|
356
|
+
lastCommandIdx,
|
|
380
357
|
prepopulateActorNames,
|
|
381
358
|
metadata
|
|
382
359
|
} = message.val;
|
|
360
|
+
const lastCommandIdxStr = lastCommandIdx === null ? "null" : stringifyBigInt(lastCommandIdx);
|
|
383
361
|
const prepopulateActorNamesStr = prepopulateActorNames === null ? "null" : `Map(${prepopulateActorNames.size})`;
|
|
384
362
|
const metadataStr = metadata === null ? "null" : `"${metadata}"`;
|
|
385
|
-
return `ToServerInit{name: "${name}", version: ${version}, totalSlots: ${totalSlots}, prepopulateActorNames: ${prepopulateActorNamesStr}, metadata: ${metadataStr}}`;
|
|
363
|
+
return `ToServerInit{name: "${name}", version: ${version}, totalSlots: ${totalSlots}, lastCommandIdx: ${lastCommandIdxStr}, prepopulateActorNames: ${prepopulateActorNamesStr}, metadata: ${metadataStr}}`;
|
|
386
364
|
}
|
|
387
365
|
case "ToServerEvents": {
|
|
388
366
|
const events = message.val;
|
|
389
367
|
return `ToServerEvents{count: ${events.length}, events: [${events.map((e) => stringifyEventWrapper(e)).join(", ")}]}`;
|
|
390
368
|
}
|
|
391
369
|
case "ToServerAckCommands": {
|
|
392
|
-
const {
|
|
393
|
-
|
|
394
|
-
return `ToServerAckCommands{lastCommandCheckpoints: ${checkpointsStr}}`;
|
|
370
|
+
const { lastCommandIdx } = message.val;
|
|
371
|
+
return `ToServerAckCommands{lastCommandIdx: ${stringifyBigInt(lastCommandIdx)}}`;
|
|
395
372
|
}
|
|
396
373
|
case "ToServerStopping":
|
|
397
374
|
return "ToServerStopping";
|
|
398
|
-
case "
|
|
375
|
+
case "ToServerPing": {
|
|
399
376
|
const { ts } = message.val;
|
|
400
|
-
return `
|
|
377
|
+
return `ToServerPing{ts: ${stringifyBigInt(ts)}}`;
|
|
401
378
|
}
|
|
402
379
|
case "ToServerKvRequest": {
|
|
403
380
|
const { actorId, requestId, data } = message.val;
|
|
@@ -413,21 +390,19 @@ function stringifyToServer(message) {
|
|
|
413
390
|
function stringifyToClient(message) {
|
|
414
391
|
switch (message.tag) {
|
|
415
392
|
case "ToClientInit": {
|
|
416
|
-
const { runnerId, metadata } = message.val;
|
|
393
|
+
const { runnerId, lastEventIdx, metadata } = message.val;
|
|
417
394
|
const metadataStr = `{runnerLostThreshold: ${stringifyBigInt(metadata.runnerLostThreshold)}}`;
|
|
418
|
-
return `ToClientInit{runnerId: "${runnerId}", metadata: ${metadataStr}}`;
|
|
395
|
+
return `ToClientInit{runnerId: "${runnerId}", lastEventIdx: ${stringifyBigInt(lastEventIdx)}, metadata: ${metadataStr}}`;
|
|
419
396
|
}
|
|
420
|
-
case "
|
|
421
|
-
|
|
422
|
-
return `ToClientPing{ts: ${stringifyBigInt(ts)}}`;
|
|
397
|
+
case "ToClientClose":
|
|
398
|
+
return "ToClientClose";
|
|
423
399
|
case "ToClientCommands": {
|
|
424
400
|
const commands = message.val;
|
|
425
401
|
return `ToClientCommands{count: ${commands.length}, commands: [${commands.map((c) => stringifyCommandWrapper(c)).join(", ")}]}`;
|
|
426
402
|
}
|
|
427
403
|
case "ToClientAckEvents": {
|
|
428
|
-
const {
|
|
429
|
-
|
|
430
|
-
return `ToClientAckEvents{lastEventCheckpoints: ${checkpointsStr}}`;
|
|
404
|
+
const { lastEventIdx } = message.val;
|
|
405
|
+
return `ToClientAckEvents{lastEventIdx: ${stringifyBigInt(lastEventIdx)}}`;
|
|
431
406
|
}
|
|
432
407
|
case "ToClientKvResponse": {
|
|
433
408
|
const { requestId, data } = message.val;
|
|
@@ -918,6 +893,11 @@ var WebSocketTunnelAdapter = class {
|
|
|
918
893
|
};
|
|
919
894
|
|
|
920
895
|
// src/tunnel.ts
|
|
896
|
+
var RunnerShutdownError = class extends Error {
|
|
897
|
+
constructor() {
|
|
898
|
+
super("Runner shut down");
|
|
899
|
+
}
|
|
900
|
+
};
|
|
921
901
|
var Tunnel = class {
|
|
922
902
|
#runner;
|
|
923
903
|
/** Maps request IDs to actor IDs for lookup */
|
|
@@ -1039,7 +1019,7 @@ var Tunnel = class {
|
|
|
1039
1019
|
(_a2 = this.log) == null ? void 0 : _a2.error({
|
|
1040
1020
|
msg: "error creating websocket during restore",
|
|
1041
1021
|
requestId: requestIdStr,
|
|
1042
|
-
|
|
1022
|
+
err
|
|
1043
1023
|
});
|
|
1044
1024
|
this.#sendMessage(gatewayId, requestId, {
|
|
1045
1025
|
tag: "ToServerWebSocketClose",
|
|
@@ -1088,7 +1068,7 @@ var Tunnel = class {
|
|
|
1088
1068
|
(_a2 = this.log) == null ? void 0 : _a2.error({
|
|
1089
1069
|
msg: "error creating stale websocket during restore",
|
|
1090
1070
|
requestId: requestIdStr,
|
|
1091
|
-
|
|
1071
|
+
err
|
|
1092
1072
|
});
|
|
1093
1073
|
});
|
|
1094
1074
|
backgroundOperations.push(cleanupOperation);
|
|
@@ -1368,6 +1348,8 @@ var Tunnel = class {
|
|
|
1368
1348
|
message.messageKind.val
|
|
1369
1349
|
);
|
|
1370
1350
|
break;
|
|
1351
|
+
case "DeprecatedTunnelAck":
|
|
1352
|
+
break;
|
|
1371
1353
|
default:
|
|
1372
1354
|
unreachable(message.messageKind);
|
|
1373
1355
|
}
|
|
@@ -1756,14 +1738,10 @@ async function importWebSocket() {
|
|
|
1756
1738
|
|
|
1757
1739
|
// src/mod.ts
|
|
1758
1740
|
var KV_EXPIRE = 3e4;
|
|
1759
|
-
var PROTOCOL_VERSION =
|
|
1741
|
+
var PROTOCOL_VERSION = 3;
|
|
1742
|
+
var RUNNER_PING_INTERVAL = 3e3;
|
|
1760
1743
|
var EVENT_BACKLOG_WARN_THRESHOLD = 1e4;
|
|
1761
1744
|
var SIGNAL_HANDLERS = [];
|
|
1762
|
-
var RunnerShutdownError = class extends Error {
|
|
1763
|
-
constructor() {
|
|
1764
|
-
super("Runner shut down");
|
|
1765
|
-
}
|
|
1766
|
-
};
|
|
1767
1745
|
var Runner = class {
|
|
1768
1746
|
#config;
|
|
1769
1747
|
get config() {
|
|
@@ -1773,6 +1751,9 @@ var Runner = class {
|
|
|
1773
1751
|
// WebSocket
|
|
1774
1752
|
__pegboardWebSocket;
|
|
1775
1753
|
runnerId;
|
|
1754
|
+
#lastCommandIdx = -1;
|
|
1755
|
+
#pingLoop;
|
|
1756
|
+
#nextEventIdx = 0n;
|
|
1776
1757
|
#started = false;
|
|
1777
1758
|
#shutdown = false;
|
|
1778
1759
|
#shuttingDown = false;
|
|
@@ -1782,6 +1763,7 @@ var Runner = class {
|
|
|
1782
1763
|
#runnerLostThreshold;
|
|
1783
1764
|
#runnerLostTimeout;
|
|
1784
1765
|
// Event storage for resending
|
|
1766
|
+
#eventHistory = [];
|
|
1785
1767
|
#eventBacklogWarned = false;
|
|
1786
1768
|
// Command acknowledgment
|
|
1787
1769
|
#ackInterval;
|
|
@@ -1812,15 +1794,7 @@ var Runner = class {
|
|
|
1812
1794
|
this.#config = config;
|
|
1813
1795
|
if (this.#config.logger) setLogger(this.#config.logger);
|
|
1814
1796
|
this.#kvCleanupInterval = setInterval(() => {
|
|
1815
|
-
|
|
1816
|
-
try {
|
|
1817
|
-
this.#cleanupOldKvRequests();
|
|
1818
|
-
} catch (err) {
|
|
1819
|
-
(_a = this.log) == null ? void 0 : _a.error({
|
|
1820
|
-
msg: "error cleaning up kv requests",
|
|
1821
|
-
error: stringifyError(err)
|
|
1822
|
-
});
|
|
1823
|
-
}
|
|
1797
|
+
this.#cleanupOldKvRequests();
|
|
1824
1798
|
}, 15e3);
|
|
1825
1799
|
}
|
|
1826
1800
|
// MARK: Manage actors
|
|
@@ -1847,28 +1821,14 @@ var Runner = class {
|
|
|
1847
1821
|
this.#removeActor(actorId, generation);
|
|
1848
1822
|
this.#sendActorStateUpdate(actorId, actor.generation, "stopped");
|
|
1849
1823
|
}
|
|
1850
|
-
#
|
|
1824
|
+
#stopAllActors() {
|
|
1851
1825
|
var _a;
|
|
1852
1826
|
(_a = this.log) == null ? void 0 : _a.info({
|
|
1853
|
-
msg: "stopping all actors due to runner lost threshold"
|
|
1827
|
+
msg: "stopping all actors due to runner lost threshold exceeded"
|
|
1854
1828
|
});
|
|
1855
|
-
for (const [_, request] of this.#kvRequests.entries()) {
|
|
1856
|
-
request.reject(new RunnerShutdownError());
|
|
1857
|
-
}
|
|
1858
|
-
this.#kvRequests.clear();
|
|
1859
|
-
this.#stopAllActors();
|
|
1860
|
-
}
|
|
1861
|
-
#stopAllActors() {
|
|
1862
1829
|
const actorIds = Array.from(this.#actors.keys());
|
|
1863
1830
|
for (const actorId of actorIds) {
|
|
1864
|
-
this.forceStopActor(actorId)
|
|
1865
|
-
var _a;
|
|
1866
|
-
(_a = this.log) == null ? void 0 : _a.error({
|
|
1867
|
-
msg: "error stopping actor",
|
|
1868
|
-
actorId,
|
|
1869
|
-
error: stringifyError(err)
|
|
1870
|
-
});
|
|
1871
|
-
});
|
|
1831
|
+
this.forceStopActor(actorId);
|
|
1872
1832
|
}
|
|
1873
1833
|
}
|
|
1874
1834
|
getActor(actorId, generation) {
|
|
@@ -1996,6 +1956,10 @@ var Runner = class {
|
|
|
1996
1956
|
clearTimeout(this.#runnerLostTimeout);
|
|
1997
1957
|
this.#runnerLostTimeout = void 0;
|
|
1998
1958
|
}
|
|
1959
|
+
if (this.#pingLoop) {
|
|
1960
|
+
clearInterval(this.#pingLoop);
|
|
1961
|
+
this.#pingLoop = void 0;
|
|
1962
|
+
}
|
|
1999
1963
|
if (this.#ackInterval) {
|
|
2000
1964
|
clearInterval(this.#ackInterval);
|
|
2001
1965
|
this.#ackInterval = void 0;
|
|
@@ -2203,6 +2167,7 @@ var Runner = class {
|
|
|
2203
2167
|
name: this.#config.runnerName,
|
|
2204
2168
|
version: this.#config.version,
|
|
2205
2169
|
totalSlots: this.#config.totalSlots,
|
|
2170
|
+
lastCommandIdx: this.#lastCommandIdx >= 0 ? BigInt(this.#lastCommandIdx) : null,
|
|
2206
2171
|
prepopulateActorNames: new Map(
|
|
2207
2172
|
Object.entries(this.#config.prepopulateActorNames).map(
|
|
2208
2173
|
([name, data]) => [
|
|
@@ -2217,29 +2182,39 @@ var Runner = class {
|
|
|
2217
2182
|
tag: "ToServerInit",
|
|
2218
2183
|
val: init
|
|
2219
2184
|
});
|
|
2185
|
+
const pingLoop = setInterval(() => {
|
|
2186
|
+
var _a3;
|
|
2187
|
+
if (ws.readyState === 1) {
|
|
2188
|
+
this.__sendToServer({
|
|
2189
|
+
tag: "ToServerPing",
|
|
2190
|
+
val: {
|
|
2191
|
+
ts: BigInt(Date.now())
|
|
2192
|
+
}
|
|
2193
|
+
});
|
|
2194
|
+
} else {
|
|
2195
|
+
clearInterval(pingLoop);
|
|
2196
|
+
(_a3 = this.log) == null ? void 0 : _a3.info({
|
|
2197
|
+
msg: "WebSocket not open, stopping ping loop"
|
|
2198
|
+
});
|
|
2199
|
+
}
|
|
2200
|
+
}, RUNNER_PING_INTERVAL);
|
|
2201
|
+
this.#pingLoop = pingLoop;
|
|
2220
2202
|
const ackInterval = 5 * 60 * 1e3;
|
|
2221
2203
|
const ackLoop = setInterval(() => {
|
|
2222
|
-
var _a3
|
|
2223
|
-
|
|
2224
|
-
|
|
2225
|
-
|
|
2226
|
-
|
|
2227
|
-
|
|
2228
|
-
|
|
2229
|
-
msg: "WebSocket not open, stopping ack loop"
|
|
2230
|
-
});
|
|
2231
|
-
}
|
|
2232
|
-
} catch (err) {
|
|
2233
|
-
(_b2 = this.log) == null ? void 0 : _b2.error({
|
|
2234
|
-
msg: "error in command acknowledgment loop",
|
|
2235
|
-
error: stringifyError(err)
|
|
2204
|
+
var _a3;
|
|
2205
|
+
if (ws.readyState === 1) {
|
|
2206
|
+
this.#sendCommandAcknowledgment();
|
|
2207
|
+
} else {
|
|
2208
|
+
clearInterval(ackLoop);
|
|
2209
|
+
(_a3 = this.log) == null ? void 0 : _a3.info({
|
|
2210
|
+
msg: "WebSocket not open, stopping ack loop"
|
|
2236
2211
|
});
|
|
2237
2212
|
}
|
|
2238
2213
|
}, ackInterval);
|
|
2239
2214
|
this.#ackInterval = ackLoop;
|
|
2240
2215
|
});
|
|
2241
2216
|
ws.addEventListener("message", async (ev) => {
|
|
2242
|
-
var _a2, _b, _c, _d, _e;
|
|
2217
|
+
var _a2, _b, _c, _d, _e, _f;
|
|
2243
2218
|
let buf;
|
|
2244
2219
|
if (ev.data instanceof Blob) {
|
|
2245
2220
|
buf = new Uint8Array(await ev.data.arrayBuffer());
|
|
@@ -2257,15 +2232,16 @@ var Runner = class {
|
|
|
2257
2232
|
const init = message.val;
|
|
2258
2233
|
if (this.runnerId !== init.runnerId) {
|
|
2259
2234
|
this.runnerId = init.runnerId;
|
|
2260
|
-
this.#
|
|
2235
|
+
this.#eventHistory.length = 0;
|
|
2261
2236
|
}
|
|
2262
2237
|
this.#runnerLostThreshold = ((_b = init.metadata) == null ? void 0 : _b.runnerLostThreshold) ? Number(init.metadata.runnerLostThreshold) : void 0;
|
|
2263
2238
|
(_c = this.log) == null ? void 0 : _c.info({
|
|
2264
2239
|
msg: "received init",
|
|
2240
|
+
lastEventIdx: init.lastEventIdx,
|
|
2265
2241
|
runnerLostThreshold: this.#runnerLostThreshold
|
|
2266
2242
|
});
|
|
2267
2243
|
this.#processUnsentKvRequests();
|
|
2268
|
-
this.#resendUnacknowledgedEvents();
|
|
2244
|
+
this.#resendUnacknowledgedEvents(init.lastEventIdx);
|
|
2269
2245
|
(_d = this.#tunnel) == null ? void 0 : _d.resendBufferedEvents();
|
|
2270
2246
|
this.#config.onConnected();
|
|
2271
2247
|
} else if (message.tag === "ToClientCommands") {
|
|
@@ -2277,20 +2253,10 @@ var Runner = class {
|
|
|
2277
2253
|
const kvResponse = message.val;
|
|
2278
2254
|
this.#handleKvResponse(kvResponse);
|
|
2279
2255
|
} else if (message.tag === "ToClientTunnelMessage") {
|
|
2280
|
-
(_e = this.#tunnel) == null ? void 0 : _e.handleTunnelMessage(message.val)
|
|
2281
|
-
|
|
2282
|
-
|
|
2283
|
-
|
|
2284
|
-
error: stringifyError(err)
|
|
2285
|
-
});
|
|
2286
|
-
});
|
|
2287
|
-
} else if (message.tag === "ToClientPing") {
|
|
2288
|
-
this.__sendToServer({
|
|
2289
|
-
tag: "ToServerPong",
|
|
2290
|
-
val: {
|
|
2291
|
-
ts: message.val.ts
|
|
2292
|
-
}
|
|
2293
|
-
});
|
|
2256
|
+
(_e = this.#tunnel) == null ? void 0 : _e.handleTunnelMessage(message.val);
|
|
2257
|
+
} else if (message.tag === "ToClientClose") {
|
|
2258
|
+
(_f = this.#tunnel) == null ? void 0 : _f.shutdown();
|
|
2259
|
+
ws.close(1e3, "manual closure");
|
|
2294
2260
|
} else {
|
|
2295
2261
|
unreachable(message);
|
|
2296
2262
|
}
|
|
@@ -2307,15 +2273,7 @@ var Runner = class {
|
|
|
2307
2273
|
seconds: this.#runnerLostThreshold / 1e3
|
|
2308
2274
|
});
|
|
2309
2275
|
this.#runnerLostTimeout = setTimeout(() => {
|
|
2310
|
-
|
|
2311
|
-
try {
|
|
2312
|
-
this.#handleLost();
|
|
2313
|
-
} catch (err) {
|
|
2314
|
-
(_a3 = this.log) == null ? void 0 : _a3.error({
|
|
2315
|
-
msg: "error handling runner lost",
|
|
2316
|
-
error: stringifyError(err)
|
|
2317
|
-
});
|
|
2318
|
-
}
|
|
2276
|
+
this.#stopAllActors();
|
|
2319
2277
|
}, this.#runnerLostThreshold);
|
|
2320
2278
|
}
|
|
2321
2279
|
this.#scheduleReconnect();
|
|
@@ -2341,6 +2299,10 @@ var Runner = class {
|
|
|
2341
2299
|
}
|
|
2342
2300
|
this.#config.onDisconnected(ev.code, ev.reason);
|
|
2343
2301
|
}
|
|
2302
|
+
if (this.#pingLoop) {
|
|
2303
|
+
clearInterval(this.#pingLoop);
|
|
2304
|
+
this.#pingLoop = void 0;
|
|
2305
|
+
}
|
|
2344
2306
|
if (this.#ackInterval) {
|
|
2345
2307
|
clearInterval(this.#ackInterval);
|
|
2346
2308
|
this.#ackInterval = void 0;
|
|
@@ -2352,15 +2314,7 @@ var Runner = class {
|
|
|
2352
2314
|
seconds: this.#runnerLostThreshold / 1e3
|
|
2353
2315
|
});
|
|
2354
2316
|
this.#runnerLostTimeout = setTimeout(() => {
|
|
2355
|
-
|
|
2356
|
-
try {
|
|
2357
|
-
this.#handleLost();
|
|
2358
|
-
} catch (err) {
|
|
2359
|
-
(_a3 = this.log) == null ? void 0 : _a3.error({
|
|
2360
|
-
msg: "error handling runner lost",
|
|
2361
|
-
error: stringifyError(err)
|
|
2362
|
-
});
|
|
2363
|
-
}
|
|
2317
|
+
this.#stopAllActors();
|
|
2364
2318
|
}, this.#runnerLostThreshold);
|
|
2365
2319
|
}
|
|
2366
2320
|
this.#scheduleReconnect();
|
|
@@ -2375,75 +2329,43 @@ var Runner = class {
|
|
|
2375
2329
|
});
|
|
2376
2330
|
for (const commandWrapper of commands) {
|
|
2377
2331
|
if (commandWrapper.inner.tag === "CommandStartActor") {
|
|
2378
|
-
this.#handleCommandStartActor(commandWrapper)
|
|
2379
|
-
var _a2;
|
|
2380
|
-
(_a2 = this.log) == null ? void 0 : _a2.error({
|
|
2381
|
-
msg: "error handling start actor command",
|
|
2382
|
-
actorId: commandWrapper.checkpoint.actorId,
|
|
2383
|
-
error: stringifyError(err)
|
|
2384
|
-
});
|
|
2385
|
-
});
|
|
2332
|
+
this.#handleCommandStartActor(commandWrapper);
|
|
2386
2333
|
} else if (commandWrapper.inner.tag === "CommandStopActor") {
|
|
2387
|
-
this.#handleCommandStopActor(commandWrapper)
|
|
2388
|
-
var _a2;
|
|
2389
|
-
(_a2 = this.log) == null ? void 0 : _a2.error({
|
|
2390
|
-
msg: "error handling stop actor command",
|
|
2391
|
-
actorId: commandWrapper.checkpoint.actorId,
|
|
2392
|
-
error: stringifyError(err)
|
|
2393
|
-
});
|
|
2394
|
-
});
|
|
2334
|
+
this.#handleCommandStopActor(commandWrapper);
|
|
2395
2335
|
} else {
|
|
2396
2336
|
unreachable(commandWrapper.inner);
|
|
2397
2337
|
}
|
|
2398
|
-
|
|
2399
|
-
commandWrapper.checkpoint.actorId,
|
|
2400
|
-
commandWrapper.inner.val.generation
|
|
2401
|
-
);
|
|
2402
|
-
if (actor) actor.lastCommandIdx = commandWrapper.checkpoint.index;
|
|
2338
|
+
this.#lastCommandIdx = Number(commandWrapper.index);
|
|
2403
2339
|
}
|
|
2404
2340
|
}
|
|
2405
2341
|
#handleAckEvents(ack) {
|
|
2406
2342
|
var _a;
|
|
2407
|
-
|
|
2408
|
-
|
|
2409
|
-
|
|
2410
|
-
|
|
2411
|
-
for (const [_, actor] of this.#actors) {
|
|
2412
|
-
let checkpoint = ack.lastEventCheckpoints.find(
|
|
2413
|
-
(x) => x.actorId == actor.actorId
|
|
2414
|
-
);
|
|
2415
|
-
if (checkpoint) actor.handleAckEvents(checkpoint.index);
|
|
2416
|
-
}
|
|
2417
|
-
const totalEvents = Array.from(this.#actors).reduce(
|
|
2418
|
-
(s, [_, actor]) => s + actor.eventHistory.length,
|
|
2419
|
-
0
|
|
2343
|
+
const lastAckedIdx = ack.lastEventIdx;
|
|
2344
|
+
const originalLength = this.#eventHistory.length;
|
|
2345
|
+
this.#eventHistory = this.#eventHistory.filter(
|
|
2346
|
+
(event) => event.index > lastAckedIdx
|
|
2420
2347
|
);
|
|
2421
|
-
const prunedCount =
|
|
2348
|
+
const prunedCount = originalLength - this.#eventHistory.length;
|
|
2422
2349
|
if (prunedCount > 0) {
|
|
2423
2350
|
(_a = this.log) == null ? void 0 : _a.info({
|
|
2424
2351
|
msg: "pruned acknowledged events",
|
|
2352
|
+
lastAckedIdx: lastAckedIdx.toString(),
|
|
2425
2353
|
prunedCount
|
|
2426
2354
|
});
|
|
2427
2355
|
}
|
|
2428
|
-
if (
|
|
2356
|
+
if (this.#eventHistory.length <= EVENT_BACKLOG_WARN_THRESHOLD) {
|
|
2429
2357
|
this.#eventBacklogWarned = false;
|
|
2430
2358
|
}
|
|
2431
2359
|
}
|
|
2432
2360
|
/** Track events to send to the server in case we need to resend it on disconnect. */
|
|
2433
2361
|
#recordEvent(eventWrapper) {
|
|
2434
2362
|
var _a;
|
|
2435
|
-
|
|
2436
|
-
if (!
|
|
2437
|
-
actor.recordEvent(eventWrapper);
|
|
2438
|
-
let totalEvents = Array.from(this.#actors).reduce(
|
|
2439
|
-
(s, [_, actor2]) => s + actor2.eventHistory.length,
|
|
2440
|
-
0
|
|
2441
|
-
);
|
|
2442
|
-
if (totalEvents > EVENT_BACKLOG_WARN_THRESHOLD && !this.#eventBacklogWarned) {
|
|
2363
|
+
this.#eventHistory.push(eventWrapper);
|
|
2364
|
+
if (this.#eventHistory.length > EVENT_BACKLOG_WARN_THRESHOLD && !this.#eventBacklogWarned) {
|
|
2443
2365
|
this.#eventBacklogWarned = true;
|
|
2444
2366
|
(_a = this.log) == null ? void 0 : _a.warn({
|
|
2445
2367
|
msg: "unacknowledged event backlog exceeds threshold",
|
|
2446
|
-
backlogSize:
|
|
2368
|
+
backlogSize: this.#eventHistory.length,
|
|
2447
2369
|
threshold: EVENT_BACKLOG_WARN_THRESHOLD
|
|
2448
2370
|
});
|
|
2449
2371
|
}
|
|
@@ -2452,7 +2374,7 @@ var Runner = class {
|
|
|
2452
2374
|
var _a, _b, _c, _d;
|
|
2453
2375
|
if (!this.#tunnel) throw new Error("missing tunnel on actor start");
|
|
2454
2376
|
const startCommand = commandWrapper.inner.val;
|
|
2455
|
-
const actorId =
|
|
2377
|
+
const actorId = startCommand.actorId;
|
|
2456
2378
|
const generation = startCommand.generation;
|
|
2457
2379
|
const config = startCommand.config;
|
|
2458
2380
|
const actorConfig = {
|
|
@@ -2511,13 +2433,11 @@ var Runner = class {
|
|
|
2511
2433
|
}
|
|
2512
2434
|
async #handleCommandStopActor(commandWrapper) {
|
|
2513
2435
|
const stopCommand = commandWrapper.inner.val;
|
|
2514
|
-
const actorId =
|
|
2436
|
+
const actorId = stopCommand.actorId;
|
|
2515
2437
|
const generation = stopCommand.generation;
|
|
2516
2438
|
await this.forceStopActor(actorId, generation);
|
|
2517
2439
|
}
|
|
2518
2440
|
#sendActorIntent(actorId, generation, intentType) {
|
|
2519
|
-
const actor = this.getActor(actorId, generation);
|
|
2520
|
-
if (!actor) return;
|
|
2521
2441
|
let actorIntent;
|
|
2522
2442
|
if (intentType === "sleep") {
|
|
2523
2443
|
actorIntent = { tag: "ActorIntentSleep", val: null };
|
|
@@ -2534,11 +2454,9 @@ var Runner = class {
|
|
|
2534
2454
|
generation,
|
|
2535
2455
|
intent: actorIntent
|
|
2536
2456
|
};
|
|
2457
|
+
const eventIndex = this.#nextEventIdx++;
|
|
2537
2458
|
const eventWrapper = {
|
|
2538
|
-
|
|
2539
|
-
actorId,
|
|
2540
|
-
index: actor.nextEventIdx++
|
|
2541
|
-
},
|
|
2459
|
+
index: eventIndex,
|
|
2542
2460
|
inner: {
|
|
2543
2461
|
tag: "EventActorIntent",
|
|
2544
2462
|
val: intentEvent
|
|
@@ -2551,8 +2469,6 @@ var Runner = class {
|
|
|
2551
2469
|
});
|
|
2552
2470
|
}
|
|
2553
2471
|
#sendActorStateUpdate(actorId, generation, stateType) {
|
|
2554
|
-
const actor = this.getActor(actorId, generation);
|
|
2555
|
-
if (!actor) return;
|
|
2556
2472
|
let actorState;
|
|
2557
2473
|
if (stateType === "running") {
|
|
2558
2474
|
actorState = { tag: "ActorStateRunning", val: null };
|
|
@@ -2572,11 +2488,9 @@ var Runner = class {
|
|
|
2572
2488
|
generation,
|
|
2573
2489
|
state: actorState
|
|
2574
2490
|
};
|
|
2491
|
+
const eventIndex = this.#nextEventIdx++;
|
|
2575
2492
|
const eventWrapper = {
|
|
2576
|
-
|
|
2577
|
-
actorId,
|
|
2578
|
-
index: actor.nextEventIdx++
|
|
2579
|
-
},
|
|
2493
|
+
index: eventIndex,
|
|
2580
2494
|
inner: {
|
|
2581
2495
|
tag: "EventActorStateUpdate",
|
|
2582
2496
|
val: stateUpdateEvent
|
|
@@ -2589,20 +2503,13 @@ var Runner = class {
|
|
|
2589
2503
|
});
|
|
2590
2504
|
}
|
|
2591
2505
|
#sendCommandAcknowledgment() {
|
|
2592
|
-
|
|
2593
|
-
|
|
2594
|
-
if (actor.lastCommandIdx < 0) {
|
|
2595
|
-
continue;
|
|
2596
|
-
}
|
|
2597
|
-
lastCommandCheckpoints.push({
|
|
2598
|
-
actorId: actor.actorId,
|
|
2599
|
-
index: actor.lastCommandIdx
|
|
2600
|
-
});
|
|
2506
|
+
if (this.#lastCommandIdx < 0) {
|
|
2507
|
+
return;
|
|
2601
2508
|
}
|
|
2602
2509
|
this.__sendToServer({
|
|
2603
2510
|
tag: "ToServerAckCommands",
|
|
2604
2511
|
val: {
|
|
2605
|
-
|
|
2512
|
+
lastCommandIdx: BigInt(this.#lastCommandIdx)
|
|
2606
2513
|
}
|
|
2607
2514
|
});
|
|
2608
2515
|
}
|
|
@@ -2836,11 +2743,9 @@ var Runner = class {
|
|
|
2836
2743
|
generation: actor.generation,
|
|
2837
2744
|
alarmTs: alarmTs !== null ? BigInt(alarmTs) : null
|
|
2838
2745
|
};
|
|
2746
|
+
const eventIndex = this.#nextEventIdx++;
|
|
2839
2747
|
const eventWrapper = {
|
|
2840
|
-
|
|
2841
|
-
actorId,
|
|
2842
|
-
index: actor.nextEventIdx++
|
|
2843
|
-
},
|
|
2748
|
+
index: eventIndex,
|
|
2844
2749
|
inner: {
|
|
2845
2750
|
tag: "EventActorSetAlarm",
|
|
2846
2751
|
val: alarmEvent
|
|
@@ -2967,8 +2872,7 @@ var Runner = class {
|
|
|
2967
2872
|
const data = protocol.encodeToServerlessServer({
|
|
2968
2873
|
tag: "ToServerlessServerInit",
|
|
2969
2874
|
val: {
|
|
2970
|
-
runnerId: this.runnerId
|
|
2971
|
-
runnerProtocolVersion: PROTOCOL_VERSION
|
|
2875
|
+
runnerId: this.runnerId
|
|
2972
2876
|
}
|
|
2973
2877
|
});
|
|
2974
2878
|
const buffer = Buffer.alloc(data.length + 2);
|
|
@@ -2993,33 +2897,26 @@ var Runner = class {
|
|
|
2993
2897
|
(_b = this.log) == null ? void 0 : _b.debug({
|
|
2994
2898
|
msg: `Scheduling reconnect attempt ${this.#reconnectAttempt + 1} in ${delay}ms`
|
|
2995
2899
|
});
|
|
2996
|
-
this.#reconnectTimeout = setTimeout(() => {
|
|
2900
|
+
this.#reconnectTimeout = setTimeout(async () => {
|
|
2997
2901
|
var _a2;
|
|
2998
2902
|
if (!this.#shutdown) {
|
|
2999
2903
|
this.#reconnectAttempt++;
|
|
3000
2904
|
(_a2 = this.log) == null ? void 0 : _a2.debug({
|
|
3001
2905
|
msg: `Attempting to reconnect (attempt ${this.#reconnectAttempt})...`
|
|
3002
2906
|
});
|
|
3003
|
-
this.#openPegboardWebSocket()
|
|
3004
|
-
var _a3;
|
|
3005
|
-
(_a3 = this.log) == null ? void 0 : _a3.error({
|
|
3006
|
-
msg: "error during websocket reconnection",
|
|
3007
|
-
error: stringifyError(err)
|
|
3008
|
-
});
|
|
3009
|
-
});
|
|
2907
|
+
await this.#openPegboardWebSocket();
|
|
3010
2908
|
}
|
|
3011
2909
|
}, delay);
|
|
3012
2910
|
}
|
|
3013
|
-
#resendUnacknowledgedEvents() {
|
|
2911
|
+
#resendUnacknowledgedEvents(lastEventIdx) {
|
|
3014
2912
|
var _a;
|
|
3015
|
-
const eventsToResend =
|
|
3016
|
-
|
|
3017
|
-
|
|
3018
|
-
}
|
|
2913
|
+
const eventsToResend = this.#eventHistory.filter(
|
|
2914
|
+
(event) => event.index > lastEventIdx
|
|
2915
|
+
);
|
|
3019
2916
|
if (eventsToResend.length === 0) return;
|
|
3020
2917
|
(_a = this.log) == null ? void 0 : _a.info({
|
|
3021
2918
|
msg: "resending unacknowledged events",
|
|
3022
|
-
|
|
2919
|
+
fromIndex: lastEventIdx + 1n
|
|
3023
2920
|
});
|
|
3024
2921
|
this.__sendToServer({
|
|
3025
2922
|
tag: "ToServerEvents",
|
|
@@ -3049,7 +2946,6 @@ var Runner = class {
|
|
|
3049
2946
|
export {
|
|
3050
2947
|
Runner,
|
|
3051
2948
|
RunnerActor,
|
|
3052
|
-
RunnerShutdownError,
|
|
3053
2949
|
idToStr
|
|
3054
2950
|
};
|
|
3055
2951
|
//# sourceMappingURL=mod.js.map
|