@rivetkit/engine-runner 2.0.26 → 2.0.27

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rivetkit/engine-runner",
-  "version": "2.0.26",
+  "version": "2.0.27",
   "type": "module",
   "exports": {
     "import": {
@@ -16,7 +16,7 @@
     "uuid": "^12.0.0",
     "pino": "^9.9.5",
     "ws": "^8.18.3",
-    "@rivetkit/engine-runner-protocol": "2.0.26"
+    "@rivetkit/engine-runner-protocol": "2.0.27"
   },
   "devDependencies": {
     "@types/node": "^22.18.1",

package/src/actor.ts

@@ -27,10 +27,6 @@ export class RunnerActor {
 	}> = [];
 	actorStartPromise: ReturnType<typeof promiseWithResolvers<void>>;
 
-	lastCommandIdx: bigint = -1n;
-	nextEventIdx: bigint = 0n;
-	eventHistory: protocol.EventWrapper[] = [];
-
 	/**
 	 * If restoreHibernatingRequests has been called. This is used to assert
 	 * that the caller is implemented correctly.
@@ -85,8 +81,8 @@ export class RunnerActor {
 			gatewayId,
 			requestId,
 			request: {
-				resolve: () => { },
-				reject: () => { },
+				resolve: () => {},
+				reject: () => {},
 				actorId: this.actorId,
 				gatewayId: gatewayId,
 				requestId: requestId,
@@ -122,8 +118,8 @@ export class RunnerActor {
 			gatewayId,
 			requestId,
 			request: {
-				resolve: () => { },
-				reject: () => { },
+				resolve: () => {},
+				reject: () => {},
 				actorId: this.actorId,
 				gatewayId: gatewayId,
 				requestId: requestId,
@@ -197,14 +193,4 @@ export class RunnerActor {
 			this.webSockets.splice(index, 1);
 		}
 	}
-
-	handleAckEvents(lastEventIdx: bigint) {
-		this.eventHistory = this.eventHistory.filter(
-			(event) => event.checkpoint.index > lastEventIdx,
-		);
-	}
-
-	recordEvent(eventWrapper: protocol.EventWrapper) {
-		this.eventHistory.push(eventWrapper);
-	}
 }

package/src/mod.ts

@@ -8,7 +8,6 @@ import { type HibernatingWebSocketMetadata, Tunnel } from "./tunnel";
 import {
 	calculateBackoff,
 	parseWebSocketCloseReason,
-	stringifyError,
 	unreachable,
 } from "./utils";
 import { importWebSocket } from "./websocket.js";
@@ -18,18 +17,13 @@ export { RunnerActor, type ActorConfig };
 export { idToStr } from "./utils";
 
 const KV_EXPIRE: number = 30_000;
-const PROTOCOL_VERSION: number = 4;
+const PROTOCOL_VERSION: number = 3;
+const RUNNER_PING_INTERVAL = 3_000;
 
 /** Warn once the backlog significantly exceeds the server's ack batch size. */
 const EVENT_BACKLOG_WARN_THRESHOLD = 10_000;
 const SIGNAL_HANDLERS: (() => void | Promise<void>)[] = [];
 
-export class RunnerShutdownError extends Error {
-	constructor() {
-		super("Runner shut down");
-	}
-}
-
 export interface RunnerConfig {
 	logger?: Logger;
 	version: number;
@@ -203,6 +197,9 @@ export class Runner {
 	// WebSocket
 	__pegboardWebSocket?: WebSocket;
 	runnerId?: string;
+	#lastCommandIdx: number = -1;
+	#pingLoop?: NodeJS.Timeout;
+	#nextEventIdx: bigint = 0n;
 	#started: boolean = false;
 	#shutdown: boolean = false;
 	#shuttingDown: boolean = false;
@@ -214,6 +211,7 @@ export class Runner {
 	#runnerLostTimeout?: NodeJS.Timeout;
 
 	// Event storage for resending
+	#eventHistory: protocol.EventWrapper[] = [];
 	#eventBacklogWarned: boolean = false;
 
 	// Command acknowledgment
@@ -257,14 +255,7 @@ export class Runner {
 
 		// Start cleaning up old unsent KV requests every 15 seconds
 		this.#kvCleanupInterval = setInterval(() => {
-			try {
-				this.#cleanupOldKvRequests();
-			} catch (err) {
-				this.log?.error({
-					msg: "error cleaning up kv requests",
-					error: stringifyError(err),
-				});
-			}
+			this.#cleanupOldKvRequests();
 		}, 15000); // Run every 15 seconds
 	}
 
@@ -316,31 +307,14 @@ export class Runner {
 		this.#sendActorStateUpdate(actorId, actor.generation, "stopped");
 	}
 
-	#handleLost() {
+	#stopAllActors() {
 		this.log?.info({
-			msg: "stopping all actors due to runner lost threshold",
+			msg: "stopping all actors due to runner lost threshold exceeded",
 		});
 
-		// Remove all remaining kv requests
-		for (const [_, request] of this.#kvRequests.entries()) {
-			request.reject(new RunnerShutdownError());
-		}
-
-		this.#kvRequests.clear();
-
-		this.#stopAllActors();
-	}
-
-	#stopAllActors() {
 		const actorIds = Array.from(this.#actors.keys());
 		for (const actorId of actorIds) {
-			this.forceStopActor(actorId).catch((err) => {
-				this.log?.error({
-					msg: "error stopping actor",
-					actorId,
-					error: stringifyError(err),
-				});
-			});
+			this.forceStopActor(actorId);
 		}
 	}
 
@@ -503,6 +477,12 @@ export class Runner {
 			this.#runnerLostTimeout = undefined;
 		}
 
+		// Clear ping loop
+		if (this.#pingLoop) {
+			clearInterval(this.#pingLoop);
+			this.#pingLoop = undefined;
+		}
+
 		// Clear ack interval
 		if (this.#ackInterval) {
 			clearInterval(this.#ackInterval);
@@ -758,6 +738,10 @@ export class Runner {
 				name: this.#config.runnerName,
 				version: this.#config.version,
 				totalSlots: this.#config.totalSlots,
+				lastCommandIdx:
+					this.#lastCommandIdx >= 0
+						? BigInt(this.#lastCommandIdx)
+						: null,
 				prepopulateActorNames: new Map(
 					Object.entries(this.#config.prepopulateActorNames).map(
 						([name, data]) => [
@@ -774,22 +758,33 @@ export class Runner {
 				val: init,
 			});
 
+			// Start ping interval
+			const pingLoop = setInterval(() => {
+				if (ws.readyState === 1) {
+					this.__sendToServer({
+						tag: "ToServerPing",
+						val: {
+							ts: BigInt(Date.now()),
+						},
+					});
+				} else {
+					clearInterval(pingLoop);
+					this.log?.info({
+						msg: "WebSocket not open, stopping ping loop",
+					});
+				}
+			}, RUNNER_PING_INTERVAL);
+			this.#pingLoop = pingLoop;
+
 			// Start command acknowledgment interval (5 minutes)
 			const ackInterval = 5 * 60 * 1000; // 5 minutes in milliseconds
 			const ackLoop = setInterval(() => {
-				try {
-					if (ws.readyState === 1) {
-						this.#sendCommandAcknowledgment();
-					} else {
-						clearInterval(ackLoop);
-						this.log?.info({
-							msg: "WebSocket not open, stopping ack loop",
-						});
-					}
-				} catch (err) {
-					this.log?.error({
-						msg: "error in command acknowledgment loop",
-						error: stringifyError(err),
+				if (ws.readyState === 1) {
+					this.#sendCommandAcknowledgment();
+				} else {
+					clearInterval(ackLoop);
+					this.log?.info({
+						msg: "WebSocket not open, stopping ack loop",
 					});
 				}
 			}, ackInterval);
@@ -820,8 +815,8 @@ export class Runner {
 				if (this.runnerId !== init.runnerId) {
 					this.runnerId = init.runnerId;
 
-					// Clear actors if runner id changed
-					this.#stopAllActors();
+					// Clear history if runner id changed
+					this.#eventHistory.length = 0;
 				}
 
 				// Store the runner lost threshold from metadata
@@ -831,12 +826,13 @@ export class Runner {
 
 				this.log?.info({
 					msg: "received init",
+					lastEventIdx: init.lastEventIdx,
 					runnerLostThreshold: this.#runnerLostThreshold,
 				});
 
 				// Resend pending events
 				this.#processUnsentKvRequests();
-				this.#resendUnacknowledgedEvents();
+				this.#resendUnacknowledgedEvents(init.lastEventIdx);
 				this.#tunnel?.resendBufferedEvents();
 
 				this.#config.onConnected();
@@ -849,19 +845,10 @@ export class Runner {
 				const kvResponse = message.val;
 				this.#handleKvResponse(kvResponse);
 			} else if (message.tag === "ToClientTunnelMessage") {
-				this.#tunnel?.handleTunnelMessage(message.val).catch((err) => {
-					this.log?.error({
-						msg: "error handling tunnel message",
-						error: stringifyError(err),
-					});
-				});
-			} else if (message.tag === "ToClientPing") {
-				this.__sendToServer({
-					tag: "ToServerPong",
-					val: {
-						ts: message.val.ts,
-					},
-				});
+				this.#tunnel?.handleTunnelMessage(message.val);
+			} else if (message.tag === "ToClientClose") {
+				this.#tunnel?.shutdown();
+				ws.close(1000, "manual closure");
 			} else {
 				unreachable(message);
 			}
@@ -884,14 +871,7 @@ export class Runner {
 						seconds: this.#runnerLostThreshold / 1000,
 					});
 					this.#runnerLostTimeout = setTimeout(() => {
-						try {
-							this.#handleLost();
-						} catch (err) {
-							this.log?.error({
-								msg: "error handling runner lost",
-								error: stringifyError(err),
-							});
-						}
+						this.#stopAllActors();
 					}, this.#runnerLostThreshold);
 				}
 
@@ -929,6 +909,12 @@ export class Runner {
 				this.#config.onDisconnected(ev.code, ev.reason);
 			}
 
+			// Clear ping loop on close
+			if (this.#pingLoop) {
+				clearInterval(this.#pingLoop);
+				this.#pingLoop = undefined;
+			}
+
 			// Clear ack interval on close
 			if (this.#ackInterval) {
 				clearInterval(this.#ackInterval);
@@ -947,14 +933,7 @@ export class Runner {
 						seconds: this.#runnerLostThreshold / 1000,
 					});
 					this.#runnerLostTimeout = setTimeout(() => {
-						try {
-							this.#handleLost();
-						} catch (err) {
-							this.log?.error({
-								msg: "error handling runner lost",
-								error: stringifyError(err),
-							});
-						}
+						this.#stopAllActors();
 					}, this.#runnerLostThreshold);
 				}
 
@@ -973,86 +952,52 @@ export class Runner {
 		for (const commandWrapper of commands) {
 			if (commandWrapper.inner.tag === "CommandStartActor") {
 				// Spawn background promise
-				this.#handleCommandStartActor(commandWrapper).catch((err) => {
-					this.log?.error({
-						msg: "error handling start actor command",
-						actorId: commandWrapper.checkpoint.actorId,
-						error: stringifyError(err),
-					});
-				});
+				this.#handleCommandStartActor(commandWrapper);
 			} else if (commandWrapper.inner.tag === "CommandStopActor") {
 				// Spawn background promise
-				this.#handleCommandStopActor(commandWrapper).catch((err) => {
-					this.log?.error({
-						msg: "error handling stop actor command",
-						actorId: commandWrapper.checkpoint.actorId,
-						error: stringifyError(err),
-					});
-				});
+				this.#handleCommandStopActor(commandWrapper);
 			} else {
 				unreachable(commandWrapper.inner);
 			}
 
-			const actor = this.getActor(
-				commandWrapper.checkpoint.actorId,
-				commandWrapper.inner.val.generation,
-			);
-			if (actor) actor.lastCommandIdx = commandWrapper.checkpoint.index;
+			this.#lastCommandIdx = Number(commandWrapper.index);
 		}
 	}
 
 	#handleAckEvents(ack: protocol.ToClientAckEvents) {
-		let originalTotalEvents = Array.from(this.#actors).reduce(
-			(s, [_, actor]) => s + actor.eventHistory.length,
-			0,
-		);
-
-		for (const [_, actor] of this.#actors) {
-			let checkpoint = ack.lastEventCheckpoints.find(
-				(x) => x.actorId == actor.actorId,
-			);
-
-			if (checkpoint) actor.handleAckEvents(checkpoint.index);
-		}
+		const lastAckedIdx = ack.lastEventIdx;
 
-		const totalEvents = Array.from(this.#actors).reduce(
-			(s, [_, actor]) => s + actor.eventHistory.length,
-			0,
+		const originalLength = this.#eventHistory.length;
+		this.#eventHistory = this.#eventHistory.filter(
+			(event) => event.index > lastAckedIdx,
 		);
-		const prunedCount = originalTotalEvents - totalEvents;
 
+		const prunedCount = originalLength - this.#eventHistory.length;
 		if (prunedCount > 0) {
 			this.log?.info({
 				msg: "pruned acknowledged events",
+				lastAckedIdx: lastAckedIdx.toString(),
 				prunedCount,
 			});
 		}
 
-		if (totalEvents <= EVENT_BACKLOG_WARN_THRESHOLD) {
+		if (this.#eventHistory.length <= EVENT_BACKLOG_WARN_THRESHOLD) {
 			this.#eventBacklogWarned = false;
 		}
 	}
 
 	/** Track events to send to the server in case we need to resend it on disconnect. */
 	#recordEvent(eventWrapper: protocol.EventWrapper) {
-		const actor = this.getActor(eventWrapper.checkpoint.actorId);
-		if (!actor) return;
-
-		actor.recordEvent(eventWrapper);
-
-		let totalEvents = Array.from(this.#actors).reduce(
-			(s, [_, actor]) => s + actor.eventHistory.length,
-			0,
-		);
+		this.#eventHistory.push(eventWrapper);
 
 		if (
-			totalEvents > EVENT_BACKLOG_WARN_THRESHOLD &&
+			this.#eventHistory.length > EVENT_BACKLOG_WARN_THRESHOLD &&
 			!this.#eventBacklogWarned
 		) {
 			this.#eventBacklogWarned = true;
 			this.log?.warn({
 				msg: "unacknowledged event backlog exceeds threshold",
-				backlogSize: totalEvents,
+				backlogSize: this.#eventHistory.length,
 				threshold: EVENT_BACKLOG_WARN_THRESHOLD,
 			});
 		}
@@ -1068,7 +1013,7 @@ export class Runner {
 		const startCommand = commandWrapper.inner
 			.val as protocol.CommandStartActor;
 
-		const actorId = commandWrapper.checkpoint.actorId;
+		const actorId = startCommand.actorId;
 		const generation = startCommand.generation;
 		const config = startCommand.config;
 
@@ -1149,7 +1094,7 @@ export class Runner {
 		const stopCommand = commandWrapper.inner
 			.val as protocol.CommandStopActor;
 
-		const actorId = commandWrapper.checkpoint.actorId;
+		const actorId = stopCommand.actorId;
 		const generation = stopCommand.generation;
 
 		await this.forceStopActor(actorId, generation);
@@ -1160,9 +1105,6 @@ export class Runner {
 		generation: number,
 		intentType: "sleep" | "stop",
 	) {
-		const actor = this.getActor(actorId, generation);
-		if (!actor) return;
-
 		let actorIntent: protocol.ActorIntent;
 
 		if (intentType === "sleep") {
@@ -1182,11 +1124,9 @@ export class Runner {
 			intent: actorIntent,
 		};
 
+		const eventIndex = this.#nextEventIdx++;
 		const eventWrapper: protocol.EventWrapper = {
-			checkpoint: {
-				actorId,
-				index: actor.nextEventIdx++,
-			},
+			index: eventIndex,
 			inner: {
 				tag: "EventActorIntent",
 				val: intentEvent,
@@ -1206,9 +1146,6 @@ export class Runner {
 		generation: number,
 		stateType: "running" | "stopped",
 	) {
-		const actor = this.getActor(actorId, generation);
-		if (!actor) return;
-
 		let actorState: protocol.ActorState;
 
 		if (stateType === "running") {
@@ -1231,11 +1168,9 @@ export class Runner {
 			state: actorState,
 		};
 
+		const eventIndex = this.#nextEventIdx++;
 		const eventWrapper: protocol.EventWrapper = {
-			checkpoint: {
-				actorId,
-				index: actor.nextEventIdx++,
-			},
+			index: eventIndex,
 			inner: {
 				tag: "EventActorStateUpdate",
 				val: stateUpdateEvent,
@@ -1251,18 +1186,9 @@ export class Runner {
 	}
 
 	#sendCommandAcknowledgment() {
-		const lastCommandCheckpoints = [];
-
-		for (const [_, actor] of this.#actors) {
-			if (actor.lastCommandIdx < 0) {
-				// No commands received yet, nothing to acknowledge
-				continue;
-			}
-
-			lastCommandCheckpoints.push({
-				actorId: actor.actorId,
-				index: actor.lastCommandIdx,
-			});
+		if (this.#lastCommandIdx < 0) {
+			// No commands received yet, nothing to acknowledge
+			return;
 		}
 
 		//this.#log?.log("Sending command acknowledgment", this.#lastCommandIdx);
@@ -1270,7 +1196,7 @@ export class Runner {
 		this.__sendToServer({
 			tag: "ToServerAckCommands",
 			val: {
-				lastCommandCheckpoints,
+				lastCommandIdx: BigInt(this.#lastCommandIdx),
 			},
 		});
 	}
@@ -1574,11 +1500,9 @@ export class Runner {
 			alarmTs: alarmTs !== null ? BigInt(alarmTs) : null,
 		};
 
+		const eventIndex = this.#nextEventIdx++;
 		const eventWrapper: protocol.EventWrapper = {
-			checkpoint: {
-				actorId,
-				index: actor.nextEventIdx++,
-			},
+			index: eventIndex,
 			inner: {
 				tag: "EventActorSetAlarm",
 				val: alarmEvent,
@@ -1745,7 +1669,6 @@ export class Runner {
 			tag: "ToServerlessServerInit",
 			val: {
 				runnerId: this.runnerId,
-				runnerProtocolVersion: PROTOCOL_VERSION,
 			},
 		});
 
@@ -1776,34 +1699,27 @@ export class Runner {
 			msg: `Scheduling reconnect attempt ${this.#reconnectAttempt + 1} in ${delay}ms`,
 		});
 
-		this.#reconnectTimeout = setTimeout(() => {
+		this.#reconnectTimeout = setTimeout(async () => {
 			if (!this.#shutdown) {
 				this.#reconnectAttempt++;
 				this.log?.debug({
 					msg: `Attempting to reconnect (attempt ${this.#reconnectAttempt})...`,
 				});
-				this.#openPegboardWebSocket().catch((err) => {
-					this.log?.error({
-						msg: "error during websocket reconnection",
-						error: stringifyError(err),
-					});
-				});
+				await this.#openPegboardWebSocket();
 			}
 		}, delay);
 	}
 
-	#resendUnacknowledgedEvents() {
-		const eventsToResend = [];
-
-		for (const [_, actor] of this.#actors) {
-			eventsToResend.push(...actor.eventHistory);
-		}
+	#resendUnacknowledgedEvents(lastEventIdx: bigint) {
+		const eventsToResend = this.#eventHistory.filter(
+			(event) => event.index > lastEventIdx,
+		);
 
 		if (eventsToResend.length === 0) return;
 
 		this.log?.info({
 			msg: "resending unacknowledged events",
-			count: eventsToResend.length,
+			fromIndex: lastEventIdx + 1n,
 		});
 
 		// Resend events in batches