@rivetkit/engine-runner 25.7.3 → 25.8.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rivetkit/engine-runner",
-  "version": "25.7.3",
+  "version": "25.8.1",
   "type": "module",
   "exports": {
     "import": {
@@ -16,7 +16,7 @@
     "uuid": "^12.0.0",
     "pino": "^9.9.5",
     "ws": "^8.18.3",
-    "@rivetkit/engine-runner-protocol": "25.7.3"
+    "@rivetkit/engine-runner-protocol": "25.8.1"
   },
   "devDependencies": {
     "@types/node": "^22.18.1",

package/src/mod.ts

@@ -10,6 +10,9 @@ import { setLogger, logger } from "./log.js";
 const KV_EXPIRE: number = 30_000;
 const PROTOCOL_VERSION: number = 1;
 
+/** Warn once the backlog significantly exceeds the server's ack batch size. */
+const EVENT_BACKLOG_WARN_THRESHOLD = 10_000;
+
 export interface ActorInstance {
 	actorId: string;
 	generation: number;
@@ -92,8 +95,8 @@ export class Runner {
 	#runnerLostTimeout?: NodeJS.Timeout;
 
 	// Event storage for resending
-	#eventHistory: { event: protocol.EventWrapper; timestamp: number }[] = [];
-	#eventPruneInterval?: NodeJS.Timeout;
+	#eventHistory: protocol.EventWrapper[] = [];
+	#eventBacklogWarned: boolean = false;
 
 	// Command acknowledgment
 	#ackInterval?: NodeJS.Timeout;
@@ -112,12 +115,6 @@ export class Runner {
 
 		this.#tunnel = new Tunnel(this);
 
-		// TODO(RVT-4986): Prune when server acks events
-		// Start pruning old events every minute
-		this.#eventPruneInterval = setInterval(() => {
-			this.#pruneOldEvents();
-		}, 60000); // Run every minute
-
 		// Start cleaning up old unsent KV requests every 15 seconds
 		this.#kvCleanupInterval = setInterval(() => {
 			this.#cleanupOldKvRequests();
@@ -205,7 +202,7 @@ export class Runner {
 	): ActorInstance | undefined {
 		const actor = this.#actors.get(actorId);
 		if (!actor) {
-			logger()?.error({ msg: "actor not found", actorId });
+			logger()?.error({ msg: "actor not found for removal", actorId });
 			return undefined;
 		}
 		if (generation !== undefined && actor.generation !== generation) {
@@ -290,12 +287,6 @@ export class Runner {
 			this.#ackInterval = undefined;
 		}
 
-		// Clear event prune interval
-		if (this.#eventPruneInterval) {
-			clearInterval(this.#eventPruneInterval);
-			this.#eventPruneInterval = undefined;
-		}
-
 		// Clear KV cleanup interval
 		if (this.#kvCleanupInterval) {
 			clearInterval(this.#kvCleanupInterval);
@@ -398,21 +389,11 @@ export class Runner {
 		return `${wsEndpoint}?protocol_version=${PROTOCOL_VERSION}&namespace=${encodeURIComponent(this.#config.namespace)}&runner_key=${encodeURIComponent(this.#config.runnerKey)}`;
 	}
 
-	get pegboardTunnelUrl() {
-		const endpoint =
-			this.#config.pegboardRelayEndpoint ||
-			this.#config.pegboardEndpoint ||
-			this.#config.endpoint;
-		const wsEndpoint = endpoint
-			.replace("http://", "ws://")
-			.replace("https://", "wss://");
-		return `${wsEndpoint}?protocol_version=${PROTOCOL_VERSION}&namespace=${encodeURIComponent(this.#config.namespace)}&runner_name=${encodeURIComponent(this.#config.runnerName)}&runner_key=${encodeURIComponent(this.#config.runnerKey)}`;
-	}
-
 	// MARK: Runner protocol
 	async #openPegboardWebSocket() {
 		const protocols = ["rivet", `rivet_target.runner`];
-		if (this.config.token) protocols.push(`rivet_token.${this.config.token}`);
+		if (this.config.token)
+			protocols.push(`rivet_token.${this.config.token}`);
 
 		const WS = await importWebSocket();
 		const ws = new WS(this.pegboardUrl, protocols) as any as WebSocket;
@@ -510,7 +491,13 @@ export class Runner {
 			// Handle message
 			if (message.tag === "ToClientInit") {
 				const init = message.val;
-				this.runnerId = init.runnerId;
+
+				if (this.runnerId != init.runnerId) {
+					this.runnerId = init.runnerId;
+
+					// Clear history if runner id changed
+					this.#eventHistory.length = 0;
+				}
 
 				// Store the runner lost threshold from metadata
 				this.#runnerLostThreshold = init.metadata?.runnerLostThreshold
@@ -532,7 +519,7 @@ export class Runner {
 				const commands = message.val;
 				this.#handleCommands(commands);
 			} else if (message.tag === "ToClientAckEvents") {
-				throw new Error("TODO");
+				this.#handleAckEvents(message.val);
 			} else if (message.tag === "ToClientKvResponse") {
 				const kvResponse = message.val;
 				this.#handleKvResponse(kvResponse);
@@ -570,7 +557,7 @@ export class Runner {
 			}
 		});
 
-		ws.addEventListener("close", (ev) => {
+		ws.addEventListener("close", async (ev) => {
 			logger()?.info({
 				msg: "connection closed",
 				code: ev.code,
@@ -579,6 +566,12 @@ export class Runner {
 
 			this.#config.onDisconnected();
 
+			if (ev.reason.toString() == "ws.eviction") {
+				logger()?.info("runner evicted");
+
+				await this.shutdown(true);
+			}
+
 			// Clear ping loop on close
 			if (this.#pingLoop) {
 				clearInterval(this.#pingLoop);
@@ -633,6 +626,45 @@ export class Runner {
 		}
 	}
 
+	#handleAckEvents(ack: protocol.ToClientAckEvents) {
+		const lastAckedIdx = ack.lastEventIdx;
+
+		const originalLength = this.#eventHistory.length;
+		this.#eventHistory = this.#eventHistory.filter(
+			(event) => event.index > lastAckedIdx,
+		);
+
+		const prunedCount = originalLength - this.#eventHistory.length;
+		if (prunedCount > 0) {
+			logger()?.info({
+				msg: "pruned acknowledged events",
+				lastAckedIdx: lastAckedIdx.toString(),
+				prunedCount,
+			});
+		}
+
+		if (this.#eventHistory.length <= EVENT_BACKLOG_WARN_THRESHOLD) {
+			this.#eventBacklogWarned = false;
+		}
+	}
+
+	/** Track events to send to the server in case we need to resend it on disconnect. */
+	#recordEvent(eventWrapper: protocol.EventWrapper) {
+		this.#eventHistory.push(eventWrapper);
+
+		if (
+			this.#eventHistory.length > EVENT_BACKLOG_WARN_THRESHOLD &&
+			!this.#eventBacklogWarned
+		) {
+			this.#eventBacklogWarned = true;
+			logger()?.warn({
+				msg: "unacknowledged event backlog exceeds threshold",
+				backlogSize: this.#eventHistory.length,
+				threshold: EVENT_BACKLOG_WARN_THRESHOLD,
+			});
+		}
+	}
+
 	#handleCommandStartActor(commandWrapper: protocol.CommandWrapper) {
 		const startCommand = commandWrapper.inner
 			.val as protocol.CommandStartActor;
@@ -724,11 +756,7 @@ export class Runner {
 			},
 		};
 
-		// Store event in history for potential resending
-		this.#eventHistory.push({
-			event: eventWrapper,
-			timestamp: Date.now(),
-		});
+		this.#recordEvent(eventWrapper);
 
 		logger()?.info({
 			msg: "sending event to server",
@@ -763,7 +791,7 @@ export class Runner {
 				tag: "ActorStateStopped",
 				val: {
 					code: protocol.StopCode.Ok,
-					message: "hello",
+					message: null,
 				},
 			};
 		} else {
@@ -785,11 +813,7 @@ export class Runner {
 			},
 		};
 
-		// Store event in history for potential resending
-		this.#eventHistory.push({
-			event: eventWrapper,
-			timestamp: Date.now(),
-		});
+		this.#recordEvent(eventWrapper);
 
 		logger()?.info({
 			msg: "sending event to server",
@@ -1142,11 +1166,7 @@ export class Runner {
 			},
 		};
 
-		// Store event in history for potential resending
-		this.#eventHistory.push({
-			event: eventWrapper,
-			timestamp: Date.now(),
-		});
+		this.#recordEvent(eventWrapper);
 
 		this.__sendToServer({
 			tag: "ToServerEvents",
@@ -1272,7 +1292,7 @@ export class Runner {
 			tag: "ToServerlessServerInit",
 			val: {
 				runnerId: this.runnerId,
-			}
+			},
 		});
 
 		// Embed version
@@ -1280,7 +1300,7 @@ export class Runner {
 		buffer.writeUInt16LE(PROTOCOL_VERSION, 0);
 		Buffer.from(data).copy(buffer, 2);
 
-		return buffer.toString('base64');
+		return buffer.toString("base64");
 	}
 
 	#scheduleReconnect() {
@@ -1313,7 +1333,7 @@ export class Runner {
 
 	#resendUnacknowledgedEvents(lastEventIdx: bigint) {
 		const eventsToResend = this.#eventHistory.filter(
-			(item) => item.event.index > lastEventIdx,
+			(event) => event.index > lastEventIdx,
 		);
 
 		if (eventsToResend.length === 0) return;
@@ -1323,29 +1343,12 @@ export class Runner {
 		//);
 
 		// Resend events in batches
-		const events = eventsToResend.map((item) => item.event);
 		this.__sendToServer({
 			tag: "ToServerEvents",
-			val: events,
+			val: eventsToResend,
 		});
 	}
 
-	// TODO(RVT-4986): Prune when server acks events instead of based on old events
-	#pruneOldEvents() {
-		const fiveMinutesAgo = Date.now() - 5 * 60 * 1000;
-		const originalLength = this.#eventHistory.length;
-
-		// Remove events older than 5 minutes
-		this.#eventHistory = this.#eventHistory.filter(
-			(item) => item.timestamp > fiveMinutesAgo,
-		);
-
-		const prunedCount = originalLength - this.#eventHistory.length;
-		if (prunedCount > 0) {
-			//logger()?.log(`Pruned ${prunedCount} old events from history`);
-		}
-	}
-
 	#cleanupOldKvRequests() {
 		const thirtySecondsAgo = Date.now() - KV_EXPIRE;
 		const toDelete: number[] = [];

package/src/tunnel.ts

@@ -16,7 +16,7 @@ interface PendingRequest {
 	actorId?: string;
 }
 
-interface PendingMessage {
+interface PendingTunnelMessage {
 	sentAt: number;
 	requestIdStr: string;
 }
@@ -24,10 +24,14 @@ interface PendingMessage {
 export class Tunnel {
 	#runner: Runner;
 
+	/** Requests over the tunnel to the actor that are in flight. */
 	#actorPendingRequests: Map<string, PendingRequest> = new Map();
+	/** WebSockets over the tunnel to the actor that are in flight. */
 	#actorWebSockets: Map<string, WebSocketTunnelAdapter> = new Map();
 
-	#pendingMessages: Map<string, PendingMessage> = new Map();
+	/** Messages sent from the actor over the tunnel that have not been acked by the gateway. */
+	#pendingTunnelMessages: Map<string, PendingTunnelMessage> = new Map();
+
 	#gcInterval?: NodeJS.Timeout;
 
 	constructor(runner: Runner) {
@@ -65,7 +69,9 @@ export class Tunnel {
 	) {
 		// TODO: Switch this with runner WS
 		if (!this.#runner.__webSocketReady()) {
-			console.warn("Cannot send tunnel message, WebSocket not connected");
+			logger()?.warn(
+				"cannot send tunnel message, socket not connected to engine",
+			);
 			return;
 		}
 
@@ -73,7 +79,7 @@ export class Tunnel {
 		const messageId = generateUuidBuffer();
 
 		const requestIdStr = bufferToString(requestId);
-		this.#pendingMessages.set(bufferToString(messageId), {
+		this.#pendingTunnelMessages.set(bufferToString(messageId), {
 			sentAt: Date.now(),
 			requestIdStr,
 		});
@@ -121,7 +127,7 @@ export class Tunnel {
 		const now = Date.now();
 		const messagesToDelete: string[] = [];
 
-		for (const [messageId, pendingMessage] of this.#pendingMessages) {
+		for (const [messageId, pendingMessage] of this.#pendingTunnelMessages) {
 			// Check if message is older than timeout
 			if (now - pendingMessage.sentAt > MESSAGE_ACK_TIMEOUT) {
 				messagesToDelete.push(messageId);
@@ -161,9 +167,14 @@ export class Tunnel {
 		}
 
 		// Remove timed out messages
-		for (const messageId of messagesToDelete) {
-			this.#pendingMessages.delete(messageId);
-			console.warn(`Purged unacked message: ${messageId}`);
+		if (messagesToDelete.length > 0) {
+			logger()?.warn({
+				msg: "purging unacked tunnel messages, this indicates that the Rivet Engine is disconnected or not responding",
+				count: messagesToDelete.length,
+			});
+			for (const messageId of messagesToDelete) {
+				this.#pendingTunnelMessages.delete(messageId);
+			}
 		}
 	}
 
@@ -214,9 +225,9 @@ export class Tunnel {
 		if (message.messageKind.tag === "TunnelAck") {
 			// Mark pending message as acknowledged and remove it
 			const msgIdStr = bufferToString(message.messageId);
-			const pending = this.#pendingMessages.get(msgIdStr);
+			const pending = this.#pendingTunnelMessages.get(msgIdStr);
 			if (pending) {
-				this.#pendingMessages.delete(msgIdStr);
+				this.#pendingTunnelMessages.delete(msgIdStr);
 			}
 		} else {
 			this.#sendAck(message.requestId, message.messageId);
@@ -438,7 +449,6 @@ export class Tunnel {
 		const websocketHandler = this.#runner.config.websocket;
 
 		if (!websocketHandler) {
-			console.error("No websocket handler configured for tunnel");
 			logger()?.error({
 				msg: "no websocket handler configured for tunnel",
 			});