@rivetkit/engine-runner 2.0.22-rc.2 → 2.0.23

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rivetkit/engine-runner",
-  "version": "2.0.22-rc.2",
+  "version": "2.0.23",
   "type": "module",
   "exports": {
     "import": {
@@ -16,7 +16,7 @@
     "uuid": "^12.0.0",
     "pino": "^9.9.5",
     "ws": "^8.18.3",
-    "@rivetkit/engine-runner-protocol": "2.0.22-rc.2"
+    "@rivetkit/engine-runner-protocol": "2.0.23"
   },
   "devDependencies": {
     "@types/node": "^22.18.1",

package/src/mod.ts

@@ -2,8 +2,13 @@ import * as protocol from "@rivetkit/engine-runner-protocol";
 import type { Logger } from "pino";
 import type WebSocket from "ws";
 import { logger, setLogger } from "./log.js";
+import { stringifyCommandWrapper, stringifyEvent } from "./stringify";
 import { Tunnel } from "./tunnel";
-import { calculateBackoff, unreachable } from "./utils";
+import {
+	calculateBackoff,
+	parseWebSocketCloseReason,
+	unreachable,
+} from "./utils";
 import { importWebSocket } from "./websocket.js";
 import type { WebSocketTunnelAdapter } from "./websocket-tunnel-adapter";
 
@@ -132,6 +137,30 @@ export class Runner {
 	// Tunnel for HTTP/WebSocket forwarding
 	#tunnel: Tunnel | undefined;
 
+	// Cached child logger with runner-specific attributes
+	#logCached?: Logger;
+
+	get log(): Logger | undefined {
+		if (this.#logCached) return this.#logCached;
+
+		const l = logger();
+		if (l) {
+			// If has connected, create child logger with relevant metadata
+			//
+			// Otherwise, return default logger
+			if (this.runnerId) {
+				this.#logCached = l.child({
+					runnerId: this.runnerId,
+				});
+				return this.#logCached;
+			} else {
+				return l;
+			}
+		}
+
+		return undefined;
+	}
+
 	constructor(config: RunnerConfig) {
 		this.#config = config;
 		if (this.#config.logger) setLogger(this.#config.logger);
@@ -144,6 +173,13 @@ export class Runner {
 
 	// MARK: Manage actors
 	sleepActor(actorId: string, generation?: number) {
+		if (this.#shutdown) {
+			this.log?.warn({
+				msg: "runner is shut down, cannot sleep actor",
+			});
+			return;
+		}
+
 		const actor = this.getActor(actorId, generation);
 		if (!actor) return;
 
@@ -186,9 +222,8 @@ export class Runner {
 	}
 
 	#stopAllActors() {
-		logger()?.info({
+		this.log?.info({
 			msg: "stopping all actors due to runner lost threshold exceeded",
-			runnerId: this.runnerId,
 		});
 
 		const actorIds = Array.from(this.#actors.keys());
@@ -200,17 +235,15 @@ export class Runner {
 	getActor(actorId: string, generation?: number): ActorInstance | undefined {
 		const actor = this.#actors.get(actorId);
 		if (!actor) {
-			logger()?.error({
+			this.log?.error({
 				msg: "actor not found",
-				runnerId: this.runnerId,
 				actorId,
 			});
 			return undefined;
 		}
 		if (generation !== undefined && actor.generation !== generation) {
-			logger()?.error({
+			this.log?.error({
 				msg: "actor generation mismatch",
-				runnerId: this.runnerId,
 				actorId,
 				generation,
 			});
@@ -236,17 +269,15 @@ export class Runner {
 	): ActorInstance | undefined {
 		const actor = this.#actors.get(actorId);
 		if (!actor) {
-			logger()?.error({
+			this.log?.error({
 				msg: "actor not found for removal",
-				runnerId: this.runnerId,
 				actorId,
 			});
 			return undefined;
 		}
 		if (generation !== undefined && actor.generation !== generation) {
-			logger()?.error({
+			this.log?.error({
 				msg: "actor generation mismatch",
-				runnerId: this.runnerId,
 				actorId,
 				generation,
 			});
@@ -263,7 +294,7 @@ export class Runner {
 		if (this.#started) throw new Error("Cannot call runner.start twice");
 		this.#started = true;
 
-		logger()?.info({ msg: "starting runner" });
+		this.log?.info({ msg: "starting runner" });
 
 		this.#tunnel = new Tunnel(this);
 		this.#tunnel.start();
@@ -278,7 +309,7 @@ export class Runner {
 		if (!this.#config.noAutoShutdown) {
 			if (!SIGNAL_HANDLERS.length) {
 				process.on("SIGTERM", () => {
-					logger()?.debug("received SIGTERM");
+					this.log?.debug("received SIGTERM");
 
 					for (const handler of SIGNAL_HANDLERS) {
 						handler();
@@ -287,7 +318,7 @@ export class Runner {
 					process.exit(0);
 				});
 				process.on("SIGINT", () => {
-					logger()?.debug("received SIGINT");
+					this.log?.debug("received SIGINT");
 
 					for (const handler of SIGNAL_HANDLERS) {
 						handler();
@@ -296,7 +327,7 @@ export class Runner {
 					process.exit(0);
 				});
 
-				logger()?.debug({
+				this.log?.debug({
 					msg: "added SIGTERM listeners",
 				});
 			}
@@ -310,9 +341,8 @@ export class Runner {
 
 	// MARK: Shutdown
 	async shutdown(immediate: boolean, exit: boolean = false) {
-		logger()?.info({
+		this.log?.info({
 			msg: "starting shutdown",
-			runnerId: this.runnerId,
 			immediate,
 			exit,
 		});
@@ -364,13 +394,12 @@ export class Runner {
 			const pegboardWebSocket = this.#pegboardWebSocket;
 			if (immediate) {
 				// Stop immediately
-				pegboardWebSocket.close(1000, "Stopping");
+				pegboardWebSocket.close(1000, "pegboard.runner_shutdown");
 			} else {
 				// Wait for actors to shut down before stopping
 				try {
-					logger()?.info({
+					this.log?.info({
 						msg: "sending stopping message",
-						runnerId: this.runnerId,
 						readyState: pegboardWebSocket.readyState,
 					});
 
@@ -386,7 +415,7 @@ export class Runner {
 					) {
 						this.#pegboardWebSocket.send(encoded);
 					} else {
-						logger()?.error(
+						this.log?.error(
 							"WebSocket not available or not open for sending data",
 						);
 					}
@@ -396,9 +425,8 @@ export class Runner {
 							throw new Error("missing pegboardWebSocket");
 
 						pegboardWebSocket.addEventListener("close", (ev) => {
-							logger()?.info({
+							this.log?.info({
 								msg: "connection closed",
-								runnerId: this.runnerId,
 								code: ev.code,
 								reason: ev.reason.toString(),
 							});
@@ -408,31 +436,29 @@ export class Runner {
 
 					// TODO: Wait for all actors to stop before closing ws
 
-					logger()?.info({
+					this.log?.info({
 						msg: "closing WebSocket",
-						runnerId: this.runnerId,
 					});
-					pegboardWebSocket.close(1000, "Stopping");
+					pegboardWebSocket.close(1000, "pegboard.runner_shutdown");
 
 					await closePromise;
 
-					logger()?.info({
+					this.log?.info({
 						msg: "websocket shutdown completed",
-						runnerId: this.runnerId,
 					});
 				} catch (error) {
-					logger()?.error({
+					this.log?.error({
 						msg: "error during websocket shutdown:",
-						runnerId: this.runnerId,
 						error,
 					});
 					pegboardWebSocket.close();
 				}
 			}
 		} else {
-			logger()?.warn({
+			// This is often logged when the serverless SSE stream closes after
+			// the runner has already shut down
+			this.log?.debug({
 				msg: "no runner WebSocket to shutdown or already closed",
-				runnerId: this.runnerId,
 				readyState: this.#pegboardWebSocket?.readyState,
 			});
 		}
@@ -474,7 +500,7 @@ export class Runner {
 		const ws = new WS(this.pegboardUrl, protocols) as any as WebSocket;
 		this.#pegboardWebSocket = ws;
 
-		logger()?.info({
+		this.log?.info({
 			msg: "connecting",
 			endpoint: this.pegboardEndpoint,
 			namespace: this.#config.namespace,
@@ -483,7 +509,20 @@ export class Runner {
 		});
 
 		ws.addEventListener("open", () => {
-			logger()?.info({ msg: "connected" });
+			if (this.#reconnectAttempt > 0) {
+				this.log?.info({
+					msg: "runner reconnected",
+					namespace: this.#config.namespace,
+					runnerName: this.#config.runnerName,
+					reconnectAttempt: this.#reconnectAttempt,
+				});
+			} else {
+				this.log?.debug({
+					msg: "runner connected",
+					namespace: this.#config.namespace,
+					runnerName: this.#config.runnerName,
+				});
+			}
 
 			// Reset reconnect attempt counter on successful connection
 			this.#reconnectAttempt = 0;
@@ -539,9 +578,8 @@ export class Runner {
 					});
 				} else {
 					clearInterval(pingLoop);
-					logger()?.info({
+					this.log?.info({
 						msg: "WebSocket not open, stopping ping loop",
-						runnerId: this.runnerId,
 					});
 				}
 			}, RUNNER_PING_INTERVAL);
@@ -554,9 +592,8 @@ export class Runner {
 					this.#sendCommandAcknowledgment();
 				} else {
 					clearInterval(ackLoop);
-					logger()?.info({
+					this.log?.info({
 						msg: "WebSocket not open, stopping ack loop",
-						runnerId: this.runnerId,
 					});
 				}
 			}, ackInterval);
@@ -592,9 +629,8 @@ export class Runner {
 					? Number(init.metadata.runnerLostThreshold)
 					: undefined;
 
-				logger()?.info({
+				this.log?.info({
 					msg: "received init",
-					runnerId: init.runnerId,
 					lastEventIdx: init.lastEventIdx,
 					runnerLostThreshold: this.#runnerLostThreshold,
 				});
@@ -622,9 +658,8 @@ export class Runner {
 		});
 
 		ws.addEventListener("error", (ev) => {
-			logger()?.error({
+			this.log?.error({
 				msg: `WebSocket error: ${ev.error}`,
-				runnerId: this.runnerId,
 			});
 
 			if (!this.#shutdown) {
@@ -634,9 +669,8 @@ export class Runner {
 					this.#runnerLostThreshold &&
 					this.#runnerLostThreshold > 0
 				) {
-					logger()?.info({
+					this.log?.info({
 						msg: "starting runner lost timeout",
-						runnerId: this.runnerId,
 						seconds: this.#runnerLostThreshold / 1000,
 					});
 					this.#runnerLostTimeout = setTimeout(() => {
@@ -650,22 +684,32 @@ export class Runner {
 		});
 
 		ws.addEventListener("close", async (ev) => {
-			logger()?.info({
-				msg: "connection closed",
-				runnerId: this.runnerId,
-				code: ev.code,
-				reason: ev.reason.toString(),
-			});
+			const closeError = parseWebSocketCloseReason(ev.reason);
+			if (
+				closeError?.group === "ws" &&
+				closeError?.error === "eviction"
+			) {
+				this.log?.info("runner websocket evicted");
 
-			this.#config.onDisconnected(ev.code, ev.reason);
-
-			if (ev.reason.toString().startsWith("ws.eviction")) {
-				logger()?.info({
-					msg: "runner evicted",
-					runnerId: this.runnerId,
-				});
+				this.#config.onDisconnected(ev.code, ev.reason);
 
 				await this.shutdown(true);
+			} else {
+				if (
+					closeError?.group === "pegboard" &&
+					closeError?.error === "runner_shutdown"
+				) {
+					this.log?.info("runner shutdown");
+				} else {
+					this.log?.warn({
+						msg: "runner disconnected",
+						code: ev.code,
+						reason: ev.reason.toString(),
+						closeError,
+					});
+				}
+
+				this.#config.onDisconnected(ev.code, ev.reason);
 			}
 
 			// Clear ping loop on close
@@ -687,9 +731,8 @@ export class Runner {
 					this.#runnerLostThreshold &&
 					this.#runnerLostThreshold > 0
 				) {
-					logger()?.info({
+					this.log?.info({
 						msg: "starting runner lost timeout",
-						runnerId: this.runnerId,
 						seconds: this.#runnerLostThreshold / 1000,
 					});
 					this.#runnerLostTimeout = setTimeout(() => {
@@ -704,17 +747,15 @@ export class Runner {
 	}
 
 	#handleCommands(commands: protocol.ToClientCommands) {
-		logger()?.info({
+		this.log?.info({
 			msg: "received commands",
-			runnerId: this.runnerId,
 			commandCount: commands.length,
 		});
 
 		for (const commandWrapper of commands) {
-			logger()?.info({
+			this.log?.info({
 				msg: "received command",
-				runnerId: this.runnerId,
-				commandWrapper,
+				command: stringifyCommandWrapper(commandWrapper),
 			});
 			if (commandWrapper.inner.tag === "CommandStartActor") {
 				this.#handleCommandStartActor(commandWrapper);
@@ -738,9 +779,8 @@ export class Runner {
 
 		const prunedCount = originalLength - this.#eventHistory.length;
 		if (prunedCount > 0) {
-			logger()?.info({
+			this.log?.info({
 				msg: "pruned acknowledged events",
-				runnerId: this.runnerId,
 				lastAckedIdx: lastAckedIdx.toString(),
 				prunedCount,
 			});
@@ -760,9 +800,8 @@ export class Runner {
 			!this.#eventBacklogWarned
 		) {
 			this.#eventBacklogWarned = true;
-			logger()?.warn({
+			this.log?.warn({
 				msg: "unacknowledged event backlog exceeds threshold",
-				runnerId: this.runnerId,
 				backlogSize: this.#eventHistory.length,
 				threshold: EVENT_BACKLOG_WARN_THRESHOLD,
 			});
@@ -801,9 +840,8 @@ export class Runner {
 		this.#config
 			.onActorStart(actorId, generation, actorConfig)
 			.catch((err) => {
-				logger()?.error({
+				this.log?.error({
 					msg: "error in onactorstart for actor",
-					runnerId: this.runnerId,
 					actorId,
 					err,
 				});
@@ -830,9 +868,9 @@ export class Runner {
 		intentType: "sleep" | "stop",
 	) {
 		if (this.#shutdown) {
-			logger()?.warn({
+			console.trace("send actor intent", actorId, intentType);
+			this.log?.warn({
 				msg: "Runner is shut down, cannot send actor intent",
-				runnerId: this.runnerId,
 			});
 			return;
 		}
@@ -866,12 +904,10 @@ export class Runner {
 
 		this.#recordEvent(eventWrapper);
 
-		logger()?.info({
+		this.log?.info({
 			msg: "sending event to server",
-			runnerId: this.runnerId,
-			index: eventWrapper.index,
-			tag: eventWrapper.inner.tag,
-			val: eventWrapper.inner.val,
+			event: stringifyEvent(eventWrapper.inner),
+			index: eventWrapper.index.toString(),
 		});
 
 		this.__sendToServer({
@@ -886,9 +922,8 @@ export class Runner {
 		stateType: "running" | "stopped",
 	) {
 		if (this.#shutdown) {
-			logger()?.warn({
+			this.log?.warn({
 				msg: "Runner is shut down, cannot send actor state update",
-				runnerId: this.runnerId,
 			});
 			return;
 		}
@@ -925,12 +960,10 @@ export class Runner {
 
 		this.#recordEvent(eventWrapper);
 
-		logger()?.info({
+		this.log?.info({
 			msg: "sending event to server",
-			runnerId: this.runnerId,
-			index: eventWrapper.index,
-			tag: eventWrapper.inner.tag,
-			val: eventWrapper.inner.val,
+			event: stringifyEvent(eventWrapper.inner),
+			index: eventWrapper.index.toString(),
 		});
 
 		this.__sendToServer({
@@ -941,9 +974,8 @@ export class Runner {
 
 	#sendCommandAcknowledgment() {
 		if (this.#shutdown) {
-			logger()?.warn({
+			this.log?.warn({
 				msg: "Runner is shut down, cannot send command acknowledgment",
-				runnerId: this.runnerId,
 			});
 			return;
 		}
@@ -953,7 +985,7 @@ export class Runner {
 			return;
 		}
 
-		//logger()?.log("Sending command acknowledgment", this.#lastCommandIdx);
+		//this.#log?.log("Sending command acknowledgment", this.#lastCommandIdx);
 
 		this.__sendToServer({
 			tag: "ToServerAckCommands",
@@ -968,8 +1000,10 @@ export class Runner {
 		const request = this.#kvRequests.get(requestId);
 
 		if (!request) {
-			const msg = "received kv response for unknown request id";
-			logger()?.error({ msg, runnerId: this.runnerId, requestId });
+			this.log?.error({
+				msg: "received kv response for unknown request id",
+				requestId,
+			});
 			return;
 		}
 
@@ -1362,7 +1396,7 @@ export class Runner {
 		}
 
 		if (processedCount > 0) {
-			//logger()?.log(`Processed ${processedCount} queued KV requests`);
+			//this.#log?.log(`Processed ${processedCount} queued KV requests`);
 		}
 	}
 
@@ -1374,9 +1408,8 @@ export class Runner {
 
 	__sendToServer(message: protocol.ToServer) {
 		if (this.#shutdown) {
-			logger()?.warn({
+			this.log?.warn({
 				msg: "Runner is shut down, cannot send message to server",
-				runnerId: this.runnerId,
 			});
 			return;
 		}
@@ -1388,9 +1421,8 @@ export class Runner {
 		) {
 			this.#pegboardWebSocket.send(encoded);
 		} else {
-			logger()?.error({
+			this.log?.error({
 				msg: "WebSocket not available or not open for sending data",
-				runnerId: this.runnerId,
 			});
 		}
 	}
@@ -1419,9 +1451,8 @@ export class Runner {
 
 	#scheduleReconnect() {
 		if (this.#shutdown) {
-			logger()?.debug({
+			this.log?.debug({
 				msg: "Runner is shut down, not attempting reconnect",
-				runnerId: this.runnerId,
 			});
 			return;
 		}
@@ -1433,17 +1464,15 @@ export class Runner {
 			jitter: true,
 		});
 
-		logger()?.debug({
+		this.log?.debug({
 			msg: `Scheduling reconnect attempt ${this.#reconnectAttempt + 1} in ${delay}ms`,
-			runnerId: this.runnerId,
 		});
 
 		this.#reconnectTimeout = setTimeout(async () => {
 			if (!this.#shutdown) {
 				this.#reconnectAttempt++;
-				logger()?.debug({
+				this.log?.debug({
 					msg: `Attempting to reconnect (attempt ${this.#reconnectAttempt})...`,
-					runnerId: this.runnerId,
 				});
 				await this.#openPegboardWebSocket();
 			}
@@ -1457,7 +1486,7 @@ export class Runner {
 
 		if (eventsToResend.length === 0) return;
 
-		//logger()?.log(
+		//this.#log?.log(
 		//	`Resending ${eventsToResend.length} unacknowledged events from index ${Number(lastEventIdx) + 1}`,
 		//);
 
@@ -1488,7 +1517,7 @@ export class Runner {
 		}
 
 		if (toDelete.length > 0) {
-			//logger()?.log(`Cleaned up ${toDelete.length} expired KV requests`);
+			//this.#log?.log(`Cleaned up ${toDelete.length} expired KV requests`);
 		}
 	}
 }