@rivetkit/engine-runner 2.0.27 → 2.0.29-rc.1

package/dist/mod.js

@@ -83,6 +83,23 @@ function idToStr(id) {
   const bytes = new Uint8Array(id);
   return Array.from(bytes).map((byte) => byte.toString(16).padStart(2, "0")).join("");
 }
+function stringifyError(error) {
+  var _a;
+  if (error instanceof Error) {
+    return `${error.name}: ${error.message}${error.stack ? `
+${error.stack}` : ""}`;
+  } else if (typeof error === "string") {
+    return error;
+  } else if (typeof error === "object" && error !== null) {
+    try {
+      return `${JSON.stringify(error)}`;
+    } catch {
+      return `[object ${((_a = error.constructor) == null ? void 0 : _a.name) || "Object"}]`;
+    }
+  } else {
+    return String(error);
+  }
+}
 
 // src/actor.ts
 var RunnerActor = class {
@@ -99,6 +116,9 @@ var RunnerActor = class {
   pendingRequests = [];
   webSockets = [];
   actorStartPromise;
+  lastCommandIdx = -1n;
+  nextEventIdx = 0n;
+  eventHistory = [];
   /**
    * If restoreHibernatingRequests has been called. This is used to assert
    * that the caller is implemented correctly.
@@ -215,6 +235,14 @@ var RunnerActor = class {
       this.webSockets.splice(index, 1);
     }
   }
+  handleAckEvents(lastEventIdx) {
+    this.eventHistory = this.eventHistory.filter(
+      (event) => event.checkpoint.index > lastEventIdx
+    );
+  }
+  recordEvent(eventWrapper) {
+    this.eventHistory.push(eventWrapper);
+  }
 };
 
 // src/stringify.ts
@@ -233,8 +261,6 @@ function stringifyMessageId(messageId) {
 }
 function stringifyToServerTunnelMessageKind(kind) {
   switch (kind.tag) {
-    case "DeprecatedTunnelAck":
-      return "DeprecatedTunnelAck";
     case "ToServerResponseStart": {
       const { status, headers, body, stream } = kind.val;
       const bodyStr = body === null ? "null" : stringifyArrayBuffer(body);
@@ -268,8 +294,6 @@ function stringifyToServerTunnelMessageKind(kind) {
 }
 function stringifyToClientTunnelMessageKind(kind) {
   switch (kind.tag) {
-    case "DeprecatedTunnelAck":
-      return "DeprecatedTunnelAck";
     case "ToClientRequestStart": {
       const { actorId, method, path, headers, body, stream } = kind.val;
       const bodyStr = body === null ? "null" : stringifyArrayBuffer(body);
@@ -300,30 +324,29 @@ function stringifyToClientTunnelMessageKind(kind) {
 function stringifyCommand(command) {
   switch (command.tag) {
     case "CommandStartActor": {
-      const { actorId, generation, config, hibernatingRequests } = command.val;
+      const { config, hibernatingRequests } = command.val;
       const keyStr = config.key === null ? "null" : `"${config.key}"`;
       const inputStr = config.input === null ? "null" : stringifyArrayBuffer(config.input);
       const hibernatingRequestsStr = hibernatingRequests.length > 0 ? `[${hibernatingRequests.map((hr) => `{gatewayId: ${idToStr(hr.gatewayId)}, requestId: ${idToStr(hr.requestId)}}`).join(", ")}]` : "[]";
-      return `CommandStartActor{actorId: "${actorId}", generation: ${generation}, config: {name: "${config.name}", key: ${keyStr}, createTs: ${stringifyBigInt(config.createTs)}, input: ${inputStr}}, hibernatingRequests: ${hibernatingRequestsStr}}`;
+      return `CommandStartActor{config: {name: "${config.name}", key: ${keyStr}, createTs: ${stringifyBigInt(config.createTs)}, input: ${inputStr}}, hibernatingRequests: ${hibernatingRequestsStr}}`;
     }
     case "CommandStopActor": {
-      const { actorId, generation } = command.val;
-      return `CommandStopActor{actorId: "${actorId}", generation: ${generation}}`;
+      return `CommandStopActor`;
     }
   }
 }
 function stringifyCommandWrapper(wrapper) {
-  return `CommandWrapper{index: ${stringifyBigInt(wrapper.index)}, inner: ${stringifyCommand(wrapper.inner)}}`;
+  return `CommandWrapper{actorId: "${wrapper.checkpoint.actorId}", generation: "${wrapper.checkpoint.generation}", index: ${stringifyBigInt(wrapper.checkpoint.index)}, inner: ${stringifyCommand(wrapper.inner)}}`;
 }
 function stringifyEvent(event) {
   switch (event.tag) {
     case "EventActorIntent": {
-      const { actorId, generation, intent } = event.val;
+      const { intent } = event.val;
       const intentStr = intent.tag === "ActorIntentSleep" ? "Sleep" : intent.tag === "ActorIntentStop" ? "Stop" : "Unknown";
-      return `EventActorIntent{actorId: "${actorId}", generation: ${generation}, intent: ${intentStr}}`;
+      return `EventActorIntent{intent: ${intentStr}}`;
     }
     case "EventActorStateUpdate": {
-      const { actorId, generation, state } = event.val;
+      const { state } = event.val;
       let stateStr;
       if (state.tag === "ActorStateRunning") {
         stateStr = "Running";
@@ -334,17 +357,17 @@ function stringifyEvent(event) {
       } else {
         stateStr = "Unknown";
       }
-      return `EventActorStateUpdate{actorId: "${actorId}", generation: ${generation}, state: ${stateStr}}`;
+      return `EventActorStateUpdate{state: ${stateStr}}`;
     }
     case "EventActorSetAlarm": {
-      const { actorId, generation, alarmTs } = event.val;
+      const { alarmTs } = event.val;
       const alarmTsStr = alarmTs === null ? "null" : stringifyBigInt(alarmTs);
-      return `EventActorSetAlarm{actorId: "${actorId}", generation: ${generation}, alarmTs: ${alarmTsStr}}`;
+      return `EventActorSetAlarm{alarmTs: ${alarmTsStr}}`;
     }
   }
 }
 function stringifyEventWrapper(wrapper) {
-  return `EventWrapper{index: ${stringifyBigInt(wrapper.index)}, inner: ${stringifyEvent(wrapper.inner)}}`;
+  return `EventWrapper{actorId: ${wrapper.checkpoint.actorId}, generation: "${wrapper.checkpoint.generation}", index: ${stringifyBigInt(wrapper.checkpoint.index)}, inner: ${stringifyEvent(wrapper.inner)}}`;
 }
 function stringifyToServer(message) {
   switch (message.tag) {
@@ -353,28 +376,27 @@ function stringifyToServer(message) {
         name,
         version,
         totalSlots,
-        lastCommandIdx,
         prepopulateActorNames,
         metadata
       } = message.val;
-      const lastCommandIdxStr = lastCommandIdx === null ? "null" : stringifyBigInt(lastCommandIdx);
       const prepopulateActorNamesStr = prepopulateActorNames === null ? "null" : `Map(${prepopulateActorNames.size})`;
       const metadataStr = metadata === null ? "null" : `"${metadata}"`;
-      return `ToServerInit{name: "${name}", version: ${version}, totalSlots: ${totalSlots}, lastCommandIdx: ${lastCommandIdxStr}, prepopulateActorNames: ${prepopulateActorNamesStr}, metadata: ${metadataStr}}`;
+      return `ToServerInit{name: "${name}", version: ${version}, totalSlots: ${totalSlots}, prepopulateActorNames: ${prepopulateActorNamesStr}, metadata: ${metadataStr}}`;
     }
     case "ToServerEvents": {
       const events = message.val;
       return `ToServerEvents{count: ${events.length}, events: [${events.map((e) => stringifyEventWrapper(e)).join(", ")}]}`;
     }
     case "ToServerAckCommands": {
-      const { lastCommandIdx } = message.val;
-      return `ToServerAckCommands{lastCommandIdx: ${stringifyBigInt(lastCommandIdx)}}`;
+      const { lastCommandCheckpoints } = message.val;
+      const checkpointsStr = lastCommandCheckpoints.length > 0 ? `[${lastCommandCheckpoints.map((cp) => `{actorId: "${cp.actorId}", index: ${stringifyBigInt(cp.index)}}`).join(", ")}]` : "[]";
+      return `ToServerAckCommands{lastCommandCheckpoints: ${checkpointsStr}}`;
     }
     case "ToServerStopping":
       return "ToServerStopping";
-    case "ToServerPing": {
+    case "ToServerPong": {
       const { ts } = message.val;
-      return `ToServerPing{ts: ${stringifyBigInt(ts)}}`;
+      return `ToServerPong{ts: ${stringifyBigInt(ts)}}`;
     }
     case "ToServerKvRequest": {
       const { actorId, requestId, data } = message.val;
@@ -390,19 +412,21 @@ function stringifyToServer(message) {
 function stringifyToClient(message) {
   switch (message.tag) {
     case "ToClientInit": {
-      const { runnerId, lastEventIdx, metadata } = message.val;
+      const { runnerId, metadata } = message.val;
       const metadataStr = `{runnerLostThreshold: ${stringifyBigInt(metadata.runnerLostThreshold)}}`;
-      return `ToClientInit{runnerId: "${runnerId}", lastEventIdx: ${stringifyBigInt(lastEventIdx)}, metadata: ${metadataStr}}`;
+      return `ToClientInit{runnerId: "${runnerId}", metadata: ${metadataStr}}`;
     }
-    case "ToClientClose":
-      return "ToClientClose";
+    case "ToClientPing":
+      const { ts } = message.val;
+      return `ToClientPing{ts: ${stringifyBigInt(ts)}}`;
     case "ToClientCommands": {
       const commands = message.val;
       return `ToClientCommands{count: ${commands.length}, commands: [${commands.map((c) => stringifyCommandWrapper(c)).join(", ")}]}`;
     }
     case "ToClientAckEvents": {
-      const { lastEventIdx } = message.val;
-      return `ToClientAckEvents{lastEventIdx: ${stringifyBigInt(lastEventIdx)}}`;
+      const { lastEventCheckpoints } = message.val;
+      const checkpointsStr = lastEventCheckpoints.length > 0 ? `[${lastEventCheckpoints.map((cp) => `{actorId: "${cp.actorId}", index: ${stringifyBigInt(cp.index)}}`).join(", ")}]` : "[]";
+      return `ToClientAckEvents{lastEventCheckpoints: ${checkpointsStr}}`;
     }
     case "ToClientKvResponse": {
       const { requestId, data } = message.val;
@@ -893,11 +917,6 @@ var WebSocketTunnelAdapter = class {
 };
 
 // src/tunnel.ts
-var RunnerShutdownError = class extends Error {
-  constructor() {
-    super("Runner shut down");
-  }
-};
 var Tunnel = class {
   #runner;
   /** Maps request IDs to actor IDs for lookup */
@@ -1019,7 +1038,7 @@ var Tunnel = class {
           (_a2 = this.log) == null ? void 0 : _a2.error({
             msg: "error creating websocket during restore",
             requestId: requestIdStr,
-            err
+            error: stringifyError(err)
           });
           this.#sendMessage(gatewayId, requestId, {
             tag: "ToServerWebSocketClose",
@@ -1068,7 +1087,7 @@ var Tunnel = class {
           (_a2 = this.log) == null ? void 0 : _a2.error({
             msg: "error creating stale websocket during restore",
             requestId: requestIdStr,
-            err
+            error: stringifyError(err)
           });
         });
         backgroundOperations.push(cleanupOperation);
@@ -1348,8 +1367,6 @@ var Tunnel = class {
           message.messageKind.val
         );
         break;
-      case "DeprecatedTunnelAck":
-        break;
       default:
         unreachable(message.messageKind);
     }
@@ -1738,10 +1755,14 @@ async function importWebSocket() {
 
 // src/mod.ts
 var KV_EXPIRE = 3e4;
-var PROTOCOL_VERSION = 3;
-var RUNNER_PING_INTERVAL = 3e3;
+var PROTOCOL_VERSION = 5;
 var EVENT_BACKLOG_WARN_THRESHOLD = 1e4;
 var SIGNAL_HANDLERS = [];
+var RunnerShutdownError = class extends Error {
+  constructor() {
+    super("Runner shut down");
+  }
+};
 var Runner = class {
   #config;
   get config() {
@@ -1751,9 +1772,6 @@ var Runner = class {
   // WebSocket
   __pegboardWebSocket;
   runnerId;
-  #lastCommandIdx = -1;
-  #pingLoop;
-  #nextEventIdx = 0n;
   #started = false;
   #shutdown = false;
   #shuttingDown = false;
@@ -1763,7 +1781,6 @@ var Runner = class {
   #runnerLostThreshold;
   #runnerLostTimeout;
   // Event storage for resending
-  #eventHistory = [];
   #eventBacklogWarned = false;
   // Command acknowledgment
   #ackInterval;
@@ -1794,7 +1811,15 @@ var Runner = class {
     this.#config = config;
     if (this.#config.logger) setLogger(this.#config.logger);
     this.#kvCleanupInterval = setInterval(() => {
-      this.#cleanupOldKvRequests();
+      var _a;
+      try {
+        this.#cleanupOldKvRequests();
+      } catch (err) {
+        (_a = this.log) == null ? void 0 : _a.error({
+          msg: "error cleaning up kv requests",
+          error: stringifyError(err)
+        });
+      }
     }, 15e3);
   }
   // MARK: Manage actors
@@ -1809,7 +1834,11 @@ var Runner = class {
     this.#sendActorIntent(actorId, actor.generation, "stop");
   }
   async forceStopActor(actorId, generation) {
-    var _a;
+    var _a, _b;
+    (_a = this.log) == null ? void 0 : _a.debug({
+      msg: "force stopping actor",
+      actorId
+    });
     const actor = this.getActor(actorId, generation);
     if (!actor) return;
     try {
@@ -1817,18 +1846,32 @@ var Runner = class {
     } catch (err) {
       console.error(`Error in onActorStop for actor ${actorId}:`, err);
     }
-    (_a = this.#tunnel) == null ? void 0 : _a.closeActiveRequests(actor);
-    this.#removeActor(actorId, generation);
+    (_b = this.#tunnel) == null ? void 0 : _b.closeActiveRequests(actor);
     this.#sendActorStateUpdate(actorId, actor.generation, "stopped");
+    this.#removeActor(actorId, generation);
   }
-  #stopAllActors() {
+  #handleLost() {
     var _a;
     (_a = this.log) == null ? void 0 : _a.info({
-      msg: "stopping all actors due to runner lost threshold exceeded"
+      msg: "stopping all actors due to runner lost threshold"
     });
+    for (const [_, request] of this.#kvRequests.entries()) {
+      request.reject(new RunnerShutdownError());
+    }
+    this.#kvRequests.clear();
+    this.#stopAllActors();
+  }
+  #stopAllActors() {
     const actorIds = Array.from(this.#actors.keys());
     for (const actorId of actorIds) {
-      this.forceStopActor(actorId);
+      this.forceStopActor(actorId).catch((err) => {
+        var _a;
+        (_a = this.log) == null ? void 0 : _a.error({
+          msg: "error stopping actor",
+          actorId,
+          error: stringifyError(err)
+        });
+      });
     }
   }
   getActor(actorId, generation) {
@@ -1956,10 +1999,6 @@ var Runner = class {
       clearTimeout(this.#runnerLostTimeout);
       this.#runnerLostTimeout = void 0;
     }
-    if (this.#pingLoop) {
-      clearInterval(this.#pingLoop);
-      this.#pingLoop = void 0;
-    }
     if (this.#ackInterval) {
       clearInterval(this.#ackInterval);
       this.#ackInterval = void 0;
@@ -2167,7 +2206,6 @@ var Runner = class {
         name: this.#config.runnerName,
         version: this.#config.version,
         totalSlots: this.#config.totalSlots,
-        lastCommandIdx: this.#lastCommandIdx >= 0 ? BigInt(this.#lastCommandIdx) : null,
         prepopulateActorNames: new Map(
           Object.entries(this.#config.prepopulateActorNames).map(
             ([name, data]) => [
@@ -2182,39 +2220,29 @@ var Runner = class {
         tag: "ToServerInit",
         val: init
       });
-      const pingLoop = setInterval(() => {
-        var _a3;
-        if (ws.readyState === 1) {
-          this.__sendToServer({
-            tag: "ToServerPing",
-            val: {
-              ts: BigInt(Date.now())
-            }
-          });
-        } else {
-          clearInterval(pingLoop);
-          (_a3 = this.log) == null ? void 0 : _a3.info({
-            msg: "WebSocket not open, stopping ping loop"
-          });
-        }
-      }, RUNNER_PING_INTERVAL);
-      this.#pingLoop = pingLoop;
       const ackInterval = 5 * 60 * 1e3;
       const ackLoop = setInterval(() => {
-        var _a3;
-        if (ws.readyState === 1) {
-          this.#sendCommandAcknowledgment();
-        } else {
-          clearInterval(ackLoop);
-          (_a3 = this.log) == null ? void 0 : _a3.info({
-            msg: "WebSocket not open, stopping ack loop"
+        var _a3, _b2;
+        try {
+          if (ws.readyState === 1) {
+            this.#sendCommandAcknowledgment();
+          } else {
+            clearInterval(ackLoop);
+            (_a3 = this.log) == null ? void 0 : _a3.info({
+              msg: "WebSocket not open, stopping ack loop"
+            });
+          }
+        } catch (err) {
+          (_b2 = this.log) == null ? void 0 : _b2.error({
+            msg: "error in command acknowledgment loop",
+            error: stringifyError(err)
           });
         }
       }, ackInterval);
       this.#ackInterval = ackLoop;
     });
     ws.addEventListener("message", async (ev) => {
-      var _a2, _b, _c, _d, _e, _f;
+      var _a2, _b, _c, _d, _e;
       let buf;
       if (ev.data instanceof Blob) {
         buf = new Uint8Array(await ev.data.arrayBuffer());
@@ -2232,16 +2260,15 @@ var Runner = class {
         const init = message.val;
         if (this.runnerId !== init.runnerId) {
           this.runnerId = init.runnerId;
-          this.#eventHistory.length = 0;
+          this.#stopAllActors();
         }
         this.#runnerLostThreshold = ((_b = init.metadata) == null ? void 0 : _b.runnerLostThreshold) ? Number(init.metadata.runnerLostThreshold) : void 0;
         (_c = this.log) == null ? void 0 : _c.info({
           msg: "received init",
-          lastEventIdx: init.lastEventIdx,
           runnerLostThreshold: this.#runnerLostThreshold
         });
         this.#processUnsentKvRequests();
-        this.#resendUnacknowledgedEvents(init.lastEventIdx);
+        this.#resendUnacknowledgedEvents();
         (_d = this.#tunnel) == null ? void 0 : _d.resendBufferedEvents();
         this.#config.onConnected();
       } else if (message.tag === "ToClientCommands") {
@@ -2253,10 +2280,20 @@ var Runner = class {
         const kvResponse = message.val;
         this.#handleKvResponse(kvResponse);
       } else if (message.tag === "ToClientTunnelMessage") {
-        (_e = this.#tunnel) == null ? void 0 : _e.handleTunnelMessage(message.val);
-      } else if (message.tag === "ToClientClose") {
-        (_f = this.#tunnel) == null ? void 0 : _f.shutdown();
-        ws.close(1e3, "manual closure");
+        (_e = this.#tunnel) == null ? void 0 : _e.handleTunnelMessage(message.val).catch((err) => {
+          var _a3;
+          (_a3 = this.log) == null ? void 0 : _a3.error({
+            msg: "error handling tunnel message",
+            error: stringifyError(err)
+          });
+        });
+      } else if (message.tag === "ToClientPing") {
+        this.__sendToServer({
+          tag: "ToServerPong",
+          val: {
+            ts: message.val.ts
+          }
+        });
       } else {
         unreachable(message);
       }
@@ -2273,7 +2310,15 @@ var Runner = class {
             seconds: this.#runnerLostThreshold / 1e3
           });
           this.#runnerLostTimeout = setTimeout(() => {
-            this.#stopAllActors();
+            var _a3;
+            try {
+              this.#handleLost();
+            } catch (err) {
+              (_a3 = this.log) == null ? void 0 : _a3.error({
+                msg: "error handling runner lost",
+                error: stringifyError(err)
+              });
+            }
           }, this.#runnerLostThreshold);
         }
         this.#scheduleReconnect();
@@ -2299,10 +2344,6 @@ var Runner = class {
         }
         this.#config.onDisconnected(ev.code, ev.reason);
       }
-      if (this.#pingLoop) {
-        clearInterval(this.#pingLoop);
-        this.#pingLoop = void 0;
-      }
       if (this.#ackInterval) {
         clearInterval(this.#ackInterval);
         this.#ackInterval = void 0;
@@ -2314,7 +2355,15 @@ var Runner = class {
             seconds: this.#runnerLostThreshold / 1e3
           });
           this.#runnerLostTimeout = setTimeout(() => {
-            this.#stopAllActors();
+            var _a3;
+            try {
+              this.#handleLost();
+            } catch (err) {
+              (_a3 = this.log) == null ? void 0 : _a3.error({
+                msg: "error handling runner lost",
+                error: stringifyError(err)
+              });
+            }
           }, this.#runnerLostThreshold);
         }
         this.#scheduleReconnect();
@@ -2329,43 +2378,75 @@ var Runner = class {
     });
     for (const commandWrapper of commands) {
       if (commandWrapper.inner.tag === "CommandStartActor") {
-        this.#handleCommandStartActor(commandWrapper);
+        this.#handleCommandStartActor(commandWrapper).catch((err) => {
+          var _a2;
+          (_a2 = this.log) == null ? void 0 : _a2.error({
+            msg: "error handling start actor command",
+            actorId: commandWrapper.checkpoint.actorId,
+            error: stringifyError(err)
+          });
+        });
+        const actor = this.getActor(
+          commandWrapper.checkpoint.actorId,
+          commandWrapper.checkpoint.generation
+        );
+        if (actor) actor.lastCommandIdx = commandWrapper.checkpoint.index;
       } else if (commandWrapper.inner.tag === "CommandStopActor") {
-        this.#handleCommandStopActor(commandWrapper);
+        this.#handleCommandStopActor(commandWrapper).catch((err) => {
+          var _a2;
+          (_a2 = this.log) == null ? void 0 : _a2.error({
+            msg: "error handling stop actor command",
+            actorId: commandWrapper.checkpoint.actorId,
+            error: stringifyError(err)
+          });
+        });
       } else {
         unreachable(commandWrapper.inner);
       }
-      this.#lastCommandIdx = Number(commandWrapper.index);
     }
   }
   #handleAckEvents(ack) {
     var _a;
-    const lastAckedIdx = ack.lastEventIdx;
-    const originalLength = this.#eventHistory.length;
-    this.#eventHistory = this.#eventHistory.filter(
-      (event) => event.index > lastAckedIdx
+    let originalTotalEvents = Array.from(this.#actors).reduce(
+      (s, [_, actor]) => s + actor.eventHistory.length,
+      0
+    );
+    for (const [_, actor] of this.#actors) {
+      let checkpoint = ack.lastEventCheckpoints.find(
+        (x) => x.actorId == actor.actorId
+      );
+      if (checkpoint) actor.handleAckEvents(checkpoint.index);
+    }
+    const totalEvents = Array.from(this.#actors).reduce(
+      (s, [_, actor]) => s + actor.eventHistory.length,
+      0
     );
-    const prunedCount = originalLength - this.#eventHistory.length;
+    const prunedCount = originalTotalEvents - totalEvents;
     if (prunedCount > 0) {
       (_a = this.log) == null ? void 0 : _a.info({
         msg: "pruned acknowledged events",
-        lastAckedIdx: lastAckedIdx.toString(),
         prunedCount
       });
     }
-    if (this.#eventHistory.length <= EVENT_BACKLOG_WARN_THRESHOLD) {
+    if (totalEvents <= EVENT_BACKLOG_WARN_THRESHOLD) {
       this.#eventBacklogWarned = false;
     }
   }
   /** Track events to send to the server in case we need to resend it on disconnect. */
   #recordEvent(eventWrapper) {
     var _a;
-    this.#eventHistory.push(eventWrapper);
-    if (this.#eventHistory.length > EVENT_BACKLOG_WARN_THRESHOLD && !this.#eventBacklogWarned) {
+    const actor = this.getActor(eventWrapper.checkpoint.actorId);
+    if (!actor) return;
+    actor.recordEvent(eventWrapper);
+    let totalEvents = Array.from(this.#actors).reduce(
+      (s, [_, actor2]) => s + actor2.eventHistory.length,
+      0
+    );
+    if (totalEvents > EVENT_BACKLOG_WARN_THRESHOLD && !this.#eventBacklogWarned) {
       this.#eventBacklogWarned = true;
       (_a = this.log) == null ? void 0 : _a.warn({
         msg: "unacknowledged event backlog exceeds threshold",
-        backlogSize: this.#eventHistory.length,
+        backlogSize: totalEvents,
         threshold: EVENT_BACKLOG_WARN_THRESHOLD
       });
     }
@@ -2374,8 +2455,8 @@ var Runner = class {
     var _a, _b, _c, _d;
     if (!this.#tunnel) throw new Error("missing tunnel on actor start");
     const startCommand = commandWrapper.inner.val;
-    const actorId = startCommand.actorId;
-    const generation = startCommand.generation;
+    const actorId = commandWrapper.checkpoint.actorId;
+    const generation = commandWrapper.checkpoint.generation;
     const config = startCommand.config;
     const actorConfig = {
       name: config.name,
@@ -2433,11 +2514,13 @@ var Runner = class {
   }
   async #handleCommandStopActor(commandWrapper) {
     const stopCommand = commandWrapper.inner.val;
-    const actorId = stopCommand.actorId;
-    const generation = stopCommand.generation;
+    const actorId = commandWrapper.checkpoint.actorId;
+    const generation = commandWrapper.checkpoint.generation;
     await this.forceStopActor(actorId, generation);
   }
   #sendActorIntent(actorId, generation, intentType) {
+    const actor = this.getActor(actorId, generation);
+    if (!actor) return;
     let actorIntent;
     if (intentType === "sleep") {
       actorIntent = { tag: "ActorIntentSleep", val: null };
@@ -2450,13 +2533,14 @@ var Runner = class {
       unreachable(intentType);
     }
     const intentEvent = {
-      actorId,
-      generation,
       intent: actorIntent
     };
-    const eventIndex = this.#nextEventIdx++;
     const eventWrapper = {
-      index: eventIndex,
+      checkpoint: {
+        actorId,
+        generation,
+        index: actor.nextEventIdx++
+      },
       inner: {
         tag: "EventActorIntent",
         val: intentEvent
@@ -2469,6 +2553,8 @@ var Runner = class {
     });
   }
   #sendActorStateUpdate(actorId, generation, stateType) {
+    const actor = this.getActor(actorId, generation);
+    if (!actor) return;
     let actorState;
     if (stateType === "running") {
       actorState = { tag: "ActorStateRunning", val: null };
@@ -2484,13 +2570,14 @@ var Runner = class {
       unreachable(stateType);
     }
     const stateUpdateEvent = {
-      actorId,
-      generation,
       state: actorState
     };
-    const eventIndex = this.#nextEventIdx++;
     const eventWrapper = {
-      index: eventIndex,
+      checkpoint: {
+        actorId,
+        generation,
+        index: actor.nextEventIdx++
+      },
       inner: {
         tag: "EventActorStateUpdate",
         val: stateUpdateEvent
@@ -2503,13 +2590,21 @@ var Runner = class {
     });
   }
   #sendCommandAcknowledgment() {
-    if (this.#lastCommandIdx < 0) {
-      return;
+    const lastCommandCheckpoints = [];
+    for (const [_, actor] of this.#actors) {
+      if (actor.lastCommandIdx < 0) {
+        continue;
+      }
+      lastCommandCheckpoints.push({
+        actorId: actor.actorId,
+        generation: actor.generation,
+        index: actor.lastCommandIdx
+      });
     }
     this.__sendToServer({
       tag: "ToServerAckCommands",
       val: {
-        lastCommandIdx: BigInt(this.#lastCommandIdx)
+        lastCommandCheckpoints
       }
     });
   }
@@ -2739,13 +2834,14 @@ var Runner = class {
     const actor = this.getActor(actorId, generation);
     if (!actor) return;
     const alarmEvent = {
-      actorId,
-      generation: actor.generation,
       alarmTs: alarmTs !== null ? BigInt(alarmTs) : null
     };
-    const eventIndex = this.#nextEventIdx++;
     const eventWrapper = {
-      index: eventIndex,
+      checkpoint: {
+        actorId,
+        generation: actor.generation,
+        index: actor.nextEventIdx++
+      },
       inner: {
         tag: "EventActorSetAlarm",
         val: alarmEvent
@@ -2872,7 +2968,8 @@ var Runner = class {
     const data = protocol.encodeToServerlessServer({
       tag: "ToServerlessServerInit",
       val: {
-        runnerId: this.runnerId
+        runnerId: this.runnerId,
+        runnerProtocolVersion: PROTOCOL_VERSION
       }
     });
     const buffer = Buffer.alloc(data.length + 2);
@@ -2897,26 +2994,33 @@ var Runner = class {
     (_b = this.log) == null ? void 0 : _b.debug({
       msg: `Scheduling reconnect attempt ${this.#reconnectAttempt + 1} in ${delay}ms`
     });
-    this.#reconnectTimeout = setTimeout(async () => {
+    this.#reconnectTimeout = setTimeout(() => {
       var _a2;
       if (!this.#shutdown) {
         this.#reconnectAttempt++;
         (_a2 = this.log) == null ? void 0 : _a2.debug({
           msg: `Attempting to reconnect (attempt ${this.#reconnectAttempt})...`
         });
-        await this.#openPegboardWebSocket();
+        this.#openPegboardWebSocket().catch((err) => {
+          var _a3;
+          (_a3 = this.log) == null ? void 0 : _a3.error({
+            msg: "error during websocket reconnection",
+            error: stringifyError(err)
+          });
+        });
       }
     }, delay);
   }
-  #resendUnacknowledgedEvents(lastEventIdx) {
+  #resendUnacknowledgedEvents() {
     var _a;
-    const eventsToResend = this.#eventHistory.filter(
-      (event) => event.index > lastEventIdx
-    );
+    const eventsToResend = [];
+    for (const [_, actor] of this.#actors) {
+      eventsToResend.push(...actor.eventHistory);
+    }
     if (eventsToResend.length === 0) return;
     (_a = this.log) == null ? void 0 : _a.info({
       msg: "resending unacknowledged events",
-      fromIndex: lastEventIdx + 1n
+      count: eventsToResend.length
     });
     this.__sendToServer({
       tag: "ToServerEvents",
@@ -2946,6 +3050,7 @@ var Runner = class {
 export {
   Runner,
   RunnerActor,
+  RunnerShutdownError,
   idToStr
 };
 //# sourceMappingURL=mod.js.map