@thru/replay 0.2.32 → 0.2.34

package/dist/index.mjs

@@ -442,6 +442,7 @@ var DEFAULT_METRICS = {
   emittedReconnect: 0,
   discardedDuplicates: 0
 };
+var RECONNECT_STUCK_THRESHOLD_MS = 3e4;
 function compareBigint(a, b) {
   if (a === b) return 0;
   return a < b ? -1 : 1;
@@ -643,6 +644,7 @@ var ReplayStream = class {
       this.logger.info("replay entering STREAMING state");
       const retryConfig = DEFAULT_RETRY_CONFIG;
       let retryAttempt = 0;
+      let waitingFirstReconnectEvent = null;
       while (true) {
         if (shouldStop()) return;
         try {
@@ -657,6 +659,16 @@ var ReplayStream = class {
             throw new Error("stream ended");
           }
           const slot = extractSlot(next.value);
+          if (waitingFirstReconnectEvent) {
+            this.logger.info("Replay stream reconnect completed", {
+              event: "replay.stream.reconnect.completed",
+              attempt: waitingFirstReconnectEvent.attempt,
+              resume_slot: waitingFirstReconnectEvent.resumeSlot.toString(),
+              first_event_slot: slot.toString(),
+              duration_ms: Date.now() - waitingFirstReconnectEvent.startedAtMs
+            });
+            waitingFirstReconnectEvent = null;
+          }
           const key = keyOf(next.value);
           if (seenItem(slot, key)) {
             this.metrics.discardedDuplicates += 1;
@@ -671,13 +683,45 @@ var ReplayStream = class {
           if (shouldStop(err)) return;
           const errMsg = err instanceof Error ? err.message : String(err);
           const backoffMs = calculateBackoff(retryAttempt, retryConfig);
-          this.logger.warn(
-            `live stream disconnected (${errMsg}); reconnecting in ${backoffMs}ms from slot ${currentSlot} (attempt ${retryAttempt + 1})`
-          );
+          const attempt = retryAttempt + 1;
+          const reconnectStartedAtMs = Date.now();
+          this.logger.warn("Replay stream reconnect started", {
+            event: "replay.stream.reconnect.started",
+            reason: errMsg === "stream ended" ? "stream_ended" : "stream_error",
+            error: errMsg,
+            backoff_ms: backoffMs,
+            attempt,
+            current_slot: currentSlot.toString()
+          });
           await abortableDelay(backoffMs, signal);
           if (shouldStop()) return;
+          const cleanupStartedAtMs = Date.now();
+          this.logger.info("Replay stream reconnect cleanup started", {
+            event: "replay.stream.reconnect.cleanup_started",
+            phase: "live_pump_close",
+            attempt,
+            current_slot: currentSlot.toString()
+          });
           currentDispose();
-          await safeClose(livePump);
+          const closeResult = await safeClose(livePump, signal);
+          const cleanupDurationMs = Date.now() - cleanupStartedAtMs;
+          if (closeResult === "timed-out") {
+            this.logger.warn("Replay stream reconnect cleanup stuck", {
+              event: "replay.stream.reconnect.stuck",
+              phase: "live_pump_close",
+              attempt,
+              duration_ms: cleanupDurationMs,
+              timeout_ms: RECONNECT_STUCK_THRESHOLD_MS
+            });
+          }
+          this.logger.info("Replay stream reconnect cleanup completed", {
+            event: "replay.stream.reconnect.cleanup_completed",
+            phase: "live_pump_close",
+            attempt,
+            duration_ms: cleanupDurationMs,
+            result: closeResult
+          });
+          if (shouldStop()) return;
           retryAttempt++;
           if (onReconnect) {
             try {
@@ -688,11 +732,16 @@ var ReplayStream = class {
               }
               currentDispose = fresh.dispose ?? (() => {
               });
-              this.logger.info("created fresh client for reconnection");
+              this.logger.info("Replay stream fresh reconnect sources created", {
+                event: "replay.stream.reconnect.sources_created",
+                attempt
+              });
             } catch (factoryErr) {
-              this.logger.error(
-                `failed to create fresh client: ${factoryErr instanceof Error ? factoryErr.message : String(factoryErr)}; using existing`
-              );
+              this.logger.error("Replay stream fresh reconnect sources failed", {
+                event: "replay.stream.reconnect.sources_failed",
+                attempt,
+                error: factoryErr
+              });
             }
           }
           if (onReconnect && currentSlot > 0n) {
@@ -715,11 +764,21 @@ var ReplayStream = class {
           }
           const resumeSlot = currentSlot > 0n ? currentSlot : 0n;
           livePump = createLivePump(resumeSlot, true, currentSlot);
+          waitingFirstReconnectEvent = {
+            startedAtMs: reconnectStartedAtMs,
+            attempt,
+            resumeSlot
+          };
+          this.logger.info("Replay stream waiting for first event", {
+            event: "replay.stream.waiting_first_event",
+            attempt,
+            resume_slot: resumeSlot.toString()
+          });
         }
       }
     } finally {
       currentDispose();
-      await safeClose(livePump);
+      await safeClose(livePump, signal);
     }
   }
   /**
@@ -773,24 +832,45 @@ var ReplayStream = class {
     }
   }
 };
-async function safeClose(pump) {
+async function safeClose(pump, signal) {
   let timeoutId;
+  let onAbort;
   try {
+    if (signal?.aborted) {
+      return "aborted";
+    }
     const timeout = new Promise((resolve) => {
-      timeoutId = setTimeout(() => resolve("timeout"), 5e3);
+      timeoutId = setTimeout(() => resolve("timeout"), RECONNECT_STUCK_THRESHOLD_MS);
     });
+    const abort = signal ? new Promise((resolve) => {
+      onAbort = () => resolve("aborted");
+      signal.addEventListener("abort", onAbort, { once: true });
+    }) : null;
+    const close = pump.close().then(
+      () => "closed",
+      () => "closed"
+    );
     const result = await Promise.race([
-      pump.close().then(() => "closed"),
-      timeout
+      close,
+      timeout,
+      ...abort ? [abort] : []
     ]);
+    if (result === "aborted") {
+      return "aborted";
+    }
     if (result === "timeout") {
+      return "timed-out";
     }
   } catch {
   } finally {
     if (timeoutId !== void 0) {
       clearTimeout(timeoutId);
     }
+    if (signal && onAbort) {
+      signal.removeEventListener("abort", onAbort);
+    }
   }
+  return "closed";
 }
 function combineFilters(base, user) {
   if (!base && !user) return void 0;
@@ -1268,6 +1348,7 @@ var PageAssembler = class {
 };
 
 // src/account-replay.ts
+var DEFAULT_RECONNECT_CLEANUP_TIMEOUT_MS = 3e4;
 async function closeAsyncIterator(iterator) {
   if (!iterator || typeof iterator.return !== "function") {
     return;
@@ -1277,6 +1358,72 @@ async function closeAsyncIterator(iterator) {
   } catch {
   }
 }
+async function waitForCleanup(promise, timeoutMs, label, logger, signal) {
+  const startedAtMs = Date.now();
+  logger.info("Replay stream reconnect cleanup started", {
+    event: "replay.stream.reconnect.cleanup_started",
+    phase: label,
+    timeout_ms: timeoutMs
+  });
+  if (signal?.aborted) {
+    logger.debug("Replay stream reconnect cleanup aborted", {
+      event: "replay.stream.reconnect.cleanup_completed",
+      phase: label,
+      timeout_ms: timeoutMs,
+      result: "aborted",
+      duration_ms: 0
+    });
+    return "aborted";
+  }
+  let timer = null;
+  let onAbort = null;
+  const timeoutPromise = new Promise((resolve) => {
+    timer = setTimeout(() => resolve("timed-out"), timeoutMs);
+    timer.unref?.();
+  });
+  const abortPromise = signal ? new Promise((resolve) => {
+    onAbort = () => resolve("aborted");
+    signal.addEventListener("abort", onAbort, { once: true });
+  }) : null;
+  const completionPromise = promise.then(
+    () => "completed",
+    () => "completed"
+  );
+  const result = await Promise.race(
+    abortPromise ? [completionPromise, timeoutPromise, abortPromise] : [completionPromise, timeoutPromise]
+  );
+  if (timer) {
+    clearTimeout(timer);
+  }
+  if (signal && onAbort) {
+    signal.removeEventListener("abort", onAbort);
+  }
+  if (result === "timed-out") {
+    logger.warn("Replay stream reconnect cleanup stuck", {
+      event: "replay.stream.reconnect.stuck",
+      phase: label,
+      timeout_ms: timeoutMs,
+      duration_ms: Date.now() - startedAtMs
+    });
+  } else if (result === "aborted") {
+    logger.debug("Replay stream reconnect cleanup aborted", {
+      event: "replay.stream.reconnect.cleanup_completed",
+      phase: label,
+      timeout_ms: timeoutMs,
+      result,
+      duration_ms: Date.now() - startedAtMs
+    });
+  } else {
+    logger.info("Replay stream reconnect cleanup completed", {
+      event: "replay.stream.reconnect.cleanup_completed",
+      phase: label,
+      timeout_ms: timeoutMs,
+      result,
+      duration_ms: Date.now() - startedAtMs
+    });
+  }
+  return result;
+}
 function bytesToHex2(bytes) {
   return Array.from(bytes).map((b) => b.toString(16).padStart(2, "0")).join("");
 }
@@ -1349,6 +1496,8 @@ async function* createAccountsByOwnerReplay(options) {
     maxRetries = 3,
     pageAssemblerOptions,
     cleanupInterval = 1e4,
+    reconnectCleanupTimeoutMs = DEFAULT_RECONNECT_CLEANUP_TIMEOUT_MS,
+    retryConfig = DEFAULT_RETRY_CONFIG,
     onBackfillComplete,
     clientFactory,
     logger = NOOP_LOGGER,
@@ -1368,23 +1517,98 @@ async function* createAccountsByOwnerReplay(options) {
   let lastActivityTime = Date.now();
   let activeStreamIterator = null;
   let activeStreamProcessor = null;
-  try {
+  let streamGeneration = 0;
+  let retryAttempt = 0;
+  const retireActiveStream = () => {
+    const iterator = activeStreamIterator;
+    const processor = activeStreamProcessor;
+    activeStreamIterator = null;
+    activeStreamProcessor = null;
+    streamGeneration++;
+    return { iterator, processor };
+  };
+  const cleanupRetiredStream = async (retired, iteratorLabel, processorLabel) => {
+    if (retired.iterator) {
+      await waitForCleanup(
+        closeAsyncIterator(retired.iterator),
+        reconnectCleanupTimeoutMs,
+        iteratorLabel,
+        logger,
+        signal
+      );
+    }
     if (shouldStop()) return;
-    cleanupTimer = setInterval(() => {
-      assembler.cleanup();
-    }, cleanupInterval);
-    cleanupTimer.unref?.();
-    const streamFilter = buildOwnerFilterWithMinSlot(owner, dataSizes, minUpdatedSlot);
-    const stream = client.streamAccountUpdates({ view, filter: streamFilter });
-    const streamIterator = stream[Symbol.asyncIterator]();
-    activeStreamIterator = streamIterator;
-    activeStreamProcessor = (async () => {
+    if (retired.processor) {
+      await waitForCleanup(
+        Promise.allSettled([retired.processor]),
+        reconnectCleanupTimeoutMs,
+        processorLabel,
+        logger,
+        signal
+      );
+    }
+  };
+  const createFreshClient = () => {
+    if (!clientFactory) {
+      return;
+    }
+    logger.info("Replay stream fresh client creation started", {
+      event: "replay.stream.reconnect.client_started"
+    });
+    try {
+      const previousClient = client;
+      const newClient = clientFactory();
+      closeIfCloseable(previousClient);
+      client = newClient;
+      logger.info("Replay stream fresh client creation completed", {
+        event: "replay.stream.reconnect.client_completed"
+      });
+    } catch (err) {
+      logger.error("Replay stream fresh client creation failed", {
+        event: "replay.stream.reconnect.client_failed",
+        error: err
+      });
+    }
+  };
+  const createStreamProcessor = (reason = "initial") => {
+    const minSlot = highestSlotSeen > 0n ? highestSlotSeen : minUpdatedSlot;
+    const generation = ++streamGeneration;
+    const streamStartedAtMs = Date.now();
+    let firstMessageSeen = false;
+    if (reason === "reconnect") {
+      logger.info("Replay stream waiting for first event", {
+        event: "replay.stream.waiting_first_event",
+        generation,
+        min_slot: minSlot?.toString()
+      });
+    }
+    const newStreamFilter = buildOwnerFilterWithMinSlot(owner, dataSizes, minSlot);
+    const newStream = client.streamAccountUpdates({ view, filter: newStreamFilter });
+    const newStreamIterator = newStream[Symbol.asyncIterator]();
+    streamDone = false;
+    streamError = null;
+    lastActivityTime = Date.now();
+    activeStreamIterator = newStreamIterator;
+    const newProcessor = (async () => {
       try {
         while (true) {
-          const next = await streamIterator.next();
+          const next = await newStreamIterator.next();
+          if (generation !== streamGeneration) {
+            return;
+          }
           if (next.done) break;
           const response = next.value;
+          retryAttempt = 0;
           lastActivityTime = Date.now();
+          if (reason === "reconnect" && !firstMessageSeen) {
+            firstMessageSeen = true;
+            logger.info("Replay stream reconnect completed", {
+              event: "replay.stream.reconnect.completed",
+              generation,
+              first_message: true,
+              duration_ms: Date.now() - streamStartedAtMs
+            });
+          }
           const event = processResponseMulti(response, assembler);
           if (event) {
             if (event.type === "account") {
@@ -1397,11 +1621,24 @@ async function* createAccountsByOwnerReplay(options) {
           }
         }
       } catch (err) {
-        streamError = err;
+        if (generation === streamGeneration) {
+          streamError = err;
+        }
       } finally {
-        streamDone = true;
+        if (generation === streamGeneration) {
+          streamDone = true;
+        }
       }
     })();
+    activeStreamProcessor = newProcessor;
+  };
+  try {
+    if (shouldStop()) return;
+    cleanupTimer = setInterval(() => {
+      assembler.cleanup();
+    }, cleanupInterval);
+    cleanupTimer.unref?.();
+    createStreamProcessor();
     const yieldStreamBuffer = function* () {
       while (streamBuffer.length > 0) {
         const event = streamBuffer.shift();
@@ -1479,51 +1716,8 @@ async function* createAccountsByOwnerReplay(options) {
     if (onBackfillComplete) {
       onBackfillComplete(highestSlotSeen);
     }
-    const retryConfig = DEFAULT_RETRY_CONFIG;
-    let retryAttempt = 0;
+    retryAttempt = 0;
     lastActivityTime = Date.now();
-    const createStreamProcessor = () => {
-      if (clientFactory) {
-        try {
-          const newClient = clientFactory();
-          closeIfCloseable(client);
-          client = newClient;
-          logger.info("[account-stream] created fresh client for reconnection");
-        } catch (err) {
-          logger.error("[account-stream] failed to create fresh client", { error: err });
-        }
-      }
-      const newStreamFilter = buildOwnerFilterWithMinSlot(owner, dataSizes, highestSlotSeen > 0n ? highestSlotSeen : minUpdatedSlot);
-      const newStream = client.streamAccountUpdates({ view, filter: newStreamFilter });
-      const newStreamIterator = newStream[Symbol.asyncIterator]();
-      activeStreamIterator = newStreamIterator;
-      const newProcessor = (async () => {
-        try {
-          while (true) {
-            const next = await newStreamIterator.next();
-            if (next.done) break;
-            const response = next.value;
-            retryAttempt = 0;
-            lastActivityTime = Date.now();
-            const event = processResponseMulti(response, assembler);
-            if (event) {
-              if (event.type === "account") {
-                seenFromStream.add(event.account.addressHex);
-                if (event.account.slot > highestSlotSeen) {
-                  highestSlotSeen = event.account.slot;
-                }
-              }
-              streamBuffer.push(event);
-            }
-          }
-        } catch (err) {
-          streamError = err;
-        } finally {
-          streamDone = true;
-        }
-      })();
-      return { iterator: newStreamIterator, processor: newProcessor };
-    };
     while (true) {
       if (shouldStop()) return;
       const hadEvents = streamBuffer.length > 0;
@@ -1531,10 +1725,13 @@ async function* createAccountsByOwnerReplay(options) {
       if (hadEvents) {
         lastActivityTime = Date.now();
       }
-      if (!streamDone && Date.now() - lastActivityTime > retryConfig.connectionTimeoutMs) {
-        logger.warn(
-          `[account-stream] no activity for ${retryConfig.connectionTimeoutMs}ms; forcing reconnection`
-        );
+      const idleMs = Date.now() - lastActivityTime;
+      if (!streamDone && idleMs > retryConfig.connectionTimeoutMs) {
+        logger.warn("Replay stream idle timeout detected", {
+          event: "replay.stream.idle_timeout",
+          idleMs,
+          connectionTimeoutMs: retryConfig.connectionTimeoutMs
+        });
         streamDone = true;
         streamError = new Error(`Operation timed out after ${retryConfig.connectionTimeoutMs}ms`);
       }
@@ -1542,36 +1739,44 @@ async function* createAccountsByOwnerReplay(options) {
         if (streamError) {
           if (shouldStop(streamError)) return;
           const backoffMs = calculateBackoff(retryAttempt, retryConfig);
-          logger.warn(
-            `[account-stream] disconnected (${streamError.message}); reconnecting in ${backoffMs}ms (attempt ${retryAttempt + 1})`
-          );
+          logger.warn("Replay stream reconnect started", {
+            event: "replay.stream.reconnect.started",
+            reason: "stream_error",
+            error: streamError.message,
+            backoffMs,
+            attempt: retryAttempt + 1,
+            highestSlotSeen: highestSlotSeen.toString()
+          });
           await abortableDelay(backoffMs, signal);
           if (shouldStop()) return;
           retryAttempt++;
+          const retired = retireActiveStream();
           streamDone = false;
           streamError = null;
           streamBuffer.length = 0;
           lastActivityTime = Date.now();
-          await closeAsyncIterator(activeStreamIterator);
-          if (activeStreamProcessor) {
-            await Promise.allSettled([activeStreamProcessor]);
-          }
-          const { iterator: newIterator, processor: newProcessor } = createStreamProcessor();
-          activeStreamIterator = newIterator;
-          activeStreamProcessor = newProcessor;
+          await cleanupRetiredStream(retired, "old iterator close", "old processor drain");
+          if (shouldStop()) return;
+          createFreshClient();
+          createStreamProcessor("reconnect");
           continue;
         } else {
           if (shouldStop()) return;
-          logger.warn("[account-stream] stream ended unexpectedly; reconnecting...");
+          logger.warn("Replay stream reconnect started", {
+            event: "replay.stream.reconnect.started",
+            reason: "stream_ended",
+            attempt: retryAttempt + 1,
+            highestSlotSeen: highestSlotSeen.toString()
+          });
+          const retired = retireActiveStream();
           streamDone = false;
+          streamError = null;
+          streamBuffer.length = 0;
           lastActivityTime = Date.now();
-          await closeAsyncIterator(activeStreamIterator);
-          if (activeStreamProcessor) {
-            await Promise.allSettled([activeStreamProcessor]);
-          }
-          const { iterator: newIterator, processor: newProcessor } = createStreamProcessor();
-          activeStreamIterator = newIterator;
-          activeStreamProcessor = newProcessor;
+          await cleanupRetiredStream(retired, "old iterator close", "old processor drain");
+          if (shouldStop()) return;
+          createFreshClient();
+          createStreamProcessor("reconnect");
           continue;
         }
       }
@@ -1581,15 +1786,11 @@ async function* createAccountsByOwnerReplay(options) {
     if (cleanupTimer) {
       clearInterval(cleanupTimer);
     }
-    const closeIteratorPromise = closeAsyncIterator(activeStreamIterator);
+    const retired = retireActiveStream();
     if (ownsClient) {
       closeIfCloseable(client);
     }
-    if (activeStreamProcessor) {
-      await Promise.allSettled([closeIteratorPromise, activeStreamProcessor]);
-    } else {
-      await closeIteratorPromise;
-    }
+    await cleanupRetiredStream(retired, "final iterator close", "final processor drain");
     assembler.clear();
   }
 }