@thru/replay 0.2.34 → 0.2.35

package/dist/index.cjs

@@ -1350,6 +1350,15 @@ var PageAssembler = class {
 
 // src/account-replay.ts
 var DEFAULT_RECONNECT_CLEANUP_TIMEOUT_MS = 3e4;
+var REPLAY_IDLE_TIMEOUT_ERROR = "ReplayIdleTimeoutError";
+function createIdleTimeoutError(timeoutMs) {
+  const error = new Error(`Operation timed out after ${timeoutMs}ms`);
+  error.name = REPLAY_IDLE_TIMEOUT_ERROR;
+  return error;
+}
+function isIdleTimeoutError(error) {
+  return error.name === REPLAY_IDLE_TIMEOUT_ERROR;
+}
 async function closeAsyncIterator(iterator) {
   if (!iterator || typeof iterator.return !== "function") {
     return;
@@ -1510,9 +1519,11 @@ async function* createAccountsByOwnerReplay(options) {
   const seenFromStream = /* @__PURE__ */ new Set();
   const fetchQueue = [];
   let highestSlotSeen = minUpdatedSlot ?? 0n;
+  const lastEmittedAccounts = /* @__PURE__ */ new Map();
   const assembler = new PageAssembler(pageAssemblerOptions);
   let cleanupTimer = null;
   const streamBuffer = [];
+  const deferredStreamBuffer = [];
   let streamDone = false;
   let streamError = null;
   let lastActivityTime = Date.now();
@@ -1520,6 +1531,71 @@ async function* createAccountsByOwnerReplay(options) {
   let activeStreamProcessor = null;
   let streamGeneration = 0;
   let retryAttempt = 0;
+  let pendingCatchUpFromSlot = null;
+  const pendingCleanupTasks = /* @__PURE__ */ new Set();
+  const shouldEmitAccountState = (account) => {
+    const previous = lastEmittedAccounts.get(account.addressHex);
+    if (previous && (account.slot < previous.slot || account.slot === previous.slot && account.seq <= previous.seq)) {
+      return false;
+    }
+    lastEmittedAccounts.set(account.addressHex, {
+      slot: account.slot,
+      seq: account.seq
+    });
+    if (account.slot > highestSlotSeen) {
+      highestSlotSeen = account.slot;
+    }
+    return true;
+  };
+  const queueReplayEvent = (event) => {
+    if (event.type === "account") {
+      if (!shouldEmitAccountState(event.account)) {
+        return false;
+      }
+      seenFromStream.add(event.account.addressHex);
+    }
+    streamBuffer.push(event);
+    return true;
+  };
+  const queueStreamEvent = (event) => {
+    if (pendingCatchUpFromSlot !== null) {
+      deferredStreamBuffer.push(event);
+      return true;
+    }
+    return queueReplayEvent(event);
+  };
+  const flushDeferredStreamEvents = () => {
+    while (deferredStreamBuffer.length > 0) {
+      const event = deferredStreamBuffer.shift();
+      queueReplayEvent(event);
+    }
+  };
+  const getReconnectFromSlot = () => {
+    if (pendingCatchUpFromSlot === null) {
+      return highestSlotSeen;
+    }
+    if (highestSlotSeen <= 0n) {
+      return pendingCatchUpFromSlot;
+    }
+    return pendingCatchUpFromSlot < highestSlotSeen ? pendingCatchUpFromSlot : highestSlotSeen;
+  };
+  const markCatchUpPending = (fromSlot) => {
+    if (fromSlot <= 0n) {
+      return fromSlot;
+    }
+    if (pendingCatchUpFromSlot === null || fromSlot < pendingCatchUpFromSlot) {
+      pendingCatchUpFromSlot = fromSlot;
+    }
+    return pendingCatchUpFromSlot;
+  };
+  const markCatchUpCompleted = (fromSlot) => {
+    if (fromSlot <= 0n) {
+      return;
+    }
+    if (pendingCatchUpFromSlot !== null && fromSlot <= pendingCatchUpFromSlot) {
+      pendingCatchUpFromSlot = null;
+    }
+  };
   const retireActiveStream = () => {
     const iterator = activeStreamIterator;
     const processor = activeStreamProcessor;
@@ -1549,6 +1625,18 @@ async function* createAccountsByOwnerReplay(options) {
       );
     }
   };
+  const cleanupRetiredStreamInBackground = (retired, iteratorLabel, processorLabel) => {
+    const cleanupTask = cleanupRetiredStream(retired, iteratorLabel, processorLabel).catch((err) => {
+      logger.warn("Replay stream reconnect cleanup failed", {
+        event: "replay.stream.reconnect.cleanup_failed",
+        error: err
+      });
+    });
+    pendingCleanupTasks.add(cleanupTask);
+    cleanupTask.finally(() => {
+      pendingCleanupTasks.delete(cleanupTask);
+    });
+  };
   const createFreshClient = () => {
     if (!clientFactory) {
       return;
@@ -1571,8 +1659,8 @@ async function* createAccountsByOwnerReplay(options) {
       });
     }
   };
-  const createStreamProcessor = (reason = "initial") => {
-    const minSlot = highestSlotSeen > 0n ? highestSlotSeen : minUpdatedSlot;
+  const createStreamProcessor = (reason = "initial", minSlotOverride) => {
+    const minSlot = minSlotOverride ?? (highestSlotSeen > 0n ? highestSlotSeen : minUpdatedSlot);
     const generation = ++streamGeneration;
     const streamStartedAtMs = Date.now();
     let firstMessageSeen = false;
@@ -1580,7 +1668,8 @@ async function* createAccountsByOwnerReplay(options) {
       logger.info("Replay stream waiting for first event", {
         event: "replay.stream.waiting_first_event",
         generation,
-        min_slot: minSlot?.toString()
+        min_slot: minSlot?.toString(),
+        highest_slot_seen: highestSlotSeen.toString()
       });
     }
     const newStreamFilter = buildOwnerFilterWithMinSlot(owner, dataSizes, minSlot);
@@ -1612,13 +1701,7 @@ async function* createAccountsByOwnerReplay(options) {
           }
           const event = processResponseMulti(response, assembler);
           if (event) {
-            if (event.type === "account") {
-              seenFromStream.add(event.account.addressHex);
-              if (event.account.slot > highestSlotSeen) {
-                highestSlotSeen = event.account.slot;
-              }
-            }
-            streamBuffer.push(event);
+            queueStreamEvent(event);
           }
         }
       } catch (err) {
@@ -1633,6 +1716,141 @@ async function* createAccountsByOwnerReplay(options) {
     })();
     activeStreamProcessor = newProcessor;
   };
+  const queueCatchUpAccounts = async (fromSlot, reason) => {
+    if (fromSlot <= 0n) {
+      return 0;
+    }
+    const startedAtMs = Date.now();
+    let accountsListed = 0;
+    let accountsFetched = 0;
+    let accountsQueued = 0;
+    let accountsSkipped = 0;
+    logger.info("Replay stream reconnect catch-up started", {
+      event: "replay.stream.reconnect.catch_up_started",
+      reason,
+      from_slot: fromSlot.toString()
+    });
+    const catchUpFilter = buildListAccountsOwnerFilter(owner, dataSizes, fromSlot);
+    let pageToken;
+    do {
+      if (shouldStop()) return accountsQueued;
+      const request = {
+        view: proto.AccountView.META_ONLY,
+        filter: catchUpFilter,
+        page: protobuf.create(proto.PageRequestSchema, {
+          pageSize,
+          pageToken
+        })
+      };
+      let response = null;
+      for (let attempt = 0; attempt < maxRetries; attempt++) {
+        try {
+          response = await client.listAccounts(request);
+          break;
+        } catch (err) {
+          if (shouldStop(err)) return accountsQueued;
+          if (attempt === maxRetries - 1) {
+            logger.error("Replay stream reconnect catch-up list accounts failed", {
+              event: "replay.stream.reconnect.catch_up_list_failed",
+              reason,
+              from_slot: fromSlot.toString(),
+              page_token: pageToken,
+              attempts: maxRetries,
+              error: err
+            });
+            throw err;
+          }
+          logger.warn("Replay stream reconnect catch-up list accounts retrying", {
+            event: "replay.stream.reconnect.catch_up_list_retry",
+            reason,
+            from_slot: fromSlot.toString(),
+            page_token: pageToken,
+            attempt: attempt + 1,
+            max_retries: maxRetries,
+            error: err
+          });
+          await abortableDelay(100 * (attempt + 1), signal);
+        }
+      }
+      if (!response) {
+        return accountsQueued;
+      }
+      for (const listedAccount of response.accounts) {
+        if (shouldStop()) return accountsQueued;
+        const address = listedAccount.address?.value;
+        if (!address) {
+          accountsSkipped++;
+          continue;
+        }
+        accountsListed++;
+        const addressHex = bytesToHex2(address);
+        let account = null;
+        for (let attempt = 0; attempt < maxRetries; attempt++) {
+          try {
+            account = await client.getAccount({
+              address: protobuf.create(proto.PubkeySchema, { value: address }),
+              view: proto.AccountView.FULL
+            });
+            break;
+          } catch (err) {
+            if (shouldStop(err)) return accountsQueued;
+            if (attempt === maxRetries - 1) {
+              logger.error(`[catch-up] failed to fetch account ${addressHex} after ${maxRetries} attempts`, { error: err });
+            } else {
+              await abortableDelay(100 * (attempt + 1), signal);
+            }
+          }
+        }
+        if (!account) {
+          accountsSkipped++;
+          continue;
+        }
+        accountsFetched++;
+        const state = getAccountToState(account);
+        if (!state || state.slot < fromSlot) {
+          accountsSkipped++;
+          continue;
+        }
+        if (queueReplayEvent({ type: "account", account: state })) {
+          accountsQueued++;
+        } else {
+          accountsSkipped++;
+        }
+      }
+      pageToken = response.page?.nextPageToken;
+    } while (pageToken);
+    logger.info("Replay stream reconnect catch-up completed", {
+      event: "replay.stream.reconnect.catch_up_completed",
+      reason,
+      from_slot: fromSlot.toString(),
+      duration_ms: Date.now() - startedAtMs,
+      accounts_listed: accountsListed,
+      accounts_fetched: accountsFetched,
+      accounts_queued: accountsQueued,
+      accounts_skipped: accountsSkipped
+    });
+    return accountsQueued;
+  };
+  const queueCatchUpAccountsForReconnect = async (fromSlot, reason) => {
+    const catchUpFromSlot = markCatchUpPending(fromSlot);
+    try {
+      await queueCatchUpAccounts(catchUpFromSlot, reason);
+      if (shouldStop()) return;
+      markCatchUpCompleted(catchUpFromSlot);
+      flushDeferredStreamEvents();
+    } catch (err) {
+      if (shouldStop(err)) return;
+      deferredStreamBuffer.length = 0;
+      logger.warn("Replay stream reconnect catch-up failed", {
+        event: "replay.stream.reconnect.catch_up_failed",
+        reason,
+        from_slot: catchUpFromSlot.toString(),
+        error: err
+      });
+      streamDone = true;
+      streamError = err instanceof Error ? err : new Error(String(err));
+    }
+  };
   try {
     if (shouldStop()) return;
     cleanupTimer = setInterval(() => {
@@ -1643,9 +1861,6 @@ async function* createAccountsByOwnerReplay(options) {
     const yieldStreamBuffer = function* () {
       while (streamBuffer.length > 0) {
         const event = streamBuffer.shift();
-        if (event.type === "account") {
-          seenFromStream.add(event.account.addressHex);
-        }
         yield event;
       }
     };
@@ -1706,10 +1921,7 @@ async function* createAccountsByOwnerReplay(options) {
       }
       if (account) {
         const state = getAccountToState(account);
-        if (state) {
-          if (state.slot > highestSlotSeen) {
-            highestSlotSeen = state.slot;
-          }
+        if (state && shouldEmitAccountState(state)) {
           yield { type: "account", account: state };
         }
       }
@@ -1734,32 +1946,42 @@ async function* createAccountsByOwnerReplay(options) {
           connectionTimeoutMs: retryConfig.connectionTimeoutMs
         });
         streamDone = true;
-        streamError = new Error(`Operation timed out after ${retryConfig.connectionTimeoutMs}ms`);
+        streamError = createIdleTimeoutError(retryConfig.connectionTimeoutMs);
       }
       if (streamDone) {
         if (streamError) {
           if (shouldStop(streamError)) return;
-          const backoffMs = calculateBackoff(retryAttempt, retryConfig);
+          const idleTimeout = isIdleTimeoutError(streamError);
+          const backoffMs = idleTimeout ? 0 : calculateBackoff(retryAttempt, retryConfig);
           logger.warn("Replay stream reconnect started", {
             event: "replay.stream.reconnect.started",
             reason: "stream_error",
             error: streamError.message,
             backoffMs,
             attempt: retryAttempt + 1,
+            idle_timeout: idleTimeout,
             highestSlotSeen: highestSlotSeen.toString()
           });
-          await abortableDelay(backoffMs, signal);
-          if (shouldStop()) return;
-          retryAttempt++;
+          if (backoffMs > 0) {
+            await abortableDelay(backoffMs, signal);
+            if (shouldStop()) return;
+          }
+          if (idleTimeout) {
+            retryAttempt = 0;
+          } else {
+            retryAttempt++;
+          }
+          const reconnectFromSlot = getReconnectFromSlot();
+          markCatchUpPending(reconnectFromSlot);
           const retired = retireActiveStream();
           streamDone = false;
           streamError = null;
           streamBuffer.length = 0;
           lastActivityTime = Date.now();
-          await cleanupRetiredStream(retired, "old iterator close", "old processor drain");
-          if (shouldStop()) return;
           createFreshClient();
-          createStreamProcessor("reconnect");
+          createStreamProcessor("reconnect", reconnectFromSlot > 0n ? reconnectFromSlot : void 0);
+          cleanupRetiredStreamInBackground(retired, "old iterator close", "old processor drain");
+          await queueCatchUpAccountsForReconnect(reconnectFromSlot, "stream_error");
           continue;
         } else {
           if (shouldStop()) return;
@@ -1769,15 +1991,18 @@ async function* createAccountsByOwnerReplay(options) {
             attempt: retryAttempt + 1,
             highestSlotSeen: highestSlotSeen.toString()
           });
+          retryAttempt++;
+          const reconnectFromSlot = getReconnectFromSlot();
+          markCatchUpPending(reconnectFromSlot);
           const retired = retireActiveStream();
           streamDone = false;
           streamError = null;
           streamBuffer.length = 0;
           lastActivityTime = Date.now();
-          await cleanupRetiredStream(retired, "old iterator close", "old processor drain");
-          if (shouldStop()) return;
           createFreshClient();
-          createStreamProcessor("reconnect");
+          createStreamProcessor("reconnect", reconnectFromSlot > 0n ? reconnectFromSlot : void 0);
+          cleanupRetiredStreamInBackground(retired, "old iterator close", "old processor drain");
+          await queueCatchUpAccountsForReconnect(reconnectFromSlot, "stream_ended");
           continue;
         }
       }
@@ -1792,6 +2017,9 @@ async function* createAccountsByOwnerReplay(options) {
       closeIfCloseable(client);
     }
     await cleanupRetiredStream(retired, "final iterator close", "final processor drain");
+    if (pendingCleanupTasks.size > 0) {
+      await Promise.allSettled([...pendingCleanupTasks]);
+    }
     assembler.clear();
   }
 }