@thru/replay 0.1.36 → 0.2.0

package/dist/index.cjs

@@ -17,6 +17,12 @@ var ChainClient = class {
   query;
   streaming;
   callOptions;
+  getAccount(request) {
+    return this.query.getAccount(protobuf.create(proto.GetAccountRequestSchema, request), this.callOptions);
+  }
+  listAccounts(request) {
+    return this.query.listAccounts(protobuf.create(proto.ListAccountsRequestSchema, request), this.callOptions);
+  }
   listBlocks(request) {
     return this.query.listBlocks(protobuf.create(proto.ListBlocksRequestSchema, request), this.callOptions);
   }
@@ -303,18 +309,52 @@ var LivePump = class {
   }
 };
 
+// src/retry.ts
+var DEFAULT_RETRY_CONFIG = {
+  initialDelayMs: 1e3,
+  maxDelayMs: 3e4,
+  connectionTimeoutMs: 3e4
+};
+function calculateBackoff(attempt, config) {
+  const delay2 = config.initialDelayMs * Math.pow(2, attempt);
+  return Math.min(delay2, config.maxDelayMs);
+}
+function withTimeout(promise, timeoutMs) {
+  return new Promise((resolve, reject) => {
+    const timer = setTimeout(() => {
+      reject(new TimeoutError(`Operation timed out after ${timeoutMs}ms`));
+    }, timeoutMs);
+    promise.then((value) => {
+      clearTimeout(timer);
+      resolve(value);
+    }).catch((err) => {
+      clearTimeout(timer);
+      reject(err);
+    });
+  });
+}
+var TimeoutError = class extends Error {
+  constructor(message) {
+    super(message);
+    this.name = "TimeoutError";
+  }
+};
+function delay(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
 // src/replay-stream.ts
 var DEFAULT_METRICS = {
   bufferedItems: 0,
   emittedBackfill: 0,
   emittedLive: 0,
+  emittedReconnect: 0,
   discardedDuplicates: 0
 };
 function compareBigint(a, b) {
   if (a === b) return 0;
   return a < b ? -1 : 1;
 }
-var RETRY_DELAY_MS = 1e3;
 var ReplayStream = class {
   config;
   logger;
@@ -337,12 +377,15 @@ var ReplayStream = class {
       extractSlot,
       extractKey,
       safetyMargin,
-      resubscribeOnEnd
+      resubscribeOnEnd,
+      onReconnect
     } = this.config;
     const shouldResubscribeOnEnd = resubscribeOnEnd ?? true;
     const keyOf = extractKey ?? ((item) => extractSlot(item).toString());
+    let currentSubscribeLive = subscribeLive;
+    let currentFetchBackfill = fetchBackfill;
     const createLivePump = (slot, startStreaming = false, emitFloor) => new LivePump({
-      source: subscribeLive(slot),
+      source: currentSubscribeLive(slot),
       slotOf: extractSlot,
       keyOf,
       logger: this.logger,
@@ -428,9 +471,15 @@ var ReplayStream = class {
     }
     if (!drained.length) livePump.updateEmitFloor(currentSlot);
     this.logger.info("replay entering STREAMING state");
+    const retryConfig = DEFAULT_RETRY_CONFIG;
+    let retryAttempt = 0;
     while (true) {
       try {
-        const next = await livePump.next();
+        const next = await withTimeout(
+          livePump.next(),
+          retryConfig.connectionTimeoutMs
+        );
+        retryAttempt = 0;
         if (next.done) {
           if (!shouldResubscribeOnEnd) break;
           throw new Error("stream ended");
@@ -448,20 +497,91 @@ var ReplayStream = class {
         livePump.updateEmitFloor(currentSlot);
       } catch (err) {
         const errMsg = err instanceof Error ? err.message : String(err);
+        const backoffMs = calculateBackoff(retryAttempt, retryConfig);
         this.logger.warn(
-          `live stream disconnected (${errMsg}); reconnecting in ${RETRY_DELAY_MS}ms from slot ${currentSlot}`
+          `live stream disconnected (${errMsg}); reconnecting in ${backoffMs}ms from slot ${currentSlot} (attempt ${retryAttempt + 1})`
         );
-        await delay(RETRY_DELAY_MS);
+        await delay(backoffMs);
         await safeClose(livePump);
+        retryAttempt++;
+        if (onReconnect) {
+          try {
+            const fresh = onReconnect();
+            currentSubscribeLive = fresh.subscribeLive;
+            if (fresh.fetchBackfill) {
+              currentFetchBackfill = fresh.fetchBackfill;
+            }
+            this.logger.info("created fresh client for reconnection");
+          } catch (factoryErr) {
+            this.logger.error(
+              `failed to create fresh client: ${factoryErr instanceof Error ? factoryErr.message : String(factoryErr)}; using existing`
+            );
+          }
+        }
+        if (onReconnect && currentSlot > 0n) {
+          for await (const item of this.miniBackfill(
+            currentSlot,
+            currentFetchBackfill,
+            extractSlot,
+            keyOf,
+            seenItem,
+            recordEmission
+          )) {
+            const itemSlot = extractSlot(item);
+            if (itemSlot > currentSlot) {
+              currentSlot = itemSlot;
+            }
+            yield item;
+          }
+        }
         const resumeSlot = currentSlot > 0n ? currentSlot : 0n;
         livePump = createLivePump(resumeSlot, true, currentSlot);
       }
     }
   }
+  /**
+   * Perform mini-backfill from lastProcessedSlot to catch up after reconnection.
+   * Ensures no data gaps from events that occurred during disconnection.
+   */
+  async *miniBackfill(fromSlot, fetchBackfill, extractSlot, keyOf, seenItem, recordEmission) {
+    this.logger.info(`mini-backfill starting from slot ${fromSlot}`);
+    const MINI_BACKFILL_TIMEOUT = 3e4;
+    const startTime = Date.now();
+    let cursor;
+    let itemsYielded = 0;
+    try {
+      while (true) {
+        if (Date.now() - startTime > MINI_BACKFILL_TIMEOUT) {
+          this.logger.warn(`mini-backfill timed out after ${MINI_BACKFILL_TIMEOUT}ms`);
+          break;
+        }
+        const page = await fetchBackfill({ startSlot: fromSlot, cursor });
+        const sorted = [...page.items].sort(
+          (a, b) => compareBigint(extractSlot(a), extractSlot(b))
+        );
+        for (const item of sorted) {
+          const slot = extractSlot(item);
+          const key = keyOf(item);
+          if (seenItem(slot, key)) {
+            this.metrics.discardedDuplicates += 1;
+            continue;
+          }
+          recordEmission(slot, key);
+          itemsYielded++;
+          this.metrics.emittedReconnect += 1;
+          yield item;
+        }
+        cursor = page.cursor;
+        if (page.done || cursor === void 0) break;
+      }
+      this.logger.info(`mini-backfill complete: ${itemsYielded} items yielded`);
+    } catch (err) {
+      this.logger.warn(
+        `mini-backfill failed: ${err instanceof Error ? err.message : String(err)}; proceeding with live stream`
+      );
+    }
+  }
 };
-function delay(ms) {
-  return new Promise((resolve) => setTimeout(resolve, ms));
-}
 async function safeClose(pump) {
   try {
     await pump.close();
@@ -623,12 +743,26 @@ function bytesToHex(bytes) {
   for (const byte of bytes) hex += byte.toString(16).padStart(2, "0");
   return hex;
 }
+
+// src/types.ts
+function resolveClient(opts, optionsName) {
+  if (opts.clientFactory) {
+    return opts.clientFactory();
+  }
+  if (!opts.client) {
+    throw new Error(`${optionsName} requires either client or clientFactory`);
+  }
+  return opts.client;
+}
+
+// src/replay/event-replay.ts
 var DEFAULT_PAGE_SIZE3 = 512;
 var DEFAULT_SAFETY_MARGIN3 = 64n;
 var PAGE_ORDER_ASC3 = "slot asc";
 function createEventReplay(options) {
   const safetyMargin = options.safetyMargin ?? DEFAULT_SAFETY_MARGIN3;
-  const fetchBackfill = async ({
+  let currentClient = resolveClient(options, "EventReplayOptions");
+  const createFetchBackfill = (client) => async ({
     startSlot,
     cursor
   }) => {
@@ -639,7 +773,7 @@ function createEventReplay(options) {
     });
     const baseFilter = slotLiteralFilter("event.slot", startSlot);
     const mergedFilter = combineFilters(baseFilter, options.filter);
-    const response = await options.client.listEvents(
+    const response = await client.listEvents(
       protobuf.create(proto.ListEventsRequestSchema, {
         filter: mergedFilter,
         page
@@ -647,25 +781,33 @@ function createEventReplay(options) {
     );
     return backfillPage(response.events, response.page);
   };
-  const subscribeLive = (startSlot) => {
+  const createSubscribeLive = (client) => (startSlot) => {
     const mergedFilter = combineFilters(slotLiteralFilter("event.slot", startSlot), options.filter);
     const request = protobuf.create(proto.StreamEventsRequestSchema, {
       filter: mergedFilter
     });
     return mapAsyncIterable(
-      options.client.streamEvents(request),
+      client.streamEvents(request),
       (resp) => streamResponseToEvent(resp)
     );
   };
+  const onReconnect = options.clientFactory ? () => {
+    currentClient = options.clientFactory();
+    return {
+      subscribeLive: createSubscribeLive(currentClient),
+      fetchBackfill: createFetchBackfill(currentClient)
+    };
+  } : void 0;
   return new ReplayStream({
     startSlot: options.startSlot,
     safetyMargin,
-    fetchBackfill,
-    subscribeLive,
+    fetchBackfill: createFetchBackfill(currentClient),
+    subscribeLive: createSubscribeLive(currentClient),
     extractSlot: (event) => event.slot ?? 0n,
     extractKey: eventKey,
     logger: options.logger,
-    resubscribeOnEnd: options.resubscribeOnEnd
+    resubscribeOnEnd: options.resubscribeOnEnd,
+    onReconnect
   });
 }
 function streamResponseToEvent(resp) {
@@ -859,7 +1001,7 @@ function snapshotToState(account) {
     meta: account.meta,
     data: account.data?.data ?? new Uint8Array(0),
     isDelete: account.meta.flags?.isDeleted ?? false,
-    source: "snapshot"
+    source: "stream"
   };
 }
 function assembledToState(assembled) {
@@ -871,9 +1013,248 @@ function assembledToState(assembled) {
     meta: assembled.meta,
     data: assembled.data,
     isDelete: assembled.isDelete,
-    source: "update"
+    source: "stream"
   };
 }
+function getAccountToState(account) {
+  if (!account.address?.value || !account.meta) {
+    return null;
+  }
+  return {
+    address: account.address.value,
+    addressHex: bytesToHex2(account.address.value),
+    slot: account.meta.lastUpdatedSlot ?? account.versionContext?.slot ?? 0n,
+    seq: BigInt(account.meta.seq.toString()),
+    meta: account.meta,
+    data: account.data?.data ?? new Uint8Array(0),
+    isDelete: account.meta.flags?.isDeleted ?? false,
+    source: "backfill"
+  };
+}
+function buildListAccountsOwnerFilter(owner, dataSizes, minUpdatedSlot) {
+  let expression = "account.meta.owner.value == params.owner_bytes";
+  if (dataSizes && dataSizes.length > 0) {
+    const sizeConditions = dataSizes.map((size) => `account.meta.data_size == uint(${size})`).join(" || ");
+    expression = `(${expression}) && (${sizeConditions})`;
+  }
+  if (minUpdatedSlot !== void 0 && minUpdatedSlot > 0n) {
+    expression = `(${expression}) && account.meta.last_updated_slot >= params.min_updated_slot`;
+  }
+  const params = {
+    owner_bytes: protobuf.create(proto.FilterParamValueSchema, { kind: { case: "bytesValue", value: new Uint8Array(owner) } })
+  };
+  if (minUpdatedSlot !== void 0 && minUpdatedSlot > 0n) {
+    params["min_updated_slot"] = protobuf.create(proto.FilterParamValueSchema, { kind: { case: "uintValue", value: minUpdatedSlot } });
+  }
+  return protobuf.create(proto.FilterSchema, { expression, params });
+}
+async function* createAccountsByOwnerReplay(options) {
+  const {
+    owner,
+    view = proto.AccountView.FULL,
+    dataSizes,
+    minUpdatedSlot,
+    pageSize = 100,
+    maxRetries = 3,
+    pageAssemblerOptions,
+    cleanupInterval = 1e4,
+    onBackfillComplete,
+    clientFactory,
+    logger = NOOP_LOGGER
+  } = options;
+  let client = resolveClient(options, "AccountsByOwnerReplayOptions");
+  const seenFromStream = /* @__PURE__ */ new Set();
+  const fetchQueue = [];
+  let highestSlotSeen = minUpdatedSlot ?? 0n;
+  const assembler = new PageAssembler(pageAssemblerOptions);
+  let cleanupTimer = null;
+  const streamBuffer = [];
+  let streamDone = false;
+  let streamError = null;
+  try {
+    cleanupTimer = setInterval(() => {
+      assembler.cleanup();
+    }, cleanupInterval);
+    const streamFilter = buildOwnerFilterWithMinSlot(owner, dataSizes, minUpdatedSlot);
+    const stream = client.streamAccountUpdates({ view, filter: streamFilter });
+    const streamProcessor = (async () => {
+      try {
+        for await (const response of stream) {
+          const event = processResponseMulti(response, assembler);
+          if (event) {
+            if (event.type === "account") {
+              seenFromStream.add(event.account.addressHex);
+              if (event.account.slot > highestSlotSeen) {
+                highestSlotSeen = event.account.slot;
+              }
+            }
+            streamBuffer.push(event);
+          }
+        }
+      } catch (err) {
+        streamError = err;
+      } finally {
+        streamDone = true;
+      }
+    })();
+    const yieldStreamBuffer = function* () {
+      while (streamBuffer.length > 0) {
+        const event = streamBuffer.shift();
+        if (event.type === "account") {
+          seenFromStream.add(event.account.addressHex);
+        }
+        yield event;
+      }
+    };
+    const backfillFilter = buildListAccountsOwnerFilter(owner, dataSizes, minUpdatedSlot);
+    let pageToken;
+    do {
+      const request = {
+        view: proto.AccountView.META_ONLY,
+        // Address + metadata only, no data
+        filter: backfillFilter,
+        page: protobuf.create(proto.PageRequestSchema, {
+          pageSize,
+          pageToken
+        })
+      };
+      const response = await client.listAccounts(request);
+      for (const account of response.accounts) {
+        if (account.address?.value) {
+          fetchQueue.push(account.address.value);
+        }
+      }
+      pageToken = response.page?.nextPageToken;
+      yield* yieldStreamBuffer();
+    } while (pageToken);
+    for (const address of fetchQueue) {
+      const addressHex = bytesToHex2(address);
+      if (seenFromStream.has(addressHex)) {
+        continue;
+      }
+      yield* yieldStreamBuffer();
+      if (seenFromStream.has(addressHex)) {
+        continue;
+      }
+      let account = null;
+      for (let attempt = 0; attempt < maxRetries; attempt++) {
+        try {
+          account = await client.getAccount({
+            address: protobuf.create(proto.PubkeySchema, { value: address }),
+            view: proto.AccountView.FULL
+          });
+          break;
+        } catch (err) {
+          if (attempt === maxRetries - 1) {
+            logger.error(`[backfill] failed to fetch account ${addressHex} after ${maxRetries} attempts`, { error: err });
+          } else {
+            await new Promise((resolve) => setTimeout(resolve, 100 * (attempt + 1)));
+          }
+        }
+      }
+      if (account) {
+        const state = getAccountToState(account);
+        if (state) {
+          if (state.slot > highestSlotSeen) {
+            highestSlotSeen = state.slot;
+          }
+          yield { type: "account", account: state };
+        }
+      }
+    }
+    if (onBackfillComplete) {
+      onBackfillComplete(highestSlotSeen);
+    }
+    const retryConfig = DEFAULT_RETRY_CONFIG;
+    let retryAttempt = 0;
+    let currentStream = stream;
+    let currentStreamProcessor = streamProcessor;
+    const createStreamProcessor = () => {
+      if (clientFactory) {
+        try {
+          client = clientFactory();
+          logger.info("[account-stream] created fresh client for reconnection");
+        } catch (err) {
+          logger.error("[account-stream] failed to create fresh client", { error: err });
+        }
+      }
+      const newStreamFilter = buildOwnerFilterWithMinSlot(owner, dataSizes, highestSlotSeen > 0n ? highestSlotSeen : minUpdatedSlot);
+      const newStream = client.streamAccountUpdates({ view, filter: newStreamFilter });
+      const newProcessor = (async () => {
+        try {
+          for await (const response of newStream) {
+            retryAttempt = 0;
+            const event = processResponseMulti(response, assembler);
+            if (event) {
+              if (event.type === "account") {
+                seenFromStream.add(event.account.addressHex);
+                if (event.account.slot > highestSlotSeen) {
+                  highestSlotSeen = event.account.slot;
+                }
+              }
+              streamBuffer.push(event);
+            }
+          }
+        } catch (err) {
+          streamError = err;
+        } finally {
+          streamDone = true;
+        }
+      })();
+      return { stream: newStream, processor: newProcessor };
+    };
+    while (true) {
+      yield* yieldStreamBuffer();
+      if (streamDone) {
+        if (streamError) {
+          const backoffMs = calculateBackoff(retryAttempt, retryConfig);
+          logger.warn(
+            `[account-stream] disconnected (${streamError.message}); reconnecting in ${backoffMs}ms (attempt ${retryAttempt + 1})`
+          );
+          await delay(backoffMs);
+          retryAttempt++;
+          streamDone = false;
+          streamError = null;
+          streamBuffer.length = 0;
+          const { stream: newStream, processor: newProcessor } = createStreamProcessor();
+          currentStream = newStream;
+          currentStreamProcessor = newProcessor;
+          continue;
+        } else {
+          logger.warn("[account-stream] stream ended unexpectedly; reconnecting...");
+          streamDone = false;
+          const { stream: newStream, processor: newProcessor } = createStreamProcessor();
+          currentStream = newStream;
+          currentStreamProcessor = newProcessor;
+          continue;
+        }
+      }
+      await delay(10);
+    }
+  } finally {
+    if (cleanupTimer) {
+      clearInterval(cleanupTimer);
+    }
+    assembler.clear();
+  }
+}
+function buildOwnerFilterWithMinSlot(owner, dataSizes, minSlot) {
+  let expression = "(has(snapshot.meta.owner) && snapshot.meta.owner.value == params.owner) || (has(account_update.meta.owner) && account_update.meta.owner.value == params.owner)";
+  if (dataSizes && dataSizes.length > 0) {
+    const sizeConditions = dataSizes.map((size) => `snapshot.meta.data_size == uint(${size}) || account_update.meta.data_size == uint(${size})`).join(" || ");
+    expression = `(${expression}) && (${sizeConditions})`;
+  }
+  if (minSlot !== void 0 && minSlot > 0n) {
+    expression = `(${expression}) && (has(snapshot.address) || (has(account_update.slot) && account_update.slot >= params.min_slot))`;
+  }
+  const params = {
+    owner: protobuf.create(proto.FilterParamValueSchema, { kind: { case: "bytesValue", value: new Uint8Array(owner) } })
+  };
+  if (minSlot !== void 0 && minSlot > 0n) {
+    params["min_slot"] = protobuf.create(proto.FilterParamValueSchema, { kind: { case: "uintValue", value: minSlot } });
+  }
+  return protobuf.create(proto.FilterSchema, { expression, params });
+}
 async function* createAccountReplay(options) {
   const {
     client,
@@ -975,60 +1356,6 @@ function processResponseMulti(response, assembler) {
       return null;
   }
 }
-function buildOwnerFilter(owner, dataSizes, additionalFilter) {
-  let expression = "(has(snapshot.meta.owner) && snapshot.meta.owner.value == params.owner) || (has(account_update.meta.owner) && account_update.meta.owner.value == params.owner)";
-  if (dataSizes && dataSizes.length > 0) {
-    const sizeConditions = dataSizes.map((size) => `snapshot.meta.data_size == uint(${size}) || account_update.meta.data_size == uint(${size})`).join(" || ");
-    expression = `(${expression}) && (${sizeConditions})`;
-  }
-  if (additionalFilter?.expression) {
-    expression = `(${expression}) && (${additionalFilter.expression})`;
-  }
-  const params = {
-    owner: protobuf.create(proto.FilterParamValueSchema, { kind: { case: "bytesValue", value: new Uint8Array(owner) } })
-  };
-  if (additionalFilter?.params) {
-    for (const [key, value] of Object.entries(additionalFilter.params)) {
-      params[key] = protobuf.create(proto.FilterParamValueSchema, value);
-    }
-  }
-  return protobuf.create(proto.FilterSchema, { expression, params });
-}
-async function* createAccountsByOwnerReplay(options) {
-  const {
-    client,
-    owner,
-    view = proto.AccountView.FULL,
-    dataSizes,
-    filter,
-    pageAssemblerOptions,
-    cleanupInterval = 1e4
-  } = options;
-  const assembler = new PageAssembler(pageAssemblerOptions);
-  let cleanupTimer = null;
-  try {
-    cleanupTimer = setInterval(() => {
-      assembler.cleanup();
-    }, cleanupInterval);
-    const ownerFilter = buildOwnerFilter(owner, dataSizes, filter);
-    const request = {
-      view,
-      filter: ownerFilter
-    };
-    const stream = client.streamAccountUpdates(request);
-    for await (const response of stream) {
-      const event = processResponseMulti(response, assembler);
-      if (event) {
-        yield event;
-      }
-    }
-  } finally {
-    if (cleanupTimer) {
-      clearInterval(cleanupTimer);
-    }
-    assembler.clear();
-  }
-}
 var AccountSeqTracker = class {
   seqs = /* @__PURE__ */ new Map();
   /**
@@ -1181,16 +1508,22 @@ Object.defineProperty(exports, "FilterSchema", {
 exports.AccountSeqTracker = AccountSeqTracker;
 exports.ChainClient = ChainClient;
 exports.ConsoleSink = ConsoleSink;
+exports.DEFAULT_RETRY_CONFIG = DEFAULT_RETRY_CONFIG;
 exports.MultiAccountReplay = MultiAccountReplay;
 exports.NOOP_LOGGER = NOOP_LOGGER;
 exports.PAGE_SIZE = PAGE_SIZE;
 exports.PageAssembler = PageAssembler;
 exports.ReplayStream = ReplayStream;
+exports.TimeoutError = TimeoutError;
+exports.calculateBackoff = calculateBackoff;
 exports.createAccountReplay = createAccountReplay;
 exports.createAccountsByOwnerReplay = createAccountsByOwnerReplay;
 exports.createBlockReplay = createBlockReplay;
 exports.createConsoleLogger = createConsoleLogger;
 exports.createEventReplay = createEventReplay;
 exports.createTransactionReplay = createTransactionReplay;
+exports.delay = delay;
+exports.resolveClient = resolveClient;
+exports.withTimeout = withTimeout;
 //# sourceMappingURL=index.cjs.map
 //# sourceMappingURL=index.cjs.map