@thru/replay 0.2.19 → 0.2.21

package/dist/index.d.cts

@@ -165,6 +165,11 @@ interface EventReplayOptions {
     /** Factory to create fresh clients on reconnection. Enables robust reconnection. */
     clientFactory?: () => EventSource;
     startSlot: Slot;
+    /** Last fully processed event. Allows backfill to resume within a slot. */
+    resumeAfter?: {
+        slot: Slot;
+        eventId: string;
+    };
     safetyMargin?: bigint;
     pageSize?: number;
     filter?: Filter;

package/dist/index.d.ts

@@ -165,6 +165,11 @@ interface EventReplayOptions {
     /** Factory to create fresh clients on reconnection. Enables robust reconnection. */
     clientFactory?: () => EventSource;
     startSlot: Slot;
+    /** Last fully processed event. Allows backfill to resume within a slot. */
+    resumeAfter?: {
+        slot: Slot;
+        eventId: string;
+    };
     safetyMargin?: bigint;
     pageSize?: number;
     filter?: Filter;

package/dist/index.mjs

@@ -358,6 +358,27 @@ function compareBigint(a, b) {
   if (a === b) return 0;
   return a < b ? -1 : 1;
 }
+function isNonDecreasing(items, extractSlot) {
+  for (let idx = 1; idx < items.length; idx += 1) {
+    if (extractSlot(items[idx]) < extractSlot(items[idx - 1])) return false;
+  }
+  return true;
+}
+function assertBackfillPageOrder(previousPage, currentPage, extractSlot) {
+  if (!isNonDecreasing(currentPage, extractSlot)) {
+    throw new Error(
+      "backfill source returned a page that is not ordered by ascending slot"
+    );
+  }
+  if (!previousPage?.length || !currentPage.length) return;
+  const previousMaxSlot = extractSlot(previousPage[previousPage.length - 1]);
+  const currentMinSlot = extractSlot(currentPage[0]);
+  if (currentMinSlot < previousMaxSlot) {
+    throw new Error(
+      `backfill source returned pages out of ascending slot order: page minimum slot ${currentMinSlot} is before previous page maximum slot ${previousMaxSlot}`
+    );
+  }
+}
 var ReplayStream = class {
   config;
   logger;
@@ -418,6 +439,29 @@ var ReplayStream = class {
     this.logger.info(
       `replay entering BACKFILLING state (startSlot=${startSlot}, safetyMargin=${safetyMargin})`
     );
+    let pendingOrderedPage = null;
+    const emitBackfillItems = async function* (self, items) {
+      for (const item of items) {
+        const slot = extractSlot(item);
+        const key = keyOf(item);
+        if (slot < startSlot) continue;
+        if (seenItem(slot, key)) {
+          self.metrics.discardedDuplicates += 1;
+          continue;
+        }
+        currentSlot = slot;
+        recordEmission(slot, key);
+        self.metrics.emittedBackfill += 1;
+        yield item;
+      }
+    };
+    const flushPendingBackfill = async function* (self) {
+      if (!pendingOrderedPage) return;
+      for await (const item of emitBackfillItems(self, pendingOrderedPage)) {
+        yield item;
+      }
+      pendingOrderedPage = null;
+    };
     let emptyPageRetries = 0;
     const MAX_EMPTY_PAGE_RETRIES = 10;
     while (!backfillDone) {
@@ -428,6 +472,9 @@ var ReplayStream = class {
           this.logger.error(
             `backfill returned ${MAX_EMPTY_PAGE_RETRIES} consecutive empty pages; treating as done`
           );
+          for await (const item of flushPendingBackfill(this)) {
+            yield item;
+          }
           break;
         }
         const backoffMs = calculateBackoff(emptyPageRetries - 1, DEFAULT_RETRY_CONFIG);
@@ -438,21 +485,18 @@ var ReplayStream = class {
         continue;
       }
       emptyPageRetries = 0;
-      const sorted = [...page.items].sort(
-        (a, b) => compareBigint(extractSlot(a), extractSlot(b))
-      );
-      for (const item of sorted) {
-        const slot = extractSlot(item);
-        const key = keyOf(item);
-        if (slot < startSlot) continue;
-        if (seenItem(slot, key)) {
-          this.metrics.discardedDuplicates += 1;
-          continue;
+      assertBackfillPageOrder(pendingOrderedPage, page.items, extractSlot);
+      if (pendingOrderedPage !== null) {
+        for await (const item of flushPendingBackfill(this)) {
+          yield item;
+        }
+      }
+      pendingOrderedPage = [...page.items];
+      const reachedEnd = page.done || page.cursor === void 0;
+      if (reachedEnd) {
+        for await (const item of flushPendingBackfill(this)) {
+          yield item;
         }
-        currentSlot = slot;
-        recordEmission(slot, key);
-        this.metrics.emittedBackfill += 1;
-        yield item;
       }
       const duplicatesTrimmed = livePump.discardBufferedUpTo(currentSlot);
       this.metrics.discardedDuplicates += duplicatesTrimmed;
@@ -461,13 +505,16 @@ var ReplayStream = class {
       if (maxStreamSlot !== null) {
         const catchUpSlot = maxStreamSlot > safetyMargin ? maxStreamSlot - safetyMargin : 0n;
         if (currentSlot >= catchUpSlot) {
+          for await (const item of flushPendingBackfill(this)) {
+            yield item;
+          }
           this.logger.info(
             `replay reached SWITCHING threshold (currentSlot=${currentSlot}, maxStreamSlot=${maxStreamSlot}, catchUpSlot=${catchUpSlot})`
           );
           backfillDone = true;
         }
       }
-      if (page.done || cursor === void 0) backfillDone = true;
+      if (reachedEnd) backfillDone = true;
     }
     this.logger.info(`replay entering SWITCHING state (currentSlot=${currentSlot})`);
     const { drained, discarded } = livePump.enableStreaming(currentSlot);
@@ -796,7 +843,7 @@ function createEventReplay(options) {
       orderBy: PAGE_ORDER_ASC3,
       pageToken: cursor
     });
-    const baseFilter = slotLiteralFilter("event.slot", startSlot);
+    const baseFilter = eventBackfillFilter(startSlot, options.resumeAfter);
     const mergedFilter = combineFilters(baseFilter, options.filter);
     const response = await client.listEvents(
       create(ListEventsRequestSchema, {
@@ -807,13 +854,19 @@ function createEventReplay(options) {
     return backfillPage(response.events, response.page);
   };
   const createSubscribeLive = (client) => (startSlot) => {
-    const mergedFilter = combineFilters(slotLiteralFilter("event.slot", startSlot), options.filter);
+    const mergedFilter = combineFilters(
+      slotLiteralFilter("event.slot", startSlot),
+      options.filter
+    );
     const request = create(StreamEventsRequestSchema, {
       filter: mergedFilter
     });
     return mapAsyncIterable(
       client.streamEvents(request),
-      (resp) => streamResponseToEvent(resp)
+      (resp) => {
+        const event = streamResponseToEvent(resp);
+        return shouldEmitLiveEvent(event, startSlot, options.resumeAfter) ? event : null;
+      }
     );
   };
   const onReconnect = options.clientFactory ? () => {
@@ -851,6 +904,52 @@ function eventKey(event) {
   const slotPart = event.slot?.toString() ?? "0";
   return `${slotPart}:${event.callIdx ?? 0}`;
 }
+function eventBackfillFilter(startSlot, resumeAfter) {
+  const boundary = parseEventId(resumeAfter);
+  if (!boundary || startSlot > boundary.slot) {
+    return slotLiteralFilter("event.slot", startSlot);
+  }
+  return create(FilterSchema, {
+    expression: `event.slot > uint(${boundary.slot.toString()}) || (event.slot == uint(${boundary.slot.toString()}) && (event.block_offset > uint(${boundary.blockOffset.toString()}) || (event.block_offset == uint(${boundary.blockOffset.toString()}) && event.call_idx >= uint(${boundary.callIdx.toString()}))))`
+  });
+}
+function parseEventId(resumeAfter) {
+  if (!resumeAfter?.eventId) return null;
+  return parseCanonicalEventId(resumeAfter.eventId, resumeAfter.slot);
+}
+function parseCanonicalEventId(eventId, expectedSlot) {
+  if (!eventId) return null;
+  const parts = eventId.split(":");
+  if (parts.length === 5) {
+    const [, slotPart, blockOffsetPart, callIdxPart, eventIdxPart] = parts;
+    const slot = BigInt(slotPart);
+    if (slot !== expectedSlot) return null;
+    return {
+      slot,
+      blockOffset: BigInt(blockOffsetPart),
+      callIdx: BigInt(callIdxPart),
+      eventIdx: BigInt(eventIdxPart)
+    };
+  }
+  return null;
+}
+function isAfterBoundary(event, boundary) {
+  if (event.slot !== boundary.slot) return event.slot > boundary.slot;
+  if (event.blockOffset !== boundary.blockOffset) {
+    return event.blockOffset > boundary.blockOffset;
+  }
+  if (event.callIdx !== boundary.callIdx) return event.callIdx > boundary.callIdx;
+  return event.eventIdx > boundary.eventIdx;
+}
+function shouldEmitLiveEvent(event, startSlot, resumeAfter) {
+  const boundary = parseEventId(resumeAfter);
+  if (!boundary || startSlot > boundary.slot) return true;
+  const eventSlot = event.slot ?? 0n;
+  if (eventSlot > boundary.slot) return true;
+  if (eventSlot < boundary.slot) return false;
+  const eventPosition = parseCanonicalEventId(event.eventId, boundary.slot);
+  return eventPosition ? isAfterBoundary(eventPosition, boundary) : false;
+}
 
 // src/page-assembler.ts
 var PAGE_SIZE = 4096;