@thru/replay 0.2.20 → 0.2.21

package/dist/index.cjs

@@ -359,6 +359,27 @@ function compareBigint(a, b) {
   if (a === b) return 0;
   return a < b ? -1 : 1;
 }
+function isNonDecreasing(items, extractSlot) {
+  for (let idx = 1; idx < items.length; idx += 1) {
+    if (extractSlot(items[idx]) < extractSlot(items[idx - 1])) return false;
+  }
+  return true;
+}
+function assertBackfillPageOrder(previousPage, currentPage, extractSlot) {
+  if (!isNonDecreasing(currentPage, extractSlot)) {
+    throw new Error(
+      "backfill source returned a page that is not ordered by ascending slot"
+    );
+  }
+  if (!previousPage?.length || !currentPage.length) return;
+  const previousMaxSlot = extractSlot(previousPage[previousPage.length - 1]);
+  const currentMinSlot = extractSlot(currentPage[0]);
+  if (currentMinSlot < previousMaxSlot) {
+    throw new Error(
+      `backfill source returned pages out of ascending slot order: page minimum slot ${currentMinSlot} is before previous page maximum slot ${previousMaxSlot}`
+    );
+  }
+}
 var ReplayStream = class {
   config;
   logger;
@@ -419,6 +440,29 @@ var ReplayStream = class {
     this.logger.info(
       `replay entering BACKFILLING state (startSlot=${startSlot}, safetyMargin=${safetyMargin})`
     );
+    let pendingOrderedPage = null;
+    const emitBackfillItems = async function* (self, items) {
+      for (const item of items) {
+        const slot = extractSlot(item);
+        const key = keyOf(item);
+        if (slot < startSlot) continue;
+        if (seenItem(slot, key)) {
+          self.metrics.discardedDuplicates += 1;
+          continue;
+        }
+        currentSlot = slot;
+        recordEmission(slot, key);
+        self.metrics.emittedBackfill += 1;
+        yield item;
+      }
+    };
+    const flushPendingBackfill = async function* (self) {
+      if (!pendingOrderedPage) return;
+      for await (const item of emitBackfillItems(self, pendingOrderedPage)) {
+        yield item;
+      }
+      pendingOrderedPage = null;
+    };
     let emptyPageRetries = 0;
     const MAX_EMPTY_PAGE_RETRIES = 10;
     while (!backfillDone) {
@@ -429,6 +473,9 @@ var ReplayStream = class {
           this.logger.error(
             `backfill returned ${MAX_EMPTY_PAGE_RETRIES} consecutive empty pages; treating as done`
           );
+          for await (const item of flushPendingBackfill(this)) {
+            yield item;
+          }
           break;
         }
         const backoffMs = calculateBackoff(emptyPageRetries - 1, DEFAULT_RETRY_CONFIG);
@@ -439,21 +486,18 @@ var ReplayStream = class {
         continue;
       }
       emptyPageRetries = 0;
-      const sorted = [...page.items].sort(
-        (a, b) => compareBigint(extractSlot(a), extractSlot(b))
-      );
-      for (const item of sorted) {
-        const slot = extractSlot(item);
-        const key = keyOf(item);
-        if (slot < startSlot) continue;
-        if (seenItem(slot, key)) {
-          this.metrics.discardedDuplicates += 1;
-          continue;
+      assertBackfillPageOrder(pendingOrderedPage, page.items, extractSlot);
+      if (pendingOrderedPage !== null) {
+        for await (const item of flushPendingBackfill(this)) {
+          yield item;
+        }
+      }
+      pendingOrderedPage = [...page.items];
+      const reachedEnd = page.done || page.cursor === void 0;
+      if (reachedEnd) {
+        for await (const item of flushPendingBackfill(this)) {
+          yield item;
         }
-        currentSlot = slot;
-        recordEmission(slot, key);
-        this.metrics.emittedBackfill += 1;
-        yield item;
       }
       const duplicatesTrimmed = livePump.discardBufferedUpTo(currentSlot);
       this.metrics.discardedDuplicates += duplicatesTrimmed;
@@ -462,13 +506,16 @@ var ReplayStream = class {
       if (maxStreamSlot !== null) {
         const catchUpSlot = maxStreamSlot > safetyMargin ? maxStreamSlot - safetyMargin : 0n;
         if (currentSlot >= catchUpSlot) {
+          for await (const item of flushPendingBackfill(this)) {
+            yield item;
+          }
           this.logger.info(
             `replay reached SWITCHING threshold (currentSlot=${currentSlot}, maxStreamSlot=${maxStreamSlot}, catchUpSlot=${catchUpSlot})`
           );
           backfillDone = true;
         }
       }
-      if (page.done || cursor === void 0) backfillDone = true;
+      if (reachedEnd) backfillDone = true;
     }
     this.logger.info(`replay entering SWITCHING state (currentSlot=${currentSlot})`);
     const { drained, discarded } = livePump.enableStreaming(currentSlot);
@@ -864,7 +911,7 @@ function eventBackfillFilter(startSlot, resumeAfter) {
     return slotLiteralFilter("event.slot", startSlot);
   }
   return protobuf.create(proto.FilterSchema, {
-    expression: `event.slot > uint(${boundary.slot.toString()}) || (event.slot == uint(${boundary.slot.toString()}) && (event.block_offset > uint(${boundary.blockOffset.toString()}) || (event.block_offset == uint(${boundary.blockOffset.toString()}) && event.call_idx > uint(${boundary.callIdx.toString()}))))`
+    expression: `event.slot > uint(${boundary.slot.toString()}) || (event.slot == uint(${boundary.slot.toString()}) && (event.block_offset > uint(${boundary.blockOffset.toString()}) || (event.block_offset == uint(${boundary.blockOffset.toString()}) && event.call_idx >= uint(${boundary.callIdx.toString()}))))`
   });
 }
 function parseEventId(resumeAfter) {
@@ -873,23 +920,27 @@ function parseEventId(resumeAfter) {
 }
 function parseCanonicalEventId(eventId, expectedSlot) {
   if (!eventId) return null;
-  const match = /^ts(\d+)_(\d+)_(\d+)$/.exec(eventId);
-  if (!match) return null;
-  const [slotPart, blockOffsetPart, callIdxPart] = match.slice(1);
-  const slot = BigInt(slotPart);
-  if (slot !== expectedSlot) return null;
-  return {
-    slot,
-    blockOffset: BigInt(blockOffsetPart),
-    callIdx: BigInt(callIdxPart)
-  };
+  const parts = eventId.split(":");
+  if (parts.length === 5) {
+    const [, slotPart, blockOffsetPart, callIdxPart, eventIdxPart] = parts;
+    const slot = BigInt(slotPart);
+    if (slot !== expectedSlot) return null;
+    return {
+      slot,
+      blockOffset: BigInt(blockOffsetPart),
+      callIdx: BigInt(callIdxPart),
+      eventIdx: BigInt(eventIdxPart)
+    };
+  }
+  return null;
 }
 function isAfterBoundary(event, boundary) {
   if (event.slot !== boundary.slot) return event.slot > boundary.slot;
   if (event.blockOffset !== boundary.blockOffset) {
     return event.blockOffset > boundary.blockOffset;
   }
-  return event.callIdx > boundary.callIdx;
+  if (event.callIdx !== boundary.callIdx) return event.callIdx > boundary.callIdx;
+  return event.eventIdx > boundary.eventIdx;
 }
 function shouldEmitLiveEvent(event, startSlot, resumeAfter) {
   const boundary = parseEventId(resumeAfter);