envio 3.11.0 → 3.12.1

package/index.d.ts

@@ -1216,6 +1216,9 @@ export type SvmAllTransactionFields = {
   readonly err: string | undefined;
   readonly fee: bigint;
   readonly computeUnitsConsumed: bigint | undefined;
+  /** Every account the transaction resolves to: the static keys, then the
+   *  address lookup tables' writable then readonly addresses. A transaction's
+   *  account indexes address this list. */
   readonly accountKeys: readonly string[];
   readonly recentBlockhash: string;
   readonly version: string | undefined;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "envio",
-  "version": "3.11.0",
+  "version": "3.12.1",
   "type": "module",
   "description": "A latency and sync speed optimized, developer friendly blockchain data indexer.",
   "bin": "./bin.mjs",
@@ -68,10 +68,10 @@
     "tsx": "4.21.0"
   },
   "optionalDependencies": {
-    "envio-linux-x64": "3.11.0",
-    "envio-linux-x64-musl": "3.11.0",
-    "envio-linux-arm64": "3.11.0",
-    "envio-darwin-x64": "3.11.0",
-    "envio-darwin-arm64": "3.11.0"
+    "envio-linux-x64": "3.12.1",
+    "envio-linux-x64-musl": "3.12.1",
+    "envio-linux-arm64": "3.12.1",
+    "envio-darwin-x64": "3.12.1",
+    "envio-darwin-arm64": "3.12.1"
   }
 }

package/src/Batch.res

@@ -4,6 +4,12 @@ open Utils.UnsafeIntOperators
 type chainAfterBatch = {
   batchSize: int,
   progressBlockNumber: int,
+  // Timestamp of `progressBlockNumber` itself, never of a lower block, so it
+  // measures how far behind chain time the committed progress is. `None` when
+  // the source didn't return that block's header - during backfill it only
+  // returns blocks carrying items, and an SVM slot may have produced no block
+  // at all.
+  progressBlockTime: option<int>,
   sourceBlockNumber: int,
   totalEventsProcessed: float,
   fetchState: FetchState.t,
@@ -23,6 +29,10 @@ type reorgHashSnapshot = {
 type chainBeforeBatch = {
   fetchState: FetchState.t,
   scannedHashes: reorgHashSnapshot,
+  // The progress block isn't known until the build computes it, so its
+  // timestamp can't be snapshotted with the rest. Reads the chain's store,
+  // which the synchronous build can't see mutate.
+  blockTimeAt: int => option<int>,
   shouldRollbackOnReorg: bool,
   progressBlockNumber: int,
   sourceBlockNumber: int,
@@ -34,24 +44,32 @@ type t = {
   totalBatchSize: int,
   items: array<Internal.item>,
   progressedChainsById: dict<chainAfterBatch>,
-  // Processed inside the reorg threshold. Drives whether history is saved, so
-  // writes never merge across a change in this value.
-  isInReorgThreshold: bool,
+  // Whether the batch's rows get history. Writes never merge across a change in
+  // it, so a single write can't mix the two.
+  history: HistoryPolicy.t,
   // Unnest-like checkpoint fields:
   checkpointIds: array<bigint>,
   checkpointChainIds: array<ChainId.t>,
   checkpointBlockNumbers: array<int>,
   checkpointBlockHashes: array<Null.t<string>>,
+  // Items the checkpoint carries, which is what indexes `items`.
+  checkpointItemsCount: array<int>,
+  // Logs the checkpoint carries: one log routed to several registrations is one
+  // event, however many items it made.
   checkpointEventsProcessed: array<int>,
   registeredAddresses: array<AddressRows.staged>,
 }
 
+// What a chain contributed to the batch: items taken off its buffer, and the
+// logs behind them.
+type chainBatchCounts = {size: int, eventsProcessed: int}
+
 let getProgressedChainsById = {
   let getChainAfterBatchIfProgressed = (
     ~chainBeforeBatch: chainBeforeBatch,
     ~progressBlockNumberAfterBatch,
     ~fetchStateAfterBatch,
-    ~batchSize,
+    ~counts: chainBatchCounts,
   ) => {
     // The check is sufficient, since we guarantee to include a full block in a batch
     // Also, this might be true even if batchSize is 0,
@@ -60,10 +78,12 @@ let getProgressedChainsById = {
       Some(
         (
           {
-            batchSize,
+            batchSize: counts.size,
             progressBlockNumber: progressBlockNumberAfterBatch,
+            progressBlockTime: chainBeforeBatch.blockTimeAt(progressBlockNumberAfterBatch),
             sourceBlockNumber: chainBeforeBatch.sourceBlockNumber,
-            totalEventsProcessed: chainBeforeBatch.totalEventsProcessed +. batchSize->Int.toFloat,
+            totalEventsProcessed: chainBeforeBatch.totalEventsProcessed +.
+            counts.eventsProcessed->Int.toFloat,
             fetchState: fetchStateAfterBatch,
             isProgressAtHeadWhenBatchCreated: progressBlockNumberAfterBatch >=
             chainBeforeBatch.sourceBlockNumber - chainBeforeBatch.chainConfig.blockLag,
@@ -77,7 +97,7 @@ let getProgressedChainsById = {
 
   (
     ~chainsBeforeBatch: dict<chainBeforeBatch>,
-    ~batchSizePerChain: dict<int>,
+    ~countsPerChain: dict<chainBatchCounts>,
     ~progressBlockNumberPerChain: dict<int>,
   ) => {
     let progressedChainsById = Dict.make()
@@ -98,14 +118,14 @@ let getProgressedChainsById = {
       | None => chainBeforeBatch.progressBlockNumber
       }
 
-      switch switch batchSizePerChain->Utils.Dict.dangerouslyGetNonOption(
+      switch switch countsPerChain->Utils.Dict.dangerouslyGetNonOption(
         fetchState.chainId->ChainId.toString,
       ) {
-      | Some(batchSize) =>
-        let leftItems = fetchState.buffer->Array.slice(~start=batchSize)
+      | Some(counts) =>
+        let leftItems = fetchState.buffer->Array.slice(~start=counts.size)
         getChainAfterBatchIfProgressed(
           ~chainBeforeBatch,
-          ~batchSize,
+          ~counts,
           ~fetchStateAfterBatch=fetchState->FetchState.updateInternal(~mutItems=leftItems),
           ~progressBlockNumberAfterBatch,
         )
@@ -113,7 +133,7 @@ let getProgressedChainsById = {
       | None =>
         getChainAfterBatchIfProgressed(
           ~chainBeforeBatch,
-          ~batchSize=0,
+          ~counts={size: 0, eventsProcessed: 0},
           ~fetchStateAfterBatch=chainBeforeBatch.fetchState,
           ~progressBlockNumberAfterBatch,
         )
@@ -146,7 +166,7 @@ let seekFirstAbove = (blockNumbers: array<int>, blockNumber) => {
 
 @inline
 let addReorgCheckpoints = (
-  ~prevCheckpointId,
+  ~cursor,
   ~scannedHashes: reorgHashSnapshot,
   ~shouldRollbackOnReorg,
   ~fromBlockExclusive,
@@ -156,10 +176,10 @@ let addReorgCheckpoints = (
   ~mutCheckpointChainIds,
   ~mutCheckpointBlockNumbers,
   ~mutCheckpointBlockHashes,
+  ~mutCheckpointItemsCount,
   ~mutCheckpointEventsProcessed,
 ) => {
   if shouldRollbackOnReorg {
-    let prevCheckpointId = ref(prevCheckpointId)
     // The snapshot already holds only in-threshold scanned hashes, ascending,
     // so seeking to the gap's lower bound gives the gap checkpoints without
     // rescanning the whole snapshot for every gap in the batch.
@@ -172,37 +192,30 @@ let addReorgCheckpoints = (
         scannedHashes.hashByBlockNumber
         ->Utils.Dict.dangerouslyGetByIntNonOption(blockNumber)
         ->Option.getUnsafe
-      let checkpointId = prevCheckpointId.contents->BigInt.add(1n)
-      prevCheckpointId := checkpointId
+      let checkpointId = cursor->CheckpointSequence.next(~chainId)
 
       mutCheckpointIds->Array.push(checkpointId)
       mutCheckpointChainIds->Array.push(chainId)
       mutCheckpointBlockNumbers->Array.push(blockNumber)
       mutCheckpointBlockHashes->Array.push(Null.Value(hash))
+      mutCheckpointItemsCount->Array.push(0)
       mutCheckpointEventsProcessed->Array.push(0)
 
       idx := idx.contents + 1
     }
-    prevCheckpointId.contents
-  } else {
-    prevCheckpointId
   }
 }
 
-// Checkpoint ids are handed out in ascending order as each chain's items are
-// walked in block order, so within a chain a higher id always means a later
-// block. Rollback preserves that: it deletes the chain's ids above its target
-// before any higher one is allocated, so the ids the re-indexed blocks get are
-// above everything the chain still holds. An isolated rollback leans on the
-// invariant — it deletes by `chain_id = c AND id > target`, which is only the
-// chain's stale suffix if ids and blocks agree on order within the chain. Ids
-// interleave freely *across* chains, which is exactly why that predicate can't
-// be the id bound alone.
-let prepareBatch = (
-  ~checkpointIdBeforeBatch,
+// Within a chain, ids ascend with block order, and rollback preserves that by
+// deleting the chain's ids above its target before any higher one is allocated.
+// Every rollback deletes by `id > target`, which is only the stale suffix while
+// ids and blocks agree on order within each chain.
+let make = (
+  ~sequence: CheckpointSequence.t,
+  ~frontier: Frontier.t,
   ~chainsBeforeBatch: dict<chainBeforeBatch>,
   ~batchSizeTarget,
-  ~isInReorgThreshold,
+  ~history,
 ) => {
   let preparedFetchStates =
     chainsBeforeBatch
@@ -214,8 +227,8 @@ let prepareBatch = (
   let preparedNumber = preparedFetchStates->Array.length
   let totalBatchSize = ref(0)
 
-  let prevCheckpointId = ref(checkpointIdBeforeBatch)
-  let mutBatchSizePerChain = Dict.make()
+  let cursor = sequence->CheckpointSequence.cursor(~frontier)
+  let mutCountsPerChain = Dict.make()
   let mutProgressBlockNumberPerChain = Dict.make()
 
   let items = []
@@ -223,6 +236,7 @@ let prepareBatch = (
   let checkpointChainIds = []
   let checkpointBlockNumbers = []
   let checkpointBlockHashes = []
+  let checkpointItemsCount = []
   let checkpointEventsProcessed = []
 
   // Accumulate items for all actively indexing chains
@@ -241,29 +255,38 @@ let prepareBatch = (
       ->Option.getUnsafe
 
     let prevBlockNumber = ref(chainBeforeBatch.progressBlockNumber)
+    let chainEventsProcessed = ref(0)
     if chainBatchSize > 0 {
       for idx in 0 to chainBatchSize - 1 {
         let item = fetchState.buffer->Array.getUnsafe(idx)
         let blockNumber = item->Internal.getItemBlockNumber
+        // The buffer is sorted, so a log's items are consecutive: the first of
+        // them is the only one that counts as an event.
+        let isNewEvent =
+          idx === 0 ||
+            !(fetchState.buffer->Array.getUnsafe(idx - 1)->FetchState.isSameLog(item))
+        if isNewEvent {
+          chainEventsProcessed := chainEventsProcessed.contents + 1
+        }
 
         // Every new block we should create a new checkpoint
         if blockNumber !== prevBlockNumber.contents {
-          prevCheckpointId :=
-            addReorgCheckpoints(
-              ~chainId=fetchState.chainId,
-              ~scannedHashes=chainBeforeBatch.scannedHashes,
-              ~shouldRollbackOnReorg=chainBeforeBatch.shouldRollbackOnReorg,
-              ~prevCheckpointId=prevCheckpointId.contents,
-              ~fromBlockExclusive=prevBlockNumber.contents,
-              ~toBlockExclusive=blockNumber,
-              ~mutCheckpointIds=checkpointIds,
-              ~mutCheckpointChainIds=checkpointChainIds,
-              ~mutCheckpointBlockNumbers=checkpointBlockNumbers,
-              ~mutCheckpointBlockHashes=checkpointBlockHashes,
-              ~mutCheckpointEventsProcessed=checkpointEventsProcessed,
-            )
-
-          let checkpointId = prevCheckpointId.contents->BigInt.add(1n)
+          addReorgCheckpoints(
+            ~chainId=fetchState.chainId,
+            ~scannedHashes=chainBeforeBatch.scannedHashes,
+            ~shouldRollbackOnReorg=chainBeforeBatch.shouldRollbackOnReorg,
+            ~cursor,
+            ~fromBlockExclusive=prevBlockNumber.contents,
+            ~toBlockExclusive=blockNumber,
+            ~mutCheckpointIds=checkpointIds,
+            ~mutCheckpointChainIds=checkpointChainIds,
+            ~mutCheckpointBlockNumbers=checkpointBlockNumbers,
+            ~mutCheckpointBlockHashes=checkpointBlockHashes,
+            ~mutCheckpointItemsCount=checkpointItemsCount,
+            ~mutCheckpointEventsProcessed=checkpointEventsProcessed,
+          )
+
+          let checkpointId = cursor->CheckpointSequence.next(~chainId=fetchState.chainId)
 
           checkpointIds->Array.push(checkpointId)->ignore
           checkpointChainIds->Array.push(fetchState.chainId)->ignore
@@ -278,41 +301,49 @@ let prepareBatch = (
             },
           )
           ->ignore
+          checkpointItemsCount->Array.push(1)->ignore
           checkpointEventsProcessed->Array.push(1)->ignore
 
           prevBlockNumber := blockNumber
-          prevCheckpointId := checkpointId
         } else {
-          let lastIndex = checkpointEventsProcessed->Array.length - 1
-          checkpointEventsProcessed
-          ->Array.setUnsafe(lastIndex, checkpointEventsProcessed->Array.getUnsafe(lastIndex) + 1)
+          let lastIndex = checkpointItemsCount->Array.length - 1
+          checkpointItemsCount
+          ->Array.setUnsafe(lastIndex, checkpointItemsCount->Array.getUnsafe(lastIndex) + 1)
           ->ignore
+          if isNewEvent {
+            checkpointEventsProcessed
+            ->Array.setUnsafe(lastIndex, checkpointEventsProcessed->Array.getUnsafe(lastIndex) + 1)
+            ->ignore
+          }
         }
 
         items->Array.push(item)->ignore
       }
 
       totalBatchSize := totalBatchSize.contents + chainBatchSize
-      mutBatchSizePerChain->ChainId.Dict.set(fetchState.chainId, chainBatchSize)
+      mutCountsPerChain->ChainId.Dict.set(
+        fetchState.chainId,
+        {size: chainBatchSize, eventsProcessed: chainEventsProcessed.contents},
+      )
     }
 
     let progressBlockNumberAfterBatch =
       fetchState->FetchState.getProgressBlockNumberAt(~index=chainBatchSize)
 
-    prevCheckpointId :=
-      addReorgCheckpoints(
-        ~chainId=fetchState.chainId,
-        ~scannedHashes=chainBeforeBatch.scannedHashes,
-        ~shouldRollbackOnReorg=chainBeforeBatch.shouldRollbackOnReorg,
-        ~prevCheckpointId=prevCheckpointId.contents,
-        ~fromBlockExclusive=prevBlockNumber.contents,
-        ~toBlockExclusive=progressBlockNumberAfterBatch + 1, // Make it inclusive
-        ~mutCheckpointIds=checkpointIds,
-        ~mutCheckpointChainIds=checkpointChainIds,
-        ~mutCheckpointBlockNumbers=checkpointBlockNumbers,
-        ~mutCheckpointBlockHashes=checkpointBlockHashes,
-        ~mutCheckpointEventsProcessed=checkpointEventsProcessed,
-      )
+    addReorgCheckpoints(
+      ~chainId=fetchState.chainId,
+      ~scannedHashes=chainBeforeBatch.scannedHashes,
+      ~shouldRollbackOnReorg=chainBeforeBatch.shouldRollbackOnReorg,
+      ~cursor,
+      ~fromBlockExclusive=prevBlockNumber.contents,
+      ~toBlockExclusive=progressBlockNumberAfterBatch + 1, // Make it inclusive
+      ~mutCheckpointIds=checkpointIds,
+      ~mutCheckpointChainIds=checkpointChainIds,
+      ~mutCheckpointBlockNumbers=checkpointBlockNumbers,
+      ~mutCheckpointBlockHashes=checkpointBlockHashes,
+      ~mutCheckpointItemsCount=checkpointItemsCount,
+      ~mutCheckpointEventsProcessed=checkpointEventsProcessed,
+    )
 
     mutProgressBlockNumberPerChain->ChainId.Dict.set(
       fetchState.chainId,
@@ -327,26 +358,28 @@ let prepareBatch = (
     items,
     progressedChainsById: getProgressedChainsById(
       ~chainsBeforeBatch,
-      ~batchSizePerChain=mutBatchSizePerChain,
+      ~countsPerChain=mutCountsPerChain,
       ~progressBlockNumberPerChain=mutProgressBlockNumberPerChain,
     ),
-    isInReorgThreshold,
+    history,
     checkpointIds,
     checkpointChainIds,
     checkpointBlockNumbers,
     checkpointBlockHashes,
+    checkpointItemsCount,
     checkpointEventsProcessed,
     registeredAddresses: [],
   }
 }
 
-let make = (
-  ~checkpointIdBeforeBatch,
-  ~chainsBeforeBatch: dict<chainBeforeBatch>,
-  ~batchSizeTarget,
-  ~isInReorgThreshold,
-) => {
-  prepareBatch(~checkpointIdBeforeBatch, ~chainsBeforeBatch, ~batchSizeTarget, ~isInReorgThreshold)
+// Where the batch leaves each chain it handed ids to. Ids ascend within a
+// chain, so the last one seen per chain is its highest.
+let checkpointFrontier = (batch: t): Frontier.t => {
+  let frontier = Frontier.empty()
+  batch.checkpointChainIds->Array.forEachWithIndex((chainId, index) =>
+    frontier->Frontier.set(chainId, batch.checkpointIds->Array.getUnsafe(index))
+  )
+  frontier
 }
 
 let findFirstEventBlockNumber = (batch: t, ~chainId) => {
@@ -357,7 +390,7 @@ let findFirstEventBlockNumber = (batch: t, ~chainId) => {
     let checkpointChainId = batch.checkpointChainIds->Array.getUnsafe(idx.contents)
     if (
       checkpointChainId === chainId &&
-        batch.checkpointEventsProcessed->Array.getUnsafe(idx.contents) > 0
+        batch.checkpointItemsCount->Array.getUnsafe(idx.contents) > 0
     ) {
       result := Some(batch.checkpointBlockNumbers->Array.getUnsafe(idx.contents))
     } else {

package/src/Batch.res.mjs

@@ -1,34 +1,40 @@
 // Generated by ReScript, PLEASE EDIT WITH CARE
 
 import * as ChainId from "./ChainId.res.mjs";
+import * as Frontier from "./db/Frontier.res.mjs";
 import * as FetchState from "./FetchState.res.mjs";
+import * as CheckpointSequence from "./db/CheckpointSequence.res.mjs";
 
-function getChainAfterBatchIfProgressed(chainBeforeBatch, progressBlockNumberAfterBatch, fetchStateAfterBatch, batchSize) {
+function getChainAfterBatchIfProgressed(chainBeforeBatch, progressBlockNumberAfterBatch, fetchStateAfterBatch, counts) {
   if (chainBeforeBatch.progressBlockNumber < progressBlockNumberAfterBatch) {
     return {
-      batchSize: batchSize,
+      batchSize: counts.size,
       progressBlockNumber: progressBlockNumberAfterBatch,
+      progressBlockTime: chainBeforeBatch.blockTimeAt(progressBlockNumberAfterBatch),
       sourceBlockNumber: chainBeforeBatch.sourceBlockNumber,
-      totalEventsProcessed: chainBeforeBatch.totalEventsProcessed + batchSize,
+      totalEventsProcessed: chainBeforeBatch.totalEventsProcessed + counts.eventsProcessed,
       fetchState: fetchStateAfterBatch,
       isProgressAtHeadWhenBatchCreated: progressBlockNumberAfterBatch >= chainBeforeBatch.sourceBlockNumber - chainBeforeBatch.chainConfig.blockLag
     };
   }
 }
 
-function getProgressedChainsById(chainsBeforeBatch, batchSizePerChain, progressBlockNumberPerChain) {
+function getProgressedChainsById(chainsBeforeBatch, countsPerChain, progressBlockNumberPerChain) {
   let progressedChainsById = {};
   Object.values(chainsBeforeBatch).forEach(chainBeforeBatch => {
     let fetchState = chainBeforeBatch.fetchState;
     let progressBlockNumber = progressBlockNumberPerChain[ChainId.toString(fetchState.chainId)];
     let progressBlockNumberAfterBatch = progressBlockNumber !== undefined ? progressBlockNumber : chainBeforeBatch.progressBlockNumber;
-    let batchSize = batchSizePerChain[ChainId.toString(fetchState.chainId)];
+    let counts = countsPerChain[ChainId.toString(fetchState.chainId)];
     let progressedChain;
-    if (batchSize !== undefined) {
-      let leftItems = fetchState.buffer.slice(batchSize);
-      progressedChain = getChainAfterBatchIfProgressed(chainBeforeBatch, progressBlockNumberAfterBatch, FetchState.updateInternal(fetchState, undefined, leftItems, undefined, undefined, undefined), batchSize);
+    if (counts !== undefined) {
+      let leftItems = fetchState.buffer.slice(counts.size);
+      progressedChain = getChainAfterBatchIfProgressed(chainBeforeBatch, progressBlockNumberAfterBatch, FetchState.updateInternal(fetchState, undefined, leftItems, undefined, undefined, undefined), counts);
     } else {
-      progressedChain = getChainAfterBatchIfProgressed(chainBeforeBatch, progressBlockNumberAfterBatch, chainBeforeBatch.fetchState, 0);
+      progressedChain = getChainAfterBatchIfProgressed(chainBeforeBatch, progressBlockNumberAfterBatch, chainBeforeBatch.fetchState, {
+        size: 0,
+        eventsProcessed: 0
+      });
     }
     if (progressedChain !== undefined) {
       progressedChainsById[chainBeforeBatch.fetchState.chainId] = progressedChain;
@@ -52,145 +58,146 @@ function seekFirstAbove(blockNumbers, blockNumber) {
   return low;
 }
 
-function addReorgCheckpoints(prevCheckpointId, scannedHashes, shouldRollbackOnReorg, fromBlockExclusive, toBlockExclusive, chainId, mutCheckpointIds, mutCheckpointChainIds, mutCheckpointBlockNumbers, mutCheckpointBlockHashes, mutCheckpointEventsProcessed) {
+function addReorgCheckpoints(cursor, scannedHashes, shouldRollbackOnReorg, fromBlockExclusive, toBlockExclusive, chainId, mutCheckpointIds, mutCheckpointChainIds, mutCheckpointBlockNumbers, mutCheckpointBlockHashes, mutCheckpointItemsCount, mutCheckpointEventsProcessed) {
   if (!shouldRollbackOnReorg) {
-    return prevCheckpointId;
+    return;
   }
-  let prevCheckpointId$1 = prevCheckpointId;
   let blockNumbers = scannedHashes.blockNumbers;
   let length = blockNumbers.length;
   let idx = seekFirstAbove(blockNumbers, fromBlockExclusive);
   while (idx < length && blockNumbers[idx] < toBlockExclusive) {
     let blockNumber = blockNumbers[idx];
     let hash = scannedHashes.hashByBlockNumber[blockNumber];
-    let checkpointId = prevCheckpointId$1 + 1n;
-    prevCheckpointId$1 = checkpointId;
+    let checkpointId = CheckpointSequence.next(cursor, chainId);
     mutCheckpointIds.push(checkpointId);
     mutCheckpointChainIds.push(chainId);
     mutCheckpointBlockNumbers.push(blockNumber);
     mutCheckpointBlockHashes.push(hash);
+    mutCheckpointItemsCount.push(0);
     mutCheckpointEventsProcessed.push(0);
     idx = idx + 1;
   };
-  return prevCheckpointId$1;
 }
 
-function prepareBatch(checkpointIdBeforeBatch, chainsBeforeBatch, batchSizeTarget, isInReorgThreshold) {
+function make(sequence, frontier, chainsBeforeBatch, batchSizeTarget, history) {
   let preparedFetchStates = FetchState.sortForBatch(Object.values(chainsBeforeBatch).map(chainBeforeBatch => chainBeforeBatch.fetchState), batchSizeTarget);
   let chainIdx = 0;
   let preparedNumber = preparedFetchStates.length;
   let totalBatchSize = 0;
-  let prevCheckpointId = checkpointIdBeforeBatch;
-  let mutBatchSizePerChain = {};
+  let cursor = CheckpointSequence.cursor(sequence, frontier);
+  let mutCountsPerChain = {};
   let mutProgressBlockNumberPerChain = {};
   let items = [];
   let checkpointIds = [];
   let checkpointChainIds = [];
   let checkpointBlockNumbers = [];
   let checkpointBlockHashes = [];
+  let checkpointItemsCount = [];
   let checkpointEventsProcessed = [];
   while (totalBatchSize < batchSizeTarget && chainIdx < preparedNumber) {
     let fetchState = preparedFetchStates[chainIdx];
     let chainBatchSize = FetchState.getReadyItemsCount(fetchState, batchSizeTarget - totalBatchSize, 0);
     let chainBeforeBatch = chainsBeforeBatch[fetchState.chainId];
     let prevBlockNumber = chainBeforeBatch.progressBlockNumber;
+    let chainEventsProcessed = 0;
     if (chainBatchSize > 0) {
       for (let idx = 0, idx_finish = chainBatchSize - 1; idx <= idx_finish; ++idx) {
         let item = fetchState.buffer[idx];
         let blockNumber = item.blockNumber;
+        let isNewEvent = idx === 0 || !FetchState.isSameLog(fetchState.buffer[idx - 1], item);
+        if (isNewEvent) {
+          chainEventsProcessed = chainEventsProcessed + 1;
+        }
         if (blockNumber !== prevBlockNumber) {
           let chainId = fetchState.chainId;
           let fromBlockExclusive = prevBlockNumber;
           let scannedHashes = chainBeforeBatch.scannedHashes;
-          let prevCheckpointId$1 = prevCheckpointId;
-          let tmp;
           if (chainBeforeBatch.shouldRollbackOnReorg) {
-            let prevCheckpointId$2 = prevCheckpointId$1;
             let blockNumbers = scannedHashes.blockNumbers;
             let length = blockNumbers.length;
             let idx$1 = seekFirstAbove(blockNumbers, fromBlockExclusive);
             while (idx$1 < length && blockNumbers[idx$1] < blockNumber) {
               let blockNumber$1 = blockNumbers[idx$1];
               let hash = scannedHashes.hashByBlockNumber[blockNumber$1];
-              let checkpointId = prevCheckpointId$2 + 1n;
-              prevCheckpointId$2 = checkpointId;
+              let checkpointId = CheckpointSequence.next(cursor, chainId);
               checkpointIds.push(checkpointId);
               checkpointChainIds.push(chainId);
               checkpointBlockNumbers.push(blockNumber$1);
               checkpointBlockHashes.push(hash);
+              checkpointItemsCount.push(0);
               checkpointEventsProcessed.push(0);
               idx$1 = idx$1 + 1;
             };
-            tmp = prevCheckpointId$2;
-          } else {
-            tmp = prevCheckpointId$1;
           }
-          prevCheckpointId = tmp;
-          let checkpointId$1 = prevCheckpointId + 1n;
+          let checkpointId$1 = CheckpointSequence.next(cursor, fetchState.chainId);
           checkpointIds.push(checkpointId$1);
           checkpointChainIds.push(fetchState.chainId);
           checkpointBlockNumbers.push(blockNumber);
           let hash$1 = chainBeforeBatch.scannedHashes.hashByBlockNumber[blockNumber];
           checkpointBlockHashes.push(hash$1 !== undefined ? hash$1 : null);
+          checkpointItemsCount.push(1);
           checkpointEventsProcessed.push(1);
           prevBlockNumber = blockNumber;
-          prevCheckpointId = checkpointId$1;
         } else {
-          let lastIndex = checkpointEventsProcessed.length - 1;
-          checkpointEventsProcessed[lastIndex] = checkpointEventsProcessed[lastIndex] + 1;
+          let lastIndex = checkpointItemsCount.length - 1;
+          checkpointItemsCount[lastIndex] = checkpointItemsCount[lastIndex] + 1;
+          if (isNewEvent) {
+            checkpointEventsProcessed[lastIndex] = checkpointEventsProcessed[lastIndex] + 1;
+          }
         }
         items.push(item);
       }
       totalBatchSize = totalBatchSize + chainBatchSize;
-      mutBatchSizePerChain[fetchState.chainId] = chainBatchSize;
+      mutCountsPerChain[fetchState.chainId] = {
+        size: chainBatchSize,
+        eventsProcessed: chainEventsProcessed
+      };
     }
     let progressBlockNumberAfterBatch = FetchState.getProgressBlockNumberAt(fetchState, chainBatchSize);
     let chainId$1 = fetchState.chainId;
     let toBlockExclusive = progressBlockNumberAfterBatch + 1;
     let fromBlockExclusive$1 = prevBlockNumber;
     let scannedHashes$1 = chainBeforeBatch.scannedHashes;
-    let prevCheckpointId$3 = prevCheckpointId;
-    let tmp$1;
     if (chainBeforeBatch.shouldRollbackOnReorg) {
-      let prevCheckpointId$4 = prevCheckpointId$3;
       let blockNumbers$1 = scannedHashes$1.blockNumbers;
       let length$1 = blockNumbers$1.length;
       let idx$2 = seekFirstAbove(blockNumbers$1, fromBlockExclusive$1);
       while (idx$2 < length$1 && blockNumbers$1[idx$2] < toBlockExclusive) {
         let blockNumber$2 = blockNumbers$1[idx$2];
         let hash$2 = scannedHashes$1.hashByBlockNumber[blockNumber$2];
-        let checkpointId$2 = prevCheckpointId$4 + 1n;
-        prevCheckpointId$4 = checkpointId$2;
+        let checkpointId$2 = CheckpointSequence.next(cursor, chainId$1);
         checkpointIds.push(checkpointId$2);
         checkpointChainIds.push(chainId$1);
         checkpointBlockNumbers.push(blockNumber$2);
         checkpointBlockHashes.push(hash$2);
+        checkpointItemsCount.push(0);
         checkpointEventsProcessed.push(0);
         idx$2 = idx$2 + 1;
       };
-      tmp$1 = prevCheckpointId$4;
-    } else {
-      tmp$1 = prevCheckpointId$3;
     }
-    prevCheckpointId = tmp$1;
     mutProgressBlockNumberPerChain[fetchState.chainId] = progressBlockNumberAfterBatch;
     chainIdx = chainIdx + 1;
   };
   return {
     totalBatchSize: totalBatchSize,
     items: items,
-    progressedChainsById: getProgressedChainsById(chainsBeforeBatch, mutBatchSizePerChain, mutProgressBlockNumberPerChain),
-    isInReorgThreshold: isInReorgThreshold,
+    progressedChainsById: getProgressedChainsById(chainsBeforeBatch, mutCountsPerChain, mutProgressBlockNumberPerChain),
+    history: history,
     checkpointIds: checkpointIds,
     checkpointChainIds: checkpointChainIds,
     checkpointBlockNumbers: checkpointBlockNumbers,
     checkpointBlockHashes: checkpointBlockHashes,
+    checkpointItemsCount: checkpointItemsCount,
     checkpointEventsProcessed: checkpointEventsProcessed,
     registeredAddresses: []
   };
 }
 
-let make = prepareBatch;
+function checkpointFrontier(batch) {
+  let frontier = Frontier.empty();
+  batch.checkpointChainIds.forEach((chainId, index) => Frontier.set(frontier, chainId, batch.checkpointIds[index]));
+  return frontier;
+}
 
 function findFirstEventBlockNumber(batch, chainId) {
   let idx = 0;
@@ -198,7 +205,7 @@ function findFirstEventBlockNumber(batch, chainId) {
   let checkpointsLength = batch.checkpointIds.length;
   while (idx < checkpointsLength && result === undefined) {
     let checkpointChainId = batch.checkpointChainIds[idx];
-    if (checkpointChainId === chainId && batch.checkpointEventsProcessed[idx] > 0) {
+    if (checkpointChainId === chainId && batch.checkpointItemsCount[idx] > 0) {
       result = batch.checkpointBlockNumbers[idx];
     } else {
       idx = idx + 1;
@@ -225,8 +232,8 @@ export {
   getProgressedChainsById,
   seekFirstAbove,
   addReorgCheckpoints,
-  prepareBatch,
   make,
+  checkpointFrontier,
   findFirstEventBlockNumber,
   findLastEventItem,
 }