envio 2.31.0-alpha.0 → 2.31.0-alpha.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "envio",
-  "version": "v2.31.0-alpha.0",
+  "version": "v2.31.0-alpha.2",
   "description": "A latency and sync speed optimized, developer friendly blockchain data indexer.",
   "bin": "./bin.js",
   "main": "./index.js",
@@ -25,10 +25,10 @@
   },
   "homepage": "https://envio.dev",
   "optionalDependencies": {
-    "envio-linux-x64": "v2.31.0-alpha.0",
-    "envio-linux-arm64": "v2.31.0-alpha.0",
-    "envio-darwin-x64": "v2.31.0-alpha.0",
-    "envio-darwin-arm64": "v2.31.0-alpha.0"
+    "envio-linux-x64": "v2.31.0-alpha.2",
+    "envio-linux-arm64": "v2.31.0-alpha.2",
+    "envio-darwin-x64": "v2.31.0-alpha.2",
+    "envio-darwin-arm64": "v2.31.0-alpha.2"
   },
   "dependencies": {
     "@envio-dev/hypersync-client": "0.6.6",

package/src/Batch.res

@@ -1,17 +1,34 @@
-type progressedChain = {
-  chainId: int,
+open Belt
+
+@@warning("-44")
+open Utils.UnsafeIntOperators
+
+type chainAfterBatch = {
   batchSize: int,
   progressBlockNumber: int,
-  isProgressAtHead: bool,
+  totalEventsProcessed: int,
+  fetchState: FetchState.t,
+  isProgressAtHeadWhenBatchCreated: bool,
+}
+
+type chainBeforeBatch = {
+  fetchState: FetchState.t,
+  reorgDetection: ReorgDetection.t,
+  progressBlockNumber: int,
+  sourceBlockNumber: int,
   totalEventsProcessed: int,
 }
 
 type t = {
+  totalBatchSize: int,
   items: array<Internal.item>,
-  progressedChains: array<progressedChain>,
-  updatedFetchStates: ChainMap.t<FetchState.t>,
-  dcsToStoreByChainId: dict<array<FetchState.indexingContract>>,
-  creationTimeMs: int,
+  progressedChainsById: dict<chainAfterBatch>,
+  // Unnest-like checkpoint fields:
+  checkpointIds: array<int>,
+  checkpointChainIds: array<int>,
+  checkpointBlockNumbers: array<int>,
+  checkpointBlockHashes: array<Js.Null.t<string>>,
+  checkpointEventsProcessed: array<int>,
 }
 
 /**
@@ -22,7 +39,7 @@ let getOrderedNextChain = (fetchStates: ChainMap.t<FetchState.t>, ~batchSizePerC
   let earliestChainTimestamp = ref(0)
   let chainKeys = fetchStates->ChainMap.keys
   for idx in 0 to chainKeys->Array.length - 1 {
-    let chain = chainKeys->Array.get(idx)
+    let chain = chainKeys->Array.getUnsafe(idx)
     let fetchState = fetchStates->ChainMap.get(chain)
     if fetchState->FetchState.isActivelyIndexing {
       let timestamp = fetchState->FetchState.getTimestampAt(
@@ -75,37 +92,250 @@ let hasMultichainReadyItem = (
   }
 }
 
+let getProgressedChainsById = {
+  let getChainAfterBatchIfProgressed = (
+    ~chainBeforeBatch: chainBeforeBatch,
+    ~progressBlockNumberAfterBatch,
+    ~fetchStateAfterBatch,
+    ~batchSize,
+  ) => {
+    // The check is sufficient, since we guarantee to include a full block in a batch
+    // Also, this might be true even if batchSize is 0,
+    // eg when indexing at the head and chain doesn't have items in a block
+    if chainBeforeBatch.progressBlockNumber < progressBlockNumberAfterBatch {
+      Some(
+        (
+          {
+            batchSize,
+            progressBlockNumber: progressBlockNumberAfterBatch,
+            totalEventsProcessed: chainBeforeBatch.totalEventsProcessed + batchSize,
+            fetchState: fetchStateAfterBatch,
+            isProgressAtHeadWhenBatchCreated: progressBlockNumberAfterBatch >=
+            chainBeforeBatch.sourceBlockNumber,
+          }: chainAfterBatch
+        ),
+      )
+    } else {
+      None
+    }
+  }
+
+  (
+    ~chainsBeforeBatch: ChainMap.t<chainBeforeBatch>,
+    ~batchSizePerChain: dict<int>,
+    ~progressBlockNumberPerChain: dict<int>,
+  ) => {
+    let progressedChainsById = Js.Dict.empty()
+
+    // Needed to:
+    // - Recalculate the computed queue sizes
+    // - Accumulate registered dynamic contracts to store in the db
+    // - Trigger onBlock pointer update
+    chainsBeforeBatch
+    ->ChainMap.values
+    ->Array.forEachU(chainBeforeBatch => {
+      let fetchState = chainBeforeBatch.fetchState
+
+      let progressBlockNumberAfterBatch = switch progressBlockNumberPerChain->Utils.Dict.dangerouslyGetNonOption(
+        fetchState.chainId->Int.toString,
+      ) {
+      | Some(progressBlockNumber) => progressBlockNumber
+      | None => chainBeforeBatch.progressBlockNumber
+      }
+
+      switch switch batchSizePerChain->Utils.Dict.dangerouslyGetNonOption(
+        fetchState.chainId->Int.toString,
+      ) {
+      | Some(batchSize) =>
+        let leftItems = fetchState.buffer->Js.Array2.sliceFrom(batchSize)
+        getChainAfterBatchIfProgressed(
+          ~chainBeforeBatch,
+          ~batchSize,
+          ~fetchStateAfterBatch=fetchState->FetchState.updateInternal(~mutItems=leftItems),
+          ~progressBlockNumberAfterBatch,
+        )
+      // Skip not affected chains
+      | None =>
+        getChainAfterBatchIfProgressed(
+          ~chainBeforeBatch,
+          ~batchSize=0,
+          ~fetchStateAfterBatch=chainBeforeBatch.fetchState,
+          ~progressBlockNumberAfterBatch,
+        )
+      } {
+      | Some(progressedChain) =>
+        progressedChainsById->Utils.Dict.setByInt(
+          chainBeforeBatch.fetchState.chainId,
+          progressedChain,
+        )
+      | None => ()
+      }
+    })
+
+    progressedChainsById
+  }
+}
+
+@inline
+let addReorgCheckpoints = (
+  ~prevCheckpointId,
+  ~reorgDetection: ReorgDetection.t,
+  ~fromBlockExclusive,
+  ~toBlockExclusive,
+  ~chainId,
+  ~mutCheckpointIds,
+  ~mutCheckpointChainIds,
+  ~mutCheckpointBlockNumbers,
+  ~mutCheckpointBlockHashes,
+  ~mutCheckpointEventsProcessed,
+) => {
+  if (
+    reorgDetection.shouldRollbackOnReorg && !(reorgDetection.dataByBlockNumber->Utils.Dict.isEmpty)
+  ) {
+    let prevCheckpointId = ref(prevCheckpointId)
+    for blockNumber in fromBlockExclusive + 1 to toBlockExclusive - 1 {
+      switch reorgDetection->ReorgDetection.getHashByBlockNumber(~blockNumber) {
+      | Js.Null.Value(hash) =>
+        let checkpointId = prevCheckpointId.contents + 1
+        prevCheckpointId := checkpointId
+
+        mutCheckpointIds->Js.Array2.push(checkpointId)->ignore
+        mutCheckpointChainIds->Js.Array2.push(chainId)->ignore
+        mutCheckpointBlockNumbers->Js.Array2.push(blockNumber)->ignore
+        mutCheckpointBlockHashes->Js.Array2.push(Js.Null.Value(hash))->ignore
+        mutCheckpointEventsProcessed->Js.Array2.push(0)->ignore
+      | Js.Null.Null => ()
+      }
+    }
+    prevCheckpointId.contents
+  } else {
+    prevCheckpointId
+  }
+}
+
 let prepareOrderedBatch = (
+  ~checkpointIdBeforeBatch,
+  ~chainsBeforeBatch: ChainMap.t<chainBeforeBatch>,
   ~batchSizeTarget,
-  ~fetchStates: ChainMap.t<FetchState.t>,
-  ~mutBatchSizePerChain: dict<int>,
 ) => {
-  let batchSize = ref(0)
+  let totalBatchSize = ref(0)
   let isFinished = ref(false)
+  let prevCheckpointId = ref(checkpointIdBeforeBatch)
+  let mutBatchSizePerChain = Js.Dict.empty()
+  let mutProgressBlockNumberPerChain = Js.Dict.empty()
+
+  let fetchStates = chainsBeforeBatch->ChainMap.map(chainBeforeBatch => chainBeforeBatch.fetchState)
+
   let items = []
+  let checkpointIds = []
+  let checkpointChainIds = []
+  let checkpointBlockNumbers = []
+  let checkpointBlockHashes = []
+  let checkpointEventsProcessed = []
 
-  while batchSize.contents < batchSizeTarget && !isFinished.contents {
+  while totalBatchSize.contents < batchSizeTarget && !isFinished.contents {
     switch fetchStates->getOrderedNextChain(~batchSizePerChain=mutBatchSizePerChain) {
     | Some(fetchState) => {
+        let chainBeforeBatch =
+          chainsBeforeBatch->ChainMap.get(ChainMap.Chain.makeUnsafe(~chainId=fetchState.chainId))
         let itemsCountBefore = switch mutBatchSizePerChain->Utils.Dict.dangerouslyGetByIntNonOption(
           fetchState.chainId,
         ) {
         | Some(batchSize) => batchSize
         | None => 0
         }
-        let newItemsCount =
-          fetchState->FetchState.getReadyItemsCount(~targetSize=1, ~fromItem=itemsCountBefore)
+
+        let prevBlockNumber = switch mutProgressBlockNumberPerChain->Utils.Dict.dangerouslyGetByIntNonOption(
+          fetchState.chainId,
+        ) {
+        | Some(progressBlockNumber) => progressBlockNumber
+        | None => chainBeforeBatch.progressBlockNumber
+        }
+
+        let newItemsCount = fetchState->FetchState.getReadyItemsCount(
+          // We should get items only for a single block
+          // Since for the ordered mode next block could be after another chain's block
+          ~targetSize=1,
+          ~fromItem=itemsCountBefore,
+        )
 
         if newItemsCount > 0 {
-          for idx in itemsCountBefore to itemsCountBefore + newItemsCount - 1 {
-            items->Js.Array2.push(fetchState.buffer->Belt.Array.getUnsafe(idx))->ignore
+          let item0 = fetchState.buffer->Array.getUnsafe(itemsCountBefore)
+          let blockNumber = item0->Internal.getItemBlockNumber
+
+          prevCheckpointId :=
+            addReorgCheckpoints(
+              ~chainId=fetchState.chainId,
+              ~reorgDetection=chainBeforeBatch.reorgDetection,
+              ~prevCheckpointId=prevCheckpointId.contents,
+              ~fromBlockExclusive=prevBlockNumber,
+              ~toBlockExclusive=blockNumber,
+              ~mutCheckpointIds=checkpointIds,
+              ~mutCheckpointChainIds=checkpointChainIds,
+              ~mutCheckpointBlockNumbers=checkpointBlockNumbers,
+              ~mutCheckpointBlockHashes=checkpointBlockHashes,
+              ~mutCheckpointEventsProcessed=checkpointEventsProcessed,
+            )
+
+          let checkpointId = prevCheckpointId.contents + 1
+
+          items
+          ->Js.Array2.push(item0)
+          ->ignore
+          for idx in 1 to newItemsCount - 1 {
+            items
+            ->Js.Array2.push(fetchState.buffer->Belt.Array.getUnsafe(itemsCountBefore + idx))
+            ->ignore
           }
-          batchSize := batchSize.contents + newItemsCount
+
+          checkpointIds
+          ->Js.Array2.push(checkpointId)
+          ->ignore
+          checkpointChainIds
+          ->Js.Array2.push(fetchState.chainId)
+          ->ignore
+          checkpointBlockNumbers
+          ->Js.Array2.push(blockNumber)
+          ->ignore
+          checkpointBlockHashes
+          ->Js.Array2.push(
+            chainBeforeBatch.reorgDetection->ReorgDetection.getHashByBlockNumber(~blockNumber),
+          )
+          ->ignore
+          checkpointEventsProcessed
+          ->Js.Array2.push(newItemsCount)
+          ->ignore
+
+          prevCheckpointId := checkpointId
+          totalBatchSize := totalBatchSize.contents + newItemsCount
           mutBatchSizePerChain->Utils.Dict.setByInt(
             fetchState.chainId,
             itemsCountBefore + newItemsCount,
           )
+          mutProgressBlockNumberPerChain->Utils.Dict.setByInt(fetchState.chainId, blockNumber)
         } else {
+          let blockNumberAfterBatch = fetchState->FetchState.bufferBlockNumber
+
+          prevCheckpointId :=
+            addReorgCheckpoints(
+              ~chainId=fetchState.chainId,
+              ~reorgDetection=chainBeforeBatch.reorgDetection,
+              ~prevCheckpointId=prevCheckpointId.contents,
+              ~fromBlockExclusive=prevBlockNumber,
+              ~toBlockExclusive=blockNumberAfterBatch + 1, // Make it inclusive
+              ~mutCheckpointIds=checkpointIds,
+              ~mutCheckpointChainIds=checkpointChainIds,
+              ~mutCheckpointBlockNumbers=checkpointBlockNumbers,
+              ~mutCheckpointBlockHashes=checkpointBlockHashes,
+              ~mutCheckpointEventsProcessed=checkpointEventsProcessed,
+            )
+
+          // Since the chain was chosen as next
+          // the fact that it doesn't have new items means that it reached the buffer block number
+          mutProgressBlockNumberPerChain->Utils.Dict.setByInt(
+            fetchState.chainId,
+            blockNumberAfterBatch,
+          )
           isFinished := true
         }
       }
@@ -114,45 +344,187 @@ let prepareOrderedBatch = (
     }
   }
 
-  items
+  {
+    totalBatchSize: totalBatchSize.contents,
+    items,
+    progressedChainsById: getProgressedChainsById(
+      ~chainsBeforeBatch,
+      ~batchSizePerChain=mutBatchSizePerChain,
+      ~progressBlockNumberPerChain=mutProgressBlockNumberPerChain,
+    ),
+    checkpointIds,
+    checkpointChainIds,
+    checkpointBlockNumbers,
+    checkpointBlockHashes,
+    checkpointEventsProcessed,
+  }
 }
 
 let prepareUnorderedBatch = (
+  ~checkpointIdBeforeBatch,
+  ~chainsBeforeBatch: ChainMap.t<chainBeforeBatch>,
   ~batchSizeTarget,
-  ~fetchStates: ChainMap.t<FetchState.t>,
-  ~mutBatchSizePerChain: dict<int>,
 ) => {
   let preparedFetchStates =
-    fetchStates
+    chainsBeforeBatch
     ->ChainMap.values
-    ->FetchState.filterAndSortForUnorderedBatch(~batchSizeTarget)
+    ->Js.Array2.map(chainBeforeBatch => chainBeforeBatch.fetchState)
+    ->FetchState.sortForUnorderedBatch(~batchSizeTarget)
 
   let chainIdx = ref(0)
   let preparedNumber = preparedFetchStates->Array.length
-  let batchSize = ref(0)
+  let totalBatchSize = ref(0)
+
+  let prevCheckpointId = ref(checkpointIdBeforeBatch)
+  let mutBatchSizePerChain = Js.Dict.empty()
+  let mutProgressBlockNumberPerChain = Js.Dict.empty()
 
   let items = []
+  let checkpointIds = []
+  let checkpointChainIds = []
+  let checkpointBlockNumbers = []
+  let checkpointBlockHashes = []
+  let checkpointEventsProcessed = []
 
   // Accumulate items for all actively indexing chains
   // the way to group as many items from a single chain as possible
   // This way the loaders optimisations will hit more often
-  while batchSize.contents < batchSizeTarget && chainIdx.contents < preparedNumber {
+  while totalBatchSize.contents < batchSizeTarget && chainIdx.contents < preparedNumber {
     let fetchState = preparedFetchStates->Js.Array2.unsafe_get(chainIdx.contents)
     let chainBatchSize =
       fetchState->FetchState.getReadyItemsCount(
-        ~targetSize=batchSizeTarget - batchSize.contents,
+        ~targetSize=batchSizeTarget - totalBatchSize.contents,
         ~fromItem=0,
       )
+    let chainBeforeBatch =
+      chainsBeforeBatch->ChainMap.get(ChainMap.Chain.makeUnsafe(~chainId=fetchState.chainId))
+
+    let prevBlockNumber = ref(chainBeforeBatch.progressBlockNumber)
     if chainBatchSize > 0 {
       for idx in 0 to chainBatchSize - 1 {
-        items->Js.Array2.push(fetchState.buffer->Belt.Array.getUnsafe(idx))->ignore
+        let item = fetchState.buffer->Belt.Array.getUnsafe(idx)
+        let blockNumber = item->Internal.getItemBlockNumber
+
+        // Every new block we should create a new checkpoint
+        if blockNumber !== prevBlockNumber.contents {
+          prevCheckpointId :=
+            addReorgCheckpoints(
+              ~chainId=fetchState.chainId,
+              ~reorgDetection=chainBeforeBatch.reorgDetection,
+              ~prevCheckpointId=prevCheckpointId.contents,
+              ~fromBlockExclusive=prevBlockNumber.contents,
+              ~toBlockExclusive=blockNumber,
+              ~mutCheckpointIds=checkpointIds,
+              ~mutCheckpointChainIds=checkpointChainIds,
+              ~mutCheckpointBlockNumbers=checkpointBlockNumbers,
+              ~mutCheckpointBlockHashes=checkpointBlockHashes,
+              ~mutCheckpointEventsProcessed=checkpointEventsProcessed,
+            )
+
+          let checkpointId = prevCheckpointId.contents + 1
+
+          checkpointIds->Js.Array2.push(checkpointId)->ignore
+          checkpointChainIds->Js.Array2.push(fetchState.chainId)->ignore
+          checkpointBlockNumbers->Js.Array2.push(blockNumber)->ignore
+          checkpointBlockHashes
+          ->Js.Array2.push(
+            chainBeforeBatch.reorgDetection->ReorgDetection.getHashByBlockNumber(~blockNumber),
+          )
+          ->ignore
+          checkpointEventsProcessed->Js.Array2.push(1)->ignore
+
+          prevBlockNumber := blockNumber
+          prevCheckpointId := checkpointId
+        } else {
+          let lastIndex = checkpointEventsProcessed->Array.length - 1
+          checkpointEventsProcessed
+          ->Belt.Array.setUnsafe(
+            lastIndex,
+            checkpointEventsProcessed->Array.getUnsafe(lastIndex) + 1,
+          )
+          ->ignore
+        }
+
+        items->Js.Array2.push(item)->ignore
       }
-      batchSize := batchSize.contents + chainBatchSize
+
+      totalBatchSize := totalBatchSize.contents + chainBatchSize
       mutBatchSizePerChain->Utils.Dict.setByInt(fetchState.chainId, chainBatchSize)
     }
 
+    let progressBlockNumberAfterBatch =
+      fetchState->FetchState.getUnorderedMultichainProgressBlockNumberAt(~index=chainBatchSize)
+
+    prevCheckpointId :=
+      addReorgCheckpoints(
+        ~chainId=fetchState.chainId,
+        ~reorgDetection=chainBeforeBatch.reorgDetection,
+        ~prevCheckpointId=prevCheckpointId.contents,
+        ~fromBlockExclusive=prevBlockNumber.contents,
+        ~toBlockExclusive=progressBlockNumberAfterBatch + 1, // Make it inclusive
+        ~mutCheckpointIds=checkpointIds,
+        ~mutCheckpointChainIds=checkpointChainIds,
+        ~mutCheckpointBlockNumbers=checkpointBlockNumbers,
+        ~mutCheckpointBlockHashes=checkpointBlockHashes,
+        ~mutCheckpointEventsProcessed=checkpointEventsProcessed,
+      )
+
+    mutProgressBlockNumberPerChain->Utils.Dict.setByInt(
+      fetchState.chainId,
+      progressBlockNumberAfterBatch,
+    )
+
     chainIdx := chainIdx.contents + 1
   }
 
-  items
+  {
+    totalBatchSize: totalBatchSize.contents,
+    items,
+    progressedChainsById: getProgressedChainsById(
+      ~chainsBeforeBatch,
+      ~batchSizePerChain=mutBatchSizePerChain,
+      ~progressBlockNumberPerChain=mutProgressBlockNumberPerChain,
+    ),
+    checkpointIds,
+    checkpointChainIds,
+    checkpointBlockNumbers,
+    checkpointBlockHashes,
+    checkpointEventsProcessed,
+  }
+}
+
+let make = (
+  ~checkpointIdBeforeBatch,
+  ~chainsBeforeBatch: ChainMap.t<chainBeforeBatch>,
+  ~multichain: InternalConfig.multichain,
+  ~batchSizeTarget,
+) => {
+  if (
+    switch multichain {
+    | Unordered => true
+    | Ordered => chainsBeforeBatch->ChainMap.size === 1
+    }
+  ) {
+    prepareUnorderedBatch(~checkpointIdBeforeBatch, ~chainsBeforeBatch, ~batchSizeTarget)
+  } else {
+    prepareOrderedBatch(~checkpointIdBeforeBatch, ~chainsBeforeBatch, ~batchSizeTarget)
+  }
+}
+
+let findFirstEventBlockNumber = (batch: t, ~chainId) => {
+  let idx = ref(0)
+  let result = ref(None)
+  let checkpointsLength = batch.checkpointIds->Array.length
+  while idx.contents < checkpointsLength && result.contents === None {
+    let checkpointChainId = batch.checkpointChainIds->Array.getUnsafe(idx.contents)
+    if (
+      checkpointChainId === chainId &&
+        batch.checkpointEventsProcessed->Array.getUnsafe(idx.contents) > 0
+    ) {
+      result := Some(batch.checkpointBlockNumbers->Array.getUnsafe(idx.contents))
+    } else {
+      idx := idx.contents + 1
+    }
+  }
+  result.contents
 }