envio 3.1.1 → 3.1.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "envio",
-  "version": "3.1.1",
+  "version": "3.1.2",
   "type": "module",
   "description": "A latency and sync speed optimized, developer friendly blockchain data indexer.",
   "bin": "./bin.mjs",
@@ -70,10 +70,10 @@
     "tsx": "4.21.0"
   },
   "optionalDependencies": {
-    "envio-linux-x64": "3.1.1",
-    "envio-linux-x64-musl": "3.1.1",
-    "envio-linux-arm64": "3.1.1",
-    "envio-darwin-x64": "3.1.1",
-    "envio-darwin-arm64": "3.1.1"
+    "envio-linux-x64": "3.1.2",
+    "envio-linux-x64-musl": "3.1.2",
+    "envio-linux-arm64": "3.1.2",
+    "envio-darwin-x64": "3.1.2",
+    "envio-darwin-arm64": "3.1.2"
   }
 }

package/src/FetchState.res

@@ -555,6 +555,63 @@ let blockItemLogIndex = 16777216
 
 let numAddresses = fetchState => fetchState.indexingAddresses->Utils.Dict.size
 
+// Appends Block items produced by the onBlock handlers for every block in
+// (fromBlock, maxBlockNumber] into mutItems and returns the new
+// latestOnBlockBlockNumber pointer. targetBufferSize bounds how many items
+// are generated at once to prevent OOM.
+let appendOnBlockItems = (
+  ~mutItems: array<Internal.item>,
+  ~onBlockConfigs: array<Internal.onBlockConfig>,
+  ~indexerStartBlock,
+  ~fromBlock,
+  ~maxBlockNumber,
+  ~targetBufferSize,
+) => {
+  let newItemsCounter = ref(0)
+  let latestOnBlockBlockNumber = ref(fromBlock)
+
+  // Simply iterate over every block
+  // could have a better algorithm to iterate over blocks in a more efficient way
+  // but raw loops are fast enough
+  while (
+    latestOnBlockBlockNumber.contents < maxBlockNumber &&
+      // Additional safeguard to prevent OOM
+      newItemsCounter.contents <= targetBufferSize
+  ) {
+    let blockNumber = latestOnBlockBlockNumber.contents + 1
+    latestOnBlockBlockNumber := blockNumber
+
+    for configIdx in 0 to onBlockConfigs->Array.length - 1 {
+      let onBlockConfig = onBlockConfigs->Array.getUnsafe(configIdx)
+
+      let handlerStartBlock = switch onBlockConfig.startBlock {
+      | Some(startBlock) => startBlock
+      | None => indexerStartBlock
+      }
+
+      if (
+        blockNumber >= handlerStartBlock &&
+        switch onBlockConfig.endBlock {
+        | Some(endBlock) => blockNumber <= endBlock
+        | None => true
+        } &&
+        (blockNumber - handlerStartBlock)->Pervasives.mod(onBlockConfig.interval) === 0
+      ) {
+        mutItems->Array.push(
+          Block({
+            onBlockConfig,
+            blockNumber,
+            logIndex: blockItemLogIndex + onBlockConfig.index,
+          }),
+        )
+        newItemsCounter := newItemsCounter.contents + 1
+      }
+    }
+  }
+
+  latestOnBlockBlockNumber.contents
+}
+
 /*
 Update fetchState, merge registers and recompute derived values.
 Runs partition optimization when partitions change.
@@ -597,49 +654,14 @@ let updateInternal = (
       }
       mutItemsRef := Some(mutItems)
 
-      let newItemsCounter = ref(0)
-      let latestOnBlockBlockNumber = ref(fetchState.latestOnBlockBlockNumber)
-
-      // Simply iterate over every block
-      // could have a better algorithm to iterate over blocks in a more efficient way
-      // but raw loops are fast enough
-      while (
-        latestOnBlockBlockNumber.contents < maxBlockNumber &&
-          // Additional safeguard to prevent OOM
-          newItemsCounter.contents <= fetchState.targetBufferSize
-      ) {
-        let blockNumber = latestOnBlockBlockNumber.contents + 1
-        latestOnBlockBlockNumber := blockNumber
-
-        for configIdx in 0 to onBlockConfigs->Array.length - 1 {
-          let onBlockConfig = onBlockConfigs->Array.getUnsafe(configIdx)
-
-          let handlerStartBlock = switch onBlockConfig.startBlock {
-          | Some(startBlock) => startBlock
-          | None => fetchState.startBlock
-          }
-
-          if (
-            blockNumber >= handlerStartBlock &&
-            switch onBlockConfig.endBlock {
-            | Some(endBlock) => blockNumber <= endBlock
-            | None => true
-            } &&
-            (blockNumber - handlerStartBlock)->Pervasives.mod(onBlockConfig.interval) === 0
-          ) {
-            mutItems->Array.push(
-              Block({
-                onBlockConfig,
-                blockNumber,
-                logIndex: blockItemLogIndex + onBlockConfig.index,
-              }),
-            )
-            newItemsCounter := newItemsCounter.contents + 1
-          }
-        }
-      }
-
-      latestOnBlockBlockNumber.contents
+      appendOnBlockItems(
+        ~mutItems,
+        ~onBlockConfigs,
+        ~indexerStartBlock=fetchState.startBlock,
+        ~fromBlock=fetchState.latestOnBlockBlockNumber,
+        ~maxBlockNumber,
+        ~targetBufferSize=fetchState.targetBufferSize,
+      )
     }
   }
 
@@ -1661,35 +1683,59 @@ let make = (
     )
   }
 
-  let numAddresses = indexingAddresses->Utils.Dict.size
-  Prometheus.IndexingAddresses.set(~addressesCount=numAddresses, ~chainId)
-  Prometheus.IndexingPartitions.set(
-    ~partitionsCount=optimizedPartitions->OptimizedPartitions.count,
-    ~chainId,
-  )
-  Prometheus.IndexingBufferSize.set(~bufferSize=0, ~chainId)
-  Prometheus.IndexingBufferBlockNumber.set(~blockNumber=latestFetchedBlock.blockNumber, ~chainId)
-  switch endBlock {
-  | Some(endBlock) => Prometheus.IndexingEndBlock.set(~endBlock, ~chainId)
-  | None => ()
+  // On resume knownHeight is restored from the DB but the buffer starts empty.
+  // For onBlock-only indexers (e.g. SVM onSlot) there are no partitions to drive
+  // fetching, so without seeding the buffer here getNextQuery would return
+  // NothingToQuery and the indexer would get stuck.
+  let buffer = []
+  let latestOnBlockBlockNumber = if knownHeight > 0 && onBlockConfigs->Utils.Array.notEmpty {
+    let maxBlockNumber = switch optimizedPartitions->OptimizedPartitions.getLatestFullyFetchedBlock {
+    | None => knownHeight
+    | Some(latestFullyFetchedBlock) => latestFullyFetchedBlock.blockNumber
+    }
+    appendOnBlockItems(
+      ~mutItems=buffer,
+      ~onBlockConfigs,
+      ~indexerStartBlock=startBlock,
+      ~fromBlock=progressBlockNumber,
+      ~maxBlockNumber,
+      ~targetBufferSize,
+    )
+  } else {
+    progressBlockNumber
   }
 
-  {
+  let fetchState = {
     optimizedPartitions,
     contractConfigs,
     chainId,
     startBlock,
     endBlock,
-    latestOnBlockBlockNumber: progressBlockNumber,
+    latestOnBlockBlockNumber,
     normalSelection,
     indexingAddresses,
     blockLag,
     onBlockConfigs,
     targetBufferSize,
     knownHeight,
-    buffer: [],
+    buffer,
     firstEventBlock,
   }
+
+  let numAddresses = indexingAddresses->Utils.Dict.size
+  Prometheus.IndexingAddresses.set(~addressesCount=numAddresses, ~chainId)
+  Prometheus.IndexingPartitions.set(
+    ~partitionsCount=optimizedPartitions->OptimizedPartitions.count,
+    ~chainId,
+  )
+  Prometheus.IndexingBufferSize.set(~bufferSize=buffer->Array.length, ~chainId)
+  Prometheus.IndexingBufferBlockNumber.set(~blockNumber=fetchState->bufferBlockNumber, ~chainId)
+  switch endBlock {
+  | Some(endBlock) => Prometheus.IndexingEndBlock.set(~endBlock, ~chainId)
+  | None => ()
+  }
+
+  fetchState
 }
 
 let bufferSize = ({buffer}: t) => buffer->Array.length

package/src/FetchState.res.mjs

@@ -450,6 +450,35 @@ function numAddresses(fetchState) {
   return Utils.Dict.size(fetchState.indexingAddresses);
 }
 
+function appendOnBlockItems(mutItems, onBlockConfigs, indexerStartBlock, fromBlock, maxBlockNumber, targetBufferSize) {
+  let newItemsCounter = 0;
+  let latestOnBlockBlockNumber = fromBlock;
+  while (latestOnBlockBlockNumber < maxBlockNumber && newItemsCounter <= targetBufferSize) {
+    let blockNumber = latestOnBlockBlockNumber + 1 | 0;
+    latestOnBlockBlockNumber = blockNumber;
+    for (let configIdx = 0, configIdx_finish = onBlockConfigs.length; configIdx < configIdx_finish; ++configIdx) {
+      let onBlockConfig = onBlockConfigs[configIdx];
+      let startBlock = onBlockConfig.startBlock;
+      let handlerStartBlock = startBlock !== undefined ? startBlock : indexerStartBlock;
+      let tmp = false;
+      if (blockNumber >= handlerStartBlock) {
+        let endBlock = onBlockConfig.endBlock;
+        tmp = endBlock !== undefined ? blockNumber <= endBlock : true;
+      }
+      if (tmp && Primitive_int.mod_(blockNumber - handlerStartBlock | 0, onBlockConfig.interval) === 0) {
+        mutItems.push({
+          kind: 1,
+          onBlockConfig: onBlockConfig,
+          blockNumber: blockNumber,
+          logIndex: 16777216 + onBlockConfig.index | 0
+        });
+        newItemsCounter = newItemsCounter + 1 | 0;
+      }
+    }
+  };
+  return latestOnBlockBlockNumber;
+}
+
 function updateInternal(fetchState, optimizedPartitionsOpt, indexingAddressesOpt, mutItems, blockLagOpt, knownHeightOpt) {
   let optimizedPartitions = optimizedPartitionsOpt !== undefined ? optimizedPartitionsOpt : fetchState.optimizedPartitions;
   let indexingAddresses = indexingAddressesOpt !== undefined ? indexingAddressesOpt : fetchState.indexingAddresses;
@@ -472,32 +501,7 @@ function updateInternal(fetchState, optimizedPartitionsOpt, indexingAddressesOpt
     let mutItems$2 = mutItemsRef;
     let mutItems$3 = mutItems$2 !== undefined ? mutItems$2 : fetchState.buffer.slice();
     mutItemsRef = mutItems$3;
-    let newItemsCounter = 0;
-    let latestOnBlockBlockNumber$1 = fetchState.latestOnBlockBlockNumber;
-    while (latestOnBlockBlockNumber$1 < maxBlockNumber && newItemsCounter <= fetchState.targetBufferSize) {
-      let blockNumber = latestOnBlockBlockNumber$1 + 1 | 0;
-      latestOnBlockBlockNumber$1 = blockNumber;
-      for (let configIdx = 0, configIdx_finish = onBlockConfigs.length; configIdx < configIdx_finish; ++configIdx) {
-        let onBlockConfig = onBlockConfigs[configIdx];
-        let startBlock = onBlockConfig.startBlock;
-        let handlerStartBlock = startBlock !== undefined ? startBlock : fetchState.startBlock;
-        let tmp = false;
-        if (blockNumber >= handlerStartBlock) {
-          let endBlock = onBlockConfig.endBlock;
-          tmp = endBlock !== undefined ? blockNumber <= endBlock : true;
-        }
-        if (tmp && Primitive_int.mod_(blockNumber - handlerStartBlock | 0, onBlockConfig.interval) === 0) {
-          mutItems$3.push({
-            kind: 1,
-            onBlockConfig: onBlockConfig,
-            blockNumber: blockNumber,
-            logIndex: 16777216 + onBlockConfig.index | 0
-          });
-          newItemsCounter = newItemsCounter + 1 | 0;
-        }
-      }
-    };
-    latestOnBlockBlockNumber = latestOnBlockBlockNumber$1;
+    latestOnBlockBlockNumber = appendOnBlockItems(mutItems$3, onBlockConfigs, fetchState.startBlock, fetchState.latestOnBlockBlockNumber, maxBlockNumber, fetchState.targetBufferSize);
   } else {
     latestOnBlockBlockNumber = knownHeight;
   }
@@ -1299,15 +1303,17 @@ function make$1(startBlock, endBlock, eventConfigs, addresses, maxAddrInPartitio
   if (optimizedPartitions.idsInAscOrder.length === 0 && Utils.$$Array.isEmpty(onBlockConfigs)) {
     Stdlib_JsError.throwWithMessage(`Invalid configuration: Nothing to fetch on chain ` + chainId.toString() + `. ` + (`addresses=` + addresses.length.toString() + `, `) + (`eventConfigs=` + eventConfigs.length.toString() + `, `) + (`normalEventConfigs=` + normalEventConfigs.length.toString() + `. `) + `Make sure that you provided at least one contract address to index, or have events with Wildcard mode enabled, or have onBlock handlers.`);
   }
-  let numAddresses = Utils.Dict.size(indexingAddresses);
-  Prometheus.IndexingAddresses.set(numAddresses, chainId);
-  Prometheus.IndexingPartitions.set(optimizedPartitions.idsInAscOrder.length, chainId);
-  Prometheus.IndexingBufferSize.set(0, chainId);
-  Prometheus.IndexingBufferBlockNumber.set(progressBlockNumber, chainId);
-  if (endBlock !== undefined) {
-    Prometheus.IndexingEndBlock.set(endBlock, chainId);
+  let buffer = [];
+  let latestOnBlockBlockNumber;
+  if (knownHeight > 0 && Utils.$$Array.notEmpty(onBlockConfigs)) {
+    let id = optimizedPartitions.idsInAscOrder[0];
+    let latestFullyFetchedBlock = id !== undefined ? optimizedPartitions.entities[id].latestFetchedBlock : undefined;
+    let maxBlockNumber = latestFullyFetchedBlock !== undefined ? latestFullyFetchedBlock.blockNumber : knownHeight;
+    latestOnBlockBlockNumber = appendOnBlockItems(buffer, onBlockConfigs, startBlock, progressBlockNumber, maxBlockNumber, targetBufferSize);
+  } else {
+    latestOnBlockBlockNumber = progressBlockNumber;
   }
-  return {
+  let fetchState = {
     optimizedPartitions: optimizedPartitions,
     startBlock: startBlock,
     endBlock: endBlock,
@@ -1315,14 +1321,23 @@ function make$1(startBlock, endBlock, eventConfigs, addresses, maxAddrInPartitio
     indexingAddresses: indexingAddresses,
     contractConfigs: contractConfigs,
     chainId: chainId,
-    latestOnBlockBlockNumber: progressBlockNumber,
+    latestOnBlockBlockNumber: latestOnBlockBlockNumber,
     blockLag: blockLag,
-    buffer: [],
+    buffer: buffer,
     targetBufferSize: targetBufferSize,
     onBlockConfigs: onBlockConfigs,
     knownHeight: knownHeight,
     firstEventBlock: firstEventBlock
   };
+  let numAddresses = Utils.Dict.size(indexingAddresses);
+  Prometheus.IndexingAddresses.set(numAddresses, chainId);
+  Prometheus.IndexingPartitions.set(optimizedPartitions.idsInAscOrder.length, chainId);
+  Prometheus.IndexingBufferSize.set(buffer.length, chainId);
+  Prometheus.IndexingBufferBlockNumber.set(bufferBlockNumber(fetchState), chainId);
+  if (endBlock !== undefined) {
+    Prometheus.IndexingEndBlock.set(endBlock, chainId);
+  }
+  return fetchState;
 }
 
 function bufferSize(param) {
@@ -1607,6 +1622,7 @@ export {
   compareBufferItem,
   blockItemLogIndex,
   numAddresses,
+  appendOnBlockItems,
   updateInternal,
   warnDifferentContractType,
   addressesByContractNameCount,