envio 2.28.0 → 2.29.0-alpha.1

package/src/EventUtils.res

@@ -1,32 +1,26 @@
-type multiChainEventIndex = {
-  timestamp: int,
-  chainId: int,
-  blockNumber: int,
-  logIndex: int,
-}
-
 //Comparator used when ordering multichain events
-let getEventComparator = (multiChainEventIndex: multiChainEventIndex) => {
-  let {timestamp, chainId, blockNumber, logIndex} = multiChainEventIndex
-  (timestamp, chainId, blockNumber, logIndex)
-}
-
-let getEventComparatorFromQueueItem = (
-  {chain, timestamp, blockNumber, logIndex}: Internal.eventItem,
-) => {
-  let chainId = chain->ChainMap.Chain.toChainId
-  (timestamp, chainId, blockNumber, logIndex)
+let getOrderedBatchItemComparator = (item: Internal.item) => {
+  switch item {
+  | Internal.Event({timestamp, chain, blockNumber, logIndex}) => (
+      timestamp,
+      chain->ChainMap.Chain.toChainId,
+      blockNumber,
+      logIndex,
+    )
+  | Internal.Block(_) =>
+    Js.Exn.raiseError("Block handlers are not supported for ordered multichain mode.")
+  }
 }
 
-//Function used to determine if one event is earlier than another
-let isEarlierEvent = (event1: multiChainEventIndex, event2: multiChainEventIndex) => {
-  event1->getEventComparator < event2->getEventComparator
+let isEarlier = (item1: (int, int, int, int), item2: (int, int, int, int)) => {
+  item1 < item2
 }
 
-type eventIndex = {
-  blockNumber: int,
-  logIndex: int,
-}
+// type eventIndex = {
+//   blockNumber: int,
+//   logIndex: int,
+// }
+//
 
 // takes blockNumber, logIndex and packs them into a number with
 //32 bits, 16 bits and 16 bits respectively
@@ -38,44 +32,44 @@ let packEventIndex = (~blockNumber, ~logIndex) => {
   blockNumber->BigInt.Bitwise.logor(logIndex)
 }
 
-//Currently not used but keeping in utils
-//using @live flag for dead code analyser
-@live
-let packMultiChainEventIndex = (~timestamp, ~chainId, ~blockNumber, ~logIndex) => {
-  let timestamp = timestamp->BigInt.fromInt
-  let chainId = chainId->BigInt.fromInt
-  let blockNumber = blockNumber->BigInt.fromInt
-  let logIndex = logIndex->BigInt.fromInt
+// //Currently not used but keeping in utils
+// //using @live flag for dead code analyser
+// @live
+// let packMultiChainEventIndex = (~timestamp, ~chainId, ~blockNumber, ~logIndex) => {
+//   let timestamp = timestamp->BigInt.fromInt
+//   let chainId = chainId->BigInt.fromInt
+//   let blockNumber = blockNumber->BigInt.fromInt
+//   let logIndex = logIndex->BigInt.fromInt
 
-  let timestamp = BigInt.Bitwise.shift_left(timestamp, 48->BigInt.fromInt)
-  let chainId = BigInt.Bitwise.shift_left(chainId, 16->BigInt.fromInt)
-  let blockNumber = BigInt.Bitwise.shift_left(blockNumber, 16->BigInt.fromInt)
+//   let timestamp = BigInt.Bitwise.shift_left(timestamp, 48->BigInt.fromInt)
+//   let chainId = BigInt.Bitwise.shift_left(chainId, 16->BigInt.fromInt)
+//   let blockNumber = BigInt.Bitwise.shift_left(blockNumber, 16->BigInt.fromInt)
 
-  timestamp
-  ->BigInt.Bitwise.logor(chainId)
-  ->BigInt.Bitwise.logor(blockNumber)
-  ->BigInt.Bitwise.logor(logIndex)
-}
+//   timestamp
+//   ->BigInt.Bitwise.logor(chainId)
+//   ->BigInt.Bitwise.logor(blockNumber)
+//   ->BigInt.Bitwise.logor(logIndex)
+// }
 
-//Currently not used but keeping in utils
-//using @live flag for dead code analyser
-@live
-let unpackEventIndex = (packedEventIndex: bigint) => {
-  let blockNumber = packedEventIndex->BigInt.Bitwise.shift_right(16->BigInt.fromInt)
-  let logIndexMask = 65535->BigInt.fromInt
-  let logIndex = packedEventIndex->BigInt.Bitwise.logand(logIndexMask)
-  {
-    blockNumber: blockNumber->BigInt.toString->Belt.Int.fromString->Belt.Option.getUnsafe,
-    logIndex: logIndex->BigInt.toString->Belt.Int.fromString->Belt.Option.getUnsafe,
-  }
-}
+// //Currently not used but keeping in utils
+// //using @live flag for dead code analyser
+// @live
+// let unpackEventIndex = (packedEventIndex: bigint) => {
+//   let blockNumber = packedEventIndex->BigInt.Bitwise.shift_right(16->BigInt.fromInt)
+//   let logIndexMask = 65535->BigInt.fromInt
+//   let logIndex = packedEventIndex->BigInt.Bitwise.logand(logIndexMask)
+//   {
+//     blockNumber: blockNumber->BigInt.toString->Belt.Int.fromString->Belt.Option.getUnsafe,
+//     logIndex: logIndex->BigInt.toString->Belt.Int.fromString->Belt.Option.getUnsafe,
+//   }
+// }
 
-//takes an eventIndex record and returnts a packed event index
-//used in TS tests
-@live
-let packEventIndexFromRecord = (eventIndex: eventIndex) => {
-  packEventIndex(~blockNumber=eventIndex.blockNumber, ~logIndex=eventIndex.logIndex)
-}
+// //takes an eventIndex record and returnts a packed event index
+// //used in TS tests
+// @live
+// let packEventIndexFromRecord = (eventIndex: eventIndex) => {
+//   packEventIndex(~blockNumber=eventIndex.blockNumber, ~logIndex=eventIndex.logIndex)
+// }
 
 //Returns unique string id for an event using its chain id combined with event id
 //Used in IO for the key in the in mem rawEvents table

package/src/EventUtils.res.js

@@ -2,30 +2,23 @@
 'use strict';
 
 var $$BigInt = require("./bindings/BigInt.res.js");
-var Belt_Int = require("rescript/lib/js/belt_Int.js");
+var Js_exn = require("rescript/lib/js/js_exn.js");
 var Caml_obj = require("rescript/lib/js/caml_obj.js");
 
-function getEventComparator(multiChainEventIndex) {
-  return [
-          multiChainEventIndex.timestamp,
-          multiChainEventIndex.chainId,
-          multiChainEventIndex.blockNumber,
-          multiChainEventIndex.logIndex
-        ];
+function getOrderedBatchItemComparator(item) {
+  if (item.kind === 0) {
+    return [
+            item.timestamp,
+            item.chain,
+            item.blockNumber,
+            item.logIndex
+          ];
+  } else {
+    return Js_exn.raiseError("Block handlers are not supported for ordered multichain mode.");
+  }
 }
 
-function getEventComparatorFromQueueItem(param) {
-  return [
-          param.timestamp,
-          param.chain,
-          param.blockNumber,
-          param.logIndex
-        ];
-}
-
-function isEarlierEvent(event1, event2) {
-  return Caml_obj.lessthan(getEventComparator(event1), getEventComparator(event2));
-}
+var isEarlier = Caml_obj.lessthan;
 
 function packEventIndex(blockNumber, logIndex) {
   var blockNumber$1 = BigInt(blockNumber);
@@ -34,42 +27,13 @@ function packEventIndex(blockNumber, logIndex) {
   return $$BigInt.Bitwise.logor(blockNumber$2, logIndex$1);
 }
 
-function packMultiChainEventIndex(timestamp, chainId, blockNumber, logIndex) {
-  var timestamp$1 = BigInt(timestamp);
-  var chainId$1 = BigInt(chainId);
-  var blockNumber$1 = BigInt(blockNumber);
-  var logIndex$1 = BigInt(logIndex);
-  var timestamp$2 = $$BigInt.Bitwise.shift_left(timestamp$1, BigInt(48));
-  var chainId$2 = $$BigInt.Bitwise.shift_left(chainId$1, BigInt(16));
-  var blockNumber$2 = $$BigInt.Bitwise.shift_left(blockNumber$1, BigInt(16));
-  return $$BigInt.Bitwise.logor($$BigInt.Bitwise.logor($$BigInt.Bitwise.logor(timestamp$2, chainId$2), blockNumber$2), logIndex$1);
-}
-
-function unpackEventIndex(packedEventIndex) {
-  var blockNumber = $$BigInt.Bitwise.shift_right(packedEventIndex, BigInt(16));
-  var logIndexMask = BigInt(65535);
-  var logIndex = $$BigInt.Bitwise.logand(packedEventIndex, logIndexMask);
-  return {
-          blockNumber: Belt_Int.fromString(blockNumber.toString()),
-          logIndex: Belt_Int.fromString(logIndex.toString())
-        };
-}
-
-function packEventIndexFromRecord(eventIndex) {
-  return packEventIndex(eventIndex.blockNumber, eventIndex.logIndex);
-}
-
 function getEventIdKeyString(chainId, eventId) {
   var chainIdStr = String(chainId);
   return chainIdStr + "_" + eventId;
 }
 
-exports.getEventComparator = getEventComparator;
-exports.getEventComparatorFromQueueItem = getEventComparatorFromQueueItem;
-exports.isEarlierEvent = isEarlierEvent;
+exports.getOrderedBatchItemComparator = getOrderedBatchItemComparator;
+exports.isEarlier = isEarlier;
 exports.packEventIndex = packEventIndex;
-exports.packMultiChainEventIndex = packMultiChainEventIndex;
-exports.unpackEventIndex = unpackEventIndex;
-exports.packEventIndexFromRecord = packEventIndexFromRecord;
 exports.getEventIdKeyString = getEventIdKeyString;
 /* BigInt Not a pure module */

package/src/FetchState.res

@@ -54,7 +54,6 @@ type t = {
   isFetchingAtHead: bool,
   endBlock: option<int>,
   maxAddrInPartition: int,
-  firstEventBlockNumber: option<int>,
   normalSelection: selection,
   // By address
   indexingContracts: dict<indexingContract>,
@@ -71,7 +70,8 @@ type t = {
   // Needed to query before entering reorg threshold
   blockLag: int,
   //Items ordered from latest to earliest
-  queue: array<Internal.eventItem>,
+  queue: array<Internal.item>,
+  onBlockConfigs: option<array<Internal.onBlockConfig>>,
 }
 
 let copy = (fetchState: t) => {
@@ -83,13 +83,13 @@ let copy = (fetchState: t) => {
     isFetchingAtHead: fetchState.isFetchingAtHead,
     latestFullyFetchedBlock: fetchState.latestFullyFetchedBlock,
     normalSelection: fetchState.normalSelection,
-    firstEventBlockNumber: fetchState.firstEventBlockNumber,
     chainId: fetchState.chainId,
     contractConfigs: fetchState.contractConfigs,
     indexingContracts: fetchState.indexingContracts,
     dcsToStore: fetchState.dcsToStore,
     blockLag: fetchState.blockLag,
     queue: fetchState.queue->Array.copy,
+    onBlockConfigs: fetchState.onBlockConfigs,
   }
 }
 
@@ -256,11 +256,8 @@ let updateInternal = (
     contractConfigs: fetchState.contractConfigs,
     normalSelection: fetchState.normalSelection,
     chainId: fetchState.chainId,
+    onBlockConfigs: fetchState.onBlockConfigs,
     nextPartitionIndex,
-    firstEventBlockNumber: switch queue->Utils.Array.last {
-    | Some(item) => Utils.Math.minOptInt(fetchState.firstEventBlockNumber, Some(item.blockNumber))
-    | None => fetchState.firstEventBlockNumber
-    },
     partitions,
     isFetchingAtHead,
     latestFullyFetchedBlock,
@@ -549,15 +546,18 @@ exception UnexpectedMergeQueryResponse({message: string})
 /*
 Comparitor for two events from the same chain. No need for chain id or timestamp
 */
-let compareBufferItem = (a: Internal.eventItem, b: Internal.eventItem) => {
-  let blockDiff = b.blockNumber - a.blockNumber
+let compareBufferItem = (a: Internal.item, b: Internal.item) => {
+  let blockDiff = b->Internal.getItemBlockNumber - a->Internal.getItemBlockNumber
   if blockDiff === 0 {
-    b.logIndex - a.logIndex
+    b->Internal.getItemLogIndex - a->Internal.getItemLogIndex
   } else {
     blockDiff
   }
 }
 
+// Some big number which should be bigger than any log index
+let blockItemLogIndex = 16777216
+
 /*
 Updates fetchState with a response for a given query.
 Returns Error if the partition with given query cannot be found (unexpected)
@@ -623,11 +623,39 @@ let handleQueryResult = (
       )
     }
   }->Result.map(partitions => {
+    let newQueue = fetchState.queue->Array.concat(newItems)
+
+    switch fetchState.onBlockConfigs {
+    | Some(onBlockConfigs) => {
+        let prevLatestFetchedBlockNumber = fetchState.latestFullyFetchedBlock.blockNumber
+        let nextLatestFullyFetchedBlockNumber = {
+          let nextLatestFullyFetchedBlockNumber = ref(latestFetchedBlock.blockNumber)
+          for idx in 0 to partitions->Array.length - 1 {
+            let p = partitions->Js.Array2.unsafe_get(idx)
+            if nextLatestFullyFetchedBlockNumber.contents > p.latestFetchedBlock.blockNumber {
+              nextLatestFullyFetchedBlockNumber := p.latestFetchedBlock.blockNumber
+            }
+          }
+          nextLatestFullyFetchedBlockNumber.contents
+        }
+
+        if nextLatestFullyFetchedBlockNumber > prevLatestFetchedBlockNumber {
+          for blockNumber in prevLatestFetchedBlockNumber + 1 to nextLatestFullyFetchedBlockNumber {
+            for configIdx in 0 to onBlockConfigs->Array.length - 1 {
+              let onBlockConfig = onBlockConfigs->Js.Array2.unsafe_get(configIdx)
+              newQueue->Array.push(Block({onBlockConfig, blockNumber, logIndex: blockItemLogIndex}))
+            }
+          }
+        }
+      }
+
+    | None => ()
+    }
+
     fetchState->updateInternal(
       ~partitions,
       ~currentBlockHeight,
-      ~queue=fetchState.queue
-      ->Array.concat(newItems)
+      ~queue=newQueue
       // Theoretically it could be faster to asume that
       // the items are sorted, but there are cases
       // when the data source returns them unsorted
@@ -807,7 +835,7 @@ let getNextQuery = (
         currentBlockHeight
       } else {
         switch queue->Array.get(targetBlockIdx) {
-        | Some(item) => Pervasives.min(item.blockNumber, currentBlockHeight) // Just in case check that we don't query beyond the current block
+        | Some(item) => Pervasives.min(item->Internal.getItemBlockNumber, currentBlockHeight) // Just in case check that we don't query beyond the current block
         | None => currentBlockHeight
         }
       }
@@ -881,7 +909,7 @@ let getNextQuery = (
   }
 }
 
-type itemWithPopFn = {item: Internal.eventItem, popItemOffQueue: unit => unit}
+type itemWithPopFn = {item: Internal.item, popItemOffQueue: unit => unit}
 
 /**
 Represents a fetchState partitions head of the  fetchedEventQueue as either
@@ -891,13 +919,6 @@ type queueItem =
   | Item(itemWithPopFn)
   | NoItem({latestFetchedBlock: blockNumberAndTimestamp})
 
-let queueItemBlockNumber = (queueItem: queueItem) => {
-  switch queueItem {
-  | Item({item}) => item.blockNumber
-  | NoItem({latestFetchedBlock: {blockNumber}}) => blockNumber === 0 ? 0 : blockNumber + 1
-  }
-}
-
 /**
 Simple constructor for no item from partition
 */
@@ -905,30 +926,6 @@ let makeNoItem = ({latestFetchedBlock}: partition) => NoItem({
   latestFetchedBlock: latestFetchedBlock,
 })
 
-/**
-Creates a compareable value for items and no items on partition queues.
-Block number takes priority here. Since a latest fetched timestamp could
-be zero from initialization of partition but a higher latest fetched block number exists
-
-Note: on the chain manager, when comparing multi chain, the timestamp is the highest priority compare value
-*/
-let qItemLt = (a, b) => {
-  let aBlockNumber = a->queueItemBlockNumber
-  let bBlockNumber = b->queueItemBlockNumber
-  if aBlockNumber < bBlockNumber {
-    true
-  } else if aBlockNumber === bBlockNumber {
-    switch (a, b) {
-    | (Item(a), Item(b)) => a.item.logIndex < b.item.logIndex
-    | (NoItem(_), Item(_)) => true
-    | (Item(_), NoItem(_))
-    | (NoItem(_), NoItem(_)) => false
-    }
-  } else {
-    false
-  }
-}
-
 /**
 Gets the earliest queueItem from thgetNodeEarliestEventWithUpdatedQueue.
 
@@ -938,7 +935,7 @@ queue item with an update fetch state.
 let getEarliestEvent = ({queue, latestFullyFetchedBlock}: t) => {
   switch queue->Utils.Array.last {
   | Some(item) =>
-    if item.blockNumber <= latestFullyFetchedBlock.blockNumber {
+    if item->Internal.getItemBlockNumber <= latestFullyFetchedBlock.blockNumber {
       Item({item, popItemOffQueue: () => queue->Js.Array2.pop->ignore})
     } else {
       NoItem({
@@ -962,6 +959,7 @@ let make = (
   ~contracts: array<indexingContract>,
   ~maxAddrInPartition,
   ~chainId,
+  ~onBlockConfigs=?,
   ~blockLag=0,
 ): t => {
   let latestFetchedBlock = {
@@ -1082,11 +1080,11 @@ let make = (
     chainId,
     endBlock,
     latestFullyFetchedBlock: latestFetchedBlock,
-    firstEventBlockNumber: None,
     normalSelection,
     indexingContracts,
     dcsToStore: None,
     blockLag,
+    onBlockConfigs,
     queue: [],
   }
 }
@@ -1099,11 +1097,15 @@ let bufferSize = ({queue}: t) => queue->Array.length
 let getLatestFullyFetchedBlock = ({latestFullyFetchedBlock}: t) => latestFullyFetchedBlock
 
 let pruneQueueFromFirstChangeEvent = (
-  queue: array<Internal.eventItem>,
+  queue: array<Internal.item>,
   ~firstChangeEvent: blockNumberAndLogIndex,
 ) => {
   queue->Array.keep(item =>
-    (item.blockNumber, item.logIndex) < (firstChangeEvent.blockNumber, firstChangeEvent.logIndex)
+    switch item {
+    | Event({blockNumber, logIndex})
+    | Block({blockNumber, logIndex}) => (blockNumber, logIndex)
+    } <
+    (firstChangeEvent.blockNumber, firstChangeEvent.logIndex)
   )
 }
 
@@ -1235,7 +1237,7 @@ let isReadyToEnterReorgThreshold = (
 let filterAndSortForUnorderedBatch = {
   let hasBatchItem = ({queue, latestFullyFetchedBlock}: t) => {
     switch queue->Utils.Array.last {
-    | Some(item) => item.blockNumber <= latestFullyFetchedBlock.blockNumber
+    | Some(item) => item->Internal.getItemBlockNumber <= latestFullyFetchedBlock.blockNumber
     | None => false
     }
   }
@@ -1251,8 +1253,8 @@ let filterAndSortForUnorderedBatch = {
     } else {
       // Unsafe can fail when maxBatchSize is 0,
       // but we ignore the case
-      (queue->Js.Array2.unsafe_get(targetBlockIdx)).blockNumber <=
-      latestFullyFetchedBlock.blockNumber
+      queue->Js.Array2.unsafe_get(targetBlockIdx)->Internal.getItemBlockNumber <=
+        latestFullyFetchedBlock.blockNumber
     }
   }
 
@@ -1264,7 +1266,15 @@ let filterAndSortForUnorderedBatch = {
       | (true, true)
       | (false, false) =>
         // Use unsafe since we filtered out all queues without batch items
-        (a.queue->Utils.Array.lastUnsafe).timestamp - (b.queue->Utils.Array.lastUnsafe).timestamp
+        switch (a.queue->Utils.Array.lastUnsafe, b.queue->Utils.Array.lastUnsafe) {
+        | (Event({timestamp: aTimestamp}), Event({timestamp: bTimestamp})) =>
+          aTimestamp - bTimestamp
+        | (Block(_), _)
+        | (_, Block(_)) =>
+          // Currently block items don't have a timestamp,
+          // so we sort chains with them in a random order
+          Js.Math.random_int(-1, 1)
+        }
       | (true, false) => -1
       | (false, true) => 1
       }
@@ -1274,15 +1284,17 @@ let filterAndSortForUnorderedBatch = {
 
 let getProgressBlockNumber = ({latestFullyFetchedBlock, queue}: t) => {
   switch queue->Utils.Array.last {
-  | Some(item) if latestFullyFetchedBlock.blockNumber >= item.blockNumber => item.blockNumber - 1
+  | Some(item) if latestFullyFetchedBlock.blockNumber >= item->Internal.getItemBlockNumber =>
+    item->Internal.getItemBlockNumber - 1
   | _ => latestFullyFetchedBlock.blockNumber
   }
 }
 
 let getProgressNextBlockLogIndex = ({queue, latestFullyFetchedBlock}: t) => {
   switch queue->Utils.Array.last {
-  | Some(item) if latestFullyFetchedBlock.blockNumber >= item.blockNumber && item.logIndex > 0 =>
-    Some(item.logIndex - 1)
+  | Some(Event({logIndex, blockNumber}))
+    if latestFullyFetchedBlock.blockNumber >= blockNumber && logIndex > 0 =>
+    Some(logIndex - 1)
   | _ => None
   }
 }