envio 2.31.0-alpha.0 → 2.31.0-alpha.2

package/src/FetchState.res.js

@@ -8,7 +8,6 @@ var Js_dict = require("rescript/lib/js/js_dict.js");
 var Js_math = require("rescript/lib/js/js_math.js");
 var Logging = require("./Logging.res.js");
 var Belt_Int = require("rescript/lib/js/belt_Int.js");
-var Caml_obj = require("rescript/lib/js/caml_obj.js");
 var Belt_Array = require("rescript/lib/js/belt_Array.js");
 var Caml_int32 = require("rescript/lib/js/caml_int32.js");
 var Prometheus = require("./Prometheus.res.js");
@@ -34,11 +33,11 @@ function mergeIntoPartition(p, target, maxAddrInPartition) {
   var allowedAddressesNumber = {
     contents: maxAddrInPartition
   };
-  Utils.Dict.forEachWithKey(target.addressesByContractName, (function (contractName, addresses) {
+  Utils.Dict.forEachWithKey(target.addressesByContractName, (function (addresses, contractName) {
           allowedAddressesNumber.contents = allowedAddressesNumber.contents - addresses.length | 0;
           mergedAddresses[contractName] = addresses;
         }));
-  Utils.Dict.forEachWithKey(p.addressesByContractName, (function (contractName, addresses) {
+  Utils.Dict.forEachWithKey(p.addressesByContractName, (function (addresses, contractName) {
           allowedAddressesNumber.contents = allowedAddressesNumber.contents - addresses.length | 0;
           var targetAddresses = mergedAddresses[contractName];
           if (targetAddresses !== undefined) {
@@ -50,7 +49,7 @@ function mergeIntoPartition(p, target, maxAddrInPartition) {
   var rest;
   if (allowedAddressesNumber.contents < 0) {
     var restAddresses = {};
-    Utils.Dict.forEachWithKey(mergedAddresses, (function (contractName, addresses) {
+    Utils.Dict.forEachWithKey(mergedAddresses, (function (addresses, contractName) {
             if (allowedAddressesNumber.contents === 0) {
               return ;
             }
@@ -123,11 +122,10 @@ function compareBufferItem(a, b) {
   }
 }
 
-function updateInternal(fetchState, partitionsOpt, nextPartitionIndexOpt, indexingContractsOpt, dcsToStoreOpt, mutItems, blockLagOpt) {
+function updateInternal(fetchState, partitionsOpt, nextPartitionIndexOpt, indexingContractsOpt, mutItems, blockLagOpt) {
   var partitions = partitionsOpt !== undefined ? partitionsOpt : fetchState.partitions;
   var nextPartitionIndex = nextPartitionIndexOpt !== undefined ? nextPartitionIndexOpt : fetchState.nextPartitionIndex;
   var indexingContracts = indexingContractsOpt !== undefined ? indexingContractsOpt : fetchState.indexingContracts;
-  var dcsToStore = dcsToStoreOpt !== undefined ? dcsToStoreOpt : fetchState.dcsToStore;
   var blockLag = blockLagOpt !== undefined ? blockLagOpt : fetchState.blockLag;
   var firstPartition = partitions[0];
   var latestFullyFetchedBlock = firstPartition.latestFetchedBlock;
@@ -198,7 +196,6 @@ function updateInternal(fetchState, partitionsOpt, nextPartitionIndexOpt, indexi
     normalSelection: updatedFetchState_normalSelection,
     indexingContracts: indexingContracts,
     contractConfigs: updatedFetchState_contractConfigs,
-    dcsToStore: dcsToStore,
     chainId: updatedFetchState_chainId,
     latestFullyFetchedBlock: latestFullyFetchedBlock$1,
     latestOnBlockBlockNumber: latestOnBlockBlockNumber,
@@ -227,7 +224,7 @@ function warnDifferentContractType(fetchState, existingContract, dc) {
   Logging.childWarn(logger, "Skipping contract registration: Contract address is already registered for one contract and cannot be registered for another contract.");
 }
 
-function registerDynamicContracts(fetchState, dynamicContracts) {
+function registerDynamicContracts(fetchState, items) {
   if (Utils.$$Array.isEmpty(fetchState.normalSelection.eventConfigs)) {
     Js_exn.raiseError("Invalid configuration. No events to fetch for the dynamic contract registration.");
   }
@@ -236,55 +233,62 @@ function registerDynamicContracts(fetchState, dynamicContracts) {
   var addressesByContractName = {};
   var earliestRegisteringEventBlockNumber = Infinity;
   var hasDCWithFilterByAddresses = false;
-  for(var idx = 0 ,idx_finish = dynamicContracts.length; idx < idx_finish; ++idx){
-    var dc = dynamicContracts[idx];
-    var match = fetchState.contractConfigs[dc.contractName];
-    if (match !== undefined) {
-      var existingContract = indexingContracts[dc.address];
-      if (existingContract !== undefined) {
-        if (existingContract.contractName !== dc.contractName) {
-          warnDifferentContractType(fetchState, existingContract, dc);
-        } else if (existingContract.startBlock > dc.startBlock) {
-          var logger = Logging.createChild({
-                chainId: fetchState.chainId,
-                contractAddress: dc.address,
-                existingBlockNumber: existingContract.startBlock,
-                newBlockNumber: dc.startBlock
-              });
-          Logging.childWarn(logger, "Skipping contract registration: Contract address is already registered at a later block number. Currently registration of the same contract address is not supported by Envio. Reach out to us if it's a problem for you.");
-        }
-        
-      } else {
-        var registeringContract = registeringContracts[dc.address];
-        var shouldUpdate;
-        if (registeringContract !== undefined) {
-          if (registeringContract.contractName !== dc.contractName) {
-            warnDifferentContractType(fetchState, registeringContract, dc);
-            shouldUpdate = false;
+  for(var itemIdx = 0 ,itemIdx_finish = items.length; itemIdx < itemIdx_finish; ++itemIdx){
+    var item = items[itemIdx];
+    var dcs = item.dcs;
+    if (dcs !== undefined) {
+      for(var idx = 0 ,idx_finish = dcs.length; idx < idx_finish; ++idx){
+        var dc = dcs[idx];
+        var match = fetchState.contractConfigs[dc.contractName];
+        if (match !== undefined) {
+          var existingContract = indexingContracts[dc.address];
+          if (existingContract !== undefined) {
+            if (existingContract.contractName !== dc.contractName) {
+              warnDifferentContractType(fetchState, existingContract, dc);
+            } else if (existingContract.startBlock > dc.startBlock) {
+              var logger = Logging.createChild({
+                    chainId: fetchState.chainId,
+                    contractAddress: dc.address,
+                    existingBlockNumber: existingContract.startBlock,
+                    newBlockNumber: dc.startBlock
+                  });
+              Logging.childWarn(logger, "Skipping contract registration: Contract address is already registered at a later block number. Currently registration of the same contract address is not supported by Envio. Reach out to us if it's a problem for you.");
+            }
+            dcs.splice(idx, 1);
           } else {
-            var match$1 = registeringContract.register;
-            var match$2 = dc.register;
-            shouldUpdate = typeof match$1 !== "object" || typeof match$2 !== "object" ? Js_exn.raiseError("Unexpected case: Config registration should be handled in a different function") : registeringContract.startBlock > dc.startBlock || registeringContract.startBlock === dc.startBlock && match$1.registeringEventLogIndex > match$2.registeringEventLogIndex;
+            var registeringContract = registeringContracts[dc.address];
+            var shouldUpdate;
+            if (registeringContract !== undefined) {
+              if (registeringContract.contractName !== dc.contractName) {
+                warnDifferentContractType(fetchState, registeringContract, dc);
+                shouldUpdate = false;
+              } else {
+                shouldUpdate = false;
+              }
+            } else {
+              hasDCWithFilterByAddresses = hasDCWithFilterByAddresses || match.filterByAddresses;
+              Utils.Dict.push(addressesByContractName, dc.contractName, dc.address);
+              shouldUpdate = true;
+            }
+            if (shouldUpdate) {
+              earliestRegisteringEventBlockNumber = earliestRegisteringEventBlockNumber < dc.startBlock ? earliestRegisteringEventBlockNumber : dc.startBlock;
+              registeringContracts[dc.address] = dc;
+            } else {
+              dcs.splice(idx, 1);
+            }
           }
         } else {
-          hasDCWithFilterByAddresses = hasDCWithFilterByAddresses || match.filterByAddresses;
-          Utils.Dict.push(addressesByContractName, dc.contractName, dc.address);
-          shouldUpdate = true;
-        }
-        if (shouldUpdate) {
-          earliestRegisteringEventBlockNumber = earliestRegisteringEventBlockNumber < dc.startBlock ? earliestRegisteringEventBlockNumber : dc.startBlock;
-          registeringContracts[dc.address] = dc;
+          var logger$1 = Logging.createChild({
+                chainId: fetchState.chainId,
+                contractAddress: dc.address,
+                contractName: dc.contractName
+              });
+          Logging.childWarn(logger$1, "Skipping contract registration: Contract doesn't have any events to fetch.");
+          dcs.splice(idx, 1);
         }
-        
       }
-    } else {
-      var logger$1 = Logging.createChild({
-            chainId: fetchState.chainId,
-            contractAddress: dc.address,
-            contractName: dc.contractName
-          });
-      Logging.childWarn(logger$1, "Skipping contract registration: Contract doesn't have any events to fetch.");
     }
+    
   }
   var dcsToStore = Js_dict.values(registeringContracts);
   if (dcsToStore.length === 0) {
@@ -379,8 +383,7 @@ function registerDynamicContracts(fetchState, dynamicContracts) {
     newPartitions = partitions;
   }
   Prometheus.IndexingAddresses.set(Object.keys(fetchState.indexingContracts).length + dcsToStore.length | 0, fetchState.chainId);
-  var existingDcs = fetchState.dcsToStore;
-  return updateInternal(fetchState, fetchState.partitions.concat(newPartitions), fetchState.nextPartitionIndex + newPartitions.length | 0, Object.assign(registeringContracts, indexingContracts), existingDcs.length !== 0 ? Belt_Array.concat(existingDcs, dcsToStore) : dcsToStore, undefined, undefined);
+  return updateInternal(fetchState, fetchState.partitions.concat(newPartitions), fetchState.nextPartitionIndex + newPartitions.length | 0, Object.assign(registeringContracts, indexingContracts), undefined, undefined);
 }
 
 var UnexpectedPartitionNotFound = /* @__PURE__ */Caml_exceptions.create("FetchState.UnexpectedPartitionNotFound");
@@ -457,7 +460,7 @@ function handleQueryResult(fetchState, query, latestFetchedBlock, newItems) {
     };
   }
   return Belt_Result.map(tmp, (function (partitions) {
-                return updateInternal(fetchState, partitions, undefined, undefined, undefined, newItems.length !== 0 ? Belt_Array.concat(fetchState.buffer, newItems) : undefined, undefined);
+                return updateInternal(fetchState, partitions, undefined, undefined, newItems.length !== 0 ? Belt_Array.concat(fetchState.buffer, newItems) : undefined, undefined);
               }));
 }
 
@@ -835,7 +838,6 @@ function make(startBlock, endBlock, eventConfigs, contracts, maxAddrInPartition,
           normalSelection: normalSelection,
           indexingContracts: indexingContracts,
           contractConfigs: contractConfigs,
-          dcsToStore: [],
           chainId: chainId,
           latestFullyFetchedBlock: latestFetchedBlock,
           latestOnBlockBlockNumber: progressBlockNumber,
@@ -850,27 +852,10 @@ function bufferSize(param) {
   return param.buffer.length;
 }
 
-function pruneQueueFromFirstChangeEvent(buffer, firstChangeEvent) {
-  return Belt_Array.keep(buffer, (function (item) {
-                var tmp;
-                tmp = item.kind === 0 ? [
-                    item.blockNumber,
-                    item.logIndex
-                  ] : [
-                    item.blockNumber,
-                    item.logIndex
-                  ];
-                return Caml_obj.lessthan(tmp, [
-                            firstChangeEvent.blockNumber,
-                            firstChangeEvent.logIndex
-                          ]);
-              }));
-}
-
-function rollbackPartition(p, firstChangeEvent, addressesToRemove) {
-  var shouldRollbackFetched = p.latestFetchedBlock.blockNumber >= firstChangeEvent.blockNumber;
+function rollbackPartition(p, targetBlockNumber, addressesToRemove) {
+  var shouldRollbackFetched = p.latestFetchedBlock.blockNumber > targetBlockNumber;
   var latestFetchedBlock = shouldRollbackFetched ? ({
-        blockNumber: firstChangeEvent.blockNumber - 1 | 0,
+        blockNumber: targetBlockNumber,
         blockTimestamp: 0
       }) : p.latestFetchedBlock;
   if (!p.selection.dependsOnAddresses) {
@@ -885,7 +870,7 @@ function rollbackPartition(p, firstChangeEvent, addressesToRemove) {
           };
   }
   var rollbackedAddressesByContractName = {};
-  Utils.Dict.forEachWithKey(p.addressesByContractName, (function (contractName, addresses) {
+  Utils.Dict.forEachWithKey(p.addressesByContractName, (function (addresses, contractName) {
           var keptAddresses = Belt_Array.keep(addresses, (function (address) {
                   return !addressesToRemove.has(address);
                 }));
@@ -910,24 +895,21 @@ function rollbackPartition(p, firstChangeEvent, addressesToRemove) {
   }
 }
 
-function rollback(fetchState, firstChangeEvent) {
+function rollback(fetchState, targetBlockNumber) {
   var addressesToRemove = new Set();
   var indexingContracts = {};
   Belt_Array.forEach(Object.keys(fetchState.indexingContracts), (function (address) {
           var indexingContract = fetchState.indexingContracts[address];
-          var dc = indexingContract.register;
-          var tmp;
-          tmp = typeof dc !== "object" ? true : indexingContract.startBlock < firstChangeEvent.blockNumber || indexingContract.startBlock === firstChangeEvent.blockNumber && dc.registeringEventLogIndex < firstChangeEvent.logIndex;
-          if (tmp) {
-            indexingContracts[address] = indexingContract;
-          } else {
+          var registrationBlock = indexingContract.registrationBlock;
+          if (registrationBlock !== undefined && registrationBlock > targetBlockNumber) {
             addressesToRemove.add(address);
+          } else {
+            indexingContracts[address] = indexingContract;
           }
         }));
   var partitions = Belt_Array.keepMap(fetchState.partitions, (function (p) {
-          return rollbackPartition(p, firstChangeEvent, addressesToRemove);
+          return rollbackPartition(p, targetBlockNumber, addressesToRemove);
         }));
-  var empty = fetchState.dcsToStore;
   return updateInternal({
               partitions: fetchState.partitions,
               nextPartitionIndex: fetchState.nextPartitionIndex,
@@ -937,17 +919,18 @@ function rollback(fetchState, firstChangeEvent) {
               normalSelection: fetchState.normalSelection,
               indexingContracts: fetchState.indexingContracts,
               contractConfigs: fetchState.contractConfigs,
-              dcsToStore: fetchState.dcsToStore,
               chainId: fetchState.chainId,
               latestFullyFetchedBlock: fetchState.latestFullyFetchedBlock,
-              latestOnBlockBlockNumber: firstChangeEvent.blockNumber - 1 | 0,
+              latestOnBlockBlockNumber: targetBlockNumber,
               blockLag: fetchState.blockLag,
               buffer: fetchState.buffer,
               targetBufferSize: fetchState.targetBufferSize,
               onBlockConfigs: fetchState.onBlockConfigs
-            }, partitions, undefined, indexingContracts, empty.length !== 0 ? empty.filter(function (dc) {
-                    return !addressesToRemove.has(dc.address);
-                  }) : empty, pruneQueueFromFirstChangeEvent(fetchState.buffer, firstChangeEvent), undefined);
+            }, partitions, undefined, indexingContracts, Belt_Array.keep(fetchState.buffer, (function (item) {
+                    var tmp;
+                    tmp = item.kind === 0 ? item.blockNumber : item.blockNumber;
+                    return tmp <= targetBlockNumber;
+                  })), undefined);
 }
 
 function isActivelyIndexing(fetchState) {
@@ -994,33 +977,61 @@ function hasFullBatch(fetchState, batchSizeTarget) {
         );
 }
 
-function filterAndSortForUnorderedBatch(fetchStates, batchSizeTarget) {
-  return Belt_Array.keepU(fetchStates, hasReadyItem).sort(function (a, b) {
+function sortForUnorderedBatch(fetchStates, batchSizeTarget) {
+  return fetchStates.slice(0).sort(function (a, b) {
               var match = hasFullBatch(a, batchSizeTarget);
               var match$1 = hasFullBatch(b, batchSizeTarget);
+              var exit = 0;
               if (match) {
                 if (!match$1) {
                   return -1;
                 }
-                
-              } else if (match$1) {
-                return 1;
-              }
-              var match$2 = a.buffer[0];
-              var match$3 = b.buffer[0];
-              if (match$2.kind === 0 && match$3.kind === 0) {
-                return match$2.timestamp - match$3.timestamp | 0;
+                exit = 1;
               } else {
-                return Js_math.random_int(-1, 1);
+                if (match$1) {
+                  return 1;
+                }
+                exit = 1;
               }
+              if (exit === 1) {
+                var match$2 = Belt_Array.get(a.buffer, 0);
+                var match$3 = Belt_Array.get(b.buffer, 0);
+                var exit$1 = 0;
+                if (match$2 !== undefined) {
+                  if (match$2.kind !== 0) {
+                    return Js_math.random_int(-1, 1);
+                  }
+                  if (match$3 === undefined) {
+                    return -1;
+                  }
+                  if (match$3.kind === 0) {
+                    return match$2.timestamp - match$3.timestamp | 0;
+                  }
+                  exit$1 = 2;
+                } else {
+                  if (match$3 === undefined) {
+                    return 0;
+                  }
+                  exit$1 = 2;
+                }
+                if (exit$1 === 2) {
+                  if (match$3.kind === 0) {
+                    return 1;
+                  } else {
+                    return Js_math.random_int(-1, 1);
+                  }
+                }
+                
+              }
+              
             });
 }
 
-function getProgressBlockNumber(fetchState) {
+function getUnorderedMultichainProgressBlockNumberAt(fetchState, index) {
   var latestOnBlockBlockNumber = fetchState.latestOnBlockBlockNumber;
   var latestFullyFetchedBlock = fetchState.latestFullyFetchedBlock;
   var bufferBlockNumber = latestOnBlockBlockNumber < latestFullyFetchedBlock.blockNumber ? latestOnBlockBlockNumber : latestFullyFetchedBlock.blockNumber;
-  var item = Belt_Array.get(fetchState.buffer, 0);
+  var item = Belt_Array.get(fetchState.buffer, index);
   if (item !== undefined && bufferBlockNumber >= item.blockNumber) {
     return item.blockNumber - 1 | 0;
   } else {
@@ -1053,11 +1064,10 @@ exports.hasReadyItem = hasReadyItem;
 exports.getReadyItemsCount = getReadyItemsCount;
 exports.make = make;
 exports.bufferSize = bufferSize;
-exports.pruneQueueFromFirstChangeEvent = pruneQueueFromFirstChangeEvent;
 exports.rollbackPartition = rollbackPartition;
 exports.rollback = rollback;
 exports.isActivelyIndexing = isActivelyIndexing;
 exports.isReadyToEnterReorgThreshold = isReadyToEnterReorgThreshold;
-exports.filterAndSortForUnorderedBatch = filterAndSortForUnorderedBatch;
-exports.getProgressBlockNumber = getProgressBlockNumber;
+exports.sortForUnorderedBatch = sortForUnorderedBatch;
+exports.getUnorderedMultichainProgressBlockNumberAt = getUnorderedMultichainProgressBlockNumberAt;
 /* Utils Not a pure module */

package/src/Internal.res

@@ -145,6 +145,18 @@ type evmContractConfig = {
   events: array<evmEventConfig>,
 }
 
+type indexingContract = {
+  address: Address.t,
+  contractName: string,
+  startBlock: int,
+  // Needed for rollback
+  // If not set, assume the contract comes from config
+  // and shouldn't be rolled back
+  registrationBlock: option<int>,
+}
+
+type dcs = array<indexingContract>
+
 // Duplicate the type from item
 // to make item properly unboxed
 type eventItem = private {
@@ -200,6 +212,11 @@ external getItemBlockNumber: item => int = "blockNumber"
 @get
 external getItemLogIndex: item => int = "logIndex"
 
+@get
+external getItemDcs: item => option<dcs> = "dcs"
+@set
+external setItemDcs: (item, dcs) => unit = "dcs"
+
 @genType
 type eventOptions<'eventFilters> = {
   wildcard?: bool,
@@ -297,3 +314,35 @@ let makeCacheTable = (~effectName) => {
 
 @genType.import(("./Types.ts", "Invalid"))
 type noEventFilters
+
+type reorgCheckpoint = {
+  @as("id")
+  checkpointId: int,
+  @as("chain_id")
+  chainId: int,
+  @as("block_number")
+  blockNumber: int,
+  @as("block_hash")
+  blockHash: string,
+}
+
+type entityValueAtStartOfBatch<'entityType> =
+  | NotSet // The entity isn't in the DB yet
+  | AlreadySet('entityType)
+
+type updatedValue<'entityType> = {
+  latest: EntityHistory.entityUpdate<'entityType>,
+  history: array<EntityHistory.entityUpdate<'entityType>>,
+  // In the event of a rollback, some entity updates may have been
+  // been affected by a rollback diff. If there was no rollback diff
+  // this will always be false.
+  // If there was a rollback diff, this will be false in the case of a
+  // new entity update (where entity affected is not present in the diff) b
+  // but true if the update is related to an entity that is
+  // currently present in the diff
+  containsRollbackDiffChange: bool,
+}
+
+type inMemoryStoreRowEntity<'entityType> =
+  | Updated(updatedValue<'entityType>)
+  | InitialReadFromDb(entityValueAtStartOfBatch<'entityType>) // This means there is no change from the db.

package/src/InternalConfig.res

@@ -30,7 +30,7 @@ type chain = {
   id: int,
   startBlock: int,
   endBlock?: int,
-  confirmedBlockThreshold: int,
+  maxReorgDepth: int,
   contracts: array<contract>,
   sources: array<Source.t>,
 }

package/src/Persistence.res

@@ -13,10 +13,25 @@ type effectCacheRecord = {
   mutable count: int,
 }
 
+type initialChainState = {
+  id: int,
+  startBlock: int,
+  endBlock: option<int>,
+  maxReorgDepth: int,
+  progressBlockNumber: int,
+  numEventsProcessed: int,
+  firstEventBlockNumber: option<int>,
+  timestampCaughtUpToHeadOrEndblock: option<Js.Date.t>,
+  dynamicContracts: array<Internal.indexingContract>,
+}
+
 type initialState = {
   cleanRun: bool,
   cache: dict<effectCacheRecord>,
-  chains: array<InternalTable.Chains.t>,
+  chains: array<initialChainState>,
+  checkpointId: int,
+  // Needed to keep reorg detection logic between restarts
+  reorgCheckpoints: array<Internal.reorgCheckpoint>,
 }
 
 type operator = [#">" | #"="]

package/src/Persistence.res.js

@@ -84,7 +84,7 @@ async function init(persistence, chainConfigs, resetOpt) {
         };
         var checkpoints = {};
         initialState$1.chains.forEach(function (c) {
-              checkpoints[c.id] = c.progress_block;
+              checkpoints[c.id] = c.progressBlockNumber;
             });
         Logging.info({
               msg: "Successfully resumed indexing state! Continuing from the last checkpoint.",