envio 3.2.1 → 3.3.0-alpha.0

package/src/FetchState.res.mjs

@@ -449,10 +449,10 @@ function numAddresses(fetchState) {
   return Utils.Dict.size(fetchState.indexingAddresses);
 }
 
-function appendOnBlockItems(mutItems, onBlockConfigs, indexerStartBlock, fromBlock, maxBlockNumber, targetBufferSize) {
+function appendOnBlockItems(mutItems, onBlockConfigs, indexerStartBlock, fromBlock, maxBlockNumber, maxOnBlockBufferSize) {
   let newItemsCounter = 0;
   let latestOnBlockBlockNumber = fromBlock;
-  while (latestOnBlockBlockNumber < maxBlockNumber && newItemsCounter <= targetBufferSize) {
+  while (latestOnBlockBlockNumber < maxBlockNumber && newItemsCounter <= maxOnBlockBufferSize) {
     let blockNumber = latestOnBlockBlockNumber + 1 | 0;
     latestOnBlockBlockNumber = blockNumber;
     for (let configIdx = 0, configIdx_finish = onBlockConfigs.length; configIdx < configIdx_finish; ++configIdx) {
@@ -490,7 +490,7 @@ function updateInternal(fetchState, optimizedPartitionsOpt, indexingAddressesOpt
     let mutItems$1 = mutItemsRef;
     let item = (
         mutItems$1 !== undefined ? mutItems$1 : fetchState.buffer
-      )[fetchState.targetBufferSize - 1 | 0];
+      )[fetchState.maxOnBlockBufferSize - 1 | 0];
     let maxBlockNumber;
     if (item !== undefined) {
       maxBlockNumber = item.blockNumber;
@@ -502,7 +502,7 @@ function updateInternal(fetchState, optimizedPartitionsOpt, indexingAddressesOpt
     let mutItems$2 = mutItemsRef;
     let mutItems$3 = mutItems$2 !== undefined ? mutItems$2 : fetchState.buffer.slice();
     mutItemsRef = mutItems$3;
-    latestOnBlockBlockNumber = appendOnBlockItems(mutItems$3, onBlockConfigs, fetchState.startBlock, fetchState.latestOnBlockBlockNumber, maxBlockNumber, fetchState.targetBufferSize);
+    latestOnBlockBlockNumber = appendOnBlockItems(mutItems$3, onBlockConfigs, fetchState.startBlock, fetchState.latestOnBlockBlockNumber, maxBlockNumber, fetchState.maxOnBlockBufferSize);
   } else {
     latestOnBlockBlockNumber = knownHeight;
   }
@@ -513,7 +513,7 @@ function updateInternal(fetchState, optimizedPartitionsOpt, indexingAddressesOpt
   let updatedFetchState_contractConfigs = fetchState.contractConfigs;
   let updatedFetchState_chainId = fetchState.chainId;
   let updatedFetchState_buffer = mutItems$4 !== undefined ? (mutItems$4.sort(compareBufferItem), mutItems$4) : fetchState.buffer;
-  let updatedFetchState_targetBufferSize = fetchState.targetBufferSize;
+  let updatedFetchState_maxOnBlockBufferSize = fetchState.maxOnBlockBufferSize;
   let updatedFetchState_onBlockConfigs = fetchState.onBlockConfigs;
   let updatedFetchState_firstEventBlock = fetchState.firstEventBlock;
   let updatedFetchState = {
@@ -527,7 +527,7 @@ function updateInternal(fetchState, optimizedPartitionsOpt, indexingAddressesOpt
     latestOnBlockBlockNumber: latestOnBlockBlockNumber,
     blockLag: blockLag,
     buffer: updatedFetchState_buffer,
-    targetBufferSize: updatedFetchState_targetBufferSize,
+    maxOnBlockBufferSize: updatedFetchState_maxOnBlockBufferSize,
     onBlockConfigs: updatedFetchState_onBlockConfigs,
     knownHeight: knownHeight,
     firstEventBlock: updatedFetchState_firstEventBlock
@@ -903,7 +903,7 @@ function handleQueryResult(fetchState, query, latestFetchedBlock, newItems) {
     }
     let filter = item.eventConfig.clientAddressFilter;
     if (filter !== undefined) {
-      return filter(item.event, item.blockNumber, fetchState.indexingAddresses);
+      return filter(item.payload, item.blockNumber, fetchState.indexingAddresses);
     } else {
       return true;
     }
@@ -949,36 +949,42 @@ function pushQueriesForRange(queries, partitionId, rangeFromBlock, rangeEndBlock
   if (maybeChunkRange !== undefined) {
     let maxBlock = rangeEndBlock !== undefined ? rangeEndBlock : maxQueryBlockNumber;
     let chunkSize = Js_math.ceil_int(maybeChunkRange * 1.8);
-    if (((rangeFromBlock + (chunkSize << 1) | 0) - 1 | 0) <= maxBlock) {
-      queries.push({
-        partitionId: partitionId,
-        fromBlock: rangeFromBlock,
-        toBlock: (rangeFromBlock + chunkSize | 0) - 1 | 0,
-        isChunk: true,
-        selection: selection,
-        addressesByContractName: addressesByContractName,
-        indexingAddresses: indexingAddresses
-      });
-      queries.push({
-        partitionId: partitionId,
-        fromBlock: rangeFromBlock + chunkSize | 0,
-        toBlock: (rangeFromBlock + (chunkSize << 1) | 0) - 1 | 0,
-        isChunk: true,
-        selection: selection,
-        addressesByContractName: addressesByContractName,
-        indexingAddresses: indexingAddresses
-      });
-    } else {
-      queries.push({
-        partitionId: partitionId,
-        fromBlock: rangeFromBlock,
-        toBlock: rangeEndBlock,
-        isChunk: rangeEndBlock !== undefined,
-        selection: selection,
-        addressesByContractName: addressesByContractName,
-        indexingAddresses: indexingAddresses
-      });
+    let probeSize = Js_math.ceil_int(maybeChunkRange * 0.9);
+    let getChunkSize = chunkIdx => {
+      if (chunkIdx < 2) {
+        return probeSize;
+      } else {
+        return chunkSize;
+      }
+    };
+    if ((((rangeFromBlock + probeSize | 0) + probeSize | 0) - 1 | 0) <= maxBlock) {
+      let chunkFromBlock = rangeFromBlock;
+      let chunkIdx = 0;
+      while (chunkIdx < 5 && ((chunkFromBlock + getChunkSize(chunkIdx) | 0) - 1 | 0) <= maxBlock) {
+        let chunkToBlock = (chunkFromBlock + getChunkSize(chunkIdx) | 0) - 1 | 0;
+        queries.push({
+          partitionId: partitionId,
+          fromBlock: chunkFromBlock,
+          toBlock: chunkToBlock,
+          isChunk: true,
+          selection: selection,
+          addressesByContractName: addressesByContractName,
+          indexingAddresses: indexingAddresses
+        });
+        chunkFromBlock = chunkToBlock + 1 | 0;
+        chunkIdx = chunkIdx + 1 | 0;
+      };
+      return;
     }
+    queries.push({
+      partitionId: partitionId,
+      fromBlock: rangeFromBlock,
+      toBlock: rangeEndBlock,
+      isChunk: rangeEndBlock !== undefined,
+      selection: selection,
+      addressesByContractName: addressesByContractName,
+      indexingAddresses: indexingAddresses
+    });
     return;
   }
   queries.push({
@@ -992,7 +998,7 @@ function pushQueriesForRange(queries, partitionId, rangeFromBlock, rangeEndBlock
   });
 }
 
-function getNextQuery(fetchState, concurrencyLimit) {
+function getNextQuery(fetchState, concurrencyLimit, bufferLimit) {
   let knownHeight = fetchState.knownHeight;
   let blockLag = fetchState.blockLag;
   let indexingAddresses = fetchState.indexingAddresses;
@@ -1004,12 +1010,15 @@ function getNextQuery(fetchState, concurrencyLimit) {
   if (concurrencyLimit === 0) {
     return "ReachedMaxConcurrency";
   }
+  if (bufferLimit <= 0) {
+    return "NothingToQuery";
+  }
   let isOnBlockBehindTheHead = fetchState.latestOnBlockBlockNumber < headBlockNumber;
   let endBlock = fetchState.endBlock;
   let shouldWaitForNewBlock = (
     endBlock !== undefined ? headBlockNumber < endBlock : true
   ) && !isOnBlockBehindTheHead;
-  let item = fetchState.buffer[fetchState.targetBufferSize - 1 | 0];
+  let item = fetchState.buffer[bufferLimit - 1 | 0];
   let maxQueryBlockNumber = item !== undefined ? Primitive_int.min(item.blockNumber, knownHeight) : knownHeight;
   let queries = [];
   let partitionsCount = optimizedPartitions.idsInAscOrder.length;
@@ -1018,11 +1027,13 @@ function getNextQuery(fetchState, concurrencyLimit) {
     let idx = idxRef;
     let partitionId = optimizedPartitions.idsInAscOrder[idx];
     let p = optimizedPartitions.entities[partitionId];
+    let pendingCount = p.mutPendingQueries.length;
     let isBehindTheHead = p.latestFetchedBlock.blockNumber < headBlockNumber;
-    let hasPendingQueries = Utils.$$Array.notEmpty(p.mutPendingQueries);
+    let hasPendingQueries = pendingCount > 0;
     if (hasPendingQueries || isBehindTheHead) {
       shouldWaitForNewBlock = false;
     }
+    let partitionQueriesStart = queries.length;
     let queryEndBlock = Utils.$$Math.minOptInt(fetchState.endBlock, p.mergeBlock);
     let queryEndBlock$1 = blockLag !== 0 ? Utils.$$Math.minOptInt(headBlockNumber, queryEndBlock) : queryEndBlock;
     let match = maxQueryBlockNumber < knownHeight;
@@ -1051,25 +1062,31 @@ function getNextQuery(fetchState, concurrencyLimit) {
             if (maybeChunkRange !== undefined) {
               let maxBlock = rangeEndBlock !== undefined ? rangeEndBlock : maxQueryBlockNumber;
               let chunkSize = Js_math.ceil_int(maybeChunkRange * 1.8);
-              if (((rangeFromBlock + (chunkSize << 1) | 0) - 1 | 0) <= maxBlock) {
-                queries.push({
-                  partitionId: partitionId,
-                  fromBlock: rangeFromBlock,
-                  toBlock: (rangeFromBlock + chunkSize | 0) - 1 | 0,
-                  isChunk: true,
-                  selection: selection,
-                  addressesByContractName: addressesByContractName,
-                  indexingAddresses: indexingAddresses
-                });
-                queries.push({
-                  partitionId: partitionId,
-                  fromBlock: rangeFromBlock + chunkSize | 0,
-                  toBlock: (rangeFromBlock + (chunkSize << 1) | 0) - 1 | 0,
-                  isChunk: true,
-                  selection: selection,
-                  addressesByContractName: addressesByContractName,
-                  indexingAddresses: indexingAddresses
-                });
+              let probeSize = Js_math.ceil_int(maybeChunkRange * 0.9);
+              let getChunkSize = chunkIdx => {
+                if (chunkIdx < 2) {
+                  return probeSize;
+                } else {
+                  return chunkSize;
+                }
+              };
+              if ((((rangeFromBlock + getChunkSize(0) | 0) + getChunkSize(1) | 0) - 1 | 0) <= maxBlock) {
+                let chunkFromBlock = rangeFromBlock;
+                let chunkIdx = 0;
+                while (chunkIdx < 5 && ((chunkFromBlock + getChunkSize(chunkIdx) | 0) - 1 | 0) <= maxBlock) {
+                  let chunkToBlock = (chunkFromBlock + getChunkSize(chunkIdx) | 0) - 1 | 0;
+                  queries.push({
+                    partitionId: partitionId,
+                    fromBlock: chunkFromBlock,
+                    toBlock: chunkToBlock,
+                    isChunk: true,
+                    selection: selection,
+                    addressesByContractName: addressesByContractName,
+                    indexingAddresses: indexingAddresses
+                  });
+                  chunkFromBlock = chunkToBlock + 1 | 0;
+                  chunkIdx = chunkIdx + 1 | 0;
+                };
               } else {
                 queries.push({
                   partitionId: partitionId,
@@ -1122,25 +1139,31 @@ function getNextQuery(fetchState, concurrencyLimit) {
           if (maybeChunkRange !== undefined) {
             let maxBlock$1 = queryEndBlock$2 !== undefined ? queryEndBlock$2 : maxQueryBlockNumber;
             let chunkSize$1 = Js_math.ceil_int(maybeChunkRange * 1.8);
-            if (((rangeFromBlock$1 + (chunkSize$1 << 1) | 0) - 1 | 0) <= maxBlock$1) {
-              queries.push({
-                partitionId: partitionId,
-                fromBlock: rangeFromBlock$1,
-                toBlock: (rangeFromBlock$1 + chunkSize$1 | 0) - 1 | 0,
-                isChunk: true,
-                selection: selection$1,
-                addressesByContractName: addressesByContractName$1,
-                indexingAddresses: indexingAddresses
-              });
-              queries.push({
-                partitionId: partitionId,
-                fromBlock: rangeFromBlock$1 + chunkSize$1 | 0,
-                toBlock: (rangeFromBlock$1 + (chunkSize$1 << 1) | 0) - 1 | 0,
-                isChunk: true,
-                selection: selection$1,
-                addressesByContractName: addressesByContractName$1,
-                indexingAddresses: indexingAddresses
-              });
+            let probeSize$1 = Js_math.ceil_int(maybeChunkRange * 0.9);
+            let getChunkSize$1 = chunkIdx => {
+              if (chunkIdx < 2) {
+                return probeSize$1;
+              } else {
+                return chunkSize$1;
+              }
+            };
+            if ((((rangeFromBlock$1 + getChunkSize$1(0) | 0) + getChunkSize$1(1) | 0) - 1 | 0) <= maxBlock$1) {
+              let chunkFromBlock$1 = rangeFromBlock$1;
+              let chunkIdx$1 = 0;
+              while (chunkIdx$1 < 5 && ((chunkFromBlock$1 + getChunkSize$1(chunkIdx$1) | 0) - 1 | 0) <= maxBlock$1) {
+                let chunkToBlock$1 = (chunkFromBlock$1 + getChunkSize$1(chunkIdx$1) | 0) - 1 | 0;
+                queries.push({
+                  partitionId: partitionId,
+                  fromBlock: chunkFromBlock$1,
+                  toBlock: chunkToBlock$1,
+                  isChunk: true,
+                  selection: selection$1,
+                  addressesByContractName: addressesByContractName$1,
+                  indexingAddresses: indexingAddresses
+                });
+                chunkFromBlock$1 = chunkToBlock$1 + 1 | 0;
+                chunkIdx$1 = chunkIdx$1 + 1 | 0;
+              };
             } else {
               queries.push({
                 partitionId: partitionId,
@@ -1166,6 +1189,11 @@ function getNextQuery(fetchState, concurrencyLimit) {
         }
       }
     }
+    let maxNewChunks = Primitive_int.max(0, 10 - pendingCount | 0);
+    let generatedCount = queries.length - partitionQueriesStart | 0;
+    if (generatedCount > maxNewChunks) {
+      queries.splice(partitionQueriesStart + maxNewChunks | 0, generatedCount - maxNewChunks | 0);
+    }
     idxRef = idxRef + 1 | 0;
   };
   if (Utils.$$Array.isEmpty(queries)) {
@@ -1214,7 +1242,7 @@ function getReadyItemsCount(fetchState, targetSize, fromItem) {
   return acc;
 }
 
-function make$1(startBlock, endBlock, eventConfigs, addresses, maxAddrInPartition, chainId, targetBufferSize, knownHeight, progressBlockNumberOpt, onBlockConfigsOpt, blockLagOpt, firstEventBlockOpt) {
+function make$1(startBlock, endBlock, eventConfigs, addresses, maxAddrInPartition, chainId, maxOnBlockBufferSize, knownHeight, progressBlockNumberOpt, onBlockConfigsOpt, blockLagOpt, firstEventBlockOpt) {
   let progressBlockNumber = progressBlockNumberOpt !== undefined ? progressBlockNumberOpt : startBlock - 1 | 0;
   let onBlockConfigs = onBlockConfigsOpt !== undefined ? onBlockConfigsOpt : [];
   let blockLag = blockLagOpt !== undefined ? blockLagOpt : 0;
@@ -1311,7 +1339,7 @@ function make$1(startBlock, endBlock, eventConfigs, addresses, maxAddrInPartitio
     let id = optimizedPartitions.idsInAscOrder[0];
     let latestFullyFetchedBlock = id !== undefined ? optimizedPartitions.entities[id].latestFetchedBlock : undefined;
     let maxBlockNumber = latestFullyFetchedBlock !== undefined ? latestFullyFetchedBlock.blockNumber : knownHeight;
-    latestOnBlockBlockNumber = appendOnBlockItems(buffer, onBlockConfigs, startBlock, progressBlockNumber, maxBlockNumber, targetBufferSize);
+    latestOnBlockBlockNumber = appendOnBlockItems(buffer, onBlockConfigs, startBlock, progressBlockNumber, maxBlockNumber, maxOnBlockBufferSize);
   } else {
     latestOnBlockBlockNumber = progressBlockNumber;
   }
@@ -1326,7 +1354,7 @@ function make$1(startBlock, endBlock, eventConfigs, addresses, maxAddrInPartitio
     latestOnBlockBlockNumber: latestOnBlockBlockNumber,
     blockLag: blockLag,
     buffer: buffer,
-    targetBufferSize: targetBufferSize,
+    maxOnBlockBufferSize: maxOnBlockBufferSize,
     onBlockConfigs: onBlockConfigs,
     knownHeight: knownHeight,
     firstEventBlock: firstEventBlock
@@ -1346,6 +1374,22 @@ function bufferSize(param) {
   return param.buffer.length;
 }
 
+function bufferReadyCount(fetchState) {
+  let frontier = bufferBlockNumber(fetchState);
+  let buffer = fetchState.buffer;
+  let lo = 0;
+  let hi = buffer.length;
+  while (lo < hi) {
+    let mid = (lo + hi | 0) / 2 | 0;
+    if (buffer[mid].blockNumber <= frontier) {
+      lo = mid + 1 | 0;
+    } else {
+      hi = mid;
+    }
+  };
+  return lo;
+}
+
 function rollbackPendingQueries(mutPendingQueries, targetBlockNumber) {
   let adjusted = [];
   for (let qIdx = 0, qIdx_finish = mutPendingQueries.length; qIdx < qIdx_finish; ++qIdx) {
@@ -1461,7 +1505,7 @@ function rollback(fetchState, targetBlockNumber) {
     latestOnBlockBlockNumber: Primitive_int.min(fetchState.latestOnBlockBlockNumber, targetBlockNumber),
     blockLag: fetchState.blockLag,
     buffer: fetchState.buffer,
-    targetBufferSize: fetchState.targetBufferSize,
+    maxOnBlockBufferSize: fetchState.maxOnBlockBufferSize,
     onBlockConfigs: fetchState.onBlockConfigs,
     knownHeight: fetchState.knownHeight,
     firstEventBlock: fetchState.firstEventBlock
@@ -1511,7 +1555,7 @@ function resetPendingQueries(fetchState) {
     latestOnBlockBlockNumber: fetchState.latestOnBlockBlockNumber,
     blockLag: fetchState.blockLag,
     buffer: fetchState.buffer,
-    targetBufferSize: fetchState.targetBufferSize,
+    maxOnBlockBufferSize: fetchState.maxOnBlockBufferSize,
     onBlockConfigs: fetchState.onBlockConfigs,
     knownHeight: fetchState.knownHeight,
     firstEventBlock: fetchState.firstEventBlock
@@ -1531,24 +1575,31 @@ function isActivelyIndexing(fetchState) {
   }
 }
 
-function isReadyToEnterReorgThreshold(fetchState) {
+function isFetchingAtHead(fetchState) {
   let knownHeight = fetchState.knownHeight;
   let blockLag = fetchState.blockLag;
   let endBlock = fetchState.endBlock;
   let bufferBlockNumber$1 = bufferBlockNumber(fetchState);
-  if (knownHeight !== 0 && (
-      endBlock !== undefined && bufferBlockNumber$1 >= endBlock ? true : bufferBlockNumber$1 >= (knownHeight - blockLag | 0)
-    )) {
-    return Utils.$$Array.isEmpty(fetchState.buffer);
+  if (knownHeight !== 0) {
+    if (endBlock !== undefined && bufferBlockNumber$1 >= endBlock) {
+      return true;
+    } else {
+      return bufferBlockNumber$1 >= (knownHeight - blockLag | 0);
+    }
   } else {
     return false;
   }
 }
 
-function hasFullBatch(fetchState, batchSizeTarget) {
-  let item = fetchState.buffer[batchSizeTarget - 1 | 0];
-  if (item !== undefined) {
-    return item.blockNumber <= bufferBlockNumber(fetchState);
+function isReadyToEnterReorgThreshold(fetchState) {
+  let knownHeight = fetchState.knownHeight;
+  let blockLag = fetchState.blockLag;
+  let endBlock = fetchState.endBlock;
+  let bufferBlockNumber$1 = bufferBlockNumber(fetchState);
+  if (knownHeight !== 0 && (
+      endBlock !== undefined && bufferBlockNumber$1 >= endBlock ? true : bufferBlockNumber$1 >= (knownHeight - blockLag | 0)
+    )) {
+    return Utils.$$Array.isEmpty(fetchState.buffer);
   } else {
     return false;
   }
@@ -1568,7 +1619,16 @@ function getProgressPercentage(fetchState) {
   return progress / totalRange;
 }
 
-function sortForUnorderedBatch(fetchStates, batchSizeTarget) {
+function hasFullBatch(fetchState, batchSizeTarget) {
+  let item = fetchState.buffer[batchSizeTarget - 1 | 0];
+  if (item !== undefined) {
+    return item.blockNumber <= bufferBlockNumber(fetchState);
+  } else {
+    return false;
+  }
+}
+
+function sortForBatch(fetchStates, batchSizeTarget) {
   let copied = fetchStates.slice();
   copied.sort((a, b) => {
     let match = hasFullBatch(a, batchSizeTarget);
@@ -1614,6 +1674,8 @@ function updateKnownHeight(fetchState, knownHeight) {
 
 let blockItemLogIndex = 16777216;
 
+let maxPendingChunksPerPartition = 10;
+
 export {
   deriveEffectiveStartBlock,
   getMinHistoryRange,
@@ -1634,17 +1696,21 @@ export {
   handleQueryResult,
   startFetchingQueries,
   pushQueriesForRange,
+  maxPendingChunksPerPartition,
   getNextQuery,
   hasReadyItem,
   getReadyItemsCount,
   make$1 as make,
   bufferSize,
+  bufferReadyCount,
   rollbackPendingQueries,
   rollback,
   resetPendingQueries,
   isActivelyIndexing,
+  isFetchingAtHead,
   isReadyToEnterReorgThreshold,
-  sortForUnorderedBatch,
+  getProgressPercentage,
+  sortForBatch,
   getProgressBlockNumberAt,
   updateKnownHeight,
 }