envio 3.2.1 → 3.3.0-alpha.1

package/src/FetchState.res.mjs

@@ -14,6 +14,14 @@ function deriveEffectiveStartBlock(registrationBlock, contractStartBlock) {
   return Primitive_int.max(Primitive_int.max(registrationBlock, 0), Stdlib_Option.getOr(contractStartBlock, 0));
 }
 
+function calculateEstResponseSize(p, fromBlock, toBlock, maxQueryBlockNumber) {
+  if (p.prevQueryRange <= 0) {
+    return 10000;
+  }
+  let density = p.prevRangeSize / p.prevQueryRange;
+  return ((Stdlib_Option.getOr(toBlock, maxQueryBlockNumber) - fromBlock | 0) + 1 | 0) * density;
+}
+
 function getMinHistoryRange(p) {
   let match = p.prevQueryRange;
   let match$1 = p.prevPrevQueryRange;
@@ -69,6 +77,7 @@ function mergePartitionsAtBlock(p1, p2, potentialMergeBlock, contractName, maxAd
         mutPendingQueries: p1.mutPendingQueries,
         prevQueryRange: p1.prevQueryRange,
         prevPrevQueryRange: p1.prevPrevQueryRange,
+        prevRangeSize: p1.prevRangeSize,
         latestBlockRangeUpdateBlock: p1.latestBlockRangeUpdateBlock
       });
       completed.push({
@@ -81,6 +90,7 @@ function mergePartitionsAtBlock(p1, p2, potentialMergeBlock, contractName, maxAd
         mutPendingQueries: p2.mutPendingQueries,
         prevQueryRange: p2.prevQueryRange,
         prevPrevQueryRange: p2.prevPrevQueryRange,
+        prevRangeSize: p2.prevRangeSize,
         latestBlockRangeUpdateBlock: p2.latestBlockRangeUpdateBlock
       });
       let newId = nextPartitionIndexRef.contents.toString();
@@ -102,6 +112,7 @@ function mergePartitionsAtBlock(p1, p2, potentialMergeBlock, contractName, maxAd
         mutPendingQueries: [],
         prevQueryRange: minRange,
         prevPrevQueryRange: minRange,
+        prevRangeSize: 0,
         latestBlockRangeUpdateBlock: 0
       };
     } else {
@@ -115,6 +126,7 @@ function mergePartitionsAtBlock(p1, p2, potentialMergeBlock, contractName, maxAd
         mutPendingQueries: p1.mutPendingQueries,
         prevQueryRange: p1.prevQueryRange,
         prevPrevQueryRange: p1.prevPrevQueryRange,
+        prevRangeSize: p1.prevRangeSize,
         latestBlockRangeUpdateBlock: p1.latestBlockRangeUpdateBlock
       });
       continuingBase = p2;
@@ -130,6 +142,7 @@ function mergePartitionsAtBlock(p1, p2, potentialMergeBlock, contractName, maxAd
       mutPendingQueries: p2.mutPendingQueries,
       prevQueryRange: p2.prevQueryRange,
       prevPrevQueryRange: p2.prevPrevQueryRange,
+      prevRangeSize: p2.prevRangeSize,
       latestBlockRangeUpdateBlock: p2.latestBlockRangeUpdateBlock
     });
     continuingBase = p1;
@@ -153,6 +166,7 @@ function mergePartitionsAtBlock(p1, p2, potentialMergeBlock, contractName, maxAd
       mutPendingQueries: continuingBase.mutPendingQueries,
       prevQueryRange: continuingBase.prevQueryRange,
       prevPrevQueryRange: continuingBase.prevPrevQueryRange,
+      prevRangeSize: continuingBase.prevRangeSize,
       latestBlockRangeUpdateBlock: continuingBase.latestBlockRangeUpdateBlock
     });
     let restId = nextPartitionIndexRef.contents.toString();
@@ -167,6 +181,7 @@ function mergePartitionsAtBlock(p1, p2, potentialMergeBlock, contractName, maxAd
       mutPendingQueries: [],
       prevQueryRange: continuingBase.prevQueryRange,
       prevPrevQueryRange: continuingBase.prevPrevQueryRange,
+      prevRangeSize: continuingBase.prevRangeSize,
       latestBlockRangeUpdateBlock: continuingBase.latestBlockRangeUpdateBlock
     });
     return completed;
@@ -183,6 +198,7 @@ function mergePartitionsAtBlock(p1, p2, potentialMergeBlock, contractName, maxAd
     mutPendingQueries: continuingBase.mutPendingQueries,
     prevQueryRange: continuingBase.prevQueryRange,
     prevPrevQueryRange: continuingBase.prevPrevQueryRange,
+    prevRangeSize: continuingBase.prevRangeSize,
     latestBlockRangeUpdateBlock: continuingBase.latestBlockRangeUpdateBlock
   });
   return completed;
@@ -320,7 +336,7 @@ function getPendingQueryOrThrow(p, fromBlock) {
   }
 }
 
-function handleQueryResponse(optimizedPartitions, query, knownHeight, latestFetchedBlock) {
+function handleQueryResponse(optimizedPartitions, query, knownHeight, itemsCount, latestFetchedBlock) {
   let partitionId = query.partitionId;
   let p = optimizedPartitions.entities[partitionId];
   let p$1 = p !== undefined ? p : Stdlib_JsError.throwWithMessage(`Unexpected case: Couldn't find partition ` + partitionId);
@@ -346,6 +362,7 @@ function handleQueryResponse(optimizedPartitions, query, knownHeight, latestFetc
   }
   let updatedPrevQueryRange = shouldUpdateBlockRange ? blockRange : p$1.prevQueryRange;
   let updatedPrevPrevQueryRange = shouldUpdateBlockRange ? p$1.prevQueryRange : p$1.prevPrevQueryRange;
+  let updatedPrevRangeSize = shouldUpdateBlockRange ? itemsCount : p$1.prevRangeSize;
   let mutPendingQueries = p$1.mutPendingQueries;
   let latestFetchedBlock$1 = p$1.latestFetchedBlock;
   while ((() => {
@@ -382,6 +399,7 @@ function handleQueryResponse(optimizedPartitions, query, knownHeight, latestFetc
       mutPendingQueries: updatedMainPartition_mutPendingQueries,
       prevQueryRange: updatedPrevQueryRange,
       prevPrevQueryRange: updatedPrevPrevQueryRange,
+      prevRangeSize: updatedPrevRangeSize,
       latestBlockRangeUpdateBlock: updatedMainPartition_latestBlockRangeUpdateBlock
     };
     mutEntities[p$1.id] = updatedMainPartition;
@@ -449,10 +467,10 @@ function numAddresses(fetchState) {
   return Utils.Dict.size(fetchState.indexingAddresses);
 }
 
-function appendOnBlockItems(mutItems, onBlockConfigs, indexerStartBlock, fromBlock, maxBlockNumber, targetBufferSize) {
+function appendOnBlockItems(mutItems, onBlockConfigs, indexerStartBlock, fromBlock, maxBlockNumber, maxOnBlockBufferSize) {
   let newItemsCounter = 0;
   let latestOnBlockBlockNumber = fromBlock;
-  while (latestOnBlockBlockNumber < maxBlockNumber && newItemsCounter <= targetBufferSize) {
+  while (latestOnBlockBlockNumber < maxBlockNumber && newItemsCounter <= maxOnBlockBufferSize) {
     let blockNumber = latestOnBlockBlockNumber + 1 | 0;
     latestOnBlockBlockNumber = blockNumber;
     for (let configIdx = 0, configIdx_finish = onBlockConfigs.length; configIdx < configIdx_finish; ++configIdx) {
@@ -490,7 +508,7 @@ function updateInternal(fetchState, optimizedPartitionsOpt, indexingAddressesOpt
     let mutItems$1 = mutItemsRef;
     let item = (
         mutItems$1 !== undefined ? mutItems$1 : fetchState.buffer
-      )[fetchState.targetBufferSize - 1 | 0];
+      )[fetchState.maxOnBlockBufferSize - 1 | 0];
     let maxBlockNumber;
     if (item !== undefined) {
       maxBlockNumber = item.blockNumber;
@@ -502,7 +520,7 @@ function updateInternal(fetchState, optimizedPartitionsOpt, indexingAddressesOpt
     let mutItems$2 = mutItemsRef;
     let mutItems$3 = mutItems$2 !== undefined ? mutItems$2 : fetchState.buffer.slice();
     mutItemsRef = mutItems$3;
-    latestOnBlockBlockNumber = appendOnBlockItems(mutItems$3, onBlockConfigs, fetchState.startBlock, fetchState.latestOnBlockBlockNumber, maxBlockNumber, fetchState.targetBufferSize);
+    latestOnBlockBlockNumber = appendOnBlockItems(mutItems$3, onBlockConfigs, fetchState.startBlock, fetchState.latestOnBlockBlockNumber, maxBlockNumber, fetchState.maxOnBlockBufferSize);
   } else {
     latestOnBlockBlockNumber = knownHeight;
   }
@@ -513,7 +531,7 @@ function updateInternal(fetchState, optimizedPartitionsOpt, indexingAddressesOpt
   let updatedFetchState_contractConfigs = fetchState.contractConfigs;
   let updatedFetchState_chainId = fetchState.chainId;
   let updatedFetchState_buffer = mutItems$4 !== undefined ? (mutItems$4.sort(compareBufferItem), mutItems$4) : fetchState.buffer;
-  let updatedFetchState_targetBufferSize = fetchState.targetBufferSize;
+  let updatedFetchState_maxOnBlockBufferSize = fetchState.maxOnBlockBufferSize;
   let updatedFetchState_onBlockConfigs = fetchState.onBlockConfigs;
   let updatedFetchState_firstEventBlock = fetchState.firstEventBlock;
   let updatedFetchState = {
@@ -527,7 +545,7 @@ function updateInternal(fetchState, optimizedPartitionsOpt, indexingAddressesOpt
     latestOnBlockBlockNumber: latestOnBlockBlockNumber,
     blockLag: blockLag,
     buffer: updatedFetchState_buffer,
-    targetBufferSize: updatedFetchState_targetBufferSize,
+    maxOnBlockBufferSize: updatedFetchState_maxOnBlockBufferSize,
     onBlockConfigs: updatedFetchState_onBlockConfigs,
     knownHeight: knownHeight,
     firstEventBlock: updatedFetchState_firstEventBlock
@@ -628,6 +646,7 @@ function createPartitionsFromIndexingAddresses(registeringContractsByContract, d
             mutPendingQueries: [],
             prevQueryRange: 0,
             prevPrevQueryRange: 0,
+            prevRangeSize: 0,
             latestBlockRangeUpdateBlock: 0
           });
           nextPartitionIndexRef = nextPartitionIndexRef + 1 | 0;
@@ -678,6 +697,7 @@ function createPartitionsFromIndexingAddresses(registeringContractsByContract, d
             mutPendingQueries: currentP.mutPendingQueries,
             prevQueryRange: currentP.prevQueryRange,
             prevPrevQueryRange: currentP.prevPrevQueryRange,
+            prevRangeSize: currentP.prevRangeSize,
             latestBlockRangeUpdateBlock: currentP.latestBlockRangeUpdateBlock
           });
           currentPRef = {
@@ -690,6 +710,7 @@ function createPartitionsFromIndexingAddresses(registeringContractsByContract, d
             mutPendingQueries: nextP.mutPendingQueries,
             prevQueryRange: nextP.prevQueryRange,
             prevPrevQueryRange: nextP.prevPrevQueryRange,
+            prevRangeSize: nextP.prevRangeSize,
             latestBlockRangeUpdateBlock: nextP.latestBlockRangeUpdateBlock
           };
         } else {
@@ -703,6 +724,7 @@ function createPartitionsFromIndexingAddresses(registeringContractsByContract, d
             mutPendingQueries: currentP.mutPendingQueries,
             prevQueryRange: currentP.prevQueryRange,
             prevPrevQueryRange: currentP.prevPrevQueryRange,
+            prevRangeSize: currentP.prevRangeSize,
             latestBlockRangeUpdateBlock: currentP.latestBlockRangeUpdateBlock
           };
         }
@@ -847,6 +869,7 @@ function registerDynamicContracts(fetchState, items) {
                   mutPendingQueries: p.mutPendingQueries,
                   prevQueryRange: p.prevQueryRange,
                   prevPrevQueryRange: p.prevPrevQueryRange,
+                  prevRangeSize: p.prevRangeSize,
                   latestBlockRangeUpdateBlock: p.latestBlockRangeUpdateBlock
                 };
                 let addressesByContractName = {};
@@ -861,6 +884,7 @@ function registerDynamicContracts(fetchState, items) {
                   mutPendingQueries: p.mutPendingQueries,
                   prevQueryRange: p.prevQueryRange,
                   prevPrevQueryRange: p.prevPrevQueryRange,
+                  prevRangeSize: p.prevRangeSize,
                   latestBlockRangeUpdateBlock: p.latestBlockRangeUpdateBlock
                 });
               }
@@ -875,6 +899,7 @@ function registerDynamicContracts(fetchState, items) {
                 mutPendingQueries: p.mutPendingQueries,
                 prevQueryRange: p.prevQueryRange,
                 prevPrevQueryRange: p.prevPrevQueryRange,
+                prevRangeSize: p.prevRangeSize,
                 latestBlockRangeUpdateBlock: p.latestBlockRangeUpdateBlock
               };
             }
@@ -903,12 +928,12 @@ function handleQueryResult(fetchState, query, latestFetchedBlock, newItems) {
     }
     let filter = item.eventConfig.clientAddressFilter;
     if (filter !== undefined) {
-      return filter(item.event, item.blockNumber, fetchState.indexingAddresses);
+      return filter(item.payload, item.blockNumber, fetchState.indexingAddresses);
     } else {
       return true;
     }
   });
-  return updateInternal(fetchState, handleQueryResponse(fetchState.optimizedPartitions, query, fetchState.knownHeight, latestFetchedBlock), undefined, newItems$1.length !== 0 ? fetchState.buffer.concat(newItems$1) : undefined, undefined, undefined);
+  return updateInternal(fetchState, handleQueryResponse(fetchState.optimizedPartitions, query, fetchState.knownHeight, newItems$1.length, latestFetchedBlock), undefined, newItems$1.length !== 0 ? fetchState.buffer.concat(newItems$1) : undefined, undefined, undefined);
 }
 
 function startFetchingQueries(param, queries) {
@@ -922,6 +947,7 @@ function startFetchingQueries(param, queries) {
       fromBlock: q.fromBlock,
       toBlock: q.toBlock,
       isChunk: q.isChunk,
+      estResponseSize: q.estResponseSize,
       fetchedBlock: undefined
     };
     let inserted = false;
@@ -939,7 +965,7 @@ function startFetchingQueries(param, queries) {
   }
 }
 
-function pushQueriesForRange(queries, partitionId, rangeFromBlock, rangeEndBlock, maxQueryBlockNumber, maybeChunkRange, selection, addressesByContractName, indexingAddresses) {
+function pushQueriesForRange(queries, partitionId, rangeFromBlock, rangeEndBlock, maxQueryBlockNumber, maybeChunkRange, partition, selection, addressesByContractName, indexingAddresses) {
   if (rangeFromBlock > maxQueryBlockNumber) {
     return;
   }
@@ -949,36 +975,48 @@ function pushQueriesForRange(queries, partitionId, rangeFromBlock, rangeEndBlock
   if (maybeChunkRange !== undefined) {
     let maxBlock = rangeEndBlock !== undefined ? rangeEndBlock : maxQueryBlockNumber;
     let chunkSize = Js_math.ceil_int(maybeChunkRange * 1.8);
-    if (((rangeFromBlock + (chunkSize << 1) | 0) - 1 | 0) <= maxBlock) {
-      queries.push({
-        partitionId: partitionId,
-        fromBlock: rangeFromBlock,
-        toBlock: (rangeFromBlock + chunkSize | 0) - 1 | 0,
-        isChunk: true,
-        selection: selection,
-        addressesByContractName: addressesByContractName,
-        indexingAddresses: indexingAddresses
-      });
-      queries.push({
-        partitionId: partitionId,
-        fromBlock: rangeFromBlock + chunkSize | 0,
-        toBlock: (rangeFromBlock + (chunkSize << 1) | 0) - 1 | 0,
-        isChunk: true,
-        selection: selection,
-        addressesByContractName: addressesByContractName,
-        indexingAddresses: indexingAddresses
-      });
-    } else {
-      queries.push({
-        partitionId: partitionId,
-        fromBlock: rangeFromBlock,
-        toBlock: rangeEndBlock,
-        isChunk: rangeEndBlock !== undefined,
-        selection: selection,
-        addressesByContractName: addressesByContractName,
-        indexingAddresses: indexingAddresses
-      });
+    let probeSize = Js_math.ceil_int(maybeChunkRange * 0.9);
+    let getChunkSize = chunkIdx => {
+      if (chunkIdx < 2) {
+        return probeSize;
+      } else {
+        return chunkSize;
+      }
+    };
+    if ((((rangeFromBlock + probeSize | 0) + probeSize | 0) - 1 | 0) <= maxBlock) {
+      let chunkFromBlock = rangeFromBlock;
+      let chunkIdx = 0;
+      while (chunkIdx < 5 && ((chunkFromBlock + getChunkSize(chunkIdx) | 0) - 1 | 0) <= maxBlock) {
+        let chunkToBlock = (chunkFromBlock + getChunkSize(chunkIdx) | 0) - 1 | 0;
+        queries.push({
+          partitionId: partitionId,
+          fromBlock: chunkFromBlock,
+          toBlock: chunkToBlock,
+          isChunk: true,
+          estResponseSize: calculateEstResponseSize(partition, chunkFromBlock, chunkToBlock, maxQueryBlockNumber),
+          chainId: 0,
+          progress: 0,
+          selection: selection,
+          addressesByContractName: addressesByContractName,
+          indexingAddresses: indexingAddresses
+        });
+        chunkFromBlock = chunkToBlock + 1 | 0;
+        chunkIdx = chunkIdx + 1 | 0;
+      };
+      return;
     }
+    queries.push({
+      partitionId: partitionId,
+      fromBlock: rangeFromBlock,
+      toBlock: rangeEndBlock,
+      isChunk: rangeEndBlock !== undefined,
+      estResponseSize: calculateEstResponseSize(partition, rangeFromBlock, rangeEndBlock, maxQueryBlockNumber),
+      chainId: 0,
+      progress: 0,
+      selection: selection,
+      addressesByContractName: addressesByContractName,
+      indexingAddresses: indexingAddresses
+    });
     return;
   }
   queries.push({
@@ -986,13 +1024,16 @@ function pushQueriesForRange(queries, partitionId, rangeFromBlock, rangeEndBlock
     fromBlock: rangeFromBlock,
     toBlock: rangeEndBlock,
     isChunk: false,
+    estResponseSize: calculateEstResponseSize(partition, rangeFromBlock, rangeEndBlock, maxQueryBlockNumber),
+    chainId: 0,
+    progress: 0,
     selection: selection,
     addressesByContractName: addressesByContractName,
     indexingAddresses: indexingAddresses
   });
 }
 
-function getNextQuery(fetchState, concurrencyLimit) {
+function getNextQuery(fetchState, budget, chainPendingBudget) {
   let knownHeight = fetchState.knownHeight;
   let blockLag = fetchState.blockLag;
   let indexingAddresses = fetchState.indexingAddresses;
@@ -1001,15 +1042,15 @@ function getNextQuery(fetchState, concurrencyLimit) {
   if (headBlockNumber <= 0) {
     return "WaitingForNewBlock";
   }
-  if (concurrencyLimit === 0) {
-    return "ReachedMaxConcurrency";
+  if (budget <= 0) {
+    return "NothingToQuery";
   }
   let isOnBlockBehindTheHead = fetchState.latestOnBlockBlockNumber < headBlockNumber;
   let endBlock = fetchState.endBlock;
   let shouldWaitForNewBlock = (
     endBlock !== undefined ? headBlockNumber < endBlock : true
   ) && !isOnBlockBehindTheHead;
-  let item = fetchState.buffer[fetchState.targetBufferSize - 1 | 0];
+  let item = fetchState.buffer[(budget + (chainPendingBudget | 0) | 0) - 1 | 0];
   let maxQueryBlockNumber = item !== undefined ? Primitive_int.min(item.blockNumber, knownHeight) : knownHeight;
   let queries = [];
   let partitionsCount = optimizedPartitions.idsInAscOrder.length;
@@ -1018,11 +1059,13 @@ function getNextQuery(fetchState, concurrencyLimit) {
     let idx = idxRef;
     let partitionId = optimizedPartitions.idsInAscOrder[idx];
     let p = optimizedPartitions.entities[partitionId];
+    let pendingCount = p.mutPendingQueries.length;
     let isBehindTheHead = p.latestFetchedBlock.blockNumber < headBlockNumber;
-    let hasPendingQueries = Utils.$$Array.notEmpty(p.mutPendingQueries);
+    let hasPendingQueries = pendingCount > 0;
     if (hasPendingQueries || isBehindTheHead) {
       shouldWaitForNewBlock = false;
     }
+    let partitionQueriesStart = queries.length;
     let queryEndBlock = Utils.$$Math.minOptInt(fetchState.endBlock, p.mergeBlock);
     let queryEndBlock$1 = blockLag !== 0 ? Utils.$$Math.minOptInt(headBlockNumber, queryEndBlock) : queryEndBlock;
     let match = maxQueryBlockNumber < knownHeight;
@@ -1051,31 +1094,43 @@ function getNextQuery(fetchState, concurrencyLimit) {
             if (maybeChunkRange !== undefined) {
               let maxBlock = rangeEndBlock !== undefined ? rangeEndBlock : maxQueryBlockNumber;
               let chunkSize = Js_math.ceil_int(maybeChunkRange * 1.8);
-              if (((rangeFromBlock + (chunkSize << 1) | 0) - 1 | 0) <= maxBlock) {
-                queries.push({
-                  partitionId: partitionId,
-                  fromBlock: rangeFromBlock,
-                  toBlock: (rangeFromBlock + chunkSize | 0) - 1 | 0,
-                  isChunk: true,
-                  selection: selection,
-                  addressesByContractName: addressesByContractName,
-                  indexingAddresses: indexingAddresses
-                });
-                queries.push({
-                  partitionId: partitionId,
-                  fromBlock: rangeFromBlock + chunkSize | 0,
-                  toBlock: (rangeFromBlock + (chunkSize << 1) | 0) - 1 | 0,
-                  isChunk: true,
-                  selection: selection,
-                  addressesByContractName: addressesByContractName,
-                  indexingAddresses: indexingAddresses
-                });
+              let probeSize = Js_math.ceil_int(maybeChunkRange * 0.9);
+              let getChunkSize = chunkIdx => {
+                if (chunkIdx < 2) {
+                  return probeSize;
+                } else {
+                  return chunkSize;
+                }
+              };
+              if ((((rangeFromBlock + getChunkSize(0) | 0) + getChunkSize(1) | 0) - 1 | 0) <= maxBlock) {
+                let chunkFromBlock = rangeFromBlock;
+                let chunkIdx = 0;
+                while (chunkIdx < 5 && ((chunkFromBlock + getChunkSize(chunkIdx) | 0) - 1 | 0) <= maxBlock) {
+                  let chunkToBlock = (chunkFromBlock + getChunkSize(chunkIdx) | 0) - 1 | 0;
+                  queries.push({
+                    partitionId: partitionId,
+                    fromBlock: chunkFromBlock,
+                    toBlock: chunkToBlock,
+                    isChunk: true,
+                    estResponseSize: calculateEstResponseSize(p, chunkFromBlock, chunkToBlock, maxQueryBlockNumber),
+                    chainId: 0,
+                    progress: 0,
+                    selection: selection,
+                    addressesByContractName: addressesByContractName,
+                    indexingAddresses: indexingAddresses
+                  });
+                  chunkFromBlock = chunkToBlock + 1 | 0;
+                  chunkIdx = chunkIdx + 1 | 0;
+                };
               } else {
                 queries.push({
                   partitionId: partitionId,
                   fromBlock: rangeFromBlock,
                   toBlock: rangeEndBlock,
                   isChunk: rangeEndBlock !== undefined,
+                  estResponseSize: calculateEstResponseSize(p, rangeFromBlock, rangeEndBlock, maxQueryBlockNumber),
+                  chainId: 0,
+                  progress: 0,
                   selection: selection,
                   addressesByContractName: addressesByContractName,
                   indexingAddresses: indexingAddresses
@@ -1087,6 +1142,9 @@ function getNextQuery(fetchState, concurrencyLimit) {
                 fromBlock: rangeFromBlock,
                 toBlock: rangeEndBlock,
                 isChunk: false,
+                estResponseSize: calculateEstResponseSize(p, rangeFromBlock, rangeEndBlock, maxQueryBlockNumber),
+                chainId: 0,
+                progress: 0,
                 selection: selection,
                 addressesByContractName: addressesByContractName,
                 indexingAddresses: indexingAddresses
@@ -1122,31 +1180,43 @@ function getNextQuery(fetchState, concurrencyLimit) {
           if (maybeChunkRange !== undefined) {
             let maxBlock$1 = queryEndBlock$2 !== undefined ? queryEndBlock$2 : maxQueryBlockNumber;
             let chunkSize$1 = Js_math.ceil_int(maybeChunkRange * 1.8);
-            if (((rangeFromBlock$1 + (chunkSize$1 << 1) | 0) - 1 | 0) <= maxBlock$1) {
-              queries.push({
-                partitionId: partitionId,
-                fromBlock: rangeFromBlock$1,
-                toBlock: (rangeFromBlock$1 + chunkSize$1 | 0) - 1 | 0,
-                isChunk: true,
-                selection: selection$1,
-                addressesByContractName: addressesByContractName$1,
-                indexingAddresses: indexingAddresses
-              });
-              queries.push({
-                partitionId: partitionId,
-                fromBlock: rangeFromBlock$1 + chunkSize$1 | 0,
-                toBlock: (rangeFromBlock$1 + (chunkSize$1 << 1) | 0) - 1 | 0,
-                isChunk: true,
-                selection: selection$1,
-                addressesByContractName: addressesByContractName$1,
-                indexingAddresses: indexingAddresses
-              });
+            let probeSize$1 = Js_math.ceil_int(maybeChunkRange * 0.9);
+            let getChunkSize$1 = chunkIdx => {
+              if (chunkIdx < 2) {
+                return probeSize$1;
+              } else {
+                return chunkSize$1;
+              }
+            };
+            if ((((rangeFromBlock$1 + getChunkSize$1(0) | 0) + getChunkSize$1(1) | 0) - 1 | 0) <= maxBlock$1) {
+              let chunkFromBlock$1 = rangeFromBlock$1;
+              let chunkIdx$1 = 0;
+              while (chunkIdx$1 < 5 && ((chunkFromBlock$1 + getChunkSize$1(chunkIdx$1) | 0) - 1 | 0) <= maxBlock$1) {
+                let chunkToBlock$1 = (chunkFromBlock$1 + getChunkSize$1(chunkIdx$1) | 0) - 1 | 0;
+                queries.push({
+                  partitionId: partitionId,
+                  fromBlock: chunkFromBlock$1,
+                  toBlock: chunkToBlock$1,
+                  isChunk: true,
+                  estResponseSize: calculateEstResponseSize(p, chunkFromBlock$1, chunkToBlock$1, maxQueryBlockNumber),
+                  chainId: 0,
+                  progress: 0,
+                  selection: selection$1,
+                  addressesByContractName: addressesByContractName$1,
+                  indexingAddresses: indexingAddresses
+                });
+                chunkFromBlock$1 = chunkToBlock$1 + 1 | 0;
+                chunkIdx$1 = chunkIdx$1 + 1 | 0;
+              };
             } else {
               queries.push({
                 partitionId: partitionId,
                 fromBlock: rangeFromBlock$1,
                 toBlock: queryEndBlock$2,
                 isChunk: queryEndBlock$2 !== undefined,
+                estResponseSize: calculateEstResponseSize(p, rangeFromBlock$1, queryEndBlock$2, maxQueryBlockNumber),
+                chainId: 0,
+                progress: 0,
                 selection: selection$1,
                 addressesByContractName: addressesByContractName$1,
                 indexingAddresses: indexingAddresses
@@ -1158,6 +1228,9 @@ function getNextQuery(fetchState, concurrencyLimit) {
               fromBlock: rangeFromBlock$1,
               toBlock: queryEndBlock$2,
               isChunk: false,
+              estResponseSize: calculateEstResponseSize(p, rangeFromBlock$1, queryEndBlock$2, maxQueryBlockNumber),
+              chainId: 0,
+              progress: 0,
               selection: selection$1,
               addressesByContractName: addressesByContractName$1,
               indexingAddresses: indexingAddresses
@@ -1166,6 +1239,11 @@ function getNextQuery(fetchState, concurrencyLimit) {
         }
       }
     }
+    let maxNewChunks = Primitive_int.max(0, 10 - pendingCount | 0);
+    let generatedCount = queries.length - partitionQueriesStart | 0;
+    if (generatedCount > maxNewChunks) {
+      queries.splice(partitionQueriesStart + maxNewChunks | 0, generatedCount - maxNewChunks | 0);
+    }
     idxRef = idxRef + 1 | 0;
   };
   if (Utils.$$Array.isEmpty(queries)) {
@@ -1174,12 +1252,12 @@ function getNextQuery(fetchState, concurrencyLimit) {
     } else {
       return "NothingToQuery";
     }
+  } else {
+    return {
+      TAG: "Ready",
+      _0: queries
+    };
   }
-  let queries$1 = queries.length > concurrencyLimit ? (queries.sort((a, b) => Primitive_int.compare(a.fromBlock, b.fromBlock)), queries.slice(0, concurrencyLimit)) : queries;
-  return {
-    TAG: "Ready",
-    _0: queries$1
-  };
 }
 
 function hasReadyItem(fetchState) {
@@ -1214,7 +1292,7 @@ function getReadyItemsCount(fetchState, targetSize, fromItem) {
   return acc;
 }
 
-function make$1(startBlock, endBlock, eventConfigs, addresses, maxAddrInPartition, chainId, targetBufferSize, knownHeight, progressBlockNumberOpt, onBlockConfigsOpt, blockLagOpt, firstEventBlockOpt) {
+function make$1(startBlock, endBlock, eventConfigs, addresses, maxAddrInPartition, chainId, maxOnBlockBufferSize, knownHeight, progressBlockNumberOpt, onBlockConfigsOpt, blockLagOpt, firstEventBlockOpt) {
   let progressBlockNumber = progressBlockNumberOpt !== undefined ? progressBlockNumberOpt : startBlock - 1 | 0;
   let onBlockConfigs = onBlockConfigsOpt !== undefined ? onBlockConfigsOpt : [];
   let blockLag = blockLagOpt !== undefined ? blockLagOpt : 0;
@@ -1267,6 +1345,7 @@ function make$1(startBlock, endBlock, eventConfigs, addresses, maxAddrInPartitio
       mutPendingQueries: [],
       prevQueryRange: 0,
       prevPrevQueryRange: 0,
+      prevRangeSize: 0,
       latestBlockRangeUpdateBlock: 0
     });
   }
@@ -1311,7 +1390,7 @@ function make$1(startBlock, endBlock, eventConfigs, addresses, maxAddrInPartitio
     let id = optimizedPartitions.idsInAscOrder[0];
     let latestFullyFetchedBlock = id !== undefined ? optimizedPartitions.entities[id].latestFetchedBlock : undefined;
     let maxBlockNumber = latestFullyFetchedBlock !== undefined ? latestFullyFetchedBlock.blockNumber : knownHeight;
-    latestOnBlockBlockNumber = appendOnBlockItems(buffer, onBlockConfigs, startBlock, progressBlockNumber, maxBlockNumber, targetBufferSize);
+    latestOnBlockBlockNumber = appendOnBlockItems(buffer, onBlockConfigs, startBlock, progressBlockNumber, maxBlockNumber, maxOnBlockBufferSize);
   } else {
     latestOnBlockBlockNumber = progressBlockNumber;
   }
@@ -1326,7 +1405,7 @@ function make$1(startBlock, endBlock, eventConfigs, addresses, maxAddrInPartitio
     latestOnBlockBlockNumber: latestOnBlockBlockNumber,
     blockLag: blockLag,
     buffer: buffer,
-    targetBufferSize: targetBufferSize,
+    maxOnBlockBufferSize: maxOnBlockBufferSize,
     onBlockConfigs: onBlockConfigs,
     knownHeight: knownHeight,
     firstEventBlock: firstEventBlock
@@ -1346,6 +1425,22 @@ function bufferSize(param) {
   return param.buffer.length;
 }
 
+function bufferReadyCount(fetchState) {
+  let frontier = bufferBlockNumber(fetchState);
+  let buffer = fetchState.buffer;
+  let lo = 0;
+  let hi = buffer.length;
+  while (lo < hi) {
+    let mid = (lo + hi | 0) / 2 | 0;
+    if (buffer[mid].blockNumber <= frontier) {
+      lo = mid + 1 | 0;
+    } else {
+      hi = mid;
+    }
+  };
+  return lo;
+}
+
 function rollbackPendingQueries(mutPendingQueries, targetBlockNumber) {
   let adjusted = [];
   for (let qIdx = 0, qIdx_finish = mutPendingQueries.length; qIdx < qIdx_finish; ++qIdx) {
@@ -1358,6 +1453,7 @@ function rollbackPendingQueries(mutPendingQueries, targetBlockNumber) {
             fromBlock: pq.fromBlock,
             toBlock: pq.toBlock,
             isChunk: pq.isChunk,
+            estResponseSize: pq.estResponseSize,
             fetchedBlock: {
               blockNumber: targetBlockNumber,
               blockTimestamp: 0
@@ -1425,6 +1521,7 @@ function rollback(fetchState, targetBlockNumber) {
             mutPendingQueries: rollbackPendingQueries(p.mutPendingQueries, targetBlockNumber),
             prevQueryRange: p.prevQueryRange,
             prevPrevQueryRange: p.prevPrevQueryRange,
+            prevRangeSize: p.prevRangeSize,
             latestBlockRangeUpdateBlock: p.latestBlockRangeUpdateBlock
           });
         }
@@ -1445,6 +1542,7 @@ function rollback(fetchState, targetBlockNumber) {
         mutPendingQueries: rollbackPendingQueries(p.mutPendingQueries, targetBlockNumber),
         prevQueryRange: p.prevQueryRange,
         prevPrevQueryRange: p.prevPrevQueryRange,
+        prevRangeSize: p.prevRangeSize,
         latestBlockRangeUpdateBlock: p.latestBlockRangeUpdateBlock
       });
     }
@@ -1461,7 +1559,7 @@ function rollback(fetchState, targetBlockNumber) {
     latestOnBlockBlockNumber: Primitive_int.min(fetchState.latestOnBlockBlockNumber, targetBlockNumber),
     blockLag: fetchState.blockLag,
     buffer: fetchState.buffer,
-    targetBufferSize: fetchState.targetBufferSize,
+    maxOnBlockBufferSize: fetchState.maxOnBlockBufferSize,
     onBlockConfigs: fetchState.onBlockConfigs,
     knownHeight: fetchState.knownHeight,
     firstEventBlock: fetchState.firstEventBlock
@@ -1489,6 +1587,7 @@ function resetPendingQueries(fetchState) {
         mutPendingQueries: kept,
         prevQueryRange: partition.prevQueryRange,
         prevPrevQueryRange: partition.prevPrevQueryRange,
+        prevRangeSize: partition.prevRangeSize,
         latestBlockRangeUpdateBlock: partition.latestBlockRangeUpdateBlock
       };
     }
@@ -1511,7 +1610,7 @@ function resetPendingQueries(fetchState) {
     latestOnBlockBlockNumber: fetchState.latestOnBlockBlockNumber,
     blockLag: fetchState.blockLag,
     buffer: fetchState.buffer,
-    targetBufferSize: fetchState.targetBufferSize,
+    maxOnBlockBufferSize: fetchState.maxOnBlockBufferSize,
     onBlockConfigs: fetchState.onBlockConfigs,
     knownHeight: fetchState.knownHeight,
     firstEventBlock: fetchState.firstEventBlock
@@ -1531,24 +1630,31 @@ function isActivelyIndexing(fetchState) {
   }
 }
 
-function isReadyToEnterReorgThreshold(fetchState) {
+function isFetchingAtHead(fetchState) {
   let knownHeight = fetchState.knownHeight;
   let blockLag = fetchState.blockLag;
   let endBlock = fetchState.endBlock;
   let bufferBlockNumber$1 = bufferBlockNumber(fetchState);
-  if (knownHeight !== 0 && (
-      endBlock !== undefined && bufferBlockNumber$1 >= endBlock ? true : bufferBlockNumber$1 >= (knownHeight - blockLag | 0)
-    )) {
-    return Utils.$$Array.isEmpty(fetchState.buffer);
+  if (knownHeight !== 0) {
+    if (endBlock !== undefined && bufferBlockNumber$1 >= endBlock) {
+      return true;
+    } else {
+      return bufferBlockNumber$1 >= (knownHeight - blockLag | 0);
+    }
   } else {
     return false;
   }
 }
 
-function hasFullBatch(fetchState, batchSizeTarget) {
-  let item = fetchState.buffer[batchSizeTarget - 1 | 0];
-  if (item !== undefined) {
-    return item.blockNumber <= bufferBlockNumber(fetchState);
+function isReadyToEnterReorgThreshold(fetchState) {
+  let knownHeight = fetchState.knownHeight;
+  let blockLag = fetchState.blockLag;
+  let endBlock = fetchState.endBlock;
+  let bufferBlockNumber$1 = bufferBlockNumber(fetchState);
+  if (knownHeight !== 0 && (
+      endBlock !== undefined && bufferBlockNumber$1 >= endBlock ? true : bufferBlockNumber$1 >= (knownHeight - blockLag | 0)
+    )) {
+    return Utils.$$Array.isEmpty(fetchState.buffer);
   } else {
     return false;
   }
@@ -1568,7 +1674,29 @@ function getProgressPercentage(fetchState) {
   return progress / totalRange;
 }
 
-function sortForUnorderedBatch(fetchStates, batchSizeTarget) {
+function getProgressPercentageAt(fetchState, blockNumber) {
+  let firstEventBlock = fetchState.firstEventBlock;
+  if (firstEventBlock === undefined) {
+    return 0;
+  }
+  let totalRange = fetchState.knownHeight - firstEventBlock | 0;
+  if (totalRange <= 0) {
+    return 0;
+  } else {
+    return (blockNumber - firstEventBlock | 0) / totalRange;
+  }
+}
+
+function hasFullBatch(fetchState, batchSizeTarget) {
+  let item = fetchState.buffer[batchSizeTarget - 1 | 0];
+  if (item !== undefined) {
+    return item.blockNumber <= bufferBlockNumber(fetchState);
+  } else {
+    return false;
+  }
+}
+
+function sortForBatch(fetchStates, batchSizeTarget) {
   let copied = fetchStates.slice();
   copied.sort((a, b) => {
     let match = hasFullBatch(a, batchSizeTarget);
@@ -1612,10 +1740,16 @@ function updateKnownHeight(fetchState, knownHeight) {
   }
 }
 
+let defaultEstResponseSize = 10000;
+
 let blockItemLogIndex = 16777216;
 
+let maxPendingChunksPerPartition = 10;
+
 export {
   deriveEffectiveStartBlock,
+  defaultEstResponseSize,
+  calculateEstResponseSize,
   getMinHistoryRange,
   getMinQueryRange,
   OptimizedPartitions,
@@ -1634,17 +1768,22 @@ export {
   handleQueryResult,
   startFetchingQueries,
   pushQueriesForRange,
+  maxPendingChunksPerPartition,
   getNextQuery,
   hasReadyItem,
   getReadyItemsCount,
   make$1 as make,
   bufferSize,
+  bufferReadyCount,
   rollbackPendingQueries,
   rollback,
   resetPendingQueries,
   isActivelyIndexing,
+  isFetchingAtHead,
   isReadyToEnterReorgThreshold,
-  sortForUnorderedBatch,
+  getProgressPercentage,
+  getProgressPercentageAt,
+  sortForBatch,
   getProgressBlockNumberAt,
   updateKnownHeight,
 }