envio 3.3.0-alpha.8 → 3.3.0-rc.0

package/src/FetchState.res.mjs

@@ -23,25 +23,28 @@ let deriveContractNameByAddress = Utils.$$WeakMap.memoize(addressesByContractNam
   return result;
 });
 
-function densityItemsTarget(p, fromBlock, toBlock, chainTargetBlock) {
-  let density = p.prevRangeSize / p.prevQueryRange;
-  return ((Stdlib_Option.getOr(toBlock, chainTargetBlock) - fromBlock | 0) + 1 | 0) * density;
+function densityItemsTarget(density, fromBlock, toBlock, chainTargetBlock) {
+  return Primitive_int.max(1, Math.ceil(((Stdlib_Option.getOr(toBlock, chainTargetBlock) - fromBlock | 0) + 1 | 0) * density) | 0);
 }
 
 function getMinHistoryRange(p) {
-  let match = p.prevQueryRange;
-  let match$1 = p.prevPrevQueryRange;
+  let match = p.sourceRangeCapacity;
+  let match$1 = p.prevSourceRangeCapacity;
   if (match !== 0 && match$1 !== 0) {
     return match < match$1 ? match : match$1;
   }
 }
 
+function getTrustedDensity(p) {
+  return p.eventDensity;
+}
+
 function getMinQueryRange(partitions) {
   let min = 0;
   for (let i = 0, i_finish = partitions.length; i < i_finish; ++i) {
     let p = partitions[i];
-    let a = p.prevQueryRange;
-    let b = p.prevPrevQueryRange;
+    let a = p.sourceRangeCapacity;
+    let b = p.prevSourceRangeCapacity;
     if (a > 0 && (min === 0 || a < min)) {
       min = a;
     }
@@ -81,10 +84,10 @@ function mergePartitionsAtBlock(p1, p2, potentialMergeBlock, contractName, maxAd
         mergeBlock: potentialMergeBlock,
         dynamicContract: p1.dynamicContract,
         mutPendingQueries: p1.mutPendingQueries,
-        prevQueryRange: p1.prevQueryRange,
-        prevPrevQueryRange: p1.prevPrevQueryRange,
-        prevRangeSize: p1.prevRangeSize,
-        latestBlockRangeUpdateBlock: p1.latestBlockRangeUpdateBlock
+        sourceRangeCapacity: p1.sourceRangeCapacity,
+        prevSourceRangeCapacity: p1.prevSourceRangeCapacity,
+        eventDensity: p1.eventDensity,
+        latestSourceRangeCapacityUpdateBlock: p1.latestSourceRangeCapacityUpdateBlock
       });
       completed.push({
         id: p2.id,
@@ -94,10 +97,10 @@ function mergePartitionsAtBlock(p1, p2, potentialMergeBlock, contractName, maxAd
         mergeBlock: potentialMergeBlock,
         dynamicContract: p2.dynamicContract,
         mutPendingQueries: p2.mutPendingQueries,
-        prevQueryRange: p2.prevQueryRange,
-        prevPrevQueryRange: p2.prevPrevQueryRange,
-        prevRangeSize: p2.prevRangeSize,
-        latestBlockRangeUpdateBlock: p2.latestBlockRangeUpdateBlock
+        sourceRangeCapacity: p2.sourceRangeCapacity,
+        prevSourceRangeCapacity: p2.prevSourceRangeCapacity,
+        eventDensity: p2.eventDensity,
+        latestSourceRangeCapacityUpdateBlock: p2.latestSourceRangeCapacityUpdateBlock
       });
       let newId = nextPartitionIndexRef.contents.toString();
       nextPartitionIndexRef.contents = nextPartitionIndexRef.contents + 1 | 0;
@@ -105,6 +108,13 @@ function mergePartitionsAtBlock(p1, p2, potentialMergeBlock, contractName, maxAd
         p1,
         p2
       ]);
+      let match = p1.eventDensity;
+      let match$1 = p2.eventDensity;
+      let inheritedDensity = match !== undefined ? (
+          match$1 !== undefined ? match + match$1 : match
+        ) : (
+          match$1 !== undefined ? match$1 : undefined
+        );
       continuingBase = {
         id: newId,
         latestFetchedBlock: {
@@ -116,10 +126,10 @@ function mergePartitionsAtBlock(p1, p2, potentialMergeBlock, contractName, maxAd
         mergeBlock: undefined,
         dynamicContract: contractName,
         mutPendingQueries: [],
-        prevQueryRange: minRange,
-        prevPrevQueryRange: minRange,
-        prevRangeSize: 0,
-        latestBlockRangeUpdateBlock: 0
+        sourceRangeCapacity: minRange,
+        prevSourceRangeCapacity: minRange,
+        eventDensity: inheritedDensity,
+        latestSourceRangeCapacityUpdateBlock: 0
       };
     } else {
       completed.push({
@@ -130,10 +140,10 @@ function mergePartitionsAtBlock(p1, p2, potentialMergeBlock, contractName, maxAd
         mergeBlock: potentialMergeBlock,
         dynamicContract: p1.dynamicContract,
         mutPendingQueries: p1.mutPendingQueries,
-        prevQueryRange: p1.prevQueryRange,
-        prevPrevQueryRange: p1.prevPrevQueryRange,
-        prevRangeSize: p1.prevRangeSize,
-        latestBlockRangeUpdateBlock: p1.latestBlockRangeUpdateBlock
+        sourceRangeCapacity: p1.sourceRangeCapacity,
+        prevSourceRangeCapacity: p1.prevSourceRangeCapacity,
+        eventDensity: p1.eventDensity,
+        latestSourceRangeCapacityUpdateBlock: p1.latestSourceRangeCapacityUpdateBlock
       });
       continuingBase = p2;
     }
@@ -146,10 +156,10 @@ function mergePartitionsAtBlock(p1, p2, potentialMergeBlock, contractName, maxAd
       mergeBlock: potentialMergeBlock,
       dynamicContract: p2.dynamicContract,
       mutPendingQueries: p2.mutPendingQueries,
-      prevQueryRange: p2.prevQueryRange,
-      prevPrevQueryRange: p2.prevPrevQueryRange,
-      prevRangeSize: p2.prevRangeSize,
-      latestBlockRangeUpdateBlock: p2.latestBlockRangeUpdateBlock
+      sourceRangeCapacity: p2.sourceRangeCapacity,
+      prevSourceRangeCapacity: p2.prevSourceRangeCapacity,
+      eventDensity: p2.eventDensity,
+      latestSourceRangeCapacityUpdateBlock: p2.latestSourceRangeCapacityUpdateBlock
     });
     continuingBase = p1;
   } else {
@@ -170,10 +180,10 @@ function mergePartitionsAtBlock(p1, p2, potentialMergeBlock, contractName, maxAd
       mergeBlock: continuingBase.mergeBlock,
       dynamicContract: continuingBase.dynamicContract,
       mutPendingQueries: continuingBase.mutPendingQueries,
-      prevQueryRange: continuingBase.prevQueryRange,
-      prevPrevQueryRange: continuingBase.prevPrevQueryRange,
-      prevRangeSize: continuingBase.prevRangeSize,
-      latestBlockRangeUpdateBlock: continuingBase.latestBlockRangeUpdateBlock
+      sourceRangeCapacity: continuingBase.sourceRangeCapacity,
+      prevSourceRangeCapacity: continuingBase.prevSourceRangeCapacity,
+      eventDensity: continuingBase.eventDensity,
+      latestSourceRangeCapacityUpdateBlock: continuingBase.latestSourceRangeCapacityUpdateBlock
     });
     let restId = nextPartitionIndexRef.contents.toString();
     nextPartitionIndexRef.contents = nextPartitionIndexRef.contents + 1 | 0;
@@ -185,10 +195,10 @@ function mergePartitionsAtBlock(p1, p2, potentialMergeBlock, contractName, maxAd
       mergeBlock: continuingBase.mergeBlock,
       dynamicContract: continuingBase.dynamicContract,
       mutPendingQueries: [],
-      prevQueryRange: continuingBase.prevQueryRange,
-      prevPrevQueryRange: continuingBase.prevPrevQueryRange,
-      prevRangeSize: continuingBase.prevRangeSize,
-      latestBlockRangeUpdateBlock: continuingBase.latestBlockRangeUpdateBlock
+      sourceRangeCapacity: continuingBase.sourceRangeCapacity,
+      prevSourceRangeCapacity: continuingBase.prevSourceRangeCapacity,
+      eventDensity: continuingBase.eventDensity,
+      latestSourceRangeCapacityUpdateBlock: continuingBase.latestSourceRangeCapacityUpdateBlock
     });
     return completed;
   }
@@ -202,10 +212,10 @@ function mergePartitionsAtBlock(p1, p2, potentialMergeBlock, contractName, maxAd
     mergeBlock: continuingBase.mergeBlock,
     dynamicContract: continuingBase.dynamicContract,
     mutPendingQueries: continuingBase.mutPendingQueries,
-    prevQueryRange: continuingBase.prevQueryRange,
-    prevPrevQueryRange: continuingBase.prevPrevQueryRange,
-    prevRangeSize: continuingBase.prevRangeSize,
-    latestBlockRangeUpdateBlock: continuingBase.latestBlockRangeUpdateBlock
+    sourceRangeCapacity: continuingBase.sourceRangeCapacity,
+    prevSourceRangeCapacity: continuingBase.prevSourceRangeCapacity,
+    eventDensity: continuingBase.eventDensity,
+    latestSourceRangeCapacityUpdateBlock: continuingBase.latestSourceRangeCapacityUpdateBlock
   });
   return completed;
 }
@@ -310,17 +320,25 @@ function make(partitions, maxAddrInPartition, nextPartitionIndex, dynamicContrac
 
 function consumeFetchedQueries(mutPendingQueries, initialLatestFetchedBlock) {
   let latestFetchedBlock = initialLatestFetchedBlock;
-  while ((() => {
-      let tmp = false;
-      if (mutPendingQueries.length !== 0) {
-        let pq = Utils.$$Array.firstUnsafe(mutPendingQueries);
-        tmp = pq.fetchedBlock !== undefined && pq.fromBlock <= (latestFetchedBlock.blockNumber + 1 | 0);
+  let consumedCount = 0;
+  let canContinue = true;
+  while (canContinue) {
+    let match = mutPendingQueries[consumedCount];
+    if (match !== undefined) {
+      let fetchedBlock = match.fetchedBlock;
+      if (fetchedBlock !== undefined && match.fromBlock <= (latestFetchedBlock.blockNumber + 1 | 0)) {
+        latestFetchedBlock = fetchedBlock;
+        consumedCount = consumedCount + 1 | 0;
+      } else {
+        canContinue = false;
       }
-      return tmp;
-    })()) {
-    let removedQuery = mutPendingQueries.shift();
-    latestFetchedBlock = removedQuery.fetchedBlock;
+    } else {
+      canContinue = false;
+    }
   };
+  if (consumedCount > 0) {
+    mutPendingQueries.splice(0, consumedCount);
+  }
   return latestFetchedBlock;
 }
 
@@ -350,67 +368,112 @@ function handleQueryResponse(optimizedPartitions, query, knownHeight, itemsCount
   let pendingQuery = getPendingQueryOrThrow(p$1, query.fromBlock);
   pendingQuery.fetchedBlock = latestFetchedBlock;
   let blockRange = (latestFetchedBlock.blockNumber - query.fromBlock | 0) + 1 | 0;
-  let shouldUpdateBlockRange = false;
-  if (latestFetchedBlock.blockNumber > p$1.latestBlockRangeUpdateBlock) {
+  let observedEventDensity = itemsCount / blockRange;
+  let eventDensity = p$1.eventDensity;
+  let updatedEventDensity = eventDensity !== undefined ? (eventDensity + observedEventDensity) / 2 : observedEventDensity;
+  let shouldUpdateSourceRangeCapacity = false;
+  if (latestFetchedBlock.blockNumber > p$1.latestSourceRangeCapacityUpdateBlock) {
     let queryToBlock = query.toBlock;
     let tmp;
     if (queryToBlock !== undefined) {
-      if (latestFetchedBlock.blockNumber >= queryToBlock) {
+      if (latestFetchedBlock.blockNumber < queryToBlock) {
+        tmp = itemsCount < query.itemsTarget;
+      } else {
         let minHistoryRange = getMinHistoryRange(p$1);
         tmp = minHistoryRange !== undefined ? ((queryToBlock - query.fromBlock | 0) + 1 | 0) >= minHistoryRange : false;
-      } else {
-        tmp = true;
       }
     } else {
       tmp = latestFetchedBlock.blockNumber < (knownHeight - 10 | 0);
     }
-    shouldUpdateBlockRange = tmp;
+    shouldUpdateSourceRangeCapacity = tmp;
   }
-  let updatedPrevQueryRange = shouldUpdateBlockRange ? blockRange : p$1.prevQueryRange;
-  let updatedPrevPrevQueryRange = shouldUpdateBlockRange ? p$1.prevQueryRange : p$1.prevPrevQueryRange;
-  let updatedPrevRangeSize = shouldUpdateBlockRange ? itemsCount : p$1.prevRangeSize;
+  let updatedSourceRangeCapacity = shouldUpdateSourceRangeCapacity ? blockRange : p$1.sourceRangeCapacity;
+  let updatedPrevSourceRangeCapacity = shouldUpdateSourceRangeCapacity ? p$1.sourceRangeCapacity : p$1.prevSourceRangeCapacity;
   let mutPendingQueries = p$1.mutPendingQueries;
   let latestFetchedBlock$1 = p$1.latestFetchedBlock;
-  while ((() => {
-      let tmp = false;
-      if (mutPendingQueries.length !== 0) {
-        let pq = Utils.$$Array.firstUnsafe(mutPendingQueries);
-        tmp = pq.fetchedBlock !== undefined && pq.fromBlock <= (latestFetchedBlock$1.blockNumber + 1 | 0);
+  let consumedCount = 0;
+  let canContinue = true;
+  while (canContinue) {
+    let match = mutPendingQueries[consumedCount];
+    if (match !== undefined) {
+      let fetchedBlock = match.fetchedBlock;
+      if (fetchedBlock !== undefined && match.fromBlock <= (latestFetchedBlock$1.blockNumber + 1 | 0)) {
+        latestFetchedBlock$1 = fetchedBlock;
+        consumedCount = consumedCount + 1 | 0;
+      } else {
+        canContinue = false;
       }
-      return tmp;
-    })()) {
-    let removedQuery = mutPendingQueries.shift();
-    latestFetchedBlock$1 = removedQuery.fetchedBlock;
+    } else {
+      canContinue = false;
+    }
   };
+  if (consumedCount > 0) {
+    mutPendingQueries.splice(0, consumedCount);
+  }
   let updatedLatestFetchedBlock = latestFetchedBlock$1;
   let mergeBlock = p$1.mergeBlock;
   let partitionReachedMergeBlock = mergeBlock !== undefined ? updatedLatestFetchedBlock.blockNumber >= mergeBlock : false;
   if (partitionReachedMergeBlock) {
     Utils.Dict.deleteInPlace(mutEntities, p$1.id);
-  } else {
-    let updatedMainPartition_id = p$1.id;
-    let updatedMainPartition_selection = p$1.selection;
-    let updatedMainPartition_addressesByContractName = p$1.addressesByContractName;
-    let updatedMainPartition_mergeBlock = p$1.mergeBlock;
-    let updatedMainPartition_dynamicContract = p$1.dynamicContract;
-    let updatedMainPartition_mutPendingQueries = p$1.mutPendingQueries;
-    let updatedMainPartition_latestBlockRangeUpdateBlock = shouldUpdateBlockRange ? latestFetchedBlock.blockNumber : p$1.latestBlockRangeUpdateBlock;
-    let updatedMainPartition = {
-      id: updatedMainPartition_id,
-      latestFetchedBlock: updatedLatestFetchedBlock,
-      selection: updatedMainPartition_selection,
-      addressesByContractName: updatedMainPartition_addressesByContractName,
-      mergeBlock: updatedMainPartition_mergeBlock,
-      dynamicContract: updatedMainPartition_dynamicContract,
-      mutPendingQueries: updatedMainPartition_mutPendingQueries,
-      prevQueryRange: updatedPrevQueryRange,
-      prevPrevQueryRange: updatedPrevPrevQueryRange,
-      prevRangeSize: updatedPrevRangeSize,
-      latestBlockRangeUpdateBlock: updatedMainPartition_latestBlockRangeUpdateBlock
+    return make(Object.values(mutEntities), optimizedPartitions.maxAddrInPartition, optimizedPartitions.nextPartitionIndex, optimizedPartitions.dynamicContracts);
+  }
+  let updatedMainPartition_id = p$1.id;
+  let updatedMainPartition_selection = p$1.selection;
+  let updatedMainPartition_addressesByContractName = p$1.addressesByContractName;
+  let updatedMainPartition_mergeBlock = p$1.mergeBlock;
+  let updatedMainPartition_dynamicContract = p$1.dynamicContract;
+  let updatedMainPartition_mutPendingQueries = p$1.mutPendingQueries;
+  let updatedMainPartition_latestSourceRangeCapacityUpdateBlock = shouldUpdateSourceRangeCapacity ? latestFetchedBlock.blockNumber : p$1.latestSourceRangeCapacityUpdateBlock;
+  let updatedMainPartition = {
+    id: updatedMainPartition_id,
+    latestFetchedBlock: updatedLatestFetchedBlock,
+    selection: updatedMainPartition_selection,
+    addressesByContractName: updatedMainPartition_addressesByContractName,
+    mergeBlock: updatedMainPartition_mergeBlock,
+    dynamicContract: updatedMainPartition_dynamicContract,
+    mutPendingQueries: updatedMainPartition_mutPendingQueries,
+    sourceRangeCapacity: updatedSourceRangeCapacity,
+    prevSourceRangeCapacity: updatedPrevSourceRangeCapacity,
+    eventDensity: updatedEventDensity,
+    latestSourceRangeCapacityUpdateBlock: updatedMainPartition_latestSourceRangeCapacityUpdateBlock
+  };
+  mutEntities[p$1.id] = updatedMainPartition;
+  if (optimizedPartitions.dynamicContracts.size !== 0) {
+    return make(Object.values(mutEntities), optimizedPartitions.maxAddrInPartition, optimizedPartitions.nextPartitionIndex, optimizedPartitions.dynamicContracts);
+  }
+  let ids = optimizedPartitions.idsInAscOrder;
+  let count = ids.length;
+  let idx = ids.indexOf(p$1.id);
+  let isAfter = jdx => {
+    if (jdx < count) {
+      return mutEntities[ids[jdx]].latestFetchedBlock.blockNumber < updatedLatestFetchedBlock.blockNumber;
+    } else {
+      return false;
+    }
+  };
+  if (!isAfter(idx + 1 | 0)) {
+    return {
+      idsInAscOrder: optimizedPartitions.idsInAscOrder,
+      entities: mutEntities,
+      maxAddrInPartition: optimizedPartitions.maxAddrInPartition,
+      nextPartitionIndex: optimizedPartitions.nextPartitionIndex,
+      dynamicContracts: optimizedPartitions.dynamicContracts
     };
-    mutEntities[p$1.id] = updatedMainPartition;
   }
-  return make(Object.values(mutEntities), optimizedPartitions.maxAddrInPartition, optimizedPartitions.nextPartitionIndex, optimizedPartitions.dynamicContracts);
+  let reordered = ids.slice();
+  let jdx = idx;
+  while (isAfter(jdx + 1 | 0)) {
+    reordered[jdx] = ids[jdx + 1 | 0];
+    jdx = jdx + 1 | 0;
+  };
+  reordered[jdx] = p$1.id;
+  return {
+    idsInAscOrder: reordered,
+    entities: mutEntities,
+    maxAddrInPartition: optimizedPartitions.maxAddrInPartition,
+    nextPartitionIndex: optimizedPartitions.nextPartitionIndex,
+    dynamicContracts: optimizedPartitions.dynamicContracts
+  };
 }
 
 function getLatestFullyFetchedBlock(optimizedPartitions) {
@@ -476,13 +539,90 @@ function bufferReadyCount(fetchState) {
   return lo;
 }
 
+function getRegistrationIndex(item) {
+  if (item.kind === 0) {
+    return item.onEventRegistration.index;
+  } else {
+    return item.onBlockRegistration.index;
+  }
+}
+
 function compareBufferItem(a, b) {
-  let blockOrdering = Primitive_int.compare(a.blockNumber, b.blockNumber);
-  if (blockOrdering === 0) {
-    return Primitive_int.compare(a.logIndex, b.logIndex);
+  let ba = a.blockNumber;
+  let bb = b.blockNumber;
+  if (ba !== bb) {
+    if (ba < bb) {
+      return -1;
+    } else {
+      return 1;
+    }
+  }
+  let la = a.logIndex;
+  let lb = b.logIndex;
+  if (la !== lb) {
+    if (la < lb) {
+      return -1;
+    } else {
+      return 1;
+    }
+  }
+  let ia = getRegistrationIndex(a);
+  let ib = getRegistrationIndex(b);
+  if (ia < ib) {
+    return -1;
+  } else if (ia > ib) {
+    return 1;
   } else {
-    return blockOrdering;
+    return 0;
+  }
+}
+
+function mergeIntoBuffer(buffer, newItems) {
+  let n = newItems.length;
+  for (let i = 1; i < n; ++i) {
+    let x = newItems[i];
+    let j = i - 1 | 0;
+    while (j >= 0 && compareBufferItem(newItems[j], x) > 0) {
+      newItems[j + 1 | 0] = newItems[j];
+      j = j - 1 | 0;
+    };
+    newItems[j + 1 | 0] = x;
   }
+  let m = buffer.length;
+  let merged = [];
+  let last = {
+    contents: undefined
+  };
+  let push = item => {
+    let l = last.contents;
+    if (l !== undefined && compareBufferItem(l, item) === 0) {
+      return;
+    }
+    merged.push(item);
+    last.contents = item;
+  };
+  let i$1 = 0;
+  let j$1 = 0;
+  while (i$1 < m && j$1 < n) {
+    let a = buffer[i$1];
+    let b = newItems[j$1];
+    if (compareBufferItem(a, b) <= 0) {
+      push(a);
+      i$1 = i$1 + 1 | 0;
+    } else {
+      push(b);
+      j$1 = j$1 + 1 | 0;
+    }
+  };
+  while (i$1 < m) {
+    push(buffer[i$1]);
+    i$1 = i$1 + 1 | 0;
+  };
+  while (j$1 < n) {
+    push(newItems[j$1]);
+    j$1 = j$1 + 1 | 0;
+  };
+  return merged;
 }
 
 function appendOnBlockItems(mutItems, onBlockRegistrations, indexerStartBlock, fromBlock, maxBlockNumber, maxOnBlockBufferSize) {
@@ -514,18 +654,19 @@ function appendOnBlockItems(mutItems, onBlockRegistrations, indexerStartBlock, f
   return latestOnBlockBlockNumber;
 }
 
-function updateInternal(fetchState, optimizedPartitionsOpt, mutItems, blockLagOpt, knownHeightOpt) {
+function updateInternal(fetchState, optimizedPartitionsOpt, mutItems, mutItemsSortedOpt, blockLagOpt, knownHeightOpt) {
   let optimizedPartitions = optimizedPartitionsOpt !== undefined ? optimizedPartitionsOpt : fetchState.optimizedPartitions;
+  let mutItemsSorted = mutItemsSortedOpt !== undefined ? mutItemsSortedOpt : false;
   let blockLag = blockLagOpt !== undefined ? blockLagOpt : fetchState.blockLag;
   let knownHeight = knownHeightOpt !== undefined ? knownHeightOpt : fetchState.knownHeight;
-  let mutItemsRef = mutItems;
+  let base = mutItems !== undefined ? (
+      mutItemsSorted ? mutItems : mergeIntoBuffer([], mutItems)
+    ) : fetchState.buffer;
+  let blockItems = [];
   let onBlockRegistrations = fetchState.onBlockRegistrations;
   let latestOnBlockBlockNumber;
   if (onBlockRegistrations.length !== 0) {
-    let mutItems$1 = mutItemsRef;
-    let item = (
-        mutItems$1 !== undefined ? mutItems$1 : fetchState.buffer
-      )[fetchState.maxOnBlockBufferSize - 1 | 0];
+    let item = base[fetchState.maxOnBlockBufferSize - 1 | 0];
     let maxBlockNumber;
     if (item !== undefined) {
       maxBlockNumber = item.blockNumber;
@@ -534,20 +675,16 @@ function updateInternal(fetchState, optimizedPartitionsOpt, mutItems, blockLagOp
       let latestFullyFetchedBlock = id !== undefined ? optimizedPartitions.entities[id].latestFetchedBlock : undefined;
       maxBlockNumber = latestFullyFetchedBlock !== undefined ? latestFullyFetchedBlock.blockNumber : knownHeight;
     }
-    let mutItems$2 = mutItemsRef;
-    let mutItems$3 = mutItems$2 !== undefined ? mutItems$2 : fetchState.buffer.slice();
-    mutItemsRef = mutItems$3;
-    latestOnBlockBlockNumber = appendOnBlockItems(mutItems$3, onBlockRegistrations, fetchState.startBlock, fetchState.latestOnBlockBlockNumber, maxBlockNumber, fetchState.maxOnBlockBufferSize);
+    latestOnBlockBlockNumber = appendOnBlockItems(blockItems, onBlockRegistrations, fetchState.startBlock, fetchState.latestOnBlockBlockNumber, maxBlockNumber, fetchState.maxOnBlockBufferSize);
   } else {
     latestOnBlockBlockNumber = knownHeight;
   }
-  let mutItems$4 = mutItemsRef;
   let updatedFetchState_startBlock = fetchState.startBlock;
   let updatedFetchState_endBlock = fetchState.endBlock;
   let updatedFetchState_normalSelection = fetchState.normalSelection;
   let updatedFetchState_contractConfigs = fetchState.contractConfigs;
   let updatedFetchState_chainId = fetchState.chainId;
-  let updatedFetchState_buffer = mutItems$4 !== undefined ? (mutItems$4.sort(compareBufferItem), mutItems$4) : fetchState.buffer;
+  let updatedFetchState_buffer = blockItems.length !== 0 ? mergeIntoBuffer(base, blockItems) : base;
   let updatedFetchState_maxOnBlockBufferSize = fetchState.maxOnBlockBufferSize;
   let updatedFetchState_onBlockRegistrations = fetchState.onBlockRegistrations;
   let updatedFetchState_firstEventBlock = fetchState.firstEventBlock;
@@ -657,10 +794,10 @@ function createPartitionsFromIndexingAddresses(registeringContractsByContract, d
             mergeBlock: undefined,
             dynamicContract: isDynamic ? contractName : undefined,
             mutPendingQueries: [],
-            prevQueryRange: 0,
-            prevPrevQueryRange: 0,
-            prevRangeSize: 0,
-            latestBlockRangeUpdateBlock: 0
+            sourceRangeCapacity: 0,
+            prevSourceRangeCapacity: 0,
+            eventDensity: undefined,
+            latestSourceRangeCapacityUpdateBlock: 0
           });
           nextPartitionIndexRef = nextPartitionIndexRef + 1 | 0;
         };
@@ -708,10 +845,10 @@ function createPartitionsFromIndexingAddresses(registeringContractsByContract, d
             mergeBlock: currentPBlock < nextPBlock ? nextPBlock : undefined,
             dynamicContract: currentP.dynamicContract,
             mutPendingQueries: currentP.mutPendingQueries,
-            prevQueryRange: currentP.prevQueryRange,
-            prevPrevQueryRange: currentP.prevPrevQueryRange,
-            prevRangeSize: currentP.prevRangeSize,
-            latestBlockRangeUpdateBlock: currentP.latestBlockRangeUpdateBlock
+            sourceRangeCapacity: currentP.sourceRangeCapacity,
+            prevSourceRangeCapacity: currentP.prevSourceRangeCapacity,
+            eventDensity: currentP.eventDensity,
+            latestSourceRangeCapacityUpdateBlock: currentP.latestSourceRangeCapacityUpdateBlock
           });
           currentPRef = {
             id: nextP.id,
@@ -721,10 +858,10 @@ function createPartitionsFromIndexingAddresses(registeringContractsByContract, d
             mergeBlock: nextP.mergeBlock,
             dynamicContract: nextP.dynamicContract,
             mutPendingQueries: nextP.mutPendingQueries,
-            prevQueryRange: nextP.prevQueryRange,
-            prevPrevQueryRange: nextP.prevPrevQueryRange,
-            prevRangeSize: nextP.prevRangeSize,
-            latestBlockRangeUpdateBlock: nextP.latestBlockRangeUpdateBlock
+            sourceRangeCapacity: nextP.sourceRangeCapacity,
+            prevSourceRangeCapacity: nextP.prevSourceRangeCapacity,
+            eventDensity: nextP.eventDensity,
+            latestSourceRangeCapacityUpdateBlock: nextP.latestSourceRangeCapacityUpdateBlock
           };
         } else {
           currentPRef = {
@@ -735,10 +872,10 @@ function createPartitionsFromIndexingAddresses(registeringContractsByContract, d
             mergeBlock: currentP.mergeBlock,
             dynamicContract: currentP.dynamicContract,
             mutPendingQueries: currentP.mutPendingQueries,
-            prevQueryRange: currentP.prevQueryRange,
-            prevPrevQueryRange: currentP.prevPrevQueryRange,
-            prevRangeSize: currentP.prevRangeSize,
-            latestBlockRangeUpdateBlock: currentP.latestBlockRangeUpdateBlock
+            sourceRangeCapacity: currentP.sourceRangeCapacity,
+            prevSourceRangeCapacity: currentP.prevSourceRangeCapacity,
+            eventDensity: currentP.eventDensity,
+            latestSourceRangeCapacityUpdateBlock: currentP.latestSourceRangeCapacityUpdateBlock
           };
         }
       }
@@ -885,10 +1022,10 @@ function registerDynamicContracts(fetchState, indexingAddresses, items) {
                 mergeBlock: p.mergeBlock,
                 dynamicContract: p.dynamicContract,
                 mutPendingQueries: p.mutPendingQueries,
-                prevQueryRange: p.prevQueryRange,
-                prevPrevQueryRange: p.prevPrevQueryRange,
-                prevRangeSize: p.prevRangeSize,
-                latestBlockRangeUpdateBlock: p.latestBlockRangeUpdateBlock
+                sourceRangeCapacity: p.sourceRangeCapacity,
+                prevSourceRangeCapacity: p.prevSourceRangeCapacity,
+                eventDensity: p.eventDensity,
+                latestSourceRangeCapacityUpdateBlock: p.latestSourceRangeCapacityUpdateBlock
               };
               let addressesByContractName = {};
               addressesByContractName[contractName] = addresses;
@@ -900,10 +1037,10 @@ function registerDynamicContracts(fetchState, indexingAddresses, items) {
                 mergeBlock: undefined,
                 dynamicContract: contractName,
                 mutPendingQueries: p.mutPendingQueries,
-                prevQueryRange: p.prevQueryRange,
-                prevPrevQueryRange: p.prevPrevQueryRange,
-                prevRangeSize: p.prevRangeSize,
-                latestBlockRangeUpdateBlock: p.latestBlockRangeUpdateBlock
+                sourceRangeCapacity: p.sourceRangeCapacity,
+                prevSourceRangeCapacity: p.prevSourceRangeCapacity,
+                eventDensity: p.eventDensity,
+                latestSourceRangeCapacityUpdateBlock: p.latestSourceRangeCapacityUpdateBlock
               });
             }
           } else {
@@ -915,10 +1052,10 @@ function registerDynamicContracts(fetchState, indexingAddresses, items) {
               mergeBlock: p.mergeBlock,
               dynamicContract: contractName,
               mutPendingQueries: p.mutPendingQueries,
-              prevQueryRange: p.prevQueryRange,
-              prevPrevQueryRange: p.prevPrevQueryRange,
-              prevRangeSize: p.prevRangeSize,
-              latestBlockRangeUpdateBlock: p.latestBlockRangeUpdateBlock
+              sourceRangeCapacity: p.sourceRangeCapacity,
+              prevSourceRangeCapacity: p.prevSourceRangeCapacity,
+              eventDensity: p.eventDensity,
+              latestSourceRangeCapacityUpdateBlock: p.latestSourceRangeCapacityUpdateBlock
             };
           }
         }
@@ -929,11 +1066,11 @@ function registerDynamicContracts(fetchState, indexingAddresses, items) {
   }
   IndexingAddresses.register(indexingAddresses, noEventsAddresses);
   let optimizedPartitions = createPartitionsFromIndexingAddresses(registeringContractsByContract, dynamicContractsRef, fetchState.normalSelection, fetchState.optimizedPartitions.maxAddrInPartition, fetchState.optimizedPartitions.nextPartitionIndex + newPartitions.length | 0, mutExistingPartitions.concat(newPartitions), 0);
-  return updateInternal(fetchState, optimizedPartitions, undefined, undefined, undefined);
+  return updateInternal(fetchState, optimizedPartitions, undefined, undefined, undefined, undefined);
 }
 
-function handleQueryResult(fetchState, indexingAddresses, query, latestFetchedBlock, newItems) {
-  let newItems$1 = newItems.filter(item => {
+function filterByClientAddress(items, indexingAddresses) {
+  return items.filter(item => {
     if (item.kind !== 0) {
       return true;
     }
@@ -944,7 +1081,10 @@ function handleQueryResult(fetchState, indexingAddresses, query, latestFetchedBl
       return true;
     }
   });
-  return updateInternal(fetchState, handleQueryResponse(fetchState.optimizedPartitions, query, fetchState.knownHeight, newItems$1.length, latestFetchedBlock), newItems$1.length !== 0 ? fetchState.buffer.concat(newItems$1) : undefined, undefined, undefined);
+}
+
+function handleQueryResult(fetchState, query, latestFetchedBlock, newItems) {
+  return updateInternal(fetchState, handleQueryResponse(fetchState.optimizedPartitions, query, fetchState.knownHeight, newItems.length, latestFetchedBlock), newItems.length !== 0 ? mergeIntoBuffer(fetchState.buffer, newItems) : undefined, true, undefined, undefined);
 }
 
 function startFetchingQueries(param, queries) {
@@ -959,6 +1099,7 @@ function startFetchingQueries(param, queries) {
       toBlock: q.toBlock,
       isChunk: q.isChunk,
       itemsTarget: q.itemsTarget,
+      itemsEst: q.itemsEst,
       fetchedBlock: undefined
     };
     let inserted = false;
@@ -976,73 +1117,223 @@ function startFetchingQueries(param, queries) {
   }
 }
 
-function pushGapFillQueries(queries, partitionId, rangeFromBlock, rangeEndBlock, knownHeight, chainTargetBlock, maybeChunkRange, maxChunks, partition, selection, addressesByContractName) {
-  let cost = 0;
-  if (rangeFromBlock <= knownHeight && maxChunks > 0) {
-    let exit = 0;
-    if (!(rangeEndBlock !== undefined && rangeFromBlock > rangeEndBlock)) {
-      exit = 1;
+function pushDensityPricedQuery(queries, partitionId, fromBlock, toBlock, isChunk, density, chunkItemsMultiplier, chainTargetBlock, itemsTargetFloor, selection, addressesByContractName) {
+  let itemsEst = densityItemsTarget(density, fromBlock, toBlock, chainTargetBlock);
+  let itemsTarget = densityItemsTarget(density * chunkItemsMultiplier, fromBlock, toBlock, chainTargetBlock);
+  queries.push({
+    partitionId: partitionId,
+    fromBlock: fromBlock,
+    toBlock: toBlock,
+    isChunk: isChunk,
+    itemsTarget: toBlock !== undefined ? Primitive_int.max(itemsTarget, itemsTargetFloor) : itemsTarget,
+    itemsEst: itemsEst,
+    selection: selection,
+    addressesByContractName: addressesByContractName
+  });
+  return itemsEst;
+}
+
+function pushGapFillQueries(queries, partitionId, rangeFromBlock, rangeEndBlock, headBlockNumber, chainTargetBlock, maybeChunkRange, maxChunks, partition, partitionBudget, chunkItemsMultiplier, itemsTargetFloor, selection, addressesByContractName) {
+  if (!(rangeFromBlock <= Primitive_int.min(headBlockNumber, chainTargetBlock) && maxChunks > 0)) {
+    return;
+  }
+  let exit = 0;
+  if (rangeEndBlock !== undefined) {
+    if (rangeFromBlock > rangeEndBlock) {
+      return;
     }
-    if (exit === 1) {
+    exit = 1;
+  } else {
+    exit = 1;
+  }
+  if (exit === 1) {
+    let maxBlock = rangeEndBlock !== undefined ? rangeEndBlock : chainTargetBlock;
+    let pushSingleQuery = (density, isChunk) => {
+      pushDensityPricedQuery(queries, partitionId, rangeFromBlock, rangeEndBlock, isChunk, density, chunkItemsMultiplier, chainTargetBlock, itemsTargetFloor, selection, addressesByContractName);
+    };
+    let match = partition.eventDensity;
+    let exit$1 = 0;
+    if (match !== undefined) {
       if (maybeChunkRange !== undefined) {
-        let maxBlock = rangeEndBlock !== undefined ? rangeEndBlock : chainTargetBlock;
-        let chunkSize = Js_math.ceil_int(maybeChunkRange * 1.8);
-        if (((rangeFromBlock + (chunkSize << 1) | 0) - 1 | 0) <= maxBlock) {
+        if (match > 0) {
+          let chunkSize = Js_math.ceil_int(maybeChunkRange * 1.8);
+          if (((rangeFromBlock + (chunkSize << 1) | 0) - 1 | 0) > maxBlock) {
+            return pushSingleQuery(match, rangeEndBlock !== undefined);
+          }
           let chunkFromBlock = rangeFromBlock;
           let chunkIdx = 0;
           while (chunkIdx < maxChunks && ((chunkFromBlock + chunkSize | 0) - 1 | 0) <= maxBlock) {
             let chunkToBlock = (chunkFromBlock + chunkSize | 0) - 1 | 0;
-            let itemsTarget = densityItemsTarget(partition, chunkFromBlock, chunkToBlock, chainTargetBlock);
-            queries.push({
-              partitionId: partitionId,
-              fromBlock: chunkFromBlock,
-              toBlock: chunkToBlock,
-              isChunk: true,
-              itemsTarget: itemsTarget,
-              selection: selection,
-              addressesByContractName: addressesByContractName
-            });
-            cost = cost + itemsTarget;
+            pushDensityPricedQuery(queries, partitionId, chunkFromBlock, chunkToBlock, true, match, chunkItemsMultiplier, chainTargetBlock, itemsTargetFloor, selection, addressesByContractName);
             chunkFromBlock = chunkToBlock + 1 | 0;
             chunkIdx = chunkIdx + 1 | 0;
           };
-        } else {
-          let itemsTarget$1 = densityItemsTarget(partition, rangeFromBlock, rangeEndBlock, chainTargetBlock);
-          queries.push({
-            partitionId: partitionId,
-            fromBlock: rangeFromBlock,
-            toBlock: rangeEndBlock,
-            isChunk: rangeEndBlock !== undefined,
-            itemsTarget: itemsTarget$1,
-            selection: selection,
-            addressesByContractName: addressesByContractName
-          });
-          cost = cost + itemsTarget$1;
+          return;
         }
+        exit$1 = 2;
       } else {
-        let itemsTarget$2 = densityItemsTarget(partition, rangeFromBlock, rangeEndBlock, chainTargetBlock);
-        queries.push({
-          partitionId: partitionId,
-          fromBlock: rangeFromBlock,
-          toBlock: rangeEndBlock,
-          isChunk: false,
-          itemsTarget: itemsTarget$2,
-          selection: selection,
-          addressesByContractName: addressesByContractName
-        });
-        cost = cost + itemsTarget$2;
+        if (match > 0) {
+          return pushSingleQuery(match, false);
+        }
+        exit$1 = 2;
+      }
+    } else {
+      exit$1 = 2;
+    }
+    if (exit$1 === 2) {
+      let remainingRange = Primitive_int.max(1, (chainTargetBlock - rangeFromBlock | 0) + 1 | 0);
+      return pushSingleQuery(partitionBudget / remainingRange, false);
+    }
+  }
+}
+
+function walkPartitionPending(p, partitionId, inFlightCount, candidates, headBlockNumber, chainTargetBlock, chunkItemsMultiplier, itemsTargetFloor, partitionBudget, queryEndBlock) {
+  let maybeChunkRange = getMinHistoryRange(p);
+  let pendingCount = p.mutPendingQueries.length;
+  let cursor = p.latestFetchedBlock.blockNumber + 1 | 0;
+  let canContinue = true;
+  let chunksUsedThisCall = 0;
+  let pqIdx = 0;
+  while (pqIdx < pendingCount && canContinue) {
+    let pq = p.mutPendingQueries[pqIdx];
+    if (pq.fromBlock > cursor) {
+      let beforeLen = candidates.length;
+      pushGapFillQueries(candidates, partitionId, cursor, Utils.$$Math.minOptInt(pq.fromBlock - 1 | 0, queryEndBlock), headBlockNumber, chainTargetBlock, maybeChunkRange, (12 - inFlightCount | 0) - chunksUsedThisCall | 0, p, partitionBudget, chunkItemsMultiplier, itemsTargetFloor, p.selection, p.addressesByContractName);
+      chunksUsedThisCall = chunksUsedThisCall + (candidates.length - beforeLen | 0) | 0;
+    }
+    let toBlock = pq.toBlock;
+    if (toBlock !== undefined && pq.isChunk) {
+      let match = pq.fetchedBlock;
+      if (match !== undefined) {
+        let blockNumber = match.blockNumber;
+        cursor = blockNumber < toBlock ? blockNumber + 1 | 0 : toBlock + 1 | 0;
+      } else {
+        cursor = toBlock + 1 | 0;
       }
+    } else {
+      canContinue = false;
     }
+    pqIdx = pqIdx + 1 | 0;
+  };
+  if (canContinue) {
+    return {
+      partitionId: partitionId,
+      p: p,
+      cursor: cursor,
+      chunksUsedThisCall: chunksUsedThisCall,
+      inFlightCount: inFlightCount,
+      queryEndBlock: queryEndBlock,
+      maybeChunkRange: maybeChunkRange
+    };
   }
-  return cost;
 }
 
-function getNextQuery(param, chainTargetBlock, chainTargetItems) {
-  let knownHeight = param.knownHeight;
+function pushForwardCandidates(candidates, inRangeStates, inRangeCount, chainTargetBlock, freshBudget, chunkItemsMultiplier, itemsTargetFloor) {
+  let probeShare = inRangeCount === 0 ? 0 : freshBudget / inRangeCount;
+  let frontierCursor = Stdlib_Array.reduce(inRangeStates, chainTargetBlock, (min, fs) => {
+    if (fs.cursor < min) {
+      return fs.cursor;
+    } else {
+      return min;
+    }
+  });
+  let rangeToTarget = (chainTargetBlock - frontierCursor | 0) + 1 | 0;
+  let rangeTargetDensity = inRangeCount > 0 && rangeToTarget > 0 ? freshBudget / rangeToTarget : 0;
+  inRangeStates.forEach(fs => {
+    let p = fs.p;
+    let eb = fs.queryEndBlock;
+    let maxBlock = eb !== undefined ? eb : chainTargetBlock;
+    let match = fs.maybeChunkRange;
+    let match$1 = p.eventDensity;
+    if (match !== undefined && match$1 !== undefined && match$1 > 0) {
+      let chunkSize = Js_math.ceil_int(match * 1.8);
+      let maxChunksRemaining = (12 - fs.inFlightCount | 0) - fs.chunksUsedThisCall | 0;
+      let chunkStartCeiling = Primitive_int.min(maxBlock, chainTargetBlock);
+      let created = 0;
+      let chunkFromBlock = fs.cursor;
+      let generatedItems = 0;
+      while (created < maxChunksRemaining && chunkFromBlock <= chunkStartCeiling && generatedItems <= freshBudget) {
+        let chunkToBlock = Primitive_int.min((chunkFromBlock + chunkSize | 0) - 1 | 0, maxBlock);
+        let itemsEst = pushDensityPricedQuery(candidates, fs.partitionId, chunkFromBlock, chunkToBlock, true, match$1, chunkItemsMultiplier, chainTargetBlock, itemsTargetFloor, p.selection, p.addressesByContractName);
+        generatedItems = generatedItems + itemsEst;
+        chunkFromBlock = chunkToBlock + 1 | 0;
+        created = created + 1 | 0;
+      };
+      return;
+    }
+    let itemsTarget = rangeToTarget > 0 ? Primitive_int.max(1, Math.round(rangeTargetDensity * ((chainTargetBlock - fs.cursor | 0) + 1 | 0) / inRangeCount) | 0) : Primitive_int.max(1, Math.round(probeShare) | 0);
+    candidates.push({
+      partitionId: fs.partitionId,
+      fromBlock: fs.cursor,
+      toBlock: fs.queryEndBlock,
+      isChunk: false,
+      itemsTarget: itemsTarget,
+      itemsEst: itemsTarget,
+      selection: p.selection,
+      addressesByContractName: p.addressesByContractName
+    });
+  });
+}
+
+function acceptCandidates(candidates, reservations, chainTargetItems, partitionIndexById, queriesByPartitionIndex) {
+  let acceptanceStream = [];
+  candidates.forEach(query => {
+    acceptanceStream.push([
+      query.fromBlock,
+      query.itemsEst,
+      query
+    ]);
+  });
+  reservations.forEach(param => {
+    acceptanceStream.push([
+      param[0],
+      param[1],
+      undefined
+    ]);
+  });
+  acceptanceStream.sort((param, param$1) => {
+    let bFrom = param$1[0];
+    let aFrom = param[0];
+    if (aFrom !== bFrom) {
+      return Primitive_int.compare(aFrom, bFrom);
+    }
+    let bQuery = param$1[2];
+    if (param[2] !== undefined) {
+      if (bQuery !== undefined) {
+        return 0;
+      } else {
+        return 1;
+      }
+    } else if (bQuery !== undefined) {
+      return -1;
+    } else {
+      return 0;
+    }
+  });
+  let streamCount = acceptanceStream.length;
+  let remainingBudget = chainTargetItems;
+  let acceptIdx = 0;
+  let usedConcurrency = reservations.length;
+  while (remainingBudget > 0 && acceptIdx < streamCount && usedConcurrency < 100) {
+    let match = acceptanceStream[acceptIdx];
+    let maybeQuery = match[2];
+    if (maybeQuery !== undefined) {
+      let partitionIdx = partitionIndexById[maybeQuery.partitionId];
+      queriesByPartitionIndex[partitionIdx].push(maybeQuery);
+      usedConcurrency = usedConcurrency + 1 | 0;
+    }
+    remainingBudget = remainingBudget - match[1];
+    acceptIdx = acceptIdx + 1 | 0;
+  };
+}
+
+function getNextQuery(param, chainTargetBlock, chainTargetItems, $staropt$star, $staropt$star$1) {
   let blockLag = param.blockLag;
   let endBlock = param.endBlock;
   let optimizedPartitions = param.optimizedPartitions;
-  let headBlockNumber = knownHeight - blockLag | 0;
+  let chunkItemsMultiplier = $staropt$star !== undefined ? $staropt$star : 1;
+  let itemsTargetFloor = $staropt$star$1 !== undefined ? $staropt$star$1 : 0;
+  let headBlockNumber = param.knownHeight - blockLag | 0;
   if (headBlockNumber <= 0) {
     return "WaitingForNewBlock";
   }
@@ -1051,14 +1342,34 @@ function getNextQuery(param, chainTargetBlock, chainTargetItems) {
     endBlock !== undefined ? headBlockNumber < endBlock : true
   ) && !isOnBlockBehindTheHead;
   let partitionsCount = optimizedPartitions.idsInAscOrder.length;
+  let inFlightCounts = Array(partitionsCount);
+  let reservations = [];
+  let chainReserved = 0;
+  let partitionIndexById = {};
   for (let idx = 0; idx < partitionsCount; ++idx) {
     let partitionId = optimizedPartitions.idsInAscOrder[idx];
     let p = optimizedPartitions.entities[partitionId];
+    partitionIndexById[partitionId] = idx;
+    let inFlightCount = 0;
+    for (let pqIdx = 0, pqIdx_finish = p.mutPendingQueries.length; pqIdx < pqIdx_finish; ++pqIdx) {
+      let pq = p.mutPendingQueries[pqIdx];
+      if (pq.fetchedBlock === undefined) {
+        inFlightCount = inFlightCount + 1 | 0;
+        chainReserved = chainReserved + pq.itemsEst;
+        reservations.push([
+          pq.fromBlock,
+          pq.itemsEst
+        ]);
+      }
+    }
+    inFlightCounts[idx] = inFlightCount;
     if (p.mutPendingQueries.length !== 0 || p.latestFetchedBlock.blockNumber < headBlockNumber) {
       shouldWaitForNewBlock = false;
     }
   }
-  let queriesByPartitionIndex = Stdlib_Array.fromInitializer(partitionsCount, param => []);
+  let availableConcurrency = 100 - reservations.length | 0;
+  let partitionsCount$1 = chainTargetItems <= 0 || shouldWaitForNewBlock || availableConcurrency <= 0 ? 0 : partitionsCount;
+  let queriesByPartitionIndex = Stdlib_Array.fromInitializer(partitionsCount$1, param => []);
   let computeQueryEndBlock = p => {
     let queryEndBlock = Utils.$$Math.minOptInt(endBlock, p.mergeBlock);
     if (blockLag !== 0) {
@@ -1067,124 +1378,18 @@ function getNextQuery(param, chainTargetBlock, chainTargetItems) {
       return queryEndBlock;
     }
   };
-  let existingReservedByPartition = {};
-  let chainReserved = 0;
-  for (let idx$1 = 0; idx$1 < partitionsCount; ++idx$1) {
+  let candidates = [];
+  let partitionBudget = partitionsCount$1 === 0 ? 0 : chainTargetItems / partitionsCount$1;
+  let fillStates = [];
+  for (let idx$1 = 0; idx$1 < partitionsCount$1; ++idx$1) {
     let partitionId$1 = optimizedPartitions.idsInAscOrder[idx$1];
     let p$1 = optimizedPartitions.entities[partitionId$1];
-    let cost = 0;
-    for (let pqIdx = 0, pqIdx_finish = p$1.mutPendingQueries.length; pqIdx < pqIdx_finish; ++pqIdx) {
-      cost = cost + p$1.mutPendingQueries[pqIdx].itemsTarget;
+    let fillState = walkPartitionPending(p$1, partitionId$1, inFlightCounts[idx$1], candidates, headBlockNumber, chainTargetBlock, chunkItemsMultiplier, itemsTargetFloor, partitionBudget, computeQueryEndBlock(p$1));
+    if (fillState !== undefined) {
+      fillStates.push(fillState);
     }
-    existingReservedByPartition[partitionId$1] = cost;
-    chainReserved = chainReserved + cost;
   }
-  let fillStates = [];
-  let gapFillCost = 0;
-  for (let idx$2 = 0; idx$2 < partitionsCount; ++idx$2) {
-    let partitionId$2 = optimizedPartitions.idsInAscOrder[idx$2];
-    let p$2 = optimizedPartitions.entities[partitionId$2];
-    let bucket = queriesByPartitionIndex[idx$2];
-    let pendingCount = p$2.mutPendingQueries.length;
-    let queryEndBlock = computeQueryEndBlock(p$2);
-    let maybeChunkRange = getMinHistoryRange(p$2);
-    let cursor = p$2.latestFetchedBlock.blockNumber + 1 | 0;
-    let canContinue = true;
-    let chunksUsedThisCall = 0;
-    let pqIdx$1 = 0;
-    while (pqIdx$1 < pendingCount && canContinue) {
-      let pq = p$2.mutPendingQueries[pqIdx$1];
-      if (pq.fromBlock > cursor) {
-        let beforeLen = bucket.length;
-        let cost$1 = pushGapFillQueries(bucket, partitionId$2, cursor, Utils.$$Math.minOptInt(pq.fromBlock - 1 | 0, queryEndBlock), knownHeight, chainTargetBlock, maybeChunkRange, (10 - pendingCount | 0) - chunksUsedThisCall | 0, p$2, p$2.selection, p$2.addressesByContractName);
-        chunksUsedThisCall = chunksUsedThisCall + (bucket.length - beforeLen | 0) | 0;
-        gapFillCost = gapFillCost + cost$1;
-      }
-      let toBlock = pq.toBlock;
-      if (toBlock !== undefined && pq.isChunk) {
-        let match = pq.fetchedBlock;
-        if (match !== undefined) {
-          let blockNumber = match.blockNumber;
-          cursor = blockNumber < toBlock ? blockNumber + 1 | 0 : toBlock + 1 | 0;
-        } else {
-          cursor = toBlock + 1 | 0;
-        }
-      } else {
-        canContinue = false;
-      }
-      pqIdx$1 = pqIdx$1 + 1 | 0;
-    };
-    if (canContinue) {
-      fillStates.push({
-        partitionId: partitionId$2,
-        p: p$2,
-        cursor: cursor,
-        chunksUsedThisCall: chunksUsedThisCall,
-        pendingCount: pendingCount,
-        queryEndBlock: queryEndBlock,
-        maybeChunkRange: maybeChunkRange,
-        bucket: bucket
-      });
-    }
-  }
-  let rangeItemsTarget = Primitive_float.max(0, chainTargetItems - chainReserved - gapFillCost);
-  let reservedByPartition = {};
-  fillStates.forEach(fs => {
-    let cost = existingReservedByPartition[fs.partitionId];
-    for (let i = 0, i_finish = fs.bucket.length; i < i_finish; ++i) {
-      cost = cost + fs.bucket[i].itemsTarget;
-    }
-    reservedByPartition[fs.partitionId] = cost;
-  });
-  let emitQueries = (fs, budget) => {
-    let p = fs.p;
-    let eb = fs.queryEndBlock;
-    let maxBlock = eb !== undefined ? eb : chainTargetBlock;
-    let minHistoryRange = fs.maybeChunkRange;
-    if (minHistoryRange !== undefined) {
-      let density = p.prevRangeSize / p.prevQueryRange;
-      let chunkSize = Js_math.ceil_int(minHistoryRange * 1.8);
-      let chunkCost = density * chunkSize;
-      let maxChunksRemaining = (10 - fs.pendingCount | 0) - fs.chunksUsedThisCall | 0;
-      let affordable = chunkCost > 0 ? Math.floor(budget / chunkCost) | 0 : maxChunksRemaining;
-      let numChunks = Primitive_int.max(1, Primitive_int.min(affordable, maxChunksRemaining));
-      let consumed = 0;
-      let created = 0;
-      let chunkFromBlock = fs.cursor;
-      while (created < numChunks && chunkFromBlock <= maxBlock) {
-        let chunkToBlock = Primitive_int.min((chunkFromBlock + chunkSize | 0) - 1 | 0, maxBlock);
-        let itemsTarget = density * ((chunkToBlock - chunkFromBlock | 0) + 1 | 0);
-        fs.bucket.push({
-          partitionId: fs.partitionId,
-          fromBlock: chunkFromBlock,
-          toBlock: chunkToBlock,
-          isChunk: true,
-          itemsTarget: itemsTarget,
-          selection: p.selection,
-          addressesByContractName: p.addressesByContractName
-        });
-        consumed = consumed + itemsTarget;
-        chunkFromBlock = chunkToBlock + 1 | 0;
-        created = created + 1 | 0;
-      };
-      fs.cursor = chunkFromBlock;
-      fs.chunksUsedThisCall = fs.chunksUsedThisCall + created | 0;
-      return consumed;
-    }
-    let itemsTarget$1 = Primitive_float.min(Math.round(budget), 10000);
-    fs.bucket.push({
-      partitionId: fs.partitionId,
-      fromBlock: fs.cursor,
-      toBlock: fs.queryEndBlock,
-      isChunk: false,
-      itemsTarget: itemsTarget$1,
-      selection: p.selection,
-      addressesByContractName: p.addressesByContractName
-    });
-    fs.cursor = maxBlock + 1 | 0;
-    fs.chunksUsedThisCall = fs.chunksUsedThisCall + 1 | 0;
-    return itemsTarget$1;
-  };
+  let freshBudget = Primitive_float.max(0, chainTargetItems - chainReserved);
   let isInRange = fs => {
     let tmp = false;
     if (fs.cursor <= chainTargetBlock) {
@@ -1192,41 +1397,16 @@ function getNextQuery(param, chainTargetBlock, chainTargetItems) {
       tmp = eb !== undefined ? fs.cursor <= eb : true;
     }
     if (tmp) {
-      return (fs.pendingCount + fs.chunksUsedThisCall | 0) < 10;
+      return (fs.inFlightCount + fs.chunksUsedThisCall | 0) < 12;
     } else {
       return false;
     }
   };
-  let notFilledPartitions = fillStates.filter(isInRange);
-  let reservedFromRange = {
-    contents: 0
-  };
-  while (notFilledPartitions.length !== 0 && reservedFromRange.contents < rangeItemsTarget) {
-    let n = notFilledPartitions.length;
-    let footprintSum = {
-      contents: 0
-    };
-    notFilledPartitions.forEach(fs => {
-      footprintSum.contents = footprintSum.contents + reservedByPartition[fs.partitionId];
-    });
-    let line = (rangeItemsTarget - reservedFromRange.contents + footprintSum.contents) / n;
-    let next = [];
-    notFilledPartitions.forEach(fs => {
-      let reserved = reservedByPartition[fs.partitionId];
-      let budget = line - reserved;
-      if (budget <= 0) {
-        return;
-      }
-      let consumed = emitQueries(fs, budget);
-      reservedByPartition[fs.partitionId] = reserved + consumed;
-      reservedFromRange.contents = reservedFromRange.contents + consumed;
-      if (isInRange(fs)) {
-        next.push(fs);
-        return;
-      }
-    });
-    notFilledPartitions = next;
-  };
+  let inRangeStates = fillStates.filter(isInRange);
+  let inRangeCount = inRangeStates.length;
+  let inRangeStates$1 = inRangeCount > availableConcurrency ? (inRangeStates.sort((a, b) => Primitive_int.compare(a.cursor, b.cursor)), inRangeStates.slice(0, availableConcurrency)) : inRangeStates;
+  pushForwardCandidates(candidates, inRangeStates$1, inRangeCount, chainTargetBlock, freshBudget, chunkItemsMultiplier, itemsTargetFloor);
+  acceptCandidates(candidates, reservations, chainTargetItems, partitionIndexById, queriesByPartitionIndex);
   let queries = queriesByPartitionIndex.flat();
   if (Utils.$$Array.isEmpty(queries)) {
     if (shouldWaitForNewBlock) {
@@ -1307,10 +1487,10 @@ function make$1(startBlock, endBlock, onEventRegistrations, contractConfigs, add
       mergeBlock: undefined,
       dynamicContract: undefined,
       mutPendingQueries: [],
-      prevQueryRange: 0,
-      prevPrevQueryRange: 0,
-      prevRangeSize: 0,
-      latestBlockRangeUpdateBlock: 0
+      sourceRangeCapacity: 0,
+      prevSourceRangeCapacity: 0,
+      eventDensity: undefined,
+      latestSourceRangeCapacityUpdateBlock: 0
     });
   }
   let normalSelection = {
@@ -1386,6 +1566,7 @@ function rollbackPendingQueries(mutPendingQueries, targetBlockNumber) {
             toBlock: pq.toBlock,
             isChunk: pq.isChunk,
             itemsTarget: pq.itemsTarget,
+            itemsEst: pq.itemsEst,
             fetchedBlock: {
               blockNumber: targetBlockNumber,
               blockTimestamp: 0
@@ -1444,10 +1625,10 @@ function rollback(fetchState, indexingAddresses, targetBlockNumber) {
             mergeBlock: mergeBlock$1,
             dynamicContract: p.dynamicContract,
             mutPendingQueries: rollbackPendingQueries(p.mutPendingQueries, targetBlockNumber),
-            prevQueryRange: p.prevQueryRange,
-            prevPrevQueryRange: p.prevPrevQueryRange,
-            prevRangeSize: p.prevRangeSize,
-            latestBlockRangeUpdateBlock: p.latestBlockRangeUpdateBlock
+            sourceRangeCapacity: p.sourceRangeCapacity,
+            prevSourceRangeCapacity: p.prevSourceRangeCapacity,
+            eventDensity: p.eventDensity,
+            latestSourceRangeCapacityUpdateBlock: p.latestSourceRangeCapacityUpdateBlock
           });
         }
       }
@@ -1465,10 +1646,10 @@ function rollback(fetchState, indexingAddresses, targetBlockNumber) {
         mergeBlock: p.mergeBlock,
         dynamicContract: p.dynamicContract,
         mutPendingQueries: rollbackPendingQueries(p.mutPendingQueries, targetBlockNumber),
-        prevQueryRange: p.prevQueryRange,
-        prevPrevQueryRange: p.prevPrevQueryRange,
-        prevRangeSize: p.prevRangeSize,
-        latestBlockRangeUpdateBlock: p.latestBlockRangeUpdateBlock
+        sourceRangeCapacity: p.sourceRangeCapacity,
+        prevSourceRangeCapacity: p.prevSourceRangeCapacity,
+        eventDensity: p.eventDensity,
+        latestSourceRangeCapacityUpdateBlock: p.latestSourceRangeCapacityUpdateBlock
       });
     }
   }
@@ -1491,7 +1672,7 @@ function rollback(fetchState, indexingAddresses, targetBlockNumber) {
     let tmp;
     tmp = item.kind === 0 ? item.blockNumber : item.blockNumber;
     return tmp <= targetBlockNumber;
-  }), undefined, undefined);
+  }), true, undefined, undefined);
 }
 
 function resetPendingQueries(fetchState) {
@@ -1509,10 +1690,10 @@ function resetPendingQueries(fetchState) {
         mergeBlock: partition.mergeBlock,
         dynamicContract: partition.dynamicContract,
         mutPendingQueries: kept,
-        prevQueryRange: partition.prevQueryRange,
-        prevPrevQueryRange: partition.prevPrevQueryRange,
-        prevRangeSize: partition.prevRangeSize,
-        latestBlockRangeUpdateBlock: partition.latestBlockRangeUpdateBlock
+        sourceRangeCapacity: partition.sourceRangeCapacity,
+        prevSourceRangeCapacity: partition.prevSourceRangeCapacity,
+        eventDensity: partition.eventDensity,
+        latestSourceRangeCapacityUpdateBlock: partition.latestSourceRangeCapacityUpdateBlock
       };
     }
   }
@@ -1588,7 +1769,7 @@ function getProgressPercentage(fetchState) {
   if (firstEventBlock === undefined) {
     return 0;
   }
-  let totalRange = fetchState.knownHeight - firstEventBlock | 0;
+  let totalRange = (fetchState.knownHeight - fetchState.blockLag | 0) - firstEventBlock | 0;
   if (totalRange <= 0) {
     return 0;
   }
@@ -1644,32 +1825,33 @@ function getProgressBlockNumberAt(fetchState, index) {
 function updateKnownHeight(fetchState, knownHeight) {
   if (knownHeight > fetchState.knownHeight) {
     Prometheus.IndexingKnownHeight.set(knownHeight, fetchState.chainId);
-    return updateInternal(fetchState, undefined, undefined, undefined, knownHeight);
+    return updateInternal(fetchState, undefined, undefined, undefined, undefined, knownHeight);
   } else {
     return fetchState;
   }
 }
 
-let minItemsTarget = 2000;
+let blockItemLogIndex = 16777216;
 
-let maxItemsTarget = 10000;
+let maxInFlightChunksPerPartition = 12;
 
-let blockItemLogIndex = 16777216;
+let maxChainConcurrency = 100;
 
-let maxPendingChunksPerPartition = 10;
+let chunkRangeGrowthFactor = 1.8;
 
 export {
   deriveContractNameByAddress,
-  minItemsTarget,
-  maxItemsTarget,
   densityItemsTarget,
   getMinHistoryRange,
+  getTrustedDensity,
   getMinQueryRange,
   OptimizedPartitions,
   bufferBlockNumber,
   bufferBlock,
   bufferReadyCount,
+  getRegistrationIndex,
   compareBufferItem,
+  mergeIntoBuffer,
   blockItemLogIndex,
   appendOnBlockItems,
   updateInternal,
@@ -1678,10 +1860,17 @@ export {
   addressesByContractNameGetAll,
   createPartitionsFromIndexingAddresses,
   registerDynamicContracts,
+  filterByClientAddress,
   handleQueryResult,
   startFetchingQueries,
-  maxPendingChunksPerPartition,
+  maxInFlightChunksPerPartition,
+  maxChainConcurrency,
+  chunkRangeGrowthFactor,
+  pushDensityPricedQuery,
   pushGapFillQueries,
+  walkPartitionPending,
+  pushForwardCandidates,
+  acceptCandidates,
   getNextQuery,
   hasReadyItem,
   getReadyItemsCount,