envio 3.3.0-alpha.11 → 3.3.0-alpha.13

package/src/FetchState.res.mjs

@@ -28,27 +28,23 @@ function densityItemsTarget(density, fromBlock, toBlock, chainTargetBlock) {
 }
 
 function getMinHistoryRange(p) {
-  let match = p.prevQueryRange;
-  let match$1 = p.prevPrevQueryRange;
+  let match = p.sourceRangeCapacity;
+  let match$1 = p.prevSourceRangeCapacity;
   if (match !== 0 && match$1 !== 0) {
     return match < match$1 ? match : match$1;
   }
 }
 
 function getTrustedDensity(p) {
-  let match = p.prevQueryRange;
-  let match$1 = p.prevPrevQueryRange;
-  if (match !== 0 && match$1 !== 0) {
-    return p.prevRangeSize / match;
-  }
+  return p.eventDensity;
 }
 
 function getMinQueryRange(partitions) {
   let min = 0;
   for (let i = 0, i_finish = partitions.length; i < i_finish; ++i) {
     let p = partitions[i];
-    let a = p.prevQueryRange;
-    let b = p.prevPrevQueryRange;
+    let a = p.sourceRangeCapacity;
+    let b = p.prevSourceRangeCapacity;
     if (a > 0 && (min === 0 || a < min)) {
       min = a;
     }
@@ -88,10 +84,10 @@ function mergePartitionsAtBlock(p1, p2, potentialMergeBlock, contractName, maxAd
         mergeBlock: potentialMergeBlock,
         dynamicContract: p1.dynamicContract,
         mutPendingQueries: p1.mutPendingQueries,
-        prevQueryRange: p1.prevQueryRange,
-        prevPrevQueryRange: p1.prevPrevQueryRange,
-        prevRangeSize: p1.prevRangeSize,
-        latestBlockRangeUpdateBlock: p1.latestBlockRangeUpdateBlock
+        sourceRangeCapacity: p1.sourceRangeCapacity,
+        prevSourceRangeCapacity: p1.prevSourceRangeCapacity,
+        eventDensity: p1.eventDensity,
+        latestSourceRangeCapacityUpdateBlock: p1.latestSourceRangeCapacityUpdateBlock
       });
       completed.push({
         id: p2.id,
@@ -101,10 +97,10 @@ function mergePartitionsAtBlock(p1, p2, potentialMergeBlock, contractName, maxAd
         mergeBlock: potentialMergeBlock,
         dynamicContract: p2.dynamicContract,
         mutPendingQueries: p2.mutPendingQueries,
-        prevQueryRange: p2.prevQueryRange,
-        prevPrevQueryRange: p2.prevPrevQueryRange,
-        prevRangeSize: p2.prevRangeSize,
-        latestBlockRangeUpdateBlock: p2.latestBlockRangeUpdateBlock
+        sourceRangeCapacity: p2.sourceRangeCapacity,
+        prevSourceRangeCapacity: p2.prevSourceRangeCapacity,
+        eventDensity: p2.eventDensity,
+        latestSourceRangeCapacityUpdateBlock: p2.latestSourceRangeCapacityUpdateBlock
       });
       let newId = nextPartitionIndexRef.contents.toString();
       nextPartitionIndexRef.contents = nextPartitionIndexRef.contents + 1 | 0;
@@ -112,7 +108,13 @@ function mergePartitionsAtBlock(p1, p2, potentialMergeBlock, contractName, maxAd
         p1,
         p2
       ]);
-      let inheritedDensity = Stdlib_Option.getOr(getTrustedDensity(p1), 0) + Stdlib_Option.getOr(getTrustedDensity(p2), 0);
+      let match = p1.eventDensity;
+      let match$1 = p2.eventDensity;
+      let inheritedDensity = match !== undefined ? (
+          match$1 !== undefined ? match + match$1 : match
+        ) : (
+          match$1 !== undefined ? match$1 : undefined
+        );
       continuingBase = {
         id: newId,
         latestFetchedBlock: {
@@ -124,10 +126,10 @@ function mergePartitionsAtBlock(p1, p2, potentialMergeBlock, contractName, maxAd
         mergeBlock: undefined,
         dynamicContract: contractName,
         mutPendingQueries: [],
-        prevQueryRange: minRange,
-        prevPrevQueryRange: minRange,
-        prevRangeSize: Math.ceil(inheritedDensity * minRange) | 0,
-        latestBlockRangeUpdateBlock: 0
+        sourceRangeCapacity: minRange,
+        prevSourceRangeCapacity: minRange,
+        eventDensity: inheritedDensity,
+        latestSourceRangeCapacityUpdateBlock: 0
       };
     } else {
       completed.push({
@@ -138,10 +140,10 @@ function mergePartitionsAtBlock(p1, p2, potentialMergeBlock, contractName, maxAd
         mergeBlock: potentialMergeBlock,
         dynamicContract: p1.dynamicContract,
         mutPendingQueries: p1.mutPendingQueries,
-        prevQueryRange: p1.prevQueryRange,
-        prevPrevQueryRange: p1.prevPrevQueryRange,
-        prevRangeSize: p1.prevRangeSize,
-        latestBlockRangeUpdateBlock: p1.latestBlockRangeUpdateBlock
+        sourceRangeCapacity: p1.sourceRangeCapacity,
+        prevSourceRangeCapacity: p1.prevSourceRangeCapacity,
+        eventDensity: p1.eventDensity,
+        latestSourceRangeCapacityUpdateBlock: p1.latestSourceRangeCapacityUpdateBlock
       });
       continuingBase = p2;
     }
@@ -154,10 +156,10 @@ function mergePartitionsAtBlock(p1, p2, potentialMergeBlock, contractName, maxAd
       mergeBlock: potentialMergeBlock,
       dynamicContract: p2.dynamicContract,
       mutPendingQueries: p2.mutPendingQueries,
-      prevQueryRange: p2.prevQueryRange,
-      prevPrevQueryRange: p2.prevPrevQueryRange,
-      prevRangeSize: p2.prevRangeSize,
-      latestBlockRangeUpdateBlock: p2.latestBlockRangeUpdateBlock
+      sourceRangeCapacity: p2.sourceRangeCapacity,
+      prevSourceRangeCapacity: p2.prevSourceRangeCapacity,
+      eventDensity: p2.eventDensity,
+      latestSourceRangeCapacityUpdateBlock: p2.latestSourceRangeCapacityUpdateBlock
     });
     continuingBase = p1;
   } else {
@@ -178,10 +180,10 @@ function mergePartitionsAtBlock(p1, p2, potentialMergeBlock, contractName, maxAd
       mergeBlock: continuingBase.mergeBlock,
       dynamicContract: continuingBase.dynamicContract,
       mutPendingQueries: continuingBase.mutPendingQueries,
-      prevQueryRange: continuingBase.prevQueryRange,
-      prevPrevQueryRange: continuingBase.prevPrevQueryRange,
-      prevRangeSize: continuingBase.prevRangeSize,
-      latestBlockRangeUpdateBlock: continuingBase.latestBlockRangeUpdateBlock
+      sourceRangeCapacity: continuingBase.sourceRangeCapacity,
+      prevSourceRangeCapacity: continuingBase.prevSourceRangeCapacity,
+      eventDensity: continuingBase.eventDensity,
+      latestSourceRangeCapacityUpdateBlock: continuingBase.latestSourceRangeCapacityUpdateBlock
     });
     let restId = nextPartitionIndexRef.contents.toString();
     nextPartitionIndexRef.contents = nextPartitionIndexRef.contents + 1 | 0;
@@ -193,10 +195,10 @@ function mergePartitionsAtBlock(p1, p2, potentialMergeBlock, contractName, maxAd
       mergeBlock: continuingBase.mergeBlock,
       dynamicContract: continuingBase.dynamicContract,
       mutPendingQueries: [],
-      prevQueryRange: continuingBase.prevQueryRange,
-      prevPrevQueryRange: continuingBase.prevPrevQueryRange,
-      prevRangeSize: continuingBase.prevRangeSize,
-      latestBlockRangeUpdateBlock: continuingBase.latestBlockRangeUpdateBlock
+      sourceRangeCapacity: continuingBase.sourceRangeCapacity,
+      prevSourceRangeCapacity: continuingBase.prevSourceRangeCapacity,
+      eventDensity: continuingBase.eventDensity,
+      latestSourceRangeCapacityUpdateBlock: continuingBase.latestSourceRangeCapacityUpdateBlock
     });
     return completed;
   }
@@ -210,10 +212,10 @@ function mergePartitionsAtBlock(p1, p2, potentialMergeBlock, contractName, maxAd
     mergeBlock: continuingBase.mergeBlock,
     dynamicContract: continuingBase.dynamicContract,
     mutPendingQueries: continuingBase.mutPendingQueries,
-    prevQueryRange: continuingBase.prevQueryRange,
-    prevPrevQueryRange: continuingBase.prevPrevQueryRange,
-    prevRangeSize: continuingBase.prevRangeSize,
-    latestBlockRangeUpdateBlock: continuingBase.latestBlockRangeUpdateBlock
+    sourceRangeCapacity: continuingBase.sourceRangeCapacity,
+    prevSourceRangeCapacity: continuingBase.prevSourceRangeCapacity,
+    eventDensity: continuingBase.eventDensity,
+    latestSourceRangeCapacityUpdateBlock: continuingBase.latestSourceRangeCapacityUpdateBlock
   });
   return completed;
 }
@@ -358,8 +360,11 @@ function handleQueryResponse(optimizedPartitions, query, knownHeight, itemsCount
   let pendingQuery = getPendingQueryOrThrow(p$1, query.fromBlock);
   pendingQuery.fetchedBlock = latestFetchedBlock;
   let blockRange = (latestFetchedBlock.blockNumber - query.fromBlock | 0) + 1 | 0;
-  let shouldUpdateBlockRange = false;
-  if (latestFetchedBlock.blockNumber > p$1.latestBlockRangeUpdateBlock) {
+  let observedEventDensity = itemsCount / blockRange;
+  let eventDensity = p$1.eventDensity;
+  let updatedEventDensity = eventDensity !== undefined ? (eventDensity + observedEventDensity) / 2 : observedEventDensity;
+  let shouldUpdateSourceRangeCapacity = false;
+  if (latestFetchedBlock.blockNumber > p$1.latestSourceRangeCapacityUpdateBlock) {
     let queryToBlock = query.toBlock;
     let tmp;
     if (queryToBlock !== undefined) {
@@ -372,11 +377,10 @@ function handleQueryResponse(optimizedPartitions, query, knownHeight, itemsCount
     } else {
       tmp = latestFetchedBlock.blockNumber < (knownHeight - 10 | 0);
     }
-    shouldUpdateBlockRange = tmp;
+    shouldUpdateSourceRangeCapacity = tmp;
   }
-  let updatedPrevQueryRange = shouldUpdateBlockRange ? blockRange : p$1.prevQueryRange;
-  let updatedPrevPrevQueryRange = shouldUpdateBlockRange ? p$1.prevQueryRange : p$1.prevPrevQueryRange;
-  let updatedPrevRangeSize = shouldUpdateBlockRange ? itemsCount : p$1.prevRangeSize;
+  let updatedSourceRangeCapacity = shouldUpdateSourceRangeCapacity ? blockRange : p$1.sourceRangeCapacity;
+  let updatedPrevSourceRangeCapacity = shouldUpdateSourceRangeCapacity ? p$1.sourceRangeCapacity : p$1.prevSourceRangeCapacity;
   let mutPendingQueries = p$1.mutPendingQueries;
   let latestFetchedBlock$1 = p$1.latestFetchedBlock;
   while ((() => {
@@ -402,7 +406,7 @@ function handleQueryResponse(optimizedPartitions, query, knownHeight, itemsCount
     let updatedMainPartition_mergeBlock = p$1.mergeBlock;
     let updatedMainPartition_dynamicContract = p$1.dynamicContract;
     let updatedMainPartition_mutPendingQueries = p$1.mutPendingQueries;
-    let updatedMainPartition_latestBlockRangeUpdateBlock = shouldUpdateBlockRange ? latestFetchedBlock.blockNumber : p$1.latestBlockRangeUpdateBlock;
+    let updatedMainPartition_latestSourceRangeCapacityUpdateBlock = shouldUpdateSourceRangeCapacity ? latestFetchedBlock.blockNumber : p$1.latestSourceRangeCapacityUpdateBlock;
     let updatedMainPartition = {
       id: updatedMainPartition_id,
       latestFetchedBlock: updatedLatestFetchedBlock,
@@ -411,10 +415,10 @@ function handleQueryResponse(optimizedPartitions, query, knownHeight, itemsCount
       mergeBlock: updatedMainPartition_mergeBlock,
       dynamicContract: updatedMainPartition_dynamicContract,
       mutPendingQueries: updatedMainPartition_mutPendingQueries,
-      prevQueryRange: updatedPrevQueryRange,
-      prevPrevQueryRange: updatedPrevPrevQueryRange,
-      prevRangeSize: updatedPrevRangeSize,
-      latestBlockRangeUpdateBlock: updatedMainPartition_latestBlockRangeUpdateBlock
+      sourceRangeCapacity: updatedSourceRangeCapacity,
+      prevSourceRangeCapacity: updatedPrevSourceRangeCapacity,
+      eventDensity: updatedEventDensity,
+      latestSourceRangeCapacityUpdateBlock: updatedMainPartition_latestSourceRangeCapacityUpdateBlock
     };
     mutEntities[p$1.id] = updatedMainPartition;
   }
@@ -739,10 +743,10 @@ function createPartitionsFromIndexingAddresses(registeringContractsByContract, d
             mergeBlock: undefined,
             dynamicContract: isDynamic ? contractName : undefined,
             mutPendingQueries: [],
-            prevQueryRange: 0,
-            prevPrevQueryRange: 0,
-            prevRangeSize: 0,
-            latestBlockRangeUpdateBlock: 0
+            sourceRangeCapacity: 0,
+            prevSourceRangeCapacity: 0,
+            eventDensity: undefined,
+            latestSourceRangeCapacityUpdateBlock: 0
           });
           nextPartitionIndexRef = nextPartitionIndexRef + 1 | 0;
         };
@@ -790,10 +794,10 @@ function createPartitionsFromIndexingAddresses(registeringContractsByContract, d
             mergeBlock: currentPBlock < nextPBlock ? nextPBlock : undefined,
             dynamicContract: currentP.dynamicContract,
             mutPendingQueries: currentP.mutPendingQueries,
-            prevQueryRange: currentP.prevQueryRange,
-            prevPrevQueryRange: currentP.prevPrevQueryRange,
-            prevRangeSize: currentP.prevRangeSize,
-            latestBlockRangeUpdateBlock: currentP.latestBlockRangeUpdateBlock
+            sourceRangeCapacity: currentP.sourceRangeCapacity,
+            prevSourceRangeCapacity: currentP.prevSourceRangeCapacity,
+            eventDensity: currentP.eventDensity,
+            latestSourceRangeCapacityUpdateBlock: currentP.latestSourceRangeCapacityUpdateBlock
           });
           currentPRef = {
             id: nextP.id,
@@ -803,10 +807,10 @@ function createPartitionsFromIndexingAddresses(registeringContractsByContract, d
             mergeBlock: nextP.mergeBlock,
             dynamicContract: nextP.dynamicContract,
             mutPendingQueries: nextP.mutPendingQueries,
-            prevQueryRange: nextP.prevQueryRange,
-            prevPrevQueryRange: nextP.prevPrevQueryRange,
-            prevRangeSize: nextP.prevRangeSize,
-            latestBlockRangeUpdateBlock: nextP.latestBlockRangeUpdateBlock
+            sourceRangeCapacity: nextP.sourceRangeCapacity,
+            prevSourceRangeCapacity: nextP.prevSourceRangeCapacity,
+            eventDensity: nextP.eventDensity,
+            latestSourceRangeCapacityUpdateBlock: nextP.latestSourceRangeCapacityUpdateBlock
           };
         } else {
           currentPRef = {
@@ -817,10 +821,10 @@ function createPartitionsFromIndexingAddresses(registeringContractsByContract, d
             mergeBlock: currentP.mergeBlock,
             dynamicContract: currentP.dynamicContract,
             mutPendingQueries: currentP.mutPendingQueries,
-            prevQueryRange: currentP.prevQueryRange,
-            prevPrevQueryRange: currentP.prevPrevQueryRange,
-            prevRangeSize: currentP.prevRangeSize,
-            latestBlockRangeUpdateBlock: currentP.latestBlockRangeUpdateBlock
+            sourceRangeCapacity: currentP.sourceRangeCapacity,
+            prevSourceRangeCapacity: currentP.prevSourceRangeCapacity,
+            eventDensity: currentP.eventDensity,
+            latestSourceRangeCapacityUpdateBlock: currentP.latestSourceRangeCapacityUpdateBlock
           };
         }
       }
@@ -967,10 +971,10 @@ function registerDynamicContracts(fetchState, indexingAddresses, items) {
                 mergeBlock: p.mergeBlock,
                 dynamicContract: p.dynamicContract,
                 mutPendingQueries: p.mutPendingQueries,
-                prevQueryRange: p.prevQueryRange,
-                prevPrevQueryRange: p.prevPrevQueryRange,
-                prevRangeSize: p.prevRangeSize,
-                latestBlockRangeUpdateBlock: p.latestBlockRangeUpdateBlock
+                sourceRangeCapacity: p.sourceRangeCapacity,
+                prevSourceRangeCapacity: p.prevSourceRangeCapacity,
+                eventDensity: p.eventDensity,
+                latestSourceRangeCapacityUpdateBlock: p.latestSourceRangeCapacityUpdateBlock
               };
               let addressesByContractName = {};
               addressesByContractName[contractName] = addresses;
@@ -982,10 +986,10 @@ function registerDynamicContracts(fetchState, indexingAddresses, items) {
                 mergeBlock: undefined,
                 dynamicContract: contractName,
                 mutPendingQueries: p.mutPendingQueries,
-                prevQueryRange: p.prevQueryRange,
-                prevPrevQueryRange: p.prevPrevQueryRange,
-                prevRangeSize: p.prevRangeSize,
-                latestBlockRangeUpdateBlock: p.latestBlockRangeUpdateBlock
+                sourceRangeCapacity: p.sourceRangeCapacity,
+                prevSourceRangeCapacity: p.prevSourceRangeCapacity,
+                eventDensity: p.eventDensity,
+                latestSourceRangeCapacityUpdateBlock: p.latestSourceRangeCapacityUpdateBlock
               });
             }
           } else {
@@ -997,10 +1001,10 @@ function registerDynamicContracts(fetchState, indexingAddresses, items) {
               mergeBlock: p.mergeBlock,
               dynamicContract: contractName,
               mutPendingQueries: p.mutPendingQueries,
-              prevQueryRange: p.prevQueryRange,
-              prevPrevQueryRange: p.prevPrevQueryRange,
-              prevRangeSize: p.prevRangeSize,
-              latestBlockRangeUpdateBlock: p.latestBlockRangeUpdateBlock
+              sourceRangeCapacity: p.sourceRangeCapacity,
+              prevSourceRangeCapacity: p.prevSourceRangeCapacity,
+              eventDensity: p.eventDensity,
+              latestSourceRangeCapacityUpdateBlock: p.latestSourceRangeCapacityUpdateBlock
             };
           }
         }
@@ -1072,7 +1076,7 @@ function pushGapFillQueries(queries, partitionId, rangeFromBlock, rangeEndBlock,
       exit = 1;
     }
     if (exit === 1) {
-      let trustedDensity = getTrustedDensity(partition);
+      let trustedDensity = partition.eventDensity;
       let maxBlock = rangeEndBlock !== undefined ? rangeEndBlock : chainTargetBlock;
       let pushSingleQuery = (density, isChunk) => {
         let itemsTarget = densityItemsTarget(density * chunkItemsMultiplier, rangeFromBlock, rangeEndBlock, chainTargetBlock);
@@ -1150,7 +1154,8 @@ function getNextQuery(param, chainTargetBlock, chainTargetItems, $staropt$star)
       shouldWaitForNewBlock = false;
     }
   }
-  let queriesByPartitionIndex = Stdlib_Array.fromInitializer(partitionsCount, param => []);
+  let partitionsCount$1 = chainTargetItems <= 0 || shouldWaitForNewBlock ? 0 : partitionsCount;
+  let queriesByPartitionIndex = Stdlib_Array.fromInitializer(partitionsCount$1, param => []);
   let computeQueryEndBlock = p => {
     let queryEndBlock = Utils.$$Math.minOptInt(endBlock, p.mergeBlock);
     if (blockLag !== 0) {
@@ -1164,7 +1169,7 @@ function getNextQuery(param, chainTargetBlock, chainTargetItems, $staropt$star)
   let partitionIndexById = {};
   let candidates = [];
   let fillStates = [];
-  for (let idx$1 = 0; idx$1 < partitionsCount; ++idx$1) {
+  for (let idx$1 = 0; idx$1 < partitionsCount$1; ++idx$1) {
     let partitionId$1 = optimizedPartitions.idsInAscOrder[idx$1];
     let p$1 = optimizedPartitions.entities[partitionId$1];
     partitionIndexById[partitionId$1] = idx$1;
@@ -1189,7 +1194,7 @@ function getNextQuery(param, chainTargetBlock, chainTargetItems, $staropt$star)
       let pq$1 = p$1.mutPendingQueries[pqIdx$1];
       if (pq$1.fromBlock > cursor) {
         let beforeLen = candidates.length;
-        pushGapFillQueries(candidates, partitionId$1, cursor, Utils.$$Math.minOptInt(pq$1.fromBlock - 1 | 0, queryEndBlock), knownHeight, chainTargetBlock, maybeChunkRange, (12 - pendingCount | 0) - chunksUsedThisCall | 0, p$1, chainTargetItems / partitionsCount, chunkItemsMultiplier, p$1.selection, p$1.addressesByContractName);
+        pushGapFillQueries(candidates, partitionId$1, cursor, Utils.$$Math.minOptInt(pq$1.fromBlock - 1 | 0, queryEndBlock), knownHeight, chainTargetBlock, maybeChunkRange, (12 - pendingCount | 0) - chunksUsedThisCall | 0, p$1, chainTargetItems / partitionsCount$1, chunkItemsMultiplier, p$1.selection, p$1.addressesByContractName);
         chunksUsedThisCall = chunksUsedThisCall + (candidates.length - beforeLen | 0) | 0;
       }
       let toBlock = pq$1.toBlock;
@@ -1248,7 +1253,7 @@ function getNextQuery(param, chainTargetBlock, chainTargetItems, $staropt$star)
     let eb = fs.queryEndBlock;
     let maxBlock = eb !== undefined ? eb : chainTargetBlock;
     let match = fs.maybeChunkRange;
-    let match$1 = getTrustedDensity(p);
+    let match$1 = p.eventDensity;
     if (match !== undefined && match$1 !== undefined && match$1 > 0) {
       let chunkSize = Js_math.ceil_int(match * 1.8);
       let maxChunksRemaining = (12 - fs.pendingCount | 0) - fs.chunksUsedThisCall | 0;
@@ -1326,12 +1331,14 @@ function getNextQuery(param, chainTargetBlock, chainTargetItems, $staropt$star)
   let streamCount = acceptanceStream.length;
   let remainingBudget = chainTargetItems;
   let acceptIdx = 0;
-  while (remainingBudget > 0 && acceptIdx < streamCount) {
+  let usedConcurrency = reservations.length;
+  while (remainingBudget > 0 && acceptIdx < streamCount && usedConcurrency < 100) {
     let match$1 = acceptanceStream[acceptIdx];
     let maybeQuery = match$1[2];
     if (maybeQuery !== undefined) {
       let partitionIdx = partitionIndexById[maybeQuery.partitionId];
       queriesByPartitionIndex[partitionIdx].push(maybeQuery);
+      usedConcurrency = usedConcurrency + 1 | 0;
     }
     remainingBudget = remainingBudget - match$1[1];
     acceptIdx = acceptIdx + 1 | 0;
@@ -1416,10 +1423,10 @@ function make$1(startBlock, endBlock, onEventRegistrations, contractConfigs, add
       mergeBlock: undefined,
       dynamicContract: undefined,
       mutPendingQueries: [],
-      prevQueryRange: 0,
-      prevPrevQueryRange: 0,
-      prevRangeSize: 0,
-      latestBlockRangeUpdateBlock: 0
+      sourceRangeCapacity: 0,
+      prevSourceRangeCapacity: 0,
+      eventDensity: undefined,
+      latestSourceRangeCapacityUpdateBlock: 0
     });
   }
   let normalSelection = {
@@ -1554,10 +1561,10 @@ function rollback(fetchState, indexingAddresses, targetBlockNumber) {
             mergeBlock: mergeBlock$1,
             dynamicContract: p.dynamicContract,
             mutPendingQueries: rollbackPendingQueries(p.mutPendingQueries, targetBlockNumber),
-            prevQueryRange: p.prevQueryRange,
-            prevPrevQueryRange: p.prevPrevQueryRange,
-            prevRangeSize: p.prevRangeSize,
-            latestBlockRangeUpdateBlock: p.latestBlockRangeUpdateBlock
+            sourceRangeCapacity: p.sourceRangeCapacity,
+            prevSourceRangeCapacity: p.prevSourceRangeCapacity,
+            eventDensity: p.eventDensity,
+            latestSourceRangeCapacityUpdateBlock: p.latestSourceRangeCapacityUpdateBlock
           });
         }
       }
@@ -1575,10 +1582,10 @@ function rollback(fetchState, indexingAddresses, targetBlockNumber) {
         mergeBlock: p.mergeBlock,
         dynamicContract: p.dynamicContract,
         mutPendingQueries: rollbackPendingQueries(p.mutPendingQueries, targetBlockNumber),
-        prevQueryRange: p.prevQueryRange,
-        prevPrevQueryRange: p.prevPrevQueryRange,
-        prevRangeSize: p.prevRangeSize,
-        latestBlockRangeUpdateBlock: p.latestBlockRangeUpdateBlock
+        sourceRangeCapacity: p.sourceRangeCapacity,
+        prevSourceRangeCapacity: p.prevSourceRangeCapacity,
+        eventDensity: p.eventDensity,
+        latestSourceRangeCapacityUpdateBlock: p.latestSourceRangeCapacityUpdateBlock
       });
     }
   }
@@ -1619,10 +1626,10 @@ function resetPendingQueries(fetchState) {
         mergeBlock: partition.mergeBlock,
         dynamicContract: partition.dynamicContract,
         mutPendingQueries: kept,
-        prevQueryRange: partition.prevQueryRange,
-        prevPrevQueryRange: partition.prevPrevQueryRange,
-        prevRangeSize: partition.prevRangeSize,
-        latestBlockRangeUpdateBlock: partition.latestBlockRangeUpdateBlock
+        sourceRangeCapacity: partition.sourceRangeCapacity,
+        prevSourceRangeCapacity: partition.prevSourceRangeCapacity,
+        eventDensity: partition.eventDensity,
+        latestSourceRangeCapacityUpdateBlock: partition.latestSourceRangeCapacityUpdateBlock
       };
     }
   }
@@ -1698,7 +1705,7 @@ function getProgressPercentage(fetchState) {
   if (firstEventBlock === undefined) {
     return 0;
   }
-  let totalRange = fetchState.knownHeight - firstEventBlock | 0;
+  let totalRange = (fetchState.knownHeight - fetchState.blockLag | 0) - firstEventBlock | 0;
   if (totalRange <= 0) {
     return 0;
   }
@@ -1764,6 +1771,8 @@ let blockItemLogIndex = 16777216;
 
 let maxPendingChunksPerPartition = 12;
 
+let maxChainConcurrency = 100;
+
 export {
   deriveContractNameByAddress,
   densityItemsTarget,
@@ -1789,6 +1798,7 @@ export {
   handleQueryResult,
   startFetchingQueries,
   maxPendingChunksPerPartition,
+  maxChainConcurrency,
   pushGapFillQueries,
   getNextQuery,
   hasReadyItem,

package/src/InMemoryStore.res

@@ -8,25 +8,13 @@ let getInMemTable = (
 ): InMemoryTable.Entity.t =>
   state->IndexerState.entities->IndexerState.EntityTables.get(~entityName=entityConfig.name)
 
-let getEffectInMemTable = (state: IndexerState.t, ~effect: Internal.effect) => {
-  let key = effect.name
-  let effects = state->IndexerState.effects
-  switch effects->Utils.Dict.dangerouslyGetNonOption(key) {
-  | Some(table) => table
-  | None =>
-    let table: IndexerState.effectCacheInMemTable = {
-      idsToStore: [],
-      dict: Dict.make(),
-      changesCount: 0.,
-      invalidationsCount: 0,
-      effect,
-    }
-    effects->Dict.set(key, table)
-    table
-  }
-}
+let getEffectInMemTable = (
+  state: IndexerState.t,
+  ~effect: Internal.effect,
+  ~scope: Internal.chainScope,
+) => state->IndexerState.effectState->EffectState.getTable(~effect, ~scope)
 
-let hasEffectOutput = (inMemTable: IndexerState.effectCacheInMemTable, key) =>
+let hasEffectOutput = (inMemTable: EffectState.effectCacheInMemTable, key) =>
   switch inMemTable.dict->Utils.Dict.dangerouslyGetNonOption(key) {
   | Some(Set(_)) => true
   | Some(Delete(_)) | None => false
@@ -36,7 +24,7 @@ let hasEffectOutput = (inMemTable: IndexerState.effectCacheInMemTable, key) =>
 // optional output, so it must never be wrapped in another option here: Some(None)
 // is encoded as the nested-option sentinel and would leak to the handler.
 let getEffectOutputUnsafe = (
-  inMemTable: IndexerState.effectCacheInMemTable,
+  inMemTable: EffectState.effectCacheInMemTable,
   key,
 ): Internal.effectOutput =>
   switch inMemTable.dict->Utils.Dict.dangerouslyGetNonOption(key) {
@@ -47,7 +35,7 @@ let getEffectOutputUnsafe = (
 // Records a handler output. Persisted on the next write only when shouldCache;
 // otherwise kept in memory (re-run on a later miss) but never written to the db.
 let setEffectOutput = (
-  inMemTable: IndexerState.effectCacheInMemTable,
+  inMemTable: EffectState.effectCacheInMemTable,
   ~checkpointId,
   ~cacheKey,
   ~output,
@@ -65,7 +53,7 @@ let setEffectOutput = (
 
 // Seeds an entry from a db read. Stamped with loadedFromDbCheckpointId so it's
 // always droppable (re-readable from the db) and never re-persisted.
-let initEffectOutputFromDb = (inMemTable: IndexerState.effectCacheInMemTable, ~cacheKey, ~output) =>
+let initEffectOutputFromDb = (inMemTable: EffectState.effectCacheInMemTable, ~cacheKey, ~output) =>
   if inMemTable.dict->Utils.Dict.dangerouslyGetNonOption(cacheKey)->Option.isNone {
     inMemTable.changesCount = inMemTable.changesCount +. 1.
     inMemTable.dict->Dict.set(
@@ -78,7 +66,7 @@ let initEffectOutputFromDb = (inMemTable: IndexerState.effectCacheInMemTable, ~c
 // cache:false). Uncommitted entries stay warm. With keepLoadedFromDb, entries
 // seeded from a db read are spared. Mirrors entity dropCommittedChanges.
 let dropCommittedEffects = (
-  inMemTable: IndexerState.effectCacheInMemTable,
+  inMemTable: EffectState.effectCacheInMemTable,
   ~committedCheckpointId,
   ~keepLoadedFromDb,
 ) => {
@@ -117,17 +105,17 @@ let prepareRollbackDiff = async (
   ->Array.map(async entityConfig => {
     let entityTable = state->getInMemTable(~entityConfig)
 
-    let (removedIdsResult, restoredEntitiesResult) = await persistence.storage.getRollbackData(
+    let (removedIds, restoredEntitiesResult) = await persistence.storage.getRollbackData(
       ~entityConfig,
       ~rollbackTargetCheckpointId,
     )
 
-    removedIdsResult->Array.forEach(data => {
-      deletedEntities->Utils.Dict.push(entityConfig.name, data["id"])
+    removedIds->Array.forEach(entityId => {
+      deletedEntities->Utils.Dict.push(entityConfig.name, entityId)
       entityTable->InMemoryTable.Entity.set(
         ~committedCheckpointId,
         Delete({
-          entityId: data["id"],
+          entityId,
           checkpointId: rollbackDiffCheckpointId,
         }),
       )

package/src/InMemoryStore.res.mjs

@@ -4,6 +4,7 @@ import * as Table from "./db/Table.res.mjs";
 import * as Utils from "./Utils.res.mjs";
 import * as Config from "./Config.res.mjs";
 import * as Internal from "./Internal.res.mjs";
+import * as EffectState from "./EffectState.res.mjs";
 import * as IndexerState from "./IndexerState.res.mjs";
 import * as InMemoryTable from "./InMemoryTable.res.mjs";
 import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
@@ -13,22 +14,8 @@ function getInMemTable(state, entityConfig) {
   return IndexerState.EntityTables.get(IndexerState.entities(state), entityConfig.name);
 }
 
-function getEffectInMemTable(state, effect) {
-  let key = effect.name;
-  let effects = IndexerState.effects(state);
-  let table = effects[key];
-  if (table !== undefined) {
-    return table;
-  }
-  let table$1 = {
-    idsToStore: [],
-    invalidationsCount: 0,
-    dict: {},
-    changesCount: 0,
-    effect: effect
-  };
-  effects[key] = table$1;
-  return table$1;
+function getEffectInMemTable(state, effect, scope) {
+  return EffectState.getTable(IndexerState.effectState(state), effect, scope);
 }
 
 function hasEffectOutput(inMemTable, key) {
@@ -102,11 +89,11 @@ async function prepareRollbackDiff(state, rollbackTargetCheckpointId, rollbackDi
   await Promise.all(persistence.allEntities.map(async entityConfig => {
     let entityTable = getInMemTable(state, entityConfig);
     let match = await persistence.storage.getRollbackData(entityConfig, rollbackTargetCheckpointId);
-    match[0].forEach(data => {
-      Utils.Dict.push(deletedEntities, entityConfig.name, data.id);
+    match[0].forEach(entityId => {
+      Utils.Dict.push(deletedEntities, entityConfig.name, entityId);
       InMemoryTable.Entity.set(entityTable, committedCheckpointId, {
         type: "DELETE",
-        entityId: data.id,
+        entityId: entityId,
         checkpointId: rollbackDiffCheckpointId
       });
     });