envio 3.3.0-alpha.8 → 3.3.0-alpha.9

package/src/FetchState.res.mjs

@@ -23,9 +23,8 @@ let deriveContractNameByAddress = Utils.$$WeakMap.memoize(addressesByContractNam
   return result;
 });
 
-function densityItemsTarget(p, fromBlock, toBlock, chainTargetBlock) {
-  let density = p.prevRangeSize / p.prevQueryRange;
-  return ((Stdlib_Option.getOr(toBlock, chainTargetBlock) - fromBlock | 0) + 1 | 0) * density;
+function densityItemsTarget(density, fromBlock, toBlock, chainTargetBlock) {
+  return Primitive_int.max(1, Math.ceil(((Stdlib_Option.getOr(toBlock, chainTargetBlock) - fromBlock | 0) + 1 | 0) * density) | 0);
 }
 
 function getMinHistoryRange(p) {
@@ -36,6 +35,14 @@ function getMinHistoryRange(p) {
   }
 }
 
+function getTrustedDensity(p) {
+  let match = p.prevQueryRange;
+  let match$1 = p.prevPrevQueryRange;
+  if (match !== 0 && match$1 !== 0) {
+    return p.prevRangeSize / match;
+  }
+}
+
 function getMinQueryRange(partitions) {
   let min = 0;
   for (let i = 0, i_finish = partitions.length; i < i_finish; ++i) {
@@ -105,6 +112,7 @@ function mergePartitionsAtBlock(p1, p2, potentialMergeBlock, contractName, maxAd
         p1,
         p2
       ]);
+      let inheritedDensity = Stdlib_Option.getOr(getTrustedDensity(p1), 0) + Stdlib_Option.getOr(getTrustedDensity(p2), 0);
       continuingBase = {
         id: newId,
         latestFetchedBlock: {
@@ -118,7 +126,7 @@ function mergePartitionsAtBlock(p1, p2, potentialMergeBlock, contractName, maxAd
         mutPendingQueries: [],
         prevQueryRange: minRange,
         prevPrevQueryRange: minRange,
-        prevRangeSize: 0,
+        prevRangeSize: Math.ceil(inheritedDensity * minRange) | 0,
         latestBlockRangeUpdateBlock: 0
       };
     } else {
@@ -976,65 +984,84 @@ function startFetchingQueries(param, queries) {
   }
 }
 
-function pushGapFillQueries(queries, partitionId, rangeFromBlock, rangeEndBlock, knownHeight, chainTargetBlock, maybeChunkRange, maxChunks, partition, selection, addressesByContractName) {
-  let cost = 0;
+function pushGapFillQueries(queries, partitionId, rangeFromBlock, rangeEndBlock, knownHeight, chainTargetBlock, maybeChunkRange, maxChunks, partition, partitionBudget, selection, addressesByContractName) {
+  let cost = {
+    contents: 0
+  };
   if (rangeFromBlock <= knownHeight && maxChunks > 0) {
     let exit = 0;
     if (!(rangeEndBlock !== undefined && rangeFromBlock > rangeEndBlock)) {
       exit = 1;
     }
     if (exit === 1) {
-      if (maybeChunkRange !== undefined) {
-        let maxBlock = rangeEndBlock !== undefined ? rangeEndBlock : chainTargetBlock;
-        let chunkSize = Js_math.ceil_int(maybeChunkRange * 1.8);
-        if (((rangeFromBlock + (chunkSize << 1) | 0) - 1 | 0) <= maxBlock) {
-          let chunkFromBlock = rangeFromBlock;
-          let chunkIdx = 0;
-          while (chunkIdx < maxChunks && ((chunkFromBlock + chunkSize | 0) - 1 | 0) <= maxBlock) {
-            let chunkToBlock = (chunkFromBlock + chunkSize | 0) - 1 | 0;
-            let itemsTarget = densityItemsTarget(partition, chunkFromBlock, chunkToBlock, chainTargetBlock);
-            queries.push({
-              partitionId: partitionId,
-              fromBlock: chunkFromBlock,
-              toBlock: chunkToBlock,
-              isChunk: true,
-              itemsTarget: itemsTarget,
-              selection: selection,
-              addressesByContractName: addressesByContractName
-            });
-            cost = cost + itemsTarget;
-            chunkFromBlock = chunkToBlock + 1 | 0;
-            chunkIdx = chunkIdx + 1 | 0;
-          };
-        } else {
-          let itemsTarget$1 = densityItemsTarget(partition, rangeFromBlock, rangeEndBlock, chainTargetBlock);
-          queries.push({
-            partitionId: partitionId,
-            fromBlock: rangeFromBlock,
-            toBlock: rangeEndBlock,
-            isChunk: rangeEndBlock !== undefined,
-            itemsTarget: itemsTarget$1,
-            selection: selection,
-            addressesByContractName: addressesByContractName
-          });
-          cost = cost + itemsTarget$1;
-        }
-      } else {
-        let itemsTarget$2 = densityItemsTarget(partition, rangeFromBlock, rangeEndBlock, chainTargetBlock);
+      let trustedDensity = getTrustedDensity(partition);
+      let maxBlock = rangeEndBlock !== undefined ? rangeEndBlock : chainTargetBlock;
+      let pushSingleQuery = (density, isChunk) => {
+        let itemsTarget = densityItemsTarget(density, rangeFromBlock, rangeEndBlock, chainTargetBlock);
         queries.push({
           partitionId: partitionId,
           fromBlock: rangeFromBlock,
           toBlock: rangeEndBlock,
-          isChunk: false,
-          itemsTarget: itemsTarget$2,
+          isChunk: isChunk,
+          itemsTarget: itemsTarget,
           selection: selection,
           addressesByContractName: addressesByContractName
         });
-        cost = cost + itemsTarget$2;
+        cost.contents = cost.contents + itemsTarget;
+      };
+      if (trustedDensity !== undefined) {
+        if (maybeChunkRange !== undefined && trustedDensity > 0) {
+          let chunkSize = Js_math.ceil_int(maybeChunkRange * 1.8);
+          if (((rangeFromBlock + (chunkSize << 1) | 0) - 1 | 0) <= maxBlock) {
+            let chunkFromBlock = rangeFromBlock;
+            let chunkIdx = 0;
+            while (chunkIdx < maxChunks && ((chunkFromBlock + chunkSize | 0) - 1 | 0) <= maxBlock) {
+              let chunkToBlock = (chunkFromBlock + chunkSize | 0) - 1 | 0;
+              let itemsTarget = densityItemsTarget(trustedDensity, chunkFromBlock, chunkToBlock, chainTargetBlock);
+              queries.push({
+                partitionId: partitionId,
+                fromBlock: chunkFromBlock,
+                toBlock: chunkToBlock,
+                isChunk: true,
+                itemsTarget: itemsTarget,
+                selection: selection,
+                addressesByContractName: addressesByContractName
+              });
+              cost.contents = cost.contents + itemsTarget;
+              chunkFromBlock = chunkToBlock + 1 | 0;
+              chunkIdx = chunkIdx + 1 | 0;
+            };
+          } else {
+            pushSingleQuery(trustedDensity, rangeEndBlock !== undefined);
+          }
+        } else {
+          pushSingleQuery(trustedDensity, false);
+        }
+      } else {
+        let remainingRange = Primitive_int.max(1, (chainTargetBlock - rangeFromBlock | 0) + 1 | 0);
+        pushSingleQuery(partitionBudget / remainingRange, false);
       }
     }
   }
-  return cost;
+  return cost.contents;
+}
+
+function waterLevel(budget, footprints) {
+  let sorted = footprints.toSorted(Primitive_float.compare);
+  let n = sorted.length;
+  let prefix = 0;
+  let level;
+  let idx = 0;
+  while (level === undefined && idx < n) {
+    let i = idx;
+    prefix = prefix + sorted[i];
+    let candidate = (budget + prefix) / (i + 1 | 0);
+    if (i === (n - 1 | 0) || candidate <= sorted[i + 1 | 0]) {
+      level = candidate;
+    }
+    idx = idx + 1 | 0;
+  };
+  return Stdlib_Option.getOr(level, 0);
 }
 
 function getNextQuery(param, chainTargetBlock, chainTargetItems) {
@@ -1096,7 +1123,7 @@ function getNextQuery(param, chainTargetBlock, chainTargetItems) {
       let pq = p$2.mutPendingQueries[pqIdx$1];
       if (pq.fromBlock > cursor) {
         let beforeLen = bucket.length;
-        let cost$1 = pushGapFillQueries(bucket, partitionId$2, cursor, Utils.$$Math.minOptInt(pq.fromBlock - 1 | 0, queryEndBlock), knownHeight, chainTargetBlock, maybeChunkRange, (10 - pendingCount | 0) - chunksUsedThisCall | 0, p$2, p$2.selection, p$2.addressesByContractName);
+        let cost$1 = pushGapFillQueries(bucket, partitionId$2, cursor, Utils.$$Math.minOptInt(pq.fromBlock - 1 | 0, queryEndBlock), knownHeight, chainTargetBlock, maybeChunkRange, (10 - pendingCount | 0) - chunksUsedThisCall | 0, p$2, chainTargetItems / partitionsCount, p$2.selection, p$2.addressesByContractName);
         chunksUsedThisCall = chunksUsedThisCall + (bucket.length - beforeLen | 0) | 0;
         gapFillCost = gapFillCost + cost$1;
       }
@@ -1136,24 +1163,37 @@ function getNextQuery(param, chainTargetBlock, chainTargetItems) {
     }
     reservedByPartition[fs.partitionId] = cost;
   });
+  let isInRange = fs => {
+    let tmp = false;
+    if (fs.cursor <= chainTargetBlock) {
+      let eb = fs.queryEndBlock;
+      tmp = eb !== undefined ? fs.cursor <= eb : true;
+    }
+    if (tmp) {
+      return (fs.pendingCount + fs.chunksUsedThisCall | 0) < 10;
+    } else {
+      return false;
+    }
+  };
+  let inRangeStates = fillStates.filter(isInRange);
   let emitQueries = (fs, budget) => {
     let p = fs.p;
     let eb = fs.queryEndBlock;
     let maxBlock = eb !== undefined ? eb : chainTargetBlock;
-    let minHistoryRange = fs.maybeChunkRange;
-    if (minHistoryRange !== undefined) {
-      let density = p.prevRangeSize / p.prevQueryRange;
-      let chunkSize = Js_math.ceil_int(minHistoryRange * 1.8);
-      let chunkCost = density * chunkSize;
+    let match = fs.maybeChunkRange;
+    let match$1 = getTrustedDensity(p);
+    if (match !== undefined && match$1 !== undefined && match$1 > 0) {
+      let chunkSize = Js_math.ceil_int(match * 1.8);
+      let chunkCost = match$1 * chunkSize;
       let maxChunksRemaining = (10 - fs.pendingCount | 0) - fs.chunksUsedThisCall | 0;
-      let affordable = chunkCost > 0 ? Math.floor(budget / chunkCost) | 0 : maxChunksRemaining;
+      let affordable = Math.floor(budget / chunkCost) | 0;
       let numChunks = Primitive_int.max(1, Primitive_int.min(affordable, maxChunksRemaining));
       let consumed = 0;
       let created = 0;
       let chunkFromBlock = fs.cursor;
       while (created < numChunks && chunkFromBlock <= maxBlock) {
         let chunkToBlock = Primitive_int.min((chunkFromBlock + chunkSize | 0) - 1 | 0, maxBlock);
-        let itemsTarget = density * ((chunkToBlock - chunkFromBlock | 0) + 1 | 0);
+        let itemsTarget = Primitive_int.max(1, Math.ceil(match$1 * ((chunkToBlock - chunkFromBlock | 0) + 1 | 0)) | 0);
         fs.bucket.push({
           partitionId: fs.partitionId,
           fromBlock: chunkFromBlock,
@@ -1171,7 +1211,7 @@ function getNextQuery(param, chainTargetBlock, chainTargetItems) {
       fs.chunksUsedThisCall = fs.chunksUsedThisCall + created | 0;
       return consumed;
     }
-    let itemsTarget$1 = Primitive_float.min(Math.round(budget), 10000);
+    let itemsTarget$1 = Primitive_int.max(1, Math.round(budget) | 0);
     fs.bucket.push({
       partitionId: fs.partitionId,
       fromBlock: fs.cursor,
@@ -1185,41 +1225,22 @@ function getNextQuery(param, chainTargetBlock, chainTargetItems) {
     fs.chunksUsedThisCall = fs.chunksUsedThisCall + 1 | 0;
     return itemsTarget$1;
   };
-  let isInRange = fs => {
-    let tmp = false;
-    if (fs.cursor <= chainTargetBlock) {
-      let eb = fs.queryEndBlock;
-      tmp = eb !== undefined ? fs.cursor <= eb : true;
-    }
-    if (tmp) {
-      return (fs.pendingCount + fs.chunksUsedThisCall | 0) < 10;
-    } else {
-      return false;
-    }
-  };
-  let notFilledPartitions = fillStates.filter(isInRange);
-  let reservedFromRange = {
-    contents: 0
+  let notFilledPartitions = inRangeStates;
+  let remainingBudget = {
+    contents: rangeItemsTarget
   };
-  while (notFilledPartitions.length !== 0 && reservedFromRange.contents < rangeItemsTarget) {
-    let n = notFilledPartitions.length;
-    let footprintSum = {
-      contents: 0
-    };
-    notFilledPartitions.forEach(fs => {
-      footprintSum.contents = footprintSum.contents + reservedByPartition[fs.partitionId];
-    });
-    let line = (rangeItemsTarget - reservedFromRange.contents + footprintSum.contents) / n;
+  while (notFilledPartitions.length !== 0 && remainingBudget.contents > 0) {
+    let level = waterLevel(remainingBudget.contents, notFilledPartitions.map(fs => reservedByPartition[fs.partitionId]));
     let next = [];
     notFilledPartitions.forEach(fs => {
       let reserved = reservedByPartition[fs.partitionId];
-      let budget = line - reserved;
+      let budget = level - reserved;
       if (budget <= 0) {
         return;
       }
       let consumed = emitQueries(fs, budget);
       reservedByPartition[fs.partitionId] = reserved + consumed;
-      reservedFromRange.contents = reservedFromRange.contents + consumed;
+      remainingBudget.contents = remainingBudget.contents - consumed;
       if (isInRange(fs)) {
         next.push(fs);
         return;
@@ -1650,20 +1671,15 @@ function updateKnownHeight(fetchState, knownHeight) {
   }
 }
 
-let minItemsTarget = 2000;
-
-let maxItemsTarget = 10000;
-
 let blockItemLogIndex = 16777216;
 
 let maxPendingChunksPerPartition = 10;
 
 export {
   deriveContractNameByAddress,
-  minItemsTarget,
-  maxItemsTarget,
   densityItemsTarget,
   getMinHistoryRange,
+  getTrustedDensity,
   getMinQueryRange,
   OptimizedPartitions,
   bufferBlockNumber,
@@ -1682,6 +1698,7 @@ export {
   startFetchingQueries,
   maxPendingChunksPerPartition,
   pushGapFillQueries,
+  waterLevel,
   getNextQuery,
   hasReadyItem,
   getReadyItemsCount,

package/src/HandlerLoader.res

@@ -78,7 +78,7 @@ let autoLoadFromSrcHandlers = async (~handlers: string) => {
 // (populating the global `HandlerRegister` registry as a side effect) and
 // returns the resulting per-chain registrations.
 let registerAllHandlers = async (~config: Config.t): HandlerRegister.registrationsByChainId => {
-  HandlerRegister.startRegistration(~ecosystem=config.ecosystem)
+  HandlerRegister.startRegistration(~config)
 
   // Auto-load all .js files from src/handlers directory
   await autoLoadFromSrcHandlers(~handlers=config.handlers)

package/src/HandlerLoader.res.mjs

@@ -57,7 +57,7 @@ async function autoLoadFromSrcHandlers(handlers) {
 }
 
 async function registerAllHandlers(config) {
-  HandlerRegister.startRegistration(config.ecosystem);
+  HandlerRegister.startRegistration(config);
   await autoLoadFromSrcHandlers(config.handlers);
   await Promise.all(config.contractHandlers.map(param => registerContractHandlers(param.name, param.handler)));
   return HandlerRegister.finishRegistration(config);