envio 3.2.0 → 3.3.0-alpha.0

package/src/FetchState.res

@@ -1,9 +1,6 @@
-type contractConfig = {filterByAddresses: bool, startBlock: option<int>}
+type contractConfig = {startBlock: option<int>}
 
-type indexingAddress = {
-  ...Internal.indexingAddress,
-  effectiveStartBlock: int,
-}
+type indexingAddress = Internal.indexingContract
 
 let deriveEffectiveStartBlock = (~registrationBlock: int, ~contractStartBlock: option<int>) => {
   Pervasives.max(Pervasives.max(registrationBlock, 0), contractStartBlock->Option.getOr(0))
@@ -499,9 +496,10 @@ type t = {
   blockLag: int,
   // Buffer of items ordered from earliest to latest
   buffer: array<Internal.item>,
-  // How many items we should aim to have in the buffer
-  // ready for processing
-  targetBufferSize: int,
+  // Caps how far ahead onBlock items are pre-generated (set to 2x the batch
+  // size). Fetch depth is bounded separately by getNextQuery's bufferLimit, the
+  // chain's per-tick slice of the indexer-wide pool.
+  maxOnBlockBufferSize: int,
   onBlockConfigs: array<Internal.onBlockConfig>,
   knownHeight: int,
   firstEventBlock: option<int>,
@@ -557,7 +555,7 @@ let numAddresses = fetchState => fetchState.indexingAddresses->Utils.Dict.size
 
 // Appends Block items produced by the onBlock handlers for every block in
 // (fromBlock, maxBlockNumber] into mutItems and returns the new
-// latestOnBlockBlockNumber pointer. targetBufferSize bounds how many items
+// latestOnBlockBlockNumber pointer. maxOnBlockBufferSize bounds how many items
 // are generated at once to prevent OOM.
 let appendOnBlockItems = (
   ~mutItems: array<Internal.item>,
@@ -565,7 +563,7 @@ let appendOnBlockItems = (
   ~indexerStartBlock,
   ~fromBlock,
   ~maxBlockNumber,
-  ~targetBufferSize,
+  ~maxOnBlockBufferSize,
 ) => {
   let newItemsCounter = ref(0)
   let latestOnBlockBlockNumber = ref(fromBlock)
@@ -576,7 +574,7 @@ let appendOnBlockItems = (
   while (
     latestOnBlockBlockNumber.contents < maxBlockNumber &&
       // Additional safeguard to prevent OOM
-      newItemsCounter.contents <= targetBufferSize
+      newItemsCounter.contents <= maxOnBlockBufferSize
   ) {
     let blockNumber = latestOnBlockBlockNumber.contents + 1
     latestOnBlockBlockNumber := blockNumber
@@ -630,7 +628,7 @@ let updateInternal = (
   | [] => knownHeight
   | onBlockConfigs => {
       // Calculate the max block number we are going to create items for
-      // Use targetBufferSize to get the last target item in the buffer
+      // Use maxOnBlockBufferSize to get the last target item in the buffer
       //
       // mutItems is not very reliable, since it might not be sorted,
       // but the chances for it happen are very low and not critical
@@ -639,7 +637,7 @@ let updateInternal = (
       let maxBlockNumber = switch switch mutItemsRef.contents {
       | Some(mutItems) => mutItems
       | None => fetchState.buffer
-      }->Array.get(fetchState.targetBufferSize - 1) {
+      }->Array.get(fetchState.maxOnBlockBufferSize - 1) {
       | Some(item) => item->Internal.getItemBlockNumber
       | None =>
         switch optimizedPartitions->OptimizedPartitions.getLatestFullyFetchedBlock {
@@ -660,7 +658,7 @@ let updateInternal = (
         ~indexerStartBlock=fetchState.startBlock,
         ~fromBlock=fetchState.latestOnBlockBlockNumber,
         ~maxBlockNumber,
-        ~targetBufferSize=fetchState.targetBufferSize,
+        ~maxOnBlockBufferSize=fetchState.maxOnBlockBufferSize,
       )
     }
   }
@@ -672,7 +670,7 @@ let updateInternal = (
     normalSelection: fetchState.normalSelection,
     chainId: fetchState.chainId,
     onBlockConfigs: fetchState.onBlockConfigs,
-    targetBufferSize: fetchState.targetBufferSize,
+    maxOnBlockBufferSize: fetchState.maxOnBlockBufferSize,
     optimizedPartitions,
     latestOnBlockBlockNumber,
     indexingAddresses,
@@ -756,7 +754,6 @@ Returns OptimizedPartitions.t directly.
 */
 let createPartitionsFromIndexingAddresses = (
   ~registeringContractsByContract: dict<dict<indexingAddress>>,
-  ~contractConfigs: dict<contractConfig>,
   ~dynamicContracts: Utils.Set.t<string>,
   ~normalSelection: selection,
   ~maxAddrInPartition: int,
@@ -778,9 +775,6 @@ OptimizedPartitions.t => {
     let addresses =
       registeringContracts->Dict.keysToArray->(Utils.magic: array<string> => array<Address.t>)
 
-    // Can unsafely get it, because we already filtered out the contracts
-    // that don't have any events to fetch
-    let contractConfig = contractConfigs->Dict.getUnsafe(contractName)
     let isDynamic = dynamicContracts->Utils.Set.has(contractName)
     let partitions = isDynamic ? dynamicPartitions : nonDynamicPartitions
 
@@ -802,30 +796,23 @@ OptimizedPartitions.t => {
       let maybeNextStartBlockKey =
         ascKeys->Array.getUnsafe(idx + 1)->(Utils.magic: string => option<string>)
 
-      // For this case we can't filter out events earlier than contract registration
-      // on the client side, so we need to keep the old logic of creating
-      // a partition for every block range, so there are no irrelevant events
-      let shouldAllocateNewPartition = if contractConfig.filterByAddresses {
-        true
-      } else {
-        switch maybeNextStartBlockKey {
-        | None => true
-        | Some(nextStartBlockKey) => {
-            let nextStartBlock = nextStartBlockKey->Int.fromString->Option.getUnsafe
-            let shouldJoinCurrentStartBlock =
-              nextStartBlock - startBlockRef.contents < OptimizedPartitions.tooFarBlockRange
-
-            // If dynamic contract registration are close to eachother
-            // and it's possible to use dc.effectiveStartBlock to filter out events on client side
-            // then we can optimize the number of partitions,
-            // by putting dcs with different startBlocks in the same partition
-            if shouldJoinCurrentStartBlock {
-              addressesRef :=
-                addressesRef.contents->Array.concat(byStartBlock->Dict.getUnsafe(nextStartBlockKey))
-              false
-            } else {
-              true
-            }
+      let shouldAllocateNewPartition = switch maybeNextStartBlockKey {
+      | None => true
+      | Some(nextStartBlockKey) => {
+          let nextStartBlock = nextStartBlockKey->Int.fromString->Option.getUnsafe
+          let shouldJoinCurrentStartBlock =
+            nextStartBlock - startBlockRef.contents < OptimizedPartitions.tooFarBlockRange
+
+          // Addresses with different start blocks within range share a partition;
+          // events before each address's effectiveStartBlock are dropped on the
+          // client side (EventRouter for the srcAddress, the event's
+          // clientAddressFilter for address-valued params).
+          if shouldJoinCurrentStartBlock {
+            addressesRef :=
+              addressesRef.contents->Array.concat(byStartBlock->Dict.getUnsafe(nextStartBlockKey))
+            false
+          } else {
+            true
           }
         }
       }
@@ -907,15 +894,10 @@ OptimizedPartitions.t => {
           }
         }
 
-        let nextContractName =
-          nextP.addressesByContractName->Dict.keysToArray->Utils.Array.firstUnsafe
-        let hasFilterByAddresses = (
-          contractConfigs->Dict.getUnsafe(nextContractName)
-        ).filterByAddresses
         let isTooFar = currentPBlock + OptimizedPartitions.tooFarBlockRange < nextPBlock
 
-        if isTooFar || hasFilterByAddresses {
-          // Too far or address-filtered: mergeBlock on current, merge addresses into next
+        if isTooFar {
+          // Too far: mergeBlock on current, merge addresses into next
           mergedNonDynamic
           ->Array.push({
             ...currentP,
@@ -927,7 +909,7 @@ OptimizedPartitions.t => {
               addressesByContractName: mergedAddresses,
             }
         } else {
-          // Close and not address-filtered: push next's addresses into current
+          // Close: push next's addresses into current
           currentPRef := {
               ...currentP,
               addressesByContractName: mergedAddresses,
@@ -968,7 +950,6 @@ let registerDynamicContracts = (
   let indexingAddresses = fetchState.indexingAddresses
   let registeringContractsByContract: dict<dict<indexingAddress>> = Dict.make()
   let earliestRegisteringEventBlockNumber = ref(%raw(`Infinity`))
-  let hasDCWithFilterByAddresses = ref(false)
   // Addresses registered for contracts without matching events. These are not
   // added to partitions, but they are tracked on fetchState.indexingAddresses
   // so that later conflicting registrations are detected, and are persisted
@@ -991,7 +972,7 @@ let registerDynamicContracts = (
         let shouldRemove = ref(false)
 
         switch fetchState.contractConfigs->Utils.Dict.dangerouslyGetNonOption(dc.contractName) {
-        | Some({filterByAddresses, startBlock: contractStartBlock}) =>
+        | Some({startBlock: contractStartBlock}) =>
           let dcWithStartBlock: indexingAddress = {
             address: dc.address,
             contractName: dc.contractName,
@@ -1038,9 +1019,7 @@ let registerDynamicContracts = (
               // FIXME: This unsafely relies on the asc order of the items
               // which is 99% true, but there were cases when the source ordering was wrong
               false
-            | None =>
-              hasDCWithFilterByAddresses := hasDCWithFilterByAddresses.contents || filterByAddresses
-              true
+            | None => true
             }
             if shouldUpdate {
               earliestRegisteringEventBlockNumber :=
@@ -1218,7 +1197,6 @@ let registerDynamicContracts = (
 
       let optimizedPartitions = createPartitionsFromIndexingAddresses(
         ~registeringContractsByContract,
-        ~contractConfigs=fetchState.contractConfigs,
         ~dynamicContracts=dynamicContractsRef.contents,
         ~normalSelection=fetchState.normalSelection,
         ~maxAddrInPartition=fetchState.optimizedPartitions.maxAddrInPartition,
@@ -1245,6 +1223,20 @@ let handleQueryResult = (
   ~latestFetchedBlock: blockNumberAndTimestamp,
   ~newItems,
 ): t => {
+  // Drop events an address-param filter rejects. A merged partition may
+  // over-fetch a wildcard event whose indexed address param references an
+  // address registered after the log's block; `clientAddressFilter` is the
+  // param-level analogue of EventRouter's srcAddress effectiveStartBlock check.
+  let newItems = newItems->Array.filter(item =>
+    switch item {
+    | Internal.Event({eventConfig, payload, blockNumber}) =>
+      switch eventConfig.clientAddressFilter {
+      | Some(filter) => filter(payload, blockNumber, fetchState.indexingAddresses)
+      | None => true
+      }
+    | _ => true
+    }
+  )
   fetchState->updateInternal(
     ~optimizedPartitions=fetchState.optimizedPartitions->OptimizedPartitions.handleQueryResponse(
       ~query,
@@ -1328,26 +1320,30 @@ let pushQueriesForRange = (
         | None => maxQueryBlockNumber
         }
         let chunkSize = Js.Math.ceil_int(chunkRange->Int.toFloat *. 1.8)
-        if rangeFromBlock + 2 * chunkSize - 1 <= maxBlock {
-          // Create 2 chunks of ceil(1.8 * chunkRange) each
-          queries->Array.push({
-            partitionId,
-            fromBlock: rangeFromBlock,
-            toBlock: Some(rangeFromBlock + chunkSize - 1),
-            isChunk: true,
-            selection,
-            addressesByContractName,
-            indexingAddresses,
-          })
-          queries->Array.push({
-            partitionId,
-            fromBlock: rangeFromBlock + chunkSize,
-            toBlock: Some(rangeFromBlock + 2 * chunkSize - 1),
-            isChunk: true,
-            selection,
-            addressesByContractName,
-            indexingAddresses,
-          })
+        // Probe with two smaller chunks first so their responses come back
+        // quickly and refresh the chunking heuristic, then three full-size chunks.
+        let probeSize = Js.Math.ceil_int(chunkRange->Int.toFloat *. 0.9)
+        let getChunkSize = chunkIdx => chunkIdx < 2 ? probeSize : chunkSize
+        if rangeFromBlock + getChunkSize(0) + getChunkSize(1) - 1 <= maxBlock {
+          let chunkFromBlock = ref(rangeFromBlock)
+          let chunkIdx = ref(0)
+          while (
+            chunkIdx.contents < 5 &&
+              chunkFromBlock.contents + getChunkSize(chunkIdx.contents) - 1 <= maxBlock
+          ) {
+            let chunkToBlock = chunkFromBlock.contents + getChunkSize(chunkIdx.contents) - 1
+            queries->Array.push({
+              partitionId,
+              fromBlock: chunkFromBlock.contents,
+              toBlock: Some(chunkToBlock),
+              isChunk: true,
+              selection,
+              addressesByContractName,
+              indexingAddresses,
+            })
+            chunkFromBlock := chunkToBlock + 1
+            chunkIdx := chunkIdx.contents + 1
+          }
         } else {
           // Not enough room for 2 chunks, fall back to a single query
           queries->Array.push({
@@ -1365,23 +1361,30 @@ let pushQueriesForRange = (
   }
 }
 
+// Most parallel in-flight chunk queries a single partition may have at once.
+let maxPendingChunksPerPartition = 10
+
 let getNextQuery = (
   {
     buffer,
     optimizedPartitions,
-    targetBufferSize,
     indexingAddresses,
     blockLag,
     latestOnBlockBlockNumber,
     knownHeight,
   } as fetchState: t,
   ~concurrencyLimit,
+  ~bufferLimit,
 ) => {
   let headBlockNumber = knownHeight - blockLag
   if headBlockNumber <= 0 {
     WaitingForNewBlock
   } else if concurrencyLimit === 0 {
     ReachedMaxConcurrency
+  } else if bufferLimit <= 0 {
+    // No room left in the shared buffer pool for this chain; wait for processing
+    // to drain before fetching more.
+    NothingToQuery
   } else {
     let isOnBlockBehindTheHead = latestOnBlockBlockNumber < headBlockNumber
     let shouldWaitForNewBlock = ref(
@@ -1392,13 +1395,12 @@ let getNextQuery = (
       !isOnBlockBehindTheHead,
     )
 
-    // We want to limit the buffer size to targetBufferSize (usually 3 * batchSize)
-    // To make sure the processing always has some buffer
-    // and not increase the memory usage too much
-    // If a partition fetched further
-    // it should be skipped until the buffer is consumed
+    // Limit how far ahead we fetch to bufferLimit items (this chain's share of
+    // the indexer-wide buffer pool) so processing always has buffer without
+    // ballooning memory. A partition that fetched further is skipped until the
+    // buffer drains.
     let maxQueryBlockNumber = {
-      switch buffer->Array.get(targetBufferSize - 1) {
+      switch buffer->Array.get(bufferLimit - 1) {
       | Some(item) =>
         // Just in case check that we don't query beyond the current block
         Pervasives.min(item->Internal.getItemBlockNumber, knownHeight)
@@ -1415,8 +1417,9 @@ let getNextQuery = (
       let partitionId = optimizedPartitions.idsInAscOrder->Array.getUnsafe(idx)
       let p = optimizedPartitions.entities->Dict.getUnsafe(partitionId)
 
+      let pendingCount = p.mutPendingQueries->Array.length
       let isBehindTheHead = p.latestFetchedBlock.blockNumber < headBlockNumber
-      let hasPendingQueries = p.mutPendingQueries->Utils.Array.notEmpty
+      let hasPendingQueries = pendingCount > 0
 
       if hasPendingQueries || isBehindTheHead {
         // Even if there are some partitions waiting for the new block
@@ -1427,6 +1430,8 @@ let getNextQuery = (
         shouldWaitForNewBlock := false
       }
 
+      let partitionQueriesStart = queries->Array.length
+
       // Compute queryEndBlock for this partition
       let queryEndBlock = Utils.Math.minOptInt(fetchState.endBlock, p.mergeBlock)
       let queryEndBlock = switch blockLag {
@@ -1494,6 +1499,22 @@ let getNextQuery = (
         )
       }
 
+      // Cap parallel in-flight chunks per partition so a single partition can't
+      // drain the whole concurrency budget (especially the high realtime one).
+      // Keep the earliest new chunks; the furthest-ahead ones wait for the next
+      // round once these resolve.
+      let maxNewChunks = Pervasives.max(0, maxPendingChunksPerPartition - pendingCount)
+      let generatedCount = queries->Array.length - partitionQueriesStart
+      if generatedCount > maxNewChunks {
+        queries
+        ->Array.splice(
+          ~start=partitionQueriesStart + maxNewChunks,
+          ~remove=generatedCount - maxNewChunks,
+          ~insert=[],
+        )
+        ->ignore
+      }
+
       idxRef := idxRef.contents + 1
     }
 
@@ -1556,7 +1577,7 @@ let make = (
   ~addresses: array<Internal.indexingAddress>,
   ~maxAddrInPartition,
   ~chainId,
-  ~targetBufferSize,
+  ~maxOnBlockBufferSize,
   ~knownHeight,
   ~progressBlockNumber=startBlock - 1,
   ~onBlockConfigs=[],
@@ -1572,15 +1593,14 @@ let make = (
   let normalEventConfigs = []
   let contractNamesWithNormalEvents = Utils.Set.make()
   let indexingAddresses = Dict.make()
-  let contractConfigs = Dict.make()
+  let contractConfigs: dict<contractConfig> = Dict.make()
 
   eventConfigs->Array.forEach(ec => {
     switch contractConfigs->Utils.Dict.dangerouslyGetNonOption(ec.contractName) {
-    | Some({filterByAddresses, startBlock}) =>
+    | Some({startBlock}) =>
       contractConfigs->Dict.set(
         ec.contractName,
         {
-          filterByAddresses: filterByAddresses || ec.filterByAddresses,
           startBlock: switch (startBlock, ec.startBlock) {
           | (Some(a), Some(b)) => Some(Pervasives.min(a, b))
           | (Some(_) as s, None) | (None, Some(_) as s) => s
@@ -1592,7 +1612,6 @@ let make = (
       contractConfigs->Dict.set(
         ec.contractName,
         {
-          filterByAddresses: ec.filterByAddresses,
           startBlock: ec.startBlock,
         },
       )
@@ -1672,7 +1691,6 @@ let make = (
 
   let optimizedPartitions = createPartitionsFromIndexingAddresses(
     ~registeringContractsByContract,
-    ~contractConfigs,
     ~dynamicContracts,
     ~normalSelection,
     ~maxAddrInPartition,
@@ -1708,7 +1726,7 @@ let make = (
       ~indexerStartBlock=startBlock,
       ~fromBlock=progressBlockNumber,
       ~maxBlockNumber,
-      ~targetBufferSize,
+      ~maxOnBlockBufferSize,
     )
   } else {
     progressBlockNumber
@@ -1725,7 +1743,7 @@ let make = (
     indexingAddresses,
     blockLag,
     onBlockConfigs,
-    targetBufferSize,
+    maxOnBlockBufferSize,
     knownHeight,
     buffer,
     firstEventBlock,
@@ -1749,6 +1767,25 @@ let make = (
 
 let bufferSize = ({buffer}: t) => buffer->Array.length
 
+// Number of buffered items at or below the ready frontier (processable now,
+// i.e. not stuck behind a gap from a lagging partition or out-of-order chunk).
+// The buffer is kept sorted, so binary-search the frontier in O(log n).
+let bufferReadyCount = (fetchState: t) => {
+  let frontier = fetchState->bufferBlockNumber
+  let buffer = fetchState.buffer
+  let lo = ref(0)
+  let hi = ref(buffer->Array.length)
+  while lo.contents < hi.contents {
+    let mid = (lo.contents + hi.contents) / 2
+    if buffer->Array.getUnsafe(mid)->Internal.getItemBlockNumber <= frontier {
+      lo := mid + 1
+    } else {
+      hi := mid
+    }
+  }
+  lo.contents
+}
+
 let rollbackPendingQueries = (mutPendingQueries: array<pendingQuery>, ~targetBlockNumber) => {
   // - Remove queries where fromBlock > target
   // - Cap fetchedBlock at target where fetchedBlock > target
@@ -1875,7 +1912,6 @@ let rollback = (fetchState: t, ~targetBlockNumber) => {
   // Step 3: Recreate partitions from deleted partition addresses
   let optimizedPartitions = createPartitionsFromIndexingAddresses(
     ~registeringContractsByContract,
-    ~contractConfigs=fetchState.contractConfigs,
     ~dynamicContracts=fetchState.optimizedPartitions.dynamicContracts,
     ~normalSelection=fetchState.normalSelection,
     ~maxAddrInPartition=fetchState.optimizedPartitions.maxAddrInPartition,
@@ -1948,6 +1984,18 @@ let isActivelyIndexing = ({endBlock} as fetchState: t) => {
   }
 }
 
+// True once the fetch frontier has reached the (lagged) head or endBlock,
+// regardless of whether the buffer has been consumed yet. Unlike
+// isReadyToEnterReorgThreshold, fetched-but-unprocessed items still count.
+let isFetchingAtHead = ({endBlock, blockLag, knownHeight} as fetchState: t) => {
+  let bufferBlockNumber = fetchState->bufferBlockNumber
+  knownHeight !== 0 &&
+    switch endBlock {
+    | Some(endBlock) if bufferBlockNumber >= endBlock => true
+    | _ => bufferBlockNumber >= knownHeight - blockLag
+    }
+}
+
 let isReadyToEnterReorgThreshold = ({endBlock, blockLag, buffer, knownHeight} as fetchState: t) => {
   let bufferBlockNumber = fetchState->bufferBlockNumber
   knownHeight !== 0 &&
@@ -1958,7 +2006,26 @@ let isReadyToEnterReorgThreshold = ({endBlock, blockLag, buffer, knownHeight} as
   buffer->Utils.Array.isEmpty
 }
 
-let sortForUnorderedBatch = {
+// Lower progress percentage = further behind = higher priority. Shared by the
+// batch ordering and the cross-chain fetch priority.
+let getProgressPercentage = (fetchState: t) => {
+  switch fetchState.firstEventBlock {
+  | None => 0.
+  | Some(firstEventBlock) =>
+    let totalRange = fetchState.knownHeight - firstEventBlock
+    if totalRange <= 0 {
+      0.
+    } else {
+      let progress = switch fetchState.buffer->Array.get(0) {
+      | Some(item) => item->Internal.getItemBlockNumber - firstEventBlock
+      | None => fetchState->bufferBlockNumber - firstEventBlock
+      }
+      progress->Int.toFloat /. totalRange->Int.toFloat
+    }
+  }
+}
+
+let sortForBatch = {
   let hasFullBatch = ({buffer} as fetchState: t, ~batchSizeTarget) => {
     switch buffer->Array.get(batchSizeTarget - 1) {
     | Some(item) => item->Internal.getItemBlockNumber <= fetchState->bufferBlockNumber
@@ -1966,24 +2033,6 @@ let sortForUnorderedBatch = {
     }
   }
 
-  // Lower progress percentage = further behind = higher priority
-  let getProgressPercentage = (fetchState: t) => {
-    switch fetchState.firstEventBlock {
-    | None => 0.
-    | Some(firstEventBlock) =>
-      let totalRange = fetchState.knownHeight - firstEventBlock
-      if totalRange <= 0 {
-        0.
-      } else {
-        let progress = switch fetchState.buffer->Array.get(0) {
-        | Some(item) => item->Internal.getItemBlockNumber - firstEventBlock
-        | None => fetchState->bufferBlockNumber - firstEventBlock
-        }
-        progress->Int.toFloat /. totalRange->Int.toFloat
-      }
-    }
-  }
-
   (fetchStates: array<t>, ~batchSizeTarget: int) => {
     let copied = fetchStates->Array.copy
     copied->Array.sort((a: t, b: t) => {