envio 3.5.0-alpha.3 → 3.5.0-rc.0

package/src/FetchState.res

@@ -19,6 +19,42 @@ type selection = {
   // gates each item against the chain's address store instead of the
   // partition's set. Absent for normal partitions.
   clientFilteredContracts?: array<string>,
+  // The earliest block any of these registrations can produce an item at.
+  // Derived once here, by `makeSelection`, because `getNextQuery` reads it for
+  // every partition on every tick and a chain can hold hundreds of them.
+  //
+  // Absent when a registration is unrestricted and so can fire from the chain
+  // start — which is also the only case where nothing can be skipped, so absent
+  // and "block 0" are the same answer here. Build selections through
+  // `makeSelection` rather than as literals, or this goes stale.
+  startBlock?: int,
+}
+
+// The earliest block any of the registrations can produce an item at, or `None`
+// when one of them is unrestricted. Nothing below it is worth querying.
+let deriveSelectionStartBlock = (onEventRegistrations: array<Internal.onEventRegistration>) => {
+  let earliest = ref(None)
+  let idx = ref(0)
+  let unrestricted = ref(onEventRegistrations->Utils.Array.isEmpty)
+  while !unrestricted.contents && idx.contents < onEventRegistrations->Array.length {
+    switch (onEventRegistrations->Array.getUnsafe(idx.contents)).startBlock {
+    | None => unrestricted := true
+    | Some(startBlock) =>
+      switch earliest.contents {
+      | Some(current) if current <= startBlock => ()
+      | _ => earliest := Some(startBlock)
+      }
+    }
+    idx := idx.contents + 1
+  }
+  unrestricted.contents ? None : earliest.contents
+}
+
+let makeSelection = (~onEventRegistrations, ~dependsOnAddresses, ~clientFilteredContracts=?) => {
+  onEventRegistrations,
+  dependsOnAddresses,
+  ?clientFilteredContracts,
+  startBlock: ?deriveSelectionStartBlock(onEventRegistrations),
 }
 
 type pendingQuery = {
@@ -100,6 +136,43 @@ type query = {
 
 let withAddresses = (p: partition, addresses: AddressSet.t) => {...p, addresses}
 
+// The selection a query over [fromBlock, toBlock] actually needs: a registration
+// whose own start block sits past the range can't produce an item there, so
+// leaving it out keeps the source from asking the server for its logs at all.
+// The address gate can't express this — it's contract-wide, so an unrestricted
+// sibling holds it open from the chain start for every registration alike.
+//
+// Returns the selection as-is when nothing is dropped (the common case), and
+// for an open-ended query, which may reach any block and so can exclude nothing.
+// Also when every registration would be dropped: `selectionStartBlock` keeps a
+// partition's cursor at or above its earliest start block, so a query below all
+// of them shouldn't exist — and an empty selection is one no source can build.
+let narrowSelectionToRange = (selection: selection, ~toBlock) =>
+  switch toBlock {
+  | None => selection
+  | Some(toBlock) =>
+    let inRange = selection.onEventRegistrations->Array.filter(reg =>
+      switch reg.startBlock {
+      | Some(startBlock) => startBlock <= toBlock
+      | None => true
+      }
+    )
+    switch inRange->Array.length {
+    // Every registration starts above the range. Narrowing to nothing would
+    // build a selection with no log filters, which each source reads as
+    // "select everything" — the opposite of the intent. The partition is
+    // queried unnarrowed instead and the routers drop what hasn't started.
+    | 0 => selection
+    | length if length === selection.onEventRegistrations->Array.length => selection
+    | _ =>
+      makeSelection(
+        ~onEventRegistrations=inRange,
+        ~dependsOnAddresses=selection.dependsOnAddresses,
+        ~clientFilteredContracts=?selection.clientFilteredContracts,
+      )
+    }
+  }
+
 // itemsEst for a query over [fromBlock, toBlock] at the given event density
 // (items/block). toBlock None is the open-ended tail, capped at
 // chainTargetBlock — the soft per-tick horizon the owning chain wants to reach
@@ -127,6 +200,14 @@ let getMinHistoryRange = (p: partition) => {
 // source-capacity history, including when an itemsTarget cap truncates a response.
 let getTrustedDensity = (p: partition) => p.eventDensity
 
+// A response is still owed for this partition, so it owns range nothing else
+// accounts for: removing it would let the frontier advance over blocks the
+// response can still deliver items from, and orphan the response itself.
+// Narrower than a non-empty queue on purpose — a settled query parked behind a
+// gap is data already in hand, and waiting on it would wedge a partition that
+// no longer queries.
+let isFetching = (p: partition) => p.mutPendingQueries->Array.some(pq => pq.fetchedBlock === None)
+
 let getMinQueryRange = (partitions: array<partition>) => {
   let min = ref(0)
   for i in 0 to partitions->Array.length - 1 {
@@ -362,9 +443,10 @@ module OptimizedPartitions = {
       // A partition already at or past its merge block is done — normally
       // handleQueryResponse removes it when the response lands, but a rollback
       // can cap the merge block at the rolled-back frontier, leaving no
-      // response to ever remove it on.
+      // response to ever remove it on. A retired partition still awaiting a
+      // response stays until it lands.
       | {mergeBlock: Some(mergeBlock)} =>
-        if p.latestFetchedBlock.blockNumber < mergeBlock {
+        if p.latestFetchedBlock.blockNumber < mergeBlock || p->isFetching {
           newPartitions->Array.push(p)->ignore
         }
       // Since it's not a dynamic contract partition,
@@ -487,6 +569,12 @@ module OptimizedPartitions = {
           | Some(anchorSafeBlock) =>
             if p.latestFetchedBlock.blockNumber < anchorSafeBlock {
               anchored->Array.push({...p, mergeBlock: Some(anchorSafeBlock)})->ignore
+            } else if p->isFetching {
+              // Caught up, but a response is still owed: keep it until that
+              // lands rather than dropping the range it owns.
+              anchored
+              ->Array.push({...p, mergeBlock: Some(p.latestFetchedBlock.blockNumber)})
+              ->ignore
             }
           }
         | _ => anchored->Array.push(p)->ignore
@@ -565,6 +653,10 @@ module OptimizedPartitions = {
     }
   }
 
+  // Every path that stops a partition from fetching keeps it until its last
+  // response lands, so a response always has a partition to advance. A rollback
+  // does delete partitions mid-run, but it bumps the indexer epoch and
+  // ChainFetching drops older responses before they reach here.
   let rec handleQueryResponse = (
     optimizedPartitions: t,
     ~query,
@@ -572,21 +664,13 @@ module OptimizedPartitions = {
     ~itemsCount,
     ~latestFetchedBlock: blockNumberAndTimestamp,
   ) =>
-    switch optimizedPartitions.entities->Utils.Dict.dangerouslyGetNonOption(query.partitionId) {
-    // The partition was absorbed into the address-free client-filtered partition
-    // while this query was in flight. Its items are still merged into the buffer
-    // by the caller (and deduped); there's no partition bookkeeping left to do,
-    // and its reservation is released by ChainState regardless.
-    | None => optimizedPartitions
-    | Some(p) =>
-      optimizedPartitions->handleQueryResponseForPartition(
-        ~p,
-        ~query,
-        ~knownHeight,
-        ~itemsCount,
-        ~latestFetchedBlock,
-      )
-    }
+    optimizedPartitions->handleQueryResponseForPartition(
+      ~p=optimizedPartitions->getOrThrow(~partitionId=query.partitionId),
+      ~query,
+      ~knownHeight,
+      ~itemsCount,
+      ~latestFetchedBlock,
+    )
 
   and handleQueryResponseForPartition = (
     optimizedPartitions: t,
@@ -659,11 +743,15 @@ module OptimizedPartitions = {
       ~initialLatestFetchedBlock=p.latestFetchedBlock,
     )
 
-    // Check if partition reached its mergeBlock and should be removed
-    let partitionReachedMergeBlock = switch p.mergeBlock {
-    | Some(mergeBlock) => updatedLatestFetchedBlock.blockNumber >= mergeBlock
-    | None => false
-    }
+    // Check if partition reached its mergeBlock and should be removed. A
+    // retired partition can hold several queries at once, so it goes only when
+    // the last of them has landed.
+    let partitionReachedMergeBlock =
+      switch p.mergeBlock {
+      | Some(mergeBlock) => updatedLatestFetchedBlock.blockNumber >= mergeBlock
+      | None => false
+      } &&
+      !(p->isFetching)
 
     if partitionReachedMergeBlock {
       mutEntities->Utils.Dict.deleteInPlace(p.id)
@@ -745,7 +833,7 @@ type t = {
   endBlock: option<int>,
   normalSelection: selection,
   // Not used for logic - only metadata
-  chainId: int,
+  chainId: ChainId.t,
   // The block number of the latest block which was added to the queue
   // by the onBlock configs
   // Need a separate pointer for this
@@ -1090,9 +1178,10 @@ let claimedFetchedBlock = (p: partition, ~knownHeight) =>
 // fetching without tearing down established state:
 // - The standing address-free partition (no mergeBlock) keeps its id, frontier,
 //   in-flight queries and learned density. Only when a newly-switched contract
-//   must be added does its selection change — under a fresh id, so in-flight
-//   responses built from the old selection are orphaned instead of advancing
-//   the frontier past ranges the new contract wasn't fetched for.
+//   must be added does its selection change — under a fresh id, so responses
+//   built from the old selection can't advance the frontier past ranges the new
+//   contract wasn't fetched for. The old generation is retired beside it,
+//   keeping the queue those responses belong to.
 // - Partitions absorbed below the standing partition's claimed block
 //   (single-contract dynamic partitions and config partitions all of whose
 //   contracts are client-filtered, plus any prior backfill) become one bounded
@@ -1137,6 +1226,21 @@ let collapseClientFilteredContracts = (
     let strippedFrontiers = []
     let anchoredContracts = partitions->OptimizedPartitions.anchoredContracts
 
+    // Retire a partition in place of removing it: same id and selection, same
+    // pending queue, capped at its own frontier so it issues nothing more. It
+    // holds the frontier down and owns its in-flight response until that lands,
+    // then handleQueryResponse drops it.
+    let retire = (p: partition) => {
+      ...p,
+      mergeBlock: Some(p.latestFetchedBlock.blockNumber),
+    }
+    let absorb = p => {
+      absorbedPartitions->Array.push(p)->ignore
+      if p->isFetching {
+        kept->Array.push(p->retire)->ignore
+      }
+    }
+
     partitions->Array.forEach(p =>
       switch p {
       | {selection: {dependsOnAddresses: false}, mergeBlock: None}
@@ -1144,7 +1248,14 @@ let collapseClientFilteredContracts = (
         standingRef := Some(p)
       | {selection: {dependsOnAddresses: false}, mergeBlock: Some(_)} =>
         backfills->Array.push(p)->ignore
-      | {selection: {dependsOnAddresses: false}} => absorbedPartitions->Array.push(p)->ignore
+
+        // The backfill this call builds covers its remaining range from the
+        // same min frontier, so it is superseded — but not before the response
+        // it is waiting on lands.
+        if p->isFetching {
+          kept->Array.push(p->retire)->ignore
+        }
+      | {selection: {dependsOnAddresses: false}} => p->absorb
       | _ =>
         let contractNames = p.addresses->AddressSet.contractNames
         let serverSideNames =
@@ -1163,7 +1274,7 @@ let collapseClientFilteredContracts = (
           // hand that job back to a guessed ceiling.
           kept->Array.push(p)->ignore
         } else if serverSideNames->Utils.Array.isEmpty {
-          absorbedPartitions->Array.push(p)->ignore
+          p->absorb
         } else {
           strippedFrontiers->Array.push(p.latestFetchedBlock)->ignore
           kept
@@ -1218,11 +1329,11 @@ let collapseClientFilteredContracts = (
           clientFilteredContracts->Utils.Set.has(reg.eventConfig.contractName)
         ),
       )
-      let newSelection = {
-        dependsOnAddresses: false,
-        onEventRegistrations: regByIndex->Dict.valuesToArray,
-        clientFilteredContracts: clientFilteredContracts->Utils.Set.toArray,
-      }
+      let newSelection = makeSelection(
+        ~dependsOnAddresses=false,
+        ~onEventRegistrations=regByIndex->Dict.valuesToArray,
+        ~clientFilteredContracts=clientFilteredContracts->Utils.Set.toArray,
+      )
 
       switch standingRef.contents {
       | None =>
@@ -1256,6 +1367,15 @@ let collapseClientFilteredContracts = (
         let standingOut = if selectionChanged {
           let id = nextPartitionIndexRef.contents->Int.toString
           nextPartitionIndexRef := nextPartitionIndexRef.contents + 1
+
+          // A response built from the old selection must not advance the new
+          // one's frontier over ranges the newly-switched contract wasn't
+          // fetched for, so the continuing partition takes a fresh id and an
+          // empty queue. The old generation is retired beside it rather than
+          // dropped: it keeps the queue those responses belong to.
+          if standing->isFetching {
+            kept->Array.push(standing->retire)->ignore
+          }
           {...standing, id, selection: newSelection, mutPendingQueries: []}
         } else {
           standing
@@ -1297,6 +1417,65 @@ let collapseClientFilteredContracts = (
   }
 }
 
+let warnAddressRegistration = (
+  ~chainId: ChainId.t,
+  ~contractAddress: Address.t,
+  ~params,
+  message,
+) =>
+  Logging.createChild(
+    ~params={
+      "chainId": chainId,
+      "contractAddress": contractAddress->Address.toString,
+      "details": params,
+    },
+  )->Logging.childWarn(message)
+
+// A rejected registration is simply absent from every partition, so without a
+// warning the user sees a contract that never indexes and nothing saying why.
+// Shared by config-time registration in `make` and by dynamic registration.
+let warnRejectedRegistration = (
+  verdict: AddressStore.verdict,
+  ~chainId: ChainId.t,
+  ~contractAddress: Address.t,
+  ~contractName: string,
+) =>
+  switch verdict {
+  | Conflict({existingContractName}) =>
+    warnAddressRegistration(
+      ~chainId,
+      ~contractAddress,
+      ~params={
+        "existingContractType": existingContractName,
+        "newContractType": contractName,
+      },
+      `Skipping contract registration: Contract address is already registered for one contract and cannot be registered for another contract.`,
+    )
+  | Duplicate({effectiveStartBlock, existingEffectiveStartBlock}) =>
+    // FIXME: Instead of filtering out duplicates, we should check the block
+    // number first. If a new registration has an earlier block number we
+    // should register it for the missing block range.
+    if existingEffectiveStartBlock > effectiveStartBlock {
+      warnAddressRegistration(
+        ~chainId,
+        ~contractAddress,
+        ~params={
+          "existingBlockNumber": existingEffectiveStartBlock,
+          "newBlockNumber": effectiveStartBlock,
+        },
+        `Skipping contract registration: Contract address is already registered at a later block number. Currently registration of the same contract address is not supported by Envio. Reach out to us if it's a problem for you.`,
+      )
+    }
+  | Invalid =>
+    warnAddressRegistration(
+      ~chainId,
+      ~contractAddress,
+      ~params={"contractName": contractName},
+      `Skipping contract registration: Not a valid address for this chain's ecosystem.`,
+    )
+  | Added(_) | NoEvents(_) => ()
+  }
+
 /**
 Creates partitions from indexing addresses with two phases:
 Phase 1: Create per-contract-name partitions (smart grouping by startBlock)
@@ -1543,21 +1722,12 @@ let registerDynamicContracts = (
   // single batch conflict the same way as across batches.
   let verdicts = addressStore->AddressStore.registerBatch(registrations)
 
-  let warn = (~contractAddress, ~params, message) =>
-    Logging.createChild(
-      ~params={
-        "chainId": fetchState.chainId,
-        "contractAddress": contractAddress,
-        "details": params,
-      },
-    )->Logging.childWarn(message)
-
   let registeringContractNames = []
   let hasNoEventsUpdatesRef = ref(false)
   for idx in 0 to verdicts->Array.length - 1 {
     let registration = registrations->Array.getUnsafe(idx)
-    let contractAddress = registration.address->Address.toString
-    let keep = switch verdicts->Array.getUnsafe(idx) {
+    let verdict = verdicts->Array.getUnsafe(idx)
+    let keep = switch verdict {
     | Added(_) =>
       if !(registeringContractNames->Array.includes(registration.contractName)) {
         registeringContractNames->Array.push(registration.contractName)->ignore
@@ -1568,42 +1738,18 @@ let registerDynamicContracts = (
       // Persist the address to the db so a future config change that adds
       // events for this contract can pick it up on restart, but skip partition
       // registration since there's nothing to fetch.
-      warn(
-        ~contractAddress,
+      warnAddressRegistration(
+        ~chainId=fetchState.chainId,
+        ~contractAddress=registration.address,
         ~params={"contractName": registration.contractName},
         `Persisting contract registration without fetching: Contract doesn't have any events to fetch. It'll be picked up on restart if you add events for the contract.`,
       )
       true
-    | Conflict({existingContractName}) =>
-      warn(
-        ~contractAddress,
-        ~params={
-          "existingContractType": existingContractName,
-          "newContractType": registration.contractName,
-        },
-        `Skipping contract registration: Contract address is already registered for one contract and cannot be registered for another contract.`,
-      )
-      false
-    | Duplicate({effectiveStartBlock, existingEffectiveStartBlock}) =>
-      // FIXME: Instead of filtering out duplicates, we should check the block
-      // number first. If a new registration has an earlier block number we
-      // should register it for the missing block range.
-      if existingEffectiveStartBlock > effectiveStartBlock {
-        warn(
-          ~contractAddress,
-          ~params={
-            "existingBlockNumber": existingEffectiveStartBlock,
-            "newBlockNumber": effectiveStartBlock,
-          },
-          `Skipping contract registration: Contract address is already registered at a later block number. Currently registration of the same contract address is not supported by Envio. Reach out to us if it's a problem for you.`,
-        )
-      }
-      false
-    | Invalid =>
-      warn(
-        ~contractAddress,
-        ~params={"contractName": registration.contractName},
-        `Skipping contract registration: Not a valid address for this chain's ecosystem.`,
+    | Conflict(_) | Duplicate(_) | Invalid =>
+      verdict->warnRejectedRegistration(
+        ~chainId=fetchState.chainId,
+        ~contractAddress=registration.address,
+        ~contractName=registration.contractName,
       )
       false
     }
@@ -1764,7 +1910,7 @@ let registerDynamicContracts = (
 
 /*
 Updates fetchState with a response for a given query.
-Returns Error if the partition with given query cannot be found (unexpected)
+Throws if the partition with given query cannot be found (unexpected)
 
 newItems are ordered earliest to latest (as they are returned from the worker)
 */
@@ -2038,11 +2184,27 @@ let walkPartitionPending = (
     pqIdx := pqIdx.contents + 1
   }
 
+  // Nothing in this partition's selection can match below its earliest start
+  // block, so forward work skips straight to it instead of scanning up to it and
+  // discarding whole pages. Only the cursor moves — `latestFetchedBlock` still
+  // advances solely on a response, so no block is ever reported fetched that
+  // wasn't. Bounded by the head and the query end block: past either, the
+  // partition would offer no candidate at all, so it queries as before rather
+  // than going quiet until the chain reaches its start block.
+  let cursor = switch p.selection.startBlock {
+  | Some(startBlock) if startBlock > cursor.contents =>
+    switch Utils.Math.minOptInt(Some(headBlockNumber), queryEndBlock) {
+    | Some(reachable) if startBlock <= reachable => startBlock
+    | _ => cursor.contents
+    }
+  | _ => cursor.contents
+  }
+
   canContinue.contents
     ? Some({
         partitionId,
         p,
-        cursor: cursor.contents,
+        cursor,
         chunksUsedThisCall: chunksUsedThisCall.contents,
         inFlightCount,
         queryEndBlock,
@@ -2489,7 +2651,7 @@ let make = (
   ~addressStore: AddressStore.t,
   ~addresses: array<Internal.indexingAddress>,
   ~maxAddrInPartition,
-  ~chainId,
+  ~chainId: ChainId.t,
   ~maxOnBlockBufferSize,
   ~knownHeight,
   ~progressBlockNumber=startBlock - 1,
@@ -2523,10 +2685,10 @@ let make = (
     partitions->Array.push({
       id: partitions->Array.length->Int.toString,
       latestFetchedBlock,
-      selection: {
-        dependsOnAddresses: false,
-        onEventRegistrations: notDependingOnAddresses,
-      },
+      selection: makeSelection(
+        ~dependsOnAddresses=false,
+        ~onEventRegistrations=notDependingOnAddresses,
+      ),
       addresses: addressStore->AddressStore.emptySet,
       mergeBlock: None,
       dynamicContract: None,
@@ -2538,10 +2700,10 @@ let make = (
     })
   }
 
-  let normalSelection = {
-    dependsOnAddresses: true,
-    onEventRegistrations: normalRegistrations,
-  }
+  let normalSelection = makeSelection(
+    ~dependsOnAddresses=true,
+    ~onEventRegistrations=normalRegistrations,
+  )
 
   // Every address the chain indexes goes into the store — including ones whose
   // contract has no address-dependent events, so a later registration of the
@@ -2554,7 +2716,17 @@ let make = (
       registrationBlock: contract.registrationBlock,
     }),
   )
-  ->ignore
+  // Verdicts are in the batch's order, so they line up with `addresses`. A
+  // config address the store rejects is dropped exactly like a dynamic one, and
+  // needs the same warning — restored dynamic addresses come through here too.
+  ->Array.forEachWithIndex((verdict, idx) => {
+    let contract = addresses->Array.getUnsafe(idx)
+    verdict->warnRejectedRegistration(
+      ~chainId,
+      ~contractAddress=contract.address,
+      ~contractName=contract.contractName,
+    )
+  })
 
   let dynamicContracts = Utils.Set.make()
   let clientFilteredContracts = Utils.Set.make()
@@ -2618,7 +2790,7 @@ let make = (
       onBlockRegistrations->Utils.Array.isEmpty
   ) {
     JsError.throwWithMessage(
-      `Invalid configuration: Nothing to fetch on chain ${chainId->Int.toString}. ` ++
+      `Invalid configuration: Nothing to fetch on chain ${chainId->ChainId.toString}. ` ++
       `addresses=${addresses->Array.length->Int.toString}, ` ++
       `onEventRegistrations=${onEventRegistrations->Array.length->Int.toString}, ` ++
       `normalRegistrations=${normalRegistrations