envio 3.15.0 → 3.16.0-subgraph

package/src/FetchState.res.mjs

@@ -4,6 +4,7 @@ import * as Env from "./Env.res.mjs";
 import * as Utils from "./Utils.res.mjs";
 import * as ChainId from "./ChainId.res.mjs";
 import * as Js_math from "@rescript/runtime/lib/es6/Js_math.js";
+import * as Internal from "./Internal.res.mjs";
 import * as BlockItem from "./sources/BlockItem.res.mjs";
 import * as AddressSet from "./sources/AddressSet.res.mjs";
 import * as Belt_Array from "@rescript/runtime/lib/es6/Belt_Array.js";
@@ -793,6 +794,77 @@ function compareTiebreak(a, b) {
   }
 }
 
+function blockOrderRank(item) {
+  let round = item.runsAfterBlock;
+  if (round !== undefined) {
+    return 3 + round | 0;
+  } else if (item.kind !== 0) {
+    if (Internal.itemRunsFirst(item)) {
+      return -1;
+    } else {
+      return 3;
+    }
+  } else if (item.callRank === 2) {
+    return 2;
+  } else {
+    return 0;
+  }
+}
+
+function compareWithCalls(a, b) {
+  let ra = blockOrderRank(a);
+  let rb = blockOrderRank(b);
+  if (ra !== rb) {
+    if (ra < rb) {
+      return -1;
+    } else {
+      return 1;
+    }
+  }
+  if (ra === 3 || ra === -1) {
+    let la = a.logIndex;
+    let lb = b.logIndex;
+    if (la !== lb) {
+      if (la < lb) {
+        return -1;
+      } else {
+        return 1;
+      }
+    } else {
+      return compareTiebreak(a, b);
+    }
+  }
+  let ta = a.transactionIndex;
+  let tb = b.transactionIndex;
+  if (ta !== tb) {
+    if (ta < tb) {
+      return -1;
+    } else {
+      return 1;
+    }
+  }
+  let ca = Stdlib_Option.isSome(a.callRank) ? 1 : 0;
+  let cb = Stdlib_Option.isSome(b.callRank) ? 1 : 0;
+  if (ca !== cb) {
+    if (ca < cb) {
+      return -1;
+    } else {
+      return 1;
+    }
+  }
+  let la$1 = a.logIndex;
+  let lb$1 = b.logIndex;
+  if (la$1 !== lb$1) {
+    if (la$1 < lb$1) {
+      return -1;
+    } else {
+      return 1;
+    }
+  } else {
+    return compareTiebreak(a, b);
+  }
+}
+
 function compareBufferItem(a, b) {
   let ba = a.blockNumber;
   let bb = b.blockNumber;
@@ -803,8 +875,29 @@ function compareBufferItem(a, b) {
       return 1;
     }
   }
-  let ka = a.kind;
-  let kb = b.kind;
+  if (Internal.callOrderingActive.contents) {
+    return compareWithCalls(a, b);
+  }
+  let round = a.runsAfterBlock;
+  let ka;
+  if (round !== undefined) {
+    ka = 1 + round | 0;
+  } else {
+    let kind = a.kind;
+    ka = kind !== 0 ? (
+        Internal.itemRunsFirst(a) ? -1 : kind
+      ) : 0;
+  }
+  let round$1 = b.runsAfterBlock;
+  let kb;
+  if (round$1 !== undefined) {
+    kb = 1 + round$1 | 0;
+  } else {
+    let kind$1 = b.kind;
+    kb = kind$1 !== 0 ? (
+        Internal.itemRunsFirst(b) ? -1 : kind$1
+      ) : 0;
+  }
   if (ka !== kb) {
     if (ka < kb) {
       return -1;
@@ -826,7 +919,7 @@ function compareBufferItem(a, b) {
 }
 
 function isSameLog(a, b) {
-  if (a.kind !== 0 || b.kind !== 0 || a.blockNumber !== b.blockNumber || a.logIndex !== b.logIndex) {
+  if (a.kind !== 0 || b.kind !== 0 || a.callRank !== b.callRank || a.blockNumber !== b.blockNumber || a.logIndex !== b.logIndex) {
     return false;
   }
   let match = a.orderPath;
@@ -929,7 +1022,9 @@ function updateInternal(fetchState, optimizedPartitionsOpt, mutItems, mutItemsSo
       tmp = latestFullyFetchedBlock !== undefined ? latestFullyFetchedBlock : knownHeight;
     }
     let maxBlockNumber = Primitive_int.min(tmp, knownHeight);
-    latestOnBlockBlockNumber = appendOnBlockItems(blockItems, onBlockRegistrations, fetchState.startBlock, fetchState.latestOnBlockBlockNumber, maxBlockNumber, fetchState.maxOnBlockBufferSize);
+    let through = fetchState.blockRegisterThrough;
+    let maxBlockNumber$1 = through !== undefined ? Primitive_int.min(maxBlockNumber, through) : maxBlockNumber;
+    latestOnBlockBlockNumber = appendOnBlockItems(blockItems, onBlockRegistrations, fetchState.startBlock, fetchState.latestOnBlockBlockNumber, maxBlockNumber$1, fetchState.maxOnBlockBufferSize);
   } else {
     latestOnBlockBlockNumber = knownHeight;
   }
@@ -941,26 +1036,49 @@ function updateInternal(fetchState, optimizedPartitionsOpt, mutItems, mutItemsSo
     chainId: fetchState.chainId,
     latestOnBlockBlockNumber: latestOnBlockBlockNumber,
     blockLag: blockLag,
+    blockThrottle: fetchState.blockThrottle,
     buffer: blockItems.length !== 0 ? mergeIntoBuffer(base, blockItems) : base,
     maxOnBlockBufferSize: fetchState.maxOnBlockBufferSize,
     onBlockRegistrations: fetchState.onBlockRegistrations,
     knownHeight: knownHeight,
     firstEventBlock: fetchState.firstEventBlock,
-    clientFilterAddressThreshold: fetchState.clientFilterAddressThreshold
+    clientFilterPolicy: fetchState.clientFilterPolicy,
+    creationBlocks: fetchState.creationBlocks,
+    blockRegisterThrough: fetchState.blockRegisterThrough
   };
 }
 
-function addClientFilteredContract(clientFilteredContracts, contractName, onNotice, addressCount, threshold, isAlreadyReportedOpt) {
+function switchToClientSideFiltering(clientFilteredContracts, contractNames, policy, dynamicContracts, addressStore, onNotice, isAlreadyReportedOpt) {
   let isAlreadyReported = isAlreadyReportedOpt !== undefined ? isAlreadyReportedOpt : false;
-  clientFilteredContracts.add(contractName);
-  if (!isAlreadyReported) {
-    return onNotice({
-      TAG: "SwitchedToClientSideFiltering",
-      contractName: contractName,
-      addressCount: addressCount,
-      threshold: threshold
-    });
-  }
+  let switched = new Set(Array.from(clientFilteredContracts));
+  contractNames.forEach(contractName => {
+    if (switched.has(contractName)) {
+      return;
+    }
+    if (policy.allDynamicContracts && dynamicContracts.has(contractName)) {
+      switched.add(contractName);
+      return;
+    }
+    let threshold = policy.addressThreshold;
+    if (threshold === undefined) {
+      return;
+    }
+    let addressCount = addressStore.contractCount(contractName);
+    if (addressCount > threshold) {
+      switched.add(contractName);
+      if (!isAlreadyReported) {
+        return onNotice({
+          TAG: "SwitchedToClientSideFiltering",
+          contractName: contractName,
+          addressCount: addressCount,
+          threshold: threshold
+        });
+      } else {
+        return;
+      }
+    }
+  });
+  return switched;
 }
 
 function claimedFetchedBlock(p, knownHeight) {
@@ -1286,6 +1404,43 @@ function createPartitions(registeringSetsByContract, addressStore, dynamicContra
   return make(allPartitions, maxAddrInPartition, nextPartitionIndexRef.contents, dynamicContracts, clientFilteredContracts);
 }
 
+function creationBlockKey(contractName, address) {
+  return contractName + `:` + address.toLowerCase();
+}
+
+function markCreationBlockEvents(creationBlocks, newItems, fetchedThrough) {
+  Object.entries(creationBlocks).forEach(param => {
+    if (param[1].block < fetchedThrough) {
+      return Utils.Dict.deleteInPlace(creationBlocks, param[0]);
+    }
+  });
+  newItems.forEach(item => {
+    if (item.kind !== 0) {
+      return;
+    }
+    let key = creationBlockKey(item.onEventRegistration.eventConfig.contractName, item.payload.srcAddress);
+    let match = creationBlocks[key];
+    if (match !== undefined && match.block === item.blockNumber) {
+      item.runsAfterBlock = match.round;
+      return;
+    }
+  });
+}
+
+function creationRoundOfRegistration(fetchState, item) {
+  let creationBlocks = fetchState.creationBlocks;
+  if (creationBlocks === undefined) {
+    return 1;
+  }
+  let key = creationBlockKey(item.onEventRegistration.eventConfig.contractName, item.payload.srcAddress);
+  let match = creationBlocks[key];
+  if (match !== undefined && match.block === item.blockNumber) {
+    return match.round + 1 | 0;
+  } else {
+    return 1;
+  }
+}
+
 function registerDynamicContracts(fetchState, addressStore, onNotice, claimCeilingOpt, registrations) {
   let claimCeiling = claimCeilingOpt !== undefined ? claimCeilingOpt : fetchState.knownHeight;
   if (Utils.$$Array.isEmpty(fetchState.normalSelection.onEventRegistrations)) {
@@ -1300,8 +1455,17 @@ function registerDynamicContracts(fetchState, addressStore, onNotice, claimCeili
     let exit = 0;
     if (typeof verdict !== "object" || verdict.TAG !== "Added") {
       exit = 1;
-    } else if (verdict.fetchable && !registeringContractNames.includes(registration.contractName)) {
-      registeringContractNames.push(registration.contractName);
+    } else if (verdict.fetchable) {
+      let creationBlocks = fetchState.creationBlocks;
+      if (creationBlocks !== undefined) {
+        creationBlocks[creationBlockKey(registration.contractName, registration.address)] = {
+          block: registration.registrationBlock,
+          round: Stdlib_Option.getOr(registration.creationRound, 1)
+        };
+      }
+      if (!registeringContractNames.includes(registration.contractName)) {
+        registeringContractNames.push(registration.contractName);
+      }
     }
     if (exit === 1) {
       reportRejectedRegistration(verdict, onNotice, registration.address, registration.contractName);
@@ -1362,20 +1526,7 @@ function registerDynamicContracts(fetchState, addressStore, onNotice, claimCeili
       }
     }
   }
-  let threshold = fetchState.clientFilterAddressThreshold;
-  let clientFilteredContracts;
-  if (threshold !== undefined) {
-    let clientFilteredContracts$1 = new Set(Array.from(fetchState.optimizedPartitions.clientFilteredContracts));
-    Array.from(dynamicContractsRef).forEach(contractName => {
-      let addressCount = addressStore.contractCount(contractName);
-      if (!clientFilteredContracts$1.has(contractName) && addressCount > threshold) {
-        return addClientFilteredContract(clientFilteredContracts$1, contractName, onNotice, addressCount, threshold, undefined);
-      }
-    });
-    clientFilteredContracts = clientFilteredContracts$1;
-  } else {
-    clientFilteredContracts = fetchState.optimizedPartitions.clientFilteredContracts;
-  }
+  let clientFilteredContracts = switchToClientSideFiltering(fetchState.optimizedPartitions.clientFilteredContracts, Array.from(dynamicContractsRef), fetchState.clientFilterPolicy, dynamicContractsRef, addressStore, onNotice, undefined);
   let registeringSetsByContract = {};
   registeringContractNames.forEach(contractName => {
     registeringSetsByContract[contractName] = AddressStore.makeSet(addressStore, contractName, {
@@ -1386,8 +1537,100 @@ function registerDynamicContracts(fetchState, addressStore, onNotice, claimCeili
   return updateInternal(fetchState, optimizedPartitions, undefined, undefined, undefined, undefined);
 }
 
+function switchDynamicContractsToClientSideFiltering(fetchState, addressStore) {
+  if (fetchState.clientFilterPolicy.allDynamicContracts) {
+    return fetchState;
+  }
+  let newrecord = {...fetchState};
+  let init = fetchState.clientFilterPolicy;
+  newrecord.clientFilterPolicy = {
+    addressThreshold: init.addressThreshold,
+    allDynamicContracts: true
+  };
+  let match = newrecord.optimizedPartitions;
+  let clientFilteredContracts = match.clientFilteredContracts;
+  let dynamicContracts = match.dynamicContracts;
+  let switched = new Set(Array.from(clientFilteredContracts).concat(Array.from(dynamicContracts)));
+  if (switched.size === clientFilteredContracts.size) {
+    return newrecord;
+  } else {
+    return updateInternal(newrecord, createPartitions({}, addressStore, dynamicContracts, switched, newrecord.normalSelection, match.maxAddrInPartition, match.nextPartitionIndex, Object.values(newrecord.optimizedPartitions.entities), 0, newrecord.knownHeight), undefined, undefined, undefined, undefined);
+  }
+}
+
 function handleQueryResult(fetchState, query, latestFetchedBlock, newItems) {
-  return updateInternal(fetchState, handleQueryResponse(fetchState.optimizedPartitions, query, fetchState.knownHeight, newItems.length, latestFetchedBlock), newItems.length !== 0 ? mergeIntoBuffer(fetchState.buffer, newItems) : undefined, true, undefined, undefined);
+  let tmp;
+  if (newItems.length !== 0) {
+    let creationBlocks = fetchState.creationBlocks;
+    if (creationBlocks !== undefined) {
+      markCreationBlockEvents(creationBlocks, newItems, bufferBlockNumber(fetchState));
+    }
+    tmp = mergeIntoBuffer(fetchState.buffer, newItems);
+  } else {
+    tmp = undefined;
+  }
+  return updateInternal(fetchState, handleQueryResponse(fetchState.optimizedPartitions, query, fetchState.knownHeight, newItems.length, latestFetchedBlock), tmp, true, undefined, undefined);
+}
+
+function capAtEndBlock(fetchState, block) {
+  let endBlock = fetchState.endBlock;
+  if (endBlock !== undefined) {
+    return Primitive_int.min(block, endBlock);
+  } else {
+    return block;
+  }
+}
+
+function canAdvanceBlockRegisters(fetchState, through) {
+  let after = fetchState.blockRegisterThrough;
+  if (after !== undefined) {
+    return capAtEndBlock(fetchState, through) > after;
+  } else {
+    return false;
+  }
+}
+
+function blockRegisterItemsAfter(fetchState, after, through, limit) {
+  let match = fetchState.blockRegisterThrough;
+  if (match === undefined) {
+    return;
+  }
+  if (capAtEndBlock(fetchState, through) <= after) {
+    return;
+  }
+  let through$1 = capAtEndBlock(fetchState, through);
+  let items = [];
+  let covered = appendOnBlockItems(items, fetchState.onBlockRegistrations.filter(r => r.contractRegister !== undefined), fetchState.startBlock, after, through$1, limit);
+  return [
+    items,
+    covered
+  ];
+}
+
+function nextBlockRegisterItems(fetchState, through, limit) {
+  let after = fetchState.blockRegisterThrough;
+  if (after !== undefined) {
+    return blockRegisterItemsAfter(fetchState, after, through, limit);
+  }
+}
+
+function setBlockRegisterThrough(fetchState, through) {
+  let current = fetchState.blockRegisterThrough;
+  if (current === undefined) {
+    return fetchState;
+  }
+  if (through <= current) {
+    return fetchState;
+  }
+  let newrecord = {...fetchState};
+  return updateInternal((newrecord.blockRegisterThrough = through, newrecord), undefined, undefined, undefined, undefined, undefined);
+}
+
+function blockRegisterFrontier(fetchState) {
+  let optimizedPartitions = fetchState.optimizedPartitions;
+  let id = optimizedPartitions.idsInAscOrder[0];
+  let block = id !== undefined ? optimizedPartitions.entities[id].latestFetchedBlock : undefined;
+  return capAtEndBlock(fetchState, block !== undefined ? Primitive_int.min(block, fetchState.knownHeight) : fetchState.knownHeight);
 }
 
 function startFetchingQueries(param, queries) {
@@ -1624,15 +1867,30 @@ function acceptCandidates(candidates, reservations, chainTargetItems, partitionI
   };
 }
 
-function getNextQuery(param, chainTargetBlock, chainTargetItems) {
-  let blockLag = param.blockLag;
+function blocksBehindHeadAllowedByThrottle(param) {
+  let blockThrottle = param.blockThrottle;
   let endBlock = param.endBlock;
-  let optimizedPartitions = param.optimizedPartitions;
-  let headBlockNumber = param.knownHeight - blockLag | 0;
+  if (endBlock !== undefined && (param.knownHeight - param.blockLag | 0) >= endBlock) {
+    return 0;
+  } else {
+    return Primitive_int.max(0, blockThrottle - 1 | 0);
+  }
+}
+
+function headFetchTarget(fetchState) {
+  return (fetchState.knownHeight - fetchState.blockLag | 0) - blocksBehindHeadAllowedByThrottle(fetchState) | 0;
+}
+
+function getNextQuery(fetchState, chainTargetBlock, chainTargetItems) {
+  let blockLag = fetchState.blockLag;
+  let endBlock = fetchState.endBlock;
+  let optimizedPartitions = fetchState.optimizedPartitions;
+  let headBlockNumber = fetchState.knownHeight - blockLag | 0;
   if (headBlockNumber <= 0) {
     return "WaitingForNewBlock";
   }
-  let isOnBlockBehindTheHead = param.latestOnBlockBlockNumber < headBlockNumber;
+  let caughtUpBlockNumber = headBlockNumber - blocksBehindHeadAllowedByThrottle(fetchState) | 0;
+  let isOnBlockBehindTheHead = fetchState.latestOnBlockBlockNumber < caughtUpBlockNumber;
   let shouldWaitForNewBlock = (
     endBlock !== undefined ? headBlockNumber < endBlock : true
   ) && !isOnBlockBehindTheHead;
@@ -1658,7 +1916,7 @@ function getNextQuery(param, chainTargetBlock, chainTargetItems) {
       }
     }
     inFlightCounts[idx] = inFlightCount;
-    if (p.mutPendingQueries.length !== 0 || p.latestFetchedBlock < headBlockNumber) {
+    if (p.mutPendingQueries.length !== 0 || p.latestFetchedBlock < caughtUpBlockNumber) {
       shouldWaitForNewBlock = false;
     }
   }
@@ -1749,13 +2007,15 @@ function getReadyItemsCount(fetchState, targetSize, fromItem) {
   return acc;
 }
 
-function make$1(startBlock, endBlock, onEventRegistrations, addressStore, addressRows, maxAddrInPartition, chainId, onNotice, maxOnBlockBufferSize, knownHeight, progressBlockNumberOpt, onBlockRegistrationsOpt, blockLagOpt, firstEventBlockOpt, clientFilterAddressThresholdOpt, isResumedOpt) {
+function make$1(startBlock, endBlock, onEventRegistrations, addressStore, addressRows, maxAddrInPartition, chainId, onNotice, maxOnBlockBufferSize, knownHeight, progressBlockNumberOpt, onBlockRegistrationsOpt, blockLagOpt, blockThrottleOpt, firstEventBlockOpt, clientFilterAddressThresholdOpt, isResumedOpt, deferCreationBlockEventsOpt) {
   let progressBlockNumber = progressBlockNumberOpt !== undefined ? progressBlockNumberOpt : startBlock - 1 | 0;
   let onBlockRegistrations = onBlockRegistrationsOpt !== undefined ? onBlockRegistrationsOpt : [];
   let blockLag = blockLagOpt !== undefined ? blockLagOpt : 0;
+  let blockThrottle = blockThrottleOpt !== undefined ? blockThrottleOpt : 0;
   let firstEventBlock = firstEventBlockOpt !== undefined ? Primitive_option.valFromOption(firstEventBlockOpt) : undefined;
   let clientFilterAddressThreshold = clientFilterAddressThresholdOpt !== undefined ? Primitive_option.valFromOption(clientFilterAddressThresholdOpt) : undefined;
   let isResumed = isResumedOpt !== undefined ? isResumedOpt : false;
+  let deferCreationBlockEvents = deferCreationBlockEventsOpt !== undefined ? deferCreationBlockEventsOpt : false;
   let notDependingOnAddresses = [];
   let normalRegistrations = [];
   let contractNamesWithNormalEvents = new Set();
@@ -1801,7 +2061,6 @@ function make$1(startBlock, endBlock, onEventRegistrations, addressStore, addres
     requestedFromBlock: rejected.effectiveStartBlock
   }));
   let dynamicContracts = new Set();
-  let clientFilteredContracts = new Set();
   let registeringSetsByContract = {};
   Array.from(contractNamesWithNormalEvents).forEach(contractName => {
     if (addressStore.contractCount(contractName) > 0) {
@@ -1815,19 +2074,17 @@ function make$1(startBlock, endBlock, onEventRegistrations, addressStore, addres
       return;
     }
   });
-  if (clientFilterAddressThreshold !== undefined) {
-    Utils.Dict.forEachWithKey(registeringSetsByContract, (set, contractName) => {
-      let addressCount = set.size();
-      if (addressCount > clientFilterAddressThreshold) {
-        return addClientFilteredContract(clientFilteredContracts, contractName, onNotice, addressCount, clientFilterAddressThreshold, isResumed);
-      }
-    });
-  }
+  let clientFilterPolicy = {
+    addressThreshold: clientFilterAddressThreshold,
+    allDynamicContracts: false
+  };
+  let clientFilteredContracts = switchToClientSideFiltering(new Set(), Object.keys(registeringSetsByContract), clientFilterPolicy, dynamicContracts, addressStore, onNotice, isResumed);
   let optimizedPartitions = createPartitions(registeringSetsByContract, addressStore, dynamicContracts, clientFilteredContracts, normalSelection, maxAddrInPartition, partitions.length, partitions, progressBlockNumber, knownHeight);
   if (optimizedPartitions.idsInAscOrder.length === 0 && Utils.$$Array.isEmpty(onBlockRegistrations$1)) {
     Stdlib_JsError.throwWithMessage(`Invalid configuration: Nothing to fetch on chain ` + ChainId.toString(chainId) + `. ` + (`addresses=` + addressRows.addresses.length.toString() + `, `) + (`onEventRegistrations=` + onEventRegistrations.length.toString() + `, `) + (`normalRegistrations=` + normalRegistrations.length.toString() + `. `) + `Make sure that you provided at least one contract address to index, or have events with Wildcard mode enabled, or have onBlock handlers.`);
   }
   let buffer = [];
+  let blockRegisterThrough = onBlockRegistrations$1.some(r => r.contractRegister !== undefined) ? progressBlockNumber : undefined;
   let latestOnBlockBlockNumber;
   let exit = 0;
   if (onBlockRegistrations$1.length !== 0 || knownHeight <= 0) {
@@ -1840,7 +2097,8 @@ function make$1(startBlock, endBlock, onEventRegistrations, addressStore, addres
       let id = optimizedPartitions.idsInAscOrder[0];
       let latestFullyFetchedBlock = id !== undefined ? optimizedPartitions.entities[id].latestFetchedBlock : undefined;
       let maxBlockNumber = latestFullyFetchedBlock !== undefined ? Primitive_int.min(latestFullyFetchedBlock, knownHeight) : knownHeight;
-      latestOnBlockBlockNumber = appendOnBlockItems(buffer, onBlockRegistrations$1, startBlock, progressBlockNumber, maxBlockNumber, maxOnBlockBufferSize);
+      let maxBlockNumber$1 = blockRegisterThrough !== undefined ? Primitive_int.min(maxBlockNumber, blockRegisterThrough) : maxBlockNumber;
+      latestOnBlockBlockNumber = appendOnBlockItems(buffer, onBlockRegistrations$1, startBlock, progressBlockNumber, maxBlockNumber$1, maxOnBlockBufferSize);
     } else {
       latestOnBlockBlockNumber = progressBlockNumber;
     }
@@ -1853,12 +2111,15 @@ function make$1(startBlock, endBlock, onEventRegistrations, addressStore, addres
     chainId: chainId,
     latestOnBlockBlockNumber: latestOnBlockBlockNumber,
     blockLag: blockLag,
+    blockThrottle: blockThrottle,
     buffer: buffer,
     maxOnBlockBufferSize: maxOnBlockBufferSize,
     onBlockRegistrations: onBlockRegistrations$1,
     knownHeight: knownHeight,
     firstEventBlock: firstEventBlock,
-    clientFilterAddressThreshold: clientFilterAddressThreshold
+    clientFilterPolicy: clientFilterPolicy,
+    creationBlocks: deferCreationBlockEvents ? ({}) : undefined,
+    blockRegisterThrough: blockRegisterThrough
   };
 }
 
@@ -1958,21 +2219,8 @@ function rollback(fetchState, rolledBackAddressStore, targetBlockNumber) {
     }
   }
   let optimizedPartitions = createPartitions(registeringSetsByContract, rolledBackAddressStore, fetchState.optimizedPartitions.dynamicContracts, fetchState.optimizedPartitions.clientFilteredContracts, fetchState.normalSelection, fetchState.optimizedPartitions.maxAddrInPartition, nextKeptIdRef, keptPartitions, targetBlockNumber, fetchState.knownHeight);
-  let rolledBack = updateInternal({
-    optimizedPartitions: fetchState.optimizedPartitions,
-    startBlock: fetchState.startBlock,
-    endBlock: fetchState.endBlock,
-    normalSelection: fetchState.normalSelection,
-    chainId: fetchState.chainId,
-    latestOnBlockBlockNumber: Primitive_int.min(fetchState.latestOnBlockBlockNumber, targetBlockNumber),
-    blockLag: fetchState.blockLag,
-    buffer: fetchState.buffer,
-    maxOnBlockBufferSize: fetchState.maxOnBlockBufferSize,
-    onBlockRegistrations: fetchState.onBlockRegistrations,
-    knownHeight: fetchState.knownHeight,
-    firstEventBlock: fetchState.firstEventBlock,
-    clientFilterAddressThreshold: fetchState.clientFilterAddressThreshold
-  }, optimizedPartitions, fetchState.buffer.filter(item => {
+  let newrecord = {...fetchState};
+  let rolledBack = updateInternal((newrecord.blockRegisterThrough = Stdlib_Option.map(fetchState.blockRegisterThrough, through => Primitive_int.min(through, targetBlockNumber)), newrecord.latestOnBlockBlockNumber = Primitive_int.min(fetchState.latestOnBlockBlockNumber, targetBlockNumber), newrecord), optimizedPartitions, fetchState.buffer.filter(item => {
     let tmp;
     tmp = item.kind === 0 ? item.blockNumber : item.blockNumber;
     return tmp <= targetBlockNumber;
@@ -2005,29 +2253,17 @@ function resetPendingQueries(fetchState) {
       };
     }
   }
+  let newrecord = {...fetchState};
   let init = fetchState.optimizedPartitions;
-  return {
-    optimizedPartitions: {
-      idsInAscOrder: init.idsInAscOrder,
-      entities: newEntities,
-      maxAddrInPartition: init.maxAddrInPartition,
-      nextPartitionIndex: init.nextPartitionIndex,
-      dynamicContracts: init.dynamicContracts,
-      clientFilteredContracts: init.clientFilteredContracts
-    },
-    startBlock: fetchState.startBlock,
-    endBlock: fetchState.endBlock,
-    normalSelection: fetchState.normalSelection,
-    chainId: fetchState.chainId,
-    latestOnBlockBlockNumber: fetchState.latestOnBlockBlockNumber,
-    blockLag: fetchState.blockLag,
-    buffer: fetchState.buffer,
-    maxOnBlockBufferSize: fetchState.maxOnBlockBufferSize,
-    onBlockRegistrations: fetchState.onBlockRegistrations,
-    knownHeight: fetchState.knownHeight,
-    firstEventBlock: fetchState.firstEventBlock,
-    clientFilterAddressThreshold: fetchState.clientFilterAddressThreshold
+  newrecord.optimizedPartitions = {
+    idsInAscOrder: init.idsInAscOrder,
+    entities: newEntities,
+    maxAddrInPartition: init.maxAddrInPartition,
+    nextPartitionIndex: init.nextPartitionIndex,
+    dynamicContracts: init.dynamicContracts,
+    clientFilteredContracts: init.clientFilteredContracts
   };
+  return newrecord;
 }
 
 function isActivelyIndexing(fetchState) {
@@ -2044,21 +2280,37 @@ function isActivelyIndexing(fetchState) {
 }
 
 function isFetchingAtHead(fetchState) {
-  let knownHeight = fetchState.knownHeight;
-  let blockLag = fetchState.blockLag;
   let endBlock = fetchState.endBlock;
   let bufferBlockNumber$1 = bufferBlockNumber(fetchState);
-  if (knownHeight !== 0) {
+  if (fetchState.knownHeight !== 0) {
     if (endBlock !== undefined && bufferBlockNumber$1 >= endBlock) {
       return true;
     } else {
-      return bufferBlockNumber$1 >= (knownHeight - blockLag | 0);
+      return bufferBlockNumber$1 >= headFetchTarget(fetchState);
     }
   } else {
     return false;
   }
 }
 
+function isWithinHeadClientFilteringDistance(fetchState) {
+  let knownHeight = fetchState.knownHeight;
+  let endBlock = fetchState.endBlock;
+  let optimizedPartitions = fetchState.optimizedPartitions;
+  if (!(knownHeight !== 0 && (
+      endBlock !== undefined ? endBlock > (knownHeight - fetchState.blockLag | 0) : true
+    ))) {
+    return false;
+  }
+  let id = optimizedPartitions.idsInAscOrder[0];
+  let latestFetchedBlock = id !== undefined ? optimizedPartitions.entities[id].latestFetchedBlock : undefined;
+  if (latestFetchedBlock !== undefined) {
+    return (headFetchTarget(fetchState) - latestFetchedBlock | 0) <= 200;
+  } else {
+    return true;
+  }
+}
+
 function isReadyToEnterReorgThreshold(tolerance, fetchState) {
   let knownHeight = fetchState.knownHeight;
   let endBlock = fetchState.endBlock;
@@ -2145,6 +2397,8 @@ let maxChainConcurrency = Env.maxChainConcurrency;
 
 let chunkRangeGrowthFactor = 1.8;
 
+let headClientFilteringDistance = 200;
+
 export {
   deriveSelectionStartBlock,
   makeSelection,
@@ -2163,18 +2417,30 @@ export {
   getRegistrationIndex,
   comparePath,
   compareTiebreak,
+  blockOrderRank,
+  compareWithCalls,
   compareBufferItem,
   isSameLog,
   mergeIntoBuffer,
   appendOnBlockItems,
   updateInternal,
-  addClientFilteredContract,
+  switchToClientSideFiltering,
   claimedFetchedBlock,
   collapseClientFilteredContracts,
   reportRejectedRegistration,
   createPartitions,
+  creationBlockKey,
+  markCreationBlockEvents,
+  creationRoundOfRegistration,
   registerDynamicContracts,
+  switchDynamicContractsToClientSideFiltering,
   handleQueryResult,
+  capAtEndBlock,
+  canAdvanceBlockRegisters,
+  blockRegisterItemsAfter,
+  nextBlockRegisterItems,
+  setBlockRegisterThrough,
+  blockRegisterFrontier,
   startFetchingQueries,
   maxInFlightChunksPerPartition,
   maxChainConcurrency,
@@ -2184,6 +2450,8 @@ export {
   walkPartitionPending,
   pushForwardCandidates,
   acceptCandidates,
+  blocksBehindHeadAllowedByThrottle,
+  headFetchTarget,
   getNextQuery,
   hasReadyItem,
   getReadyItemsCount,
@@ -2195,6 +2463,8 @@ export {
   resetPendingQueries,
   isActivelyIndexing,
   isFetchingAtHead,
+  headClientFilteringDistance,
+  isWithinHeadClientFilteringDistance,
   isReadyToEnterReorgThreshold,
   getProgressPercentage,
   sortForBatch,

package/src/FinalizeBackfill.res

@@ -40,7 +40,7 @@ let runOnce = async (state: IndexerState.t) => {
     let storage = persistence->Persistence.getInitializedStorageOrThrow
     let readyAt = Date.make()
 
-    await storage.finalizeBackfill(~entities=persistence.allEntities, ~chainIds, ~readyAt)
+    await storage.finalizeBackfill(~chainIds, ~readyAt)
 
     // Only after the commit: in-memory readiness must never run ahead of the
     // `ready_at` a restart would read back.
@@ -68,10 +68,8 @@ let run = (state: IndexerState.t) =>
 // Best-effort and not awaited by the loop: indexing is already live and correct
 // without the index, just slower.
 let repairSchemaIndexes = (state: IndexerState.t) => {
-  let persistence = state->IndexerState.persistence
-  let storage = persistence->Persistence.getInitializedStorageOrThrow
+  let storage = state->IndexerState.persistence->Persistence.getInitializedStorageOrThrow
   storage.ensureSchemaIndexes(
-    ~entities=persistence.allEntities,
     ~chainIds=state->IndexerState.crossChainState->CrossChainState.chainIds,
   )
 }