envio 3.15.0 → 3.16.0-subgraph

package/src/ChainState.res

@@ -26,6 +26,9 @@ type t = {
   // scheduler doesn't re-sum pending queries on every tick. Incremented when
   // queries are dispatched, decremented as their responses land.
   mutable pendingBudget: float,
+  // Counts resets of pendingBudget: a hold taken before one has nothing left
+  // to release.
+  mutable pendingBudgetResets: int,
   // Chain-wide events/block, used to turn a chain's item budget into a target
   // block for query sizing. Seeded from cumulative progress on construction,
   // then smoothed with an EMA on every batch (see applyBatchProgress). None
@@ -73,6 +76,12 @@ type t = {
   // Timestamp of the committed progress block itself. See
   // `Batch.chainAfterBatch.progressBlockTime`.
   mutable committedProgressBlockTime: option<int>,
+  // Subgraph mode, with a block handler that has a register pass: the chain's
+  // register work runs one unit at a time, a response's passes or a run of
+  // block passes alone, so a pass never runs ahead of a block pass of an
+  // earlier block. Settles when the latest unit is applied.
+  mutable blockRegisterTail: promise<unit>,
+  mutable blockRegisterRuns: int,
 }
 
 // The chain's config-declared addresses as storage rows: what a clean run
@@ -167,6 +176,7 @@ let make = (
     processingBlockNumber: committedProgressBlockNumber,
     numEventsProcessed,
     pendingBudget: 0.,
+    pendingBudgetResets: 0,
     chainDensity,
     safeCheckpointTracking,
     shouldRollbackOnReorg,
@@ -187,6 +197,8 @@ let make = (
     rollbackTargetBlock: None,
     progressLatencyMs: None,
     committedProgressBlockTime,
+    blockRegisterTail: Promise.resolve(),
+    blockRegisterRuns: 0,
   }
 }
 
@@ -308,13 +320,19 @@ let makeInternal = (
     // FIXME: Shouldn't set with full history
     // The resumed depth, not the config one: the pre-threshold lag must match
     // the window reorg detection compares, or a reduced config depth would let
-    // fetching enter the stored rollback window without history.
+    // fetching enter the stored rollback window without history. One block
+    // more than the window, so the last pre-threshold page ends below it and
+    // the window's first block is read by a page that knows it is reorgable.
     ~blockLag=Pervasives.max(
-      !config.shouldRollbackOnReorg || isInReorgThreshold ? 0 : maxReorgDepth,
+      !config.shouldRollbackOnReorg || isInReorgThreshold || maxReorgDepth == 0
+        ? 0
+        : maxReorgDepth + 1,
       chainConfig.blockLag,
     ),
+    ~blockThrottle=chainConfig.blockThrottle,
     ~onBlockRegistrations,
     ~firstEventBlock,
+    ~deferCreationBlockEvents=config.subgraph->Option.isSome,
     // Fuel and SVM route through the address store like EVM does, so the
     // client-side path works for them - but their address counts are nowhere
     // near the threshold, so switching would only trade a server-side filter
@@ -349,6 +367,7 @@ let makeInternal = (
     ~blockStore,
     ~transactionStore,
     ~logger,
+    ~maxAddressesPerRequest=config.maxAddrInPartition,
   )
 
   // Seed the stored reorg checkpoints (hash-only rows) so detection resumes
@@ -459,6 +478,7 @@ let logger = (cs: t) => cs.logger
 let blockStore = (cs: t) => cs.blockStore
 let transactionStore = (cs: t) => cs.transactionStore
 let shouldChecksum = (cs: t) => cs.shouldChecksum
+let fetchState = (cs: t) => cs.fetchState
 let entities = (cs: t) => cs.entities
 
 // Rollback discards every uncommitted change, so this chain's partition is
@@ -545,8 +565,70 @@ let startFetchingQueries = (cs: t, ~queries: array<FetchState.query>) => {
 let resetPendingQueries = (cs: t) => {
   cs.fetchState = cs.fetchState->FetchState.resetPendingQueries
   cs.pendingBudget = 0.
+  cs.pendingBudgetResets = cs.pendingBudgetResets + 1
 }
 
+// A response that landed with more items than its query was priced at keeps the
+// excess reserved while it waits for its register passes: until then its items
+// are neither in the buffer nor in the query's estimate, and the pool would
+// admit the next queries on top of them. Released when the response is applied.
+type heldBudget = {items: int, resets: int}
+
+let holdExtraBudget = (cs: t, ~items) => {
+  cs.pendingBudget = cs.pendingBudget +. items->Int.toFloat
+  {items, resets: cs.pendingBudgetResets}
+}
+
+let releaseExtraBudget = (cs: t, held: heldBudget) =>
+  if held.resets === cs.pendingBudgetResets {
+    cs.pendingBudget = Pervasives.max(0., cs.pendingBudget -. held.items->Int.toFloat)
+  }
+
+let hasBlockRegisters = (cs: t) => cs.fetchState.blockRegisterThrough !== None
+let isBlockRegisterIdle = (cs: t) => cs.blockRegisterRuns === 0
+
+// Queues a unit of register work behind the chain's latest one. Returns the
+// promise to wait on before running it, and the callback that settles it,
+// which must be called however the unit ends.
+let queueBlockRegisterRun = (cs: t) => {
+  let previous = cs.blockRegisterTail
+  let settle = ref(() => ())
+  cs.blockRegisterTail = Promise.make((resolve, _) => settle := resolve)
+  cs.blockRegisterRuns = cs.blockRegisterRuns + 1
+  let finish = () => {
+    cs.blockRegisterRuns = cs.blockRegisterRuns - 1
+    settle.contents()
+  }
+  (previous, finish)
+}
+
+let canAdvanceBlockRegisters = (cs: t, ~through) =>
+  cs.fetchState->FetchState.canAdvanceBlockRegisters(~through)
+let nextBlockRegisterItems = (cs: t, ~through, ~limit) =>
+  cs.fetchState->FetchState.nextBlockRegisterItems(~through, ~limit)
+let blockRegisterItemsAfter = (cs: t, ~after, ~through, ~limit) =>
+  cs.fetchState->FetchState.blockRegisterItemsAfter(~after, ~through, ~limit)
+let blockRegisterThrough = (cs: t) => cs.fetchState.blockRegisterThrough
+let blockRegisterFrontier = (cs: t) => cs.fetchState->FetchState.blockRegisterFrontier
+
+// Applies what a run of block passes alone registered, then lets block items
+// through its last block be generated.
+let applyBlockRegisters = (cs: t, ~registrations, ~through) => {
+  let fs = switch registrations {
+  | [] => cs.fetchState
+  | _ =>
+    cs.fetchState->FetchState.registerDynamicContracts(
+      ~addressStore=cs.addressStore,
+      ~onNotice=notice => cs.logger->logFetchStateNotice(notice),
+      registrations,
+    )
+  }
+  cs.fetchState = fs->FetchState.setBlockRegisterThrough(~through)
+}
+
+let markBlockRegistered = (cs: t, ~through) =>
+  cs.fetchState = cs.fetchState->FetchState.setBlockRegisterThrough(~through)
+
 let isReady = (cs: t) => cs.timestampCaughtUpToHeadOrEndblock !== None
 
 // Block span over which a batch fully replaces the chain density estimate;
@@ -776,10 +858,10 @@ let takeFinished = (cs: t) =>
     }
   }
 
-let getHighestBlockBelowThreshold = (cs: t): int => {
-  let highestBlockBelowThreshold = cs.fetchState.knownHeight - cs.maxReorgDepth
-  highestBlockBelowThreshold < 0 ? 0 : highestBlockBelowThreshold
-}
+// The block reorg detection compares from is itself in the window, so the
+// highest block below it is the one before, where catch-up stops.
+let getHighestBlockBelowThreshold = (cs: t): int =>
+  Pervasives.max(cs.fetchState.knownHeight - cs.maxReorgDepth - 1, 0)
 
 // True once the fetch frontier has reached the head/endBlock for this chain.
 let isFetchingAtHead = (cs: t) => cs.fetchState->FetchState.isFetchingAtHead
@@ -977,6 +1059,34 @@ let materializePageItems = async (
   ))
 }
 
+// Near the head: each new block in as few queries as possible, of a shape that
+// stays the same while contracts keep being registered. Once every partition is
+// close enough, the chain's dynamic contracts leave the server-side address
+// filter for the rest of the run.
+//
+// Judged only when the chain learns a newer head. An RPC response repeats the
+// height its query went out with, which can be hours old by the end of a
+// backfill: judged against it, every block produced since would be fetched
+// address-free.
+let switchDynamicContractsToClientSideFilteringNearHead = (cs: t) =>
+  if (
+    !cs.fetchState.clientFilterPolicy.allDynamicContracts &&
+    cs.fetchState->FetchState.isWithinHeadClientFilteringDistance
+  ) {
+    cs.fetchState =
+      cs.fetchState->FetchState.switchDynamicContractsToClientSideFiltering(
+        ~addressStore=cs.addressStore,
+      )
+    if cs.onEventRegistrations->Array.some(registration => registration.contractRegister !== None) {
+      cs.logger->Logger.trace(
+        "Near the head: dynamic contracts are fetched without their addresses for the rest of the run, and filtered on our side.",
+        ~params={
+          "contractNames": cs.fetchState.optimizedPartitions.dynamicContracts->Utils.Set.toArray,
+        },
+      )
+    }
+  }
+
 let handleQueryResult = (
   cs: t,
   ~query: FetchState.query,
@@ -1000,6 +1110,7 @@ let handleQueryResult = (
     )
   }
 
+  let previousKnownHeight = cs.fetchState.knownHeight
   cs.fetchState =
     fs
     ->FetchState.handleQueryResult(~query, ~latestFetchedBlock, ~newItems)
@@ -1007,6 +1118,9 @@ let handleQueryResult = (
 
   // The query is no longer in flight, so release its reservation.
   cs.pendingBudget = Pervasives.max(0., cs.pendingBudget -. query.itemsEst->Int.toFloat)
+  if cs.fetchState.knownHeight > previousKnownHeight {
+    cs->switchDynamicContractsToClientSideFilteringNearHead
+  }
 }
 
 // Run reorg detection against a fetch response by merging its block-store page
@@ -1041,8 +1155,11 @@ let prepareReorg = (cs: t, ~eventsProcessedDiff) => {
   cs->resetPendingQueries
 }
 
-let updateKnownHeight = (cs: t, ~knownHeight) =>
+// A chain waiting at its head goes address-free before querying the new block.
+let updateKnownHeight = (cs: t, ~knownHeight) => {
   cs.fetchState = cs.fetchState->FetchState.updateKnownHeight(~knownHeight)
+  cs->switchDynamicContractsToClientSideFilteringNearHead
+}
 
 // In auto-exit mode, pin the endBlock to the earliest observed event block.
 let setEndBlockToFirstEvent = (cs: t, ~blockNumber) =>

package/src/ChainState.res.mjs

@@ -83,6 +83,7 @@ function make(chainConfig, fetchState, onEventRegistrationsOpt, addressStore, so
     processingBlockNumber: committedProgressBlockNumber,
     numEventsProcessed: numEventsProcessed,
     pendingBudget: 0,
+    pendingBudgetResets: 0,
     chainDensity: chainDensity,
     entities: EntityTables.make(perChainEntities),
     safeCheckpointTracking: safeCheckpointTracking,
@@ -103,7 +104,9 @@ function make(chainConfig, fetchState, onEventRegistrationsOpt, addressStore, so
     reorgDetectedBlock: undefined,
     rollbackTargetBlock: undefined,
     progressLatencyMs: undefined,
-    committedProgressBlockTime: committedProgressBlockTime
+    committedProgressBlockTime: committedProgressBlockTime,
+    blockRegisterTail: Promise.resolve(),
+    blockRegisterRuns: 0
   };
 }
 
@@ -159,16 +162,11 @@ function makeFromDbState(chainConfig, resumedChainState, reorgCheckpoints, isInR
     onBlockRegistrations: []
   });
   let onEventRegistrations = match.onEventRegistrations;
-  let onBlockRegistrations = match.onBlockRegistrations.map(reg => ({
-    index: reg.index,
-    name: reg.name,
-    chainId: reg.chainId,
-    startBlock: Stdlib_Option.getOr(reg.startBlock, startBlock),
-    endBlock: reg.endBlock,
-    interval: reg.interval,
-    handler: reg.handler,
-    blockFieldMask: reg.blockFieldMask
-  }));
+  let onBlockRegistrations = match.onBlockRegistrations.map(reg => {
+    let newrecord = {...reg};
+    newrecord.startBlock = Stdlib_Option.getOr(reg.startBlock, startBlock);
+    return newrecord;
+  });
   chainConfig.contracts.forEach(contract => {
     let contractStartBlock = contract.startBlock;
     if (contractStartBlock !== undefined && contractStartBlock < startBlock) {
@@ -188,7 +186,7 @@ function makeFromDbState(chainConfig, resumedChainState, reorgCheckpoints, isInR
       tmp = undefined;
       break;
   }
-  let fetchState = FetchState.make(startBlock, endBlock, onEventRegistrations, addressStore, addressRows, config.maxAddrInPartition, chainConfig.id, notice => logFetchStateNotice(logger$1, notice), (config.batchSize << 1), knownHeight, progressBlockNumber, onBlockRegistrations, Primitive_int.max(!config.shouldRollbackOnReorg || isInReorgThreshold ? 0 : maxReorgDepth, chainConfig.blockLag), Primitive_option.some(firstEventBlock), Primitive_option.some(tmp), isResumed);
+  let fetchState = FetchState.make(startBlock, endBlock, onEventRegistrations, addressStore, addressRows, config.maxAddrInPartition, chainConfig.id, notice => logFetchStateNotice(logger$1, notice), (config.batchSize << 1), knownHeight, progressBlockNumber, onBlockRegistrations, Primitive_int.max(!config.shouldRollbackOnReorg || isInReorgThreshold || maxReorgDepth === 0 ? 0 : maxReorgDepth + 1 | 0, chainConfig.blockLag), chainConfig.blockThrottle, Primitive_option.some(firstEventBlock), Primitive_option.some(tmp), isResumed, Stdlib_Option.isSome(config.subgraph));
   let chainReorgCheckpoints = Stdlib_Array.filterMap(reorgCheckpoints, reorgCheckpoint => {
     if (reorgCheckpoint.chain_id === chainConfig.id) {
       return reorgCheckpoint;
@@ -196,7 +194,7 @@ function makeFromDbState(chainConfig, resumedChainState, reorgCheckpoints, isInR
   });
   let blockStore = BlockStore.make(config.ecosystem.name);
   let transactionStore = TransactionStore.make(config.ecosystem.name);
-  let sources = ChainSources.make(chainConfig, onEventRegistrations, onBlockRegistrations, addressStore, lowercaseAddresses, blockStore, transactionStore, logger$1);
+  let sources = ChainSources.make(chainConfig, onEventRegistrations, onBlockRegistrations, addressStore, lowercaseAddresses, blockStore, transactionStore, logger$1, config.maxAddrInPartition);
   if (Utils.$$Array.notEmpty(chainReorgCheckpoints)) {
     let seedPage = BlockStore.fromJs(chainReorgCheckpoints.map(cp => ({
       blockNumber: cp.block_number,
@@ -225,6 +223,10 @@ function shouldChecksum(cs) {
   return cs.shouldChecksum;
 }
 
+function fetchState(cs) {
+  return cs.fetchState;
+}
+
 function entities(cs) {
   return cs.entities;
 }
@@ -338,6 +340,78 @@ function startFetchingQueries(cs, queries) {
 function resetPendingQueries(cs) {
   cs.fetchState = FetchState.resetPendingQueries(cs.fetchState);
   cs.pendingBudget = 0;
+  cs.pendingBudgetResets = cs.pendingBudgetResets + 1 | 0;
+}
+
+function holdExtraBudget(cs, items) {
+  cs.pendingBudget = cs.pendingBudget + items;
+  return {
+    items: items,
+    resets: cs.pendingBudgetResets
+  };
+}
+
+function releaseExtraBudget(cs, held) {
+  if (held.resets === cs.pendingBudgetResets) {
+    cs.pendingBudget = Primitive_float.max(0, cs.pendingBudget - held.items);
+    return;
+  }
+}
+
+function hasBlockRegisters(cs) {
+  return cs.fetchState.blockRegisterThrough !== undefined;
+}
+
+function isBlockRegisterIdle(cs) {
+  return cs.blockRegisterRuns === 0;
+}
+
+function queueBlockRegisterRun(cs) {
+  let previous = cs.blockRegisterTail;
+  let settle = {
+    contents: () => {}
+  };
+  cs.blockRegisterTail = new Promise((resolve, param) => {
+    settle.contents = resolve;
+  });
+  cs.blockRegisterRuns = cs.blockRegisterRuns + 1 | 0;
+  let finish = () => {
+    cs.blockRegisterRuns = cs.blockRegisterRuns - 1 | 0;
+    settle.contents();
+  };
+  return [
+    previous,
+    finish
+  ];
+}
+
+function canAdvanceBlockRegisters(cs, through) {
+  return FetchState.canAdvanceBlockRegisters(cs.fetchState, through);
+}
+
+function nextBlockRegisterItems(cs, through, limit) {
+  return FetchState.nextBlockRegisterItems(cs.fetchState, through, limit);
+}
+
+function blockRegisterItemsAfter(cs, after, through, limit) {
+  return FetchState.blockRegisterItemsAfter(cs.fetchState, after, through, limit);
+}
+
+function blockRegisterThrough(cs) {
+  return cs.fetchState.blockRegisterThrough;
+}
+
+function blockRegisterFrontier(cs) {
+  return FetchState.blockRegisterFrontier(cs.fetchState);
+}
+
+function applyBlockRegisters(cs, registrations, through) {
+  let fs = registrations.length !== 0 ? FetchState.registerDynamicContracts(cs.fetchState, cs.addressStore, notice => logFetchStateNotice(cs.logger, notice), undefined, registrations) : cs.fetchState;
+  cs.fetchState = FetchState.setBlockRegisterThrough(fs, through);
+}
+
+function markBlockRegistered(cs, through) {
+  cs.fetchState = FetchState.setBlockRegisterThrough(cs.fetchState, through);
 }
 
 function isReady(cs) {
@@ -494,12 +568,7 @@ function takeFinished(cs) {
 }
 
 function getHighestBlockBelowThreshold(cs) {
-  let highestBlockBelowThreshold = cs.fetchState.knownHeight - cs.maxReorgDepth | 0;
-  if (highestBlockBelowThreshold < 0) {
-    return 0;
-  } else {
-    return highestBlockBelowThreshold;
-  }
+  return Primitive_int.max((cs.fetchState.knownHeight - cs.maxReorgDepth | 0) - 1 | 0, 0);
 }
 
 function isFetchingAtHead(cs) {
@@ -647,10 +716,27 @@ async function materializePageItems(items, transactionStore, blockStore, shouldC
   ]);
 }
 
+function switchDynamicContractsToClientSideFilteringNearHead(cs) {
+  if (!cs.fetchState.clientFilterPolicy.allDynamicContracts && FetchState.isWithinHeadClientFilteringDistance(cs.fetchState)) {
+    cs.fetchState = FetchState.switchDynamicContractsToClientSideFiltering(cs.fetchState, cs.addressStore);
+    if (cs.onEventRegistrations.some(registration => registration.contractRegister !== undefined)) {
+      return Logger.trace(cs.logger, "Near the head: dynamic contracts are fetched without their addresses for the rest of the run, and filtered on our side.", {
+        contractNames: Array.from(cs.fetchState.optimizedPartitions.dynamicContracts)
+      });
+    } else {
+      return;
+    }
+  }
+}
+
 function handleQueryResult(cs, query, newItems, newRegistrations, latestFetchedBlock, knownHeight) {
   let fs = newRegistrations.length !== 0 ? FetchState.registerDynamicContracts(cs.fetchState, cs.addressStore, notice => logFetchStateNotice(cs.logger, notice), Primitive_int.max(knownHeight, latestFetchedBlock), newRegistrations) : cs.fetchState;
+  let previousKnownHeight = cs.fetchState.knownHeight;
   cs.fetchState = FetchState.updateKnownHeight(FetchState.handleQueryResult(fs, query, latestFetchedBlock, newItems), knownHeight);
   cs.pendingBudget = Primitive_float.max(0, cs.pendingBudget - query.itemsEst);
+  if (cs.fetchState.knownHeight > previousKnownHeight) {
+    return switchDynamicContractsToClientSideFilteringNearHead(cs);
+  }
 }
 
 function registerReorgGuard(cs, blockStore, knownHeight) {
@@ -683,6 +769,7 @@ function prepareReorg(cs, eventsProcessedDiff) {
 
 function updateKnownHeight(cs, knownHeight) {
   cs.fetchState = FetchState.updateKnownHeight(cs.fetchState, knownHeight);
+  switchDynamicContractsToClientSideFilteringNearHead(cs);
 }
 
 function setEndBlockToFirstEvent(cs, blockNumber) {
@@ -691,40 +778,14 @@ function setEndBlockToFirstEvent(cs, blockNumber) {
     if (blockNumber >= currentEndBlock) {
       return;
     }
-    let init = cs.fetchState;
-    cs.fetchState = {
-      optimizedPartitions: init.optimizedPartitions,
-      startBlock: init.startBlock,
-      endBlock: blockNumber,
-      normalSelection: init.normalSelection,
-      chainId: init.chainId,
-      latestOnBlockBlockNumber: init.latestOnBlockBlockNumber,
-      blockLag: init.blockLag,
-      buffer: init.buffer,
-      maxOnBlockBufferSize: init.maxOnBlockBufferSize,
-      onBlockRegistrations: init.onBlockRegistrations,
-      knownHeight: init.knownHeight,
-      firstEventBlock: init.firstEventBlock,
-      clientFilterAddressThreshold: init.clientFilterAddressThreshold
-    };
+    let newrecord = {...cs.fetchState};
+    newrecord.endBlock = blockNumber;
+    cs.fetchState = newrecord;
     return;
   }
-  let init$1 = cs.fetchState;
-  cs.fetchState = {
-    optimizedPartitions: init$1.optimizedPartitions,
-    startBlock: init$1.startBlock,
-    endBlock: blockNumber,
-    normalSelection: init$1.normalSelection,
-    chainId: init$1.chainId,
-    latestOnBlockBlockNumber: init$1.latestOnBlockBlockNumber,
-    blockLag: init$1.blockLag,
-    buffer: init$1.buffer,
-    maxOnBlockBufferSize: init$1.maxOnBlockBufferSize,
-    onBlockRegistrations: init$1.onBlockRegistrations,
-    knownHeight: init$1.knownHeight,
-    firstEventBlock: init$1.firstEventBlock,
-    clientFilterAddressThreshold: init$1.clientFilterAddressThreshold
-  };
+  let newrecord$1 = {...cs.fetchState};
+  newrecord$1.endBlock = blockNumber;
+  cs.fetchState = newrecord$1;
 }
 
 function enterReorgThreshold(cs) {
@@ -872,22 +933,9 @@ function applyBatchProgress(cs, batch, blockTimestampName) {
   if (match === undefined) {
     let firstEventBlock = Batch.findFirstEventBlockNumber(batch, chainId);
     if (firstEventBlock !== undefined) {
-      let init = cs.fetchState;
-      cs.fetchState = {
-        optimizedPartitions: init.optimizedPartitions,
-        startBlock: init.startBlock,
-        endBlock: init.endBlock,
-        normalSelection: init.normalSelection,
-        chainId: init.chainId,
-        latestOnBlockBlockNumber: init.latestOnBlockBlockNumber,
-        blockLag: init.blockLag,
-        buffer: init.buffer,
-        maxOnBlockBufferSize: init.maxOnBlockBufferSize,
-        onBlockRegistrations: init.onBlockRegistrations,
-        knownHeight: init.knownHeight,
-        firstEventBlock: firstEventBlock,
-        clientFilterAddressThreshold: init.clientFilterAddressThreshold
-      };
+      let newrecord = {...cs.fetchState};
+      newrecord.firstEventBlock = firstEventBlock;
+      cs.fetchState = newrecord;
     }
   }
   let deltaBlocks = chainAfterBatch.progressBlockNumber - cs.committedProgressBlockNumber | 0;
@@ -990,6 +1038,7 @@ export {
   blockStore,
   transactionStore,
   shouldChecksum,
+  fetchState,
   entities,
   resetEntities,
   sourceManager,
@@ -1002,6 +1051,18 @@ export {
   committedProgressBlockTime,
   numEventsProcessed,
   pendingBudget,
+  holdExtraBudget,
+  releaseExtraBudget,
+  hasBlockRegisters,
+  isBlockRegisterIdle,
+  queueBlockRegisterRun,
+  canAdvanceBlockRegisters,
+  nextBlockRegisterItems,
+  blockRegisterItemsAfter,
+  blockRegisterThrough,
+  blockRegisterFrontier,
+  applyBlockRegisters,
+  markBlockRegistered,
   startFetchingQueries,
   timestampCaughtUpToHeadOrEndblock,
   recordBlockRangeFetch,

package/src/ChainState.resi

@@ -58,6 +58,8 @@ let logger: t => Logger.t
 let blockStore: t => BlockStore.t
 let transactionStore: t => TransactionStore.t
 let shouldChecksum: t => bool
+// The chain's fetch state, read-only: for the round a registration belongs to.
+let fetchState: t => FetchState.t
 let entities: t => EntityTables.t
 let resetEntities: (t, ~perChainEntities: array<Internal.entityConfig>) => unit
 let sourceManager: t => SourceManager.t
@@ -75,6 +77,30 @@ let committedProgressBlockNumber: t => int
 let committedProgressBlockTime: t => option<int>
 let numEventsProcessed: t => float
 let pendingBudget: t => float
+type heldBudget
+let holdExtraBudget: (t, ~items: int) => heldBudget
+let releaseExtraBudget: (t, heldBudget) => unit
+// Subgraph mode: the register passes of block handlers that create data
+// sources. See `FetchState.blockRegisterThrough`.
+let hasBlockRegisters: t => bool
+let isBlockRegisterIdle: t => bool
+let queueBlockRegisterRun: t => (promise<unit>, unit => unit)
+let canAdvanceBlockRegisters: (t, ~through: int) => bool
+let nextBlockRegisterItems: (t, ~through: int, ~limit: int) => option<(array<Internal.item>, int)>
+let blockRegisterItemsAfter: (
+  t,
+  ~after: int,
+  ~through: int,
+  ~limit: int,
+) => option<(array<Internal.item>, int)>
+let blockRegisterThrough: t => option<int>
+let blockRegisterFrontier: t => int
+let applyBlockRegisters: (
+  t,
+  ~registrations: array<AddressStore.registration>,
+  ~through: int,
+) => unit
+let markBlockRegistered: (t, ~through: int) => unit
 let startFetchingQueries: (t, ~queries: array<FetchState.query>) => unit
 let timestampCaughtUpToHeadOrEndblock: t => option<Date.t>
 

package/src/Clock.res

@@ -0,0 +1,57 @@
+// What the source layer reads time from and arms its timers on. A test can hand
+// it a clock that only moves when the test moves it, which takes the scheduler
+// out of every race between a timer and the step a test is taking. The label
+// names a timer in that clock's timeline; the real clock ignores it.
+type timer
+
+type t = {
+  now: unit => float,
+  setTimeout: (~label: string, unit => unit, int) => timer,
+  clearTimeout: timer => unit,
+  random: unit => float,
+}
+
+// Test-only. Set while a test drives a run on virtual time, so code that reaches
+// past the clock it was given fails where it does, instead of arming a timer
+// the test can neither see nor move.
+let refuseReal = ref(false)
+
+let refused = what =>
+  JsError.throwWithMessage(
+    `Clock.real ${what} on virtual time. Use the clock the code was given, so advanceTime can see and move it.`,
+  )
+
+// Monotonic: every reading is only ever compared with another, and a wall
+// clock corrected mid-run would make a connection look like it served for
+// minutes, or a benched source look recovered.
+let real: t = {
+  now: () => refuseReal.contents ? refused("read the time") : Performance.now(),
+  // A zero delay is a turn of the event loop on either clock.
+  setTimeout: (~label, callback, millis) =>
+    refuseReal.contents && millis > 0
+      ? refused(`armed "${label}"`)
+      : setTimeout(callback, millis)->(Utils.magic: timeoutId => timer),
+  clearTimeout: timer => clearTimeout(timer->(Utils.magic: timer => timeoutId)),
+  random: Math.random,
+}
+
+let delay = (clock: t, ~label, millis) =>
+  Promise.make((resolve, _reject) => clock.setTimeout(~label, () => resolve(), millis)->ignore)
+
+// Half the delay fixed, half spread across it. Every indexer pointed at one
+// provider loses its stream, or gives up on a quiet chain, in the same instant
+// that provider blinks, and putting them all back on the same schedule is how a
+// blip becomes a stampede.
+let jitter = (clock: t, delay) =>
+  delay / 2 + (clock.random() *. (delay / 2)->Int.toFloat)->Float.toInt
+
+// Clearing a pending timer and forgetting it are one action. A ref left holding
+// a spent timer reads as one still pending, and the next clear looks like it
+// did something.
+let clearRef = (clock: t, timer: ref<option<timer>>) => {
+  switch timer.contents {
+  | Some(timer) => clock.clearTimeout(timer)
+  | None => ()
+  }
+  timer := None
+}

package/src/Clock.res.mjs

@@ -0,0 +1,72 @@
+// Generated by ReScript, PLEASE EDIT WITH CARE
+
+import * as Performance from "./bindings/Performance.res.mjs";
+import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
+import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js";
+
+let refuseReal = {
+  contents: false
+};
+
+function refused(what) {
+  return Stdlib_JsError.throwWithMessage(`Clock.real ` + what + ` on virtual time. Use the clock the code was given, so advanceTime can see and move it.`);
+}
+
+function real_now() {
+  if (refuseReal.contents) {
+    return refused("read the time");
+  } else {
+    return Performance.now();
+  }
+}
+
+function real_setTimeout(label, callback, millis) {
+  if (refuseReal.contents && millis > 0) {
+    return refused(`armed "` + label + `"`);
+  } else {
+    return setTimeout(callback, millis);
+  }
+}
+
+function real_clearTimeout(timer) {
+  clearTimeout(timer);
+}
+
+function real_random(prim) {
+  return Math.random();
+}
+
+let real = {
+  now: real_now,
+  setTimeout: real_setTimeout,
+  clearTimeout: real_clearTimeout,
+  random: real_random
+};
+
+function delay(clock, label, millis) {
+  return new Promise((resolve, _reject) => {
+    clock.setTimeout(label, () => resolve(), millis);
+  });
+}
+
+function jitter(clock, delay) {
+  return (delay / 2 | 0) + (clock.random() * (delay / 2 | 0) | 0) | 0;
+}
+
+function clearRef(clock, timer) {
+  let timer$1 = timer.contents;
+  if (timer$1 !== undefined) {
+    clock.clearTimeout(Primitive_option.valFromOption(timer$1));
+  }
+  timer.contents = undefined;
+}
+
+export {
+  refuseReal,
+  refused,
+  real,
+  delay,
+  jitter,
+  clearRef,
+}
+/* No side effect */