envio 3.5.0-alpha.0 → 3.5.0-alpha.2

package/index.d.ts

@@ -585,14 +585,18 @@ export type SvmOnSlotContext<Config extends IndexerConfigTypes = GlobalConfig> =
   BaseHandlerContext<Config, SvmChainIds<Config>>
 >;
 
+/** The entity's `id` type. `ID!`/`String!` ids are `string`; `Int!` is `number`
+ * and `BigInt!` is `bigint`, so id-keyed operations accept the real scalar. */
+type EntityId<Entity> = Entity extends { readonly id: infer Id } ? Id : string;
+
 /** Entity operations available in handler contexts. */
 type EntityOperations<Entity> = {
-  readonly get: (id: string) => Promise<Entity | undefined>;
-  readonly getOrThrow: (id: string, message?: string) => Promise<Entity>;
+  readonly get: (id: EntityId<Entity>) => Promise<Entity | undefined>;
+  readonly getOrThrow: (id: EntityId<Entity>, message?: string) => Promise<Entity>;
   readonly getWhere: (filter: GetWhereFilter<Entity>) => Promise<Entity[]>;
   readonly getOrCreate: (entity: Entity) => Promise<Entity>;
   readonly set: (entity: Entity) => void;
-  readonly deleteUnsafe: (id: string) => void;
+  readonly deleteUnsafe: (id: EntityId<Entity>) => void;
 };
 
 /** Contract registration handle. */
@@ -1510,7 +1514,7 @@ type EntityChangeValue<Entity> = {
   /** Entities that were created or updated. */
   readonly sets?: readonly Entity[];
   /** IDs of entities that were deleted. */
-  readonly deleted?: readonly string[];
+  readonly deleted?: readonly EntityId<Entity>[];
 };
 
 /** A dynamic contract address registration. */
@@ -1528,9 +1532,9 @@ type ConfigEntities<Config extends IndexerConfigTypes = GlobalConfig> =
 /** Entity operations available on test indexer for direct entity manipulation. */
 type TestIndexerEntityOperations<Entity> = {
   /** Get an entity by ID. Returns undefined if not found. */
-  readonly get: (id: string) => Promise<Entity | undefined>;
+  readonly get: (id: EntityId<Entity>) => Promise<Entity | undefined>;
   /** Get an entity by ID or throw if not found. */
-  readonly getOrThrow: (id: string, message?: string) => Promise<Entity>;
+  readonly getOrThrow: (id: EntityId<Entity>, message?: string) => Promise<Entity>;
   /** Get all entities. */
   readonly getAll: () => Promise<Entity[]>;
   /** Set (create or update) an entity. */

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "envio",
-  "version": "3.5.0-alpha.0",
+  "version": "3.5.0-alpha.2",
   "type": "module",
   "description": "A latency and sync speed optimized, developer friendly blockchain data indexer.",
   "bin": "./bin.mjs",
@@ -69,10 +69,10 @@
     "tsx": "4.21.0"
   },
   "optionalDependencies": {
-    "envio-linux-x64": "3.5.0-alpha.0",
-    "envio-linux-x64-musl": "3.5.0-alpha.0",
-    "envio-linux-arm64": "3.5.0-alpha.0",
-    "envio-darwin-x64": "3.5.0-alpha.0",
-    "envio-darwin-arm64": "3.5.0-alpha.0"
+    "envio-linux-x64": "3.5.0-alpha.2",
+    "envio-linux-x64-musl": "3.5.0-alpha.2",
+    "envio-linux-arm64": "3.5.0-alpha.2",
+    "envio-darwin-x64": "3.5.0-alpha.2",
+    "envio-darwin-arm64": "3.5.0-alpha.2"
   }
 }

package/src/BatchProcessing.res

@@ -45,6 +45,19 @@ let rec startProcessing = async (state: IndexerState.t, ~scheduleFetch, ~schedul
     // Hand off now that no batch is in flight.
     if state->IndexerState.isResolvingReorg {
       scheduleRollback()
+    } else if (
+      !(state->IndexerState.isStopped) &&
+      // Only a genuine fetch stall when some chain still has blocks to bring in.
+      // If every chain has buffered up to its known head, the loop is idling at
+      // the tip waiting for new blocks, not bottlenecked on fetch.
+      !(
+        state
+        ->IndexerState.chainStates
+        ->Dict.valuesToArray
+        ->Array.every(ChainState.isFetchingAtHead)
+      )
+    ) {
+      state->IndexerState.markProcessingStalledOnFetch
     }
   }
 }

package/src/BatchProcessing.res.mjs

@@ -108,6 +108,10 @@ async function startProcessing(state, scheduleFetch, scheduleRollback) {
   IndexerState.endProcessing(state);
   if (IndexerState.isResolvingReorg(state)) {
     return scheduleRollback();
+  } else if (!IndexerState.isStopped(state) && !Object.values(IndexerState.chainStates(state)).every(ChainState.isFetchingAtHead)) {
+    return IndexerState.markProcessingStalledOnFetch(state);
+  } else {
+    return;
   }
 }
 

package/src/Bin.res

@@ -52,8 +52,8 @@ let applyEnv = (env: dict<JSON.t>) =>
 let run = async args => {
   try {
     switch (await Core.runCli(args))->Null.toOption {
-    // Rust-only command (codegen / init / serve / stop / docker / metrics /
-    // help / version / scripts) — nothing for JS to do, exit cleanly.
+    // Rust-only command (codegen / init / stop / docker / metrics / help /
+    // version / scripts) — nothing for JS to do, exit cleanly.
     | None => ()
     | Some(json) =>
       switch decodeCommand(json->JSON.parseOrThrow) {

package/src/ChainFetching.res

@@ -209,11 +209,11 @@ let rec onQueryResponse = async (
       // kick (eg from the processing loop quiescing) collapses into this one.
       scheduleRollback()
     | None =>
-      // Drop over-fetched events (a merged partition returning an address before
-      // its effectiveStartBlock, or a wildcard param referencing an address
-      // registered after the log's block) before contract registration, so they
-      // neither spawn dynamic contracts nor enter the buffer.
-      let newItems = chainState->ChainState.filterByClientAddress(parsedQueueItems)
+      // Over-fetched events (a merged partition returning an address before its
+      // effectiveStartBlock, or a wildcard param referencing an address
+      // registered after the log's block) are already dropped by the source's
+      // native address gate, so everything here is indexable.
+      let newItems = parsedQueueItems
       let itemsWithContractRegister = []
       for idx in 0 to newItems->Array.length - 1 {
         let item = newItems->Array.getUnsafe(idx)

package/src/ChainFetching.res.mjs

@@ -145,17 +145,16 @@ async function onQueryResponse(state, param, stateId, scheduleFetch, schedulePro
     IndexerState.beginReorg(state, chain, rollbackWithReorgDetectedBlockNumber);
     return scheduleRollback();
   }
-  let newItems = ChainState.filterByClientAddress(chainState, parsedQueueItems);
   let itemsWithContractRegister = [];
-  for (let idx = 0, idx_finish = newItems.length; idx < idx_finish; ++idx) {
-    let item = newItems[idx];
+  for (let idx = 0, idx_finish = parsedQueueItems.length; idx < idx_finish; ++idx) {
+    let item = parsedQueueItems[idx];
     if (item.onEventRegistration.contractRegister !== undefined) {
       itemsWithContractRegister.push(item);
     }
   }
   let proceed = newItemsWithDcs => {
     if (!IndexerState.isStale(state, stateId)) {
-      applyQueryResponse(state, chain, newItems, newItemsWithDcs, knownHeight, {
+      applyQueryResponse(state, chain, parsedQueueItems, newItemsWithDcs, knownHeight, {
         blockNumber: latestFetchedBlockNumber,
         blockTimestamp: latestFetchedBlockTimestamp
       }, query, transactionStore, blockStore);

package/src/ChainState.res

@@ -7,9 +7,10 @@ type t = {
   // The registrations used to build this chain's sources and route native items.
   onEventRegistrations: array<Internal.onEventRegistration>,
   mutable fetchState: FetchState.t,
-  // The chain-wide address index. Not `mutable`: the dict is mutated in place by
-  // register/rollback, so the reference is stable across fetchState versions.
-  indexingAddresses: IndexingAddresses.t,
+  // The chain-wide address index, kept in Rust. Not `mutable`: registration and
+  // rollback mutate it in place, so the handle is stable across fetchState
+  // versions — and it's the same handle this chain's source clients hold.
+  addressStore: AddressStore.t,
   sourceManager: SourceManager.t,
   chainConfig: Config.chain,
   mutable isProgressAtHead: bool,
@@ -83,7 +84,7 @@ let make = (
   ~chainConfig: Config.chain,
   ~fetchState: FetchState.t,
   ~onEventRegistrations=[],
-  ~indexingAddresses: IndexingAddresses.t,
+  ~addressStore: AddressStore.t,
   ~sourceManager: SourceManager.t,
   ~reorgDetection: ReorgDetection.t,
   ~committedProgressBlockNumber: int,
@@ -102,7 +103,7 @@ let make = (
     logger,
     onEventRegistrations,
     fetchState,
-    indexingAddresses,
+    addressStore,
     sourceManager,
     chainConfig,
     isProgressAtHead,
@@ -166,12 +167,18 @@ let makeInternal = (
     }
   })
 
-  let contractConfigs = IndexingAddresses.makeContractConfigs(~onEventRegistrations)
-  let indexingAddressIndex = IndexingAddresses.make(~contractConfigs, ~addresses=indexingAddresses)
+  let lowercaseAddresses = config.lowercaseAddresses
+  // Created before the fetch state and the sources: `make` registers this
+  // chain's addresses into it, and every source client holds the same handle.
+  let addressStore = AddressStore.make(
+    ~ecosystem=config.ecosystem.name,
+    ~shouldChecksum=!lowercaseAddresses,
+    ~contracts=AddressStore.contractsOf(~onEventRegistrations),
+  )
 
   let fetchState = FetchState.make(
     ~maxAddrInPartition=config.maxAddrInPartition,
-    ~contractConfigs,
+    ~addressStore,
     ~addresses=indexingAddresses,
     ~progressBlockNumber,
     ~startBlock,
@@ -187,6 +194,15 @@ let makeInternal = (
     ),
     ~onBlockRegistrations,
     ~firstEventBlock,
+    // Fuel and SVM route through the address store like EVM does, so the
+    // client-side path works for them — but their address counts are nowhere
+    // near the threshold, so switching would only trade a server-side filter
+    // that costs nothing today for an over-fetch. Keep them server-side until a
+    // chain actually needs it.
+    ~clientFilterAddressThreshold=switch config.ecosystem.name {
+    | Evm => Some(config.clientFilterAddressThreshold)
+    | Fuel | Svm => None
+    },
   )
 
   let chainReorgCheckpoints = reorgCheckpoints->Array.filterMap(reorgCheckpoint => {
@@ -199,7 +215,6 @@ let makeInternal = (
 
   // Create sources lazily here - this is where API token validation happens
   let chain = ChainMap.Chain.makeUnsafe(~chainId=chainConfig.id)
-  let lowercaseAddresses = config.lowercaseAddresses
   let sources = switch chainConfig.sourceConfig {
   | Config.EvmSourceConfig({hypersync, rpcs}) =>
     let evmRpcs: array<EvmChain.rpc> = rpcs->Array.map((rpc): EvmChain.rpc => {
@@ -222,6 +237,7 @@ let makeInternal = (
       ~hyperSync=hypersync,
       ~rpcs=evmRpcs,
       ~lowercaseAddresses,
+      ~addressStore,
     )
   | Config.FuelSourceConfig({hypersync}) => [
       FuelHyperSyncSource.make({
@@ -229,6 +245,7 @@ let makeInternal = (
         endpointUrl: hypersync,
         apiToken: Env.envioApiToken,
         onEventRegistrations,
+        addressStore,
       }),
     ]
   | Config.SvmSourceConfig({hypersync, rpc}) =>
@@ -249,6 +266,7 @@ let makeInternal = (
           apiToken,
           onEventRegistrations,
           clientTimeoutMillis: Env.hyperSyncClientTimeoutMillis,
+          addressStore,
         }),
       ]
     }
@@ -268,7 +286,7 @@ let makeInternal = (
     ~chainConfig,
     ~fetchState,
     ~onEventRegistrations,
-    ~indexingAddresses=indexingAddressIndex,
+    ~addressStore,
     ~sourceManager=SourceManager.make(~sources, ~isRealtime, ~reducedPollingInterval?),
     ~reorgDetection=ReorgDetection.make(
       ~chainReorgCheckpoints,
@@ -409,7 +427,7 @@ let setRollbackTargetBlock = (cs: t, ~blockNumber) => cs.rollbackTargetBlock = S
 // rather than reaching into it.
 let knownHeight = (cs: t) => cs.fetchState.knownHeight
 let contractAddresses = (cs: t, ~contractName) =>
-  cs.indexingAddresses->IndexingAddresses.getContractAddresses(~contractName)
+  cs.addressStore->AddressStore.contractAddresses(contractName)
 let bufferSize = (cs: t) => cs.fetchState->FetchState.bufferSize
 let bufferReadyCount = (cs: t) => cs.fetchState->FetchState.bufferReadyCount
 let getProgressPercentage = (cs: t) => cs.fetchState->FetchState.getProgressPercentage
@@ -541,13 +559,7 @@ let frontierProgress = (cs: t) =>
 // maxTargetBlock set to the most-behind chain's progress mapped onto this
 // chain, so a chain with budget can't run further ahead than the chain the
 // whole pool is prioritizing.
-let getNextQuery = (
-  cs: t,
-  ~chainTargetItems: float,
-  ~chunkItemsMultiplier=1.,
-  ~itemsTargetFloor=0,
-  ~maxTargetBlock=?,
-) => {
+let getNextQuery = (cs: t, ~chainTargetItems: float, ~maxTargetBlock=?) => {
   let chainTargetBlock = cs->targetBlock(~chainTargetItems)
   let chainTargetBlock = switch maxTargetBlock {
   | Some(maxTargetBlock) => Pervasives.min(chainTargetBlock, maxTargetBlock)
@@ -561,10 +573,6 @@ let getNextQuery = (
   // of being held by an oversized probe.
   let chainTargetItems = switch cs->effectiveDensity {
   | Some(density) if density > 0. =>
-    // No extra headroom here: the budget is reserved in honest itemsEst units,
-    // and truncation safety lives in the itemsTarget server cap (sized with
-    // chunkItemsMultiplier at query creation) — multiplying the budget cap too
-    // would compound the two and hold budget away from other chains.
     let rangeCost =
       density *. (chainTargetBlock - cs.fetchState->FetchState.bufferBlockNumber)->Int.toFloat
     Pervasives.min(chainTargetItems, Math.ceil(rangeCost) +. cs.pendingBudget)
@@ -572,12 +580,7 @@ let getNextQuery = (
   // budget to the cold-chain cap, so it's used as-is.
   | _ => chainTargetItems
   }
-  cs.fetchState->FetchState.getNextQuery(
-    ~chainTargetBlock,
-    ~chainTargetItems,
-    ~chunkItemsMultiplier,
-    ~itemsTargetFloor,
-  )
+  cs.fetchState->FetchState.getNextQuery(~chainTargetBlock, ~chainTargetItems)
 }
 
 // Run a fetch tick for this chain against its sources, feeding the owned fetch
@@ -715,6 +718,9 @@ let groupBatchItems = (items: array<Internal.item>, ~includeBlocks: bool): (
           }
         }
       }
+    // onBlock items build their block from the handler's own block number, not
+    // from the stores — which is what lets the sources keep only the blocks and
+    // transactions an event item references.
     | Internal.Block(_) => ()
     }
   )
@@ -811,15 +817,12 @@ let materializePageItems = async (
   ))
 }
 
-let filterByClientAddress = (cs: t, items: array<Internal.item>): array<Internal.item> =>
-  items->FetchState.filterByClientAddress(~indexingAddresses=cs.indexingAddresses)
-
 let handleQueryResult = (
   cs: t,
   ~query: FetchState.query,
   ~newItems,
   ~newItemsWithDcs,
-  ~latestFetchedBlock,
+  ~latestFetchedBlock: FetchState.blockNumberAndTimestamp,
   ~knownHeight,
   ~transactionStore as txPage: option<TransactionStore.t>,
   ~blockStore as blockPage: option<BlockStore.t>,
@@ -839,7 +842,12 @@ let handleQueryResult = (
   | [] => cs.fetchState
   | _ =>
     cs.fetchState->FetchState.registerDynamicContracts(
-      ~indexingAddresses=cs.indexingAddresses,
+      ~addressStore=cs.addressStore,
+      // This response is applied below, after the addresses land. It was routed
+      // before they existed, so whatever it claims has to be inside the
+      // catch-up range — and an unbounded query can reach past the height that
+      // was known when it went out.
+      ~claimCeiling=Pervasives.max(knownHeight, latestFetchedBlock.blockNumber),
       newItemsWithDcs,
     )
   }
@@ -915,7 +923,7 @@ let toMetrics = (cs: t): Metrics.chainMetrics => {
   numBatchesFetched: 0,
   startBlock: cs.fetchState.startBlock,
   endBlock: cs.fetchState.endBlock,
-  numAddresses: cs.indexingAddresses->IndexingAddresses.size,
+  numAddresses: cs.addressStore->AddressStore.size,
   isReady: cs->isReady,
   sourceBlockNumber: cs.fetchState.knownHeight,
   progressBlockNumber: cs.committedProgressBlockNumber,
@@ -1120,7 +1128,7 @@ let rollback = (
     }
     cs.fetchState =
       cs.fetchState->FetchState.rollback(
-        ~indexingAddresses=cs.indexingAddresses,
+        ~addressStore=cs.addressStore,
         ~targetBlockNumber=newProgressBlockNumber,
       )
     cs.transactionStore->TransactionStore.rollback(newProgressBlockNumber)
@@ -1136,7 +1144,7 @@ let rollback = (
         )
       cs.fetchState =
         cs.fetchState->FetchState.rollback(
-          ~indexingAddresses=cs.indexingAddresses,
+          ~addressStore=cs.addressStore,
           ~targetBlockNumber=rollbackTargetBlockNumber,
         )
       cs.transactionStore->TransactionStore.rollback(rollbackTargetBlockNumber)

package/src/ChainState.res.mjs

@@ -11,6 +11,7 @@ import * as FieldMask from "./sources/FieldMask.res.mjs";
 import * as BlockStore from "./sources/BlockStore.res.mjs";
 import * as FetchState from "./FetchState.res.mjs";
 import * as Stdlib_Null from "@rescript/runtime/lib/es6/Stdlib_Null.js";
+import * as AddressStore from "./sources/AddressStore.res.mjs";
 import * as Stdlib_Array from "@rescript/runtime/lib/es6/Stdlib_Array.js";
 import * as Primitive_int from "@rescript/runtime/lib/es6/Primitive_int.js";
 import * as SourceManager from "./sources/SourceManager.res.mjs";
@@ -20,7 +21,6 @@ import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
 import * as Primitive_float from "@rescript/runtime/lib/es6/Primitive_float.js";
 import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js";
 import * as TransactionStore from "./sources/TransactionStore.res.mjs";
-import * as IndexingAddresses from "./IndexingAddresses.res.mjs";
 import * as SvmHyperSyncSource from "./sources/SvmHyperSyncSource.res.mjs";
 import * as FuelHyperSyncSource from "./sources/FuelHyperSyncSource.res.mjs";
 import * as SafeCheckpointTracking from "./SafeCheckpointTracking.res.mjs";
@@ -47,7 +47,7 @@ function validateOnEventRegistrations(chainId, registrations) {
   });
 }
 
-function make(chainConfig, fetchState, onEventRegistrationsOpt, indexingAddresses, sourceManager, reorgDetection, committedProgressBlockNumber, safeCheckpointTrackingOpt, numEventsProcessedOpt, timestampCaughtUpToHeadOrEndblockOpt, isProgressAtHeadOpt, transactionStoreOpt, chainDensityOpt, blockStoreOpt, reorgThresholdReadyToleranceOpt, logger) {
+function make(chainConfig, fetchState, onEventRegistrationsOpt, addressStore, sourceManager, reorgDetection, committedProgressBlockNumber, safeCheckpointTrackingOpt, numEventsProcessedOpt, timestampCaughtUpToHeadOrEndblockOpt, isProgressAtHeadOpt, transactionStoreOpt, chainDensityOpt, blockStoreOpt, reorgThresholdReadyToleranceOpt, logger) {
   let onEventRegistrations = onEventRegistrationsOpt !== undefined ? onEventRegistrationsOpt : [];
   let safeCheckpointTracking = safeCheckpointTrackingOpt !== undefined ? Primitive_option.valFromOption(safeCheckpointTrackingOpt) : undefined;
   let numEventsProcessed = numEventsProcessedOpt !== undefined ? numEventsProcessedOpt : 0;
@@ -62,7 +62,7 @@ function make(chainConfig, fetchState, onEventRegistrationsOpt, indexingAddresse
     logger: logger,
     onEventRegistrations: onEventRegistrations,
     fetchState: fetchState,
-    indexingAddresses: indexingAddresses,
+    addressStore: addressStore,
     sourceManager: sourceManager,
     chainConfig: chainConfig,
     isProgressAtHead: isProgressAtHead,
@@ -103,16 +103,26 @@ function makeInternal(chainConfig, indexingAddresses, startBlock, endBlock, firs
       return Stdlib_JsError.throwWithMessage(`The start block for contract "` + contract.name + `" is less than the chain start block. This is not supported yet.`);
     }
   });
-  let contractConfigs = IndexingAddresses.makeContractConfigs(onEventRegistrations);
-  let indexingAddressIndex = IndexingAddresses.make(contractConfigs, indexingAddresses);
-  let fetchState = FetchState.make(startBlock, endBlock, onEventRegistrations, contractConfigs, indexingAddresses, config.maxAddrInPartition, chainConfig.id, (config.batchSize << 1), knownHeight, progressBlockNumber, match.onBlockRegistrations, Primitive_int.max(!config.shouldRollbackOnReorg || isInReorgThreshold ? 0 : chainConfig.maxReorgDepth, chainConfig.blockLag), Primitive_option.some(firstEventBlock));
+  let lowercaseAddresses = config.lowercaseAddresses;
+  let addressStore = AddressStore.make(config.ecosystem.name, !lowercaseAddresses, AddressStore.contractsOf(onEventRegistrations));
+  let match$1 = config.ecosystem.name;
+  let tmp;
+  switch (match$1) {
+    case "evm" :
+      tmp = config.clientFilterAddressThreshold;
+      break;
+    case "fuel" :
+    case "svm" :
+      tmp = undefined;
+      break;
+  }
+  let fetchState = FetchState.make(startBlock, endBlock, onEventRegistrations, addressStore, indexingAddresses, config.maxAddrInPartition, chainConfig.id, (config.batchSize << 1), knownHeight, progressBlockNumber, match.onBlockRegistrations, Primitive_int.max(!config.shouldRollbackOnReorg || isInReorgThreshold ? 0 : chainConfig.maxReorgDepth, chainConfig.blockLag), Primitive_option.some(firstEventBlock), Primitive_option.some(tmp));
   let chainReorgCheckpoints = Stdlib_Array.filterMap(reorgCheckpoints, reorgCheckpoint => {
     if (reorgCheckpoint.chain_id === chainConfig.id) {
       return reorgCheckpoint;
     }
   });
   let chain = ChainMap.Chain.makeUnsafe(chainConfig.id);
-  let lowercaseAddresses = config.lowercaseAddresses;
   let sources = chainConfig.sourceConfig;
   let sources$1;
   switch (sources.TAG) {
@@ -129,14 +139,15 @@ function makeInternal(chainConfig, indexingAddresses, startBlock, endBlock, firs
           headers: headers
         };
       });
-      sources$1 = EvmChain.makeSources(chain, onEventRegistrations, sources.hypersync, evmRpcs, lowercaseAddresses);
+      sources$1 = EvmChain.makeSources(chain, onEventRegistrations, sources.hypersync, evmRpcs, lowercaseAddresses, addressStore);
       break;
     case "FuelSourceConfig" :
       sources$1 = [FuelHyperSyncSource.make({
           chain: chain,
           endpointUrl: sources.hypersync,
           apiToken: Env.envioApiToken,
-          onEventRegistrations: onEventRegistrations
+          onEventRegistrations: onEventRegistrations,
+          addressStore: addressStore
         })];
       break;
     case "SvmSourceConfig" :
@@ -147,7 +158,8 @@ function makeInternal(chainConfig, indexingAddresses, startBlock, endBlock, firs
             endpointUrl: hypersync,
             apiToken: Env.envioApiToken,
             onEventRegistrations: onEventRegistrations,
-            clientTimeoutMillis: Env.hyperSyncClientTimeoutMillis
+            clientTimeoutMillis: Env.hyperSyncClientTimeoutMillis,
+            addressStore: addressStore
           })] : (
           rpc !== undefined ? [Svm.makeRPCSource(chain, rpc, undefined)] : Stdlib_JsError.throwWithMessage(`Chain ` + chain.toString() + ` has no SVM data source`)
         );
@@ -158,7 +170,7 @@ function makeInternal(chainConfig, indexingAddresses, startBlock, endBlock, firs
   }
   let firstEventBlock$1 = fetchState.firstEventBlock;
   let chainDensity = firstEventBlock$1 !== undefined && progressBlockNumber > firstEventBlock$1 && numEventsProcessed > 0 ? numEventsProcessed / (progressBlockNumber - firstEventBlock$1 | 0) : undefined;
-  return make(chainConfig, fetchState, onEventRegistrations, indexingAddressIndex, SourceManager.make(sources$1, isRealtime, undefined, undefined, undefined, reducedPollingInterval, undefined, undefined), ReorgDetection.make(chainReorgCheckpoints, maxReorgDepth, config.shouldRollbackOnReorg), progressBlockNumber, Primitive_option.some(SafeCheckpointTracking.make(maxReorgDepth, config.shouldRollbackOnReorg, chainReorgCheckpoints)), numEventsProcessed, Primitive_option.some(timestampCaughtUpToHeadOrEndblock), undefined, Primitive_option.some(TransactionStore.make(config.ecosystem.name, !lowercaseAddresses)), Primitive_option.some(chainDensity), Primitive_option.some(BlockStore.make(config.ecosystem.name, !lowercaseAddresses)), config.reorgThresholdReadyTolerance, logger);
+  return make(chainConfig, fetchState, onEventRegistrations, addressStore, SourceManager.make(sources$1, isRealtime, undefined, undefined, undefined, reducedPollingInterval, undefined, undefined), ReorgDetection.make(chainReorgCheckpoints, maxReorgDepth, config.shouldRollbackOnReorg), progressBlockNumber, Primitive_option.some(SafeCheckpointTracking.make(maxReorgDepth, config.shouldRollbackOnReorg, chainReorgCheckpoints)), numEventsProcessed, Primitive_option.some(timestampCaughtUpToHeadOrEndblock), undefined, Primitive_option.some(TransactionStore.make(config.ecosystem.name, !lowercaseAddresses)), Primitive_option.some(chainDensity), Primitive_option.some(BlockStore.make(config.ecosystem.name, !lowercaseAddresses)), config.reorgThresholdReadyTolerance, logger);
 }
 
 function makeFromConfig(chainConfig, config, registrationsByChainId, knownHeight) {
@@ -239,7 +251,7 @@ function knownHeight(cs) {
 }
 
 function contractAddresses(cs, contractName) {
-  return IndexingAddresses.getContractAddresses(cs.indexingAddresses, contractName);
+  return cs.addressStore.contractAddresses(contractName);
 }
 
 function bufferSize(cs) {
@@ -348,9 +360,7 @@ function frontierProgress(cs) {
   }
 }
 
-function getNextQuery(cs, chainTargetItems, chunkItemsMultiplierOpt, itemsTargetFloorOpt, maxTargetBlock) {
-  let chunkItemsMultiplier = chunkItemsMultiplierOpt !== undefined ? chunkItemsMultiplierOpt : 1;
-  let itemsTargetFloor = itemsTargetFloorOpt !== undefined ? itemsTargetFloorOpt : 0;
+function getNextQuery(cs, chainTargetItems, maxTargetBlock) {
   let chainTargetBlock = targetBlock(cs, chainTargetItems);
   let chainTargetBlock$1 = maxTargetBlock !== undefined ? Primitive_int.min(chainTargetBlock, maxTargetBlock) : chainTargetBlock;
   let density = effectiveDensity(cs);
@@ -361,7 +371,7 @@ function getNextQuery(cs, chainTargetItems, chunkItemsMultiplierOpt, itemsTarget
   } else {
     chainTargetItems$1 = chainTargetItems;
   }
-  return FetchState.getNextQuery(cs.fetchState, chainTargetBlock$1, chainTargetItems$1, chunkItemsMultiplier, itemsTargetFloor);
+  return FetchState.getNextQuery(cs.fetchState, chainTargetBlock$1, chainTargetItems$1);
 }
 
 function dispatch(cs, executeQuery, waitForNewBlock, onNewBlock, action, stateId) {
@@ -532,10 +542,6 @@ async function materializePageItems(items, transactionStore, blockStore, ecosyst
   ]);
 }
 
-function filterByClientAddress(cs, items) {
-  return FetchState.filterByClientAddress(items, cs.indexingAddresses);
-}
-
 function handleQueryResult(cs, query, newItems, newItemsWithDcs, latestFetchedBlock, knownHeight, txPage, blockPage) {
   if (txPage !== undefined) {
     cs.transactionStore.merge(Primitive_option.valFromOption(txPage));
@@ -543,7 +549,7 @@ function handleQueryResult(cs, query, newItems, newItemsWithDcs, latestFetchedBl
   if (blockPage !== undefined) {
     cs.blockStore.merge(Primitive_option.valFromOption(blockPage));
   }
-  let fs = newItemsWithDcs.length !== 0 ? FetchState.registerDynamicContracts(cs.fetchState, cs.indexingAddresses, newItemsWithDcs) : cs.fetchState;
+  let fs = newItemsWithDcs.length !== 0 ? FetchState.registerDynamicContracts(cs.fetchState, cs.addressStore, Primitive_int.max(knownHeight, latestFetchedBlock.blockNumber), newItemsWithDcs) : cs.fetchState;
   cs.fetchState = FetchState.updateKnownHeight(FetchState.handleQueryResult(fs, query, latestFetchedBlock, newItems), knownHeight);
   cs.pendingBudget = Primitive_float.max(0, cs.pendingBudget - query.itemsEst);
 }
@@ -578,7 +584,6 @@ function setEndBlockToFirstEvent(cs, blockNumber) {
       startBlock: init.startBlock,
       endBlock: blockNumber,
       normalSelection: init.normalSelection,
-      contractConfigs: init.contractConfigs,
       chainId: init.chainId,
       latestOnBlockBlockNumber: init.latestOnBlockBlockNumber,
       blockLag: init.blockLag,
@@ -586,7 +591,8 @@ function setEndBlockToFirstEvent(cs, blockNumber) {
       maxOnBlockBufferSize: init.maxOnBlockBufferSize,
       onBlockRegistrations: init.onBlockRegistrations,
       knownHeight: init.knownHeight,
-      firstEventBlock: init.firstEventBlock
+      firstEventBlock: init.firstEventBlock,
+      clientFilterAddressThreshold: init.clientFilterAddressThreshold
     };
     return;
   }
@@ -596,7 +602,6 @@ function setEndBlockToFirstEvent(cs, blockNumber) {
     startBlock: init$1.startBlock,
     endBlock: blockNumber,
     normalSelection: init$1.normalSelection,
-    contractConfigs: init$1.contractConfigs,
     chainId: init$1.chainId,
     latestOnBlockBlockNumber: init$1.latestOnBlockBlockNumber,
     blockLag: init$1.blockLag,
@@ -604,7 +609,8 @@ function setEndBlockToFirstEvent(cs, blockNumber) {
     maxOnBlockBufferSize: init$1.maxOnBlockBufferSize,
     onBlockRegistrations: init$1.onBlockRegistrations,
     knownHeight: init$1.knownHeight,
-    firstEventBlock: init$1.firstEventBlock
+    firstEventBlock: init$1.firstEventBlock,
+    clientFilterAddressThreshold: init$1.clientFilterAddressThreshold
   };
 }
 
@@ -634,7 +640,7 @@ function toMetrics(cs) {
     numBatchesFetched: 0,
     startBlock: cs.fetchState.startBlock,
     endBlock: cs.fetchState.endBlock,
-    numAddresses: IndexingAddresses.size(cs.indexingAddresses),
+    numAddresses: cs.addressStore.size(),
     isReady: cs.timestampCaughtUpToHeadOrEndblock !== undefined,
     sourceBlockNumber: cs.fetchState.knownHeight,
     progressBlockNumber: cs.committedProgressBlockNumber,
@@ -704,7 +710,6 @@ function applyBatchProgress(cs, batch, blockTimestampName) {
         startBlock: init.startBlock,
         endBlock: init.endBlock,
         normalSelection: init.normalSelection,
-        contractConfigs: init.contractConfigs,
         chainId: init.chainId,
         latestOnBlockBlockNumber: init.latestOnBlockBlockNumber,
         blockLag: init.blockLag,
@@ -712,7 +717,8 @@ function applyBatchProgress(cs, batch, blockTimestampName) {
         maxOnBlockBufferSize: init.maxOnBlockBufferSize,
         onBlockRegistrations: init.onBlockRegistrations,
         knownHeight: init.knownHeight,
-        firstEventBlock: firstEventBlock
+        firstEventBlock: firstEventBlock,
+        clientFilterAddressThreshold: init.clientFilterAddressThreshold
       };
     }
   }
@@ -762,7 +768,7 @@ function rollback(cs, newProgressBlockNumber, eventsProcessedDiff, rollbackTarge
   if (newProgressBlockNumber === undefined) {
     if (isReorgChain) {
       cs.reorgDetection = ReorgDetection.rollbackToValidBlockNumber(cs.reorgDetection, rollbackTargetBlockNumber);
-      cs.fetchState = FetchState.rollback(cs.fetchState, cs.indexingAddresses, rollbackTargetBlockNumber);
+      cs.fetchState = FetchState.rollback(cs.fetchState, cs.addressStore, rollbackTargetBlockNumber);
       cs.transactionStore.rollback(rollbackTargetBlockNumber);
       cs.blockStore.rollback(rollbackTargetBlockNumber);
       cs.committedProgressBlockNumber = Primitive_int.min(cs.committedProgressBlockNumber, rollbackTargetBlockNumber);
@@ -780,7 +786,7 @@ function rollback(cs, newProgressBlockNumber, eventsProcessedDiff, rollbackTarge
   if (safeCheckpointTracking !== undefined) {
     cs.safeCheckpointTracking = SafeCheckpointTracking.rollback(safeCheckpointTracking, newProgressBlockNumber);
   }
-  cs.fetchState = FetchState.rollback(cs.fetchState, cs.indexingAddresses, newProgressBlockNumber);
+  cs.fetchState = FetchState.rollback(cs.fetchState, cs.addressStore, newProgressBlockNumber);
   cs.transactionStore.rollback(newProgressBlockNumber);
   cs.blockStore.rollback(newProgressBlockNumber);
   cs.committedProgressBlockNumber = newProgressBlockNumber;
@@ -830,7 +836,6 @@ export {
   isReady,
   isFetchingAtHead,
   isAtHeadWithoutEndBlock,
-  filterByClientAddress,
   handleQueryResult,
   materializeBatchItems,
   materializePageItems,

package/src/ChainState.resi

@@ -9,7 +9,7 @@ let make: (
   ~chainConfig: Config.chain,
   ~fetchState: FetchState.t,
   ~onEventRegistrations: array<Internal.onEventRegistration>=?,
-  ~indexingAddresses: IndexingAddresses.t,
+  ~addressStore: AddressStore.t,
   ~sourceManager: SourceManager.t,
   ~reorgDetection: ReorgDetection.t,
   ~committedProgressBlockNumber: int,
@@ -80,13 +80,7 @@ let hasReadyItem: t => bool
 let targetBlock: (t, ~chainTargetItems: float) => int
 let blockAtProgress: (t, ~progress: float) => int
 let frontierProgress: t => float
-let getNextQuery: (
-  t,
-  ~chainTargetItems: float,
-  ~chunkItemsMultiplier: float=?,
-  ~itemsTargetFloor: int=?,
-  ~maxTargetBlock: int=?,
-) => FetchState.nextQuery
+let getNextQuery: (t, ~chainTargetItems: float, ~maxTargetBlock: int=?) => FetchState.nextQuery
 let dispatch: (
   t,
   ~executeQuery: FetchState.query => promise<unit>,
@@ -110,10 +104,6 @@ let isReady: t => bool
 let isFetchingAtHead: t => bool
 let isAtHeadWithoutEndBlock: t => bool
 
-// Drop over-fetched events an address-param filter rejects, before contract
-// registration runs on them. Uses the chain's current indexing addresses.
-let filterByClientAddress: (t, array<Internal.item>) => array<Internal.item>
-
 // State transitions. The chain state is mutated only through these.
 let handleQueryResult: (
   t,

package/src/Change.res

@@ -1,9 +1,9 @@
 @tag("type")
 type t<'entity> =
-  | @as("SET") Set({entityId: string, entity: 'entity, checkpointId: bigint})
-  | @as("DELETE") Delete({entityId: string, checkpointId: bigint})
+  | @as("SET") Set({entityId: EntityId.t, entity: 'entity, checkpointId: bigint})
+  | @as("DELETE") Delete({entityId: EntityId.t, checkpointId: bigint})
 
 @get
-external getEntityId: t<'entity> => string = "entityId"
+external getEntityId: t<'entity> => EntityId.t = "entityId"
 @get
 external getCheckpointId: t<'entity> => bigint = "checkpointId"