envio 3.4.0 → 3.5.0-alpha.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "envio",
-  "version": "3.4.0",
+  "version": "3.5.0-alpha.1",
   "type": "module",
   "description": "A latency and sync speed optimized, developer friendly blockchain data indexer.",
   "bin": "./bin.mjs",
@@ -69,10 +69,10 @@
     "tsx": "4.21.0"
   },
   "optionalDependencies": {
-    "envio-linux-x64": "3.4.0",
-    "envio-linux-x64-musl": "3.4.0",
-    "envio-linux-arm64": "3.4.0",
-    "envio-darwin-x64": "3.4.0",
-    "envio-darwin-arm64": "3.4.0"
+    "envio-linux-x64": "3.5.0-alpha.1",
+    "envio-linux-x64-musl": "3.5.0-alpha.1",
+    "envio-linux-arm64": "3.5.0-alpha.1",
+    "envio-darwin-x64": "3.5.0-alpha.1",
+    "envio-darwin-arm64": "3.5.0-alpha.1"
   }
 }

package/src/ChainState.res

@@ -187,6 +187,12 @@ let makeInternal = (
     ),
     ~onBlockRegistrations,
     ~firstEventBlock,
+    // Only EVM sources (HyperSync + RPC) honor client-side address filtering so
+    // far, so switching contracts to it is gated to EVM chains.
+    ~clientFilterAddressThreshold=switch config.ecosystem.name {
+    | Evm => Some(config.clientFilterAddressThreshold)
+    | Fuel | Svm => None
+    },
   )
 
   let chainReorgCheckpoints = reorgCheckpoints->Array.filterMap(reorgCheckpoint => {
@@ -541,13 +547,7 @@ let frontierProgress = (cs: t) =>
 // maxTargetBlock set to the most-behind chain's progress mapped onto this
 // chain, so a chain with budget can't run further ahead than the chain the
 // whole pool is prioritizing.
-let getNextQuery = (
-  cs: t,
-  ~chainTargetItems: float,
-  ~chunkItemsMultiplier=1.,
-  ~itemsTargetFloor=0,
-  ~maxTargetBlock=?,
-) => {
+let getNextQuery = (cs: t, ~chainTargetItems: float, ~maxTargetBlock=?) => {
   let chainTargetBlock = cs->targetBlock(~chainTargetItems)
   let chainTargetBlock = switch maxTargetBlock {
   | Some(maxTargetBlock) => Pervasives.min(chainTargetBlock, maxTargetBlock)
@@ -561,10 +561,6 @@ let getNextQuery = (
   // of being held by an oversized probe.
   let chainTargetItems = switch cs->effectiveDensity {
   | Some(density) if density > 0. =>
-    // No extra headroom here: the budget is reserved in honest itemsEst units,
-    // and truncation safety lives in the itemsTarget server cap (sized with
-    // chunkItemsMultiplier at query creation) — multiplying the budget cap too
-    // would compound the two and hold budget away from other chains.
     let rangeCost =
       density *. (chainTargetBlock - cs.fetchState->FetchState.bufferBlockNumber)->Int.toFloat
     Pervasives.min(chainTargetItems, Math.ceil(rangeCost) +. cs.pendingBudget)
@@ -572,12 +568,7 @@ let getNextQuery = (
   // budget to the cold-chain cap, so it's used as-is.
   | _ => chainTargetItems
   }
-  cs.fetchState->FetchState.getNextQuery(
-    ~chainTargetBlock,
-    ~chainTargetItems,
-    ~chunkItemsMultiplier,
-    ~itemsTargetFloor,
-  )
+  cs.fetchState->FetchState.getNextQuery(~chainTargetBlock, ~chainTargetItems)
 }
 
 // Run a fetch tick for this chain against its sources, feeding the owned fetch

package/src/ChainState.res.mjs

@@ -105,7 +105,18 @@ function makeInternal(chainConfig, indexingAddresses, startBlock, endBlock, firs
   });
   let contractConfigs = IndexingAddresses.makeContractConfigs(onEventRegistrations);
   let indexingAddressIndex = IndexingAddresses.make(contractConfigs, indexingAddresses);
-  let fetchState = FetchState.make(startBlock, endBlock, onEventRegistrations, contractConfigs, indexingAddresses, config.maxAddrInPartition, chainConfig.id, (config.batchSize << 1), knownHeight, progressBlockNumber, match.onBlockRegistrations, Primitive_int.max(!config.shouldRollbackOnReorg || isInReorgThreshold ? 0 : chainConfig.maxReorgDepth, chainConfig.blockLag), Primitive_option.some(firstEventBlock));
+  let match$1 = config.ecosystem.name;
+  let tmp;
+  switch (match$1) {
+    case "evm" :
+      tmp = config.clientFilterAddressThreshold;
+      break;
+    case "fuel" :
+    case "svm" :
+      tmp = undefined;
+      break;
+  }
+  let fetchState = FetchState.make(startBlock, endBlock, onEventRegistrations, contractConfigs, indexingAddresses, config.maxAddrInPartition, chainConfig.id, (config.batchSize << 1), knownHeight, progressBlockNumber, match.onBlockRegistrations, Primitive_int.max(!config.shouldRollbackOnReorg || isInReorgThreshold ? 0 : chainConfig.maxReorgDepth, chainConfig.blockLag), Primitive_option.some(firstEventBlock), Primitive_option.some(tmp));
   let chainReorgCheckpoints = Stdlib_Array.filterMap(reorgCheckpoints, reorgCheckpoint => {
     if (reorgCheckpoint.chain_id === chainConfig.id) {
       return reorgCheckpoint;
@@ -348,9 +359,7 @@ function frontierProgress(cs) {
   }
 }
 
-function getNextQuery(cs, chainTargetItems, chunkItemsMultiplierOpt, itemsTargetFloorOpt, maxTargetBlock) {
-  let chunkItemsMultiplier = chunkItemsMultiplierOpt !== undefined ? chunkItemsMultiplierOpt : 1;
-  let itemsTargetFloor = itemsTargetFloorOpt !== undefined ? itemsTargetFloorOpt : 0;
+function getNextQuery(cs, chainTargetItems, maxTargetBlock) {
   let chainTargetBlock = targetBlock(cs, chainTargetItems);
   let chainTargetBlock$1 = maxTargetBlock !== undefined ? Primitive_int.min(chainTargetBlock, maxTargetBlock) : chainTargetBlock;
   let density = effectiveDensity(cs);
@@ -361,7 +370,7 @@ function getNextQuery(cs, chainTargetItems, chunkItemsMultiplierOpt, itemsTarget
   } else {
     chainTargetItems$1 = chainTargetItems;
   }
-  return FetchState.getNextQuery(cs.fetchState, chainTargetBlock$1, chainTargetItems$1, chunkItemsMultiplier, itemsTargetFloor);
+  return FetchState.getNextQuery(cs.fetchState, chainTargetBlock$1, chainTargetItems$1);
 }
 
 function dispatch(cs, executeQuery, waitForNewBlock, onNewBlock, action, stateId) {
@@ -586,7 +595,8 @@ function setEndBlockToFirstEvent(cs, blockNumber) {
       maxOnBlockBufferSize: init.maxOnBlockBufferSize,
       onBlockRegistrations: init.onBlockRegistrations,
       knownHeight: init.knownHeight,
-      firstEventBlock: init.firstEventBlock
+      firstEventBlock: init.firstEventBlock,
+      clientFilterAddressThreshold: init.clientFilterAddressThreshold
     };
     return;
   }
@@ -604,7 +614,8 @@ function setEndBlockToFirstEvent(cs, blockNumber) {
     maxOnBlockBufferSize: init$1.maxOnBlockBufferSize,
     onBlockRegistrations: init$1.onBlockRegistrations,
     knownHeight: init$1.knownHeight,
-    firstEventBlock: init$1.firstEventBlock
+    firstEventBlock: init$1.firstEventBlock,
+    clientFilterAddressThreshold: init$1.clientFilterAddressThreshold
   };
 }
 
@@ -712,7 +723,8 @@ function applyBatchProgress(cs, batch, blockTimestampName) {
         maxOnBlockBufferSize: init.maxOnBlockBufferSize,
         onBlockRegistrations: init.onBlockRegistrations,
         knownHeight: init.knownHeight,
-        firstEventBlock: firstEventBlock
+        firstEventBlock: firstEventBlock,
+        clientFilterAddressThreshold: init.clientFilterAddressThreshold
       };
     }
   }

package/src/ChainState.resi

@@ -80,13 +80,7 @@ let hasReadyItem: t => bool
 let targetBlock: (t, ~chainTargetItems: float) => int
 let blockAtProgress: (t, ~progress: float) => int
 let frontierProgress: t => float
-let getNextQuery: (
-  t,
-  ~chainTargetItems: float,
-  ~chunkItemsMultiplier: float=?,
-  ~itemsTargetFloor: int=?,
-  ~maxTargetBlock: int=?,
-) => FetchState.nextQuery
+let getNextQuery: (t, ~chainTargetItems: float, ~maxTargetBlock: int=?) => FetchState.nextQuery
 let dispatch: (
   t,
   ~executeQuery: FetchState.query => promise<unit>,

package/src/Config.res

@@ -78,6 +78,9 @@ type t = {
   ecosystem: Ecosystem.t,
   enableRawEvents: bool,
   maxAddrInPartition: int,
+  // Per-contract registered-address count past which a dynamic contract switches
+  // to client-side filtering. Overridable in tests.
+  clientFilterAddressThreshold: int,
   batchSize: int,
   // Slack (in blocks) below the lagged head within which a chain still counts as
   // ready to enter the reorg threshold, absorbing head advances between catch-up
@@ -995,10 +998,14 @@ let fromPublic = (publicConfigJson: JSON.t) => {
 
   let allEntities = userEntities->Array.concat([EnvioAddresses.entityConfig])
 
+  // Keyed by the capitalized entity name to match the handler-context
+  // accessor (`context.Pool_snapshots`) the generated types expose, while
+  // entityConfig.name stays the original schema name used for the physical
+  // Postgres/ClickHouse tables.
   let userEntitiesByName =
     userEntities
     ->Array.map(entityConfig => {
-      (entityConfig.name, entityConfig)
+      (entityConfig.name->Utils.String.capitalize, entityConfig)
     })
     ->Dict.fromArray
 
@@ -1029,6 +1036,7 @@ let fromPublic = (publicConfigJson: JSON.t) => {
     enableRawEvents: publicConfig["rawEvents"]->Option.getOr(false),
     ecosystem,
     maxAddrInPartition,
+    clientFilterAddressThreshold: Env.clientFilterAddressThreshold,
     batchSize: publicConfig["fullBatchSize"]->Option.getOr(5000),
     reorgThresholdReadyTolerance: 100,
     lowercaseAddresses,

package/src/Config.res.mjs

@@ -704,7 +704,7 @@ function fromPublic(publicConfigJson) {
   let userEntities = parseEntitiesFromJson(Stdlib_Option.getOr(publicConfig.entities, []), enumConfigsByName, globalStorage);
   let allEntities = userEntities.concat([entityConfig]);
   let userEntitiesByName = Object.fromEntries(userEntities.map(entityConfig => [
-    entityConfig.name,
+    Utils.$$String.capitalize(entityConfig.name),
     entityConfig
   ]));
   let contractHandlers = publicContractsConfig !== undefined ? Object.entries(publicContractsConfig).map(param => ({
@@ -724,6 +724,7 @@ function fromPublic(publicConfigJson) {
     ecosystem: ecosystem,
     enableRawEvents: Stdlib_Option.getOr(publicConfig.rawEvents, false),
     maxAddrInPartition: Env.maxAddrInPartition,
+    clientFilterAddressThreshold: Env.clientFilterAddressThreshold,
     batchSize: Stdlib_Option.getOr(publicConfig.fullBatchSize, 5000),
     reorgThresholdReadyTolerance: 100,
     lowercaseAddresses: lowercaseAddresses,

package/src/CrossChainState.res

@@ -249,19 +249,6 @@ let checkAndFetch = async (
   // while it takes its first measurements. Its probe is one admission unit.
   let coldChainBudget = minimumAdmissionBudget
 
-  // Chunk reservations get headroom over the density estimate so a
-  // denser-than-expected range doesn't truncate at the server cap; realtime
-  // gets more since a forced catch-up query there costs a head-poll roundtrip.
-  let chunkItemsMultiplier = crossChainState.isRealtime ? 3. : 1.5
-
-  // Server-cap floor for bounded queries: their block range is already the
-  // hard bound on the response, so a low density estimate shrinking the cap
-  // below this only buys self-truncated responses. Splitting the target pool
-  // across a chain's concurrency slots keeps the worst case — every in-flight
-  // bounded query returning a full floored response at once — at ~one buffer
-  // target.
-  let itemsTargetFloor = crossChainState.targetBufferSize / FetchState.maxChainConcurrency
-
   let prioritizedChainStates = crossChainState->priorityOrder
 
   // Alignment anchor: the first known-height chain in priority order — which,
@@ -308,12 +295,7 @@ let checkAndFetch = async (
         Some(cs->ChainState.blockAtProgress(~progress=progress +. 0.1))
       | _ => None
       }
-      switch cs->ChainState.getNextQuery(
-        ~chainTargetItems,
-        ~chunkItemsMultiplier,
-        ~itemsTargetFloor,
-        ~maxTargetBlock?,
-      ) {
+      switch cs->ChainState.getNextQuery(~chainTargetItems, ~maxTargetBlock?) {
       | WaitingForNewBlock as action => actionByChain->Utils.Dict.setByInt(chainId, action)
       | NothingToQuery =>
         // A chain below its head can emit no query when its budget went to
@@ -333,7 +315,7 @@ let checkAndFetch = async (
               {
                 "fromBlock": query.fromBlock,
                 "targetBlock": query.toBlock,
-                "targetEvents": query.itemsTarget,
+                "targetEvents": query.itemsEst,
               },
             )
           )

package/src/CrossChainState.res.mjs

@@ -6,9 +6,7 @@ import * as Utils from "./Utils.res.mjs";
 import * as Logging from "./Logging.res.mjs";
 import * as ChainMap from "./ChainMap.res.mjs";
 import * as ChainState from "./ChainState.res.mjs";
-import * as FetchState from "./FetchState.res.mjs";
 import * as Stdlib_Array from "@rescript/runtime/lib/es6/Stdlib_Array.js";
-import * as Primitive_int from "@rescript/runtime/lib/es6/Primitive_int.js";
 import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
 import * as Primitive_float from "@rescript/runtime/lib/es6/Primitive_float.js";
 import * as SafeCheckpointTracking from "./SafeCheckpointTracking.res.mjs";
@@ -147,8 +145,6 @@ async function checkAndFetch(crossChainState, dispatchChain) {
     contents: Primitive_float.max(0, targetBudget - totalReadyCount(crossChainState) - totalReservedSize(crossChainState))
   };
   let minimumAdmissionBudget = targetBudget * 0.1;
-  let chunkItemsMultiplier = crossChainState.isRealtime ? 3 : 1.5;
-  let itemsTargetFloor = Primitive_int.div(crossChainState.targetBufferSize, FetchState.maxChainConcurrency);
   let prioritizedChainStates = priorityOrder(crossChainState);
   let alignment = crossChainState.isRealtime ? undefined : Stdlib_Option.map(prioritizedChainStates.find(cs => ChainState.knownHeight(cs) !== 0), cs => [
       ChainState.chainConfig(cs).id,
@@ -170,7 +166,7 @@ async function checkAndFetch(crossChainState, dispatchChain) {
       isCold ? Primitive_float.min(remaining.contents, minimumAdmissionBudget) : remaining.contents
     ) + ChainState.pendingBudget(cs);
     let maxTargetBlock = alignment !== undefined && alignment[0] !== chainId ? ChainState.blockAtProgress(cs, alignment[1] + 0.1) : undefined;
-    let action = ChainState.getNextQuery(cs, chainTargetItems, chunkItemsMultiplier, itemsTargetFloor, maxTargetBlock);
+    let action = ChainState.getNextQuery(cs, chainTargetItems, maxTargetBlock);
     if (typeof action !== "object") {
       if (action === "WaitingForNewBlock") {
         actionByChain[chainId] = action;
@@ -186,7 +182,7 @@ async function checkAndFetch(crossChainState, dispatchChain) {
         partitions[query.partitionId] = {
           fromBlock: query.fromBlock,
           targetBlock: query.toBlock,
-          targetEvents: query.itemsTarget
+          targetEvents: query.itemsEst
         };
       });
       Logging.trace({

package/src/Env.res

@@ -9,6 +9,22 @@ let updateSyncTimeOnRestart =
 let targetBufferSize = envSafe->EnvSafe.get("ENVIO_INDEXING_MAX_BUFFER_SIZE", S.option(S.int))
 let maxAddrInPartition = envSafe->EnvSafe.get("MAX_PARTITION_SIZE", S.int, ~fallback=5_000)
 
+// Most parallel in-flight queries a single chain may have at once, across all
+// its partitions (consumed as FetchState.maxChainConcurrency).
+let maxChainConcurrency = 100
+
+// Switch a single contract to client-side address filtering
+// once its registered address count crosses this threshold. Keeping addresses
+// server-side spreads the contract across ceil(count / maxAddrInPartition)
+// partitions, each holding an in-flight query slot; capping a contract at half
+// the chain's concurrency budget stops one busy contract from monopolising them.
+let clientFilterAddressThreshold =
+  envSafe->EnvSafe.get(
+    "ENVIO_CLIENT_FILTER_ADDRESS_THRESHOLD",
+    S.int,
+    ~fallback=maxAddrInPartition * maxChainConcurrency / 2,
+  )
+
 // Target number of in-memory objects (uncommitted entity/effect changes plus
 // unwritten batch items) the store holds before processing waits for the write
 // cycle to catch up.

package/src/Env.res.mjs

@@ -19,6 +19,8 @@ let targetBufferSize = EnvSafe.get(envSafe, "ENVIO_INDEXING_MAX_BUFFER_SIZE", S$
 
 let maxAddrInPartition = EnvSafe.get(envSafe, "MAX_PARTITION_SIZE", S$RescriptSchema.int, undefined, 5000, undefined, undefined);
 
+let clientFilterAddressThreshold = EnvSafe.get(envSafe, "ENVIO_CLIENT_FILTER_ADDRESS_THRESHOLD", S$RescriptSchema.int, undefined, (maxAddrInPartition * 100 | 0) / 2 | 0, undefined, undefined);
+
 let inMemoryObjectsTarget = EnvSafe.get(envSafe, "ENVIO_IN_MEMORY_OBJECTS_TARGET", S$RescriptSchema.int, undefined, 100000, undefined, undefined);
 
 let serverPort = EnvSafe.get(envSafe, "ENVIO_INDEXER_PORT", S$RescriptSchema.port(S$RescriptSchema.int, undefined), undefined, EnvSafe.get(envSafe, "METRICS_PORT", S$RescriptSchema.port(S$RescriptSchema.int, undefined), undefined, 9898, undefined, undefined), undefined, undefined);
@@ -216,10 +218,14 @@ let ThrottleWrites = {
 
 EnvSafe.close(envSafe);
 
+let maxChainConcurrency = 100;
+
 export {
   updateSyncTimeOnRestart,
   targetBufferSize,
   maxAddrInPartition,
+  maxChainConcurrency,
+  clientFilterAddressThreshold,
   inMemoryObjectsTarget,
   serverPort,
   tuiEnvVar,