envio 3.2.0 → 3.3.0-alpha.0

package/src/CrossChainState.resi

@@ -0,0 +1,47 @@
+// Whole-indexer view over every chain's ChainState. `t` is opaque: other modules
+// read it through the accessors and change it only through the transitions here.
+
+type t
+
+let calculateTargetBufferSize: unit => int
+
+let make: (
+  ~chainStates: dict<ChainState.t>,
+  ~isInReorgThreshold: bool,
+  ~isRealtime: bool,
+  ~maxBackfillConcurrency: int=?,
+  ~maxRealtimeConcurrency: int=?,
+  ~targetBufferSize: int=?,
+) => t
+
+// Accessors.
+let chainStates: t => dict<ChainState.t>
+let isRealtime: t => bool
+let isInReorgThreshold: t => bool
+let maxConcurrency: t => int
+let inFlight: t => int
+
+// Derived (pure).
+let nextItemIsNone: t => bool
+let getSafeCheckpointId: t => option<Internal.checkpointId>
+
+// Cross-chain transitions.
+let createBatch: (
+  t,
+  ~processedCheckpointId: Internal.checkpointId,
+  ~batchSizeTarget: int,
+  ~isRollback: bool,
+) => Batch.t
+let enterReorgThreshold: t => unit
+let applyBatchProgress: (t, ~batch: Batch.t) => unit
+
+// Fetch control.
+let priorityOrder: t => array<ChainState.t>
+let checkAndFetch: (
+  t,
+  ~fetchChain: (
+    ~chain: ChainMap.Chain.t,
+    ~concurrencyLimit: int,
+    ~bufferLimit: int,
+  ) => promise<unit>,
+) => promise<unit>

package/src/Ecosystem.res

@@ -29,8 +29,66 @@ type t = {
       `LogSelection.res`. SVM does not support event handlers, so its
       schema always surfaces `None`. */
   onEventBlockFilterSchema: S.t<option<unknown>>,
+  /** Base logger injected at construction. Used to build per-item child
+      loggers (see `getItemLogger`). */
+  logger: Pino.t,
+  /** Materialise the user-facing event handed to handlers and contract
+      registration from an item's opaque payload. */
+  toEvent: Internal.eventItem => Internal.event,
+  /** Build the per-item child logger for an event item, with
+      ecosystem-specific log fields (EVM/Fuel: contract/event/address; SVM:
+      program/instruction/programId). Closes over the injected logger. */
+  toEventLogger: Internal.eventItem => Pino.t,
+  /** Build a raw event row for the `raw_events` table. Unsupported on SVM,
+      where the implementation throws. */
+  toRawEvent: Internal.eventItem => Internal.rawEvent,
 }
 
+// The materialised event and the child logger are both memoised on the item
+// object (an item is processed across preload + execution passes, and logged
+// from several places). Hidden keys mirror each other.
+let getItemEvent = {
+  let cacheKey = "_event"
+  (item: Internal.item, ~ecosystem: t): Internal.event => {
+    let cache = item->(Utils.magic: Internal.item => dict<Internal.event>)
+    switch cache->Utils.Dict.dangerouslyGetNonOption(cacheKey) {
+    | Some(event) => event
+    | None =>
+      let event = ecosystem.toEvent(item->Internal.castUnsafeEventItem)
+      cache->Dict.set(cacheKey, event)
+      event
+    }
+  }
+}
+
+let getItemLogger = {
+  let cacheKey = "_logger"
+  (item: Internal.item, ~ecosystem: t): Pino.t => {
+    let cache = item->(Utils.magic: Internal.item => dict<Pino.t>)
+    switch cache->Utils.Dict.dangerouslyGetNonOption(cacheKey) {
+    | Some(logger) => logger
+    | None =>
+      let logger = switch item {
+      | Internal.Event(_) => ecosystem.toEventLogger(item->Internal.castUnsafeEventItem)
+      | Block({blockNumber, onBlockConfig}) =>
+        Logging.createChildFrom(
+          ~logger=ecosystem.logger,
+          ~params={
+            "onBlock": onBlockConfig.name,
+            "chainId": onBlockConfig.chainId,
+            "block": blockNumber,
+          },
+        )
+      }
+      cache->Dict.set(cacheKey, logger)
+      logger
+    }
+  }
+}
+
+let getItemUserLogger = (item: Internal.item, ~ecosystem: t): Envio.logger =>
+  getItemLogger(item, ~ecosystem)->Logging.userLogger
+
 let makeOnBlockArgs = (~blockNumber: int, ~ecosystem: t, ~context): Internal.onBlockArgs => {
   switch ecosystem.name {
   | Svm => {slot: blockNumber, context}

package/src/Ecosystem.res.mjs

@@ -1,7 +1,46 @@
 // Generated by ReScript, PLEASE EDIT WITH CARE
 
+import * as Logging from "./Logging.res.mjs";
 import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js";
 
+let cacheKey = "_event";
+
+function getItemEvent(item, ecosystem) {
+  let event = item[cacheKey];
+  if (event !== undefined) {
+    return Primitive_option.valFromOption(event);
+  }
+  let event$1 = ecosystem.toEvent(item);
+  item[cacheKey] = event$1;
+  return event$1;
+}
+
+let cacheKey$1 = "_logger";
+
+function getItemLogger(item, ecosystem) {
+  let logger = item[cacheKey$1];
+  if (logger !== undefined) {
+    return logger;
+  }
+  let logger$1;
+  if (item.kind === 0) {
+    logger$1 = ecosystem.toEventLogger(item);
+  } else {
+    let onBlockConfig = item.onBlockConfig;
+    logger$1 = Logging.createChildFrom(ecosystem.logger, {
+      onBlock: onBlockConfig.name,
+      chainId: onBlockConfig.chainId,
+      block: item.blockNumber
+    });
+  }
+  item[cacheKey$1] = logger$1;
+  return logger$1;
+}
+
+function getItemUserLogger(item, ecosystem) {
+  return Logging.userLogger(getItemLogger(item, ecosystem));
+}
+
 function makeOnBlockArgs(blockNumber, ecosystem, context) {
   let match = ecosystem.name;
   switch (match) {
@@ -23,6 +62,9 @@ function makeOnBlockArgs(blockNumber, ecosystem, context) {
 }
 
 export {
+  getItemEvent,
+  getItemLogger,
+  getItemUserLogger,
   makeOnBlockArgs,
 }
-/* No side effect */
+/* Logging Not a pure module */

package/src/Env.res

@@ -8,8 +8,13 @@ let updateSyncTimeOnRestart =
   envSafe->EnvSafe.get("UPDATE_SYNC_TIME_ON_RESTART", S.bool, ~fallback=true)
 let targetBufferSize = envSafe->EnvSafe.get("ENVIO_INDEXING_MAX_BUFFER_SIZE", S.option(S.int))
 let maxAddrInPartition = envSafe->EnvSafe.get("MAX_PARTITION_SIZE", S.int, ~fallback=5_000)
-let maxPartitionConcurrency =
-  envSafe->EnvSafe.get("ENVIO_MAX_PARTITION_CONCURRENCY", S.int, ~fallback=10)
+// Global caps on concurrent data-source queries across the whole indexer (all
+// chains share the budget), not per chain. Realtime allows far more since head
+// queries are small and many chains follow the head at once.
+let maxBackfillConcurrency =
+  envSafe->EnvSafe.get("ENVIO_MAX_BACKFILL_CONCURRENCY", S.int, ~fallback=30)
+let maxRealtimeConcurrency =
+  envSafe->EnvSafe.get("ENVIO_MAX_REALTIME_CONCURRENCY", S.int, ~fallback=200)
 
 // Target number of in-memory objects (uncommitted entity/effect changes plus
 // unwritten batch items) the store holds before processing waits for the write

package/src/Env.res.mjs

@@ -19,7 +19,9 @@ let targetBufferSize = EnvSafe.get(envSafe, "ENVIO_INDEXING_MAX_BUFFER_SIZE", S$
 
 let maxAddrInPartition = EnvSafe.get(envSafe, "MAX_PARTITION_SIZE", S$RescriptSchema.int, undefined, 5000, undefined, undefined);
 
-let maxPartitionConcurrency = EnvSafe.get(envSafe, "ENVIO_MAX_PARTITION_CONCURRENCY", S$RescriptSchema.int, undefined, 10, undefined, undefined);
+let maxBackfillConcurrency = EnvSafe.get(envSafe, "ENVIO_MAX_BACKFILL_CONCURRENCY", S$RescriptSchema.int, undefined, 30, undefined, undefined);
+
+let maxRealtimeConcurrency = EnvSafe.get(envSafe, "ENVIO_MAX_REALTIME_CONCURRENCY", S$RescriptSchema.int, undefined, 200, undefined, undefined);
 
 let inMemoryObjectsTarget = EnvSafe.get(envSafe, "ENVIO_IN_MEMORY_OBJECTS_TARGET", S$RescriptSchema.int, undefined, 100000, undefined, undefined);
 
@@ -222,7 +224,8 @@ export {
   updateSyncTimeOnRestart,
   targetBufferSize,
   maxAddrInPartition,
-  maxPartitionConcurrency,
+  maxBackfillConcurrency,
+  maxRealtimeConcurrency,
   inMemoryObjectsTarget,
   serverPort,
   tuiEnvVar,

package/src/EventConfigBuilder.res

@@ -309,6 +309,39 @@ let resolveFieldSelection = (
   (selectedBlockFields, selectedTransactionFields)
 }
 
+// ============== Client-side address filter ==============
+
+let compileAddressFilter: string => (
+  Internal.eventPayload,
+  int,
+  dict<Internal.indexingContract>,
+) => bool = %raw(`function (body) {
+  return new Function("event", "blockNumber", "indexingAddresses", body);
+}`)
+
+// Body of the client-side address filter for a DNF of address-filtered param
+// names (OR of AND-groups): keep the event only if some group's params are all
+// registered at or before the log's block. The DNF is fixed here, so it's
+// unrolled into one boolean expression — no per-event closure, loop, or array.
+// `None` when there's no address-param filter. Exposed for snapshotting.
+let buildAddressFilterBody = (groups: array<array<string>>): option<string> => {
+  switch groups {
+  | [] => None
+  | _ =>
+    let leaf = name =>
+      `(ic = indexingAddresses[p[${JSON.stringify(
+          JSON.String(name),
+        )}]]) !== undefined && ic.effectiveStartBlock <= blockNumber`
+    let groupExprs =
+      groups->Array.map(group => "(" ++ group->Array.map(leaf)->Array.join(" && ") ++ ")")
+    Some("var p = event.params, ic; return " ++ groupExprs->Array.join(" || ") ++ ";")
+  }
+}
+
+let buildAddressFilter = (groups: array<array<string>>): option<
+  (Internal.eventPayload, int, dict<Internal.indexingContract>) => bool,
+> => buildAddressFilterBody(groups)->Option.map(compileAddressFilter)
+
 // ============== Build complete EVM event config ==============
 
 let buildEvmEventConfig = (
@@ -334,6 +367,7 @@ let buildEvmEventConfig = (
   let {
     getEventFiltersOrThrow,
     filterByAddresses,
+    addressFilterParamGroups,
     startBlock: whereStartBlock,
   } = LogSelection.parseEventFiltersOrThrow(
     ~eventFilters,
@@ -374,6 +408,7 @@ let buildEvmEventConfig = (
     simulateParamsSchema: buildSimulateParamsSchema(params),
     getEventFiltersOrThrow,
     filterByAddresses,
+    clientAddressFilter: ?buildAddressFilter(addressFilterParamGroups),
     dependsOnAddresses: !isWildcard || filterByAddresses,
     startBlock: resolvedStartBlock,
     selectedBlockFields,

package/src/EventConfigBuilder.res.mjs

@@ -251,6 +251,23 @@ function resolveFieldSelection(blockFields, transactionFields, globalBlockFields
   ];
 }
 
+let compileAddressFilter = (function (body) {
+  return new Function("event", "blockNumber", "indexingAddresses", body);
+});
+
+function buildAddressFilterBody(groups) {
+  if (groups.length === 0) {
+    return;
+  }
+  let leaf = name => `(ic = indexingAddresses[p[` + JSON.stringify(name) + `]]) !== undefined && ic.effectiveStartBlock <= blockNumber`;
+  let groupExprs = groups.map(group => "(" + group.map(leaf).join(" && ") + ")");
+  return "var p = event.params, ic; return " + groupExprs.join(" || ") + ";";
+}
+
+function buildAddressFilter(groups) {
+  return Stdlib_Option.map(buildAddressFilterBody(groups), compileAddressFilter);
+}
+
 function buildEvmEventConfig(contractName, eventName, sighash, params, isWildcard, handler, contractRegister, eventFilters, probeChainId, onEventBlockFilterSchema, blockFields, transactionFields, startBlock, globalBlockFieldsSetOpt, globalTransactionFieldsSetOpt) {
   let globalBlockFieldsSet = globalBlockFieldsSetOpt !== undefined ? Primitive_option.valFromOption(globalBlockFieldsSetOpt) : new Set();
   let globalTransactionFieldsSet = globalTransactionFieldsSetOpt !== undefined ? Primitive_option.valFromOption(globalTransactionFieldsSetOpt) : new Set();
@@ -274,6 +291,7 @@ function buildEvmEventConfig(contractName, eventName, sighash, params, isWildcar
     isWildcard: isWildcard,
     filterByAddresses: filterByAddresses,
     dependsOnAddresses: !isWildcard || filterByAddresses,
+    clientAddressFilter: Stdlib_Option.map(buildAddressFilterBody(match.addressFilterParamGroups), compileAddressFilter),
     handler: handler,
     contractRegister: contractRegister,
     paramsRawEventSchema: buildParamsSchema(params),
@@ -393,6 +411,9 @@ export {
   buildTopicGetter,
   alwaysIncludedBlockFields,
   resolveFieldSelection,
+  compileAddressFilter,
+  buildAddressFilterBody,
+  buildAddressFilter,
   buildEvmEventConfig,
   buildSvmInstructionEventConfig,
   buildFuelEventConfig,

package/src/EventProcessing.res

@@ -1,18 +1,17 @@
-let allChainsEventsProcessedToEndblock = (chainFetchers: ChainMap.t<ChainFetcher.t>) => {
-  chainFetchers
-  ->ChainMap.values
-  ->Array.every(cf => cf->ChainFetcher.hasProcessedToEndblock)
+let allChainsEventsProcessedToEndblock = (chainStates: dict<ChainState.t>) => {
+  chainStates
+  ->Dict.valuesToArray
+  ->Array.every(cs => cs->ChainState.hasProcessedToEndblock)
 }
 
-let computeChainsState = (chainFetchers: ChainMap.t<ChainFetcher.t>): Internal.chains => {
+let computeChainsState = (chainStates: dict<ChainState.t>): Internal.chains => {
   let chains = Dict.make()
 
-  let isRealtime = chainFetchers->ChainMap.values->Array.every(cf => cf->ChainFetcher.isReady)
+  let values = chainStates->Dict.valuesToArray
+  let isRealtime = values->Array.every(cs => cs->ChainState.isReady)
 
-  chainFetchers
-  ->ChainMap.keys
-  ->Array.forEach(chain => {
-    let chainId = chain->ChainMap.Chain.toChainId
+  values->Array.forEach(cs => {
+    let chainId = (cs->ChainState.chainConfig).id
     chains->Dict.set(
       chainId->Int.toString,
       {
@@ -31,7 +30,7 @@ let runEventHandlerOrThrow = async (
   item: Internal.item,
   ~checkpointId,
   ~handler,
-  ~inMemoryStore,
+  ~indexerState,
   ~loadManager,
   ~persistence,
   ~chains: Internal.chains,
@@ -46,7 +45,7 @@ let runEventHandlerOrThrow = async (
     let contextParams: UserContext.contextParams = {
       item,
       checkpointId,
-      inMemoryStore,
+      indexerState,
       loadManager,
       persistence,
       isPreload: false,
@@ -57,7 +56,7 @@ let runEventHandlerOrThrow = async (
     await handler(
       (
         {
-          event: eventItem.event,
+          event: item->Ecosystem.getItemEvent(~ecosystem=config.ecosystem),
           context: UserContext.getHandlerContext(contextParams),
         }: Internal.handlerArgs
       ),
@@ -84,9 +83,10 @@ let runEventHandlerOrThrow = async (
 let runHandlerOrThrow = async (
   item: Internal.item,
   ~checkpointId,
-  ~inMemoryStore,
+  ~indexerState,
   ~loadManager,
-  ~ctx: Ctx.t,
+  ~persistence: Persistence.t,
+  ~config: Config.t,
   ~chains: Internal.chains,
 ) => {
   switch item {
@@ -94,19 +94,19 @@ let runHandlerOrThrow = async (
     try {
       let contextParams: UserContext.contextParams = {
         item,
-        inMemoryStore,
+        indexerState,
         loadManager,
-        persistence: ctx.persistence,
+        persistence,
         checkpointId,
         isPreload: false,
         chains,
-        config: ctx.config,
+        config,
         isResolved: false,
       }
       await handler(
         Ecosystem.makeOnBlockArgs(
           ~blockNumber,
-          ~ecosystem=ctx.config.ecosystem,
+          ~ecosystem=config.ecosystem,
           ~context=UserContext.getHandlerContext(contextParams),
         ),
       )
@@ -121,16 +121,17 @@ let runHandlerOrThrow = async (
         }),
       )
     }
-  | Event({eventConfig}) => switch eventConfig.handler {
+  | Event({eventConfig}) =>
+    switch eventConfig.handler {
     | Some(handler) =>
       await item->runEventHandlerOrThrow(
         ~handler,
         ~checkpointId,
-        ~inMemoryStore,
+        ~indexerState,
         ~loadManager,
-        ~persistence=ctx.persistence,
+        ~persistence,
         ~chains,
-        ~config=ctx.config,
+        ~config,
       )
     | None => ()
     }
@@ -142,7 +143,7 @@ let preloadBatchOrThrow = async (
   ~loadManager,
   ~persistence,
   ~config: Config.t,
-  ~inMemoryStore,
+  ~indexerState,
   ~chains: Internal.chains,
 ) => {
   // On the first run of loaders, we don't care about the result,
@@ -160,7 +161,7 @@ let preloadBatchOrThrow = async (
     for idx in 0 to checkpointEventsProcessed - 1 {
       let item = batch.items->Array.getUnsafe(itemIdx.contents + idx)
       switch item {
-      | Event({eventConfig: {handler, contractName, name: eventName}, event}) =>
+      | Event({eventConfig: {handler, contractName, name: eventName}}) =>
         switch handler {
         | None => ()
         | Some(handler) =>
@@ -171,10 +172,10 @@ let preloadBatchOrThrow = async (
             )
             promises->Array.push(
               handler({
-                event,
+                event: item->Ecosystem.getItemEvent(~ecosystem=config.ecosystem),
                 context: UserContext.getHandlerContext({
                   item,
-                  inMemoryStore,
+                  indexerState,
                   loadManager,
                   persistence,
                   checkpointId,
@@ -208,7 +209,7 @@ let preloadBatchOrThrow = async (
                 ~ecosystem=config.ecosystem,
                 ~context=UserContext.getHandlerContext({
                   item,
-                  inMemoryStore,
+                  indexerState,
                   loadManager,
                   persistence,
                   checkpointId,
@@ -234,9 +235,10 @@ let preloadBatchOrThrow = async (
 
 let runBatchHandlersOrThrow = async (
   batch: Batch.t,
-  ~inMemoryStore,
+  ~indexerState,
   ~loadManager,
-  ~ctx,
+  ~persistence,
+  ~config,
   ~chains: Internal.chains,
 ) => {
   let itemIdx = ref(0)
@@ -248,7 +250,15 @@ let runBatchHandlersOrThrow = async (
     for idx in 0 to checkpointEventsProcessed - 1 {
       let item = batch.items->Array.getUnsafe(itemIdx.contents + idx)
 
-      await runHandlerOrThrow(item, ~checkpointId, ~inMemoryStore, ~loadManager, ~ctx, ~chains)
+      await runHandlerOrThrow(
+        item,
+        ~checkpointId,
+        ~indexerState,
+        ~loadManager,
+        ~persistence,
+        ~config,
+        ~chains,
+      )
     }
     itemIdx := itemIdx.contents + checkpointEventsProcessed
   }
@@ -273,14 +283,15 @@ type logPartitionInfo = {
 
 let processEventBatch = async (
   ~batch: Batch.t,
-  ~inMemoryStore: InMemoryStore.t,
+  ~indexerState: IndexerState.t,
   ~loadManager,
-  ~ctx: Ctx.t,
-  ~chainFetchers: ChainMap.t<ChainFetcher.t>,
+  ~persistence: Persistence.t,
+  ~config: Config.t,
+  ~chainStates: dict<ChainState.t>,
 ) => {
   let totalBatchSize = batch.totalBatchSize
   // Compute chains state for this batch
-  let chains: Internal.chains = chainFetchers->computeChainsState
+  let chains: Internal.chains = chainStates->computeChainsState
 
   let logger = Logging.getLogger()
   logger->Logging.childTrace({
@@ -296,29 +307,29 @@ let processEventBatch = async (
 
   try {
     // Backpressure: keep processing within keepLatestChangesLimit of the cycle.
-    await inMemoryStore->InMemoryStore.awaitCapacity
+    await indexerState->Writing.awaitCapacity
 
     let timeRef = Hrtime.makeTimer()
 
     if batch.items->Utils.Array.notEmpty {
-      await batch->preloadBatchOrThrow(
-        ~loadManager,
-        ~persistence=ctx.persistence,
-        ~inMemoryStore,
-        ~chains,
-        ~config=ctx.config,
-      )
+      await batch->preloadBatchOrThrow(~loadManager, ~persistence, ~indexerState, ~chains, ~config)
     }
 
     let elapsedTimeAfterLoaders = timeRef->Hrtime.timeSince->Hrtime.toSecondsFloat
 
     if batch.items->Utils.Array.notEmpty {
-      await batch->runBatchHandlersOrThrow(~inMemoryStore, ~loadManager, ~ctx, ~chains)
+      await batch->runBatchHandlersOrThrow(
+        ~indexerState,
+        ~loadManager,
+        ~persistence,
+        ~config,
+        ~chains,
+      )
     }
 
     let elapsedTimeAfterProcessing = timeRef->Hrtime.timeSince->Hrtime.toSecondsFloat
 
-    inMemoryStore->InMemoryStore.commitBatch(~batch)
+    indexerState->Writing.commitBatch(~batch)
 
     let loaderDuration = elapsedTimeAfterLoaders
     let handlerDuration = elapsedTimeAfterProcessing -. loaderDuration
@@ -329,7 +340,10 @@ let processEventBatch = async (
     reason->ErrorHandling.make(~msg=message, ~logger)->Error
   | ProcessingError({message, exn, item}) =>
     exn
-    ->ErrorHandling.make(~msg=message, ~logger=item->Logging.getItemLogger)
+    ->ErrorHandling.make(
+      ~msg=message,
+      ~logger=Ecosystem.getItemLogger(item, ~ecosystem=config.ecosystem),
+    )
     ->Error
   | exn => exn->ErrorHandling.make(~msg="Failed processing batch", ~logger)->Error
   }