envio 3.1.0-rc.1 → 3.1.0

package/index.d.ts

@@ -1201,6 +1201,18 @@ export type IndexerFromConfig<Config extends IndexerConfigTypes = GlobalConfig>
     readonly name: string;
     /** The indexer description from config.yaml. */
     readonly description: string | undefined;
+    /**
+     * Internal, unstable API that will be removed without notice. Registers a
+     * callback fired once per chain affected by a reorg rollback, after the
+     * rollback is durably written to the database. A throwing callback crashes
+     * the indexer through the same path as a failed write.
+     */
+    readonly "~internalAndWillBeRemovedSoon_onRollbackCommit": (
+      callback: (args: {
+        readonly chainId: number;
+        readonly rollbackToBlock: number;
+      }) => Promise<void>,
+    ) => void;
   } & SingleEcosystemChains<Config>
 >;
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "envio",
-  "version": "3.1.0-rc.1",
+  "version": "3.1.0",
   "type": "module",
   "description": "A latency and sync speed optimized, developer friendly blockchain data indexer.",
   "bin": "./bin.mjs",
@@ -70,10 +70,10 @@
     "tsx": "4.21.0"
   },
   "optionalDependencies": {
-    "envio-linux-x64": "3.1.0-rc.1",
-    "envio-linux-x64-musl": "3.1.0-rc.1",
-    "envio-linux-arm64": "3.1.0-rc.1",
-    "envio-darwin-x64": "3.1.0-rc.1",
-    "envio-darwin-arm64": "3.1.0-rc.1"
+    "envio-linux-x64": "3.1.0",
+    "envio-linux-x64-musl": "3.1.0",
+    "envio-linux-arm64": "3.1.0",
+    "envio-darwin-x64": "3.1.0",
+    "envio-darwin-arm64": "3.1.0"
   }
 }

package/src/Batch.res

@@ -23,6 +23,9 @@ type t = {
   totalBatchSize: int,
   items: array<Internal.item>,
   progressedChainsById: dict<chainAfterBatch>,
+  // Processed inside the reorg threshold. Drives whether history is saved, so
+  // writes never merge across a change in this value.
+  isInReorgThreshold: bool,
   // Unnest-like checkpoint fields:
   checkpointIds: array<bigint>,
   checkpointChainIds: array<int>,
@@ -165,6 +168,7 @@ let prepareBatch = (
   ~checkpointIdBeforeBatch,
   ~chainsBeforeBatch: ChainMap.t<chainBeforeBatch>,
   ~batchSizeTarget,
+  ~isInReorgThreshold,
 ) => {
   let preparedFetchStates =
     chainsBeforeBatch
@@ -286,6 +290,7 @@ let prepareBatch = (
       ~batchSizePerChain=mutBatchSizePerChain,
       ~progressBlockNumberPerChain=mutProgressBlockNumberPerChain,
     ),
+    isInReorgThreshold,
     checkpointIds,
     checkpointChainIds,
     checkpointBlockNumbers,
@@ -298,8 +303,9 @@ let make = (
   ~checkpointIdBeforeBatch,
   ~chainsBeforeBatch: ChainMap.t<chainBeforeBatch>,
   ~batchSizeTarget,
+  ~isInReorgThreshold,
 ) => {
-  prepareBatch(~checkpointIdBeforeBatch, ~chainsBeforeBatch, ~batchSizeTarget)
+  prepareBatch(~checkpointIdBeforeBatch, ~chainsBeforeBatch, ~batchSizeTarget, ~isInReorgThreshold)
 }
 
 let findFirstEventBlockNumber = (batch: t, ~chainId) => {

package/src/Batch.res.mjs

@@ -70,7 +70,7 @@ function addReorgCheckpoints(prevCheckpointId, reorgDetection, fromBlockExclusiv
   return prevCheckpointId$1;
 }
 
-function prepareBatch(checkpointIdBeforeBatch, chainsBeforeBatch, batchSizeTarget) {
+function prepareBatch(checkpointIdBeforeBatch, chainsBeforeBatch, batchSizeTarget, isInReorgThreshold) {
   let preparedFetchStates = FetchState.sortForUnorderedBatch(ChainMap.values(chainsBeforeBatch).map(chainBeforeBatch => chainBeforeBatch.fetchState), batchSizeTarget);
   let chainIdx = 0;
   let preparedNumber = preparedFetchStates.length;
@@ -168,6 +168,7 @@ function prepareBatch(checkpointIdBeforeBatch, chainsBeforeBatch, batchSizeTarge
     totalBatchSize: totalBatchSize,
     items: items,
     progressedChainsById: getProgressedChainsById(chainsBeforeBatch, mutBatchSizePerChain, mutProgressBlockNumberPerChain),
+    isInReorgThreshold: isInReorgThreshold,
     checkpointIds: checkpointIds,
     checkpointChainIds: checkpointChainIds,
     checkpointBlockNumbers: checkpointBlockNumbers,

package/src/ChainManager.res

@@ -135,12 +135,13 @@ let nextItemIsNone = (chainManager: t): bool => {
 
 let createBatch = (
   chainManager: t,
-  ~committedCheckpointId,
+  ~processedCheckpointId,
   ~batchSizeTarget: int,
   ~isRollback: bool,
 ): Batch.t => {
   Batch.make(
-    ~checkpointIdBeforeBatch=committedCheckpointId->BigInt.add(
+    ~isInReorgThreshold=chainManager.isInReorgThreshold,
+    ~checkpointIdBeforeBatch=processedCheckpointId->BigInt.add(
       // Since for rollback we have a diff checkpoint id.
       // This is needed to currectly overwrite old state
       // in an append-only ClickHouse insert.

package/src/ChainManager.res.mjs

@@ -99,8 +99,8 @@ function nextItemIsNone(chainManager) {
   return !Batch.hasReadyItem(ChainMap.map(chainManager.chainFetchers, cf => cf.fetchState));
 }
 
-function createBatch(chainManager, committedCheckpointId, batchSizeTarget, isRollback) {
-  return Batch.make(committedCheckpointId + (
+function createBatch(chainManager, processedCheckpointId, batchSizeTarget, isRollback) {
+  return Batch.make(processedCheckpointId + (
     isRollback ? 1n : 0n
   ), ChainMap.map(chainManager.chainFetchers, cf => ({
     fetchState: cf.fetchState,
@@ -109,7 +109,7 @@ function createBatch(chainManager, committedCheckpointId, batchSizeTarget, isRol
     sourceBlockNumber: cf.fetchState.knownHeight,
     totalEventsProcessed: cf.numEventsProcessed,
     chainConfig: cf.chainConfig
-  })), batchSizeTarget);
+  })), batchSizeTarget, chainManager.isInReorgThreshold);
 }
 
 function isProgressAtHead(chainManager) {

package/src/Env.res

@@ -11,6 +11,12 @@ let maxAddrInPartition = envSafe->EnvSafe.get("MAX_PARTITION_SIZE", S.int, ~fall
 let maxPartitionConcurrency =
   envSafe->EnvSafe.get("ENVIO_MAX_PARTITION_CONCURRENCY", S.int, ~fallback=10)
 
+// Target number of in-memory objects (uncommitted entity/effect changes plus
+// unwritten batch items) the store holds before processing waits for the write
+// cycle to catch up.
+let inMemoryObjectsTarget =
+  envSafe->EnvSafe.get("ENVIO_IN_MEMORY_OBJECTS_TARGET", S.int, ~fallback=100_000)->Belt.Int.toFloat
+
 // FIXME: This broke HS grafana dashboard. Should investigate it later. Maybe we should use :: as a default value?
 // We want to be able to set it to 0.0.0.0
 // to allow to passthrough the port from a Docker container

package/src/Env.res.mjs

@@ -21,6 +21,8 @@ let maxAddrInPartition = EnvSafe.get(envSafe, "MAX_PARTITION_SIZE", S$RescriptSc
 
 let maxPartitionConcurrency = EnvSafe.get(envSafe, "ENVIO_MAX_PARTITION_CONCURRENCY", S$RescriptSchema.int, undefined, 10, undefined, undefined);
 
+let inMemoryObjectsTarget = EnvSafe.get(envSafe, "ENVIO_IN_MEMORY_OBJECTS_TARGET", S$RescriptSchema.int, undefined, 100000, undefined, undefined);
+
 let serverPort = EnvSafe.get(envSafe, "ENVIO_INDEXER_PORT", S$RescriptSchema.port(S$RescriptSchema.int, undefined), undefined, EnvSafe.get(envSafe, "METRICS_PORT", S$RescriptSchema.port(S$RescriptSchema.int, undefined), undefined, 9898, undefined, undefined), undefined, undefined);
 
 let tuiEnvVar = EnvSafe.get(envSafe, "ENVIO_TUI", S$RescriptSchema.option(S$RescriptSchema.bool), undefined, undefined, undefined, undefined);
@@ -221,6 +223,7 @@ export {
   targetBufferSize,
   maxAddrInPartition,
   maxPartitionConcurrency,
+  inMemoryObjectsTarget,
   serverPort,
   tuiEnvVar,
   logLevelSchema,

package/src/EventProcessing.res

@@ -25,77 +25,6 @@ let computeChainsState = (chainFetchers: ChainMap.t<ChainFetcher.t>): Internal.c
   chains
 }
 
-let convertFieldsToJson = (fields: option<dict<unknown>>) => {
-  switch fields {
-  | None => %raw(`{}`)
-  | Some(fields) =>
-    // Convert bigint fields to string. There are no fields with nested
-    // bigints, so iterating only the top level is safe.
-    fields
-    ->Utils.Dict.mapValues(value =>
-      typeof(value) === #bigint
-        ? value
-          ->(Utils.magic: unknown => bigint)
-          ->BigInt.toString
-          ->(Utils.magic: string => unknown)
-        : value
-    )
-    ->(Utils.magic: dict<unknown> => JSON.t)
-  }
-}
-
-let addItemToRawEvents = (
-  eventItem: Internal.eventItem,
-  ~inMemoryStore: InMemoryStore.t,
-  ~config: Config.t,
-) => {
-  let {event, eventConfig, chain, blockNumber, timestamp: blockTimestamp} = eventItem
-  let {block, transaction, params, logIndex, srcAddress} = event
-  let chainId = chain->ChainMap.Chain.toChainId
-  let eventId = EventUtils.packEventIndex(~logIndex, ~blockNumber)
-  let blockFields =
-    block
-    ->(Utils.magic: Internal.eventBlock => option<dict<unknown>>)
-    ->convertFieldsToJson
-  let transactionFields =
-    transaction
-    ->(Utils.magic: Internal.eventTransaction => option<dict<unknown>>)
-    ->convertFieldsToJson
-
-  blockFields->config.ecosystem.cleanUpRawEventFieldsInPlace
-
-  // Serialize to unknown, because serializing to Js.Json.t fails for Bytes Fuel type, since it has unknown schema
-  let params =
-    params
-    ->S.reverseConvertOrThrow(eventConfig.paramsRawEventSchema)
-    ->(Utils.magic: unknown => JSON.t)
-  let params = if params === %raw(`null`) {
-    // Should probably make the params field nullable
-    // But this is currently needed to make events
-    // with empty params work
-    %raw(`"null"`)
-  } else {
-    params
-  }
-
-  let rawEvent: InternalTable.RawEvents.t = {
-    chainId,
-    eventId,
-    eventName: eventConfig.name,
-    contractName: eventConfig.contractName,
-    blockNumber,
-    logIndex,
-    srcAddress,
-    blockHash: block->config.ecosystem.getId,
-    blockTimestamp,
-    blockFields,
-    transactionFields,
-    params,
-  }
-
-  inMemoryStore.rawEvents->Array.push(rawEvent)
-}
-
 exception ProcessingError({message: string, exn: exn, item: Internal.item})
 
 let runEventHandlerOrThrow = async (
@@ -192,26 +121,18 @@ let runHandlerOrThrow = async (
         }),
       )
     }
-  | Event({eventConfig}) => {
-      switch eventConfig.handler {
-      | Some(handler) =>
-        await item->runEventHandlerOrThrow(
-          ~handler,
-          ~checkpointId,
-          ~inMemoryStore,
-          ~loadManager,
-          ~persistence=ctx.persistence,
-          ~chains,
-          ~config=ctx.config,
-        )
-      | None => ()
-      }
-
-      if ctx.config.enableRawEvents {
-        item
-        ->Internal.castUnsafeEventItem
-        ->addItemToRawEvents(~inMemoryStore, ~config=ctx.config)
-      }
+  | Event({eventConfig}) => switch eventConfig.handler {
+    | Some(handler) =>
+      await item->runEventHandlerOrThrow(
+        ~handler,
+        ~checkpointId,
+        ~inMemoryStore,
+        ~loadManager,
+        ~persistence=ctx.persistence,
+        ~chains,
+        ~config=ctx.config,
+      )
+    | None => ()
     }
   }
 }
@@ -333,17 +254,11 @@ let runBatchHandlersOrThrow = async (
   }
 }
 
-let registerProcessEventBatchMetrics = (
-  ~logger,
-  ~loadDuration,
-  ~handlerDuration,
-  ~dbWriteDuration,
-) => {
+let registerProcessEventBatchMetrics = (~logger, ~loadDuration, ~handlerDuration) => {
   logger->Logging.childTrace({
     "msg": "Finished processing batch",
     "loader_time_elapsed": loadDuration,
     "handlers_time_elapsed": handlerDuration,
-    "write_time_elapsed": dbWriteDuration,
   })
 
   Prometheus.ProcessingBatch.registerMetrics(~loadDuration, ~handlerDuration)
@@ -359,7 +274,6 @@ type logPartitionInfo = {
 let processEventBatch = async (
   ~batch: Batch.t,
   ~inMemoryStore: InMemoryStore.t,
-  ~isInReorgThreshold,
   ~loadManager,
   ~ctx: Ctx.t,
   ~chainFetchers: ChainMap.t<ChainFetcher.t>,
@@ -381,6 +295,9 @@ let processEventBatch = async (
   })
 
   try {
+    // Backpressure: keep processing within keepLatestChangesLimit of the cycle.
+    await inMemoryStore->InMemoryStore.awaitCapacity
+
     let timeRef = Hrtime.makeTimer()
 
     if batch.items->Utils.Array.notEmpty {
@@ -401,34 +318,19 @@ let processEventBatch = async (
 
     let elapsedTimeAfterProcessing = timeRef->Hrtime.timeSince->Hrtime.toSecondsFloat
 
-    try {
-      await inMemoryStore->InMemoryStore.writeBatch(
-        ~persistence=ctx.persistence,
-        ~batch,
-        ~config=ctx.config,
-        ~isInReorgThreshold,
-      )
+    inMemoryStore->InMemoryStore.commitBatch(~batch)
 
-      let elapsedTimeAfterDbWrite = timeRef->Hrtime.timeSince->Hrtime.toSecondsFloat
-      let loaderDuration = elapsedTimeAfterLoaders
-      let handlerDuration = elapsedTimeAfterProcessing -. loaderDuration
-      let dbWriteDuration = elapsedTimeAfterDbWrite -. elapsedTimeAfterProcessing
-      registerProcessEventBatchMetrics(
-        ~logger,
-        ~loadDuration=loaderDuration,
-        ~handlerDuration,
-        ~dbWriteDuration,
-      )
-      Ok()
-    } catch {
-    | Persistence.StorageError({message, reason}) =>
-      reason->ErrorHandling.make(~msg=message, ~logger)->Error
-    | exn => exn->ErrorHandling.make(~msg="Failed writing batch to database", ~logger)->Error
-    }
+    let loaderDuration = elapsedTimeAfterLoaders
+    let handlerDuration = elapsedTimeAfterProcessing -. loaderDuration
+    registerProcessEventBatchMetrics(~logger, ~loadDuration=loaderDuration, ~handlerDuration)
+    Ok()
   } catch {
+  | Persistence.StorageError({message, reason}) =>
+    reason->ErrorHandling.make(~msg=message, ~logger)->Error
   | ProcessingError({message, exn, item}) =>
     exn
     ->ErrorHandling.make(~msg=message, ~logger=item->Logging.getItemLogger)
     ->Error
+  | exn => exn->ErrorHandling.make(~msg="Failed processing batch", ~logger)->Error
   }
 }

package/src/EventProcessing.res.mjs

@@ -5,14 +5,12 @@ import * as Hrtime from "./bindings/Hrtime.res.mjs";
 import * as Logging from "./Logging.res.mjs";
 import * as ChainMap from "./ChainMap.res.mjs";
 import * as Ecosystem from "./Ecosystem.res.mjs";
-import * as EventUtils from "./EventUtils.res.mjs";
 import * as Prometheus from "./Prometheus.res.mjs";
 import * as Persistence from "./Persistence.res.mjs";
 import * as UserContext from "./UserContext.res.mjs";
 import * as ChainFetcher from "./ChainFetcher.res.mjs";
 import * as ErrorHandling from "./ErrorHandling.res.mjs";
 import * as InMemoryStore from "./InMemoryStore.res.mjs";
-import * as S$RescriptSchema from "rescript-schema/src/S.res.mjs";
 import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
 
 function allChainsEventsProcessedToEndblock(chainFetchers) {
@@ -31,55 +29,6 @@ function computeChainsState(chainFetchers) {
   return chains;
 }
 
-function convertFieldsToJson(fields) {
-  if (fields !== undefined) {
-    return Utils.Dict.mapValues(fields, value => {
-      if (typeof value === "bigint") {
-        return value.toString();
-      } else {
-        return value;
-      }
-    });
-  } else {
-    return {};
-  }
-}
-
-function addItemToRawEvents(eventItem, inMemoryStore, config) {
-  let event = eventItem.event;
-  let block = event.block;
-  let logIndex = event.logIndex;
-  let blockNumber = eventItem.blockNumber;
-  let eventConfig = eventItem.eventConfig;
-  let eventId = EventUtils.packEventIndex(blockNumber, logIndex);
-  let blockFields = convertFieldsToJson(block);
-  let transactionFields = convertFieldsToJson(event.transaction);
-  config.ecosystem.cleanUpRawEventFieldsInPlace(blockFields);
-  let params = S$RescriptSchema.reverseConvertOrThrow(event.params, eventConfig.paramsRawEventSchema);
-  let params$1 = params === null ? "null" : params;
-  let rawEvent_chain_id = eventItem.chain;
-  let rawEvent_event_name = eventConfig.name;
-  let rawEvent_contract_name = eventConfig.contractName;
-  let rawEvent_src_address = event.srcAddress;
-  let rawEvent_block_hash = config.ecosystem.getId(block);
-  let rawEvent_block_timestamp = eventItem.timestamp;
-  let rawEvent = {
-    chain_id: rawEvent_chain_id,
-    event_id: eventId,
-    event_name: rawEvent_event_name,
-    contract_name: rawEvent_contract_name,
-    block_number: blockNumber,
-    log_index: logIndex,
-    src_address: rawEvent_src_address,
-    block_hash: rawEvent_block_hash,
-    block_timestamp: rawEvent_block_timestamp,
-    block_fields: blockFields,
-    transaction_fields: transactionFields,
-    params: params$1
-  };
-  inMemoryStore.rawEvents.push(rawEvent);
-}
-
 let ProcessingError = /* @__PURE__ */Primitive_exceptions.create("EventProcessing.ProcessingError");
 
 async function runEventHandlerOrThrow(item, checkpointId, handler, inMemoryStore, loadManager, persistence, chains, config) {
@@ -119,10 +68,7 @@ async function runHandlerOrThrow(item, checkpointId, inMemoryStore, loadManager,
   if (item.kind === 0) {
     let handler = item.eventConfig.handler;
     if (handler !== undefined) {
-      await runEventHandlerOrThrow(item, checkpointId, handler, inMemoryStore, loadManager, ctx.persistence, chains, ctx.config);
-    }
-    if (ctx.config.enableRawEvents) {
-      return addItemToRawEvents(item, inMemoryStore, ctx.config);
+      return await runEventHandlerOrThrow(item, checkpointId, handler, inMemoryStore, loadManager, ctx.persistence, chains, ctx.config);
     } else {
       return;
     }
@@ -224,17 +170,16 @@ async function runBatchHandlersOrThrow(batch, inMemoryStore, loadManager, ctx, c
   }
 }
 
-function registerProcessEventBatchMetrics(logger, loadDuration, handlerDuration, dbWriteDuration) {
+function registerProcessEventBatchMetrics(logger, loadDuration, handlerDuration) {
   Logging.childTrace(logger, {
     msg: "Finished processing batch",
     loader_time_elapsed: loadDuration,
-    handlers_time_elapsed: handlerDuration,
-    write_time_elapsed: dbWriteDuration
+    handlers_time_elapsed: handlerDuration
   });
   Prometheus.ProcessingBatch.registerMetrics(loadDuration, handlerDuration);
 }
 
-async function processEventBatch(batch, inMemoryStore, isInReorgThreshold, loadManager, ctx, chainFetchers) {
+async function processEventBatch(batch, inMemoryStore, loadManager, ctx, chainFetchers) {
   let totalBatchSize = batch.totalBatchSize;
   let chains = computeChainsState(chainFetchers);
   let logger = Logging.getLogger();
@@ -247,6 +192,7 @@ async function processEventBatch(batch, inMemoryStore, isInReorgThreshold, loadM
     }))
   });
   try {
+    await InMemoryStore.awaitCapacity(inMemoryStore);
     let timeRef = Hrtime.makeTimer();
     if (Utils.$$Array.notEmpty(batch.items)) {
       await preloadBatchOrThrow(batch, loadManager, ctx.persistence, ctx.config, inMemoryStore, chains);
@@ -256,47 +202,37 @@ async function processEventBatch(batch, inMemoryStore, isInReorgThreshold, loadM
       await runBatchHandlersOrThrow(batch, inMemoryStore, loadManager, ctx, chains);
     }
     let elapsedTimeAfterProcessing = Hrtime.toSecondsFloat(Hrtime.timeSince(timeRef));
-    try {
-      await InMemoryStore.writeBatch(inMemoryStore, ctx.persistence, batch, ctx.config, isInReorgThreshold);
-      let elapsedTimeAfterDbWrite = Hrtime.toSecondsFloat(Hrtime.timeSince(timeRef));
-      let handlerDuration = elapsedTimeAfterProcessing - elapsedTimeAfterLoaders;
-      let dbWriteDuration = elapsedTimeAfterDbWrite - elapsedTimeAfterProcessing;
-      registerProcessEventBatchMetrics(logger, elapsedTimeAfterLoaders, handlerDuration, dbWriteDuration);
+    InMemoryStore.commitBatch(inMemoryStore, batch);
+    let handlerDuration = elapsedTimeAfterProcessing - elapsedTimeAfterLoaders;
+    registerProcessEventBatchMetrics(logger, elapsedTimeAfterLoaders, handlerDuration);
+    return {
+      TAG: "Ok",
+      _0: undefined
+    };
+  } catch (raw_exn) {
+    let exn = Primitive_exceptions.internalToException(raw_exn);
+    if (exn.RE_EXN_ID === Persistence.StorageError) {
       return {
-        TAG: "Ok",
-        _0: undefined
+        TAG: "Error",
+        _0: ErrorHandling.make(exn.reason, logger, exn.message)
       };
-    } catch (raw_exn) {
-      let exn = Primitive_exceptions.internalToException(raw_exn);
-      if (exn.RE_EXN_ID === Persistence.StorageError) {
-        return {
-          TAG: "Error",
-          _0: ErrorHandling.make(exn.reason, logger, exn.message)
-        };
-      } else {
-        return {
-          TAG: "Error",
-          _0: ErrorHandling.make(exn, logger, "Failed writing batch to database")
-        };
-      }
-    }
-  } catch (raw_exn$1) {
-    let exn$1 = Primitive_exceptions.internalToException(raw_exn$1);
-    if (exn$1.RE_EXN_ID === ProcessingError) {
+    } else if (exn.RE_EXN_ID === ProcessingError) {
+      return {
+        TAG: "Error",
+        _0: ErrorHandling.make(exn.exn, Logging.getItemLogger(exn.item), exn.message)
+      };
+    } else {
       return {
         TAG: "Error",
-        _0: ErrorHandling.make(exn$1.exn, Logging.getItemLogger(exn$1.item), exn$1.message)
+        _0: ErrorHandling.make(exn, logger, "Failed processing batch")
       };
     }
-    throw exn$1;
   }
 }
 
 export {
   allChainsEventsProcessedToEndblock,
   computeChainsState,
-  convertFieldsToJson,
-  addItemToRawEvents,
   ProcessingError,
   runEventHandlerOrThrow,
   runHandlerOrThrow,