envio 3.0.0-alpha.2 → 3.0.0-alpha.4

package/src/EventProcessing.res.mjs

@@ -0,0 +1,341 @@
+// Generated by ReScript, PLEASE EDIT WITH CARE
+
+import * as Env from "./Env.res.mjs";
+import * as Utils from "./Utils.res.mjs";
+import * as Config from "./Config.res.mjs";
+import * as Hrtime from "./bindings/Hrtime.res.mjs";
+import * as Logging from "./Logging.res.mjs";
+import * as $$Promise from "./bindings/Promise.res.mjs";
+import * as ChainMap from "./ChainMap.res.mjs";
+import * as Benchmark from "./Benchmark.res.mjs";
+import * as Ecosystem from "./Ecosystem.res.mjs";
+import * as Belt_Array from "rescript/lib/es6/belt_Array.js";
+import * as EventUtils from "./EventUtils.res.mjs";
+import * as Prometheus from "./Prometheus.res.mjs";
+import * as Persistence from "./Persistence.res.mjs";
+import * as UserContext from "./UserContext.res.mjs";
+import * as ChainFetcher from "./ChainFetcher.res.mjs";
+import * as ErrorHandling from "./ErrorHandling.res.mjs";
+import * as InMemoryTable from "./InMemoryTable.res.mjs";
+import * as Caml_exceptions from "rescript/lib/es6/caml_exceptions.js";
+import * as S$RescriptSchema from "rescript-schema/src/S.res.mjs";
+import * as Caml_js_exceptions from "rescript/lib/es6/caml_js_exceptions.js";
+
+function allChainsEventsProcessedToEndblock(chainFetchers) {
+  return Belt_Array.every(ChainMap.values(chainFetchers), (function (cf) {
+                return ChainFetcher.hasProcessedToEndblock(cf);
+              }));
+}
+
+function computeChainsState(chainFetchers) {
+  var chains = {};
+  Belt_Array.forEach(ChainMap.entries(chainFetchers), (function (param) {
+          var chain = param[0];
+          var chainId = String(chain);
+          var isLive = param[1].timestampCaughtUpToHeadOrEndblock !== undefined;
+          chains[chainId] = {
+            id: chain,
+            isLive: isLive
+          };
+        }));
+  return chains;
+}
+
+function convertFieldsToJson(fields) {
+  if (fields === undefined) {
+    return {};
+  }
+  var keys = Object.keys(fields);
+  var $$new = {};
+  for(var i = 0 ,i_finish = keys.length; i < i_finish; ++i){
+    var key = keys[i];
+    var value = fields[key];
+    $$new[key] = typeof value === "bigint" ? value.toString() : value;
+  }
+  return $$new;
+}
+
+function addItemToRawEvents(eventItem, inMemoryStore, config) {
+  var $$event = eventItem.event;
+  var block = $$event.block;
+  var logIndex = $$event.logIndex;
+  var blockNumber = eventItem.blockNumber;
+  var chain = eventItem.chain;
+  var eventConfig = eventItem.eventConfig;
+  var eventId = EventUtils.packEventIndex(blockNumber, logIndex);
+  var blockFields = convertFieldsToJson(block);
+  var transactionFields = convertFieldsToJson($$event.transaction);
+  config.ecosystem.cleanUpRawEventFieldsInPlace(blockFields);
+  var params = S$RescriptSchema.reverseConvertOrThrow($$event.params, eventConfig.paramsRawEventSchema);
+  var params$1 = params === null ? "null" : params;
+  var rawEvent_event_name = eventConfig.name;
+  var rawEvent_contract_name = eventConfig.contractName;
+  var rawEvent_src_address = $$event.srcAddress;
+  var rawEvent_block_hash = config.ecosystem.getId(block);
+  var rawEvent_block_timestamp = eventItem.timestamp;
+  var rawEvent = {
+    chain_id: chain,
+    event_id: eventId,
+    event_name: rawEvent_event_name,
+    contract_name: rawEvent_contract_name,
+    block_number: blockNumber,
+    log_index: logIndex,
+    src_address: rawEvent_src_address,
+    block_hash: rawEvent_block_hash,
+    block_timestamp: rawEvent_block_timestamp,
+    block_fields: blockFields,
+    transaction_fields: transactionFields,
+    params: params$1
+  };
+  var eventIdStr = eventId.toString();
+  InMemoryTable.set(inMemoryStore.rawEvents, {
+        chainId: chain,
+        eventId: eventIdStr
+      }, rawEvent);
+}
+
+var ProcessingError = /* @__PURE__ */Caml_exceptions.create("EventProcessing.ProcessingError");
+
+async function runEventHandlerOrThrow(item, checkpointId, handler, inMemoryStore, loadManager, persistence, shouldSaveHistory, shouldBenchmark, chains, config) {
+  var timeBeforeHandler = Hrtime.makeTimer();
+  try {
+    var contextParams = {
+      item: item,
+      checkpointId: checkpointId,
+      inMemoryStore: inMemoryStore,
+      loadManager: loadManager,
+      persistence: persistence,
+      isPreload: false,
+      shouldSaveHistory: shouldSaveHistory,
+      chains: chains,
+      config: config,
+      isResolved: false
+    };
+    await handler({
+          event: item.event,
+          context: UserContext.getHandlerContext(contextParams)
+        });
+    contextParams.isResolved = true;
+  }
+  catch (raw_exn){
+    var exn = Caml_js_exceptions.internalToOCamlException(raw_exn);
+    throw {
+          RE_EXN_ID: ProcessingError,
+          message: "Unexpected error in the event handler. Please handle the error to keep the indexer running smoothly.",
+          exn: exn,
+          item: item,
+          Error: new Error()
+        };
+  }
+  if (!shouldBenchmark) {
+    return ;
+  }
+  var timeEnd = Hrtime.floatFromMillis(Hrtime.toMillis(Hrtime.timeSince(timeBeforeHandler)));
+  return Benchmark.addSummaryData("Handlers Per Event", item.eventConfig.contractName + " " + item.eventConfig.name + " Handler (ms)", timeEnd, 4);
+}
+
+async function runHandlerOrThrow(item, checkpointId, inMemoryStore, loadManager, indexer, shouldSaveHistory, shouldBenchmark, chains) {
+  if (item.kind === 0) {
+    var handler = item.eventConfig.handler;
+    if (handler !== undefined) {
+      await runEventHandlerOrThrow(item, checkpointId, handler, inMemoryStore, loadManager, indexer.persistence, shouldSaveHistory, shouldBenchmark, chains, indexer.config);
+    }
+    if (indexer.config.enableRawEvents) {
+      return addItemToRawEvents(item, inMemoryStore, indexer.config);
+    } else {
+      return ;
+    }
+  }
+  try {
+    var contextParams = {
+      item: item,
+      checkpointId: checkpointId,
+      inMemoryStore: inMemoryStore,
+      loadManager: loadManager,
+      persistence: indexer.persistence,
+      isPreload: false,
+      shouldSaveHistory: shouldSaveHistory,
+      chains: chains,
+      config: indexer.config,
+      isResolved: false
+    };
+    await item.onBlockConfig.handler(Ecosystem.makeOnBlockArgs(item.blockNumber, indexer.config.ecosystem, UserContext.getHandlerContext(contextParams)));
+    contextParams.isResolved = true;
+    return ;
+  }
+  catch (raw_exn){
+    var exn = Caml_js_exceptions.internalToOCamlException(raw_exn);
+    throw {
+          RE_EXN_ID: ProcessingError,
+          message: "Unexpected error in the block handler. Please handle the error to keep the indexer running smoothly.",
+          exn: exn,
+          item: item,
+          Error: new Error()
+        };
+  }
+}
+
+async function preloadBatchOrThrow(batch, loadManager, persistence, config, inMemoryStore, chains) {
+  var promises = [];
+  var itemIdx = 0;
+  for(var checkpointIdx = 0 ,checkpointIdx_finish = batch.checkpointIds.length; checkpointIdx < checkpointIdx_finish; ++checkpointIdx){
+    var checkpointId = batch.checkpointIds[checkpointIdx];
+    var checkpointEventsProcessed = batch.checkpointEventsProcessed[checkpointIdx];
+    for(var idx = 0; idx < checkpointEventsProcessed; ++idx){
+      var item = batch.items[itemIdx + idx | 0];
+      if (item.kind === 0) {
+        var handler = item.eventConfig.handler;
+        if (handler !== undefined) {
+          try {
+            promises.push($$Promise.silentCatch(handler({
+                          event: item.event,
+                          context: UserContext.getHandlerContext({
+                                item: item,
+                                checkpointId: checkpointId,
+                                inMemoryStore: inMemoryStore,
+                                loadManager: loadManager,
+                                persistence: persistence,
+                                isPreload: true,
+                                shouldSaveHistory: false,
+                                chains: chains,
+                                config: config,
+                                isResolved: false
+                              })
+                        })));
+          }
+          catch (exn){
+            
+          }
+        }
+        
+      } else {
+        try {
+          promises.push($$Promise.silentCatch(item.onBlockConfig.handler(Ecosystem.makeOnBlockArgs(item.blockNumber, config.ecosystem, UserContext.getHandlerContext({
+                                item: item,
+                                checkpointId: checkpointId,
+                                inMemoryStore: inMemoryStore,
+                                loadManager: loadManager,
+                                persistence: persistence,
+                                isPreload: true,
+                                shouldSaveHistory: false,
+                                chains: chains,
+                                config: config,
+                                isResolved: false
+                              })))));
+        }
+        catch (exn$1){
+          
+        }
+      }
+    }
+    itemIdx = itemIdx + checkpointEventsProcessed | 0;
+  }
+  await Promise.all(promises);
+}
+
+async function runBatchHandlersOrThrow(batch, inMemoryStore, loadManager, indexer, shouldSaveHistory, shouldBenchmark, chains) {
+  var itemIdx = 0;
+  for(var checkpointIdx = 0 ,checkpointIdx_finish = batch.checkpointIds.length; checkpointIdx < checkpointIdx_finish; ++checkpointIdx){
+    var checkpointId = batch.checkpointIds[checkpointIdx];
+    var checkpointEventsProcessed = batch.checkpointEventsProcessed[checkpointIdx];
+    for(var idx = 0; idx < checkpointEventsProcessed; ++idx){
+      var item = batch.items[itemIdx + idx | 0];
+      await runHandlerOrThrow(item, checkpointId, inMemoryStore, loadManager, indexer, shouldSaveHistory, shouldBenchmark, chains);
+    }
+    itemIdx = itemIdx + checkpointEventsProcessed | 0;
+  }
+}
+
+function registerProcessEventBatchMetrics(logger, loadDuration, handlerDuration, dbWriteDuration) {
+  Logging.childTrace(logger, {
+        msg: "Finished processing batch",
+        loader_time_elapsed: loadDuration,
+        handlers_time_elapsed: handlerDuration,
+        write_time_elapsed: dbWriteDuration
+      });
+  Prometheus.incrementLoadEntityDurationCounter(loadDuration);
+  Prometheus.incrementEventRouterDurationCounter(handlerDuration);
+  Prometheus.incrementExecuteBatchDurationCounter(dbWriteDuration);
+  Prometheus.incrementStorageWriteTimeCounter(dbWriteDuration);
+  Prometheus.incrementStorageWriteCounter();
+}
+
+async function processEventBatch(batch, inMemoryStore, isInReorgThreshold, loadManager, indexer, chainFetchers) {
+  var totalBatchSize = batch.totalBatchSize;
+  var chains = computeChainsState(chainFetchers);
+  var logger = Logging.getLogger();
+  Logging.childTrace(logger, {
+        msg: "Started processing batch",
+        totalBatchSize: totalBatchSize,
+        chains: Utils.Dict.mapValues(batch.progressedChainsById, (function (chainAfterBatch) {
+                return {
+                        batchSize: chainAfterBatch.batchSize,
+                        progress: chainAfterBatch.progressBlockNumber
+                      };
+              }))
+      });
+  try {
+    var timeRef = Hrtime.makeTimer();
+    if (Utils.$$Array.notEmpty(batch.items)) {
+      await preloadBatchOrThrow(batch, loadManager, indexer.persistence, indexer.config, inMemoryStore, chains);
+    }
+    var elapsedTimeAfterLoaders = Hrtime.intFromMillis(Hrtime.toMillis(Hrtime.timeSince(timeRef)));
+    if (Utils.$$Array.notEmpty(batch.items)) {
+      await runBatchHandlersOrThrow(batch, inMemoryStore, loadManager, indexer, Config.shouldSaveHistory(indexer.config, isInReorgThreshold), Env.Benchmark.shouldSaveData, chains);
+    }
+    var elapsedTimeAfterProcessing = Hrtime.intFromMillis(Hrtime.toMillis(Hrtime.timeSince(timeRef)));
+    try {
+      await Persistence.writeBatch(indexer.persistence, batch, indexer.config, inMemoryStore, isInReorgThreshold);
+      var elapsedTimeAfterDbWrite = Hrtime.intFromMillis(Hrtime.toMillis(Hrtime.timeSince(timeRef)));
+      var handlerDuration = elapsedTimeAfterProcessing - elapsedTimeAfterLoaders | 0;
+      var dbWriteDuration = elapsedTimeAfterDbWrite - elapsedTimeAfterProcessing | 0;
+      registerProcessEventBatchMetrics(logger, elapsedTimeAfterLoaders, handlerDuration, dbWriteDuration);
+      if (Env.Benchmark.shouldSaveData) {
+        Benchmark.addEventProcessing(totalBatchSize, elapsedTimeAfterLoaders, handlerDuration, dbWriteDuration, elapsedTimeAfterDbWrite);
+      }
+      return {
+              TAG: "Ok",
+              _0: undefined
+            };
+    }
+    catch (raw_exn){
+      var exn = Caml_js_exceptions.internalToOCamlException(raw_exn);
+      if (exn.RE_EXN_ID === Persistence.StorageError) {
+        return {
+                TAG: "Error",
+                _0: ErrorHandling.make(exn.reason, logger, exn.message)
+              };
+      } else {
+        return {
+                TAG: "Error",
+                _0: ErrorHandling.make(exn, logger, "Failed writing batch to database")
+              };
+      }
+    }
+  }
+  catch (raw_exn$1){
+    var exn$1 = Caml_js_exceptions.internalToOCamlException(raw_exn$1);
+    if (exn$1.RE_EXN_ID === ProcessingError) {
+      return {
+              TAG: "Error",
+              _0: ErrorHandling.make(exn$1.exn, Logging.getItemLogger(exn$1.item), exn$1.message)
+            };
+    }
+    throw exn$1;
+  }
+}
+
+export {
+  allChainsEventsProcessedToEndblock ,
+  computeChainsState ,
+  convertFieldsToJson ,
+  addItemToRawEvents ,
+  ProcessingError ,
+  runEventHandlerOrThrow ,
+  runHandlerOrThrow ,
+  preloadBatchOrThrow ,
+  runBatchHandlersOrThrow ,
+  registerProcessEventBatchMetrics ,
+  processEventBatch ,
+}
+/* Env Not a pure module */

package/src/FetchState.res

@@ -59,6 +59,7 @@ type t = {
   // ready for processing
   targetBufferSize: int,
   onBlockConfigs: array<Internal.onBlockConfig>,
+  knownHeight: int,
 }
 
 let mergeIntoPartition = (p: partition, ~target: partition, ~maxAddrInPartition) => {
@@ -184,16 +185,27 @@ let updateInternal = (
   ~indexingContracts=fetchState.indexingContracts,
   ~mutItems=?,
   ~blockLag=fetchState.blockLag,
+  ~knownHeight=fetchState.knownHeight,
 ): t => {
-  let firstPartition = partitions->Js.Array2.unsafe_get(0)
-  let latestFullyFetchedBlock = ref(firstPartition.latestFetchedBlock)
-  for idx in 0 to partitions->Array.length - 1 {
-    let p = partitions->Js.Array2.unsafe_get(idx)
-    if latestFullyFetchedBlock.contents.blockNumber > p.latestFetchedBlock.blockNumber {
-      latestFullyFetchedBlock := p.latestFetchedBlock
+  let latestFullyFetchedBlock = if partitions->Utils.Array.notEmpty {
+    let firstPartition = partitions->Js.Array2.unsafe_get(0)
+    let latestFullyFetchedBlock = ref(firstPartition.latestFetchedBlock)
+    for idx in 0 to partitions->Array.length - 1 {
+      let p = partitions->Js.Array2.unsafe_get(idx)
+      if latestFullyFetchedBlock.contents.blockNumber > p.latestFetchedBlock.blockNumber {
+        latestFullyFetchedBlock := p.latestFetchedBlock
+      }
+    }
+    latestFullyFetchedBlock.contents
+  } else {
+    {
+      blockNumber: knownHeight,
+      // The case is only possible when using only block handlers
+      // so it's fine to have a zero timestamp
+      // since we don't support ordered multichain mode anyways
+      blockTimestamp: 0,
     }
   }
-  let latestFullyFetchedBlock = latestFullyFetchedBlock.contents
 
   let mutItemsRef = ref(mutItems)
 
@@ -282,6 +294,7 @@ let updateInternal = (
     latestFullyFetchedBlock,
     indexingContracts,
     blockLag,
+    knownHeight,
     buffer: switch mutItemsRef.contents {
     // Theoretically it could be faster to asume that
     // the items are sorted, but there are cases
@@ -751,13 +764,14 @@ let getNextQuery = (
     endBlock,
     indexingContracts,
     blockLag,
+    latestOnBlockBlockNumber,
+    knownHeight,
   }: t,
   ~concurrencyLimit,
-  ~currentBlockHeight,
   ~stateId,
 ) => {
-  let headBlock = currentBlockHeight - blockLag
-  if headBlock <= 0 {
+  let headBlockNumber = knownHeight - blockLag
+  if headBlockNumber <= 0 {
     WaitingForNewBlock
   } else if concurrencyLimit === 0 {
     ReachedMaxConcurrency
@@ -767,11 +781,14 @@ let getNextQuery = (
     let areMergingPartitionsFetching = ref(false)
     let mostBehindMergingPartition = ref(None)
     let mergingPartitionTarget = ref(None)
+
+    let isOnBlockBehindTheHead = latestOnBlockBlockNumber < headBlockNumber
     let shouldWaitForNewBlock = ref(
       switch endBlock {
-      | Some(endBlock) => headBlock < endBlock
+      | Some(endBlock) => headBlockNumber < endBlock
       | None => true
-      },
+      } &&
+      !isOnBlockBehindTheHead,
     )
 
     let checkIsFetchingPartition = p => {
@@ -785,12 +802,12 @@ let getNextQuery = (
       let p = partitions->Js.Array2.unsafe_get(idx)
 
       let isFetching = checkIsFetchingPartition(p)
-      let hasReachedTheHead = p.latestFetchedBlock.blockNumber >= headBlock
+      let isBehindTheHead = p.latestFetchedBlock.blockNumber < headBlockNumber
 
-      if isFetching || !hasReachedTheHead {
+      if isFetching || isBehindTheHead {
         // Even if there are some partitions waiting for the new block
         // We still want to wait for all partitions reaching the head
-        // because they might update currentBlockHeight in their response
+        // because they might update knownHeight in their response
         // Also, there are cases when some partitions fetching at 50% of the chain
         // and we don't want to poll the head for a few small partitions
         shouldWaitForNewBlock := false
@@ -845,8 +862,8 @@ let getNextQuery = (
       switch buffer->Array.get(targetBufferSize - 1) {
       | Some(item) =>
         // Just in case check that we don't query beyond the current block
-        Pervasives.min(item->Internal.getItemBlockNumber, currentBlockHeight)
-      | None => currentBlockHeight
+        Pervasives.min(item->Internal.getItemBlockNumber, knownHeight)
+      | None => knownHeight
       }
     }
     let queries = []
@@ -859,17 +876,17 @@ let getNextQuery = (
         | 0 => endBlock
         | _ =>
           switch endBlock {
-          | Some(endBlock) => Some(Pervasives.min(headBlock, endBlock))
+          | Some(endBlock) => Some(Pervasives.min(headBlockNumber, endBlock))
           // Force head block as an endBlock when blockLag is set
           // because otherwise HyperSync might return bigger range
-          | None => Some(headBlock)
+          | None => Some(headBlockNumber)
           }
         }
         // Enforce the respose range up until target block
         // Otherwise for indexers with 100+ partitions
         // we might blow up the buffer size to more than 600k events
         // simply because of HyperSync returning extra blocks
-        let endBlock = switch (endBlock, maxQueryBlockNumber < currentBlockHeight) {
+        let endBlock = switch (endBlock, maxQueryBlockNumber < knownHeight) {
         | (Some(endBlock), true) => Some(Pervasives.min(maxQueryBlockNumber, endBlock))
         | (None, true) => Some(maxQueryBlockNumber)
         | (_, false) => endBlock
@@ -968,6 +985,7 @@ let make = (
   ~maxAddrInPartition,
   ~chainId,
   ~targetBufferSize,
+  ~knownHeight,
   ~progressBlockNumber=startBlock - 1,
   ~onBlockConfigs=[],
   ~blockLag=0,
@@ -1065,9 +1083,9 @@ let make = (
     }
   }
 
-  if partitions->Array.length === 0 {
+  if partitions->Utils.Array.isEmpty && onBlockConfigs->Utils.Array.isEmpty {
     Js.Exn.raiseError(
-      "Invalid configuration: Nothing to fetch. Make sure that you provided at least one contract address to index, or have events with Wildcard mode enabled.",
+      "Invalid configuration: Nothing to fetch. Make sure that you provided at least one contract address to index, or have events with Wildcard mode enabled, or have onBlock handlers.",
     )
   }
 
@@ -1096,6 +1114,7 @@ let make = (
     blockLag,
     onBlockConfigs,
     targetBufferSize,
+    knownHeight,
     buffer: [],
   }
 }
@@ -1173,7 +1192,7 @@ let rollback = (fetchState: t, ~targetBlockNumber) => {
 
   {
     ...fetchState,
-    latestOnBlockBlockNumber: targetBlockNumber, // TODO: This is not tested. I assume there might be a possible issue of it skipping some blocks
+    latestOnBlockBlockNumber: targetBlockNumber, // FIXME: This is not tested. I assume there might be a possible issue of it skipping some blocks
   }->updateInternal(
     ~partitions,
     ~indexingContracts,
@@ -1204,15 +1223,12 @@ let isActivelyIndexing = ({endBlock} as fetchState: t) => {
   }
 }
 
-let isReadyToEnterReorgThreshold = (
-  {endBlock, blockLag, buffer} as fetchState: t,
-  ~currentBlockHeight,
-) => {
+let isReadyToEnterReorgThreshold = ({endBlock, blockLag, buffer, knownHeight} as fetchState: t) => {
   let bufferBlockNumber = fetchState->bufferBlockNumber
-  currentBlockHeight !== 0 &&
+  knownHeight !== 0 &&
   switch endBlock {
   | Some(endBlock) if bufferBlockNumber >= endBlock => true
-  | _ => bufferBlockNumber >= currentBlockHeight - blockLag
+  | _ => bufferBlockNumber >= knownHeight - blockLag
   } &&
   buffer->Utils.Array.isEmpty
 }
@@ -1262,3 +1278,12 @@ let getUnorderedMultichainProgressBlockNumberAt = ({buffer} as fetchState: t, ~i
   | _ => bufferBlockNumber
   }
 }
+
+let updateKnownHeight = (fetchState: t, ~knownHeight) => {
+  if knownHeight > fetchState.knownHeight {
+    Prometheus.setKnownHeight(~blockNumber=knownHeight, ~chainId=fetchState.chainId)
+    fetchState->updateInternal(~knownHeight)
+  } else {
+    fetchState
+  }
+}