envio 3.3.0-rc.0 → 3.3.1-rc.0

package/src/IndexerState.res.mjs

@@ -8,11 +8,12 @@ import * as ChainMap from "./ChainMap.res.mjs";
 import * as Internal from "./Internal.res.mjs";
 import * as Throttler from "./Throttler.res.mjs";
 import * as ChainState from "./ChainState.res.mjs";
-import * as Prometheus from "./Prometheus.res.mjs";
 import * as EffectState from "./EffectState.res.mjs";
 import * as LoadManager from "./LoadManager.res.mjs";
+import * as Performance from "./bindings/Performance.res.mjs";
 import * as ErrorHandling from "./ErrorHandling.res.mjs";
 import * as InMemoryTable from "./InMemoryTable.res.mjs";
+import * as SourceManager from "./sources/SourceManager.res.mjs";
 import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
 import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
 import * as CrossChainState from "./CrossChainState.res.mjs";
@@ -82,7 +83,16 @@ function make$1(config, persistence, chainStates, isInReorgThreshold, isRealtime
     onError: onError,
     isStopped: false,
     epoch: 0,
-    simulateDeadInputTracker: SimulateDeadInputTracker.makeFromConfig(config)
+    simulateDeadInputTracker: SimulateDeadInputTracker.makeFromConfig(config),
+    preloadSeconds: 0,
+    processingSeconds: 0,
+    handlerStats: {},
+    storageLoadStats: {},
+    storageWriteStats: {},
+    historyPruneStats: {},
+    rollbackSeconds: 0,
+    rollbackCount: 0,
+    rollbackEventsCount: 0
   };
 }
 
@@ -101,12 +111,6 @@ function makeFromDbState(config, persistence, initialState, registrationsByChain
         return false;
       }
     });
-  Prometheus.ProcessingMaxBatchSize.set(config.batchSize);
-  Prometheus.ReorgThreshold.set(isInReorgThreshold);
-  Utils.Dict.forEach(initialState.cache, param => {
-    let count = param.count;
-    Prometheus.EffectCacheCount.set(count, param.effectName, Internal.EffectCache.scopeToString(param.scope));
-  });
   let isRealtime = !Env.updateSyncTimeOnRestart && initialState.chains.length !== 0 && initialState.chains.every(c => Stdlib_Option.isSome(c.timestampCaughtUpToHeadOrEndblock));
   let chainStates = {};
   initialState.chains.forEach(resumedChainState => {
@@ -114,24 +118,12 @@ function makeFromDbState(config, persistence, initialState, registrationsByChain
     let chainConfig = ChainMap.get(config.chainMap, chain);
     chainStates[resumedChainState.id] = ChainState.makeFromDbState(chainConfig, resumedChainState, initialState.reorgCheckpoints, isInReorgThreshold, isRealtime, config, registrationsByChainId, reducedPollingInterval);
   });
-  let allChainsReady = {
-    contents: initialState.chains.length !== 0
-  };
-  Utils.Dict.forEach(chainStates, cs => {
-    let chainId = ChainState.chainConfig(cs).id;
-    Prometheus.ProgressBlockNumber.set(ChainState.committedProgressBlockNumber(cs), chainId);
-    Prometheus.ProgressReady.init(chainId);
-    if (ChainState.isReady(cs)) {
-      return Prometheus.ProgressReady.set(chainId);
-    } else {
-      allChainsReady.contents = false;
-      return;
-    }
+  let state = make$1(config, persistence, chainStates, isInReorgThreshold, isRealtime, targetBufferSize, initialState.checkpointId, isDevelopmentMode, shouldUseTui, exitAfterFirstEventBlock, onError);
+  Utils.Dict.forEach(initialState.cache, param => {
+    let count = param.count;
+    EffectState.setUnregisteredCacheCount(state.effectState, param.effectName, param.scope, count);
   });
-  if (allChainsReady.contents) {
-    Prometheus.ProgressReady.setAllReady();
-  }
-  return make$1(config, persistence, chainStates, isInReorgThreshold, isRealtime, targetBufferSize, initialState.checkpointId, isDevelopmentMode, shouldUseTui, exitAfterFirstEventBlock, onError);
+  return state;
 }
 
 function isStale(state, stateId) {
@@ -343,6 +335,213 @@ function simulateDeadInputTracker(state) {
   return state.simulateDeadInputTracker;
 }
 
+function toMetrics(state) {
+  let chainStates = CrossChainState.chainStates(state.crossChainState);
+  let sourceRequests = [];
+  let sourceHeights = [];
+  Utils.Dict.forEach(chainStates, cs => {
+    let sourceManager = ChainState.sourceManager(cs);
+    SourceManager.getRequestStatSamples(sourceManager).forEach(s => {
+      sourceRequests.push({
+        source: s.sourceName,
+        chainId: s.chainId,
+        method: s.method,
+        count: s.count,
+        seconds: s.seconds
+      });
+    });
+    SourceManager.getSourceHeightSamples(sourceManager).forEach(s => {
+      sourceHeights.push({
+        source: s.sourceName,
+        chainId: s.chainId,
+        height: s.height
+      });
+    });
+  });
+  let historyPrunes = [];
+  Utils.Dict.forEachWithKey(state.historyPruneStats, (s, entityName) => {
+    historyPrunes.push({
+      entity: entityName,
+      seconds: s.seconds,
+      count: s.count
+    });
+  });
+  return {
+    startTime: state.indexerStartTime,
+    targetBufferSize: CrossChainState.targetBufferSize(state.crossChainState),
+    isInReorgThreshold: CrossChainState.isInReorgThreshold(state.crossChainState),
+    rollbackEnabled: state.config.shouldRollbackOnReorg,
+    maxBatchSize: state.config.batchSize,
+    preloadSeconds: state.preloadSeconds,
+    processingSeconds: state.processingSeconds,
+    rollbackSeconds: state.rollbackSeconds,
+    rollbackCount: state.rollbackCount,
+    rollbackEventsCount: state.rollbackEventsCount,
+    chains: Object.values(chainStates).map(ChainState.toMetrics),
+    handlers: Object.values(state.handlerStats).map(s => ({
+      contract: s.contract,
+      event: s.event,
+      processingSeconds: s.processingSeconds,
+      processingCount: s.processingCount,
+      preloadSeconds: s.preloadSeconds,
+      preloadCount: s.preloadCount,
+      preloadSecondsTotal: s.preloadSecondsTotal
+    })),
+    effects: EffectState.toMetrics(state.effectState),
+    storageLoads: Object.values(state.storageLoadStats).map(s => ({
+      operation: s.operation,
+      storage: s.storage,
+      seconds: s.seconds,
+      secondsTotal: s.secondsTotal,
+      count: s.count,
+      whereSize: s.whereSize,
+      size: s.size
+    })),
+    storageWrites: Object.values(state.storageWriteStats).map(s => ({
+      storage: s.storage,
+      seconds: s.seconds,
+      count: s.count
+    })),
+    historyPrunes: historyPrunes,
+    sourceRequests: sourceRequests,
+    sourceHeights: sourceHeights
+  };
+}
+
+function recordBatchDurations(state, loadDuration, handlerDuration) {
+  state.preloadSeconds = state.preloadSeconds + loadDuration;
+  state.processingSeconds = state.processingSeconds + handlerDuration;
+}
+
+function getHandlerStat(state, contract, event) {
+  let key = contract.length.toString() + ":" + contract + event;
+  let stat = state.handlerStats[key];
+  if (stat !== undefined) {
+    return stat;
+  }
+  let stat$1 = {
+    contract: contract,
+    event: event,
+    processingSeconds: 0,
+    processingCount: 0,
+    preloadSeconds: 0,
+    preloadCount: 0,
+    preloadSecondsTotal: 0,
+    preloadPendingCount: 0,
+    preloadPendingTimerRef: null
+  };
+  state.handlerStats[key] = stat$1;
+  return stat$1;
+}
+
+function recordHandlerDuration(state, contract, event, duration) {
+  let stat = getHandlerStat(state, contract, event);
+  stat.processingSeconds = stat.processingSeconds + duration;
+  stat.processingCount = stat.processingCount + 1;
+}
+
+function startPreloadHandler(state, contract, event) {
+  let stat = getHandlerStat(state, contract, event);
+  if (stat.preloadPendingCount === 0) {
+    stat.preloadPendingTimerRef = Performance.now();
+  }
+  stat.preloadPendingCount = stat.preloadPendingCount + 1 | 0;
+  return Performance.now();
+}
+
+function endPreloadHandler(state, timerRef, contract, event) {
+  let stat = getHandlerStat(state, contract, event);
+  stat.preloadPendingCount = stat.preloadPendingCount - 1 | 0;
+  if (stat.preloadPendingCount === 0) {
+    stat.preloadSeconds = stat.preloadSeconds + Performance.secondsSince(stat.preloadPendingTimerRef);
+  }
+  stat.preloadSecondsTotal = stat.preloadSecondsTotal + Performance.secondsSince(timerRef);
+  stat.preloadCount = stat.preloadCount + 1;
+}
+
+function getStorageLoadStat(state, storage, operation) {
+  let key = storage.length.toString() + ":" + storage + operation;
+  let stat = state.storageLoadStats[key];
+  if (stat !== undefined) {
+    return stat;
+  }
+  let stat$1 = {
+    operation: operation,
+    storage: storage,
+    seconds: 0,
+    secondsTotal: 0,
+    count: 0,
+    whereSize: 0,
+    size: 0,
+    pendingCount: 0,
+    pendingTimerRef: null
+  };
+  state.storageLoadStats[key] = stat$1;
+  return stat$1;
+}
+
+function startStorageLoad(state, storage, operation) {
+  let stat = getStorageLoadStat(state, storage, operation);
+  if (stat.pendingCount === 0) {
+    stat.pendingTimerRef = Performance.now();
+  }
+  stat.pendingCount = stat.pendingCount + 1 | 0;
+  return Performance.now();
+}
+
+function endStorageLoad(state, timerRef, storage, operation, whereSize, size) {
+  let stat = getStorageLoadStat(state, storage, operation);
+  stat.pendingCount = stat.pendingCount - 1 | 0;
+  if (stat.pendingCount === 0) {
+    stat.seconds = stat.seconds + Performance.secondsSince(stat.pendingTimerRef);
+  }
+  stat.secondsTotal = stat.secondsTotal + Performance.secondsSince(timerRef);
+  stat.count = stat.count + 1;
+  stat.whereSize = stat.whereSize + whereSize;
+  stat.size = stat.size + size;
+}
+
+function recordStorageWrite(state, storage, timeSeconds) {
+  let stat = state.storageWriteStats[storage];
+  let stat$1;
+  if (stat !== undefined) {
+    stat$1 = stat;
+  } else {
+    let stat$2 = {
+      storage: storage,
+      seconds: 0,
+      count: 0
+    };
+    state.storageWriteStats[storage] = stat$2;
+    stat$1 = stat$2;
+  }
+  stat$1.seconds = stat$1.seconds + timeSeconds;
+  stat$1.count = stat$1.count + 1 | 0;
+}
+
+function recordHistoryPrune(state, entityName, timeSeconds) {
+  let stat = state.historyPruneStats[entityName];
+  let stat$1;
+  if (stat !== undefined) {
+    stat$1 = stat;
+  } else {
+    let stat$2 = {
+      seconds: 0,
+      count: 0
+    };
+    state.historyPruneStats[entityName] = stat$2;
+    stat$1 = stat$2;
+  }
+  stat$1.seconds = stat$1.seconds + timeSeconds;
+  stat$1.count = stat$1.count + 1 | 0;
+}
+
+function recordRollbackSuccess(state, timeSeconds, rollbackedProcessedEvents) {
+  state.rollbackSeconds = state.rollbackSeconds + timeSeconds;
+  state.rollbackCount = state.rollbackCount + 1 | 0;
+  state.rollbackEventsCount = state.rollbackEventsCount + rollbackedProcessedEvents;
+}
+
 function queueProcessedBatch(state, batch) {
   state.processedBatches.push(batch);
   let checkpointId = Utils.$$Array.last(batch.checkpointIds);
@@ -523,6 +722,16 @@ export {
   epoch,
   lastPrunedAtMillis,
   simulateDeadInputTracker,
+  toMetrics,
+  recordBatchDurations,
+  recordHandlerDuration,
+  startPreloadHandler,
+  endPreloadHandler,
+  startStorageLoad,
+  endStorageLoad,
+  recordStorageWrite,
+  recordHistoryPrune,
+  recordRollbackSuccess,
   queueProcessedBatch,
   drainBatchRun,
   takeRollback,

package/src/IndexerState.resi

@@ -101,6 +101,28 @@ let epoch: t => int
 let lastPrunedAtMillis: t => dict<float>
 let simulateDeadInputTracker: t => option<SimulateDeadInputTracker.t>
 
+// Read-only snapshot of every metric; the single window into the mutable
+// counters for the /metrics endpoint, the TUI and the console API.
+let toMetrics: t => Metrics.t
+
+// Metric counters.
+let recordBatchDurations: (t, ~loadDuration: float, ~handlerDuration: float) => unit
+let recordHandlerDuration: (t, ~contract: string, ~event: string, ~duration: float) => unit
+let startPreloadHandler: (t, ~contract: string, ~event: string) => Performance.timeRef
+let endPreloadHandler: (t, Performance.timeRef, ~contract: string, ~event: string) => unit
+let startStorageLoad: (t, ~storage: string, ~operation: string) => Performance.timeRef
+let endStorageLoad: (
+  t,
+  Performance.timeRef,
+  ~storage: string,
+  ~operation: string,
+  ~whereSize: int,
+  ~size: int,
+) => unit
+let recordStorageWrite: (t, ~storage: string, ~timeSeconds: float) => unit
+let recordHistoryPrune: (t, ~entityName: string, ~timeSeconds: float) => unit
+let recordRollbackSuccess: (t, ~timeSeconds: float, ~rollbackedProcessedEvents: float) => unit
+
 // Store domain operations.
 let queueProcessedBatch: (t, ~batch: Batch.t) => unit
 let drainBatchRun: t => Batch.t

package/src/LoadLayer.res

@@ -13,7 +13,8 @@ let loadById = (
 
   let load = async (idsToLoad, ~onError as _) => {
     let storage = persistence->Persistence.getInitializedStorageOrThrow
-    let timerRef = Prometheus.StorageLoad.startOperation(~storage=storage.name, ~operation=key)
+    let timerRef =
+      indexerState->IndexerState.startStorageLoad(~storage=storage.name, ~operation=key)
 
     // Since LoadManager.call prevents registering entities already in the in-memory store,
     // we can be sure that we load only the new ones.
@@ -50,7 +51,8 @@ let loadById = (
       )
     })
 
-    timerRef->Prometheus.StorageLoad.endOperation(
+    indexerState->IndexerState.endStorageLoad(
+      timerRef,
       ~storage=storage.name,
       ~operation=key,
       ~whereSize=idsToLoad->Array.length,
@@ -76,25 +78,7 @@ let callEffect = (
   ~timerRef,
   ~onError,
 ) => {
-  let effectName = effect.name
-  let scopeLabel = inMemTable.scope->Internal.EffectCache.scopeToString
-  let hadActiveCalls = inMemTable.activeCallsCount > 0
-  inMemTable.activeCallsCount = inMemTable.activeCallsCount + 1
-  Prometheus.EffectCalls.activeCallsCount->Prometheus.SafeGauge.handleInt(
-    ~labels={"effect": effectName, "scope": scopeLabel},
-    ~value=inMemTable.activeCallsCount,
-  )
-
-  if hadActiveCalls {
-    let elapsed = Performance.secondsBetween(~from=inMemTable.prevCallStartTimerRef, ~to=timerRef)
-    if elapsed > 0. {
-      Prometheus.EffectCalls.timeCounter->Prometheus.SafeCounter.handleFloat(
-        ~labels={"effect": effectName, "scope": scopeLabel},
-        ~value=elapsed,
-      )
-    }
-  }
-  inMemTable.prevCallStartTimerRef = timerRef
+  inMemTable.stats->EffectState.startCall(~timerRef)
 
   effect.handler(arg)
   ->Promise.thenResolve(output => {
@@ -109,25 +93,7 @@ let callEffect = (
     onError(~inputKey=arg.cacheKey, ~exn)
   })
   ->Promise.finally(() => {
-    inMemTable.activeCallsCount = inMemTable.activeCallsCount - 1
-    Prometheus.EffectCalls.activeCallsCount->Prometheus.SafeGauge.handleInt(
-      ~labels={"effect": effectName, "scope": scopeLabel},
-      ~value=inMemTable.activeCallsCount,
-    )
-    let newTimer = Performance.now()
-    Prometheus.EffectCalls.timeCounter->Prometheus.SafeCounter.handleFloat(
-      ~labels={"effect": effectName, "scope": scopeLabel},
-      ~value=Performance.secondsBetween(~from=inMemTable.prevCallStartTimerRef, ~to=newTimer),
-    )
-    inMemTable.prevCallStartTimerRef = newTimer
-
-    Prometheus.EffectCalls.totalCallsCount->Prometheus.SafeCounter.increment(
-      ~labels={"effect": effectName, "scope": scopeLabel},
-    )
-    Prometheus.EffectCalls.sumTimeCounter->Prometheus.SafeCounter.handleFloat(
-      ~labels={"effect": effectName, "scope": scopeLabel},
-      ~value=timerRef->Performance.secondsSince,
-    )
+    inMemTable.stats->EffectState.endCall(~startTimerRef=timerRef)
   })
 }
 
@@ -138,8 +104,6 @@ let rec executeWithRateLimit = (
   ~onError,
   ~isFromQueue: bool,
 ) => {
-  let effectName = effect.name
-
   let timerRef = Performance.now()
   let promises = []
 
@@ -194,25 +158,13 @@ let rec executeWithRateLimit = (
     }
 
     if immediateCount > 0 && isFromQueue {
-      // Update queue count metric
-      state.queueCount = state.queueCount - immediateCount
-      Prometheus.EffectQueueCount.set(
-        ~count=state.queueCount,
-        ~effectName,
-        ~scope=inMemTable.scope->Internal.EffectCache.scopeToString,
-      )
+      inMemTable.stats->EffectState.queueDequeued(~count=immediateCount)
     }
 
     // Handle queued items
     if queuedArgs->Utils.Array.notEmpty {
       if !isFromQueue {
-        // Update queue count metric
-        state.queueCount = state.queueCount + queuedArgs->Array.length
-        Prometheus.EffectQueueCount.set(
-          ~count=state.queueCount,
-          ~effectName,
-          ~scope=inMemTable.scope->Internal.EffectCache.scopeToString,
-        )
+        inMemTable.stats->EffectState.queueEnqueued(~count=queuedArgs->Array.length)
       }
 
       let millisUntilReset = ref(0)
@@ -232,9 +184,8 @@ let rec executeWithRateLimit = (
         nextWindowPromise
         ->Promise.then(() => {
           if millisUntilReset.contents > 0 {
-            Prometheus.EffectQueueCount.timeCounter->Prometheus.SafeCounter.handleFloat(
-              ~labels=effectName,
-              ~value=millisUntilReset.contents->Int.toFloat /. 1000.,
+            inMemTable.stats->EffectState.addQueueWaitSeconds(
+              ~seconds=millisUntilReset.contents->Int.toFloat /. 1000.,
             )
           }
           executeWithRateLimit(
@@ -289,7 +240,8 @@ let loadEffect = (
       }
     ) {
       let storage = persistence->Persistence.getInitializedStorageOrThrow
-      let timerRef = Prometheus.StorageLoad.startOperation(~storage=storage.name, ~operation=key)
+      let timerRef =
+        indexerState->IndexerState.startStorageLoad(~storage=storage.name, ~operation=key)
       let {outputSchema} = effect.storageMeta
 
       let dbEntities = try {
@@ -319,8 +271,7 @@ let loadEffect = (
           inMemTable->InMemoryStore.initEffectOutputFromDb(~cacheKey=dbEntity.id, ~output)
         } catch {
         | S.Raised(error) =>
-          inMemTable.invalidationsCount = inMemTable.invalidationsCount + 1
-          Prometheus.EffectCacheInvalidationsCount.increment(~effectName)
+          inMemTable->EffectState.recordInvalidation
           Ecosystem.getItemLogger(item, ~ecosystem)->Logging.childTrace({
             "msg": "Invalidated effect cache",
             "input": dbEntity.id,
@@ -330,7 +281,8 @@ let loadEffect = (
         }
       })
 
-      timerRef->Prometheus.StorageLoad.endOperation(
+      indexerState->IndexerState.endStorageLoad(
+        timerRef,
         ~storage=storage.name,
         ~operation=key,
         ~whereSize=idsToLoad->Array.length,
@@ -386,7 +338,8 @@ let loadByFilter = (
 
   let load = async (filters: array<EntityFilter.t>, ~onError as _) => {
     let storage = persistence->Persistence.getInitializedStorageOrThrow
-    let timerRef = Prometheus.StorageLoad.startOperation(~storage=storage.name, ~operation=key)
+    let timerRef =
+      indexerState->IndexerState.startStorageLoad(~storage=storage.name, ~operation=key)
 
     let size = ref(0)
 
@@ -434,7 +387,8 @@ let loadByFilter = (
     })
     ->Promise.all
 
-    timerRef->Prometheus.StorageLoad.endOperation(
+    indexerState->IndexerState.endStorageLoad(
+      timerRef,
       ~storage=storage.name,
       ~operation=key,
       ~whereSize=queries->Array.reduce(0, (acc, query) => acc + query->EntityFilter.valuesCount),

package/src/LoadLayer.res.mjs

@@ -3,9 +3,8 @@
 import * as Table from "./db/Table.res.mjs";
 import * as Utils from "./Utils.res.mjs";
 import * as Logging from "./Logging.res.mjs";
-import * as Internal from "./Internal.res.mjs";
 import * as Ecosystem from "./Ecosystem.res.mjs";
-import * as Prometheus from "./Prometheus.res.mjs";
+import * as EffectState from "./EffectState.res.mjs";
 import * as LoadManager from "./LoadManager.res.mjs";
 import * as Performance from "./bindings/Performance.res.mjs";
 import * as Persistence from "./Persistence.res.mjs";
@@ -23,7 +22,7 @@ function loadById(loadManager, persistence, entityConfig, indexerState, shouldGr
   let inMemTable = InMemoryStore.getInMemTable(indexerState, entityConfig);
   let load = async (idsToLoad, param) => {
     let storage = Persistence.getInitializedStorageOrThrow(persistence);
-    let timerRef = Prometheus.StorageLoad.startOperation(storage.name, key);
+    let timerRef = IndexerState.startStorageLoad(indexerState, storage.name, key);
     let dbEntities;
     try {
       dbEntities = await storage.loadOrThrow({
@@ -45,55 +44,17 @@ function loadById(loadManager, persistence, entityConfig, indexerState, shouldGr
       entitiesMap[entity.id] = entity;
     }
     idsToLoad.forEach(entityId => InMemoryTable.Entity.initValue(inMemTable, IndexerState.committedCheckpointId(indexerState), entityId, entitiesMap[entityId]));
-    return Prometheus.StorageLoad.endOperation(timerRef, storage.name, key, idsToLoad.length, dbEntities.length);
+    return IndexerState.endStorageLoad(indexerState, timerRef, storage.name, key, idsToLoad.length, dbEntities.length);
   };
   return LoadManager.call(loadManager, entityId, key, load, LoadManager.noopHasher, shouldGroup, hash => hash in inMemTable.latestEntityChangeById, InMemoryTable.Entity.getUnsafe(inMemTable));
 }
 
 function callEffect(effect, arg, inMemTable, timerRef, onError) {
-  let effectName = effect.name;
-  let scopeLabel = Internal.EffectCache.scopeToString(inMemTable.scope);
-  let hadActiveCalls = inMemTable.activeCallsCount > 0;
-  inMemTable.activeCallsCount = inMemTable.activeCallsCount + 1 | 0;
-  Prometheus.SafeGauge.handleInt(Prometheus.EffectCalls.activeCallsCount, {
-    effect: effectName,
-    scope: scopeLabel
-  }, inMemTable.activeCallsCount);
-  if (hadActiveCalls) {
-    let elapsed = Performance.secondsBetween(inMemTable.prevCallStartTimerRef, timerRef);
-    if (elapsed > 0) {
-      Prometheus.SafeCounter.handleFloat(Prometheus.EffectCalls.timeCounter, {
-        effect: effectName,
-        scope: scopeLabel
-      }, elapsed);
-    }
-  }
-  inMemTable.prevCallStartTimerRef = timerRef;
-  return effect.handler(arg).then(output => InMemoryStore.setEffectOutput(inMemTable, arg.checkpointId, arg.cacheKey, output, arg.context.cache)).catch(exn => onError(arg.cacheKey, exn)).finally(() => {
-    inMemTable.activeCallsCount = inMemTable.activeCallsCount - 1 | 0;
-    Prometheus.SafeGauge.handleInt(Prometheus.EffectCalls.activeCallsCount, {
-      effect: effectName,
-      scope: scopeLabel
-    }, inMemTable.activeCallsCount);
-    let newTimer = Performance.now();
-    Prometheus.SafeCounter.handleFloat(Prometheus.EffectCalls.timeCounter, {
-      effect: effectName,
-      scope: scopeLabel
-    }, Performance.secondsBetween(inMemTable.prevCallStartTimerRef, newTimer));
-    inMemTable.prevCallStartTimerRef = newTimer;
-    Prometheus.SafeCounter.increment(Prometheus.EffectCalls.totalCallsCount, {
-      effect: effectName,
-      scope: scopeLabel
-    });
-    Prometheus.SafeCounter.handleFloat(Prometheus.EffectCalls.sumTimeCounter, {
-      effect: effectName,
-      scope: scopeLabel
-    }, Performance.secondsSince(timerRef));
-  });
+  EffectState.startCall(inMemTable.stats, timerRef);
+  return effect.handler(arg).then(output => InMemoryStore.setEffectOutput(inMemTable, arg.checkpointId, arg.cacheKey, output, arg.context.cache)).catch(exn => onError(arg.cacheKey, exn)).finally(() => EffectState.endCall(inMemTable.stats, timerRef));
 }
 
 function executeWithRateLimit(effect, effectArgs, inMemTable, onError, isFromQueue) {
-  let effectName = effect.name;
   let timerRef = Performance.now();
   let promises = [];
   let state = inMemTable.rateLimitState;
@@ -112,13 +73,11 @@ function executeWithRateLimit(effect, effectArgs, inMemTable, onError, isFromQue
       promises.push(callEffect(effect, immediateArgs[idx], inMemTable, timerRef, onError));
     }
     if (immediateCount > 0 && isFromQueue) {
-      state.queueCount = state.queueCount - immediateCount | 0;
-      Prometheus.EffectQueueCount.set(state.queueCount, effectName, Internal.EffectCache.scopeToString(inMemTable.scope));
+      EffectState.queueDequeued(inMemTable.stats, immediateCount);
     }
     if (Utils.$$Array.notEmpty(queuedArgs)) {
       if (!isFromQueue) {
-        state.queueCount = state.queueCount + queuedArgs.length | 0;
-        Prometheus.EffectQueueCount.set(state.queueCount, effectName, Internal.EffectCache.scopeToString(inMemTable.scope));
+        EffectState.queueEnqueued(inMemTable.stats, queuedArgs.length);
       }
       let millisUntilReset = {
         contents: 0
@@ -135,7 +94,7 @@ function executeWithRateLimit(effect, effectArgs, inMemTable, onError, isFromQue
       }
       promises.push(nextWindowPromise.then(() => {
         if (millisUntilReset.contents > 0) {
-          Prometheus.SafeCounter.handleFloat(Prometheus.EffectQueueCount.timeCounter, effectName, millisUntilReset.contents / 1000);
+          EffectState.addQueueWaitSeconds(inMemTable.stats, millisUntilReset.contents / 1000);
         }
         return executeWithRateLimit(effect, queuedArgs, inMemTable, onError, true);
       }));
@@ -163,7 +122,7 @@ function loadEffect(loadManager, persistence, effect, effectArgs, scope, indexer
     tmp = typeof match !== "object" || match.TAG === "Initializing" ? false : tableName in match._0.cache;
     if (tmp) {
       let storage = Persistence.getInitializedStorageOrThrow(persistence);
-      let timerRef = Prometheus.StorageLoad.startOperation(storage.name, key);
+      let timerRef = IndexerState.startStorageLoad(indexerState, storage.name, key);
       let match$1 = effect.storageMeta;
       let outputSchema = match$1.outputSchema;
       let dbEntities;
@@ -190,8 +149,7 @@ function loadEffect(loadManager, persistence, effect, effectArgs, scope, indexer
         } catch (raw_error) {
           let error = Primitive_exceptions.internalToException(raw_error);
           if (error.RE_EXN_ID === S$RescriptSchema.Raised) {
-            inMemTable.invalidationsCount = inMemTable.invalidationsCount + 1 | 0;
-            Prometheus.EffectCacheInvalidationsCount.increment(effectName);
+            EffectState.recordInvalidation(inMemTable);
             return Logging.childTrace(Ecosystem.getItemLogger(item, ecosystem), {
               msg: "Invalidated effect cache",
               input: dbEntity.id,
@@ -202,7 +160,7 @@ function loadEffect(loadManager, persistence, effect, effectArgs, scope, indexer
           throw error;
         }
       });
-      Prometheus.StorageLoad.endOperation(timerRef, storage.name, key, idsToLoad.length, dbEntities.length);
+      IndexerState.endStorageLoad(indexerState, timerRef, storage.name, key, idsToLoad.length, dbEntities.length);
     }
     let remainingCallsCount = idsToLoad.length - idsFromCache.size | 0;
     if (remainingCallsCount <= 0) {
@@ -227,7 +185,7 @@ function loadByFilter(loadManager, persistence, entityConfig, indexerState, shou
   let inMemTable = InMemoryStore.getInMemTable(indexerState, entityConfig);
   let load = async (filters, param) => {
     let storage = Persistence.getInitializedStorageOrThrow(persistence);
-    let timerRef = Prometheus.StorageLoad.startOperation(storage.name, key);
+    let timerRef = IndexerState.startStorageLoad(indexerState, storage.name, key);
     let size = {
       contents: 0
     };
@@ -250,7 +208,7 @@ function loadByFilter(loadManager, persistence, entityConfig, indexerState, shou
         throw exn;
       }
     }));
-    return Prometheus.StorageLoad.endOperation(timerRef, storage.name, key, Stdlib_Array.reduce(queries, 0, (acc, query) => acc + EntityFilter.valuesCount(query) | 0), size.contents);
+    return IndexerState.endStorageLoad(indexerState, timerRef, storage.name, key, Stdlib_Array.reduce(queries, 0, (acc, query) => acc + EntityFilter.valuesCount(query) | 0), size.contents);
   };
   return LoadManager.call(loadManager, filter, key, load, EntityFilter.toString, shouldGroup, InMemoryTable.Entity.hasIndex(inMemTable), InMemoryTable.Entity.getUnsafeOnIndex(inMemTable));
 }