envio 3.2.1 → 3.3.0-alpha.1

package/src/IndexerState.res.mjs

@@ -0,0 +1,543 @@
+// Generated by ReScript, PLEASE EDIT WITH CARE
+
+import * as Env from "./Env.res.mjs";
+import * as Utils from "./Utils.res.mjs";
+import * as Config from "./Config.res.mjs";
+import * as Logging from "./Logging.res.mjs";
+import * as ChainMap from "./ChainMap.res.mjs";
+import * as Internal from "./Internal.res.mjs";
+import * as Throttler from "./Throttler.res.mjs";
+import * as ChainState from "./ChainState.res.mjs";
+import * as Prometheus from "./Prometheus.res.mjs";
+import * as LoadManager from "./LoadManager.res.mjs";
+import * as ErrorHandling from "./ErrorHandling.res.mjs";
+import * as InMemoryTable from "./InMemoryTable.res.mjs";
+import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
+import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
+import * as CrossChainState from "./CrossChainState.res.mjs";
+import * as Primitive_object from "@rescript/runtime/lib/es6/Primitive_object.js";
+import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js";
+import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
+
+function make() {
+  let intervalMillis = Env.ThrottleWrites.pruneStaleDataIntervalMillis;
+  let logger = Logging.createChild({
+    context: "Throttler for pruning stale entity history data",
+    intervalMillis: intervalMillis
+  });
+  let pruneStaleEntityHistory = Throttler.make(intervalMillis, logger);
+  return {
+    pruneStaleEntityHistory: pruneStaleEntityHistory
+  };
+}
+
+let UndefinedEntity = /* @__PURE__ */Primitive_exceptions.create("IndexerState.EntityTables.UndefinedEntity");
+
+function make$1(entities) {
+  let init = {};
+  entities.forEach(entityConfig => {
+    init[entityConfig.name] = InMemoryTable.Entity.make();
+  });
+  return init;
+}
+
+function get(self, entityName) {
+  let table = self[entityName];
+  if (table !== undefined) {
+    return table;
+  } else {
+    return ErrorHandling.mkLogAndRaise(undefined, "Unexpected, entity InMemoryTable is undefined", {
+      RE_EXN_ID: UndefinedEntity,
+      entityName: entityName
+    });
+  }
+}
+
+function make$2(config, persistence, chainStates, isInReorgThreshold, isRealtime, targetBufferSizeOpt, committedCheckpointIdOpt, isDevelopmentModeOpt, shouldUseTuiOpt, exitAfterFirstEventBlockOpt, onError) {
+  let targetBufferSize = targetBufferSizeOpt !== undefined ? targetBufferSizeOpt : CrossChainState.calculateTargetBufferSize();
+  let committedCheckpointId = committedCheckpointIdOpt !== undefined ? committedCheckpointIdOpt : Internal.initialCheckpointId;
+  let isDevelopmentMode = isDevelopmentModeOpt !== undefined ? isDevelopmentModeOpt : false;
+  let shouldUseTui = shouldUseTuiOpt !== undefined ? shouldUseTuiOpt : false;
+  let exitAfterFirstEventBlock = exitAfterFirstEventBlockOpt !== undefined ? exitAfterFirstEventBlockOpt : false;
+  let intervalMillis = Env.ThrottleWrites.chainMetadataIntervalMillis;
+  let chainMetaThrottler = Throttler.make(intervalMillis, Logging.createChild({
+    context: "Throttler for chain metadata writes",
+    intervalMillis: intervalMillis
+  }));
+  return {
+    config: config,
+    persistence: persistence,
+    allEntities: persistence.allEntities,
+    entities: make$1(persistence.allEntities),
+    effects: {},
+    rollback: undefined,
+    committedCheckpointId: committedCheckpointId,
+    processedCheckpointId: committedCheckpointId,
+    processedBatches: [],
+    processedBatchesCount: 0,
+    writeFiber: undefined,
+    hasFailedWrite: false,
+    commitWaiters: [],
+    chainMeta: {},
+    chainMetaDirty: false,
+    chainMetaThrottler: chainMetaThrottler,
+    isProcessing: false,
+    crossChainState: CrossChainState.make(chainStates, isInReorgThreshold, isRealtime, targetBufferSize),
+    rollbackState: "NoRollback",
+    indexerStartTime: new Date(),
+    writeThrottlers: make(),
+    loadManager: LoadManager.make(),
+    keepProcessAlive: isDevelopmentMode || shouldUseTui,
+    exitAfterFirstEventBlock: exitAfterFirstEventBlock,
+    onError: onError,
+    isStopped: false,
+    epoch: 0
+  };
+}
+
+function makeFromDbState(config, persistence, initialState, registrations, isDevelopmentModeOpt, shouldUseTuiOpt, exitAfterFirstEventBlockOpt, reducedPollingInterval, targetBufferSizeOpt, onError) {
+  let isDevelopmentMode = isDevelopmentModeOpt !== undefined ? isDevelopmentModeOpt : false;
+  let shouldUseTui = shouldUseTuiOpt !== undefined ? shouldUseTuiOpt : false;
+  let exitAfterFirstEventBlock = exitAfterFirstEventBlockOpt !== undefined ? exitAfterFirstEventBlockOpt : false;
+  let targetBufferSize = targetBufferSizeOpt !== undefined ? targetBufferSizeOpt : CrossChainState.calculateTargetBufferSize();
+  let isInReorgThreshold = initialState.cleanRun ? false : initialState.chains.some(chain => {
+      let progressBlockNumber = chain.progressBlockNumber;
+      let sourceBlockNumber = chain.sourceBlockNumber;
+      let maxReorgDepth = chain.maxReorgDepth;
+      if (maxReorgDepth > 0 && sourceBlockNumber > 0) {
+        return progressBlockNumber > (sourceBlockNumber - maxReorgDepth | 0);
+      } else {
+        return false;
+      }
+    });
+  Prometheus.ProcessingMaxBatchSize.set(config.batchSize);
+  Prometheus.ReorgThreshold.set(isInReorgThreshold);
+  Utils.Dict.forEach(initialState.cache, param => {
+    let count = param.count;
+    Prometheus.EffectCacheCount.set(count, param.effectName);
+  });
+  let isRealtime = !Env.updateSyncTimeOnRestart && initialState.chains.length !== 0 && initialState.chains.every(c => Stdlib_Option.isSome(c.timestampCaughtUpToHeadOrEndblock));
+  let chainStates = {};
+  initialState.chains.forEach(resumedChainState => {
+    let chain = Config.getChain(config, resumedChainState.id);
+    let chainConfig = ChainMap.get(config.chainMap, chain);
+    chainStates[resumedChainState.id] = ChainState.makeFromDbState(chainConfig, resumedChainState, initialState.reorgCheckpoints, isInReorgThreshold, isRealtime, config, registrations, reducedPollingInterval);
+  });
+  let allChainsReady = {
+    contents: initialState.chains.length !== 0
+  };
+  Utils.Dict.forEach(chainStates, cs => {
+    let chainId = ChainState.chainConfig(cs).id;
+    Prometheus.ProgressBlockNumber.set(ChainState.committedProgressBlockNumber(cs), chainId);
+    Prometheus.ProgressReady.init(chainId);
+    if (ChainState.isReady(cs)) {
+      return Prometheus.ProgressReady.set(chainId);
+    } else {
+      allChainsReady.contents = false;
+      return;
+    }
+  });
+  if (allChainsReady.contents) {
+    Prometheus.ProgressReady.setAllReady();
+  }
+  return make$2(config, persistence, chainStates, isInReorgThreshold, isRealtime, targetBufferSize, initialState.checkpointId, isDevelopmentMode, shouldUseTui, exitAfterFirstEventBlock, onError);
+}
+
+function isStale(state, stateId) {
+  if (state.isStopped) {
+    return true;
+  } else {
+    return stateId !== state.epoch;
+  }
+}
+
+function isResolvingReorg(state) {
+  let match = state.rollbackState;
+  if (typeof match !== "object") {
+    return match !== "NoRollback";
+  } else {
+    return match.TAG !== "RollbackReady";
+  }
+}
+
+function errorExit(state, errHandler) {
+  if (!state.isStopped) {
+    state.isStopped = true;
+    return state.onError(errHandler);
+  }
+}
+
+function stop(state) {
+  state.isStopped = true;
+}
+
+function getChainState(state, chain) {
+  let cs = CrossChainState.chainStates(state.crossChainState)[chain];
+  if (cs !== undefined) {
+    return Primitive_option.valFromOption(cs);
+  } else {
+    return Stdlib_JsError.throwWithMessage("No chain with id " + ChainMap.Chain.toString(chain) + " found in chain states");
+  }
+}
+
+function getSafeCheckpointId(state) {
+  return CrossChainState.getSafeCheckpointId(state.crossChainState);
+}
+
+function createBatch(state, processedCheckpointId, batchSizeTarget, isRollback) {
+  return CrossChainState.createBatch(state.crossChainState, processedCheckpointId, batchSizeTarget, isRollback);
+}
+
+function enterReorgThreshold(state) {
+  CrossChainState.enterReorgThreshold(state.crossChainState);
+}
+
+function beginReorg(state, chain, blockNumber) {
+  state.epoch = state.epoch + 1 | 0;
+  state.rollbackState = {
+    TAG: "ReorgDetected",
+    chain: chain,
+    blockNumber: blockNumber
+  };
+}
+
+function enterFindingReorgDepth(state) {
+  state.rollbackState = "FindingReorgDepth";
+}
+
+function foundReorgDepth(state, chain, rollbackTargetBlockNumber) {
+  state.rollbackState = {
+    TAG: "FoundReorgDepth",
+    chain: chain,
+    rollbackTargetBlockNumber: rollbackTargetBlockNumber
+  };
+}
+
+function completeRollback(state, eventsProcessedDiffByChain) {
+  state.rollbackState = {
+    TAG: "RollbackReady",
+    eventsProcessedDiffByChain: eventsProcessedDiffByChain
+  };
+}
+
+function clearRollback(state) {
+  state.rollbackState = "NoRollback";
+}
+
+function invalidateInflight(state) {
+  state.epoch = state.epoch + 1 | 0;
+}
+
+function applyBatchProgress(state, batch) {
+  CrossChainState.applyBatchProgress(state.crossChainState, batch);
+}
+
+function isProcessing(state) {
+  return state.isProcessing;
+}
+
+function beginProcessing(state) {
+  state.isProcessing = true;
+}
+
+function endProcessing(state) {
+  state.isProcessing = false;
+}
+
+function recordProcessedBatch(state) {
+  state.processedBatchesCount = state.processedBatchesCount + 1 | 0;
+}
+
+function config(state) {
+  return state.config;
+}
+
+function persistence(state) {
+  return state.persistence;
+}
+
+function allEntities(state) {
+  return state.allEntities;
+}
+
+function entities(state) {
+  return state.entities;
+}
+
+function effects(state) {
+  return state.effects;
+}
+
+function committedCheckpointId(state) {
+  return state.committedCheckpointId;
+}
+
+function processedCheckpointId(state) {
+  return state.processedCheckpointId;
+}
+
+function processedBatches(state) {
+  return state.processedBatches;
+}
+
+function processedBatchesCount(state) {
+  return state.processedBatchesCount;
+}
+
+function writeFiber(state) {
+  return state.writeFiber;
+}
+
+function hasFailedWrite(state) {
+  return state.hasFailedWrite;
+}
+
+function chainMetaDirty(state) {
+  return state.chainMetaDirty;
+}
+
+function chainMetaThrottler(state) {
+  return state.chainMetaThrottler;
+}
+
+function crossChainState(state) {
+  return state.crossChainState;
+}
+
+function chainStates(state) {
+  return CrossChainState.chainStates(state.crossChainState);
+}
+
+function isInReorgThreshold(state) {
+  return CrossChainState.isInReorgThreshold(state.crossChainState);
+}
+
+function isRealtime(state) {
+  return CrossChainState.isRealtime(state.crossChainState);
+}
+
+function rollbackState(state) {
+  return state.rollbackState;
+}
+
+function indexerStartTime(state) {
+  return state.indexerStartTime;
+}
+
+function loadManager(state) {
+  return state.loadManager;
+}
+
+function keepProcessAlive(state) {
+  return state.keepProcessAlive;
+}
+
+function exitAfterFirstEventBlock(state) {
+  return state.exitAfterFirstEventBlock;
+}
+
+function isStopped(state) {
+  return state.isStopped;
+}
+
+function epoch(state) {
+  return state.epoch;
+}
+
+function pruneStaleEntityHistoryThrottler(state) {
+  return state.writeThrottlers.pruneStaleEntityHistory;
+}
+
+function queueProcessedBatch(state, batch) {
+  state.processedBatches.push(batch);
+  let checkpointId = Utils.$$Array.last(batch.checkpointIds);
+  if (checkpointId !== undefined) {
+    state.processedCheckpointId = checkpointId;
+    return;
+  }
+}
+
+function drainBatchRun(state) {
+  let all = state.processedBatches;
+  let isInReorgThreshold = all[0].isInReorgThreshold;
+  let rest = [];
+  let progressedChainsById = {};
+  let totalBatchSize = {
+    contents: 0
+  };
+  let items = [];
+  let checkpointIds = [];
+  let checkpointChainIds = [];
+  let checkpointBlockNumbers = [];
+  let checkpointBlockHashes = [];
+  let checkpointEventsProcessed = [];
+  all.forEach(batch => {
+    if (Utils.$$Array.isEmpty(rest) && batch.isInReorgThreshold === isInReorgThreshold) {
+      Utils.Dict.forEachWithKey(batch.progressedChainsById, (chainAfterBatch, key) => {
+        progressedChainsById[key] = chainAfterBatch;
+      });
+      totalBatchSize.contents = totalBatchSize.contents + batch.totalBatchSize | 0;
+      items.push(...batch.items);
+      checkpointIds.push(...batch.checkpointIds);
+      checkpointChainIds.push(...batch.checkpointChainIds);
+      checkpointBlockNumbers.push(...batch.checkpointBlockNumbers);
+      checkpointBlockHashes.push(...batch.checkpointBlockHashes);
+      checkpointEventsProcessed.push(...batch.checkpointEventsProcessed);
+    } else {
+      rest.push(batch);
+    }
+  });
+  state.processedBatches = rest;
+  return {
+    totalBatchSize: totalBatchSize.contents,
+    items: items,
+    progressedChainsById: progressedChainsById,
+    isInReorgThreshold: isInReorgThreshold,
+    checkpointIds: checkpointIds,
+    checkpointChainIds: checkpointChainIds,
+    checkpointBlockNumbers: checkpointBlockNumbers,
+    checkpointBlockHashes: checkpointBlockHashes,
+    checkpointEventsProcessed: checkpointEventsProcessed
+  };
+}
+
+function takeRollback(state) {
+  let rollback = state.rollback;
+  state.rollback = undefined;
+  return rollback;
+}
+
+function markCommitted(state, upToCheckpointId) {
+  state.committedCheckpointId = upToCheckpointId;
+}
+
+function beginRollbackDiff(state, targetCheckpointId, diffCheckpointId, progressBlockNumberByChainId) {
+  state.entities = make$1(state.allEntities);
+  state.effects = {};
+  state.rollback = {
+    targetCheckpointId: targetCheckpointId,
+    diffCheckpointId: diffCheckpointId,
+    progressBlockNumberByChainId: progressBlockNumberByChainId
+  };
+}
+
+function recordWriteFailure(state, exn) {
+  state.hasFailedWrite = true;
+  state.onError(ErrorHandling.make(exn, undefined, "Failed writing batch to the database"));
+}
+
+function beginWriteFiber(state, fiber) {
+  state.writeFiber = fiber;
+}
+
+function endWriteFiber(state) {
+  state.writeFiber = undefined;
+}
+
+function wakeCommitWaiters(state) {
+  let waiters = state.commitWaiters;
+  state.commitWaiters = [];
+  waiters.forEach(resolve => resolve());
+}
+
+function addCommitWaiter(state, resolve) {
+  state.commitWaiters.push(resolve);
+}
+
+function metaFieldsEqual(a, b) {
+  if (Primitive_object.equal(a.first_event_block, b.first_event_block) && a.buffer_block === b.buffer_block && a._is_hyper_sync === b._is_hyper_sync) {
+    return Primitive_object.equal(Stdlib_Option.map(Primitive_option.fromNull(a.ready_at), prim => prim.getTime()), Stdlib_Option.map(Primitive_option.fromNull(b.ready_at), prim => prim.getTime()));
+  } else {
+    return false;
+  }
+}
+
+function stageChainMeta(state, chainsData) {
+  Utils.Dict.forEachWithKey(chainsData, (meta, chainId) => {
+    let prev = state.chainMeta[chainId];
+    let changed = prev !== undefined ? !metaFieldsEqual(meta, prev) : true;
+    if (changed) {
+      state.chainMeta[chainId] = meta;
+      state.chainMetaDirty = true;
+      return;
+    }
+  });
+}
+
+function takeChainMetaSnapshot(state) {
+  if (state.chainMetaDirty) {
+    state.chainMetaDirty = false;
+    return Utils.Dict.shallowCopy(state.chainMeta);
+  }
+}
+
+let EntityTables = {
+  make: make$1,
+  get: get
+};
+
+let unexpectedErrorMsg = "Indexer has failed with an unexpected error";
+
+export {
+  EntityTables,
+  make$2 as make,
+  makeFromDbState,
+  unexpectedErrorMsg,
+  isStale,
+  isResolvingReorg,
+  errorExit,
+  stop,
+  getChainState,
+  enterReorgThreshold,
+  beginReorg,
+  enterFindingReorgDepth,
+  foundReorgDepth,
+  completeRollback,
+  clearRollback,
+  invalidateInflight,
+  applyBatchProgress,
+  isProcessing,
+  beginProcessing,
+  endProcessing,
+  recordProcessedBatch,
+  createBatch,
+  getSafeCheckpointId,
+  config,
+  persistence,
+  allEntities,
+  entities,
+  effects,
+  committedCheckpointId,
+  processedCheckpointId,
+  processedBatches,
+  processedBatchesCount,
+  writeFiber,
+  hasFailedWrite,
+  chainMetaDirty,
+  chainMetaThrottler,
+  crossChainState,
+  chainStates,
+  isInReorgThreshold,
+  isRealtime,
+  rollbackState,
+  indexerStartTime,
+  loadManager,
+  keepProcessAlive,
+  exitAfterFirstEventBlock,
+  isStopped,
+  epoch,
+  pruneStaleEntityHistoryThrottler,
+  queueProcessedBatch,
+  drainBatchRun,
+  takeRollback,
+  markCommitted,
+  beginRollbackDiff,
+  recordWriteFailure,
+  beginWriteFiber,
+  endWriteFiber,
+  wakeCommitWaiters,
+  addCommitWaiter,
+  stageChainMeta,
+  takeChainMetaSnapshot,
+}
+/* Env Not a pure module */

package/src/IndexerState.resi

@@ -0,0 +1,128 @@
+// The indexer state. `t` is opaque: other modules read it through the accessors
+// and change it only through the transitions and setters exposed here.
+
+type chain = ChainMap.Chain.t
+
+type rollbackState =
+  | NoRollback
+  | ReorgDetected({chain: chain, blockNumber: int})
+  | FindingReorgDepth
+  | FoundReorgDepth({chain: chain, rollbackTargetBlockNumber: int})
+  | RollbackReady({eventsProcessedDiffByChain: dict<float>})
+
+module EntityTables: {
+  type t = dict<InMemoryTable.Entity.t>
+  let make: array<Internal.entityConfig> => t
+  let get: (t, ~entityName: string) => InMemoryTable.Entity.t
+}
+
+type effectCacheInMemTable = {
+  mutable idsToStore: array<string>,
+  mutable invalidationsCount: int,
+  mutable dict: dict<Change.t<Internal.effectOutput>>,
+  mutable changesCount: float,
+  effect: Internal.effect,
+}
+
+type t
+
+let make: (
+  ~config: Config.t,
+  ~persistence: Persistence.t,
+  ~chainStates: dict<ChainState.t>,
+  ~isInReorgThreshold: bool,
+  ~isRealtime: bool,
+  ~targetBufferSize: int=?,
+  ~committedCheckpointId: Internal.checkpointId=?,
+  ~isDevelopmentMode: bool=?,
+  ~shouldUseTui: bool=?,
+  ~exitAfterFirstEventBlock: bool=?,
+  ~onError: ErrorHandling.t => unit,
+) => t
+
+let makeFromDbState: (
+  ~config: Config.t,
+  ~persistence: Persistence.t,
+  ~initialState: Persistence.initialState,
+  ~registrations: HandlerRegister.registrations,
+  ~isDevelopmentMode: bool=?,
+  ~shouldUseTui: bool=?,
+  ~exitAfterFirstEventBlock: bool=?,
+  ~reducedPollingInterval: int=?,
+  ~targetBufferSize: int=?,
+  ~onError: ErrorHandling.t => unit,
+) => t
+
+let unexpectedErrorMsg: string
+
+// Read guards / transitions.
+let isStale: (t, ~stateId: int) => bool
+let isResolvingReorg: t => bool
+let errorExit: (t, ErrorHandling.t) => unit
+let stop: t => unit
+let getChainState: (t, ~chain: chain) => ChainState.t
+let enterReorgThreshold: t => unit
+let beginReorg: (t, ~chain: chain, ~blockNumber: int) => unit
+let enterFindingReorgDepth: t => unit
+let foundReorgDepth: (t, ~chain: chain, ~rollbackTargetBlockNumber: int) => unit
+let completeRollback: (t, ~eventsProcessedDiffByChain: dict<float>) => unit
+let clearRollback: t => unit
+let invalidateInflight: t => unit
+let applyBatchProgress: (t, ~batch: Batch.t) => unit
+let isProcessing: t => bool
+let beginProcessing: t => unit
+let endProcessing: t => unit
+let recordProcessedBatch: t => unit
+let createBatch: (
+  t,
+  ~processedCheckpointId: Internal.checkpointId,
+  ~batchSizeTarget: int,
+  ~isRollback: bool,
+) => Batch.t
+let getSafeCheckpointId: t => option<Internal.checkpointId>
+
+// Accessors.
+let config: t => Config.t
+let persistence: t => Persistence.t
+let allEntities: t => array<Internal.entityConfig>
+let entities: t => EntityTables.t
+let effects: t => dict<effectCacheInMemTable>
+let committedCheckpointId: t => Internal.checkpointId
+let processedCheckpointId: t => Internal.checkpointId
+let processedBatches: t => array<Batch.t>
+let processedBatchesCount: t => int
+let writeFiber: t => option<promise<unit>>
+let hasFailedWrite: t => bool
+let chainMetaDirty: t => bool
+let chainMetaThrottler: t => Throttler.t
+let crossChainState: t => CrossChainState.t
+let chainStates: t => dict<ChainState.t>
+let isInReorgThreshold: t => bool
+let isRealtime: t => bool
+let rollbackState: t => rollbackState
+let indexerStartTime: t => Date.t
+let loadManager: t => LoadManager.t
+let keepProcessAlive: t => bool
+let exitAfterFirstEventBlock: t => bool
+let isStopped: t => bool
+let epoch: t => int
+let pruneStaleEntityHistoryThrottler: t => Throttler.t
+
+// Store domain operations.
+let queueProcessedBatch: (t, ~batch: Batch.t) => unit
+let drainBatchRun: t => Batch.t
+let takeRollback: t => option<Persistence.rollback>
+let markCommitted: (t, ~upToCheckpointId: Internal.checkpointId) => unit
+let beginRollbackDiff: (
+  t,
+  ~targetCheckpointId: Internal.checkpointId,
+  ~diffCheckpointId: Internal.checkpointId,
+  ~progressBlockNumberByChainId: dict<int>,
+) => unit
+let recordWriteFailure: (t, exn) => unit
+let beginWriteFiber: (t, promise<unit>) => unit
+let endWriteFiber: t => unit
+let wakeCommitWaiters: t => unit
+let addCommitWaiter: (t, unit => unit) => unit
+let stageChainMeta: (t, dict<InternalTable.Chains.metaFields>) => unit
+let takeChainMetaSnapshot: t => option<dict<InternalTable.Chains.metaFields>>