envio 3.2.1 → 3.3.0-alpha.0

package/src/InMemoryStore.res.mjs

@@ -1,84 +1,22 @@
 // Generated by ReScript, PLEASE EDIT WITH CARE
 
-import * as Env from "./Env.res.mjs";
 import * as Table from "./db/Table.res.mjs";
 import * as Utils from "./Utils.res.mjs";
 import * as Config from "./Config.res.mjs";
-import * as Logging from "./Logging.res.mjs";
 import * as Internal from "./Internal.res.mjs";
-import * as Throttler from "./Throttler.res.mjs";
-import * as Prometheus from "./Prometheus.res.mjs";
-import * as Stdlib_Array from "@rescript/runtime/lib/es6/Stdlib_Array.js";
-import * as ErrorHandling from "./ErrorHandling.res.mjs";
+import * as IndexerState from "./IndexerState.res.mjs";
 import * as InMemoryTable from "./InMemoryTable.res.mjs";
 import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
-import * as RollbackCommit from "./RollbackCommit.res.mjs";
-import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
-import * as Primitive_object from "@rescript/runtime/lib/es6/Primitive_object.js";
-import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js";
 import * as S$RescriptSchema from "rescript-schema/src/S.res.mjs";
-import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
 
-let UndefinedEntity = /* @__PURE__ */Primitive_exceptions.create("InMemoryStore.EntityTables.UndefinedEntity");
-
-function make(entities) {
-  let init = {};
-  entities.forEach(entityConfig => {
-    init[entityConfig.name] = InMemoryTable.Entity.make();
-  });
-  return init;
+function getInMemTable(state, entityConfig) {
+  return IndexerState.EntityTables.get(IndexerState.entities(state), entityConfig.name);
 }
 
-function get(self, entityName) {
-  let table = self[entityName];
-  if (table !== undefined) {
-    return table;
-  } else {
-    return ErrorHandling.mkLogAndRaise(undefined, "Unexpected, entity InMemoryTable is undefined", {
-      RE_EXN_ID: UndefinedEntity,
-      entityName: entityName
-    });
-  }
-}
-
-let EntityTables = {
-  UndefinedEntity: UndefinedEntity,
-  make: make,
-  get: get
-};
-
-function make$1(entities, committedCheckpointIdOpt, persistence, config, onError) {
-  let committedCheckpointId = committedCheckpointIdOpt !== undefined ? committedCheckpointIdOpt : Internal.initialCheckpointId;
-  let intervalMillis = Env.ThrottleWrites.chainMetadataIntervalMillis;
-  let chainMetaThrottler = Throttler.make(intervalMillis, Logging.createChild({
-    context: "Throttler for chain metadata writes",
-    intervalMillis: intervalMillis
-  }));
-  return {
-    allEntities: entities,
-    entities: make(entities),
-    effects: {},
-    rollback: undefined,
-    committedCheckpointId: committedCheckpointId,
-    processedCheckpointId: committedCheckpointId,
-    processedBatches: [],
-    isProcessing: false,
-    processedBatchesCount: 0,
-    writeFiber: undefined,
-    hasFailedWrite: false,
-    onError: onError,
-    commitWaiters: [],
-    persistence: persistence,
-    config: config,
-    chainMeta: {},
-    chainMetaDirty: false,
-    chainMetaThrottler: chainMetaThrottler
-  };
-}
-
-function getEffectInMemTable(inMemoryStore, effect) {
+function getEffectInMemTable(state, effect) {
   let key = effect.name;
-  let table = inMemoryStore.effects[key];
+  let effects = IndexerState.effects(state);
+  let table = effects[key];
   if (table !== undefined) {
     return table;
   }
@@ -89,7 +27,7 @@ function getEffectInMemTable(inMemoryStore, effect) {
     changesCount: 0,
     effect: effect
   };
-  inMemoryStore.effects[key] = table$1;
+  effects[key] = table$1;
   return table$1;
 }
 
@@ -155,285 +93,18 @@ function dropCommittedEffects(inMemTable, committedCheckpointId, keepLoadedFromD
   inMemTable.changesCount = inMemTable.changesCount - keysToDelete.length;
 }
 
-function getInMemTable(inMemoryStore, entityConfig) {
-  return get(inMemoryStore.entities, entityConfig.name);
-}
-
-function isRollingBack(inMemoryStore) {
-  return inMemoryStore.rollback !== undefined;
-}
-
-function getChangesCount(inMemoryStore) {
-  let total = {
-    contents: 0
-  };
-  inMemoryStore.allEntities.forEach(entityConfig => {
-    total.contents = total.contents + getInMemTable(inMemoryStore, entityConfig).changesCount;
-  });
-  Utils.Dict.forEach(inMemoryStore.effects, inMemTable => {
-    total.contents = total.contents + inMemTable.changesCount;
-  });
-  inMemoryStore.processedBatches.forEach(batch => {
-    total.contents = total.contents + batch.totalBatchSize;
-  });
-  return total.contents;
-}
-
-function wakeCommitWaiters(inMemoryStore) {
-  let waiters = inMemoryStore.commitWaiters;
-  inMemoryStore.commitWaiters = [];
-  waiters.forEach(resolve => resolve());
-}
-
-function waitForCommit(inMemoryStore) {
-  return new Promise((resolve, param) => {
-    inMemoryStore.commitWaiters.push(resolve);
-  });
-}
-
-function drainBatchRun(inMemoryStore) {
-  let all = inMemoryStore.processedBatches;
-  let isInReorgThreshold = all[0].isInReorgThreshold;
-  let rest = [];
-  let progressedChainsById = {};
-  let totalBatchSize = {
-    contents: 0
-  };
-  let items = [];
-  let checkpointIds = [];
-  let checkpointChainIds = [];
-  let checkpointBlockNumbers = [];
-  let checkpointBlockHashes = [];
-  let checkpointEventsProcessed = [];
-  all.forEach(batch => {
-    if (Utils.$$Array.isEmpty(rest) && batch.isInReorgThreshold === isInReorgThreshold) {
-      Utils.Dict.forEachWithKey(batch.progressedChainsById, (chainAfterBatch, key) => {
-        progressedChainsById[key] = chainAfterBatch;
-      });
-      totalBatchSize.contents = totalBatchSize.contents + batch.totalBatchSize | 0;
-      items.push(...batch.items);
-      checkpointIds.push(...batch.checkpointIds);
-      checkpointChainIds.push(...batch.checkpointChainIds);
-      checkpointBlockNumbers.push(...batch.checkpointBlockNumbers);
-      checkpointBlockHashes.push(...batch.checkpointBlockHashes);
-      checkpointEventsProcessed.push(...batch.checkpointEventsProcessed);
-    } else {
-      rest.push(batch);
-    }
-  });
-  inMemoryStore.processedBatches = rest;
-  return {
-    totalBatchSize: totalBatchSize.contents,
-    items: items,
-    progressedChainsById: progressedChainsById,
-    isInReorgThreshold: isInReorgThreshold,
-    checkpointIds: checkpointIds,
-    checkpointChainIds: checkpointChainIds,
-    checkpointBlockNumbers: checkpointBlockNumbers,
-    checkpointBlockHashes: checkpointBlockHashes,
-    checkpointEventsProcessed: checkpointEventsProcessed
-  };
-}
-
-function snapshotEffects(inMemoryStore, cache) {
-  let acc = [];
-  Utils.Dict.forEach(inMemoryStore.effects, inMemTable => {
-    let idsToStore = inMemTable.idsToStore;
-    let invalidationsCount = inMemTable.invalidationsCount;
-    let dict = inMemTable.dict;
-    let effect = inMemTable.effect;
-    if (idsToStore.length !== 0) {
-      let items = Stdlib_Array.filterMap(idsToStore, id => {
-        let match = dict[id];
-        if (match.type === "SET") {
-          return {
-            id: id,
-            output: match.entity
-          };
-        }
-      });
-      let effectName = effect.name;
-      let c = cache[effectName];
-      let effectCacheRecord;
-      if (c !== undefined) {
-        effectCacheRecord = c;
-      } else {
-        let c$1 = {
-          effectName: effectName,
-          count: 0
-        };
-        cache[effectName] = c$1;
-        effectCacheRecord = c$1;
-      }
-      let shouldInitialize = effectCacheRecord.count === 0;
-      effectCacheRecord.count = (effectCacheRecord.count + items.length | 0) - invalidationsCount | 0;
-      Prometheus.EffectCacheCount.set(effectCacheRecord.count, effectName);
-      acc.push({
-        effect: effect,
-        items: items,
-        shouldInitialize: shouldInitialize
-      });
-    }
-    inMemTable.idsToStore = [];
-    inMemTable.invalidationsCount = 0;
-  });
-  return acc;
-}
-
-async function runOneWrite(inMemoryStore, persistence, config) {
-  let match = persistence.storageStatus;
-  let cache;
-  cache = typeof match !== "object" || match.TAG === "Initializing" ? Stdlib_JsError.throwWithMessage(`Failed to access the indexer storage. The Persistence layer is not initialized.`) : match._0.cache;
-  let chainMetaData = inMemoryStore.chainMetaDirty ? (inMemoryStore.chainMetaDirty = false, Utils.Dict.shallowCopy(inMemoryStore.chainMeta)) : undefined;
-  let match$1 = inMemoryStore.processedBatches;
-  if (match$1.length === 0) {
-    if (chainMetaData !== undefined) {
-      return await persistence.storage.setChainMeta(chainMetaData);
-    } else {
-      return;
-    }
-  }
-  let committedCheckpointId = inMemoryStore.committedCheckpointId;
-  let batch = drainBatchRun(inMemoryStore);
-  let checkpointId = Utils.$$Array.last(batch.checkpointIds);
-  let upToCheckpointId = checkpointId !== undefined ? checkpointId : committedCheckpointId;
-  let rollback = inMemoryStore.rollback;
-  inMemoryStore.rollback = undefined;
-  let updatedEntities = Stdlib_Array.filterMap(persistence.allEntities, entityConfig => {
-    let table = getInMemTable(inMemoryStore, entityConfig);
-    let changes = InMemoryTable.Entity.snapshotChanges(table, committedCheckpointId, upToCheckpointId);
-    if (Utils.$$Array.isEmpty(changes)) {
-      return;
-    } else {
-      return {
-        entityConfig: entityConfig,
-        changes: changes
-      };
-    }
-  });
-  let updatedEffectsCache = snapshotEffects(inMemoryStore, cache);
-  await persistence.storage.writeBatch(batch, rollback, batch.isInReorgThreshold, config, persistence.allEntities, updatedEffectsCache, updatedEntities, chainMetaData);
-  inMemoryStore.committedCheckpointId = upToCheckpointId;
-  if (rollback !== undefined && Utils.$$Array.notEmpty(RollbackCommit.callbacks)) {
-    return await RollbackCommit.fire(rollback.progressBlockNumberByChainId);
-  }
-}
-
-function hasPendingWrite(inMemoryStore) {
-  if (Utils.$$Array.notEmpty(inMemoryStore.processedBatches)) {
-    return true;
-  } else {
-    return inMemoryStore.chainMetaDirty;
-  }
-}
-
-async function runWriteLoop(inMemoryStore) {
-  while (hasPendingWrite(inMemoryStore) && !inMemoryStore.hasFailedWrite) {
-    try {
-      await runOneWrite(inMemoryStore, inMemoryStore.persistence, inMemoryStore.config);
-      wakeCommitWaiters(inMemoryStore);
-    } catch (raw_exn) {
-      let exn = Primitive_exceptions.internalToException(raw_exn);
-      inMemoryStore.hasFailedWrite = true;
-      inMemoryStore.onError(exn);
-    }
-  };
-  inMemoryStore.writeFiber = undefined;
-  return wakeCommitWaiters(inMemoryStore);
-}
-
-function kick(inMemoryStore) {
-  if (Stdlib_Option.isNone(inMemoryStore.writeFiber) && !inMemoryStore.hasFailedWrite && hasPendingWrite(inMemoryStore)) {
-    inMemoryStore.writeFiber = runWriteLoop(inMemoryStore);
-    return;
-  }
-}
-
-function metaFieldsEqual(a, b) {
-  if (Primitive_object.equal(a.first_event_block, b.first_event_block) && a.buffer_block === b.buffer_block && a._is_hyper_sync === b._is_hyper_sync) {
-    return Primitive_object.equal(Stdlib_Option.map(Primitive_option.fromNull(a.ready_at), prim => prim.getTime()), Stdlib_Option.map(Primitive_option.fromNull(b.ready_at), prim => prim.getTime()));
-  } else {
-    return false;
-  }
-}
-
-function setChainMeta(inMemoryStore, chainsData) {
-  Utils.Dict.forEachWithKey(chainsData, (meta, chainId) => {
-    let prev = inMemoryStore.chainMeta[chainId];
-    let changed = prev !== undefined ? !metaFieldsEqual(meta, prev) : true;
-    if (changed) {
-      inMemoryStore.chainMeta[chainId] = meta;
-      inMemoryStore.chainMetaDirty = true;
-      return;
-    }
-  });
-  if (inMemoryStore.chainMetaDirty) {
-    return Throttler.schedule(inMemoryStore.chainMetaThrottler, () => {
-      kick(inMemoryStore);
-      return Promise.resolve();
-    });
-  }
-}
-
-function commitBatch(inMemoryStore, batch) {
-  inMemoryStore.processedBatches.push(batch);
-  let checkpointId = Utils.$$Array.last(batch.checkpointIds);
-  if (checkpointId !== undefined) {
-    inMemoryStore.processedCheckpointId = checkpointId;
-  }
-  kick(inMemoryStore);
-}
-
-function dropCommitted(inMemoryStore, keepLoadedFromDb) {
-  let committedCheckpointId = inMemoryStore.committedCheckpointId;
-  inMemoryStore.allEntities.forEach(entityConfig => InMemoryTable.Entity.dropCommittedChanges(getInMemTable(inMemoryStore, entityConfig), committedCheckpointId, keepLoadedFromDb));
-  Utils.Dict.forEach(inMemoryStore.effects, inMemTable => dropCommittedEffects(inMemTable, committedCheckpointId, keepLoadedFromDb));
-}
-
-async function awaitCapacity(inMemoryStore) {
-  if (!inMemoryStore.hasFailedWrite && getChangesCount(inMemoryStore) >= Env.inMemoryObjectsTarget) {
-    dropCommitted(inMemoryStore, true);
-    if (getChangesCount(inMemoryStore) >= Env.inMemoryObjectsTarget) {
-      dropCommitted(inMemoryStore, false);
-    }
-    if (getChangesCount(inMemoryStore) >= Env.inMemoryObjectsTarget && Utils.$$Array.notEmpty(inMemoryStore.processedBatches)) {
-      kick(inMemoryStore);
-      await waitForCommit(inMemoryStore);
-      return await awaitCapacity(inMemoryStore);
-    } else {
-      return;
-    }
-  }
-}
-
-async function flush(inMemoryStore) {
-  if (inMemoryStore.hasFailedWrite) {
-    return;
-  }
-  kick(inMemoryStore);
-  let fiber = inMemoryStore.writeFiber;
-  if (fiber !== undefined) {
-    await fiber;
-    return await flush(inMemoryStore);
-  }
-}
-
-async function prepareRollbackDiff(inMemoryStore, persistence, rollbackTargetCheckpointId, rollbackDiffCheckpointId, progressBlockNumberByChainId) {
-  inMemoryStore.entities = make(inMemoryStore.allEntities);
-  inMemoryStore.effects = {};
-  inMemoryStore.rollback = {
-    targetCheckpointId: rollbackTargetCheckpointId,
-    diffCheckpointId: rollbackDiffCheckpointId,
-    progressBlockNumberByChainId: progressBlockNumberByChainId
-  };
+async function prepareRollbackDiff(state, rollbackTargetCheckpointId, rollbackDiffCheckpointId, progressBlockNumberByChainId) {
+  IndexerState.beginRollbackDiff(state, rollbackTargetCheckpointId, rollbackDiffCheckpointId, progressBlockNumberByChainId);
+  let persistence = IndexerState.persistence(state);
+  let committedCheckpointId = IndexerState.committedCheckpointId(state);
   let deletedEntities = {};
   let setEntities = {};
   await Promise.all(persistence.allEntities.map(async entityConfig => {
-    let entityTable = getInMemTable(inMemoryStore, entityConfig);
+    let entityTable = getInMemTable(state, entityConfig);
     let match = await persistence.storage.getRollbackData(entityConfig, rollbackTargetCheckpointId);
     match[0].forEach(data => {
       Utils.Dict.push(deletedEntities, entityConfig.name, data.id);
-      InMemoryTable.Entity.set(entityTable, inMemoryStore.committedCheckpointId, {
+      InMemoryTable.Entity.set(entityTable, committedCheckpointId, {
         type: "DELETE",
         entityId: data.id,
         checkpointId: rollbackDiffCheckpointId
@@ -442,7 +113,7 @@ async function prepareRollbackDiff(inMemoryStore, persistence, rollbackTargetChe
     let restoredEntities = S$RescriptSchema.parseOrThrow(match[1], Table.pgRowsSchema(entityConfig.table));
     restoredEntities.forEach(entity => {
       Utils.Dict.push(setEntities, entityConfig.name, entity.id);
-      InMemoryTable.Entity.set(entityTable, inMemoryStore.committedCheckpointId, {
+      InMemoryTable.Entity.set(entityTable, committedCheckpointId, {
         type: "SET",
         entityId: entity.id,
         entity: entity,
@@ -456,8 +127,9 @@ async function prepareRollbackDiff(inMemoryStore, persistence, rollbackTargetChe
   };
 }
 
-function setBatchDcs(inMemoryStore, batch) {
-  let inMemTable = getInMemTable(inMemoryStore, Config.EnvioAddresses.entityConfig);
+function setBatchDcs(state, batch) {
+  let inMemTable = getInMemTable(state, Config.EnvioAddresses.entityConfig);
+  let committedCheckpointId = IndexerState.committedCheckpointId(state);
   let itemIdx = 0;
   for (let checkpoint = 0, checkpoint_finish = batch.checkpointIds.length; checkpoint < checkpoint_finish; ++checkpoint) {
     let checkpointId = batch.checkpointIds[checkpoint];
@@ -480,7 +152,7 @@ function setBatchDcs(inMemoryStore, batch) {
             registration_log_index: entity_registration_log_index,
             contract_name: entity_contract_name
           };
-          InMemoryTable.Entity.set(inMemTable, inMemoryStore.committedCheckpointId, {
+          InMemoryTable.Entity.set(inMemTable, committedCheckpointId, {
             type: "SET",
             entityId: entity_id,
             entity: entity,
@@ -493,36 +165,15 @@ function setBatchDcs(inMemoryStore, batch) {
   }
 }
 
-let keepLatestChangesLimit = Env.inMemoryObjectsTarget;
-
 export {
-  EntityTables,
-  make$1 as make,
-  keepLatestChangesLimit,
+  getInMemTable,
   getEffectInMemTable,
   hasEffectOutput,
   getEffectOutputUnsafe,
   setEffectOutput,
   initEffectOutputFromDb,
   dropCommittedEffects,
-  getInMemTable,
-  isRollingBack,
-  getChangesCount,
-  wakeCommitWaiters,
-  waitForCommit,
-  drainBatchRun,
-  snapshotEffects,
-  runOneWrite,
-  hasPendingWrite,
-  runWriteLoop,
-  kick,
-  metaFieldsEqual,
-  setChainMeta,
-  commitBatch,
-  dropCommitted,
-  awaitCapacity,
-  flush,
   prepareRollbackDiff,
   setBatchDcs,
 }
-/* Env Not a pure module */
+/* Table Not a pure module */

package/src/IndexerLoop.res

@@ -0,0 +1,44 @@
+// The indexer run loop: it owns scheduling and wires the operation modules
+// together. Each operation (ChainFetching, BatchProcessing, Rollback) is handed
+// the schedule* effects it needs to re-enter the loop; this is the only module
+// that knows how they connect. State and its transitions live in IndexerState.
+
+// Fire-and-forget an async step. Every launchable owns a try/catch that routes
+// failures to errorExit, so there's no rejection to swallow here.
+@inline
+let launch = (state: IndexerState.t, work: unit => promise<unit>) =>
+  if !(state->IndexerState.isStopped) {
+    work()->Promise.ignore
+  }
+
+// Kick off the indexer loops. The schedule* effects are mutually recursive
+// (fetch kicks process/rollback, which kick fetch again), so they're defined as
+// one `let rec` block and threaded into the operations as the only way back in.
+let start = (state: IndexerState.t) => {
+  let rec scheduleFetch = () =>
+    launch(state, () =>
+      state
+      ->IndexerState.crossChainState
+      ->CrossChainState.checkAndFetch(~fetchChain=(~chain, ~concurrencyLimit, ~bufferLimit) =>
+        ChainFetching.fetchChain(
+          state,
+          chain,
+          ~concurrencyLimit,
+          ~bufferLimit,
+          ~stateId=state->IndexerState.epoch,
+          ~scheduleFetch,
+          ~scheduleProcessing,
+          ~scheduleRollback,
+        )
+      )
+    )
+  and scheduleProcessing = () =>
+    launch(state, () => BatchProcessing.startProcessing(state, ~scheduleFetch, ~scheduleRollback))
+  and scheduleRollback = () =>
+    launch(state, () =>
+      Rollback.rollback(state, ~scheduleFetch, ~scheduleProcessing, ~scheduleRollback)
+    )
+
+  scheduleFetch()
+  scheduleProcessing()
+}

package/src/IndexerLoop.res.mjs

@@ -0,0 +1,46 @@
+// Generated by ReScript, PLEASE EDIT WITH CARE
+
+import * as Rollback from "./Rollback.res.mjs";
+import * as IndexerState from "./IndexerState.res.mjs";
+import * as ChainFetching from "./ChainFetching.res.mjs";
+import * as BatchProcessing from "./BatchProcessing.res.mjs";
+import * as CrossChainState from "./CrossChainState.res.mjs";
+
+function launch(state, work) {
+  if (!IndexerState.isStopped(state)) {
+    work();
+    return;
+  }
+}
+
+function start(state) {
+  let scheduleFetch = () => {
+    let work = () => CrossChainState.checkAndFetch(IndexerState.crossChainState(state), (chain, concurrencyLimit, bufferLimit) => ChainFetching.fetchChain(state, chain, concurrencyLimit, bufferLimit, IndexerState.epoch(state), scheduleFetch, scheduleProcessing, scheduleRollback));
+    if (!IndexerState.isStopped(state)) {
+      work();
+      return;
+    }
+  };
+  let scheduleProcessing = () => {
+    let work = () => BatchProcessing.startProcessing(state, scheduleFetch, scheduleRollback);
+    if (!IndexerState.isStopped(state)) {
+      work();
+      return;
+    }
+  };
+  let scheduleRollback = () => {
+    let work = () => Rollback.rollback(state, scheduleFetch, scheduleProcessing, scheduleRollback);
+    if (!IndexerState.isStopped(state)) {
+      work();
+      return;
+    }
+  };
+  scheduleFetch();
+  scheduleProcessing();
+}
+
+export {
+  launch,
+  start,
+}
+/* Rollback Not a pure module */