envio 3.5.0-alpha.2 → 3.5.0-rc.0

package/src/FinalizeBackfill.res

@@ -0,0 +1,61 @@
+// The FinalizingIndexes phase. Reached from the processing loop when every
+// chain has caught up: processing is already paused (the loop awaits this),
+// pending writes are flushed, then storage builds every missing schema-defined
+// index and, once they all verify, commits `ready_at`. A failure part-way
+// leaves the indexes built so far in place and reaches the processing loop's
+// error boundary; the retry only owes what's left. Either way the indexer never
+// reports ready with an index the schema promised still missing.
+
+let runOnce = async (state: IndexerState.t) => {
+  Logging.info(
+    "All chains are caught up. Finalizing the indexer before switching to realtime: flushing pending writes, then creating the indexes the schema promises.",
+  )
+
+  await Writing.flush(state)
+
+  // A failed write already surfaced through onError; committing ready_at on top
+  // of an incomplete write would claim progress that isn't durable.
+  if !(state->IndexerState.hasFailedWrite) {
+    let persistence = state->IndexerState.persistence
+    let storage = persistence->Persistence.getInitializedStorageOrThrow
+    let readyAt = Date.make()
+
+    await storage.finalizeBackfill(
+      ~entities=persistence.allEntities,
+      ~chainIds=state
+      ->IndexerState.chainStates
+      ->Dict.valuesToArray
+      ->Array.map(cs => (cs->ChainState.chainConfig).id),
+      ~readyAt,
+    )
+
+    // Only after the commit: in-memory readiness must never run ahead of the
+    // `ready_at` a restart would read back.
+    state->IndexerState.markReady(~readyAt)
+    Logging.info("The indexer is ready. Switching to realtime indexing.")
+  }
+}
+
+// Several paths reach the phase — a processed batch, a tick that progressed
+// nothing — and a height update can bring another one round while the first is
+// still building. They all join the in-flight run rather than starting a second
+// pass over the same indexes.
+let run = (state: IndexerState.t) =>
+  switch state->IndexerState.finalizeFiber {
+  | Some(fiber) => fiber
+  | None =>
+    let fiber = runOnce(state)->Promise.finally(() => state->IndexerState.endFinalizeFiber)
+    state->IndexerState.beginFinalizeFiber(fiber)
+    fiber
+  }
+
+// An indexer resumed already ready never reaches `run`, so nothing above would
+// notice an index the schema promises being dropped or invalidated while it was
+// down — `ensureQueryIndexes` only covers what a getWhere actually asks for.
+// Best-effort and not awaited by the loop: indexing is already live and correct
+// without the index, just slower.
+let repairSchemaIndexes = (state: IndexerState.t) => {
+  let persistence = state->IndexerState.persistence
+  let storage = persistence->Persistence.getInitializedStorageOrThrow
+  storage.ensureSchemaIndexes(~entities=persistence.allEntities)
+}

package/src/FinalizeBackfill.res.mjs

@@ -0,0 +1,44 @@
+// Generated by ReScript, PLEASE EDIT WITH CARE
+
+import * as Logging from "./Logging.res.mjs";
+import * as Writing from "./Writing.res.mjs";
+import * as ChainState from "./ChainState.res.mjs";
+import * as Persistence from "./Persistence.res.mjs";
+import * as IndexerState from "./IndexerState.res.mjs";
+
+async function runOnce(state) {
+  Logging.info("All chains are caught up. Finalizing the indexer before switching to realtime: flushing pending writes, then creating the indexes the schema promises.");
+  await Writing.flush(state);
+  if (IndexerState.hasFailedWrite(state)) {
+    return;
+  }
+  let persistence = IndexerState.persistence(state);
+  let storage = Persistence.getInitializedStorageOrThrow(persistence);
+  let readyAt = new Date();
+  await storage.finalizeBackfill(persistence.allEntities, Object.values(IndexerState.chainStates(state)).map(cs => ChainState.chainConfig(cs).id), readyAt);
+  IndexerState.markReady(state, readyAt);
+  return Logging.info("The indexer is ready. Switching to realtime indexing.");
+}
+
+function run(state) {
+  let fiber = IndexerState.finalizeFiber(state);
+  if (fiber !== undefined) {
+    return fiber;
+  }
+  let fiber$1 = runOnce(state).finally(() => IndexerState.endFinalizeFiber(state));
+  IndexerState.beginFinalizeFiber(state, fiber$1);
+  return fiber$1;
+}
+
+function repairSchemaIndexes(state) {
+  let persistence = IndexerState.persistence(state);
+  let storage = Persistence.getInitializedStorageOrThrow(persistence);
+  return storage.ensureSchemaIndexes(persistence.allEntities);
+}
+
+export {
+  runOnce,
+  run,
+  repairSchemaIndexes,
+}
+/* Logging Not a pure module */

package/src/HandlerRegister.res

@@ -82,8 +82,8 @@ let startRegistration = (~config: Config.t) => {
   }
 }
 
-let getChainRegistrations = (r: activeRegistration, ~chainId: int): chainRegistrations => {
-  let key = chainId->Int.toString
+let getChainRegistrations = (r: activeRegistration, ~chainId: ChainId.t): chainRegistrations => {
+  let key = chainId->ChainId.toString
   switch r.registrationsByChainId->Utils.Dict.dangerouslyGetNonOption(key) {
   | Some(existing) => existing
   | None =>
@@ -98,7 +98,7 @@ let getChainRegistrations = (r: activeRegistration, ~chainId: int): chainRegistr
 
 let buildOnEventRegistrationWith = (
   ~config: Config.t,
-  ~chainId: int,
+  ~chainId: ChainId.t,
   ~eventConfig: Internal.eventConfig,
   ~isWildcard: bool,
   ~handler: option<Internal.handler>,
@@ -342,10 +342,10 @@ let setContractRegister = (~contractName, ~eventName, contractRegister, ~eventOp
 }
 
 // Raw onEvent registrations stored for a chain (empty if the chain has none).
-let storedOnEventRegistrations = (r: activeRegistration, ~chainId: int): array<
+let storedOnEventRegistrations = (r: activeRegistration, ~chainId: ChainId.t): array<
   Internal.onEventRegistration,
 > =>
-  switch r.registrationsByChainId->Utils.Dict.dangerouslyGetNonOption(chainId->Int.toString) {
+  switch r.registrationsByChainId->Utils.Dict.dangerouslyGetNonOption(chainId->ChainId.toString) {
   | Some(chainRegs) => chainRegs.onEventRegistrations
   | None => []
   }
@@ -373,7 +373,7 @@ let isWildcard = (~contractName, ~eventName) =>
 // item still produces an item to run.
 let getSimulateOnEventRegistrations = (
   ~config: Config.t,
-  ~chainId: int,
+  ~chainId: ChainId.t,
   ~eventConfig: Internal.eventConfig,
 ): array<Internal.onEventRegistration> => {
   let stored = switch getActiveRegistration() {
@@ -412,7 +412,7 @@ let finishRegistration = (~config: Config.t): registrationsByChainId => {
       ->ChainMap.values
       ->Array.forEach(chainConfig => {
         let chainId = chainConfig.id
-        let key = chainId->Int.toString
+        let key = chainId->ChainId.toString
 
         let builtRegs = mergeRegistrations(r->storedOnEventRegistrations(~chainId), ~config)
         let registeredKeys = Utils.Set.make()
@@ -638,7 +638,7 @@ let registerOnBlock = (
     ->ChainMap.values
     ->Array.forEach(chainConfig => {
       let chainId = chainConfig.id
-      let chainObj = chainsDict->Dict.getUnsafe(chainId->Int.toString)
+      let chainObj = chainsDict->Dict.getUnsafe(chainId->ChainId.toString)
 
       // Predicate returns `true` → match with no filter; `false` → skip;
       // any plain object → structured filter. `undefined`/`null` returns

package/src/HandlerRegister.res.mjs

@@ -1,6 +1,7 @@
 // Generated by ReScript, PLEASE EDIT WITH CARE
 
 import * as Utils from "./Utils.res.mjs";
+import * as ChainId from "./ChainId.res.mjs";
 import * as Logging from "./Logging.res.mjs";
 import * as ChainMap from "./ChainMap.res.mjs";
 import * as EnvioGlobal from "./EnvioGlobal.res.mjs";
@@ -54,7 +55,7 @@ function startRegistration(config) {
 }
 
 function getChainRegistrations(r, chainId) {
-  let key = chainId.toString();
+  let key = ChainId.toString(chainId);
   let existing = r.registrationsByChainId[key];
   if (existing !== undefined) {
     return existing;
@@ -220,7 +221,7 @@ function setContractRegister(contractName, eventName, contractRegister, eventOpt
 }
 
 function storedOnEventRegistrations(r, chainId) {
-  let chainRegs = r.registrationsByChainId[chainId.toString()];
+  let chainRegs = r.registrationsByChainId[ChainId.toString(chainId)];
   if (chainRegs !== undefined) {
     return chainRegs.onEventRegistrations;
   } else {
@@ -270,7 +271,7 @@ function finishRegistration(config) {
   let registrationsByChainId = {};
   ChainMap.values(config.chainMap).forEach(chainConfig => {
     let chainId = chainConfig.id;
-    let key = chainId.toString();
+    let key = ChainId.toString(chainId);
     let builtRegs = mergeRegistrations(storedOnEventRegistrations(r, chainId), config);
     let registeredKeys = new Set();
     builtRegs.forEach(reg => {
@@ -385,7 +386,7 @@ function registerOnBlock(name, where, handler, getChainsObject) {
     };
     ChainMap.values(config.chainMap).forEach(chainConfig => {
       let chainId = chainConfig.id;
-      let chainObj = chainsDict[chainId.toString()];
+      let chainObj = chainsDict[ChainId.toString(chainId)];
       let result = where$1 !== undefined ? where$1({
           chain: chainObj
         }) : true;

package/src/HandlerRegister.resi

@@ -26,7 +26,7 @@ let isWildcard: (~contractName: string, ~eventName: string) => bool
 let isDroppedByWhere: (~config: Config.t, Internal.onEventRegistration) => bool
 let getSimulateOnEventRegistrations: (
   ~config: Config.t,
-  ~chainId: int,
+  ~chainId: ChainId.t,
   ~eventConfig: Internal.eventConfig,
 ) => array<Internal.onEventRegistration>
 

package/src/InMemoryStore.res

@@ -108,7 +108,11 @@ let prepareRollbackDiff = async (
   let deletedEntities = Dict.make()
   let setEntities = Dict.make()
 
+  // Rollback data comes from Postgres entity history, which is kept only for
+  // Postgres-backed entities. ClickHouse-only entities have no history to
+  // restore from, so a reorg leaves them un-reverted in the sink.
   let _ = await persistence.allEntities
+  ->Array.filter(entityConfig => entityConfig.storage.postgres)
   ->Array.map(async entityConfig => {
     let entityTable = state->getInMemTable(~entityConfig)
 

package/src/InMemoryStore.res.mjs

@@ -86,7 +86,7 @@ async function prepareRollbackDiff(state, rollbackTargetCheckpointId, rollbackDi
   let committedCheckpointId = IndexerState.committedCheckpointId(state);
   let deletedEntities = {};
   let setEntities = {};
-  await Promise.all(persistence.allEntities.map(async entityConfig => {
+  await Promise.all(persistence.allEntities.filter(entityConfig => entityConfig.storage.postgres).map(async entityConfig => {
     let entityTable = getInMemTable(state, entityConfig);
     let match = await persistence.storage.getRollbackData(entityConfig, rollbackTargetCheckpointId);
     match[0].forEach(entityId => {

package/src/InMemoryTable.res

@@ -2,7 +2,7 @@ module Entity = {
   type relatedEntityId = string
   type filterWithRelatedIds = (EntityFilter.t, Utils.Set.t<relatedEntityId>)
   // Keyed by EntityFilter.toString
-  type filterIndices = dict<filterWithRelatedIds>
+  type filterIndexes = dict<filterWithRelatedIds>
 
   type entityFilters = Utils.Set.t<EntityFilter.t>
   type t = {
@@ -14,7 +14,7 @@ module Entity = {
     // previous changes persist in the background.
     mutable prevEntityChanges: array<Change.t<Internal.entity>>,
     mutable filtersByEntityId: dict<entityFilters>,
-    mutable filterIndices: filterIndices,
+    mutable filterIndexes: filterIndexes,
   }
 
   // Helper to extract an entity's id as a dict key. The raw id may be a
@@ -44,7 +44,7 @@ module Entity = {
     changesCount: 0.,
     prevEntityChanges: [],
     filtersByEntityId: Dict.make(),
-    filterIndices: Dict.make(),
+    filterIndexes: Dict.make(),
   }
 
   // Changes to persist for checkpoints in (committedCheckpointId, upToCheckpointId].
@@ -81,7 +81,7 @@ module Entity = {
   }
 
   // Frees committed changes: drops latest entries at or below committedCheckpointId
-  // (re-readable from the db) and clears the per-batch indices (rebuilt on the next
+  // (re-readable from the db) and clears the per-batch indexes (rebuilt on the next
   // getWhere). Uncommitted changes are kept. With keepLoadedFromDb, entries seeded
   // from a db read are spared so the cheaper-to-re-derive writes are dropped first.
   let dropCommittedChanges = (self: t, ~committedCheckpointId, ~keepLoadedFromDb) => {
@@ -98,10 +98,10 @@ module Entity = {
     keysToDelete->Array.forEach(key => self.latestEntityChangeById->Utils.Dict.deleteInPlace(key))
     self.changesCount = self.changesCount -. keysToDelete->Array.length->Int.toFloat
     self.filtersByEntityId = Dict.make()
-    self.filterIndices = Dict.make()
+    self.filterIndexes = Dict.make()
   }
 
-  let updateIndices = (self: t, ~entity: Internal.entity) => {
+  let updateIndexes = (self: t, ~entity: Internal.entity) => {
     let entityId = entity->getEntityIdUnsafe
     let entityAsDict = entity->(Utils.magic: Internal.entity => dict<EntityFilter.FieldValue.t>)
 
@@ -116,7 +116,7 @@ module Entity = {
       })
     }
 
-    self.filterIndices->Utils.Dict.forEach(((filter, relatedEntityIds)) => {
+    self.filterIndexes->Utils.Dict.forEach(((filter, relatedEntityIds)) => {
       if filter->EntityFilter.matches(~entity=entityAsDict) {
         //Add entity id to the filter index and the filter to entity filters
         relatedEntityIds->Utils.Set.add(entityId)->ignore
@@ -127,12 +127,12 @@ module Entity = {
     })
   }
 
-  let deleteEntityFromIndices = (self: t, ~entityId: string) =>
+  let deleteEntityFromIndexes = (self: t, ~entityId: string) =>
     switch self.filtersByEntityId->Utils.Dict.dangerouslyGetNonOption(entityId) {
     | None => ()
     | Some(entityFilters) =>
       entityFilters->Utils.Set.forEach(filter => {
-        switch self.filterIndices->Utils.Dict.dangerouslyGetNonOption(
+        switch self.filterIndexes->Utils.Dict.dangerouslyGetNonOption(
           filter->EntityFilter.toString,
         ) {
         | Some((_filter, relatedEntityIds)) =>
@@ -159,8 +159,8 @@ module Entity = {
     }
 
     switch change {
-    | Set({entity}) => inMemTable->updateIndices(~entity)
-    | Delete({entityId}) => inMemTable->deleteEntityFromIndices(~entityId=entityId->EntityId.toKey)
+    | Set({entity}) => inMemTable->updateIndexes(~entity)
+    | Delete({entityId}) => inMemTable->deleteEntityFromIndexes(~entityId=entityId->EntityId.toKey)
     }
     inMemTable.latestEntityChangeById->Dict.set(entityKey, change)
   }
@@ -200,12 +200,12 @@ module Entity = {
 
   let hasIndex = (inMemTable: t) =>
     (filterKey: string) =>
-      inMemTable.filterIndices->Utils.Dict.dangerouslyGetNonOption(filterKey) !== None
+      inMemTable.filterIndexes->Utils.Dict.dangerouslyGetNonOption(filterKey) !== None
 
   let getUnsafeOnIndex = (inMemTable: t) => {
     let getEntity = inMemTable->getUnsafe
     (filterKey: string) => {
-      switch inMemTable.filterIndices->Utils.Dict.dangerouslyGetNonOption(filterKey) {
+      switch inMemTable.filterIndexes->Utils.Dict.dangerouslyGetNonOption(filterKey) {
       | None =>
         JsError.throwWithMessage(`Unexpected error. Must have an index for the filter ${filterKey}`)
       | Some((_filter, relatedEntityIds)) =>
@@ -223,7 +223,7 @@ module Entity = {
 
   let addEmptyIndex = (inMemTable: t, ~filter: EntityFilter.t) => {
     let filterKey = filter->EntityFilter.toString
-    switch inMemTable.filterIndices->Utils.Dict.dangerouslyGetNonOption(filterKey) {
+    switch inMemTable.filterIndexes->Utils.Dict.dangerouslyGetNonOption(filterKey) {
     | Some(_) => () //Should not happen, this means the index already exists
     | None =>
       let relatedEntityIds = Utils.Set.make()
@@ -240,7 +240,7 @@ module Entity = {
         | None => ()
         }
       })
-      inMemTable.filterIndices->Dict.set(filterKey, (filter, relatedEntityIds))
+      inMemTable.filterIndexes->Dict.set(filterKey, (filter, relatedEntityIds))
     }
   }
 }

package/src/InMemoryTable.res.mjs

@@ -40,7 +40,7 @@ function make() {
     changesCount: 0,
     prevEntityChanges: [],
     filtersByEntityId: {},
-    filterIndices: {}
+    filterIndexes: {}
   };
 }
 
@@ -84,10 +84,10 @@ function dropCommittedChanges(self, committedCheckpointId, keepLoadedFromDb) {
   keysToDelete.forEach(key => Utils.Dict.deleteInPlace(self.latestEntityChangeById, key));
   self.changesCount = self.changesCount - keysToDelete.length;
   self.filtersByEntityId = {};
-  self.filterIndices = {};
+  self.filterIndexes = {};
 }
 
-function updateIndices(self, entity) {
+function updateIndexes(self, entity) {
   let entityId = getEntityIdUnsafe(entity);
   let entityFilters = self.filtersByEntityId[entityId];
   if (entityFilters !== undefined) {
@@ -99,7 +99,7 @@ function updateIndices(self, entity) {
       }
     });
   }
-  Utils.Dict.forEach(self.filterIndices, param => {
+  Utils.Dict.forEach(self.filterIndexes, param => {
     let relatedEntityIds = param[1];
     let filter = param[0];
     if (EntityFilter.matches(filter, entity)) {
@@ -111,14 +111,14 @@ function updateIndices(self, entity) {
   });
 }
 
-function deleteEntityFromIndices(self, entityId) {
+function deleteEntityFromIndexes(self, entityId) {
   let entityFilters = self.filtersByEntityId[entityId];
   if (entityFilters === undefined) {
     return;
   }
   let entityFilters$1 = Primitive_option.valFromOption(entityFilters);
   entityFilters$1.forEach(filter => {
-    let match = self.filterIndices[EntityFilter.toString(filter)];
+    let match = self.filterIndexes[EntityFilter.toString(filter)];
     if (match !== undefined) {
       match[1].delete(entityId);
     }
@@ -139,9 +139,9 @@ function set(inMemTable, committedCheckpointId, change) {
     inMemTable.changesCount = inMemTable.changesCount + 1;
   }
   if (change.type === "SET") {
-    updateIndices(inMemTable, change.entity);
+    updateIndexes(inMemTable, change.entity);
   } else {
-    deleteEntityFromIndices(inMemTable, EntityId.toKey(change.entityId));
+    deleteEntityFromIndexes(inMemTable, EntityId.toKey(change.entityId));
   }
   inMemTable.latestEntityChangeById[entityKey] = change;
 }
@@ -174,13 +174,13 @@ function getUnsafe(inMemTable) {
 }
 
 function hasIndex(inMemTable) {
-  return filterKey => inMemTable.filterIndices[filterKey] !== undefined;
+  return filterKey => inMemTable.filterIndexes[filterKey] !== undefined;
 }
 
 function getUnsafeOnIndex(inMemTable) {
   let getEntity = getUnsafe(inMemTable);
   return filterKey => {
-    let match = inMemTable.filterIndices[filterKey];
+    let match = inMemTable.filterIndexes[filterKey];
     if (match !== undefined) {
       return Stdlib_Array.filterMap(Array.from(match[1]), entityId => {
         if (entityId in inMemTable.latestEntityChangeById) {
@@ -195,7 +195,7 @@ function getUnsafeOnIndex(inMemTable) {
 
 function addEmptyIndex(inMemTable, filter) {
   let filterKey = EntityFilter.toString(filter);
-  let match = inMemTable.filterIndices[filterKey];
+  let match = inMemTable.filterIndexes[filterKey];
   if (match !== undefined) {
     return;
   }
@@ -212,7 +212,7 @@ function addEmptyIndex(inMemTable, filter) {
     getOrCreateEntityFilters(inMemTable, entityId).add(filter);
     relatedEntityIds.add(entityId);
   });
-  inMemTable.filterIndices[filterKey] = [
+  inMemTable.filterIndexes[filterKey] = [
     filter,
     relatedEntityIds
   ];
@@ -225,8 +225,8 @@ let Entity = {
   make: make,
   snapshotChanges: snapshotChanges,
   dropCommittedChanges: dropCommittedChanges,
-  updateIndices: updateIndices,
-  deleteEntityFromIndices: deleteEntityFromIndices,
+  updateIndexes: updateIndexes,
+  deleteEntityFromIndexes: deleteEntityFromIndexes,
   set: set,
   initValue: initValue,
   mapChangeToEntity: mapChangeToEntity,

package/src/IndexerLoop.res

@@ -19,10 +19,10 @@ let start = (state: IndexerState.t) => {
     launch(state, () =>
       state
       ->IndexerState.crossChainState
-      ->CrossChainState.checkAndFetch(~dispatchChain=(~chain, ~action) =>
+      ->CrossChainState.checkAndFetch(~dispatchChain=(~chainId, ~action) =>
         ChainFetching.fetchChain(
           state,
-          chain,
+          chainId,
           ~action,
           ~stateId=state->IndexerState.epoch,
           ~scheduleFetch,
@@ -38,6 +38,13 @@ let start = (state: IndexerState.t) => {
       Rollback.rollback(state, ~scheduleFetch, ~scheduleProcessing, ~scheduleRollback)
     )
 
+  // Resuming already ready means the FinalizingIndexes phase is behind us and
+  // won't run again, so this is the only pass that can restore an index the
+  // database lost while the indexer was down.
+  if state->IndexerState.isRealtime {
+    launch(state, () => FinalizeBackfill.repairSchemaIndexes(state))
+  }
+
   scheduleFetch()
   scheduleProcessing()
 }

package/src/IndexerLoop.res.mjs

@@ -5,6 +5,7 @@ import * as IndexerState from "./IndexerState.res.mjs";
 import * as ChainFetching from "./ChainFetching.res.mjs";
 import * as BatchProcessing from "./BatchProcessing.res.mjs";
 import * as CrossChainState from "./CrossChainState.res.mjs";
+import * as FinalizeBackfill from "./FinalizeBackfill.res.mjs";
 
 function launch(state, work) {
   if (!IndexerState.isStopped(state)) {
@@ -15,7 +16,7 @@ function launch(state, work) {
 
 function start(state) {
   let scheduleFetch = () => {
-    let work = () => CrossChainState.checkAndFetch(IndexerState.crossChainState(state), (chain, action) => ChainFetching.fetchChain(state, chain, action, IndexerState.epoch(state), scheduleFetch, scheduleProcessing, scheduleRollback));
+    let work = () => CrossChainState.checkAndFetch(IndexerState.crossChainState(state), (chainId, action) => ChainFetching.fetchChain(state, chainId, action, IndexerState.epoch(state), scheduleFetch, scheduleProcessing, scheduleRollback));
     if (!IndexerState.isStopped(state)) {
       work();
       return;
@@ -35,6 +36,12 @@ function start(state) {
       return;
     }
   };
+  if (IndexerState.isRealtime(state)) {
+    let work = () => FinalizeBackfill.repairSchemaIndexes(state);
+    if (!IndexerState.isStopped(state)) {
+      work();
+    }
+  }
   scheduleFetch();
   scheduleProcessing();
 }