envio 3.5.0-alpha.2 → 3.5.0-alpha.3

package/src/InMemoryTable.res.mjs

@@ -40,7 +40,7 @@ function make() {
     changesCount: 0,
     prevEntityChanges: [],
     filtersByEntityId: {},
-    filterIndices: {}
+    filterIndexes: {}
   };
 }
 
@@ -84,10 +84,10 @@ function dropCommittedChanges(self, committedCheckpointId, keepLoadedFromDb) {
   keysToDelete.forEach(key => Utils.Dict.deleteInPlace(self.latestEntityChangeById, key));
   self.changesCount = self.changesCount - keysToDelete.length;
   self.filtersByEntityId = {};
-  self.filterIndices = {};
+  self.filterIndexes = {};
 }
 
-function updateIndices(self, entity) {
+function updateIndexes(self, entity) {
   let entityId = getEntityIdUnsafe(entity);
   let entityFilters = self.filtersByEntityId[entityId];
   if (entityFilters !== undefined) {
@@ -99,7 +99,7 @@ function updateIndices(self, entity) {
       }
     });
   }
-  Utils.Dict.forEach(self.filterIndices, param => {
+  Utils.Dict.forEach(self.filterIndexes, param => {
     let relatedEntityIds = param[1];
     let filter = param[0];
     if (EntityFilter.matches(filter, entity)) {
@@ -111,14 +111,14 @@ function updateIndices(self, entity) {
   });
 }
 
-function deleteEntityFromIndices(self, entityId) {
+function deleteEntityFromIndexes(self, entityId) {
   let entityFilters = self.filtersByEntityId[entityId];
   if (entityFilters === undefined) {
     return;
   }
   let entityFilters$1 = Primitive_option.valFromOption(entityFilters);
   entityFilters$1.forEach(filter => {
-    let match = self.filterIndices[EntityFilter.toString(filter)];
+    let match = self.filterIndexes[EntityFilter.toString(filter)];
     if (match !== undefined) {
       match[1].delete(entityId);
     }
@@ -139,9 +139,9 @@ function set(inMemTable, committedCheckpointId, change) {
     inMemTable.changesCount = inMemTable.changesCount + 1;
   }
   if (change.type === "SET") {
-    updateIndices(inMemTable, change.entity);
+    updateIndexes(inMemTable, change.entity);
   } else {
-    deleteEntityFromIndices(inMemTable, EntityId.toKey(change.entityId));
+    deleteEntityFromIndexes(inMemTable, EntityId.toKey(change.entityId));
   }
   inMemTable.latestEntityChangeById[entityKey] = change;
 }
@@ -174,13 +174,13 @@ function getUnsafe(inMemTable) {
 }
 
 function hasIndex(inMemTable) {
-  return filterKey => inMemTable.filterIndices[filterKey] !== undefined;
+  return filterKey => inMemTable.filterIndexes[filterKey] !== undefined;
 }
 
 function getUnsafeOnIndex(inMemTable) {
   let getEntity = getUnsafe(inMemTable);
   return filterKey => {
-    let match = inMemTable.filterIndices[filterKey];
+    let match = inMemTable.filterIndexes[filterKey];
     if (match !== undefined) {
       return Stdlib_Array.filterMap(Array.from(match[1]), entityId => {
         if (entityId in inMemTable.latestEntityChangeById) {
@@ -195,7 +195,7 @@ function getUnsafeOnIndex(inMemTable) {
 
 function addEmptyIndex(inMemTable, filter) {
   let filterKey = EntityFilter.toString(filter);
-  let match = inMemTable.filterIndices[filterKey];
+  let match = inMemTable.filterIndexes[filterKey];
   if (match !== undefined) {
     return;
   }
@@ -212,7 +212,7 @@ function addEmptyIndex(inMemTable, filter) {
     getOrCreateEntityFilters(inMemTable, entityId).add(filter);
     relatedEntityIds.add(entityId);
   });
-  inMemTable.filterIndices[filterKey] = [
+  inMemTable.filterIndexes[filterKey] = [
     filter,
     relatedEntityIds
   ];
@@ -225,8 +225,8 @@ let Entity = {
   make: make,
   snapshotChanges: snapshotChanges,
   dropCommittedChanges: dropCommittedChanges,
-  updateIndices: updateIndices,
-  deleteEntityFromIndices: deleteEntityFromIndices,
+  updateIndexes: updateIndexes,
+  deleteEntityFromIndexes: deleteEntityFromIndexes,
   set: set,
   initValue: initValue,
   mapChangeToEntity: mapChangeToEntity,

package/src/IndexerState.res

@@ -468,6 +468,18 @@ let crossChainState = (state: t) => state.crossChainState
 let chainStates = (state: t) => state.crossChainState->CrossChainState.chainStates
 let isInReorgThreshold = (state: t) => state.crossChainState->CrossChainState.isInReorgThreshold
 let isRealtime = (state: t) => state.crossChainState->CrossChainState.isRealtime
+
+// The indexer runs Backfilling → FinalizingIndexes → Ready. This is true only
+// in the middle phase: every chain has caught up, but the deferred schema
+// indexes and `ready_at` haven't been committed yet.
+let isFinalizingIndexes = (state: t) =>
+  state.crossChainState->CrossChainState.isCaughtUp &&
+    !(state.crossChainState->CrossChainState.isRealtime)
+
+let markCaughtUp = (state: t) => state.crossChainState->CrossChainState.markCaughtUp
+
+let markReady = (state: t, ~readyAt) => state.crossChainState->CrossChainState.markReady(~readyAt)
+
 let rollbackState = (state: t) => state.rollbackState
 let indexerStartTime = (state: t) => state.indexerStartTime
 let loadManager = (state: t) => state.loadManager

package/src/IndexerState.res.mjs

@@ -336,6 +336,22 @@ function isRealtime(state) {
   return CrossChainState.isRealtime(state.crossChainState);
 }
 
+function isFinalizingIndexes(state) {
+  if (CrossChainState.isCaughtUp(state.crossChainState)) {
+    return !CrossChainState.isRealtime(state.crossChainState);
+  } else {
+    return false;
+  }
+}
+
+function markCaughtUp(state) {
+  CrossChainState.markCaughtUp(state.crossChainState);
+}
+
+function markReady(state, readyAt) {
+  CrossChainState.markReady(state.crossChainState, readyAt);
+}
+
 function rollbackState(state) {
   return state.rollbackState;
 }
@@ -762,6 +778,9 @@ export {
   chainStates,
   isInReorgThreshold,
   isRealtime,
+  isFinalizingIndexes,
+  markCaughtUp,
+  markReady,
   rollbackState,
   indexerStartTime,
   loadManager,

package/src/IndexerState.resi

@@ -94,6 +94,9 @@ let crossChainState: t => CrossChainState.t
 let chainStates: t => dict<ChainState.t>
 let isInReorgThreshold: t => bool
 let isRealtime: t => bool
+let isFinalizingIndexes: t => bool
+let markCaughtUp: t => unit
+let markReady: (t, ~readyAt: Date.t) => unit
 let rollbackState: t => rollbackState
 let indexerStartTime: t => Date.t
 let loadManager: t => LoadManager.t

package/src/LoadLayer.res

@@ -338,6 +338,7 @@ let loadByFilter = (
 
   let load = async (filters: array<EntityFilter.t>, ~onError as _) => {
     let storage = persistence->Persistence.getInitializedStorageOrThrow
+
     let timerRef =
       indexerState->IndexerState.startStorageLoad(~storage=storage.name, ~operation=key)
 
@@ -345,8 +346,15 @@ let loadByFilter = (
 
     filters->Array.forEach(filter => inMemTable->InMemoryTable.Entity.addEmptyIndex(~filter))
 
+    // Any non-derived field can be filtered on, so the columns this query reads
+    // are indexed on demand before it runs rather than promised by the schema.
+    // Inside the load timing: waiting on the build is time the handler spends
+    // waiting for this operation, and it's the only thing that explains an
+    // occasional very slow getWhere.
+    await storage.ensureQueryIndexes(~table=entityConfig.table, ~filters)
+
     // Loading a superset of rows via a merged query is safe: every loaded
-    // entity is matched against all registered indices, not only the
+    // entity is matched against all registered indexes, not only the
     // query's own filter.
     let queries = filters->EntityFilter.merge
 

package/src/LoadLayer.res.mjs

@@ -190,6 +190,7 @@ function loadByFilter(loadManager, persistence, entityConfig, indexerState, shou
       contents: 0
     };
     filters.forEach(filter => InMemoryTable.Entity.addEmptyIndex(inMemTable, filter));
+    await storage.ensureQueryIndexes(entityConfig.table, filters);
     let queries = EntityFilter.merge(filters);
     await Promise.all(queries.map(async filter => {
       try {

package/src/Persistence.res

@@ -89,6 +89,17 @@ type storage = {
   // Field values are serialized and rows parsed with the table's field schemas.
   @raises("StorageError")
   loadOrThrow: (~filter: EntityFilter.t, ~table: Table.table) => promise<array<unknown>>,
+  // Creates whatever indexes the filters need and aren't there yet, resolving
+  // once they're queryable. Best-effort: it resolves even when a build fails,
+  // leaving the query to run unindexed rather than failing the handler.
+  ensureQueryIndexes: (~table: Table.table, ~filters: array<EntityFilter.t>) => promise<unit>,
+  // Creates every schema-defined index still missing and stamps `ready_at` on
+  // the given chains, atomically. Called once, when backfill completes.
+  finalizeBackfill: (
+    ~entities: array<Internal.entityConfig>,
+    ~chainIds: array<int>,
+    ~readyAt: Date.t,
+  ) => promise<unit>,
   // This is to download cache from the database to .envio/cache
   dumpEffectCache: unit => promise<unit>,
   reset: unit => promise<unit>,