envio 3.5.0-alpha.2 → 3.5.0-alpha.3

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "envio",
-  "version": "3.5.0-alpha.2",
+  "version": "3.5.0-alpha.3",
   "type": "module",
   "description": "A latency and sync speed optimized, developer friendly blockchain data indexer.",
   "bin": "./bin.mjs",
@@ -69,10 +69,10 @@
     "tsx": "4.21.0"
   },
   "optionalDependencies": {
-    "envio-linux-x64": "3.5.0-alpha.2",
-    "envio-linux-x64-musl": "3.5.0-alpha.2",
-    "envio-linux-arm64": "3.5.0-alpha.2",
-    "envio-darwin-x64": "3.5.0-alpha.2",
-    "envio-darwin-arm64": "3.5.0-alpha.2"
+    "envio-linux-x64": "3.5.0-alpha.3",
+    "envio-linux-x64-musl": "3.5.0-alpha.3",
+    "envio-linux-arm64": "3.5.0-alpha.3",
+    "envio-darwin-x64": "3.5.0-alpha.3",
+    "envio-darwin-arm64": "3.5.0-alpha.3"
   }
 }

package/src/BatchProcessing.res

@@ -104,6 +104,13 @@ and processNextBatch = async (state: IndexerState.t, ~scheduleFetch): unit => {
     // When resuming from persisted state, all events may already be processed.
     if EventProcessing.allChainsEventsProcessedToEndblock(state->IndexerState.chainStates) {
       Logging.info("All chains are caught up to end blocks.")
+      // A run that has nothing left to process still owes the schema its
+      // deferred indexes — otherwise a resume of a backfill that died right
+      // before finalizing would report ready without them.
+      state->IndexerState.markCaughtUp
+      if state->IndexerState.isFinalizingIndexes {
+        await FinalizeBackfill.run(state)
+      }
       if !(state->IndexerState.keepProcessAlive) {
         await ExitOnCaughtUp.run(state)
       }
@@ -156,6 +163,13 @@ and processNextBatch = async (state: IndexerState.t, ~scheduleFetch): unit => {
         state->IndexerState.clearRollback
         state->IndexerState.applyBatchProgress(~batch)
 
+        // Backfilling → FinalizingIndexes → Ready. Awaiting here holds the
+        // processing loop for the whole finalize, which is what pauses
+        // processing while the indexes are built.
+        if state->IndexerState.isFinalizingIndexes {
+          await FinalizeBackfill.run(state)
+        }
+
         if !isRealtimeBeforeUpdate && state->IndexerState.isRealtime {
           // Catching up just flipped the chain to realtime, which changes the
           // active source for waitForNewBlock (eg sync -> live). The waiter that

package/src/BatchProcessing.res.mjs

@@ -9,6 +9,7 @@ import * as ErrorHandling from "./ErrorHandling.res.mjs";
 import * as InMemoryStore from "./InMemoryStore.res.mjs";
 import * as ExitOnCaughtUp from "./ExitOnCaughtUp.res.mjs";
 import * as EventProcessing from "./EventProcessing.res.mjs";
+import * as FinalizeBackfill from "./FinalizeBackfill.res.mjs";
 import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js";
 import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
 import * as SimulateDeadInputTracker from "./SimulateDeadInputTracker.res.mjs";
@@ -38,6 +39,10 @@ async function processNextBatch(state, scheduleFetch) {
     }
     if (EventProcessing.allChainsEventsProcessedToEndblock(IndexerState.chainStates(state))) {
       Logging.info("All chains are caught up to end blocks.");
+      IndexerState.markCaughtUp(state);
+      if (IndexerState.isFinalizingIndexes(state)) {
+        await FinalizeBackfill.run(state);
+      }
       if (!IndexerState.keepProcessAlive(state)) {
         return await ExitOnCaughtUp.run(state);
       } else {
@@ -64,6 +69,9 @@ async function processNextBatch(state, scheduleFetch) {
   }
   IndexerState.clearRollback(state);
   IndexerState.applyBatchProgress(state, batch);
+  if (IndexerState.isFinalizingIndexes(state)) {
+    await FinalizeBackfill.run(state);
+  }
   if (!isRealtimeBeforeUpdate && IndexerState.isRealtime(state)) {
     IndexerState.invalidateInflight(state);
     scheduleFetch();

package/src/ChainState.res

@@ -147,6 +147,7 @@ let makeInternal = (
   ~reorgCheckpoints: array<Internal.reorgCheckpoint>,
   ~maxReorgDepth,
   ~knownHeight=0,
+  ~isResumed=false,
   ~reducedPollingInterval=?,
 ): t => {
   // Handler binding + `where`-derived fetch state, and onBlock registrations,
@@ -203,6 +204,7 @@ let makeInternal = (
     | Evm => Some(config.clientFilterAddressThreshold)
     | Fuel | Svm => None
     },
+    ~isResumed,
   )
 
   let chainReorgCheckpoints = reorgCheckpoints->Array.filterMap(reorgCheckpoint => {
@@ -383,6 +385,7 @@ let makeFromDbState = (
     ~isInReorgThreshold,
     ~isRealtime,
     ~knownHeight=resumedChainState.sourceBlockNumber,
+    ~isResumed=true,
     ~reducedPollingInterval?,
   )
 }
@@ -1082,11 +1085,14 @@ let applyBatchProgress = (cs: t, ~batch: Batch.t, ~blockTimestampName: string) =
 }
 
 // Mark the chain caught up to head/endblock. Called by CrossChainState only once
-// every chain in the indexer is caught up, so no chain flips to ready while
-// another is still backfilling. Sticky: a chain stays ready once set.
-let markReady = (cs: t) =>
+// every chain in the indexer is caught up and the deferred schema indexes are
+// committed, so no chain flips to ready while another is still backfilling or
+// while an index the schema promises is still missing. `readyAt` is the
+// timestamp already committed to `envio_chains.ready_at` in that same
+// transaction. Sticky: a chain stays ready once set.
+let markReady = (cs: t, ~readyAt) =>
   if !(cs->isReady) {
-    cs.timestampCaughtUpToHeadOrEndblock = Date.make()->Some
+    cs.timestampCaughtUpToHeadOrEndblock = Some(readyAt)
   }
 
 // Roll a chain back to a reorg target. With a progress diff, restore fetch/

package/src/ChainState.res.mjs

@@ -89,9 +89,10 @@ function make(chainConfig, fetchState, onEventRegistrationsOpt, addressStore, so
   };
 }
 
-function makeInternal(chainConfig, indexingAddresses, startBlock, endBlock, firstEventBlockOpt, progressBlockNumber, config, registrationsByChainId, logger, timestampCaughtUpToHeadOrEndblock, numEventsProcessed, isInReorgThreshold, isRealtime, reorgCheckpoints, maxReorgDepth, knownHeightOpt, reducedPollingInterval) {
+function makeInternal(chainConfig, indexingAddresses, startBlock, endBlock, firstEventBlockOpt, progressBlockNumber, config, registrationsByChainId, logger, timestampCaughtUpToHeadOrEndblock, numEventsProcessed, isInReorgThreshold, isRealtime, reorgCheckpoints, maxReorgDepth, knownHeightOpt, isResumedOpt, reducedPollingInterval) {
   let firstEventBlock = firstEventBlockOpt !== undefined ? Primitive_option.valFromOption(firstEventBlockOpt) : undefined;
   let knownHeight = knownHeightOpt !== undefined ? knownHeightOpt : 0;
+  let isResumed = isResumedOpt !== undefined ? isResumedOpt : false;
   let match = Stdlib_Option.getOr(registrationsByChainId[chainConfig.id.toString()], {
     onEventRegistrations: [],
     onBlockRegistrations: []
@@ -116,7 +117,7 @@ function makeInternal(chainConfig, indexingAddresses, startBlock, endBlock, firs
       tmp = undefined;
       break;
   }
-  let fetchState = FetchState.make(startBlock, endBlock, onEventRegistrations, addressStore, indexingAddresses, config.maxAddrInPartition, chainConfig.id, (config.batchSize << 1), knownHeight, progressBlockNumber, match.onBlockRegistrations, Primitive_int.max(!config.shouldRollbackOnReorg || isInReorgThreshold ? 0 : chainConfig.maxReorgDepth, chainConfig.blockLag), Primitive_option.some(firstEventBlock), Primitive_option.some(tmp));
+  let fetchState = FetchState.make(startBlock, endBlock, onEventRegistrations, addressStore, indexingAddresses, config.maxAddrInPartition, chainConfig.id, (config.batchSize << 1), knownHeight, progressBlockNumber, match.onBlockRegistrations, Primitive_int.max(!config.shouldRollbackOnReorg || isInReorgThreshold ? 0 : chainConfig.maxReorgDepth, chainConfig.blockLag), Primitive_option.some(firstEventBlock), Primitive_option.some(tmp), isResumed);
   let chainReorgCheckpoints = Stdlib_Array.filterMap(reorgCheckpoints, reorgCheckpoint => {
     if (reorgCheckpoint.chain_id === chainConfig.id) {
       return reorgCheckpoint;
@@ -177,7 +178,7 @@ function makeFromConfig(chainConfig, config, registrationsByChainId, knownHeight
   let logger = Logging.createChild({
     chainId: chainConfig.id
   });
-  return makeInternal(chainConfig, configAddresses(chainConfig), chainConfig.startBlock, chainConfig.endBlock, undefined, -1, config, registrationsByChainId, logger, undefined, 0, false, false, [], chainConfig.maxReorgDepth, knownHeight, undefined);
+  return makeInternal(chainConfig, configAddresses(chainConfig), chainConfig.startBlock, chainConfig.endBlock, undefined, -1, config, registrationsByChainId, logger, undefined, 0, false, false, [], chainConfig.maxReorgDepth, knownHeight, undefined, undefined);
 }
 
 function makeFromDbState(chainConfig, resumedChainState, reorgCheckpoints, isInReorgThreshold, isRealtime, config, registrationsByChainId, reducedPollingInterval) {
@@ -186,7 +187,7 @@ function makeFromDbState(chainConfig, resumedChainState, reorgCheckpoints, isInR
     chainId: chainId
   });
   let progressBlockNumber = resumedChainState.progressBlockNumber >= 0 ? resumedChainState.progressBlockNumber : resumedChainState.startBlock - 1 | 0;
-  return makeInternal(chainConfig, resumedChainState.indexingAddresses, resumedChainState.startBlock, resumedChainState.endBlock, Primitive_option.some(resumedChainState.firstEventBlockNumber), progressBlockNumber, config, registrationsByChainId, logger, Env.updateSyncTimeOnRestart ? undefined : resumedChainState.timestampCaughtUpToHeadOrEndblock, resumedChainState.numEventsProcessed, isInReorgThreshold, isRealtime, reorgCheckpoints, resumedChainState.maxReorgDepth, resumedChainState.sourceBlockNumber, reducedPollingInterval);
+  return makeInternal(chainConfig, resumedChainState.indexingAddresses, resumedChainState.startBlock, resumedChainState.endBlock, Primitive_option.some(resumedChainState.firstEventBlockNumber), progressBlockNumber, config, registrationsByChainId, logger, Env.updateSyncTimeOnRestart ? undefined : resumedChainState.timestampCaughtUpToHeadOrEndblock, resumedChainState.numEventsProcessed, isInReorgThreshold, isRealtime, reorgCheckpoints, resumedChainState.maxReorgDepth, resumedChainState.sourceBlockNumber, true, reducedPollingInterval);
 }
 
 function logger(cs) {
@@ -757,9 +758,9 @@ function applyBatchProgress(cs, batch, blockTimestampName) {
   }
 }
 
-function markReady(cs) {
+function markReady(cs, readyAt) {
   if (cs.timestampCaughtUpToHeadOrEndblock === undefined) {
-    cs.timestampCaughtUpToHeadOrEndblock = new Date();
+    cs.timestampCaughtUpToHeadOrEndblock = readyAt;
     return;
   }
 }

package/src/ChainState.resi

@@ -137,7 +137,7 @@ let setEndBlockToFirstEvent: (t, ~blockNumber: int) => unit
 let enterReorgThreshold: t => unit
 let advanceAfterBatch: (t, ~batch: Batch.t, ~enteringReorgThreshold: bool) => unit
 let applyBatchProgress: (t, ~batch: Batch.t, ~blockTimestampName: string) => unit
-let markReady: t => unit
+let markReady: (t, ~readyAt: Date.t) => unit
 let rollback: (
   t,
   ~newProgressBlockNumber: option<int>,

package/src/Config.res

@@ -455,7 +455,7 @@ let parseEntitiesFromJson = (
         )
       )
 
-    let compositeIndices =
+    let compositeIndexes =
       entityJson["compositeIndices"]
       ->Option.getOr([])
       ->Array.map(ci =>
@@ -470,7 +470,7 @@ let parseEntitiesFromJson = (
     let table = Table.mkTable(
       entityName,
       ~fields=Array.concat(fields, derivedFields),
-      ~compositeIndices,
+      ~compositeIndexes,
       ~description=?entityJson["description"],
     )
 

package/src/Config.res.mjs

@@ -331,11 +331,11 @@ function parseEntitiesFromJson(entitiesJson, enumConfigsByName, globalStorage) {
       return Table.mkField(prop.name, match[0], match[1], undefined, match[3], match[2], prop.name === "id", match[4], prop.linkedEntity, prop.description, prop.postgresDbName, prop.clickhouseDbName);
     });
     let derivedFields = Stdlib_Option.getOr(entityJson.derivedFields, []).map(df => Table.mkDerivedFromField(df.fieldName, df.derivedFromEntity, df.derivedFromField, df.description));
-    let compositeIndices = Stdlib_Option.getOr(entityJson.compositeIndices, []).map(ci => ci.map(f => ({
+    let compositeIndexes = Stdlib_Option.getOr(entityJson.compositeIndices, []).map(ci => ci.map(f => ({
       fieldName: f.fieldName,
       direction: f.direction === "Asc" ? "Asc" : "Desc"
     })));
-    let table = Table.mkTable(entityName, compositeIndices, fields.concat(derivedFields), entityJson.description);
+    let table = Table.mkTable(entityName, compositeIndexes, fields.concat(derivedFields), entityJson.description);
     let getApiFieldName = prop => {
       let match = prop.linkedEntity;
       if (match !== undefined) {

package/src/CrossChainState.res

@@ -10,6 +10,10 @@ type t = {
   chainIds: array<int>,
   // True once every chain has caught up to head/endBlock. Monotonic during a run.
   mutable isRealtime: bool,
+  // True once every chain has caught up and there's nothing left to process,
+  // but before the deferred schema indexes and `ready_at` are committed. The
+  // gap between this and `isRealtime` is the FinalizingIndexes phase.
+  mutable isCaughtUp: bool,
   mutable isInReorgThreshold: bool,
   // Indexer-wide fetch buffer pool (item count), shared across all chains.
   targetBufferSize: int,
@@ -32,6 +36,7 @@ let make = (
     chainStates,
     chainIds: chainStates->Dict.valuesToArray->Array.map(cs => (cs->ChainState.chainConfig).id),
     isRealtime,
+    isCaughtUp: isRealtime,
     isInReorgThreshold,
     targetBufferSize,
   }
@@ -46,6 +51,7 @@ let getChainState = (crossChainState: t, chainId) =>
 
 let chainStates = (crossChainState: t) => crossChainState.chainStates
 let isRealtime = (crossChainState: t) => crossChainState.isRealtime
+let isCaughtUp = (crossChainState: t) => crossChainState.isCaughtUp
 let isInReorgThreshold = (crossChainState: t) => crossChainState.isInReorgThreshold
 let targetBufferSize = (crossChainState: t) => crossChainState.targetBufferSize
 
@@ -129,10 +135,11 @@ let enterReorgThreshold = (crossChainState: t) => {
   crossChainState.isInReorgThreshold = true
 }
 
-// Commit each progressed chain's batch progress, then decide readiness for the
-// whole indexer. A chain is marked caught up only once EVERY chain is caught up
-// (reached endblock or fetched/processed to head) with no processable events
-// left — so no chain flips to ready while another is still backfilling.
+// Commit each progressed chain's batch progress, then record whether the whole
+// indexer has caught up (every chain reached endblock or processed to head,
+// with no processable events left). Catching up doesn't make the indexer ready:
+// the deferred schema indexes still have to be created, which `markReady`
+// below concludes once they're committed.
 let applyBatchProgress = (crossChainState: t, ~batch: Batch.t, ~blockTimestampName: string) => {
   let chainIds = crossChainState.chainIds
 
@@ -145,15 +152,14 @@ let applyBatchProgress = (crossChainState: t, ~batch: Batch.t, ~blockTimestampNa
     }
   }
 
-  let indexerCaughtUp = crossChainState->nextItemIsNone && everyChainCaughtUp.contents
+  crossChainState.isCaughtUp =
+    crossChainState.isCaughtUp || (crossChainState->nextItemIsNone && everyChainCaughtUp.contents)
 
+  // A run resumed with every chain already stamped ready needs no finalize —
+  // the indexes were committed together with those stamps.
   let allChainsReady = ref(true)
   for i in 0 to chainIds->Array.length - 1 {
-    let cs = crossChainState->getChainState(chainIds->Array.getUnsafe(i))
-    if indexerCaughtUp {
-      cs->ChainState.markReady
-    }
-    if !(cs->ChainState.isReady) {
+    if !(crossChainState->getChainState(chainIds->Array.getUnsafe(i))->ChainState.isReady) {
       allChainsReady := false
     }
   }
@@ -161,6 +167,23 @@ let applyBatchProgress = (crossChainState: t, ~batch: Batch.t, ~blockTimestampNa
   crossChainState.isRealtime = crossChainState.isRealtime || allChainsReady.contents
 }
 
+// Enter the FinalizingIndexes phase without a batch. Used when the loop finds
+// every chain already processed to its endblock — a resume that has nothing
+// left to do still owes the schema its deferred indexes before it reports ready.
+let markCaughtUp = (crossChainState: t) => crossChainState.isCaughtUp = true
+
+// Concludes the FinalizingIndexes phase: stamps every chain with the `ready_at`
+// already committed alongside the deferred schema indexes and switches the
+// indexer to realtime.
+let markReady = (crossChainState: t, ~readyAt) => {
+  for i in 0 to crossChainState.chainIds->Array.length - 1 {
+    crossChainState
+    ->getChainState(crossChainState.chainIds->Array.getUnsafe(i))
+    ->ChainState.markReady(~readyAt)
+  }
+  crossChainState.isRealtime = true
+}
+
 // --- Fetch control. ---
 
 // Chains ordered furthest-behind first by fetch-frontier progress, so the

package/src/CrossChainState.res.mjs

@@ -27,6 +27,7 @@ function make(chainStates, isInReorgThreshold, isRealtime, targetBufferSizeOpt)
     chainStates: chainStates,
     chainIds: Object.values(chainStates).map(cs => ChainState.chainConfig(cs).id),
     isRealtime: isRealtime,
+    isCaughtUp: isRealtime,
     isInReorgThreshold: isInReorgThreshold,
     targetBufferSize: targetBufferSize
   };
@@ -40,6 +41,10 @@ function isRealtime(crossChainState) {
   return crossChainState.isRealtime;
 }
 
+function isCaughtUp(crossChainState) {
+  return crossChainState.isCaughtUp;
+}
+
 function isInReorgThreshold(crossChainState) {
   return crossChainState.isInReorgThreshold;
 }
@@ -104,20 +109,27 @@ function applyBatchProgress(crossChainState, batch, blockTimestampName) {
       everyChainCaughtUp = false;
     }
   }
-  let indexerCaughtUp = nextItemIsNone(crossChainState) && everyChainCaughtUp;
+  crossChainState.isCaughtUp = crossChainState.isCaughtUp || nextItemIsNone(crossChainState) && everyChainCaughtUp;
   let allChainsReady = true;
   for (let i$1 = 0, i_finish$1 = chainIds.length; i$1 < i_finish$1; ++i$1) {
-    let cs$1 = crossChainState.chainStates[chainIds[i$1]];
-    if (indexerCaughtUp) {
-      ChainState.markReady(cs$1);
-    }
-    if (!ChainState.isReady(cs$1)) {
+    if (!ChainState.isReady(crossChainState.chainStates[chainIds[i$1]])) {
       allChainsReady = false;
     }
   }
   crossChainState.isRealtime = crossChainState.isRealtime || allChainsReady;
 }
 
+function markCaughtUp(crossChainState) {
+  crossChainState.isCaughtUp = true;
+}
+
+function markReady(crossChainState, readyAt) {
+  for (let i = 0, i_finish = crossChainState.chainIds.length; i < i_finish; ++i) {
+    ChainState.markReady(crossChainState.chainStates[crossChainState.chainIds[i]], readyAt);
+  }
+  crossChainState.isRealtime = true;
+}
+
 function priorityOrder(crossChainState) {
   return Object.values(crossChainState.chainStates).toSorted((a, b) => Primitive_float.compare(ChainState.frontierProgress(a), ChainState.frontierProgress(b)));
 }
@@ -222,6 +234,7 @@ export {
   make,
   chainStates,
   isRealtime,
+  isCaughtUp,
   isInReorgThreshold,
   targetBufferSize,
   nextItemIsNone,
@@ -229,6 +242,8 @@ export {
   createBatch,
   enterReorgThreshold,
   applyBatchProgress,
+  markCaughtUp,
+  markReady,
   priorityOrder,
   checkAndFetch,
 }

package/src/CrossChainState.resi

@@ -15,6 +15,7 @@ let make: (
 // Accessors.
 let chainStates: t => dict<ChainState.t>
 let isRealtime: t => bool
+let isCaughtUp: t => bool
 let isInReorgThreshold: t => bool
 let targetBufferSize: t => int
 
@@ -31,6 +32,8 @@ let createBatch: (
 ) => Batch.t
 let enterReorgThreshold: t => unit
 let applyBatchProgress: (t, ~batch: Batch.t, ~blockTimestampName: string) => unit
+let markCaughtUp: t => unit
+let markReady: (t, ~readyAt: Date.t) => unit
 
 // Fetch control.
 let priorityOrder: t => array<ChainState.t>