envio 3.13.0-rc.0 → 3.13.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "envio",
-  "version": "3.13.0-rc.0",
+  "version": "3.13.0",
   "type": "module",
   "description": "A latency and sync speed optimized, developer friendly blockchain data indexer.",
   "bin": "./bin.mjs",
@@ -66,10 +66,10 @@
     "tsx": "4.21.0"
   },
   "optionalDependencies": {
-    "envio-linux-x64": "3.13.0-rc.0",
-    "envio-linux-x64-musl": "3.13.0-rc.0",
-    "envio-linux-arm64": "3.13.0-rc.0",
-    "envio-darwin-x64": "3.13.0-rc.0",
-    "envio-darwin-arm64": "3.13.0-rc.0"
+    "envio-linux-x64": "3.13.0",
+    "envio-linux-x64-musl": "3.13.0",
+    "envio-linux-arm64": "3.13.0",
+    "envio-darwin-x64": "3.13.0",
+    "envio-darwin-arm64": "3.13.0"
   }
 }

package/src/BatchProcessing.res

@@ -159,8 +159,6 @@ and processNextBatch = async (state: IndexerState.t, ~scheduleFetch): unit => {
         // Can safely reset rollback state, since overwrite is not possible.
         state->IndexerState.clearRollback
         state->IndexerState.applyBatchProgress(~batch)
-        // Before the finalize below, so a chain says where it finished ahead of
-        // the process saying what it does about that.
         state->IndexerState.reportFinished
 
         // Backfilling → FinalizingIndexes → Ready. Awaiting here holds the

package/src/ChainState.res

@@ -426,7 +426,6 @@ let getLatestValidScannedBlock = (cs: t, ~blockStore: BlockStore.t, ~blockNumber
   ->BlockStore.latestValidBlockFromStore(blockStore, blockNumbers)
   ->Null.toOption
 let safeCheckpointTracking = (cs: t) => cs.safeCheckpointTracking
-let isProgressAtHead = (cs: t) => cs.isProgressAtHead
 let committedProgressBlockNumber = (cs: t) => cs.committedProgressBlockNumber
 let committedProgressBlockTime = (cs: t) => cs.committedProgressBlockTime
 let numEventsProcessed = (cs: t) => cs.numEventsProcessed
@@ -660,30 +659,6 @@ let hasProcessedToEndblock = (cs: t) => {
   }
 }
 
-// Where this chain has finished indexing, the first time it gets there.
-// `EndBlock` is terminal: the chain indexed everything it was configured to.
-// `Backfill` is the rest of the history, up to the point where blocks can
-// still be reorged, which is as far as a chain indexes before the indexer
-// crosses into them.
-type finished = EndBlock(int) | Backfill(int)
-
-let takeFinished = (cs: t) =>
-  if cs.reportedFinished {
-    None
-  } else {
-    switch (cs.fetchState.endBlock, cs->hasProcessedToEndblock, cs.isProgressAtHead) {
-    | (Some(endBlock), true, _) => {
-        cs.reportedFinished = true
-        Some(EndBlock(endBlock))
-      }
-    | (_, _, true) => {
-        cs.reportedFinished = true
-        Some(Backfill(cs.committedProgressBlockNumber))
-      }
-    | _ => None
-    }
-  }
-
 // Caught up as judged by persisted values alone: progress reached the endBlock,
 // or the head the previous run had already observed (less the lag that holds the
 // tip back). Unlike `isFetchingAtHead` this doesn't move when a fresh height
@@ -708,6 +683,45 @@ let isDurablyCaughtUp = (cs: t) => {
   atEndBlock || atHead
 }
 
+// This chain has indexed everything there was to index: it reached its end
+// block, or its progress reached the head. The single reading of that, for
+// everything that turns on it — what the chain reports, and whether the whole
+// process may finalize.
+//
+// Both spellings, because neither covers the other: `isProgressAtHead` latches
+// the moment a batch's progress met the head known when it was created, which a
+// head that has since moved on would read as behind; `isDurablyCaughtUp` judges
+// the head as it stands, which is all a run resumed at the head has — it has no
+// batch to latch anything.
+let hasCaughtUp = (cs: t) => cs.isProgressAtHead || cs->isDurablyCaughtUp
+
+// Where this chain has finished indexing, the first time it gets there.
+// `EndBlock` is terminal: the chain indexed everything it was configured to.
+// `Backfill` is the rest of the history, up to the point where blocks can
+// still be reorged, which is as far as a chain indexes before the indexer
+// crosses into them.
+type finished = EndBlock(int) | Backfill(int)
+
+let takeFinished = (cs: t) =>
+  if cs.reportedFinished {
+    None
+  } else {
+    switch (cs.fetchState.endBlock, cs->hasProcessedToEndblock, cs->hasCaughtUp) {
+    | (Some(endBlock), true, _) => {
+        cs.reportedFinished = true
+        Some(EndBlock(endBlock))
+      }
+    | (_, _, true) => {
+        cs.reportedFinished = true
+        // Finishing a backfill happens in a run, and a chain resumed with its
+        // `ready_at` already committed did none in this one. Being at the end
+        // block, above, stays true across runs and is said on every one.
+        cs->isReady ? None : Some(Backfill(cs.committedProgressBlockNumber))
+      }
+    | _ => None
+    }
+  }
+
 let getHighestBlockBelowThreshold = (cs: t): int => {
   let highestBlockBelowThreshold = cs.fetchState.knownHeight - cs.maxReorgDepth
   highestBlockBelowThreshold < 0 ? 0 : highestBlockBelowThreshold

package/src/ChainState.res.mjs

@@ -220,10 +220,6 @@ function safeCheckpointTracking(cs) {
   return cs.safeCheckpointTracking;
 }
 
-function isProgressAtHead(cs) {
-  return cs.isProgressAtHead;
-}
-
 function committedProgressBlockNumber(cs) {
   return cs.committedProgressBlockNumber;
 }
@@ -421,13 +417,29 @@ function hasProcessedToEndblock(cs) {
   }
 }
 
+function hasCaughtUp(cs) {
+  if (cs.isProgressAtHead) {
+    return true;
+  } else {
+    let fetchState = cs.fetchState;
+    let committedProgressBlockNumber = cs.committedProgressBlockNumber;
+    let atEndBlock = Stdlib_Option.mapOr(fetchState.endBlock, false, endBlock => committedProgressBlockNumber >= endBlock);
+    let atHead = fetchState.knownHeight > 0 && committedProgressBlockNumber >= Primitive_int.max(0, fetchState.knownHeight - cs.chainConfig.blockLag | 0);
+    if (atEndBlock) {
+      return true;
+    } else {
+      return atHead;
+    }
+  }
+}
+
 function takeFinished(cs) {
   if (cs.reportedFinished) {
     return;
   }
   let match = cs.fetchState.endBlock;
   let match$1 = hasProcessedToEndblock(cs);
-  let match$2 = cs.isProgressAtHead;
+  let match$2 = hasCaughtUp(cs);
   if (match !== undefined && match$1) {
     cs.reportedFinished = true;
     return {
@@ -437,22 +449,14 @@ function takeFinished(cs) {
   }
   if (match$2) {
     cs.reportedFinished = true;
-    return {
-      TAG: "Backfill",
-      _0: cs.committedProgressBlockNumber
-    };
-  }
-}
-
-function isDurablyCaughtUp(cs) {
-  let fetchState = cs.fetchState;
-  let committedProgressBlockNumber = cs.committedProgressBlockNumber;
-  let atEndBlock = Stdlib_Option.mapOr(fetchState.endBlock, false, endBlock => committedProgressBlockNumber >= endBlock);
-  let atHead = fetchState.knownHeight > 0 && committedProgressBlockNumber >= Primitive_int.max(0, fetchState.knownHeight - cs.chainConfig.blockLag | 0);
-  if (atEndBlock) {
-    return true;
-  } else {
-    return atHead;
+    if (cs.timestampCaughtUpToHeadOrEndblock !== undefined) {
+      return;
+    } else {
+      return {
+        TAG: "Backfill",
+        _0: cs.committedProgressBlockNumber
+      };
+    }
   }
 }
 
@@ -946,7 +950,6 @@ export {
   getReorgThresholdBlockNumbersBelow,
   getLatestValidScannedBlock,
   safeCheckpointTracking,
-  isProgressAtHead,
   committedProgressBlockNumber,
   committedProgressBlockTime,
   numEventsProcessed,
@@ -980,7 +983,7 @@ export {
   reorgThresholdLiftsCeiling,
   reorgThresholdEntryMessage,
   hasProcessedToEndblock,
-  isDurablyCaughtUp,
+  hasCaughtUp,
   getHighestBlockBelowThreshold,
   isActivelyIndexing,
   isReady,

package/src/ChainState.resi

@@ -70,7 +70,6 @@ let getLatestValidScannedBlock: (
   ~blockNumbers: array<int>,
 ) => option<int>
 let safeCheckpointTracking: t => option<SafeCheckpointTracking.t>
-let isProgressAtHead: t => bool
 let committedProgressBlockNumber: t => int
 let committedProgressBlockTime: t => option<int>
 let numEventsProcessed: t => float
@@ -132,7 +131,7 @@ let takeFinished: t => option<finished>
 let reorgThresholdLiftsCeiling: t => bool
 let reorgThresholdEntryMessage: string
 let hasProcessedToEndblock: t => bool
-let isDurablyCaughtUp: t => bool
+let hasCaughtUp: t => bool
 let getHighestBlockBelowThreshold: t => int
 let isActivelyIndexing: t => bool
 let isReady: t => bool

package/src/CrossChainState.res

@@ -199,7 +199,7 @@ let applyBatchProgress = (crossChainState: t, ~batch: Batch.t, ~blockTimestampNa
   for i in 0 to chainIds->Array.length - 1 {
     let cs = crossChainState->getChainState(chainIds->Array.getUnsafe(i))
     cs->ChainState.applyBatchProgress(~batch, ~blockTimestampName)
-    if !(cs->ChainState.hasProcessedToEndblock || cs->ChainState.isProgressAtHead) {
+    if !(cs->ChainState.hasCaughtUp) {
       everyChainCaughtUp := false
     }
   }
@@ -254,7 +254,7 @@ let markCaughtUpOnResume = (crossChainState: t) => {
   let everyChainCaughtUp = ref(crossChainState.chainIds->Array.length > 0)
   for i in 0 to crossChainState.chainIds->Array.length - 1 {
     let cs = crossChainState->getChainState(crossChainState.chainIds->Array.getUnsafe(i))
-    if !(cs->ChainState.isDurablyCaughtUp) {
+    if !(cs->ChainState.hasCaughtUp) {
       everyChainCaughtUp := false
     }
   }
@@ -284,11 +284,19 @@ let markReady = (crossChainState: t, ~readyAt) => {
   crossChainState.isRealtime = true
 }
 
+// What a caught-up chain is waiting on before its process can finalize: the run
+// holding the process back, or another chain it drives still behind. Read when
+// the chain speaks, so "waiting" is said only while it is true.
+let isWaitingOnOthers = (crossChainState: t, cs: ChainState.t) =>
+  crossChainState.holdRealtime ||
+  crossChainState.chainStates
+  ->Dict.valuesToArray
+  ->Array.some(other => other !== cs && !(other->ChainState.hasCaughtUp))
+
 // Each chain that has just finished indexing, said once, by the chain it is
 // about — so a chain that finishes early says so then, rather than when the
 // last chain in its process catches up.
-let reportFinished = (crossChainState: t) => {
-  let waitingOnOthers = crossChainState.holdRealtime || crossChainState.chainIds->Array.length > 1
+let reportFinished = (crossChainState: t) =>
   crossChainState.chainStates
   ->Dict.valuesToArray
   ->Array.forEach(cs =>
@@ -301,7 +309,7 @@ let reportFinished = (crossChainState: t) => {
       cs
       ->ChainState.logger
       ->Logging.childInfo({
-        "msg": waitingOnOthers
+        "msg": crossChainState->isWaitingOnOthers(cs)
           ? "Finished backfill. Waiting for the other chains."
           : "Finished backfill.",
         "block": block,
@@ -309,7 +317,6 @@ let reportFinished = (crossChainState: t) => {
     | None => ()
     }
   )
-}
 
 // --- Fetch control. ---
 

package/src/CrossChainState.res.mjs

@@ -141,7 +141,7 @@ function applyBatchProgress(crossChainState, batch, blockTimestampName) {
   for (let i = 0, i_finish = chainIds.length; i < i_finish; ++i) {
     let cs = crossChainState.chainStates[chainIds[i]];
     ChainState.applyBatchProgress(cs, batch, blockTimestampName);
-    if (!(ChainState.hasProcessedToEndblock(cs) || ChainState.isProgressAtHead(cs))) {
+    if (!ChainState.hasCaughtUp(cs)) {
       everyChainCaughtUp = false;
     }
   }
@@ -169,7 +169,7 @@ function markCaughtUpOnResume(crossChainState) {
   let everyChainCaughtUp = crossChainState.chainIds.length !== 0;
   for (let i = 0, i_finish = crossChainState.chainIds.length; i < i_finish; ++i) {
     let cs = crossChainState.chainStates[crossChainState.chainIds[i]];
-    if (!ChainState.isDurablyCaughtUp(cs)) {
+    if (!ChainState.hasCaughtUp(cs)) {
       everyChainCaughtUp = false;
     }
   }
@@ -191,8 +191,21 @@ function markReady(crossChainState, readyAt) {
   crossChainState.isRealtime = true;
 }
 
+function isWaitingOnOthers(crossChainState, cs) {
+  if (crossChainState.holdRealtime) {
+    return true;
+  } else {
+    return Object.values(crossChainState.chainStates).some(other => {
+      if (other !== cs) {
+        return !ChainState.hasCaughtUp(other);
+      } else {
+        return false;
+      }
+    });
+  }
+}
+
 function reportFinished(crossChainState) {
-  let waitingOnOthers = crossChainState.holdRealtime || crossChainState.chainIds.length > 1;
   Object.values(crossChainState.chainStates).forEach(cs => {
     let match = ChainState.takeFinished(cs);
     if (match !== undefined) {
@@ -203,7 +216,7 @@ function reportFinished(crossChainState) {
         });
       } else {
         return Logging.childInfo(ChainState.logger(cs), {
-          msg: waitingOnOthers ? "Finished backfill. Waiting for the other chains." : "Finished backfill.",
+          msg: isWaitingOnOthers(crossChainState, cs) ? "Finished backfill. Waiting for the other chains." : "Finished backfill.",
           block: match._0
         });
       }

package/src/FinalizeBackfill.res

@@ -11,15 +11,25 @@
 // `envio start --chain` process is indexing and never waits on one.
 
 let runOnce = async (state: IndexerState.t) => {
+  let chainIds = state->IndexerState.crossChainState->CrossChainState.chainIds
+
+  // Whatever brought the process here — a batch, a tick that progressed
+  // nothing, a supervisor's release — its chains say where they finished
+  // before it says what it does about that. Said here rather than left to each
+  // caller to order: a chain that has already spoken says nothing again.
+  state->IndexerState.reportFinished
+
   // Said by the process rather than by each of its chains: the indexes are one
   // build over the tables, and the pause is the whole process's. A chain has
   // already said it caught up, and says it is ready once this commits. The
   // chains are named because the pause is theirs, and a split run has a process
-  // saying this for each part of it.
-  Logging.info({
-    "msg": "Building database indexes. Indexing is paused until they are ready, which can take a while on a large database.",
-    "chainIds": state->IndexerState.crossChainState->CrossChainState.chainIds,
-  })
+  // saying this for each part of it — one of them, for a process driving a
+  // single chain.
+  let msg = "Building database indexes. Indexing is paused until they are ready, which can take a while on a large database."
+  switch chainIds {
+  | [chainId] => Logging.info({"msg": msg, "chainId": chainId})
+  | chainIds => Logging.info({"msg": msg, "chainIds": chainIds})
+  }
 
   await Writing.flush(state)
 
@@ -30,11 +40,7 @@ let runOnce = async (state: IndexerState.t) => {
     let storage = persistence->Persistence.getInitializedStorageOrThrow
     let readyAt = Date.make()
 
-    await storage.finalizeBackfill(
-      ~entities=persistence.allEntities,
-      ~chainIds=state->IndexerState.crossChainState->CrossChainState.chainIds,
-      ~readyAt,
-    )
+    await storage.finalizeBackfill(~entities=persistence.allEntities, ~chainIds, ~readyAt)
 
     // Only after the commit: in-memory readiness must never run ahead of the
     // `ready_at` a restart would read back.

package/src/FinalizeBackfill.res.mjs

@@ -7,10 +7,21 @@ import * as IndexerState from "./IndexerState.res.mjs";
 import * as CrossChainState from "./CrossChainState.res.mjs";
 
 async function runOnce(state) {
-  Logging.info({
-    msg: "Building database indexes. Indexing is paused until they are ready, which can take a while on a large database.",
-    chainIds: CrossChainState.chainIds(IndexerState.crossChainState(state))
-  });
+  let chainIds = CrossChainState.chainIds(IndexerState.crossChainState(state));
+  IndexerState.reportFinished(state);
+  let msg = "Building database indexes. Indexing is paused until they are ready, which can take a while on a large database.";
+  if (chainIds.length !== 1) {
+    Logging.info({
+      msg: msg,
+      chainIds: chainIds
+    });
+  } else {
+    let chainId = chainIds[0];
+    Logging.info({
+      msg: msg,
+      chainId: chainId
+    });
+  }
   await Writing.flush(state);
   if (IndexerState.hasFailedWrite(state)) {
     return;
@@ -18,7 +29,7 @@ async function runOnce(state) {
   let persistence = IndexerState.persistence(state);
   let storage = Persistence.getInitializedStorageOrThrow(persistence);
   let readyAt = new Date();
-  await storage.finalizeBackfill(persistence.allEntities, CrossChainState.chainIds(IndexerState.crossChainState(state)), readyAt);
+  await storage.finalizeBackfill(persistence.allEntities, chainIds, readyAt);
   return IndexerState.markReady(state, readyAt);
 }
 

package/src/TestIndexer.res

@@ -254,13 +254,16 @@ let makeInitialState = (
       addressRowsByChain
       ->Utils.Dict.dangerouslyGetNonOption(chainIdStr)
       ->Option.getOr(AddressRows.emptySeedRows())
+    // The pinned startBlock resumes the chain rather than moving its start, so
+    // contract start blocks and onBlock intervals stay what config.yaml says.
+    // https://github.com/enviodev/hyperindex/issues/1656
     {
       Persistence.id: chain,
-      startBlock: processChainConfig.startBlock,
+      startBlock: config.chainMap->ChainMap.get(chain)->Config.startBlockOrZero,
       endBlock: processChainConfig.endBlock,
       sourceBlockNumber: processChainConfig.endBlock->Option.getOr(0),
       maxReorgDepth: 0, // No reorg support in test indexer
-      progressBlockNumber: -1,
+      progressBlockNumber: processChainConfig.startBlock - 1,
       progressBlockTime: None,
       numEventsProcessed: 0.,
       firstEventBlockNumber: None,

package/src/TestIndexer.res.mjs

@@ -180,10 +180,10 @@ function makeInitialState(config, processConfigChains, addressRowsByChain, contr
     let addressRows = Stdlib_Option.getOr(addressRowsByChain[chainIdStr], AddressRows.emptySeedRows());
     return {
       id: chain,
-      startBlock: processChainConfig.startBlock,
+      startBlock: Config.startBlockOrZero(ChainMap.get(config.chainMap, chain)),
       endBlock: processChainConfig.endBlock,
       maxReorgDepth: 0,
-      progressBlockNumber: -1,
+      progressBlockNumber: processChainConfig.startBlock - 1 | 0,
       progressBlockTime: undefined,
       numEventsProcessed: 0,
       firstEventBlockNumber: undefined,