envio 3.11.0 → 3.12.0

package/src/EventProcessing.res.mjs

@@ -110,8 +110,8 @@ async function preloadBatchOrThrow(batch, loadManager, persistence, config, inde
   let itemIdx = 0;
   for (let checkpointIdx = 0, checkpointIdx_finish = batch.checkpointIds.length; checkpointIdx < checkpointIdx_finish; ++checkpointIdx) {
     let checkpointId = batch.checkpointIds[checkpointIdx];
-    let checkpointEventsProcessed = batch.checkpointEventsProcessed[checkpointIdx];
-    for (let idx = 0; idx < checkpointEventsProcessed; ++idx) {
+    let checkpointItemsCount = batch.checkpointItemsCount[checkpointIdx];
+    for (let idx = 0; idx < checkpointItemsCount; ++idx) {
       let item = batch.items[itemIdx + idx | 0];
       if (item.kind === 0) {
         let match = item.onEventRegistration;
@@ -158,7 +158,7 @@ async function preloadBatchOrThrow(batch, loadManager, persistence, config, inde
         }
       }
     }
-    itemIdx = itemIdx + checkpointEventsProcessed | 0;
+    itemIdx = itemIdx + checkpointItemsCount | 0;
   }
   await Promise.all(promises);
 }
@@ -167,12 +167,12 @@ async function runBatchHandlersOrThrow(batch, indexerState, loadManager, persist
   let itemIdx = 0;
   for (let checkpointIdx = 0, checkpointIdx_finish = batch.checkpointIds.length; checkpointIdx < checkpointIdx_finish; ++checkpointIdx) {
     let checkpointId = batch.checkpointIds[checkpointIdx];
-    let checkpointEventsProcessed = batch.checkpointEventsProcessed[checkpointIdx];
-    for (let idx = 0; idx < checkpointEventsProcessed; ++idx) {
+    let checkpointItemsCount = batch.checkpointItemsCount[checkpointIdx];
+    for (let idx = 0; idx < checkpointItemsCount; ++idx) {
       let item = batch.items[itemIdx + idx | 0];
       await runHandlerOrThrow(item, checkpointId, indexerState, loadManager, persistence, config, chains);
     }
-    itemIdx = itemIdx + checkpointEventsProcessed | 0;
+    itemIdx = itemIdx + checkpointItemsCount | 0;
   }
 }
 

package/src/FetchState.res

@@ -979,6 +979,20 @@ let compareBufferItem = (a: Internal.item, b: Internal.item): int => {
   }
 }
 
+// Whether two adjacent buffer items came from one log routed to two
+// registrations: everything `compareBufferItem` orders on except the
+// registration index is equal. Only meaningful on neighbours of a sorted
+// buffer, where such items sit next to each other.
+let isSameLog = (a: Internal.item, b: Internal.item): bool =>
+  a->Internal.getItemKind === 0 &&
+  b->Internal.getItemKind === 0 &&
+  a->Internal.getItemBlockNumber === b->Internal.getItemBlockNumber &&
+  a->Internal.getItemLogIndex === b->Internal.getItemLogIndex &&
+  switch (a->Internal.getItemOrderPath, b->Internal.getItemOrderPath) {
+  | (Value(pa), Value(pb)) => comparePath(pa, pb) === 0
+  | _ => true
+  }
+
 // Merge a maybe-unsorted `newItems` run into the already-sorted, already-deduped
 // `buffer`, dropping items equal on every component of `compareBufferItem`.
 // Single linear pass over both runs after ordering `newItems` in place; every

package/src/FetchState.res.mjs

@@ -795,6 +795,19 @@ function compareBufferItem(a, b) {
   }
 }
 
+function isSameLog(a, b) {
+  if (a.kind !== 0 || b.kind !== 0 || a.blockNumber !== b.blockNumber || a.logIndex !== b.logIndex) {
+    return false;
+  }
+  let match = a.orderPath;
+  let match$1 = b.orderPath;
+  if ((match == null) || (match$1 == null)) {
+    return true;
+  } else {
+    return comparePath(match, match$1) === 0;
+  }
+}
+
 function mergeIntoBuffer(buffer, newItems) {
   let n = newItems.length;
   for (let i = 1; i < n; ++i) {
@@ -2113,6 +2126,7 @@ export {
   comparePath,
   compareTiebreak,
   compareBufferItem,
+  isSameLog,
   mergeIntoBuffer,
   appendOnBlockItems,
   updateInternal,

package/src/FinalizeBackfill.res

@@ -1,10 +1,14 @@
-// The FinalizingIndexes phase. Reached from the processing loop when every
-// chain has caught up: processing is already paused (the loop awaits this),
-// pending writes are flushed, then storage builds every missing schema-defined
-// index and, once they all verify, commits `ready_at`. A failure part-way
-// leaves the indexes built so far in place and reaches the processing loop's
-// error boundary; the retry only owes what's left. Either way the indexer never
-// reports ready with an index the schema promised still missing.
+// The FinalizingIndexes phase. Reached from the processing loop when the chains
+// this process drives have caught up: processing is already paused (the loop
+// awaits this), pending writes are flushed, then storage builds every index the
+// schema promises for those chains and, once they all verify, commits their
+// `ready_at`. A failure part-way leaves the indexes built so far in place and
+// reaches the processing loop's error boundary; the retry only owes what's left.
+// Either way a chain never reports ready with an index the schema promised for
+// it still missing.
+//
+// A chain's rows live in a partition of its own, so this touches nothing another
+// `envio start --chain` process is indexing and never waits on one.
 
 let runOnce = async (state: IndexerState.t) => {
   Logging.info(
@@ -22,10 +26,7 @@ let runOnce = async (state: IndexerState.t) => {
 
     await storage.finalizeBackfill(
       ~entities=persistence.allEntities,
-      ~chainIds=state
-      ->IndexerState.chainStates
-      ->Dict.valuesToArray
-      ->Array.map(cs => (cs->ChainState.chainConfig).id),
+      ~chainIds=state->IndexerState.crossChainState->CrossChainState.chainIds,
       ~readyAt,
     )
 
@@ -57,5 +58,8 @@ let run = (state: IndexerState.t) =>
 let repairSchemaIndexes = (state: IndexerState.t) => {
   let persistence = state->IndexerState.persistence
   let storage = persistence->Persistence.getInitializedStorageOrThrow
-  storage.ensureSchemaIndexes(~entities=persistence.allEntities)
+  storage.ensureSchemaIndexes(
+    ~entities=persistence.allEntities,
+    ~chainIds=state->IndexerState.crossChainState->CrossChainState.chainIds,
+  )
 }

package/src/FinalizeBackfill.res.mjs

@@ -2,9 +2,9 @@
 
 import * as Logging from "./Logging.res.mjs";
 import * as Writing from "./Writing.res.mjs";
-import * as ChainState from "./ChainState.res.mjs";
 import * as Persistence from "./Persistence.res.mjs";
 import * as IndexerState from "./IndexerState.res.mjs";
+import * as CrossChainState from "./CrossChainState.res.mjs";
 
 async function runOnce(state) {
   Logging.info("All chains are caught up. Finalizing the indexer before switching to realtime: flushing pending writes, then creating the indexes the schema promises.");
@@ -15,7 +15,7 @@ async function runOnce(state) {
   let persistence = IndexerState.persistence(state);
   let storage = Persistence.getInitializedStorageOrThrow(persistence);
   let readyAt = new Date();
-  await storage.finalizeBackfill(persistence.allEntities, Object.values(IndexerState.chainStates(state)).map(cs => ChainState.chainConfig(cs).id), readyAt);
+  await storage.finalizeBackfill(persistence.allEntities, CrossChainState.chainIds(IndexerState.crossChainState(state)), readyAt);
   IndexerState.markReady(state, readyAt);
   return Logging.info("The indexer is ready. Switching to realtime indexing.");
 }
@@ -33,7 +33,7 @@ function run(state) {
 function repairSchemaIndexes(state) {
   let persistence = IndexerState.persistence(state);
   let storage = Persistence.getInitializedStorageOrThrow(persistence);
-  return storage.ensureSchemaIndexes(persistence.allEntities);
+  return storage.ensureSchemaIndexes(persistence.allEntities, CrossChainState.chainIds(IndexerState.crossChainState(state)));
 }
 
 export {

package/src/HistoryPolicy.res

@@ -0,0 +1,46 @@
+// Whether each chain's rows get history on a write, keyed by chain id. History
+// is only ever kept for what a rollback could reach: under one shared sequence
+// any chain's rollback reaches every chain's rows, so the run decides together;
+// with a counter per chain only the group's own chain can reach its rows.
+type t = dict<bool>
+
+%%private(
+  let anyChainSaves = (shouldSaveHistory: dict<bool>) =>
+    shouldSaveHistory->Dict.valuesToArray->Array.some(saves => saves)
+)
+
+let decide = (config: Config.t, ~shouldSaveHistory: dict<bool>): t =>
+  if config.shouldSaveFullHistory {
+    shouldSaveHistory->Utils.Dict.mapValues(_ => true)
+  } else {
+    switch config.checkpointSequence {
+    | SharedAcrossChains =>
+      let saves = shouldSaveHistory->anyChainSaves
+      shouldSaveHistory->Utils.Dict.mapValues(_ => saves)
+    | PerChain => shouldSaveHistory
+    }
+  }
+
+// Every chain the run indexes was decided for, so an absent one is a bug —
+// answering `false` for it would silently leave the chain with nothing to roll
+// back to.
+let forChain = (t: t, chainId: ChainId.t): bool =>
+  switch t->ChainId.Dict.dangerouslyGetNonOption(chainId) {
+  | Some(saves) => saves
+  | None =>
+    JsError.throwWithMessage(
+      `Internal error: no history decision for chain ${chainId->ChainId.toString}. The policy is decided for every chain the run indexes.`,
+    )
+  }
+
+// A cross-chain group's rows are reachable by any chain's rollback.
+let forScope = (t: t, ~scope: Internal.chainScope): bool =>
+  switch scope {
+  | Chain(chainId) => t->forChain(chainId)
+  | CrossChain => t->anyChainSaves
+  }
+
+// Whether a run has stale history to prune at all: history it keeps but doesn't
+// keep forever.
+let mayPrune = (config: Config.t, ~shouldSaveHistory: dict<bool>) =>
+  !config.shouldSaveFullHistory && shouldSaveHistory->anyChainSaves

package/src/HistoryPolicy.res.mjs

@@ -0,0 +1,50 @@
+// Generated by ReScript, PLEASE EDIT WITH CARE
+
+import * as Utils from "./Utils.res.mjs";
+import * as ChainId from "./ChainId.res.mjs";
+import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
+
+function decide(config, shouldSaveHistory) {
+  if (config.shouldSaveFullHistory) {
+    return Utils.Dict.mapValues(shouldSaveHistory, param => true);
+  }
+  let match = config.checkpointSequence;
+  if (match !== "SharedAcrossChains") {
+    return shouldSaveHistory;
+  }
+  let saves = Object.values(shouldSaveHistory).some(saves => saves);
+  return Utils.Dict.mapValues(shouldSaveHistory, param => saves);
+}
+
+function forChain(t, chainId) {
+  let saves = t[chainId];
+  if (saves !== undefined) {
+    return saves;
+  } else {
+    return Stdlib_JsError.throwWithMessage(`Internal error: no history decision for chain ` + ChainId.toString(chainId) + `. The policy is decided for every chain the run indexes.`);
+  }
+}
+
+function forScope(t, scope) {
+  if (scope === "crossChain") {
+    return Object.values(t).some(saves => saves);
+  } else {
+    return forChain(t, scope);
+  }
+}
+
+function mayPrune(config, shouldSaveHistory) {
+  if (config.shouldSaveFullHistory) {
+    return false;
+  } else {
+    return Object.values(shouldSaveHistory).some(saves => saves);
+  }
+}
+
+export {
+  decide,
+  forChain,
+  forScope,
+  mayPrune,
+}
+/* Utils Not a pure module */

package/src/InMemoryStore.res

@@ -49,10 +49,7 @@ let getEffectInMemTable = (
 ) => state->IndexerState.effectState->EffectState.getTable(~effect, ~scope)
 
 let hasEffectOutput = (inMemTable: EffectState.effectCacheInMemTable, key) =>
-  switch inMemTable.dict->Utils.Dict.dangerouslyGetNonOption(key) {
-  | Some(Set(_)) => true
-  | Some(Delete(_)) | None => false
-  }
+  inMemTable.dict->Utils.Dict.dangerouslyGetNonOption(key)->Option.isSome
 
 // Returns the raw output. The output is itself an option for effects with an
 // optional output, so it must never be wrapped in another option here: Some(None)
@@ -62,14 +59,15 @@ let getEffectOutputUnsafe = (
   key,
 ): Internal.effectOutput =>
   switch inMemTable.dict->Utils.Dict.dangerouslyGetNonOption(key) {
-  | Some(Set({entity: output})) => output
-  | Some(Delete(_)) | None => %raw(`undefined`)
+  | Some({output}) => output
+  | None => %raw(`undefined`)
   }
 
 // Records a handler output. Persisted on the next write only when shouldCache;
 // otherwise kept in memory (re-run on a later miss) but never written to the db.
 let setEffectOutput = (
   inMemTable: EffectState.effectCacheInMemTable,
+  ~chainId,
   ~checkpointId,
   ~cacheKey,
   ~output,
@@ -79,10 +77,7 @@ let setEffectOutput = (
   | Some(_) => ()
   | None => inMemTable.changesCount = inMemTable.changesCount +. 1.
   }
-  inMemTable.dict->Dict.set(
-    cacheKey,
-    Set({entityId: cacheKey->EntityId.unsafeOfString, entity: output, checkpointId}),
-  )
+  inMemTable.dict->Dict.set(cacheKey, {output, chainId, checkpointId})
   if shouldCache {
     inMemTable.idsToStore->Array.push(cacheKey)->ignore
   }
@@ -90,16 +85,17 @@ let setEffectOutput = (
 
 // Seeds an entry from a db read. Stamped with loadedFromDbCheckpointId so it's
 // always droppable (re-readable from the db) and never re-persisted.
-let initEffectOutputFromDb = (inMemTable: EffectState.effectCacheInMemTable, ~cacheKey, ~output) =>
+let initEffectOutputFromDb = (
+  inMemTable: EffectState.effectCacheInMemTable,
+  ~chainId,
+  ~cacheKey,
+  ~output,
+) =>
   if inMemTable.dict->Utils.Dict.dangerouslyGetNonOption(cacheKey)->Option.isNone {
     inMemTable.changesCount = inMemTable.changesCount +. 1.
     inMemTable.dict->Dict.set(
       cacheKey,
-      Set({
-        entityId: cacheKey->EntityId.unsafeOfString,
-        entity: output,
-        checkpointId: Internal.loadedFromDbCheckpointId,
-      }),
+      {output, chainId, checkpointId: Internal.loadedFromDbCheckpointId},
     )
   }
 
@@ -108,14 +104,13 @@ let initEffectOutputFromDb = (inMemTable: EffectState.effectCacheInMemTable, ~ca
 // seeded from a db read are spared. Mirrors entity dropCommittedChanges.
 let dropCommittedEffects = (
   inMemTable: EffectState.effectCacheInMemTable,
-  ~committedCheckpointId,
+  ~committedFrontier: Frontier.t,
   ~keepLoadedFromDb,
 ) => {
   let keysToDelete = []
-  inMemTable.dict->Utils.Dict.forEachWithKey((change, key) => {
-    let checkpointId = change->Change.getCheckpointId
+  inMemTable.dict->Utils.Dict.forEachWithKey(({chainId, checkpointId}, key) => {
     if (
-      !(checkpointId > committedCheckpointId) &&
+      !(checkpointId > committedFrontier->Frontier.get(chainId)) &&
       !(keepLoadedFromDb && checkpointId == Internal.loadedFromDbCheckpointId)
     ) {
       keysToDelete->Array.push(key)
@@ -128,18 +123,21 @@ let dropCommittedEffects = (
 let prepareRollbackDiff = async (
   state: IndexerState.t,
   ~floors: RollbackFloors.t,
-  ~rollbackDiffCheckpointId,
   ~progressedChains,
   ~rolledBackAddresses,
 ) => {
-  state->IndexerState.beginRollbackDiff(
-    ~diffCheckpointId=rollbackDiffCheckpointId,
-    ~floors,
-    ~progressedChains,
-    ~rolledBackAddresses,
-  )
+  let diffFrontier =
+    state->IndexerState.beginRollbackDiff(~floors, ~progressedChains, ~rolledBackAddresses)
   let persistence = state->IndexerState.persistence
-  let committedCheckpointId = state->IndexerState.committedCheckpointId
+  let sequence = (state->IndexerState.config).checkpointSequence
+  let diffCheckpointId = scope =>
+    switch sequence->CheckpointSequence.findForScope(diffFrontier, ~scope) {
+    | Some(checkpointId) => checkpointId
+    | None =>
+      JsError.throwWithMessage(
+        "Internal error: the rollback returned rows for a chain its diff never moved.",
+      )
+    }
 
   let deletedEntities = Dict.make()
   let setEntities = Dict.make()
@@ -160,10 +158,10 @@ let prepareRollbackDiff = async (
       state
       ->getInMemTable(~entityConfig, ~scope)
       ->InMemoryTable.Entity.set(
-        ~committedCheckpointId,
+        ~committedCheckpointId=state->IndexerState.committedCheckpointIdFor(~scope),
         Delete({
           entityId,
-          checkpointId: rollbackDiffCheckpointId,
+          checkpointId: diffCheckpointId(scope),
         }),
       )
     })
@@ -174,10 +172,10 @@ let prepareRollbackDiff = async (
       state
       ->getInMemTable(~entityConfig, ~scope)
       ->InMemoryTable.Entity.set(
-        ~committedCheckpointId,
+        ~committedCheckpointId=state->IndexerState.committedCheckpointIdFor(~scope),
         Set({
           entityId: entity.id->EntityId.unsafeOfString,
-          checkpointId: rollbackDiffCheckpointId,
+          checkpointId: diffCheckpointId(scope),
           entity,
         }),
       )

package/src/InMemoryStore.res.mjs

@@ -2,6 +2,7 @@
 
 import * as Table from "./db/Table.res.mjs";
 import * as Utils from "./Utils.res.mjs";
+import * as Frontier from "./db/Frontier.res.mjs";
 import * as Internal from "./Internal.res.mjs";
 import * as ChainState from "./ChainState.res.mjs";
 import * as EffectState from "./EffectState.res.mjs";
@@ -11,6 +12,7 @@ import * as InMemoryTable from "./InMemoryTable.res.mjs";
 import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
 import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
 import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js";
+import * as CheckpointSequence from "./db/CheckpointSequence.res.mjs";
 
 function getInMemTable(state, entityConfig, scope) {
   let tmp;
@@ -45,23 +47,19 @@ function getEffectInMemTable(state, effect, scope) {
 }
 
 function hasEffectOutput(inMemTable, key) {
-  let match = inMemTable.dict[key];
-  if (match !== undefined) {
-    return match.type === "SET";
-  } else {
-    return false;
-  }
+  return Stdlib_Option.isSome(inMemTable.dict[key]);
 }
 
 function getEffectOutputUnsafe(inMemTable, key) {
   let match = inMemTable.dict[key];
-  if (match !== undefined && match.type === "SET") {
-    return match.entity;
+  if (match !== undefined) {
+    return match.output;
+  } else {
+    return undefined;
   }
-  return undefined;
 }
 
-function setEffectOutput(inMemTable, checkpointId, cacheKey, output, shouldCache) {
+function setEffectOutput(inMemTable, chainId, checkpointId, cacheKey, output, shouldCache) {
   let match = inMemTable.dict[cacheKey];
   if (match !== undefined) {
     
@@ -69,9 +67,8 @@ function setEffectOutput(inMemTable, checkpointId, cacheKey, output, shouldCache
     inMemTable.changesCount = inMemTable.changesCount + 1;
   }
   inMemTable.dict[cacheKey] = {
-    type: "SET",
-    entityId: cacheKey,
-    entity: output,
+    output: output,
+    chainId: chainId,
     checkpointId: checkpointId
   };
   if (shouldCache) {
@@ -80,24 +77,23 @@ function setEffectOutput(inMemTable, checkpointId, cacheKey, output, shouldCache
   }
 }
 
-function initEffectOutputFromDb(inMemTable, cacheKey, output) {
+function initEffectOutputFromDb(inMemTable, chainId, cacheKey, output) {
   if (Stdlib_Option.isNone(inMemTable.dict[cacheKey])) {
     inMemTable.changesCount = inMemTable.changesCount + 1;
     inMemTable.dict[cacheKey] = {
-      type: "SET",
-      entityId: cacheKey,
-      entity: output,
+      output: output,
+      chainId: chainId,
       checkpointId: Internal.loadedFromDbCheckpointId
     };
     return;
   }
 }
 
-function dropCommittedEffects(inMemTable, committedCheckpointId, keepLoadedFromDb) {
+function dropCommittedEffects(inMemTable, committedFrontier, keepLoadedFromDb) {
   let keysToDelete = [];
-  Utils.Dict.forEachWithKey(inMemTable.dict, (change, key) => {
-    let checkpointId = change.checkpointId;
-    if (checkpointId <= committedCheckpointId && !(keepLoadedFromDb && checkpointId === Internal.loadedFromDbCheckpointId)) {
+  Utils.Dict.forEachWithKey(inMemTable.dict, (param, key) => {
+    let checkpointId = param.checkpointId;
+    if (checkpointId <= Frontier.get(committedFrontier, param.chainId) && !(keepLoadedFromDb && checkpointId === Internal.loadedFromDbCheckpointId)) {
       keysToDelete.push(key);
       return;
     }
@@ -106,31 +102,40 @@ function dropCommittedEffects(inMemTable, committedCheckpointId, keepLoadedFromD
   inMemTable.changesCount = inMemTable.changesCount - keysToDelete.length;
 }
 
-async function prepareRollbackDiff(state, floors, rollbackDiffCheckpointId, progressedChains, rolledBackAddresses) {
-  IndexerState.beginRollbackDiff(state, rollbackDiffCheckpointId, floors, progressedChains, rolledBackAddresses);
+async function prepareRollbackDiff(state, floors, progressedChains, rolledBackAddresses) {
+  let diffFrontier = IndexerState.beginRollbackDiff(state, floors, progressedChains, rolledBackAddresses);
   let persistence = IndexerState.persistence(state);
-  let committedCheckpointId = IndexerState.committedCheckpointId(state);
+  let sequence = IndexerState.config(state).checkpointSequence;
+  let diffCheckpointId = scope => {
+    let checkpointId = CheckpointSequence.findForScope(sequence, diffFrontier, scope);
+    if (checkpointId !== undefined) {
+      return checkpointId;
+    } else {
+      return Stdlib_JsError.throwWithMessage("Internal error: the rollback returned rows for a chain its diff never moved.");
+    }
+  };
   let deletedEntities = {};
   let setEntities = {};
   await Promise.all(persistence.allEntities.filter(entityConfig => entityConfig.storage.postgres).map(async entityConfig => {
     let match = await persistence.storage.getRollbackData(entityConfig, floors);
     match[0].forEach(param => {
+      let scope = param.scope;
       let entityId = param.entityId;
       Utils.Dict.push(deletedEntities, entityConfig.name, entityId);
-      InMemoryTable.Entity.set(getInMemTable(state, entityConfig, param.scope), committedCheckpointId, {
+      InMemoryTable.Entity.set(getInMemTable(state, entityConfig, scope), IndexerState.committedCheckpointIdFor(state, scope), {
         type: "DELETE",
         entityId: entityId,
-        checkpointId: rollbackDiffCheckpointId
+        checkpointId: diffCheckpointId(scope)
       });
     });
     match[1].forEach(entity => {
       let scope = takeRowScope(entity, entityConfig);
       Utils.Dict.push(setEntities, entityConfig.name, entity.id);
-      InMemoryTable.Entity.set(getInMemTable(state, entityConfig, scope), committedCheckpointId, {
+      InMemoryTable.Entity.set(getInMemTable(state, entityConfig, scope), IndexerState.committedCheckpointIdFor(state, scope), {
         type: "SET",
         entityId: entity.id,
         entity: entity,
-        checkpointId: rollbackDiffCheckpointId
+        checkpointId: diffCheckpointId(scope)
       });
     });
   }));