envio 3.1.0-rc.2 → 3.1.0

package/index.d.ts

@@ -1201,6 +1201,18 @@ export type IndexerFromConfig<Config extends IndexerConfigTypes = GlobalConfig>
     readonly name: string;
     /** The indexer description from config.yaml. */
     readonly description: string | undefined;
+    /**
+     * Internal, unstable API that will be removed without notice. Registers a
+     * callback fired once per chain affected by a reorg rollback, after the
+     * rollback is durably written to the database. A throwing callback crashes
+     * the indexer through the same path as a failed write.
+     */
+    readonly "~internalAndWillBeRemovedSoon_onRollbackCommit": (
+      callback: (args: {
+        readonly chainId: number;
+        readonly rollbackToBlock: number;
+      }) => Promise<void>,
+    ) => void;
   } & SingleEcosystemChains<Config>
 >;
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "envio",
-  "version": "3.1.0-rc.2",
+  "version": "3.1.0",
   "type": "module",
   "description": "A latency and sync speed optimized, developer friendly blockchain data indexer.",
   "bin": "./bin.mjs",
@@ -70,10 +70,10 @@
     "tsx": "4.21.0"
   },
   "optionalDependencies": {
-    "envio-linux-x64": "3.1.0-rc.2",
-    "envio-linux-x64-musl": "3.1.0-rc.2",
-    "envio-linux-arm64": "3.1.0-rc.2",
-    "envio-darwin-x64": "3.1.0-rc.2",
-    "envio-darwin-arm64": "3.1.0-rc.2"
+    "envio-linux-x64": "3.1.0",
+    "envio-linux-x64-musl": "3.1.0",
+    "envio-linux-arm64": "3.1.0",
+    "envio-darwin-x64": "3.1.0",
+    "envio-darwin-arm64": "3.1.0"
   }
 }

package/src/GlobalState.res

@@ -1159,6 +1159,7 @@ let injectedTaskReducer = (
           ~persistence=state.ctx.persistence,
           ~rollbackTargetCheckpointId,
           ~rollbackDiffCheckpointId=state.ctx.inMemoryStore.committedCheckpointId->BigInt.add(1n),
+          ~progressBlockNumberByChainId=newProgressBlockNumberPerChain,
         )
 
         let chainManager = {

package/src/GlobalState.res.mjs

@@ -1151,7 +1151,7 @@ function injectedTaskReducer(waitForNewBlock, executeQuery, getLastKnownValidBlo
                 };
               });
               await InMemoryStore.flush(state.ctx.inMemoryStore);
-              let diff$1 = await InMemoryStore.prepareRollbackDiff(state.ctx.inMemoryStore, state.ctx.persistence, rollbackTargetCheckpointId, state.ctx.inMemoryStore.committedCheckpointId + 1n);
+              let diff$1 = await InMemoryStore.prepareRollbackDiff(state.ctx.inMemoryStore, state.ctx.persistence, rollbackTargetCheckpointId, state.ctx.inMemoryStore.committedCheckpointId + 1n, newProgressBlockNumberPerChain);
               let init = state.chainManager;
               let chainManager_isInReorgThreshold = init.isInReorgThreshold;
               let chainManager_isRealtime = init.isRealtime;

package/src/InMemoryStore.res

@@ -359,6 +359,12 @@ let runOneWrite = async (inMemoryStore: t, ~persistence: Persistence.t, ~config)
     )
 
     inMemoryStore.committedCheckpointId = upToCheckpointId
+
+    switch rollback {
+    | Some({progressBlockNumberByChainId}) if RollbackCommit.callbacks->Utils.Array.notEmpty =>
+      await RollbackCommit.fire(~progressBlockNumberByChainId)
+    | _ => ()
+    }
   }
 }
 
@@ -494,12 +500,14 @@ let prepareRollbackDiff = async (
   ~persistence: Persistence.t,
   ~rollbackTargetCheckpointId,
   ~rollbackDiffCheckpointId,
+  ~progressBlockNumberByChainId,
 ) => {
   inMemoryStore.entities = EntityTables.make(inMemoryStore.allEntities)
   inMemoryStore.effects = Dict.make()
   inMemoryStore.rollback = Some({
     targetCheckpointId: rollbackTargetCheckpointId,
     diffCheckpointId: rollbackDiffCheckpointId,
+    progressBlockNumberByChainId,
   })
 
   let deletedEntities = Dict.make()

package/src/InMemoryStore.res.mjs

@@ -11,6 +11,7 @@ import * as Stdlib_Array from "@rescript/runtime/lib/es6/Stdlib_Array.js";
 import * as ErrorHandling from "./ErrorHandling.res.mjs";
 import * as InMemoryTable from "./InMemoryTable.res.mjs";
 import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
+import * as RollbackCommit from "./RollbackCommit.res.mjs";
 import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
 import * as Primitive_object from "@rescript/runtime/lib/es6/Primitive_object.js";
 import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js";
@@ -300,6 +301,9 @@ async function runOneWrite(inMemoryStore, persistence, config) {
   let updatedEffectsCache = snapshotEffects(inMemoryStore, cache);
   await persistence.storage.writeBatch(batch, rollback, batch.isInReorgThreshold, config, persistence.allEntities, updatedEffectsCache, updatedEntities, chainMetaData);
   inMemoryStore.committedCheckpointId = upToCheckpointId;
+  if (rollback !== undefined && Utils.$$Array.notEmpty(RollbackCommit.callbacks)) {
+    return await RollbackCommit.fire(rollback.progressBlockNumberByChainId);
+  }
 }
 
 function hasPendingWrite(inMemoryStore) {
@@ -401,12 +405,13 @@ async function flush(inMemoryStore) {
   }
 }
 
-async function prepareRollbackDiff(inMemoryStore, persistence, rollbackTargetCheckpointId, rollbackDiffCheckpointId) {
+async function prepareRollbackDiff(inMemoryStore, persistence, rollbackTargetCheckpointId, rollbackDiffCheckpointId, progressBlockNumberByChainId) {
   inMemoryStore.entities = make(inMemoryStore.allEntities);
   inMemoryStore.effects = {};
   inMemoryStore.rollback = {
     targetCheckpointId: rollbackTargetCheckpointId,
-    diffCheckpointId: rollbackDiffCheckpointId
+    diffCheckpointId: rollbackDiffCheckpointId,
+    progressBlockNumberByChainId: progressBlockNumberByChainId
   };
   let deletedEntities = {};
   let setEntities = {};

package/src/Main.res

@@ -312,6 +312,13 @@ let getGlobalIndexer = (): 'indexer => {
     )
   }
 
+  let onRollbackCommitFn = (callback: 'a) => {
+    HandlerRegister.throwIfFinishedRegistration(
+      ~methodName="~internalAndWillBeRemovedSoon_onRollbackCommit",
+    )
+    let _ = RollbackCommit.register(callback->(Utils.magic: 'a => RollbackCommit.callback))
+  }
+
   // Two-stage parse: first the ecosystem-specific outer schema unwraps the
   // wrapper (`block.number` / `block.height` / `slot`) and surfaces the
   // inner chunk as raw `unknown`; then the shared `blockRangeSchema`
@@ -453,8 +460,16 @@ let getGlobalIndexer = (): 'indexer => {
             "onEvent",
             "contractRegister",
             "onBlock",
+            "~internalAndWillBeRemovedSoon_onRollbackCommit",
+          ]
+        | Svm => [
+            "name",
+            "description",
+            "chainIds",
+            "chains",
+            "onSlot",
+            "~internalAndWillBeRemovedSoon_onRollbackCommit",
           ]
-        | Svm => ["name", "description", "chainIds", "chains", "onSlot"]
         }
         keysMemo := Some(keys)
         keys
@@ -477,6 +492,7 @@ let getGlobalIndexer = (): 'indexer => {
     | "onEvent" => onEventFn->Utils.magic
     | "contractRegister" => contractRegisterFn->Utils.magic
     | "onBlock" | "onSlot" => onBlockFn->Utils.magic
+    | "~internalAndWillBeRemovedSoon_onRollbackCommit" => onRollbackCommitFn->Utils.magic
     | _ =>
       JsError.throwWithMessage(
         `Field \`${prop}\` does not exist on \`indexer\`. Available fields: ${getKeys()->Array.join(

package/src/Main.res.mjs

@@ -25,6 +25,7 @@ import * as Primitive_int from "@rescript/runtime/lib/es6/Primitive_int.js";
 import * as SourceManager from "./sources/SourceManager.res.mjs";
 import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
 import * as Helpers from "yargs/helpers";
+import * as RollbackCommit from "./RollbackCommit.res.mjs";
 import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
 import * as HandlerRegister from "./HandlerRegister.res.mjs";
 import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js";
@@ -255,6 +256,10 @@ function getGlobalIndexer() {
     let match = parseIdentityConfig(identityConfig);
     HandlerRegister.setContractRegister(match[0], match[1], handler, match[2], undefined);
   };
+  let onRollbackCommitFn = callback => {
+    HandlerRegister.throwIfFinishedRegistration("~internalAndWillBeRemovedSoon_onRollbackCommit");
+    RollbackCommit.register(callback);
+  };
   let extractRange = (filter, name, ecosystem) => {
     try {
       let inner = S$RescriptSchema.parseOrThrow(filter, ecosystem.onBlockFilterSchema);
@@ -341,7 +346,8 @@ function getGlobalIndexer() {
           "description",
           "chainIds",
           "chains",
-          "onSlot"
+          "onSlot",
+          "~internalAndWillBeRemovedSoon_onRollbackCommit"
         ];
         break;
     }
@@ -353,7 +359,8 @@ function getGlobalIndexer() {
         "chains",
         "onEvent",
         "contractRegister",
-        "onBlock"
+        "onBlock",
+        "~internalAndWillBeRemovedSoon_onRollbackCommit"
       ];
     }
     keysMemo.contents = keys;
@@ -376,6 +383,8 @@ function getGlobalIndexer() {
       case "onBlock" :
       case "onSlot" :
         return onBlockFn;
+      case "~internalAndWillBeRemovedSoon_onRollbackCommit" :
+        return onRollbackCommitFn;
       default:
         return Stdlib_JsError.throwWithMessage(`Field \`` + prop + `\` does not exist on \`indexer\`. Available fields: ` + getKeys().join(", ") + `.`);
     }

package/src/Persistence.res

@@ -51,6 +51,9 @@ type updatedEffectCache = {
 type rollback = {
   targetCheckpointId: Internal.checkpointId,
   diffCheckpointId: Internal.checkpointId,
+  // Last valid block per chain affected by the rollback. Read by
+  // `RollbackCommit.fire` once the diff is durably written.
+  progressBlockNumberByChainId: dict<int>,
 }
 
 type updatedEntity = {

package/src/RollbackCommit.res

@@ -0,0 +1,32 @@
+// Temporary, internal-only support for the unstable
+// `indexer.~internalAndWillBeRemovedSoon_onRollbackCommit` API. The whole
+// feature lives here plus two call sites: registration in `Main.res` and the
+// fire on a successful rollback write in `InMemoryStore.res`. Delete those
+// together with this module.
+type args = {chainId: int, rollbackToBlock: int}
+type callback = args => promise<unit>
+
+let callbacks: array<callback> = []
+
+let register = (callback: callback) => {
+  callbacks->Array.push(callback)
+  () =>
+    switch callbacks->Array.indexOf(callback) {
+    | -1 => ()
+    | index => callbacks->Array.splice(~start=index, ~remove=1, ~insert=[])
+    }
+}
+
+// Fired after a rollback diff is durably written, once per affected chain.
+// `progressBlockNumberByChainId` is the last valid block per chain, taken from
+// the in-memory store's rollback object. A throwing callback bubbles to the
+// write loop's onError, crashing the indexer like a failed write.
+let fire = async (~progressBlockNumberByChainId: dict<int>) => {
+  let _ = await progressBlockNumberByChainId
+  ->Dict.toArray
+  ->Array.flatMap(((chainIdKey, rollbackToBlock)) => {
+    let args = {chainId: chainIdKey->Int.fromString->Option.getUnsafe, rollbackToBlock}
+    callbacks->Array.map(callback => callback(args))
+  })
+  ->Promise.all
+}

package/src/RollbackCommit.res.mjs

@@ -0,0 +1,35 @@
+// Generated by ReScript, PLEASE EDIT WITH CARE
+
+import * as Stdlib_Int from "@rescript/runtime/lib/es6/Stdlib_Int.js";
+
+let callbacks = [];
+
+function register(callback) {
+  callbacks.push(callback);
+  return () => {
+    let index = callbacks.indexOf(callback);
+    if (index !== -1) {
+      callbacks.splice(index, 1);
+      return;
+    }
+  };
+}
+
+async function fire(progressBlockNumberByChainId) {
+  await Promise.all(Object.entries(progressBlockNumberByChainId).flatMap(param => {
+    let args_chainId = Stdlib_Int.fromString(param[0], undefined);
+    let args_rollbackToBlock = param[1];
+    let args = {
+      chainId: args_chainId,
+      rollbackToBlock: args_rollbackToBlock
+    };
+    return callbacks.map(callback => callback(args));
+  }));
+}
+
+export {
+  callbacks,
+  register,
+  fire,
+}
+/* No side effect */