envio 2.29.0-alpha.1 → 2.29.0-alpha.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "envio",
-  "version": "v2.29.0-alpha.1",
+  "version": "v2.29.0-alpha.2",
   "description": "A latency and sync speed optimized, developer friendly blockchain data indexer.",
   "bin": "./bin.js",
   "main": "./index.js",
@@ -25,10 +25,10 @@
   },
   "homepage": "https://envio.dev",
   "optionalDependencies": {
-    "envio-linux-x64": "v2.29.0-alpha.1",
-    "envio-linux-arm64": "v2.29.0-alpha.1",
-    "envio-darwin-x64": "v2.29.0-alpha.1",
-    "envio-darwin-arm64": "v2.29.0-alpha.1"
+    "envio-linux-x64": "v2.29.0-alpha.2",
+    "envio-linux-arm64": "v2.29.0-alpha.2",
+    "envio-darwin-x64": "v2.29.0-alpha.2",
+    "envio-darwin-arm64": "v2.29.0-alpha.2"
   },
   "dependencies": {
     "@envio-dev/hypersync-client": "0.6.6",

package/src/Envio.gen.ts

@@ -17,9 +17,13 @@ export type blockEvent = Internal_blockEvent;
 
 export type onBlockArgs<context> = { readonly block: blockEvent; readonly context: context };
 
-export type chainReference = number;
-
-export type onBlockOptions = { readonly name: string; readonly chain: chainReference };
+export type onBlockOptions<chain> = {
+  readonly name: string; 
+  readonly chain: chain; 
+  readonly interval?: number; 
+  readonly startBlock?: number; 
+  readonly endBlock?: number
+};
 
 export type logger = $$logger;
 

package/src/Envio.res

@@ -11,16 +11,13 @@ type onBlockArgs<'context> = {
   context: 'context,
 }
 
-@unboxed
-type chainReference = Id(int) // | Name(string)
-
 @genType
-type onBlockOptions = {
+type onBlockOptions<'chain> = {
   name: string,
-  chain: chainReference,
-  // interval?: int,
-  // startBlock?: int,
-  // endBlock?: int,
+  chain: 'chain,
+  interval?: int,
+  startBlock?: int,
+  endBlock?: int,
 }
 
 @genType.import(("./Types.ts", "Logger"))

package/src/EventRegister.res

@@ -10,11 +10,18 @@ type activeRegistration = {
 
 let activeRegistration = ref(None)
 
+// Might happen for tests when the handler file
+// is imported by a non-envio process (eg mocha)
+// and initialized before we started registration.
+// So we track them here to register when the startRegistration is called.
+// Theoretically we could keep preRegistration without an explicit start
+// but I want it to be this way, so for the actual indexer run
+// an error is thrown with the exact stack trace where the handler was registered.
+let preRegistered = []
+
 let withRegistration = (fn: activeRegistration => unit) => {
   switch activeRegistration.contents {
-  | None => // The file with handlers might run by a non-envio process (eg mocha)
-    // So we just ignore handlers in this case
-    ()
+  | None => preRegistered->Belt.Array.push(fn)
   | Some(r) =>
     if r.finished {
       Js.Exn.raiseError(
@@ -27,7 +34,7 @@ let withRegistration = (fn: activeRegistration => unit) => {
 }
 
 let startRegistration = (~ecosystem, ~multichain, ~preloadHandlers) => {
-  activeRegistration.contents = Some({
+  let r = {
     ecosystem,
     multichain,
     preloadHandlers,
@@ -35,7 +42,15 @@ let startRegistration = (~ecosystem, ~multichain, ~preloadHandlers) => {
       onBlockByChainId: Js.Dict.empty(),
     },
     finished: false,
-  })
+  }
+  activeRegistration.contents = Some(r)
+  while preRegistered->Js.Array2.length > 0 {
+    // Loop + cleanup in one go
+    switch preRegistered->Js.Array2.pop {
+    | Some(fn) => fn(r)
+    | None => ()
+    }
+  }
 }
 
 let finishRegistration = () => {
@@ -49,12 +64,17 @@ let finishRegistration = () => {
   }
 }
 
-let onBlockOptionsSchema = S.schema((s): Envio.onBlockOptions => {
-  name: s.matches(S.string),
-  chain: Id(s.matches(S.int)),
-})
+let onBlockOptionsSchema = S.schema(s =>
+  {
+    "name": s.matches(S.string),
+    "chain": s.matches(S.int),
+    "interval": s.matches(S.option(S.int->S.intMin(1))->S.Option.getOr(1)),
+    "startBlock": s.matches(S.option(S.int)),
+    "endBlock": s.matches(S.option(S.int)),
+  }
+)
 
-let onBlock = (options: Envio.onBlockOptions, handler: Internal.onBlockArgs => promise<unit>) => {
+let onBlock = (rawOptions: unknown, handler: Internal.onBlockArgs => promise<unit>) => {
   withRegistration(registration => {
     // There's no big reason for this. It's just more work
     switch registration.ecosystem {
@@ -82,9 +102,9 @@ let onBlock = (options: Envio.onBlockOptions, handler: Internal.onBlockArgs => p
       )
     }
 
-    options->S.assertOrThrow(onBlockOptionsSchema)
-    let chainId = switch options.chain {
-    | Id(chainId) => chainId
+    let options = rawOptions->S.parseOrThrow(onBlockOptionsSchema)
+    let chainId = switch options["chain"] {
+    | chainId => chainId
     // Dmitry: I want to add names for chains in the future
     // and to be able to use them as a lookup.
     // To do so, we'll need to pass a config during reigstration
@@ -101,7 +121,10 @@ let onBlock = (options: Envio.onBlockOptions, handler: Internal.onBlockArgs => p
           (
             {
               index: 0,
-              name: options.name,
+              name: options["name"],
+              startBlock: options["startBlock"],
+              endBlock: options["endBlock"],
+              interval: options["interval"],
               chainId,
               handler,
             }: Internal.onBlockConfig

package/src/EventRegister.res.js

@@ -12,6 +12,8 @@ var activeRegistration = {
   contents: undefined
 };
 
+var preRegistered = [];
+
 function withRegistration(fn) {
   var r = activeRegistration.contents;
   if (r !== undefined) {
@@ -20,12 +22,14 @@ function withRegistration(fn) {
     } else {
       return fn(r);
     }
+  } else {
+    preRegistered.push(fn);
+    return ;
   }
-  
 }
 
 function startRegistration(ecosystem, multichain, preloadHandlers) {
-  activeRegistration.contents = {
+  var r = {
     ecosystem: ecosystem,
     multichain: multichain,
     preloadHandlers: preloadHandlers,
@@ -34,6 +38,14 @@ function startRegistration(ecosystem, multichain, preloadHandlers) {
     },
     finished: false
   };
+  activeRegistration.contents = r;
+  while(preRegistered.length > 0) {
+    var fn = preRegistered.pop();
+    if (fn !== undefined) {
+      fn(r);
+    }
+    
+  };
 }
 
 function finishRegistration() {
@@ -49,11 +61,14 @@ function finishRegistration() {
 var onBlockOptionsSchema = S$RescriptSchema.schema(function (s) {
       return {
               name: s.m(S$RescriptSchema.string),
-              chain: s.m(S$RescriptSchema.$$int)
+              chain: s.m(S$RescriptSchema.$$int),
+              interval: s.m(S$RescriptSchema.$$Option.getOr(S$RescriptSchema.option(S$RescriptSchema.intMin(S$RescriptSchema.$$int, 1, undefined)), 1)),
+              startBlock: s.m(S$RescriptSchema.option(S$RescriptSchema.$$int)),
+              endBlock: s.m(S$RescriptSchema.option(S$RescriptSchema.$$int))
             };
     });
 
-function onBlock(options, handler) {
+function onBlock(rawOptions, handler) {
   withRegistration(function (registration) {
         var match = registration.ecosystem;
         if (match !== "evm") {
@@ -68,7 +83,7 @@ function onBlock(options, handler) {
         } else {
           Js_exn.raiseError("Block Handlers require the Preload Optimization feature. Enable it by setting the `preload_handlers` option to `true` in your config.");
         }
-        S$RescriptSchema.assertOrThrow(options, onBlockOptionsSchema);
+        var options = S$RescriptSchema.parseOrThrow(rawOptions, onBlockOptionsSchema);
         var chainId = options.chain;
         var onBlockByChainId = registration.registrations.onBlockByChainId;
         var match$2 = onBlockByChainId[String(chainId)];
@@ -79,6 +94,9 @@ function onBlock(options, handler) {
               index: 0,
               name: options.name,
               chainId: chainId,
+              startBlock: options.startBlock,
+              endBlock: options.endBlock,
+              interval: options.interval,
               handler: handler
             }];
           return ;

package/src/EventRegister.resi

@@ -29,4 +29,4 @@ let getEventFilters: t => option<Js.Json.t>
 let isWildcard: t => bool
 let hasRegistration: t => bool
 
-let onBlock: (Envio.onBlockOptions, Internal.onBlockArgs => promise<unit>) => unit
+let onBlock: (unknown, Internal.onBlockArgs => promise<unit>) => unit

package/src/FetchState.res

@@ -52,6 +52,7 @@ type t = {
   // since partitions might be deleted on merge or cleaned up
   nextPartitionIndex: int,
   isFetchingAtHead: bool,
+  startBlock: int,
   endBlock: option<int>,
   maxAddrInPartition: int,
   normalSelection: selection,
@@ -78,6 +79,7 @@ let copy = (fetchState: t) => {
   {
     maxAddrInPartition: fetchState.maxAddrInPartition,
     partitions: fetchState.partitions,
+    startBlock: fetchState.startBlock,
     endBlock: fetchState.endBlock,
     nextPartitionIndex: fetchState.nextPartitionIndex,
     isFetchingAtHead: fetchState.isFetchingAtHead,
@@ -252,6 +254,7 @@ let updateInternal = (
 
   {
     maxAddrInPartition: fetchState.maxAddrInPartition,
+    startBlock: fetchState.startBlock,
     endBlock: fetchState.endBlock,
     contractConfigs: fetchState.contractConfigs,
     normalSelection: fetchState.normalSelection,
@@ -640,10 +643,26 @@ let handleQueryResult = (
         }
 
         if nextLatestFullyFetchedBlockNumber > prevLatestFetchedBlockNumber {
-          for blockNumber in prevLatestFetchedBlockNumber + 1 to nextLatestFullyFetchedBlockNumber {
-            for configIdx in 0 to onBlockConfigs->Array.length - 1 {
-              let onBlockConfig = onBlockConfigs->Js.Array2.unsafe_get(configIdx)
-              newQueue->Array.push(Block({onBlockConfig, blockNumber, logIndex: blockItemLogIndex}))
+          for configIdx in 0 to onBlockConfigs->Array.length - 1 {
+            let onBlockConfig = onBlockConfigs->Js.Array2.unsafe_get(configIdx)
+
+            let handlerStartBlock = switch onBlockConfig.startBlock {
+            | Some(startBlock) => startBlock
+            | None => fetchState.startBlock
+            }
+            let rangeStart = Pervasives.max(handlerStartBlock, prevLatestFetchedBlockNumber + 1)
+            let rangeEnd = switch onBlockConfig.endBlock {
+            | Some(endBlock) => Pervasives.min(endBlock, nextLatestFullyFetchedBlockNumber)
+            | None => nextLatestFullyFetchedBlockNumber
+            }
+            if rangeStart <= rangeEnd {
+              for blockNumber in rangeStart to rangeEnd {
+                if (blockNumber - handlerStartBlock)->Pervasives.mod(onBlockConfig.interval) === 0 {
+                  newQueue->Array.push(
+                    Block({onBlockConfig, blockNumber, logIndex: blockItemLogIndex}),
+                  )
+                }
+              }
             }
           }
         }
@@ -959,12 +978,13 @@ let make = (
   ~contracts: array<indexingContract>,
   ~maxAddrInPartition,
   ~chainId,
+  ~progressBlockNumber=startBlock - 1,
   ~onBlockConfigs=?,
   ~blockLag=0,
 ): t => {
   let latestFetchedBlock = {
     blockTimestamp: 0,
-    blockNumber: startBlock - 1,
+    blockNumber: progressBlockNumber,
   }
 
   let notDependingOnAddresses = []
@@ -1078,6 +1098,7 @@ let make = (
     isFetchingAtHead: false,
     maxAddrInPartition,
     chainId,
+    startBlock,
     endBlock,
     latestFullyFetchedBlock: latestFetchedBlock,
     normalSelection,

package/src/FetchState.res.js

@@ -10,6 +10,7 @@ var Logging = require("./Logging.res.js");
 var Belt_Int = require("rescript/lib/js/belt_Int.js");
 var Caml_obj = require("rescript/lib/js/caml_obj.js");
 var Belt_Array = require("rescript/lib/js/belt_Array.js");
+var Caml_int32 = require("rescript/lib/js/caml_int32.js");
 var Prometheus = require("./Prometheus.res.js");
 var Belt_Option = require("rescript/lib/js/belt_Option.js");
 var Belt_Result = require("rescript/lib/js/belt_Result.js");
@@ -21,6 +22,7 @@ function copy(fetchState) {
           partitions: fetchState.partitions,
           nextPartitionIndex: fetchState.nextPartitionIndex,
           isFetchingAtHead: fetchState.isFetchingAtHead,
+          startBlock: fetchState.startBlock,
           endBlock: fetchState.endBlock,
           maxAddrInPartition: fetchState.maxAddrInPartition,
           normalSelection: fetchState.normalSelection,
@@ -142,6 +144,7 @@ function updateInternal(fetchState, partitionsOpt, nextPartitionIndexOpt, indexi
           partitions: partitions,
           nextPartitionIndex: nextPartitionIndex,
           isFetchingAtHead: isFetchingAtHead,
+          startBlock: fetchState.startBlock,
           endBlock: fetchState.endBlock,
           maxAddrInPartition: fetchState.maxAddrInPartition,
           normalSelection: fetchState.normalSelection,
@@ -423,16 +426,27 @@ function handleQueryResult(fetchState, query, latestFetchedBlock, newItems, curr
                   }
                   var nextLatestFullyFetchedBlockNumber$1 = nextLatestFullyFetchedBlockNumber;
                   if (nextLatestFullyFetchedBlockNumber$1 > prevLatestFetchedBlockNumber) {
-                    for(var blockNumber = prevLatestFetchedBlockNumber + 1 | 0; blockNumber <= nextLatestFullyFetchedBlockNumber$1; ++blockNumber){
-                      for(var configIdx = 0 ,configIdx_finish = onBlockConfigs.length; configIdx < configIdx_finish; ++configIdx){
-                        var onBlockConfig = onBlockConfigs[configIdx];
-                        newQueue.push({
-                              kind: 1,
-                              onBlockConfig: onBlockConfig,
-                              blockNumber: blockNumber,
-                              logIndex: 16777216
-                            });
+                    for(var configIdx = 0 ,configIdx_finish = onBlockConfigs.length; configIdx < configIdx_finish; ++configIdx){
+                      var onBlockConfig = onBlockConfigs[configIdx];
+                      var startBlock = onBlockConfig.startBlock;
+                      var handlerStartBlock = startBlock !== undefined ? startBlock : fetchState.startBlock;
+                      var rangeStart = Caml.int_max(handlerStartBlock, prevLatestFetchedBlockNumber + 1 | 0);
+                      var endBlock = onBlockConfig.endBlock;
+                      var rangeEnd = endBlock !== undefined && endBlock < nextLatestFullyFetchedBlockNumber$1 ? endBlock : nextLatestFullyFetchedBlockNumber$1;
+                      if (rangeStart <= rangeEnd) {
+                        for(var blockNumber = rangeStart; blockNumber <= rangeEnd; ++blockNumber){
+                          if (Caml_int32.mod_(blockNumber - handlerStartBlock | 0, onBlockConfig.interval) === 0) {
+                            newQueue.push({
+                                  kind: 1,
+                                  onBlockConfig: onBlockConfig,
+                                  blockNumber: blockNumber,
+                                  logIndex: 16777216
+                                });
+                          }
+                          
+                        }
                       }
+                      
                     }
                   }
                   
@@ -689,11 +703,11 @@ function getEarliestEvent(param) {
   }
 }
 
-function make(startBlock, endBlock, eventConfigs, contracts, maxAddrInPartition, chainId, onBlockConfigs, blockLagOpt) {
+function make(startBlock, endBlock, eventConfigs, contracts, maxAddrInPartition, chainId, progressBlockNumberOpt, onBlockConfigs, blockLagOpt) {
+  var progressBlockNumber = progressBlockNumberOpt !== undefined ? progressBlockNumberOpt : startBlock - 1 | 0;
   var blockLag = blockLagOpt !== undefined ? blockLagOpt : 0;
-  var latestFetchedBlock_blockNumber = startBlock - 1 | 0;
   var latestFetchedBlock = {
-    blockNumber: latestFetchedBlock_blockNumber,
+    blockNumber: progressBlockNumber,
     blockTimestamp: 0
   };
   var notDependingOnAddresses = [];
@@ -780,7 +794,7 @@ function make(startBlock, endBlock, eventConfigs, contracts, maxAddrInPartition,
   Prometheus.IndexingAddresses.set(numAddresses, chainId);
   Prometheus.IndexingPartitions.set(partitions.length, chainId);
   Prometheus.IndexingBufferSize.set(0, chainId);
-  Prometheus.IndexingBufferBlockNumber.set(latestFetchedBlock_blockNumber, chainId);
+  Prometheus.IndexingBufferBlockNumber.set(progressBlockNumber, chainId);
   if (endBlock !== undefined) {
     Prometheus.IndexingEndBlock.set(endBlock, chainId);
   }
@@ -788,6 +802,7 @@ function make(startBlock, endBlock, eventConfigs, contracts, maxAddrInPartition,
           partitions: partitions,
           nextPartitionIndex: partitions.length,
           isFetchingAtHead: false,
+          startBlock: startBlock,
           endBlock: endBlock,
           maxAddrInPartition: maxAddrInPartition,
           normalSelection: normalSelection,

package/src/Internal.res

@@ -162,6 +162,9 @@ type onBlockConfig = {
   index: int,
   name: string,
   chainId: int,
+  startBlock: option<int>,
+  endBlock: option<int>,
+  interval: int,
   handler: onBlockArgs => promise<unit>,
 }