envio 2.27.2 → 2.27.4

package/README.md

@@ -12,7 +12,7 @@ HyperIndex is a fast, developer-friendly multichain indexer, optimized for both
 ## Key Features
 
 - **[Indexer auto-generation](https://docs.envio.dev/docs/HyperIndex/contract-import)** – Generate Indexers directly from smart contract addresses
-- **High performance** – Historical backfills at over 5,000+ events per second ([fastest in market](https://docs.envio.dev/blog/indexer-benchmarking-results))
+- **High performance** – Historical backfills at over 10,000+ events per second ([fastest in market](https://docs.envio.dev/blog/indexer-benchmarking-results))
 - **Local development** – Full-featured local environment with Docker
 - **[Multichain indexing](https://docs.envio.dev/docs/HyperIndex/multichain-indexing)** – Index any EVM-compatible blockchain and Fuel (simultaneously)
 - **Real-time indexing** – Instantly track blockchain events

package/index.js

@@ -35,4 +35,5 @@ exports.S = {
   // Nullish type will change in "sury@10"
   // nullish: Sury.nullish,
   assertOrThrow: Sury.assertOrThrow,
+  parseOrThrow: Sury.parseOrThrow,
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "envio",
-  "version": "v2.27.2",
+  "version": "v2.27.4",
   "description": "A latency and sync speed optimized, developer friendly blockchain data indexer.",
   "bin": "./bin.js",
   "main": "./index.js",
@@ -25,10 +25,10 @@
   },
   "homepage": "https://envio.dev",
   "optionalDependencies": {
-    "envio-linux-x64": "v2.27.2",
-    "envio-linux-arm64": "v2.27.2",
-    "envio-darwin-x64": "v2.27.2",
-    "envio-darwin-arm64": "v2.27.2"
+    "envio-linux-x64": "v2.27.4",
+    "envio-linux-arm64": "v2.27.4",
+    "envio-darwin-x64": "v2.27.4",
+    "envio-darwin-arm64": "v2.27.4"
   },
   "dependencies": {
     "@envio-dev/hypersync-client": "0.6.5",

package/src/FetchState.res

@@ -258,12 +258,12 @@ let updateInternal = (
     }
   }
 
-  let queueSize = queue->Array.length
+  let bufferSize = queue->Array.length
   Prometheus.IndexingPartitions.set(
     ~partitionsCount=partitions->Array.length,
     ~chainId=fetchState.chainId,
   )
-  Prometheus.IndexingBufferSize.set(~bufferSize=queueSize, ~chainId=fetchState.chainId)
+  Prometheus.IndexingBufferSize.set(~bufferSize, ~chainId=fetchState.chainId)
   Prometheus.IndexingBufferBlockNumber.set(
     ~blockNumber=latestFullyFetchedBlock.blockNumber,
     ~chainId=fetchState.chainId,
@@ -1086,7 +1086,7 @@ let make = (
   }
 }
 
-let queueSize = ({queue}: t) => queue->Array.length
+let bufferSize = ({queue}: t) => queue->Array.length
 
 /**
 * Returns the latest block number fetched for the lowest fetcher queue (ie the earliest un-fetched dynamic contract)
@@ -1207,7 +1207,7 @@ let isActivelyIndexing = ({latestFullyFetchedBlock, endBlock} as fetchState: t)
   | Some(endBlock) =>
     let isPastEndblock = latestFullyFetchedBlock.blockNumber >= endBlock
     if isPastEndblock {
-      fetchState->queueSize > 0
+      fetchState->bufferSize > 0
     } else {
       true
     }
@@ -1226,3 +1226,43 @@ let isReadyToEnterReorgThreshold = (
   } &&
   queue->Utils.Array.isEmpty
 }
+
+let filterAndSortForUnorderedBatch = {
+  let hasBatchItem = ({queue, latestFullyFetchedBlock}: t) => {
+    switch queue->Utils.Array.last {
+    | Some(item) => item.blockNumber <= latestFullyFetchedBlock.blockNumber
+    | None => false
+    }
+  }
+
+  let hasFullBatch = ({queue, latestFullyFetchedBlock}: t, ~maxBatchSize) => {
+    // Queue is ordered from latest to earliest, so the earliest eligible
+    // item for a full batch of size B is at index (length - B).
+    // Do NOT subtract an extra 1 here; when length === B we should still
+    // classify the queue as full and probe index 0.
+    let targetBlockIdx = queue->Array.length - maxBatchSize
+    if targetBlockIdx < 0 {
+      false
+    } else {
+      // Unsafe can fail when maxBatchSize is 0,
+      // but we ignore the case
+      (queue->Js.Array2.unsafe_get(targetBlockIdx)).blockNumber <=
+      latestFullyFetchedBlock.blockNumber
+    }
+  }
+
+  (fetchStates: array<t>, ~maxBatchSize: int) => {
+    fetchStates
+    ->Array.keepU(hasBatchItem)
+    ->Js.Array2.sortInPlaceWith((a: t, b: t) => {
+      switch (a->hasFullBatch(~maxBatchSize), b->hasFullBatch(~maxBatchSize)) {
+      | (true, true)
+      | (false, false) =>
+        // Use unsafe since we filtered out all queues without batch items
+        (a.queue->Utils.Array.lastUnsafe).timestamp - (b.queue->Utils.Array.lastUnsafe).timestamp
+      | (true, false) => -1
+      | (false, true) => 1
+      }
+    })
+  }
+}

package/src/FetchState.res.js

@@ -147,9 +147,9 @@ function updateInternal(fetchState, partitionsOpt, nextPartitionIndexOpt, indexi
   var isFetchingAtHead = currentBlockHeight !== undefined ? (
       latestFullyFetchedBlock$1.blockNumber >= currentBlockHeight ? true : fetchState.isFetchingAtHead && checkIsWithinSyncRange(latestFullyFetchedBlock$1, currentBlockHeight)
     ) : fetchState.isFetchingAtHead;
-  var queueSize = queue.length;
+  var bufferSize = queue.length;
   Prometheus.IndexingPartitions.set(partitions.length, fetchState.chainId);
-  Prometheus.IndexingBufferSize.set(queueSize, fetchState.chainId);
+  Prometheus.IndexingBufferSize.set(bufferSize, fetchState.chainId);
   Prometheus.IndexingBufferBlockNumber.set(latestFullyFetchedBlock$1.blockNumber, fetchState.chainId);
   var item = Utils.$$Array.last(queue);
   return {
@@ -804,7 +804,7 @@ function make(startBlock, endBlock, eventConfigs, contracts, maxAddrInPartition,
         };
 }
 
-function queueSize(param) {
+function bufferSize(param) {
   return param.queue.length;
 }
 
@@ -902,7 +902,7 @@ function isActivelyIndexing(fetchState) {
   }
   var isPastEndblock = fetchState.latestFullyFetchedBlock.blockNumber >= endBlock;
   if (isPastEndblock) {
-    return queueSize(fetchState) > 0;
+    return bufferSize(fetchState) > 0;
   } else {
     return true;
   }
@@ -921,6 +921,41 @@ function isReadyToEnterReorgThreshold(param, currentBlockHeight) {
   }
 }
 
+function hasBatchItem(param) {
+  var item = Utils.$$Array.last(param.queue);
+  if (item !== undefined) {
+    return item.blockNumber <= param.latestFullyFetchedBlock.blockNumber;
+  } else {
+    return false;
+  }
+}
+
+function hasFullBatch(param, maxBatchSize) {
+  var queue = param.queue;
+  var targetBlockIdx = queue.length - maxBatchSize | 0;
+  if (targetBlockIdx < 0) {
+    return false;
+  } else {
+    return queue[targetBlockIdx].blockNumber <= param.latestFullyFetchedBlock.blockNumber;
+  }
+}
+
+function filterAndSortForUnorderedBatch(fetchStates, maxBatchSize) {
+  return Belt_Array.keepU(fetchStates, hasBatchItem).sort(function (a, b) {
+              var match = hasFullBatch(a, maxBatchSize);
+              var match$1 = hasFullBatch(b, maxBatchSize);
+              if (match) {
+                if (!match$1) {
+                  return -1;
+                }
+                
+              } else if (match$1) {
+                return 1;
+              }
+              return Utils.$$Array.lastUnsafe(a.queue).timestamp - Utils.$$Array.lastUnsafe(b.queue).timestamp | 0;
+            });
+}
+
 exports.copy = copy;
 exports.eventItemGt = eventItemGt;
 exports.mergeSortedEventList = mergeSortedEventList;
@@ -944,11 +979,12 @@ exports.makeNoItem = makeNoItem;
 exports.qItemLt = qItemLt;
 exports.getEarliestEvent = getEarliestEvent;
 exports.make = make;
-exports.queueSize = queueSize;
+exports.bufferSize = bufferSize;
 exports.getLatestFullyFetchedBlock = getLatestFullyFetchedBlock;
 exports.pruneQueueFromFirstChangeEvent = pruneQueueFromFirstChangeEvent;
 exports.rollbackPartition = rollbackPartition;
 exports.rollback = rollback;
 exports.isActivelyIndexing = isActivelyIndexing;
 exports.isReadyToEnterReorgThreshold = isReadyToEnterReorgThreshold;
+exports.filterAndSortForUnorderedBatch = filterAndSortForUnorderedBatch;
 /* Utils Not a pure module */

package/src/PgStorage.res

@@ -325,8 +325,8 @@ let removeInvalidUtf8InPlace = entities =>
     })
   })
 
-let pgEncodingErrorSchema = S.object(s =>
-  s.tag("message", `invalid byte sequence for encoding "UTF8": 0x00`)
+let pgErrorMessageSchema = S.object(s =>
+  s.field("message", S.string)
 )
 
 exception PgEncodingError({table: Table.table})

package/src/PgStorage.res.js

@@ -228,8 +228,8 @@ function removeInvalidUtf8InPlace(entities) {
       });
 }
 
-var pgEncodingErrorSchema = S$RescriptSchema.object(function (s) {
-      s.tag("message", "invalid byte sequence for encoding \"UTF8\": 0x00");
+var pgErrorMessageSchema = S$RescriptSchema.object(function (s) {
+      return s.f("message", S$RescriptSchema.string);
     });
 
 var PgEncodingError = /* @__PURE__ */Caml_exceptions.create("PgStorage.PgEncodingError");
@@ -631,7 +631,7 @@ exports.maxItemsPerQuery = maxItemsPerQuery;
 exports.makeTableBatchSetQuery = makeTableBatchSetQuery;
 exports.chunkArray = chunkArray;
 exports.removeInvalidUtf8InPlace = removeInvalidUtf8InPlace;
-exports.pgEncodingErrorSchema = pgEncodingErrorSchema;
+exports.pgErrorMessageSchema = pgErrorMessageSchema;
 exports.PgEncodingError = PgEncodingError;
 exports.setQueryCache = setQueryCache;
 exports.setOrThrow = setOrThrow;
@@ -641,4 +641,4 @@ exports.cacheTablePrefixLength = cacheTablePrefixLength;
 exports.makeSchemaCacheTableInfoQuery = makeSchemaCacheTableInfoQuery;
 exports.getConnectedPsqlExec = getConnectedPsqlExec;
 exports.make = make;
-/* pgEncodingErrorSchema Not a pure module */
+/* pgErrorMessageSchema Not a pure module */

package/src/Prometheus.res

@@ -494,15 +494,44 @@ module RollbackEnabled = {
   }
 }
 
-module RollbackDuration = {
-  let histogram = PromClient.Histogram.make({
-    "name": "envio_rollback_duration",
-    "help": "Rollback on reorg duration in seconds",
-    "buckets": [0.5, 1., 5., 10.],
+module RollbackSuccess = {
+  let timeCounter = PromClient.Counter.makeCounter({
+    "name": "envio_rollback_time",
+    "help": "Rollback on reorg total time in milliseconds",
+  })
+
+  let counter = PromClient.Counter.makeCounter({
+    "name": "envio_rollback_count",
+    "help": "Number of successful rollbacks on reorg",
   })
 
-  let startTimer = () => {
-    histogram->PromClient.Histogram.startTimer
+  let increment = (~timeMillis: Hrtime.milliseconds) => {
+    timeCounter->PromClient.Counter.incMany(timeMillis->Hrtime.intFromMillis)
+    counter->PromClient.Counter.inc
+  }
+}
+
+module RollbackHistoryPrune = {
+  let entityNameLabelsSchema = S.object(s => s.field("entity", S.string))
+
+  let timeCounter = SafeCounter.makeOrThrow(
+    ~name="envio_rollback_history_prune_time",
+    ~help="The total time spent pruning entity history which is not in the reorg threshold. (milliseconds)",
+    ~labelSchema=entityNameLabelsSchema,
+  )
+
+  let counter = SafeCounter.makeOrThrow(
+    ~name="envio_rollback_history_prune_count",
+    ~help="Number of successful entity history prunes",
+    ~labelSchema=entityNameLabelsSchema,
+  )
+
+  let increment = (~timeMillis, ~entityName) => {
+    timeCounter->SafeCounter.handleInt(
+      ~labels={entityName},
+      ~value=timeMillis->Hrtime.intFromMillis,
+    )
+    counter->SafeCounter.increment(~labels={entityName})
   }
 }
 

package/src/Prometheus.res.js

@@ -1,6 +1,7 @@
 // Generated by ReScript, PLEASE EDIT WITH CARE
 'use strict';
 
+var Hrtime = require("./bindings/Hrtime.res.js");
 var Js_exn = require("rescript/lib/js/js_exn.js");
 var ChainMap = require("./ChainMap.res.js");
 var Belt_Array = require("rescript/lib/js/belt_Array.js");
@@ -600,24 +601,45 @@ var RollbackEnabled = {
   set: set$15
 };
 
-var histogram = new PromClient.Histogram({
-      name: "envio_rollback_duration",
-      help: "Rollback on reorg duration in seconds",
-      buckets: [
-        0.5,
-        1,
-        5,
-        10
-      ]
+var timeCounter = new PromClient.Counter({
+      name: "envio_rollback_time",
+      help: "Rollback on reorg total time in milliseconds"
     });
 
-function startTimer$1() {
-  return histogram.startTimer();
+var counter$4 = new PromClient.Counter({
+      name: "envio_rollback_count",
+      help: "Number of successful rollbacks on reorg"
+    });
+
+function increment$3(timeMillis) {
+  timeCounter.inc(Hrtime.intFromMillis(timeMillis));
+  counter$4.inc();
+}
+
+var RollbackSuccess = {
+  timeCounter: timeCounter,
+  counter: counter$4,
+  increment: increment$3
+};
+
+var entityNameLabelsSchema = S$RescriptSchema.object(function (s) {
+      return s.f("entity", S$RescriptSchema.string);
+    });
+
+var timeCounter$1 = makeOrThrow("envio_rollback_history_prune_time", "The total time spent pruning entity history which is not in the reorg threshold. (milliseconds)", entityNameLabelsSchema);
+
+var counter$5 = makeOrThrow("envio_rollback_history_prune_count", "Number of successful entity history prunes", entityNameLabelsSchema);
+
+function increment$4(timeMillis, entityName) {
+  handleInt(timeCounter$1, entityName, Hrtime.intFromMillis(timeMillis));
+  increment(counter$5, entityName);
 }
 
-var RollbackDuration = {
-  histogram: histogram,
-  startTimer: startTimer$1
+var RollbackHistoryPrune = {
+  entityNameLabelsSchema: entityNameLabelsSchema,
+  timeCounter: timeCounter$1,
+  counter: counter$5,
+  increment: increment$4
 };
 
 var gauge$16 = makeOrThrow$1("envio_rollback_target_block_number", "The block number reorg was rollbacked to the last time.", chainIdLabelsSchema);
@@ -764,7 +786,8 @@ exports.ReorgCount = ReorgCount;
 exports.ReorgDetectionBlockNumber = ReorgDetectionBlockNumber;
 exports.ReorgThreshold = ReorgThreshold;
 exports.RollbackEnabled = RollbackEnabled;
-exports.RollbackDuration = RollbackDuration;
+exports.RollbackSuccess = RollbackSuccess;
+exports.RollbackHistoryPrune = RollbackHistoryPrune;
 exports.RollbackTargetBlockNumber = RollbackTargetBlockNumber;
 exports.ProcessingBlockNumber = ProcessingBlockNumber;
 exports.ProcessingBatchSize = ProcessingBatchSize;

package/src/Utils.res

@@ -250,6 +250,8 @@ Helper to check if a value exists in an array
 
   let last = (arr: array<'a>): option<'a> => arr->Belt.Array.get(arr->Array.length - 1)
 
+  let lastUnsafe = (arr: array<'a>): 'a => arr->Belt.Array.getUnsafe(arr->Array.length - 1)
+
   let findReverseWithIndex = (arr: array<'a>, fn: 'a => bool): option<('a, int)> => {
     let rec loop = (index: int) => {
       if index < 0 {

package/src/Utils.res.js

@@ -268,6 +268,10 @@ function last(arr) {
   return Belt_Array.get(arr, arr.length - 1 | 0);
 }
 
+function lastUnsafe(arr) {
+  return arr[arr.length - 1 | 0];
+}
+
 function findReverseWithIndex(arr, fn) {
   var _index = arr.length - 1 | 0;
   while(true) {
@@ -310,6 +314,7 @@ var $$Array$1 = {
   awaitEach: awaitEach,
   removeAtIndex: removeAtIndex,
   last: last,
+  lastUnsafe: lastUnsafe,
   findReverseWithIndex: findReverseWithIndex,
   interleave: interleave
 };

package/src/sources/HyperSyncJsonApi.res

@@ -368,9 +368,20 @@ let queryRoute = Rest.route(() => {
   responses: [s => s.data(ResponseTypes.queryResponseSchema)],
 })
 
+@unboxed
+type heightResult = Value(int) | ErrorMessage(string)
+
 let heightRoute = Rest.route(() => {
   path: "/height",
   method: Get,
   input: s => s.auth(Bearer),
-  responses: [s => s.field("height", S.int)],
+  responses: [
+    s =>
+      s.data(
+        S.union([
+          S.object(s => Value(s.field("height", S.int))),
+          S.string->S.shape(s => ErrorMessage(s)),
+        ]),
+      ),
+  ],
 })

package/src/sources/HyperSyncJsonApi.res.js

@@ -247,7 +247,14 @@ function heightRoute() {
               return s.auth("Bearer");
             }),
           responses: [(function (s) {
-                return s.field("height", S$RescriptSchema.$$int);
+                return s.data(S$RescriptSchema.union([
+                                S$RescriptSchema.object(function (s) {
+                                      return s.f("height", S$RescriptSchema.$$int);
+                                    }),
+                                S$RescriptSchema.shape(S$RescriptSchema.string, (function (s) {
+                                        return s;
+                                      }))
+                              ]));
               })]
         };
 }