envio 2.27.3 → 2.27.4

package/README.md

@@ -12,7 +12,7 @@ HyperIndex is a fast, developer-friendly multichain indexer, optimized for both
 ## Key Features
 
 - **[Indexer auto-generation](https://docs.envio.dev/docs/HyperIndex/contract-import)** – Generate Indexers directly from smart contract addresses
-- **High performance** – Historical backfills at over 5,000+ events per second ([fastest in market](https://docs.envio.dev/blog/indexer-benchmarking-results))
+- **High performance** – Historical backfills at over 10,000+ events per second ([fastest in market](https://docs.envio.dev/blog/indexer-benchmarking-results))
 - **Local development** – Full-featured local environment with Docker
 - **[Multichain indexing](https://docs.envio.dev/docs/HyperIndex/multichain-indexing)** – Index any EVM-compatible blockchain and Fuel (simultaneously)
 - **Real-time indexing** – Instantly track blockchain events

package/index.js

@@ -35,4 +35,5 @@ exports.S = {
   // Nullish type will change in "sury@10"
   // nullish: Sury.nullish,
   assertOrThrow: Sury.assertOrThrow,
+  parseOrThrow: Sury.parseOrThrow,
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "envio",
-  "version": "v2.27.3",
+  "version": "v2.27.4",
   "description": "A latency and sync speed optimized, developer friendly blockchain data indexer.",
   "bin": "./bin.js",
   "main": "./index.js",
@@ -25,10 +25,10 @@
   },
   "homepage": "https://envio.dev",
   "optionalDependencies": {
-    "envio-linux-x64": "v2.27.3",
-    "envio-linux-arm64": "v2.27.3",
-    "envio-darwin-x64": "v2.27.3",
-    "envio-darwin-arm64": "v2.27.3"
+    "envio-linux-x64": "v2.27.4",
+    "envio-linux-arm64": "v2.27.4",
+    "envio-darwin-x64": "v2.27.4",
+    "envio-darwin-arm64": "v2.27.4"
   },
   "dependencies": {
     "@envio-dev/hypersync-client": "0.6.5",

package/src/FetchState.res

@@ -1235,14 +1235,34 @@ let filterAndSortForUnorderedBatch = {
     }
   }
 
-  let compareUnorderedBatchChainPriority = (a: t, b: t) => {
-    // Use unsafe since we filtered out all queues without batch items
-    (a.queue->Utils.Array.lastUnsafe).timestamp - (b.queue->Utils.Array.lastUnsafe).timestamp
+  let hasFullBatch = ({queue, latestFullyFetchedBlock}: t, ~maxBatchSize) => {
+    // Queue is ordered from latest to earliest, so the earliest eligible
+    // item for a full batch of size B is at index (length - B).
+    // Do NOT subtract an extra 1 here; when length === B we should still
+    // classify the queue as full and probe index 0.
+    let targetBlockIdx = queue->Array.length - maxBatchSize
+    if targetBlockIdx < 0 {
+      false
+    } else {
+      // Unsafe can fail when maxBatchSize is 0,
+      // but we ignore the case
+      (queue->Js.Array2.unsafe_get(targetBlockIdx)).blockNumber <=
+      latestFullyFetchedBlock.blockNumber
+    }
   }
 
-  (fetchStates: array<t>) => {
+  (fetchStates: array<t>, ~maxBatchSize: int) => {
     fetchStates
     ->Array.keepU(hasBatchItem)
-    ->Js.Array2.sortInPlaceWith(compareUnorderedBatchChainPriority)
+    ->Js.Array2.sortInPlaceWith((a: t, b: t) => {
+      switch (a->hasFullBatch(~maxBatchSize), b->hasFullBatch(~maxBatchSize)) {
+      | (true, true)
+      | (false, false) =>
+        // Use unsafe since we filtered out all queues without batch items
+        (a.queue->Utils.Array.lastUnsafe).timestamp - (b.queue->Utils.Array.lastUnsafe).timestamp
+      | (true, false) => -1
+      | (false, true) => 1
+      }
+    })
   }
 }

package/src/FetchState.res.js

@@ -930,12 +930,30 @@ function hasBatchItem(param) {
   }
 }
 
-function compareUnorderedBatchChainPriority(a, b) {
-  return Utils.$$Array.lastUnsafe(a.queue).timestamp - Utils.$$Array.lastUnsafe(b.queue).timestamp | 0;
+function hasFullBatch(param, maxBatchSize) {
+  var queue = param.queue;
+  var targetBlockIdx = queue.length - maxBatchSize | 0;
+  if (targetBlockIdx < 0) {
+    return false;
+  } else {
+    return queue[targetBlockIdx].blockNumber <= param.latestFullyFetchedBlock.blockNumber;
+  }
 }
 
-function filterAndSortForUnorderedBatch(fetchStates) {
-  return Belt_Array.keepU(fetchStates, hasBatchItem).sort(compareUnorderedBatchChainPriority);
+function filterAndSortForUnorderedBatch(fetchStates, maxBatchSize) {
+  return Belt_Array.keepU(fetchStates, hasBatchItem).sort(function (a, b) {
+              var match = hasFullBatch(a, maxBatchSize);
+              var match$1 = hasFullBatch(b, maxBatchSize);
+              if (match) {
+                if (!match$1) {
+                  return -1;
+                }
+                
+              } else if (match$1) {
+                return 1;
+              }
+              return Utils.$$Array.lastUnsafe(a.queue).timestamp - Utils.$$Array.lastUnsafe(b.queue).timestamp | 0;
+            });
 }
 
 exports.copy = copy;

package/src/PgStorage.res

@@ -325,8 +325,8 @@ let removeInvalidUtf8InPlace = entities =>
     })
   })
 
-let pgEncodingErrorSchema = S.object(s =>
-  s.tag("message", `invalid byte sequence for encoding "UTF8": 0x00`)
+let pgErrorMessageSchema = S.object(s =>
+  s.field("message", S.string)
 )
 
 exception PgEncodingError({table: Table.table})

package/src/PgStorage.res.js

@@ -228,8 +228,8 @@ function removeInvalidUtf8InPlace(entities) {
       });
 }
 
-var pgEncodingErrorSchema = S$RescriptSchema.object(function (s) {
-      s.tag("message", "invalid byte sequence for encoding \"UTF8\": 0x00");
+var pgErrorMessageSchema = S$RescriptSchema.object(function (s) {
+      return s.f("message", S$RescriptSchema.string);
     });
 
 var PgEncodingError = /* @__PURE__ */Caml_exceptions.create("PgStorage.PgEncodingError");
@@ -631,7 +631,7 @@ exports.maxItemsPerQuery = maxItemsPerQuery;
 exports.makeTableBatchSetQuery = makeTableBatchSetQuery;
 exports.chunkArray = chunkArray;
 exports.removeInvalidUtf8InPlace = removeInvalidUtf8InPlace;
-exports.pgEncodingErrorSchema = pgEncodingErrorSchema;
+exports.pgErrorMessageSchema = pgErrorMessageSchema;
 exports.PgEncodingError = PgEncodingError;
 exports.setQueryCache = setQueryCache;
 exports.setOrThrow = setOrThrow;
@@ -641,4 +641,4 @@ exports.cacheTablePrefixLength = cacheTablePrefixLength;
 exports.makeSchemaCacheTableInfoQuery = makeSchemaCacheTableInfoQuery;
 exports.getConnectedPsqlExec = getConnectedPsqlExec;
 exports.make = make;
-/* pgEncodingErrorSchema Not a pure module */
+/* pgErrorMessageSchema Not a pure module */

package/src/Prometheus.res

@@ -494,15 +494,44 @@ module RollbackEnabled = {
   }
 }
 
-module RollbackDuration = {
-  let histogram = PromClient.Histogram.make({
-    "name": "envio_rollback_duration",
-    "help": "Rollback on reorg duration in seconds",
-    "buckets": [0.5, 1., 5., 10.],
+module RollbackSuccess = {
+  let timeCounter = PromClient.Counter.makeCounter({
+    "name": "envio_rollback_time",
+    "help": "Rollback on reorg total time in milliseconds",
+  })
+
+  let counter = PromClient.Counter.makeCounter({
+    "name": "envio_rollback_count",
+    "help": "Number of successful rollbacks on reorg",
   })
 
-  let startTimer = () => {
-    histogram->PromClient.Histogram.startTimer
+  let increment = (~timeMillis: Hrtime.milliseconds) => {
+    timeCounter->PromClient.Counter.incMany(timeMillis->Hrtime.intFromMillis)
+    counter->PromClient.Counter.inc
+  }
+}
+
+module RollbackHistoryPrune = {
+  let entityNameLabelsSchema = S.object(s => s.field("entity", S.string))
+
+  let timeCounter = SafeCounter.makeOrThrow(
+    ~name="envio_rollback_history_prune_time",
+    ~help="The total time spent pruning entity history which is not in the reorg threshold. (milliseconds)",
+    ~labelSchema=entityNameLabelsSchema,
+  )
+
+  let counter = SafeCounter.makeOrThrow(
+    ~name="envio_rollback_history_prune_count",
+    ~help="Number of successful entity history prunes",
+    ~labelSchema=entityNameLabelsSchema,
+  )
+
+  let increment = (~timeMillis, ~entityName) => {
+    timeCounter->SafeCounter.handleInt(
+      ~labels={entityName},
+      ~value=timeMillis->Hrtime.intFromMillis,
+    )
+    counter->SafeCounter.increment(~labels={entityName})
   }
 }
 

package/src/Prometheus.res.js

@@ -1,6 +1,7 @@
 // Generated by ReScript, PLEASE EDIT WITH CARE
 'use strict';
 
+var Hrtime = require("./bindings/Hrtime.res.js");
 var Js_exn = require("rescript/lib/js/js_exn.js");
 var ChainMap = require("./ChainMap.res.js");
 var Belt_Array = require("rescript/lib/js/belt_Array.js");
@@ -600,24 +601,45 @@ var RollbackEnabled = {
   set: set$15
 };
 
-var histogram = new PromClient.Histogram({
-      name: "envio_rollback_duration",
-      help: "Rollback on reorg duration in seconds",
-      buckets: [
-        0.5,
-        1,
-        5,
-        10
-      ]
+var timeCounter = new PromClient.Counter({
+      name: "envio_rollback_time",
+      help: "Rollback on reorg total time in milliseconds"
     });
 
-function startTimer$1() {
-  return histogram.startTimer();
+var counter$4 = new PromClient.Counter({
+      name: "envio_rollback_count",
+      help: "Number of successful rollbacks on reorg"
+    });
+
+function increment$3(timeMillis) {
+  timeCounter.inc(Hrtime.intFromMillis(timeMillis));
+  counter$4.inc();
+}
+
+var RollbackSuccess = {
+  timeCounter: timeCounter,
+  counter: counter$4,
+  increment: increment$3
+};
+
+var entityNameLabelsSchema = S$RescriptSchema.object(function (s) {
+      return s.f("entity", S$RescriptSchema.string);
+    });
+
+var timeCounter$1 = makeOrThrow("envio_rollback_history_prune_time", "The total time spent pruning entity history which is not in the reorg threshold. (milliseconds)", entityNameLabelsSchema);
+
+var counter$5 = makeOrThrow("envio_rollback_history_prune_count", "Number of successful entity history prunes", entityNameLabelsSchema);
+
+function increment$4(timeMillis, entityName) {
+  handleInt(timeCounter$1, entityName, Hrtime.intFromMillis(timeMillis));
+  increment(counter$5, entityName);
 }
 
-var RollbackDuration = {
-  histogram: histogram,
-  startTimer: startTimer$1
+var RollbackHistoryPrune = {
+  entityNameLabelsSchema: entityNameLabelsSchema,
+  timeCounter: timeCounter$1,
+  counter: counter$5,
+  increment: increment$4
 };
 
 var gauge$16 = makeOrThrow$1("envio_rollback_target_block_number", "The block number reorg was rollbacked to the last time.", chainIdLabelsSchema);
@@ -764,7 +786,8 @@ exports.ReorgCount = ReorgCount;
 exports.ReorgDetectionBlockNumber = ReorgDetectionBlockNumber;
 exports.ReorgThreshold = ReorgThreshold;
 exports.RollbackEnabled = RollbackEnabled;
-exports.RollbackDuration = RollbackDuration;
+exports.RollbackSuccess = RollbackSuccess;
+exports.RollbackHistoryPrune = RollbackHistoryPrune;
 exports.RollbackTargetBlockNumber = RollbackTargetBlockNumber;
 exports.ProcessingBlockNumber = ProcessingBlockNumber;
 exports.ProcessingBatchSize = ProcessingBatchSize;

package/src/sources/HyperSyncJsonApi.res

@@ -368,9 +368,20 @@ let queryRoute = Rest.route(() => {
   responses: [s => s.data(ResponseTypes.queryResponseSchema)],
 })
 
+@unboxed
+type heightResult = Value(int) | ErrorMessage(string)
+
 let heightRoute = Rest.route(() => {
   path: "/height",
   method: Get,
   input: s => s.auth(Bearer),
-  responses: [s => s.field("height", S.int)],
+  responses: [
+    s =>
+      s.data(
+        S.union([
+          S.object(s => Value(s.field("height", S.int))),
+          S.string->S.shape(s => ErrorMessage(s)),
+        ]),
+      ),
+  ],
 })

package/src/sources/HyperSyncJsonApi.res.js

@@ -247,7 +247,14 @@ function heightRoute() {
               return s.auth("Bearer");
             }),
           responses: [(function (s) {
-                return s.field("height", S$RescriptSchema.$$int);
+                return s.data(S$RescriptSchema.union([
+                                S$RescriptSchema.object(function (s) {
+                                      return s.f("height", S$RescriptSchema.$$int);
+                                    }),
+                                S$RescriptSchema.shape(S$RescriptSchema.string, (function (s) {
+                                        return s;
+                                      }))
+                              ]));
               })]
         };
 }