envio 2.27.3 → 2.27.5-rc.0

package/README.md

@@ -12,7 +12,7 @@ HyperIndex is a fast, developer-friendly multichain indexer, optimized for both
 ## Key Features
 
 - **[Indexer auto-generation](https://docs.envio.dev/docs/HyperIndex/contract-import)** – Generate Indexers directly from smart contract addresses
-- **High performance** – Historical backfills at over 5,000+ events per second ([fastest in market](https://docs.envio.dev/blog/indexer-benchmarking-results))
+- **High performance** – Historical backfills at over 10,000+ events per second ([fastest in market](https://docs.envio.dev/blog/indexer-benchmarking-results))
 - **Local development** – Full-featured local environment with Docker
 - **[Multichain indexing](https://docs.envio.dev/docs/HyperIndex/multichain-indexing)** – Index any EVM-compatible blockchain and Fuel (simultaneously)
 - **Real-time indexing** – Instantly track blockchain events

package/index.js

@@ -35,4 +35,5 @@ exports.S = {
   // Nullish type will change in "sury@10"
   // nullish: Sury.nullish,
   assertOrThrow: Sury.assertOrThrow,
+  parseOrThrow: Sury.parseOrThrow,
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "envio",
-  "version": "v2.27.3",
+  "version": "v2.27.5-rc.0",
   "description": "A latency and sync speed optimized, developer friendly blockchain data indexer.",
   "bin": "./bin.js",
   "main": "./index.js",
@@ -25,10 +25,10 @@
   },
   "homepage": "https://envio.dev",
   "optionalDependencies": {
-    "envio-linux-x64": "v2.27.3",
-    "envio-linux-arm64": "v2.27.3",
-    "envio-darwin-x64": "v2.27.3",
-    "envio-darwin-arm64": "v2.27.3"
+    "envio-linux-x64": "v2.27.5-rc.0",
+    "envio-linux-arm64": "v2.27.5-rc.0",
+    "envio-darwin-x64": "v2.27.5-rc.0",
+    "envio-darwin-arm64": "v2.27.5-rc.0"
   },
   "dependencies": {
     "@envio-dev/hypersync-client": "0.6.5",

package/src/FetchState.res

@@ -93,25 +93,6 @@ let copy = (fetchState: t) => {
   }
 }
 
-/*
-Comapritor for two events from the same chain. No need for chain id or timestamp
-*/
-let eventItemGt = (a: Internal.eventItem, b: Internal.eventItem) =>
-  if a.blockNumber > b.blockNumber {
-    true
-  } else if a.blockNumber === b.blockNumber {
-    a.logIndex > b.logIndex
-  } else {
-    false
-  }
-
-/*
-Merges two event queues on a single event fetcher
-
-Pass the shorter list into A for better performance
-*/
-let mergeSortedEventList = (a, b) => Utils.Array.mergeSorted(eventItemGt, a, b)
-
 let mergeIntoPartition = (p: partition, ~target: partition, ~maxAddrInPartition) => {
   switch (p, target) {
   | ({selection: {dependsOnAddresses: true}}, {selection: {dependsOnAddresses: true}}) => {
@@ -565,6 +546,18 @@ type query = {
 exception UnexpectedPartitionNotFound({partitionId: string})
 exception UnexpectedMergeQueryResponse({message: string})
 
+/*
+Comparitor for two events from the same chain. No need for chain id or timestamp
+*/
+let compareBufferItem = (a: Internal.eventItem, b: Internal.eventItem) => {
+  let blockDiff = b.blockNumber - a.blockNumber
+  if blockDiff === 0 {
+    b.logIndex - a.logIndex
+  } else {
+    blockDiff
+  }
+}
+
 /*
 Updates fetchState with a response for a given query.
 Returns Error if the partition with given query cannot be found (unexpected)
@@ -576,7 +569,7 @@ let handleQueryResult = (
   {partitions} as fetchState: t,
   ~query: query,
   ~latestFetchedBlock: blockNumberAndTimestamp,
-  ~reversedNewItems,
+  ~newItems,
   ~currentBlockHeight,
 ): result<t, exn> =>
   {
@@ -633,7 +626,12 @@ let handleQueryResult = (
     fetchState->updateInternal(
       ~partitions,
       ~currentBlockHeight,
-      ~queue=mergeSortedEventList(reversedNewItems, fetchState.queue),
+      ~queue=fetchState.queue
+      ->Array.concat(newItems)
+      // Theoretically it could be faster to asume that
+      // the items are sorted, but there are cases
+      // when the data source returns them unsorted
+      ->Js.Array2.sortInPlaceWith(compareBufferItem),
     )
   })
 
@@ -1235,14 +1233,34 @@ let filterAndSortForUnorderedBatch = {
     }
   }
 
-  let compareUnorderedBatchChainPriority = (a: t, b: t) => {
-    // Use unsafe since we filtered out all queues without batch items
-    (a.queue->Utils.Array.lastUnsafe).timestamp - (b.queue->Utils.Array.lastUnsafe).timestamp
+  let hasFullBatch = ({queue, latestFullyFetchedBlock}: t, ~maxBatchSize) => {
+    // Queue is ordered from latest to earliest, so the earliest eligible
+    // item for a full batch of size B is at index (length - B).
+    // Do NOT subtract an extra 1 here; when length === B we should still
+    // classify the queue as full and probe index 0.
+    let targetBlockIdx = queue->Array.length - maxBatchSize
+    if targetBlockIdx < 0 {
+      false
+    } else {
+      // Unsafe can fail when maxBatchSize is 0,
+      // but we ignore the case
+      (queue->Js.Array2.unsafe_get(targetBlockIdx)).blockNumber <=
+      latestFullyFetchedBlock.blockNumber
+    }
   }
 
-  (fetchStates: array<t>) => {
+  (fetchStates: array<t>, ~maxBatchSize: int) => {
     fetchStates
     ->Array.keepU(hasBatchItem)
-    ->Js.Array2.sortInPlaceWith(compareUnorderedBatchChainPriority)
+    ->Js.Array2.sortInPlaceWith((a: t, b: t) => {
+      switch (a->hasFullBatch(~maxBatchSize), b->hasFullBatch(~maxBatchSize)) {
+      | (true, true)
+      | (false, false) =>
+        // Use unsafe since we filtered out all queues without batch items
+        (a.queue->Utils.Array.lastUnsafe).timestamp - (b.queue->Utils.Array.lastUnsafe).timestamp
+      | (true, false) => -1
+      | (false, true) => 1
+      }
+    })
   }
 }

package/src/FetchState.res.js

@@ -34,20 +34,6 @@ function copy(fetchState) {
         };
 }
 
-function eventItemGt(a, b) {
-  if (a.blockNumber > b.blockNumber) {
-    return true;
-  } else if (a.blockNumber === b.blockNumber) {
-    return a.logIndex > b.logIndex;
-  } else {
-    return false;
-  }
-}
-
-function mergeSortedEventList(a, b) {
-  return Utils.$$Array.mergeSorted(eventItemGt, a, b);
-}
-
 function mergeIntoPartition(p, target, maxAddrInPartition) {
   if (!p.selection.dependsOnAddresses) {
     return [
@@ -344,7 +330,16 @@ var UnexpectedPartitionNotFound = /* @__PURE__ */Caml_exceptions.create("FetchSt
 
 var UnexpectedMergeQueryResponse = /* @__PURE__ */Caml_exceptions.create("FetchState.UnexpectedMergeQueryResponse");
 
-function handleQueryResult(fetchState, query, latestFetchedBlock, reversedNewItems, currentBlockHeight) {
+function compareBufferItem(a, b) {
+  var blockDiff = b.blockNumber - a.blockNumber | 0;
+  if (blockDiff === 0) {
+    return b.logIndex - a.logIndex | 0;
+  } else {
+    return blockDiff;
+  }
+}
+
+function handleQueryResult(fetchState, query, latestFetchedBlock, newItems, currentBlockHeight) {
   var partitions = fetchState.partitions;
   var partitionId = query.partitionId;
   var pIndex = Belt_Array.getIndexBy(partitions, (function (p) {
@@ -414,7 +409,7 @@ function handleQueryResult(fetchState, query, latestFetchedBlock, reversedNewIte
     };
   }
   return Belt_Result.map(tmp, (function (partitions) {
-                return updateInternal(fetchState, partitions, undefined, undefined, undefined, currentBlockHeight, mergeSortedEventList(reversedNewItems, fetchState.queue), undefined);
+                return updateInternal(fetchState, partitions, undefined, undefined, undefined, currentBlockHeight, Belt_Array.concat(fetchState.queue, newItems).sort(compareBufferItem), undefined);
               }));
 }
 
@@ -930,17 +925,33 @@ function hasBatchItem(param) {
   }
 }
 
-function compareUnorderedBatchChainPriority(a, b) {
-  return Utils.$$Array.lastUnsafe(a.queue).timestamp - Utils.$$Array.lastUnsafe(b.queue).timestamp | 0;
+function hasFullBatch(param, maxBatchSize) {
+  var queue = param.queue;
+  var targetBlockIdx = queue.length - maxBatchSize | 0;
+  if (targetBlockIdx < 0) {
+    return false;
+  } else {
+    return queue[targetBlockIdx].blockNumber <= param.latestFullyFetchedBlock.blockNumber;
+  }
 }
 
-function filterAndSortForUnorderedBatch(fetchStates) {
-  return Belt_Array.keepU(fetchStates, hasBatchItem).sort(compareUnorderedBatchChainPriority);
+function filterAndSortForUnorderedBatch(fetchStates, maxBatchSize) {
+  return Belt_Array.keepU(fetchStates, hasBatchItem).sort(function (a, b) {
+              var match = hasFullBatch(a, maxBatchSize);
+              var match$1 = hasFullBatch(b, maxBatchSize);
+              if (match) {
+                if (!match$1) {
+                  return -1;
+                }
+                
+              } else if (match$1) {
+                return 1;
+              }
+              return Utils.$$Array.lastUnsafe(a.queue).timestamp - Utils.$$Array.lastUnsafe(b.queue).timestamp | 0;
+            });
 }
 
 exports.copy = copy;
-exports.eventItemGt = eventItemGt;
-exports.mergeSortedEventList = mergeSortedEventList;
 exports.mergeIntoPartition = mergeIntoPartition;
 exports.checkIsWithinSyncRange = checkIsWithinSyncRange;
 exports.updateInternal = updateInternal;
@@ -949,6 +960,7 @@ exports.warnDifferentContractType = warnDifferentContractType;
 exports.registerDynamicContracts = registerDynamicContracts;
 exports.UnexpectedPartitionNotFound = UnexpectedPartitionNotFound;
 exports.UnexpectedMergeQueryResponse = UnexpectedMergeQueryResponse;
+exports.compareBufferItem = compareBufferItem;
 exports.handleQueryResult = handleQueryResult;
 exports.makePartitionQuery = makePartitionQuery;
 exports.startFetchingQueries = startFetchingQueries;

package/src/PgStorage.res

@@ -325,8 +325,8 @@ let removeInvalidUtf8InPlace = entities =>
     })
   })
 
-let pgEncodingErrorSchema = S.object(s =>
-  s.tag("message", `invalid byte sequence for encoding "UTF8": 0x00`)
+let pgErrorMessageSchema = S.object(s =>
+  s.field("message", S.string)
 )
 
 exception PgEncodingError({table: Table.table})

package/src/PgStorage.res.js

@@ -228,8 +228,8 @@ function removeInvalidUtf8InPlace(entities) {
       });
 }
 
-var pgEncodingErrorSchema = S$RescriptSchema.object(function (s) {
-      s.tag("message", "invalid byte sequence for encoding \"UTF8\": 0x00");
+var pgErrorMessageSchema = S$RescriptSchema.object(function (s) {
+      return s.f("message", S$RescriptSchema.string);
     });
 
 var PgEncodingError = /* @__PURE__ */Caml_exceptions.create("PgStorage.PgEncodingError");
@@ -631,7 +631,7 @@ exports.maxItemsPerQuery = maxItemsPerQuery;
 exports.makeTableBatchSetQuery = makeTableBatchSetQuery;
 exports.chunkArray = chunkArray;
 exports.removeInvalidUtf8InPlace = removeInvalidUtf8InPlace;
-exports.pgEncodingErrorSchema = pgEncodingErrorSchema;
+exports.pgErrorMessageSchema = pgErrorMessageSchema;
 exports.PgEncodingError = PgEncodingError;
 exports.setQueryCache = setQueryCache;
 exports.setOrThrow = setOrThrow;
@@ -641,4 +641,4 @@ exports.cacheTablePrefixLength = cacheTablePrefixLength;
 exports.makeSchemaCacheTableInfoQuery = makeSchemaCacheTableInfoQuery;
 exports.getConnectedPsqlExec = getConnectedPsqlExec;
 exports.make = make;
-/* pgEncodingErrorSchema Not a pure module */
+/* pgErrorMessageSchema Not a pure module */

package/src/Prometheus.res

@@ -241,25 +241,6 @@ module BenchmarkCounters = {
   }
 }
 
-module PartitionBlockFetched = {
-  type labels = {chainId: int, partitionId: string}
-
-  let labelSchema = S.schema(s => {
-    chainId: s.matches(S.string->S.coerce(S.int)),
-    partitionId: s.matches(S.string),
-  })
-
-  let counter = SafeGauge.makeOrThrow(
-    ~name="partition_block_fetched",
-    ~help="The latest fetched block number for each partition",
-    ~labelSchema,
-  )
-
-  let set = (~blockNumber, ~partitionId, ~chainId) => {
-    counter->SafeGauge.handleInt(~labels={chainId, partitionId}, ~value=blockNumber)
-  }
-}
-
 let chainIdLabelsSchema = S.object(s => {
   s.field("chainId", S.string->S.coerce(S.int))
 })
@@ -440,12 +421,6 @@ module SourceGetHeightDuration = {
 }
 
 module ReorgCount = {
-  let deprecatedCounter = PromClient.Counter.makeCounter({
-    "name": "reorgs_detected",
-    "help": "Total number of reorgs detected",
-    "labelNames": ["chainId"],
-  })
-
   let gauge = SafeGauge.makeOrThrow(
     ~name="envio_reorg_count",
     ~help="Total number of reorgs detected",
@@ -453,9 +428,6 @@ module ReorgCount = {
   )
 
   let increment = (~chain) => {
-    deprecatedCounter
-    ->PromClient.Counter.labels({"chainId": chain->ChainMap.Chain.toString})
-    ->PromClient.Counter.inc
     gauge->SafeGauge.increment(~labels=chain->ChainMap.Chain.toChainId)
   }
 }
@@ -494,15 +466,44 @@ module RollbackEnabled = {
   }
 }
 
-module RollbackDuration = {
-  let histogram = PromClient.Histogram.make({
-    "name": "envio_rollback_duration",
-    "help": "Rollback on reorg duration in seconds",
-    "buckets": [0.5, 1., 5., 10.],
+module RollbackSuccess = {
+  let timeCounter = PromClient.Counter.makeCounter({
+    "name": "envio_rollback_time",
+    "help": "Rollback on reorg total time in milliseconds",
+  })
+
+  let counter = PromClient.Counter.makeCounter({
+    "name": "envio_rollback_count",
+    "help": "Number of successful rollbacks on reorg",
   })
 
-  let startTimer = () => {
-    histogram->PromClient.Histogram.startTimer
+  let increment = (~timeMillis: Hrtime.milliseconds) => {
+    timeCounter->PromClient.Counter.incMany(timeMillis->Hrtime.intFromMillis)
+    counter->PromClient.Counter.inc
+  }
+}
+
+module RollbackHistoryPrune = {
+  let entityNameLabelsSchema = S.object(s => s.field("entity", S.string))
+
+  let timeCounter = SafeCounter.makeOrThrow(
+    ~name="envio_rollback_history_prune_time",
+    ~help="The total time spent pruning entity history which is not in the reorg threshold. (milliseconds)",
+    ~labelSchema=entityNameLabelsSchema,
+  )
+
+  let counter = SafeCounter.makeOrThrow(
+    ~name="envio_rollback_history_prune_count",
+    ~help="Number of successful entity history prunes",
+    ~labelSchema=entityNameLabelsSchema,
+  )
+
+  let increment = (~timeMillis, ~entityName) => {
+    timeCounter->SafeCounter.handleInt(
+      ~labels={entityName},
+      ~value=timeMillis->Hrtime.intFromMillis,
+    )
+    counter->SafeCounter.increment(~labels={entityName})
   }
 }
 
@@ -613,3 +614,79 @@ module EffectCacheCount = {
     gauge->SafeGauge.handleInt(~labels=effectName, ~value=count)
   }
 }
+
+module StorageLoad = {
+  let operationLabelsSchema = S.object(s => s.field("operation", S.string))
+
+  let timeCounter = SafeCounter.makeOrThrow(
+    ~name="envio_storage_load_time",
+    ~help="Processing time taken to load data from storage. (milliseconds)",
+    ~labelSchema=operationLabelsSchema,
+  )
+
+  let totalTimeCounter = SafeCounter.makeOrThrow(
+    ~name="envio_storage_load_total_time",
+    ~help="Cumulative time spent loading data from storage during the indexing process. (milliseconds)",
+    ~labelSchema=operationLabelsSchema,
+  )
+
+  let counter = SafeCounter.makeOrThrow(
+    ~name="envio_storage_load_count",
+    ~help="Cumulative number of successful storage load operations during the indexing process.",
+    ~labelSchema=operationLabelsSchema,
+  )
+
+  let whereSizeCounter = SafeCounter.makeOrThrow(
+    ~name="envio_storage_load_where_size",
+    ~help="Cumulative number of filter conditions ('where' items) used in storage load operations during the indexing process.",
+    ~labelSchema=operationLabelsSchema,
+  )
+
+  let sizeCounter = SafeCounter.makeOrThrow(
+    ~name="envio_storage_load_size",
+    ~help="Cumulative number of records loaded from storage during the indexing process.",
+    ~labelSchema=operationLabelsSchema,
+  )
+
+  type operationRef = {
+    mutable pendingCount: int,
+    timerRef: Hrtime.timeRef,
+  }
+  let operations = Js.Dict.empty()
+
+  let startOperation = (~operation) => {
+    switch operations->Utils.Dict.dangerouslyGetNonOption(operation) {
+    | Some(operationRef) => operationRef.pendingCount = operationRef.pendingCount + 1
+    | None =>
+      operations->Js.Dict.set(
+        operation,
+        (
+          {
+            pendingCount: 1,
+            timerRef: Hrtime.makeTimer(),
+          }: operationRef
+        ),
+      )
+    }
+    Hrtime.makeTimer()
+  }
+
+  let endOperation = (timerRef, ~operation, ~whereSize, ~size) => {
+    let operationRef = operations->Js.Dict.unsafeGet(operation)
+    operationRef.pendingCount = operationRef.pendingCount - 1
+    if operationRef.pendingCount === 0 {
+      timeCounter->SafeCounter.handleInt(
+        ~labels={operation},
+        ~value=operationRef.timerRef->Hrtime.timeSince->Hrtime.toMillis->Hrtime.intFromMillis,
+      )
+      operations->Utils.Dict.deleteInPlace(operation)
+    }
+    totalTimeCounter->SafeCounter.handleInt(
+      ~labels={operation},
+      ~value=timerRef->Hrtime.timeSince->Hrtime.toMillis->Hrtime.intFromMillis,
+    )
+    counter->SafeCounter.increment(~labels={operation})
+    whereSizeCounter->SafeCounter.handleInt(~labels={operation}, ~value=whereSize)
+    sizeCounter->SafeCounter.handleInt(~labels={operation}, ~value=size)
+  }
+}

package/src/Prometheus.res.js

@@ -1,6 +1,8 @@
 // Generated by ReScript, PLEASE EDIT WITH CARE
 'use strict';
 
+var Utils = require("./Utils.res.js");
+var Hrtime = require("./bindings/Hrtime.res.js");
 var Js_exn = require("rescript/lib/js/js_exn.js");
 var ChainMap = require("./ChainMap.res.js");
 var Belt_Array = require("rescript/lib/js/belt_Array.js");
@@ -346,28 +348,6 @@ var BenchmarkCounters = {
   set: set$1
 };
 
-var labelSchema$2 = S$RescriptSchema.schema(function (s) {
-      return {
-              chainId: s.m(S$RescriptSchema.coerce(S$RescriptSchema.string, S$RescriptSchema.$$int)),
-              partitionId: s.m(S$RescriptSchema.string)
-            };
-    });
-
-var counter = makeOrThrow$1("partition_block_fetched", "The latest fetched block number for each partition", labelSchema$2);
-
-function set$2(blockNumber, partitionId, chainId) {
-  handleInt$1(counter, {
-        chainId: chainId,
-        partitionId: partitionId
-      }, blockNumber);
-}
-
-var PartitionBlockFetched = {
-  labelSchema: labelSchema$2,
-  counter: counter,
-  set: set$2
-};
-
 var chainIdLabelsSchema = S$RescriptSchema.object(function (s) {
       return s.f("chainId", S$RescriptSchema.coerce(S$RescriptSchema.string, S$RescriptSchema.$$int));
     });
@@ -378,7 +358,7 @@ var gauge$2 = makeOrThrow$1("envio_info", "Information about the indexer", S$Res
                 };
         }));
 
-function set$3(version) {
+function set$2(version) {
   handleInt$1(gauge$2, {
         version: version
       }, 1);
@@ -386,80 +366,80 @@ function set$3(version) {
 
 var Info = {
   gauge: gauge$2,
-  set: set$3
+  set: set$2
 };
 
 var gauge$3 = makeOrThrow$1("envio_indexing_addresses", "The number of addresses indexed on chain. Includes both static and dynamic addresses.", chainIdLabelsSchema);
 
-function set$4(addressesCount, chainId) {
+function set$3(addressesCount, chainId) {
   handleInt$1(gauge$3, chainId, addressesCount);
 }
 
 var IndexingAddresses = {
   gauge: gauge$3,
-  set: set$4
+  set: set$3
 };
 
 var gauge$4 = makeOrThrow$1("envio_indexing_max_concurrency", "The maximum number of concurrent queries to the chain data-source.", chainIdLabelsSchema);
 
-function set$5(maxConcurrency, chainId) {
+function set$4(maxConcurrency, chainId) {
   handleInt$1(gauge$4, chainId, maxConcurrency);
 }
 
 var IndexingMaxConcurrency = {
   gauge: gauge$4,
-  set: set$5
+  set: set$4
 };
 
 var gauge$5 = makeOrThrow$1("envio_indexing_concurrency", "The number of executing concurrent queries to the chain data-source.", chainIdLabelsSchema);
 
-function set$6(concurrency, chainId) {
+function set$5(concurrency, chainId) {
   handleInt$1(gauge$5, chainId, concurrency);
 }
 
 var IndexingConcurrency = {
   gauge: gauge$5,
-  set: set$6
+  set: set$5
 };
 
 var gauge$6 = makeOrThrow$1("envio_indexing_partitions", "The number of partitions used to split fetching logic by addresses and block ranges.", chainIdLabelsSchema);
 
-function set$7(partitionsCount, chainId) {
+function set$6(partitionsCount, chainId) {
   handleInt$1(gauge$6, chainId, partitionsCount);
 }
 
 var IndexingPartitions = {
   gauge: gauge$6,
-  set: set$7
+  set: set$6
 };
 
-var counter$1 = makeOrThrow("envio_indexing_idle_time", "The number of milliseconds the indexer source syncing has been idle. A high value may indicate the source sync is a bottleneck.", chainIdLabelsSchema);
+var counter = makeOrThrow("envio_indexing_idle_time", "The number of milliseconds the indexer source syncing has been idle. A high value may indicate the source sync is a bottleneck.", chainIdLabelsSchema);
 
 var IndexingIdleTime = {
-  counter: counter$1
+  counter: counter
 };
 
-var counter$2 = makeOrThrow("envio_indexing_source_waiting_time", "The number of milliseconds the indexer has been waiting for new blocks.", chainIdLabelsSchema);
+var counter$1 = makeOrThrow("envio_indexing_source_waiting_time", "The number of milliseconds the indexer has been waiting for new blocks.", chainIdLabelsSchema);
 
 var IndexingSourceWaitingTime = {
-  counter: counter$2
+  counter: counter$1
 };
 
-var counter$3 = makeOrThrow("envio_indexing_query_time", "The number of milliseconds spent performing queries to the chain data-source.", chainIdLabelsSchema);
+var counter$2 = makeOrThrow("envio_indexing_query_time", "The number of milliseconds spent performing queries to the chain data-source.", chainIdLabelsSchema);
 
 var IndexingQueryTime = {
-  counter: counter$3
+  counter: counter$2
 };
 
 var gauge$7 = makeOrThrow$1("envio_indexing_buffer_size", "The current number of items in the indexing buffer.", chainIdLabelsSchema);
 
-function set$8(bufferSize, chainId) {
+function set$7(bufferSize, chainId) {
   handleInt$1(gauge$7, chainId, bufferSize);
 }
 
 var IndexingBufferSize = {
   gauge: gauge$7,
-  set: set$8
+  set: set$7
 };
 
 var gauge$8 = new PromClient.Gauge({
@@ -467,13 +447,13 @@ var gauge$8 = new PromClient.Gauge({
       help: "The target buffer size per chain for indexing. The actual number of items in the queue may exceed this value, but the indexer always tries to keep the buffer filled up to this target."
     });
 
-function set$9(targetBufferSize) {
+function set$8(targetBufferSize) {
   gauge$8.set(targetBufferSize);
 }
 
 var IndexingTargetBufferSize = {
   gauge: gauge$8,
-  set: set$9
+  set: set$8
 };
 
 var deprecatedGauge = new PromClient.Gauge({
@@ -484,7 +464,7 @@ var deprecatedGauge = new PromClient.Gauge({
 
 var gauge$9 = makeOrThrow$1("envio_indexing_buffer_block_number", "The highest block number that has been fully fetched by the indexer.", chainIdLabelsSchema);
 
-function set$10(blockNumber, chainId) {
+function set$9(blockNumber, chainId) {
   deprecatedGauge.labels({
           chainId: chainId
         }).set(blockNumber);
@@ -494,18 +474,18 @@ function set$10(blockNumber, chainId) {
 var IndexingBufferBlockNumber = {
   deprecatedGauge: deprecatedGauge,
   gauge: gauge$9,
-  set: set$10
+  set: set$9
 };
 
 var gauge$10 = makeOrThrow$1("envio_indexing_end_block", "The block number to stop indexing at. (inclusive)", chainIdLabelsSchema);
 
-function set$11(endBlock, chainId) {
+function set$10(endBlock, chainId) {
   handleInt$1(gauge$10, chainId, endBlock);
 }
 
 var IndexingEndBlock = {
   gauge: gauge$10,
-  set: set$11
+  set: set$10
 };
 
 var sourceLabelsSchema = S$RescriptSchema.schema(function (s) {
@@ -517,7 +497,7 @@ var sourceLabelsSchema = S$RescriptSchema.schema(function (s) {
 
 var gauge$11 = makeOrThrow$1("envio_source_height", "The latest known block number reported by the source. This value may lag behind the actual chain height, as it is updated only when queried.", sourceLabelsSchema);
 
-function set$12(sourceName, chainId, blockNumber) {
+function set$11(sourceName, chainId, blockNumber) {
   handleInt$1(gauge$11, {
         source: sourceName,
         chainId: chainId
@@ -526,7 +506,7 @@ function set$12(sourceName, chainId, blockNumber) {
 
 var SourceHeight = {
   gauge: gauge$11,
-  set: set$12
+  set: set$11
 };
 
 var startTimer = makeSafeHistogramOrThrow("envio_source_get_height_duration", "Duration of the source get height requests in seconds", sourceLabelsSchema, [
@@ -540,36 +520,26 @@ var SourceGetHeightDuration = {
   startTimer: startTimer
 };
 
-var deprecatedCounter = new PromClient.Counter({
-      name: "reorgs_detected",
-      help: "Total number of reorgs detected",
-      labelNames: ["chainId"]
-    });
-
 var gauge$12 = makeOrThrow$1("envio_reorg_count", "Total number of reorgs detected", chainIdLabelsSchema);
 
 function increment$2(chain) {
-  deprecatedCounter.labels({
-          chainId: ChainMap.Chain.toString(chain)
-        }).inc();
   increment$1(gauge$12, chain);
 }
 
 var ReorgCount = {
-  deprecatedCounter: deprecatedCounter,
   gauge: gauge$12,
   increment: increment$2
 };
 
 var gauge$13 = makeOrThrow$1("envio_reorg_detection_block_number", "The block number where reorg was detected the last time. This doesn't mean that the block was reorged, this is simply where we found block hash to be different.", chainIdLabelsSchema);
 
-function set$13(blockNumber, chain) {
+function set$12(blockNumber, chain) {
   handleInt$1(gauge$13, chain, blockNumber);
 }
 
 var ReorgDetectionBlockNumber = {
   gauge: gauge$13,
-  set: set$13
+  set: set$12
 };
 
 var gauge$14 = new PromClient.Gauge({
@@ -577,13 +547,13 @@ var gauge$14 = new PromClient.Gauge({
       help: "Whether indexing is currently within the reorg threshold"
     });
 
-function set$14(isInReorgThreshold) {
+function set$13(isInReorgThreshold) {
   gauge$14.set(isInReorgThreshold ? 1 : 0);
 }
 
 var ReorgThreshold = {
   gauge: gauge$14,
-  set: set$14
+  set: set$13
 };
 
 var gauge$15 = new PromClient.Gauge({
@@ -591,66 +561,87 @@ var gauge$15 = new PromClient.Gauge({
       help: "Whether rollback on reorg is enabled"
     });
 
-function set$15(enabled) {
+function set$14(enabled) {
   gauge$15.set(enabled ? 1 : 0);
 }
 
 var RollbackEnabled = {
   gauge: gauge$15,
-  set: set$15
+  set: set$14
 };
 
-var histogram = new PromClient.Histogram({
-      name: "envio_rollback_duration",
-      help: "Rollback on reorg duration in seconds",
-      buckets: [
-        0.5,
-        1,
-        5,
-        10
-      ]
+var timeCounter = new PromClient.Counter({
+      name: "envio_rollback_time",
+      help: "Rollback on reorg total time in milliseconds"
+    });
+
+var counter$3 = new PromClient.Counter({
+      name: "envio_rollback_count",
+      help: "Number of successful rollbacks on reorg"
     });
 
-function startTimer$1() {
-  return histogram.startTimer();
+function increment$3(timeMillis) {
+  timeCounter.inc(Hrtime.intFromMillis(timeMillis));
+  counter$3.inc();
+}
+
+var RollbackSuccess = {
+  timeCounter: timeCounter,
+  counter: counter$3,
+  increment: increment$3
+};
+
+var entityNameLabelsSchema = S$RescriptSchema.object(function (s) {
+      return s.f("entity", S$RescriptSchema.string);
+    });
+
+var timeCounter$1 = makeOrThrow("envio_rollback_history_prune_time", "The total time spent pruning entity history which is not in the reorg threshold. (milliseconds)", entityNameLabelsSchema);
+
+var counter$4 = makeOrThrow("envio_rollback_history_prune_count", "Number of successful entity history prunes", entityNameLabelsSchema);
+
+function increment$4(timeMillis, entityName) {
+  handleInt(timeCounter$1, entityName, Hrtime.intFromMillis(timeMillis));
+  increment(counter$4, entityName);
 }
 
-var RollbackDuration = {
-  histogram: histogram,
-  startTimer: startTimer$1
+var RollbackHistoryPrune = {
+  entityNameLabelsSchema: entityNameLabelsSchema,
+  timeCounter: timeCounter$1,
+  counter: counter$4,
+  increment: increment$4
 };
 
 var gauge$16 = makeOrThrow$1("envio_rollback_target_block_number", "The block number reorg was rollbacked to the last time.", chainIdLabelsSchema);
 
-function set$16(blockNumber, chain) {
+function set$15(blockNumber, chain) {
   handleInt$1(gauge$16, chain, blockNumber);
 }
 
 var RollbackTargetBlockNumber = {
   gauge: gauge$16,
-  set: set$16
+  set: set$15
 };
 
 var gauge$17 = makeOrThrow$1("envio_processing_block_number", "The latest item block number included in the currently processing batch for the chain.", chainIdLabelsSchema);
 
-function set$17(blockNumber, chainId) {
+function set$16(blockNumber, chainId) {
   handleInt$1(gauge$17, chainId, blockNumber);
 }
 
 var ProcessingBlockNumber = {
   gauge: gauge$17,
-  set: set$17
+  set: set$16
 };
 
 var gauge$18 = makeOrThrow$1("envio_processing_batch_size", "The number of items included in the currently processing batch for the chain.", chainIdLabelsSchema);
 
-function set$18(batchSize, chainId) {
+function set$17(batchSize, chainId) {
   handleInt$1(gauge$18, chainId, batchSize);
 }
 
 var ProcessingBatchSize = {
   gauge: gauge$18,
-  set: set$18
+  set: set$17
 };
 
 var gauge$19 = new PromClient.Gauge({
@@ -658,24 +649,24 @@ var gauge$19 = new PromClient.Gauge({
       help: "The maximum number of items to process in a single batch."
     });
 
-function set$19(maxBatchSize) {
+function set$18(maxBatchSize) {
   gauge$19.set(maxBatchSize);
 }
 
 var ProcessingMaxBatchSize = {
   gauge: gauge$19,
-  set: set$19
+  set: set$18
 };
 
 var gauge$20 = makeOrThrow$1("envio_progress_block_number", "The block number of the latest block processed and stored in the database.", chainIdLabelsSchema);
 
-function set$20(blockNumber, chainId) {
+function set$19(blockNumber, chainId) {
   handleInt$1(gauge$20, chainId, blockNumber);
 }
 
 var ProgressBlockNumber = {
   gauge: gauge$20,
-  set: set$20
+  set: set$19
 };
 
 var deprecatedGauge$1 = new PromClient.Gauge({
@@ -686,7 +677,7 @@ var deprecatedGauge$1 = new PromClient.Gauge({
 
 var gauge$21 = makeOrThrow$1("envio_progress_events_count", "The number of events processed and reflected in the database.", chainIdLabelsSchema);
 
-function set$21(processedCount, chainId) {
+function set$20(processedCount, chainId) {
   deprecatedGauge$1.labels({
           chainId: chainId
         }).set(processedCount);
@@ -696,7 +687,7 @@ function set$21(processedCount, chainId) {
 var ProgressEventsCount = {
   deprecatedGauge: deprecatedGauge$1,
   gauge: gauge$21,
-  set: set$21
+  set: set$20
 };
 
 var effectLabelsSchema = S$RescriptSchema.object(function (s) {
@@ -705,24 +696,78 @@ var effectLabelsSchema = S$RescriptSchema.object(function (s) {
 
 var gauge$22 = makeOrThrow$1("envio_effect_calls_count", "The number of calls to the effect. Including both handler execution and cache hits.", effectLabelsSchema);
 
-function set$22(callsCount, effectName) {
+function set$21(callsCount, effectName) {
   handleInt$1(gauge$22, effectName, callsCount);
 }
 
 var EffectCallsCount = {
   gauge: gauge$22,
-  set: set$22
+  set: set$21
 };
 
 var gauge$23 = makeOrThrow$1("envio_effect_cache_count", "The number of items in the effect cache.", effectLabelsSchema);
 
-function set$23(count, effectName) {
+function set$22(count, effectName) {
   handleInt$1(gauge$23, effectName, count);
 }
 
 var EffectCacheCount = {
   gauge: gauge$23,
-  set: set$23
+  set: set$22
+};
+
+var operationLabelsSchema = S$RescriptSchema.object(function (s) {
+      return s.f("operation", S$RescriptSchema.string);
+    });
+
+var timeCounter$2 = makeOrThrow("envio_storage_load_time", "Processing time taken to load data from storage. (milliseconds)", operationLabelsSchema);
+
+var totalTimeCounter = makeOrThrow("envio_storage_load_total_time", "Cumulative time spent loading data from storage during the indexing process. (milliseconds)", operationLabelsSchema);
+
+var counter$5 = makeOrThrow("envio_storage_load_count", "Cumulative number of successful storage load operations during the indexing process.", operationLabelsSchema);
+
+var whereSizeCounter = makeOrThrow("envio_storage_load_where_size", "Cumulative number of filter conditions ('where' items) used in storage load operations during the indexing process.", operationLabelsSchema);
+
+var sizeCounter = makeOrThrow("envio_storage_load_size", "Cumulative number of records loaded from storage during the indexing process.", operationLabelsSchema);
+
+var operations = {};
+
+function startOperation(operation) {
+  var operationRef = operations[operation];
+  if (operationRef !== undefined) {
+    operationRef.pendingCount = operationRef.pendingCount + 1 | 0;
+  } else {
+    operations[operation] = {
+      pendingCount: 1,
+      timerRef: Hrtime.makeTimer()
+    };
+  }
+  return Hrtime.makeTimer();
+}
+
+function endOperation(timerRef, operation, whereSize, size) {
+  var operationRef = operations[operation];
+  operationRef.pendingCount = operationRef.pendingCount - 1 | 0;
+  if (operationRef.pendingCount === 0) {
+    handleInt(timeCounter$2, operation, Hrtime.intFromMillis(Hrtime.toMillis(Hrtime.timeSince(operationRef.timerRef))));
+    Utils.Dict.deleteInPlace(operations, operation);
+  }
+  handleInt(totalTimeCounter, operation, Hrtime.intFromMillis(Hrtime.toMillis(Hrtime.timeSince(timerRef))));
+  increment(counter$5, operation);
+  handleInt(whereSizeCounter, operation, whereSize);
+  handleInt(sizeCounter, operation, size);
+}
+
+var StorageLoad = {
+  operationLabelsSchema: operationLabelsSchema,
+  timeCounter: timeCounter$2,
+  totalTimeCounter: totalTimeCounter,
+  counter: counter$5,
+  whereSizeCounter: whereSizeCounter,
+  sizeCounter: sizeCounter,
+  operations: operations,
+  startOperation: startOperation,
+  endOperation: endOperation
 };
 
 exports.loadEntitiesDurationCounter = loadEntitiesDurationCounter;
@@ -743,7 +788,6 @@ exports.incrementExecuteBatchDurationCounter = incrementExecuteBatchDurationCoun
 exports.setSourceChainHeight = setSourceChainHeight;
 exports.setAllChainsSyncedToHead = setAllChainsSyncedToHead;
 exports.BenchmarkCounters = BenchmarkCounters;
-exports.PartitionBlockFetched = PartitionBlockFetched;
 exports.chainIdLabelsSchema = chainIdLabelsSchema;
 exports.Info = Info;
 exports.IndexingAddresses = IndexingAddresses;
@@ -764,7 +808,8 @@ exports.ReorgCount = ReorgCount;
 exports.ReorgDetectionBlockNumber = ReorgDetectionBlockNumber;
 exports.ReorgThreshold = ReorgThreshold;
 exports.RollbackEnabled = RollbackEnabled;
-exports.RollbackDuration = RollbackDuration;
+exports.RollbackSuccess = RollbackSuccess;
+exports.RollbackHistoryPrune = RollbackHistoryPrune;
 exports.RollbackTargetBlockNumber = RollbackTargetBlockNumber;
 exports.ProcessingBlockNumber = ProcessingBlockNumber;
 exports.ProcessingBatchSize = ProcessingBatchSize;
@@ -774,4 +819,5 @@ exports.ProgressEventsCount = ProgressEventsCount;
 exports.effectLabelsSchema = effectLabelsSchema;
 exports.EffectCallsCount = EffectCallsCount;
 exports.EffectCacheCount = EffectCacheCount;
+exports.StorageLoad = StorageLoad;
 /* loadEntitiesDurationCounter Not a pure module */

package/src/db/Table.res

@@ -240,6 +240,7 @@ let toSqlParams = (table: table, ~schema, ~pgSchema) => {
           switch field {
           | Field(f) =>
             switch f.fieldType {
+            | Custom(fieldType) if fieldType->Js.String2.startsWith("NUMERIC(") => fieldType
             | Custom(fieldType) => `${(Text :> string)}[]::"${pgSchema}".${(fieldType :> string)}`
             | Boolean => `${(Integer :> string)}[]::${(f.fieldType :> string)}`
             | fieldType => (fieldType :> string)

package/src/db/Table.res.js

@@ -261,7 +261,9 @@ function toSqlParams(table, schema, pgSchema) {
                   var fieldType = f.fieldType;
                   tmp = fieldType === "TIMESTAMP" || fieldType === "TIMESTAMP WITH TIME ZONE" || fieldType === "JSONB" || fieldType === "SERIAL" || fieldType === "TEXT" || fieldType === "DOUBLE PRECISION" || fieldType === "NUMERIC" || fieldType === "BOOLEAN" || fieldType === "INTEGER" || fieldType === "TIMESTAMP WITH TIME ZONE NULL" ? (
                       fieldType === "BOOLEAN" ? "INTEGER[]::" + f.fieldType : fieldType
-                    ) : "TEXT[]::\"" + pgSchema + "\"." + fieldType;
+                    ) : (
+                      fieldType.startsWith("NUMERIC(") ? fieldType : "TEXT[]::\"" + pgSchema + "\"." + fieldType
+                    );
                 } else {
                   tmp = "TEXT";
                 }

package/src/sources/HyperSyncJsonApi.res

@@ -368,9 +368,20 @@ let queryRoute = Rest.route(() => {
   responses: [s => s.data(ResponseTypes.queryResponseSchema)],
 })
 
+@unboxed
+type heightResult = Value(int) | ErrorMessage(string)
+
 let heightRoute = Rest.route(() => {
   path: "/height",
   method: Get,
   input: s => s.auth(Bearer),
-  responses: [s => s.field("height", S.int)],
+  responses: [
+    s =>
+      s.data(
+        S.union([
+          S.object(s => Value(s.field("height", S.int))),
+          S.string->S.shape(s => ErrorMessage(s)),
+        ]),
+      ),
+  ],
 })

package/src/sources/HyperSyncJsonApi.res.js

@@ -247,7 +247,14 @@ function heightRoute() {
               return s.auth("Bearer");
             }),
           responses: [(function (s) {
-                return s.field("height", S$RescriptSchema.$$int);
+                return s.data(S$RescriptSchema.union([
+                                S$RescriptSchema.object(function (s) {
+                                      return s.f("height", S$RescriptSchema.$$int);
+                                    }),
+                                S$RescriptSchema.shape(S$RescriptSchema.string, (function (s) {
+                                        return s;
+                                      }))
+                              ]));
               })]
         };
 }