envio 2.19.2 → 2.20.1

package/evm.schema.json

@@ -27,7 +27,7 @@
       ]
     },
     "schema": {
-      "description": "Custom path to schema.yaml file",
+      "description": "Custom path to schema.graphql file",
       "type": [
         "string",
         "null"

package/fuel.schema.json

@@ -20,7 +20,7 @@
       "$ref": "#/$defs/EcosystemTag"
     },
     "schema": {
-      "description": "Custom path to schema.yaml file",
+      "description": "Custom path to schema.graphql file",
       "type": [
         "string",
         "null"

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "envio",
-  "version": "v2.19.2",
+  "version": "v2.20.1",
   "description": "A latency and sync speed optimized, developer friendly blockchain data indexer.",
   "bin": "./bin.js",
   "main": "./index.js",
@@ -25,10 +25,10 @@
   },
   "homepage": "https://envio.dev",
   "optionalDependencies": {
-    "envio-linux-x64": "v2.19.2",
-    "envio-linux-arm64": "v2.19.2",
-    "envio-darwin-x64": "v2.19.2",
-    "envio-darwin-arm64": "v2.19.2"
+    "envio-linux-x64": "v2.20.1",
+    "envio-linux-arm64": "v2.20.1",
+    "envio-darwin-x64": "v2.20.1",
+    "envio-darwin-arm64": "v2.20.1"
   },
   "dependencies": {
     "@envio-dev/hypersync-client": "0.6.3",

package/src/Envio.gen.ts

@@ -9,7 +9,7 @@ import type {Effect as $$effect} from './Types.ts';
 
 import type {Logger as $$logger} from './Types.ts';
 
-import type {S_t as RescriptSchema_S_t} from './RescriptSchema.gen';
+import type {S_t as RescriptSchema_S_t} from 'rescript-schema/RescriptSchema.gen';
 
 export type logger = $$logger;
 

package/src/Prometheus.res

@@ -22,12 +22,6 @@ let eventsProcessedCounter = PromClient.Gauge.makeGauge({
   "labelNames": ["chainId"],
 })
 
-let reorgsDetectedCounter = PromClient.Counter.makeCounter({
-  "name": "reorgs_detected",
-  "help": "Total number of reorgs detected",
-  "labelNames": ["chainId"],
-})
-
 let allChainsSyncedToHead = PromClient.Gauge.makeGauge({
   "name": "hyperindex_synced_to_head",
   "help": "All chains fully synced",
@@ -88,6 +82,7 @@ module MakeSafePromMetric = (
   let makeOrThrow: (~name: string, ~help: string, ~labelSchema: S.t<'a>) => t<'a>
   let handleInt: (t<'a>, ~labels: 'a, ~value: int) => unit
   let handleFloat: (t<'a>, ~labels: 'a, ~value: float) => unit
+  let increment: (t<'a>, ~labels: 'a) => unit
 } => {
   type t<'a> = {metric: M.t, labelSchema: S.t<'a>}
 
@@ -119,6 +114,13 @@ module MakeSafePromMetric = (
     metric
     ->M.labels(labels->S.reverseConvertToJsonOrThrow(labelSchema))
     ->M.handleInt(value)
+
+  let increment = ({metric, labelSchema}: t<'a>, ~labels: 'a) =>
+    (
+      metric
+      ->M.labels(labels->S.reverseConvertToJsonOrThrow(labelSchema))
+      ->Obj.magic
+    )["inc"]()
 }
 
 module SafeCounter = MakeSafePromMetric({
@@ -137,6 +139,21 @@ module SafeGauge = MakeSafePromMetric({
   let handleFloat = PromClient.Gauge.setFloat
 })
 
+let makeSafeHistogramOrThrow = (~name, ~help, ~labelSchema, ~backets=?) => {
+  let histogram = PromClient.Histogram.make({
+    "name": name,
+    "help": help,
+    "labelNames": labelSchema->Labels.getLabelNames->Belt.Result.getExn,
+    "buckets": backets,
+  })
+
+  labels => {
+    histogram
+    ->PromClient.Histogram.labels(labels->S.reverseConvertToJsonOrThrow(labelSchema))
+    ->PromClient.Histogram.startTimer
+  }
+}
+
 module BenchmarkSummaryData = {
   type labels = {
     group: string,
@@ -188,12 +205,6 @@ let processedUntilHeight = PromClient.Gauge.makeGauge({
   "labelNames": ["chainId"],
 })
 
-let fetchedUntilHeight = PromClient.Gauge.makeGauge({
-  "name": "chain_block_height_fully_fetched",
-  "help": "Block height fully fetched by indexer",
-  "labelNames": ["chainId"],
-})
-
 let incrementLoadEntityDurationCounter = (~duration) => {
   loadEntitiesDurationCounter->PromClient.Counter.incMany(duration)
 }
@@ -212,10 +223,6 @@ let setEventsProcessedGuage = (~number, ~chainId) => {
   ->PromClient.Gauge.set(number)
 }
 
-let incrementReorgsDetected = (~chain) => {
-  reorgsDetectedCounter->PromClient.Counter.incLabels({"chainId": chain->ChainMap.Chain.toString})
-}
-
 let setSourceChainHeight = (~blockNumber, ~chain) => {
   sourceChainHeight
   ->PromClient.Gauge.labels({"chainId": chain->ChainMap.Chain.toString})
@@ -232,12 +239,6 @@ let setProcessedUntilHeight = (~blockNumber, ~chain) => {
   ->PromClient.Gauge.set(blockNumber)
 }
 
-let setFetchedUntilHeight = (~blockNumber, ~chain) => {
-  fetchedUntilHeight
-  ->PromClient.Gauge.labels({"chainId": chain->ChainMap.Chain.toString})
-  ->PromClient.Gauge.set(blockNumber)
-}
-
 module BenchmarkCounters = {
   type labels = {label: string}
   let labelSchema = S.schema(s => {
@@ -279,6 +280,34 @@ let chainIdLabelsSchema = S.object(s => {
   s.field("chainId", S.string->S.coerce(S.int))
 })
 
+module Info = {
+  let gauge = SafeGauge.makeOrThrow(
+    ~name="envio_info",
+    ~help="Information about the indexer",
+    ~labelSchema=S.schema(s =>
+      {
+        "version": s.matches(S.string),
+      }
+    ),
+  )
+
+  let set = (~version) => {
+    gauge->SafeGauge.handleInt(~labels={"version": version}, ~value=1)
+  }
+}
+
+module ProgressBlockNumber = {
+  let gauge = SafeGauge.makeOrThrow(
+    ~name="envio_progress_block_number",
+    ~help="The block number to track the progress of indexing at. Currently uses the fully fetched block number. In the future will be changed to block number processed and stored in the database.",
+    ~labelSchema=chainIdLabelsSchema,
+  )
+
+  let set = (~blockNumber, ~chainId) => {
+    gauge->SafeGauge.handleInt(~labels=chainId, ~value=blockNumber)
+  }
+}
+
 module IndexingAddresses = {
   let gauge = SafeGauge.makeOrThrow(
     ~name="envio_indexing_addresses",
@@ -291,6 +320,77 @@ module IndexingAddresses = {
   }
 }
 
+module IndexingMaxConcurrency = {
+  let gauge = SafeGauge.makeOrThrow(
+    ~name="envio_indexing_max_concurrency",
+    ~help="The maximum number of concurrent queries to the chain data-source.",
+    ~labelSchema=chainIdLabelsSchema,
+  )
+
+  let set = (~maxConcurrency, ~chainId) => {
+    gauge->SafeGauge.handleInt(~labels=chainId, ~value=maxConcurrency)
+  }
+}
+
+module IndexingConcurrency = {
+  let gauge = SafeGauge.makeOrThrow(
+    ~name="envio_indexing_concurrency",
+    ~help="The current number of concurrent queries to the chain data-source.",
+    ~labelSchema=chainIdLabelsSchema,
+  )
+
+  let set = (~concurrency, ~chainId) => {
+    gauge->SafeGauge.handleInt(~labels=chainId, ~value=concurrency)
+  }
+}
+
+module IndexingBufferSize = {
+  let gauge = SafeGauge.makeOrThrow(
+    ~name="envio_indexing_buffer_size",
+    ~help="The current number of items in the indexing buffer.",
+    ~labelSchema=chainIdLabelsSchema,
+  )
+
+  let set = (~bufferSize, ~chainId) => {
+    gauge->SafeGauge.handleInt(~labels=chainId, ~value=bufferSize)
+  }
+}
+
+module IndexingMaxBufferSize = {
+  let gauge = SafeGauge.makeOrThrow(
+    ~name="envio_indexing_max_buffer_size",
+    ~help="The maximum number of items allowed in the indexing buffer for the chain.",
+    ~labelSchema=chainIdLabelsSchema,
+  )
+
+  let set = (~maxBufferSize, ~chainId) => {
+    gauge->SafeGauge.handleInt(~labels=chainId, ~value=maxBufferSize)
+  }
+}
+
+module IndexingBufferBlockNumber = {
+  let deprecatedGauge = PromClient.Gauge.makeGauge({
+    "name": "chain_block_height_fully_fetched",
+    "help": "Block height fully fetched by indexer",
+    "labelNames": ["chainId"],
+  })
+
+  let gauge = SafeGauge.makeOrThrow(
+    ~name="envio_indexing_buffer_block_number",
+    ~help="The highest block number that has been fully fetched by the indexer.",
+    ~labelSchema=chainIdLabelsSchema,
+  )
+
+  let set = (~blockNumber, ~chainId) => {
+    deprecatedGauge
+    ->PromClient.Gauge.labels({"chainId": chainId})
+    ->PromClient.Gauge.set(blockNumber)
+    // TODO: Use the block number stored in the database instead
+    ProgressBlockNumber.set(~blockNumber, ~chainId)
+    gauge->SafeGauge.handleInt(~labels=chainId, ~value=blockNumber)
+  }
+}
+
 module IndexingEndBlock = {
   let gauge = SafeGauge.makeOrThrow(
     ~name="envio_indexing_end_block",
@@ -303,14 +403,101 @@ module IndexingEndBlock = {
   }
 }
 
-module ProgressBlockNumber = {
+let sourceLabelsSchema = S.schema(s =>
+  {
+    "source": s.matches(S.string),
+    "chainId": s.matches(S.string->S.coerce(S.int)),
+  }
+)
+
+module SourceHeight = {
   let gauge = SafeGauge.makeOrThrow(
-    ~name="envio_progress_block_number",
-    ~help="The block number to track the progress of indexing at. Currently uses the fully fetched block number. In the future will be changed to block number processed and stored in the database.",
+    ~name="envio_source_height",
+    ~help="The latest known block number reported by the source. This value may lag behind the actual chain height, as it is updated only when queried.",
+    ~labelSchema=sourceLabelsSchema,
+  )
+
+  let set = (~sourceName, ~chainId, ~blockNumber) => {
+    gauge->SafeGauge.handleInt(
+      ~labels={"source": sourceName, "chainId": chainId},
+      ~value=blockNumber,
+    )
+  }
+}
+
+module SourceGetHeightDuration = {
+  let startTimer = makeSafeHistogramOrThrow(
+    ~name="envio_source_get_height_duration",
+    ~help="Duration of the source get height requests in seconds",
+    ~labelSchema=sourceLabelsSchema,
+    ~backets=[0.1, 0.5, 1., 10.],
+  )
+}
+
+module ReorgCount = {
+  let deprecatedCounter = PromClient.Counter.makeCounter({
+    "name": "reorgs_detected",
+    "help": "Total number of reorgs detected",
+    "labelNames": ["chainId"],
+  })
+
+  let counter = SafeGauge.makeOrThrow(
+    ~name="envio_reorg_count",
+    ~help="Total number of reorgs detected",
     ~labelSchema=chainIdLabelsSchema,
   )
 
-  let set = (~endBlock, ~chainId) => {
-    gauge->SafeGauge.handleInt(~labels=chainId, ~value=endBlock)
+  let increment = (~chain) => {
+    deprecatedCounter
+    ->PromClient.Counter.labels({"chainId": chain->ChainMap.Chain.toString})
+    ->PromClient.Counter.inc
+    counter->SafeGauge.increment(~labels=chain->ChainMap.Chain.toChainId)
+  }
+}
+
+module ReorgDetectionBlockNumber = {
+  let gauge = SafeGauge.makeOrThrow(
+    ~name="envio_reorg_detection_block_number",
+    ~help="The block number where reorg was detected the last time. This doesn't mean that the block was reorged, this is simply where we found block hash to be different.",
+    ~labelSchema=chainIdLabelsSchema,
+  )
+
+  let set = (~blockNumber, ~chain) => {
+    gauge->SafeGauge.handleInt(~labels=chain->ChainMap.Chain.toChainId, ~value=blockNumber)
+  }
+}
+
+module RollbackEnabled = {
+  let gauge = PromClient.Gauge.makeGauge({
+    "name": "envio_rollback_enabled",
+    "help": "Whether rollback on reorg is enabled",
+  })
+
+  let set = (~enabled) => {
+    gauge->PromClient.Gauge.set(enabled ? 1 : 0)
+  }
+}
+
+module RollbackDuration = {
+  let histogram = PromClient.Histogram.make({
+    "name": "envio_rollback_duration",
+    "help": "Rollback on reorg duration in seconds",
+    "buckets": [0.5, 1., 5., 10.],
+  })
+
+  let startTimer = () => {
+    histogram->PromClient.Histogram.startTimer
+  }
+}
+
+module RollbackTargetBlockNumber = {
+  let gauge = SafeGauge.makeOrThrow(
+    ~name="envio_rollback_target_block_number",
+    ~help="The block number reorg was rollbacked to the last time.",
+    ~labelSchema=chainIdLabelsSchema,
+  )
+
+  let set = (~blockNumber, ~chain) => {
+    gauge->SafeGauge.handleInt(~labels=chain->ChainMap.Chain.toChainId, ~value=blockNumber)
   }
 }

package/src/bindings/BigDecimal.gen.ts

@@ -5,7 +5,7 @@
 
 const BigDecimalJS = require('./BigDecimal.bs.js');
 
-import type {S_t as RescriptSchema_S_t} from './RescriptSchema.gen';
+import type {S_t as RescriptSchema_S_t} from 'rescript-schema/RescriptSchema.gen';
 
 import type {default as $$t} from 'bignumber.js';
 

package/src/bindings/Hrtime.res

@@ -0,0 +1,41 @@
+type seconds = float
+type milliseconds = float
+type nanoseconds = float
+
+type timeTuple = (seconds, nanoseconds)
+
+type timeRef = timeTuple
+
+type timeElapsed = timeTuple
+
+@val @scope("process") external makeTimer: unit => timeRef = "hrtime"
+
+@val @scope("process") external timeSince: timeRef => timeElapsed = "hrtime"
+
+let nanoToMilli = (nano: nanoseconds): milliseconds => nano /. 1_000_000.
+let secToMilli = (sec: seconds): milliseconds => sec *. 1_000.
+
+let nanoToTimeTuple = (nano: nanoseconds): timeTuple => {
+  let factor = 1_000_000_000.
+  let seconds = Js.Math.floor_float(nano /. factor)
+  let nanos = mod_float(nano, factor)
+  (seconds, nanos)
+}
+
+let timeElapsedToNewRef = (elapsed: timeElapsed, ref: timeRef): timeRef => {
+  let (elapsedSeconds, elapsedNano) = elapsed
+  let (refSeconds, refNano) = ref
+
+  let (nanoExtraSeconds, remainderNanos) = nanoToTimeTuple(elapsedNano +. refNano)
+  (elapsedSeconds +. refSeconds +. nanoExtraSeconds, remainderNanos)
+}
+
+let toMillis = ((sec, nano): timeElapsed): milliseconds => {
+  sec->secToMilli +. nano->nanoToMilli
+}
+
+let toInt = float => float->Belt.Int.fromFloat
+let intFromMillis = toInt
+let intFromNanos = toInt
+let intFromSeconds = toInt
+let floatFromMillis = Utils.magic

package/src/bindings/Hrtime.resi

@@ -0,0 +1,24 @@
+type seconds
+type milliseconds
+type nanoseconds
+
+type timeTuple = (seconds, nanoseconds)
+
+type timeRef
+
+type timeElapsed = timeTuple
+
+let makeTimer: unit => timeRef
+
+let timeSince: timeRef => timeElapsed
+
+let nanoToMilli: nanoseconds => milliseconds
+let secToMilli: seconds => milliseconds
+
+let timeElapsedToNewRef: (timeElapsed, timeRef) => timeRef
+let toMillis: timeElapsed => milliseconds
+
+let intFromMillis: milliseconds => int
+let intFromNanos: nanoseconds => int
+let intFromSeconds: seconds => int
+let floatFromMillis: milliseconds => float

package/src/bindings/PromClient.res

@@ -18,7 +18,6 @@ module Counter = {
   @send external inc: counter => unit = "inc"
   @send external incMany: (counter, int) => unit = "inc"
 
-  @send external incLabels: (counter, 'labelsObject) => unit = "labels"
   @send external labels: (counter, 'labelsObject) => counter = "labels"
 }
 
@@ -41,10 +40,10 @@ module Gauge = {
 
 module Histogram = {
   type histogram
-  @new @module("prom-client") external makeHistogram: customMetric<'a> => histogram = "Histogram"
+  @new @module("prom-client") external make: customMetric<'a> => histogram = "Histogram"
 
   @send external observe: (histogram, float) => unit = "observe"
-  @send external startTimer: (histogram, unit) => float = "startTimer"
+  @send external startTimer: histogram => unit => unit = "startTimer"
 
   @send external labels: (histogram, 'labelsObject) => histogram = "labels"
 }

package/src/db/EntityHistory.res

@@ -24,6 +24,15 @@ type historyRow<'entity> = {
   current: historyFields,
   previous: option<historyFields>,
   entityData: entityData<'entity>,
+  // In the event of a rollback, some entity updates may have been
+  // been affected by a rollback diff. If there was no rollback diff
+  // this will always be false.
+  // If there was a rollback diff, this will be false in the case of a
+  // new entity update (where entity affected is not present in the diff) b
+  // but true if the update is related to an entity that is
+  // currently present in the diff
+  // Optional since it's discarded during parsing/serialization
+  containsRollbackDiffChange?: bool,
 }
 
 type previousHistoryFields = historyFieldsGeneral<option<int>>
@@ -134,6 +143,7 @@ type t<'entity> = {
   table: table,
   createInsertFnQuery: string,
   schema: S.t<historyRow<'entity>>,
+  // Used for parsing
   schemaRows: S.t<array<historyRow<'entity>>>,
   insertFn: (Postgres.sql, Js.Json.t, ~shouldCopyCurrentEntity: bool) => promise<unit>,
 }
@@ -148,19 +158,13 @@ let insertRow = (
   self.insertFn(sql, row, ~shouldCopyCurrentEntity)
 }
 
-let batchInsertRows = (
-  self: t<'entity>,
-  ~sql,
-  ~rows: array<historyRow<'entity>>,
-  ~shouldCopyCurrentEntity,
-) => {
-  let rows =
-    rows
-    ->S.reverseConvertToJsonOrThrow(self.schemaRows)
-    ->(Utils.magic: Js.Json.t => array<Js.Json.t>)
+let batchInsertRows = (self: t<'entity>, ~sql, ~rows: array<historyRow<'entity>>) => {
   rows
-  ->Belt.Array.map(row => {
-    self.insertFn(sql, row, ~shouldCopyCurrentEntity)
+  ->Belt.Array.map(historyRow => {
+    let containsRollbackDiffChange =
+      historyRow.containsRollbackDiffChange->Belt.Option.getWithDefault(false)
+    let shouldCopyCurrentEntity = !containsRollbackDiffChange
+    self->insertRow(~sql, ~historyRow, ~shouldCopyCurrentEntity)
   })
   ->Promise.all
   ->Promise.thenResolve(_ => ())