envio 2.17.1 → 2.18.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "envio",
-  "version": "v2.17.1",
+  "version": "v2.18.0",
   "description": "A latency and sync speed optimized, developer friendly blockchain data indexer.",
   "bin": "./bin.js",
   "types": "index.d.ts",
@@ -24,15 +24,15 @@
   },
   "homepage": "https://envio.dev",
   "optionalDependencies": {
-    "envio-linux-x64": "v2.17.1",
-    "envio-linux-arm64": "v2.17.1",
-    "envio-darwin-x64": "v2.17.1",
-    "envio-darwin-arm64": "v2.17.1"
+    "envio-linux-x64": "v2.18.0",
+    "envio-linux-arm64": "v2.18.0",
+    "envio-darwin-x64": "v2.18.0",
+    "envio-darwin-arm64": "v2.18.0"
   },
   "dependencies": {
     "@envio-dev/hypersync-client": "0.6.3",
     "rescript": "11.1.3",
-    "rescript-schema": "9.1.0",
+    "rescript-schema": "9.3.0",
     "viem": "2.21.0"
   },
   "files": [

package/src/Prometheus.res

@@ -0,0 +1,316 @@
+let loadEntitiesDurationCounter = PromClient.Counter.makeCounter({
+  "name": "load_entities_processing_time_spent",
+  "help": "Duration spend on loading entities",
+  "labelNames": [],
+})
+
+let eventRouterDurationCounter = PromClient.Counter.makeCounter({
+  "name": "event_router_processing_time_spent",
+  "help": "Duration spend on event routing",
+  "labelNames": [],
+})
+
+let executeBatchDurationCounter = PromClient.Counter.makeCounter({
+  "name": "execute_batch_processing_time_spent",
+  "help": "Duration spend on executing batch",
+  "labelNames": [],
+})
+
+let eventsProcessedCounter = PromClient.Gauge.makeGauge({
+  "name": "events_processed",
+  "help": "Total number of events processed",
+  "labelNames": ["chainId"],
+})
+
+let reorgsDetectedCounter = PromClient.Counter.makeCounter({
+  "name": "reorgs_detected",
+  "help": "Total number of reorgs detected",
+  "labelNames": ["chainId"],
+})
+
+let allChainsSyncedToHead = PromClient.Gauge.makeGauge({
+  "name": "hyperindex_synced_to_head",
+  "help": "All chains fully synced",
+  "labelNames": [],
+})
+
+let sourceChainHeight = PromClient.Gauge.makeGauge({
+  "name": "chain_block_height",
+  "help": "Chain Height of Source Chain",
+  "labelNames": ["chainId"],
+})
+
+module Labels = {
+  let rec schemaIsString = (schema: S.t<'a>) =>
+    switch schema->S.classify {
+    | String => true
+    | Null(s)
+    | Option(s) =>
+      s->schemaIsString
+    | _ => false
+    }
+
+  let getLabelNames = (schema: S.t<'a>) =>
+    switch schema->S.classify {
+    | Object({items}) =>
+      let nonStringFields = items->Belt.Array.reduce([], (nonStringFields, item) => {
+        if item.schema->schemaIsString {
+          nonStringFields
+        } else {
+          nonStringFields->Belt.Array.concat([item.location])
+        }
+      })
+
+      switch nonStringFields {
+      | [] => items->Belt.Array.map(item => item.location)->Ok
+      | nonStringItems =>
+        let nonStringItems = nonStringItems->Js.Array2.joinWith(", ")
+        Error(
+          `Label schema must be an object with string (or optional string) values. Non string values: ${nonStringItems}`,
+        )
+      }
+    | _ => Error("Label schema must be an object")
+    }
+}
+
+let metricNames: Utils.Set.t<string> = Utils.Set.make()
+
+module MakeSafePromMetric = (
+  M: {
+    type t
+    let make: {"name": string, "help": string, "labelNames": array<string>} => t
+    let labels: (t, 'a) => t
+    let handleFloat: (t, float) => unit
+    let handleInt: (t, int) => unit
+  },
+): {
+  type t<'a>
+  let makeOrThrow: (~name: string, ~help: string, ~labelSchema: S.t<'a>) => t<'a>
+  let handleInt: (t<'a>, ~labels: 'a, ~value: int) => unit
+  let handleFloat: (t<'a>, ~labels: 'a, ~value: float) => unit
+} => {
+  type t<'a> = {metric: M.t, labelSchema: S.t<'a>}
+
+  let makeOrThrow = (~name, ~help, ~labelSchema: S.t<'a>): t<'a> =>
+    switch labelSchema->Labels.getLabelNames {
+    | Ok(labelNames) =>
+      if metricNames->Utils.Set.has(name) {
+        Js.Exn.raiseError("Duplicate prometheus metric name: " ++ name)
+      } else {
+        metricNames->Utils.Set.add(name)->ignore
+        let metric = M.make({
+          "name": name,
+          "help": help,
+          "labelNames": labelNames,
+        })
+
+        {metric, labelSchema}
+      }
+
+    | Error(error) => Js.Exn.raiseError(error)
+    }
+
+  let handleFloat = ({metric, labelSchema}: t<'a>, ~labels: 'a, ~value) =>
+    metric
+    ->M.labels(labels->S.reverseConvertToJsonOrThrow(labelSchema))
+    ->M.handleFloat(value)
+
+  let handleInt = ({metric, labelSchema}: t<'a>, ~labels: 'a, ~value) =>
+    metric
+    ->M.labels(labels->S.reverseConvertToJsonOrThrow(labelSchema))
+    ->M.handleInt(value)
+}
+
+module SafeCounter = MakeSafePromMetric({
+  type t = PromClient.Counter.counter
+  let make = PromClient.Counter.makeCounter
+  let labels = PromClient.Counter.labels
+  let handleInt = PromClient.Counter.incMany
+  let handleFloat = PromClient.Counter.incMany->Utils.magic
+})
+
+module SafeGauge = MakeSafePromMetric({
+  type t = PromClient.Gauge.gauge
+  let make = PromClient.Gauge.makeGauge
+  let labels = PromClient.Gauge.labels
+  let handleInt = PromClient.Gauge.set
+  let handleFloat = PromClient.Gauge.setFloat
+})
+
+module BenchmarkSummaryData = {
+  type labels = {
+    group: string,
+    stat: string,
+    label: string,
+  }
+  let labelSchema = S.schema(s => {
+    group: s.matches(S.string),
+    stat: s.matches(S.string),
+    label: s.matches(S.string),
+  })
+
+  let gauge = SafeGauge.makeOrThrow(
+    ~name="benchmark_summary_data",
+    ~help="All data points collected during indexer benchmark",
+    ~labelSchema,
+  )
+
+  let set = (
+    ~group: string,
+    ~label: string,
+    ~n: float,
+    ~mean: float,
+    ~stdDev: option<float>,
+    ~min: float,
+    ~max: float,
+    ~sum: float,
+  ) => {
+    let mk = stat => {
+      group,
+      stat,
+      label,
+    }
+    gauge->SafeGauge.handleFloat(~labels=mk("n"), ~value=n)
+    gauge->SafeGauge.handleFloat(~labels=mk("mean"), ~value=mean)
+    gauge->SafeGauge.handleFloat(~labels=mk("min"), ~value=min)
+    gauge->SafeGauge.handleFloat(~labels=mk("max"), ~value=max)
+    gauge->SafeGauge.handleFloat(~labels=mk("sum"), ~value=sum)
+    switch stdDev {
+    | Some(stdDev) => gauge->SafeGauge.handleFloat(~labels=mk("stdDev"), ~value=stdDev)
+    | None => ()
+    }
+  }
+}
+
+let processedUntilHeight = PromClient.Gauge.makeGauge({
+  "name": "chain_block_height_processed",
+  "help": "Block height processed by indexer",
+  "labelNames": ["chainId"],
+})
+
+let fetchedUntilHeight = PromClient.Gauge.makeGauge({
+  "name": "chain_block_height_fully_fetched",
+  "help": "Block height fully fetched by indexer",
+  "labelNames": ["chainId"],
+})
+
+let incrementLoadEntityDurationCounter = (~duration) => {
+  loadEntitiesDurationCounter->PromClient.Counter.incMany(duration)
+}
+
+let incrementEventRouterDurationCounter = (~duration) => {
+  eventRouterDurationCounter->PromClient.Counter.incMany(duration)
+}
+
+let incrementExecuteBatchDurationCounter = (~duration) => {
+  executeBatchDurationCounter->PromClient.Counter.incMany(duration)
+}
+
+let setEventsProcessedGuage = (~number, ~chainId) => {
+  eventsProcessedCounter
+  ->PromClient.Gauge.labels({"chainId": chainId})
+  ->PromClient.Gauge.set(number)
+}
+
+let incrementReorgsDetected = (~chain) => {
+  reorgsDetectedCounter->PromClient.Counter.incLabels({"chainId": chain->ChainMap.Chain.toString})
+}
+
+let setSourceChainHeight = (~blockNumber, ~chain) => {
+  sourceChainHeight
+  ->PromClient.Gauge.labels({"chainId": chain->ChainMap.Chain.toString})
+  ->PromClient.Gauge.set(blockNumber)
+}
+
+let setAllChainsSyncedToHead = () => {
+  allChainsSyncedToHead->PromClient.Gauge.set(1)
+}
+
+let setProcessedUntilHeight = (~blockNumber, ~chain) => {
+  processedUntilHeight
+  ->PromClient.Gauge.labels({"chainId": chain->ChainMap.Chain.toString})
+  ->PromClient.Gauge.set(blockNumber)
+}
+
+let setFetchedUntilHeight = (~blockNumber, ~chain) => {
+  fetchedUntilHeight
+  ->PromClient.Gauge.labels({"chainId": chain->ChainMap.Chain.toString})
+  ->PromClient.Gauge.set(blockNumber)
+}
+
+module BenchmarkCounters = {
+  type labels = {label: string}
+  let labelSchema = S.schema(s => {
+    label: s.matches(S.string),
+  })
+
+  let gauge = SafeGauge.makeOrThrow(
+    ~name="benchmark_counters",
+    ~help="All counters collected during indexer benchmark",
+    ~labelSchema,
+  )
+
+  let set = (~label, ~millis, ~totalRuntimeMillis) => {
+    gauge->SafeGauge.handleFloat(~labels={label: label}, ~value=millis)
+    gauge->SafeGauge.handleFloat(~labels={label: "Total Run Time (ms)"}, ~value=totalRuntimeMillis)
+  }
+}
+
+module PartitionBlockFetched = {
+  type labels = {chainId: int, partitionId: string}
+
+  let labelSchema = S.schema(s => {
+    chainId: s.matches(S.string->S.coerce(S.int)),
+    partitionId: s.matches(S.string),
+  })
+
+  let counter = SafeGauge.makeOrThrow(
+    ~name="partition_block_fetched",
+    ~help="The latest fetched block number for each partition",
+    ~labelSchema,
+  )
+
+  let set = (~blockNumber, ~partitionId, ~chainId) => {
+    counter->SafeGauge.handleInt(~labels={chainId, partitionId}, ~value=blockNumber)
+  }
+}
+
+let chainIdLabelsSchema = S.object(s => {
+  s.field("chainId", S.string->S.coerce(S.int))
+})
+
+module IndexingAddresses = {
+  let gauge = SafeGauge.makeOrThrow(
+    ~name="envio_indexing_addresses",
+    ~help="The number of addresses indexed on chain. Includes both static and dynamic addresses.",
+    ~labelSchema=chainIdLabelsSchema,
+  )
+
+  let set = (~addressesCount, ~chainId) => {
+    gauge->SafeGauge.handleInt(~labels=chainId, ~value=addressesCount)
+  }
+}
+
+module IndexingEndBlock = {
+  let gauge = SafeGauge.makeOrThrow(
+    ~name="envio_indexing_end_block",
+    ~help="The block number to stop indexing at. (inclusive)",
+    ~labelSchema=chainIdLabelsSchema,
+  )
+
+  let set = (~endBlock, ~chainId) => {
+    gauge->SafeGauge.handleInt(~labels=chainId, ~value=endBlock)
+  }
+}
+
+module ProgressBlockNumber = {
+  let gauge = SafeGauge.makeOrThrow(
+    ~name="envio_progress_block_number",
+    ~help="The block number to track the progress of indexing at. Currently uses the fully fetched block number. In the future will be changed to block number processed and stored in the database.",
+    ~labelSchema=chainIdLabelsSchema,
+  )
+
+  let set = (~endBlock, ~chainId) => {
+    gauge->SafeGauge.handleInt(~labels=chainId, ~value=endBlock)
+  }
+}

package/src/bindings/BigDecimal.gen.ts

@@ -0,0 +1,8 @@
+/* TypeScript file generated from BigDecimal.res by genType. */
+
+/* eslint-disable */
+/* tslint:disable */
+
+import type {default as $$t} from 'bignumber.js';
+
+export type t = $$t;

package/src/bindings/BigDecimal.res

@@ -0,0 +1,60 @@
+@genType.import(("bignumber.js", "default"))
+type rec t = {
+  toString: unit => string,
+  toFixed: int => string,
+  plus: t => t,
+  minus: t => t,
+  times: t => t,
+  div: t => t,
+  isEqualTo: t => bool,
+  gt: t => bool,
+  gte: t => bool,
+  lt: t => bool,
+  lte: t => bool,
+}
+
+// Constructors
+@new @module external fromBigInt: bigint => t = "bignumber.js"
+@new @module external fromFloat: float => t = "bignumber.js"
+@new @module external fromInt: int => t = "bignumber.js"
+@new @module external fromStringUnsafe: string => t = "bignumber.js"
+@new @module external fromString: string => option<t> = "bignumber.js"
+
+// Methods
+@send external toString: t => string = "toString"
+@send external toFixed: t => string = "toFixed"
+let toInt = (b: t): option<int> => b->toString->Belt.Int.fromString
+@send external toNumber: t => float = "toNumber"
+
+// Arithmetic Operations
+@send external plus: (t, t) => t = "plus"
+@send external minus: (t, t) => t = "minus"
+@send external times: (t, t) => t = "multipliedBy"
+@send external div: (t, t) => t = "dividedBy"
+@send external sqrt: t => t = "sqrt"
+
+// Comparison
+@send external equals: (t, t) => bool = "isEqualTo"
+let notEquals: (t, t) => bool = (a, b) => !equals(a, b)
+@send external gt: (t, t) => bool = "isGreaterThan"
+@send external gte: (t, t) => bool = "isGreaterThanOrEqualTo"
+@send external lt: (t, t) => bool = "isLessThan"
+@send external lte: (t, t) => bool = "isLessThanOrEqualTo"
+
+// Utilities
+let zero = fromInt(0)
+let one = fromInt(1)
+@send external decimalPlaces: (t, int) => t = "decimalPlaces"
+
+// Serialization
+let schema =
+  S.string
+  ->S.setName("BigDecimal")
+  ->S.transform(s => {
+    parser: string =>
+      switch string->fromString {
+      | Some(bigDecimal) => bigDecimal
+      | None => s.fail("The string is not valid BigDecimal")
+      },
+    serializer: bigDecimal => bigDecimal.toString(),
+  })

package/src/bindings/PromClient.res

@@ -0,0 +1,58 @@
+/** All metric types have two mandatory parameters: name and help. Refer to https://prometheus.io/docs/practices/naming/ for guidance on naming metrics. */
+type customMetric<'a> = {.."name": string, "help": string} as 'a
+
+@module("prom-client") external collectDefaultMetrics: 'a => unit = "collectDefaultMetrics"
+
+type registry
+@new @module("prom-client") external makeRegistry: unit => registry = "Registry"
+
+@module("prom-client") external defaultRegister: registry = "register"
+
+@send external metrics: registry => Promise.t<string> = "metrics"
+@get external getContentType: registry => string = "contentType"
+
+module Counter = {
+  type counter
+  @new @module("prom-client") external makeCounter: customMetric<'a> => counter = "Counter"
+
+  @send external inc: counter => unit = "inc"
+  @send external incMany: (counter, int) => unit = "inc"
+
+  @send external incLabels: (counter, 'labelsObject) => unit = "labels"
+  @send external labels: (counter, 'labelsObject) => counter = "labels"
+}
+
+module Gauge = {
+  type gauge
+  @new @module("prom-client") external makeGauge: customMetric<'a> => gauge = "Gauge"
+
+  @send external inc: gauge => unit = "inc"
+  @send external incMany: (gauge, int) => unit = "inc"
+
+  @send external dec: gauge => unit = "dec"
+  @send external decMany: (gauge, int) => unit = "dec"
+
+  @send external set: (gauge, int) => unit = "set"
+
+  @send external setFloat: (gauge, float) => unit = "set"
+
+  @send external labels: (gauge, 'labelsObject) => gauge = "labels"
+}
+
+module Histogram = {
+  type histogram
+  @new @module("prom-client") external makeHistogram: customMetric<'a> => histogram = "Histogram"
+
+  @send external observe: (histogram, float) => unit = "observe"
+  @send external startTimer: (histogram, unit) => float = "startTimer"
+
+  @send external labels: (histogram, 'labelsObject) => histogram = "labels"
+}
+
+module Summary = {
+  type summary
+  @new @module("prom-client") external makeSummary: customMetric<'a> => summary = "Summary"
+
+  @send external observe: (summary, float) => unit = "observe"
+  @send external startTimer: (summary, unit) => float = "startTimer"
+}

package/src/db/Table.res

@@ -214,7 +214,10 @@ let toSqlParams = (table: table, ~schema) => {
             }
           | Bool =>
             // Workaround for https://github.com/porsager/postgres/issues/471
-            S.union([S.literal(1)->S.to(_ => true), S.literal(0)->S.to(_ => false)])->S.toUnknown
+            S.union([
+              S.literal(1)->S.shape(_ => true),
+              S.literal(0)->S.shape(_ => false),
+            ])->S.toUnknown
           | _ => schema
           }