envio 2.21.5 → 2.22.1

package/src/PgStorage.res.js

@@ -0,0 +1,125 @@
+// Generated by ReScript, PLEASE EDIT WITH CARE
+'use strict';
+
+var $$Array = require("rescript/lib/js/array.js");
+var Table = require("./db/Table.res.js");
+var Utils = require("./Utils.res.js");
+var Schema = require("./db/Schema.res.js");
+var Belt_Array = require("rescript/lib/js/belt_Array.js");
+
+function makeCreateIndexSqlUnsafe(tableName, indexFields, pgSchema) {
+  var indexName = tableName + "_" + indexFields.join("_");
+  var index = Belt_Array.map(indexFields, (function (idx) {
+            return "\"" + idx + "\"";
+          })).join(", ");
+  return "CREATE INDEX IF NOT EXISTS \"" + indexName + "\" ON \"" + pgSchema + "\".\"" + tableName + "\"(" + index + ");";
+}
+
+function makeCreateTableIndicesSqlUnsafe(table, pgSchema) {
+  var tableName = table.tableName;
+  var createIndex = function (indexField) {
+    return makeCreateIndexSqlUnsafe(tableName, [indexField], pgSchema);
+  };
+  var createCompositeIndex = function (indexFields) {
+    return makeCreateIndexSqlUnsafe(tableName, indexFields, pgSchema);
+  };
+  var singleIndices = Table.getSingleIndices(table);
+  var compositeIndices = Table.getCompositeIndices(table);
+  return Belt_Array.map(singleIndices, createIndex).join("\n") + Belt_Array.map(compositeIndices, createCompositeIndex).join("\n");
+}
+
+function makeCreateTableSqlUnsafe(table, pgSchema) {
+  var fieldsMapped = Belt_Array.map(Table.getFields(table), (function (field) {
+            var defaultValue = field.defaultValue;
+            var fieldType = field.fieldType;
+            var fieldName = Table.getDbFieldName(field);
+            var tmp;
+            tmp = fieldType === "TIMESTAMP" || fieldType === "TIMESTAMP WITH TIME ZONE" || fieldType === "JSONB" || fieldType === "SERIAL" || fieldType === "TEXT" || fieldType === "DOUBLE PRECISION" || fieldType === "NUMERIC" || fieldType === "BOOLEAN" || fieldType === "INTEGER" || fieldType === "TIMESTAMP WITH TIME ZONE NULL" || fieldType.startsWith("NUMERIC(") ? fieldType : "\"" + pgSchema + "\"." + fieldType;
+            return "\"" + fieldName + "\" " + tmp + (
+                    field.isArray ? "[]" : ""
+                  ) + (
+                    defaultValue !== undefined ? " DEFAULT " + defaultValue : (
+                        field.isNullable ? "" : " NOT NULL"
+                      )
+                  );
+          })).join(", ");
+  var primaryKeyFieldNames = Table.getPrimaryKeyFieldNames(table);
+  var primaryKey = Belt_Array.map(primaryKeyFieldNames, (function (field) {
+            return "\"" + field + "\"";
+          })).join(", ");
+  return "CREATE TABLE IF NOT EXISTS \"" + pgSchema + "\".\"" + table.tableName + "\"(" + fieldsMapped + (
+          primaryKeyFieldNames.length !== 0 ? ", PRIMARY KEY(" + primaryKey + ")" : ""
+        ) + ");";
+}
+
+function makeInitializeTransaction(pgSchema, staticTables, entities, enums, cleanRun) {
+  var allTables = $$Array.copy(staticTables);
+  var allEntityTables = [];
+  entities.forEach(function (entity) {
+        allEntityTables.push(entity.table);
+        allTables.push(entity.table);
+        allTables.push(entity.entityHistory.table);
+      });
+  var derivedSchema = Schema.make(allEntityTables);
+  var query = {
+    contents: (
+      cleanRun ? "DROP SCHEMA IF EXISTS \"" + pgSchema + "\" CASCADE;\nCREATE SCHEMA \"" + pgSchema + "\";" : "CREATE SCHEMA IF NOT EXISTS \"" + pgSchema + "\";"
+    ) + ("GRANT ALL ON SCHEMA \"" + pgSchema + "\" TO postgres;\nGRANT ALL ON SCHEMA \"" + pgSchema + "\" TO public;")
+  };
+  enums.forEach(function (enumConfig) {
+        var enumCreateQuery = "CREATE TYPE \"" + pgSchema + "\"." + enumConfig.name + " AS ENUM(" + enumConfig.variants.map(function (v) {
+                return "'" + v + "'";
+              }).join(", ") + ");";
+        query.contents = query.contents + "\n" + (
+          cleanRun ? enumCreateQuery : "IF NOT EXISTS (\n  SELECT 1 FROM pg_type \n  WHERE typname = '" + enumConfig.name.toLowerCase() + "' \n  AND typnamespace = (SELECT oid FROM pg_namespace WHERE nspname = '" + pgSchema + "')\n) THEN \n    " + enumCreateQuery + "\nEND IF;"
+        );
+      });
+  allTables.forEach(function (table) {
+        query.contents = query.contents + "\n" + makeCreateTableSqlUnsafe(table, pgSchema);
+      });
+  allTables.forEach(function (table) {
+        var indices = makeCreateTableIndicesSqlUnsafe(table, pgSchema);
+        if (indices !== "") {
+          query.contents = query.contents + "\n" + indices;
+          return ;
+        }
+        
+      });
+  var functionsQuery = {
+    contents: ""
+  };
+  entities.forEach(function (entity) {
+        functionsQuery.contents = functionsQuery.contents + "\n" + entity.entityHistory.createInsertFnQuery;
+        Table.getDerivedFromFields(entity.table).forEach(function (derivedFromField) {
+              var indexField = Utils.unwrapResultExn(Schema.getDerivedFromFieldName(derivedSchema, derivedFromField));
+              query.contents = query.contents + "\n" + makeCreateIndexSqlUnsafe(derivedFromField.derivedFromEntity, [indexField], pgSchema);
+            });
+      });
+  return [cleanRun ? query.contents : "DO $$ BEGIN " + query.contents + " END $$;"].concat(functionsQuery.contents !== "" ? [functionsQuery.contents] : []);
+}
+
+function make(sql, pgSchema) {
+  var isInitialized = async function () {
+    var schemas = await sql.unsafe("SELECT schema_name FROM information_schema.schemata WHERE schema_name = '" + pgSchema + "';");
+    return Utils.$$Array.notEmpty(schemas);
+  };
+  var initialize = async function (entities, staticTables, enums, cleanRun) {
+    var queries = makeInitializeTransaction(pgSchema, staticTables, entities, enums, cleanRun);
+    await sql.begin(function (sql) {
+          return queries.map(function (query) {
+                      return sql.unsafe(query);
+                    });
+        });
+  };
+  return {
+          isInitialized: isInitialized,
+          initialize: initialize
+        };
+}
+
+exports.makeCreateIndexSqlUnsafe = makeCreateIndexSqlUnsafe;
+exports.makeCreateTableIndicesSqlUnsafe = makeCreateTableIndicesSqlUnsafe;
+exports.makeCreateTableSqlUnsafe = makeCreateTableSqlUnsafe;
+exports.makeInitializeTransaction = makeInitializeTransaction;
+exports.make = make;
+/* Table Not a pure module */

package/src/Prometheus.res

@@ -16,12 +16,6 @@ let executeBatchDurationCounter = PromClient.Counter.makeCounter({
   "labelNames": [],
 })
 
-let eventsProcessedCounter = PromClient.Gauge.makeGauge({
-  "name": "events_processed",
-  "help": "Total number of events processed",
-  "labelNames": ["chainId"],
-})
-
 let allChainsSyncedToHead = PromClient.Gauge.makeGauge({
   "name": "hyperindex_synced_to_head",
   "help": "All chains fully synced",
@@ -207,12 +201,6 @@ module BenchmarkSummaryData = {
   }
 }
 
-let processedUntilHeight = PromClient.Gauge.makeGauge({
-  "name": "chain_block_height_processed",
-  "help": "Block height processed by indexer",
-  "labelNames": ["chainId"],
-})
-
 let incrementLoadEntityDurationCounter = (~duration) => {
   loadEntitiesDurationCounter->PromClient.Counter.incMany(duration)
 }
@@ -225,12 +213,6 @@ let incrementExecuteBatchDurationCounter = (~duration) => {
   executeBatchDurationCounter->PromClient.Counter.incMany(duration)
 }
 
-let setEventsProcessedGuage = (~number, ~chainId) => {
-  eventsProcessedCounter
-  ->PromClient.Gauge.labels({"chainId": chainId})
-  ->PromClient.Gauge.set(number)
-}
-
 let setSourceChainHeight = (~blockNumber, ~chain) => {
   sourceChainHeight
   ->PromClient.Gauge.labels({"chainId": chain->ChainMap.Chain.toString})
@@ -241,12 +223,6 @@ let setAllChainsSyncedToHead = () => {
   allChainsSyncedToHead->PromClient.Gauge.set(1)
 }
 
-let setProcessedUntilHeight = (~blockNumber, ~chain) => {
-  processedUntilHeight
-  ->PromClient.Gauge.labels({"chainId": chain->ChainMap.Chain.toString})
-  ->PromClient.Gauge.set(blockNumber)
-}
-
 module BenchmarkCounters = {
   type labels = {label: string}
   let labelSchema = S.schema(s => {
@@ -304,18 +280,6 @@ module Info = {
   }
 }
 
-module ProgressBlockNumber = {
-  let gauge = SafeGauge.makeOrThrow(
-    ~name="envio_progress_block_number",
-    ~help="The block number to track the progress of indexing at. Currently uses the fully fetched block number. In the future will be changed to block number processed and stored in the database.",
-    ~labelSchema=chainIdLabelsSchema,
-  )
-
-  let set = (~blockNumber, ~chainId) => {
-    gauge->SafeGauge.handleInt(~labels=chainId, ~value=blockNumber)
-  }
-}
-
 module IndexingAddresses = {
   let gauge = SafeGauge.makeOrThrow(
     ~name="envio_indexing_addresses",
@@ -428,8 +392,6 @@ module IndexingBufferBlockNumber = {
     deprecatedGauge
     ->PromClient.Gauge.labels({"chainId": chainId})
     ->PromClient.Gauge.set(blockNumber)
-    // TODO: Use the block number stored in the database instead
-    ProgressBlockNumber.set(~blockNumber, ~chainId)
     gauge->SafeGauge.handleInt(~labels=chainId, ~value=blockNumber)
   }
 }
@@ -484,7 +446,7 @@ module ReorgCount = {
     "labelNames": ["chainId"],
   })
 
-  let counter = SafeGauge.makeOrThrow(
+  let gauge = SafeGauge.makeOrThrow(
     ~name="envio_reorg_count",
     ~help="Total number of reorgs detected",
     ~labelSchema=chainIdLabelsSchema,
@@ -494,7 +456,7 @@ module ReorgCount = {
     deprecatedCounter
     ->PromClient.Counter.labels({"chainId": chain->ChainMap.Chain.toString})
     ->PromClient.Counter.inc
-    counter->SafeGauge.increment(~labels=chain->ChainMap.Chain.toChainId)
+    gauge->SafeGauge.increment(~labels=chain->ChainMap.Chain.toChainId)
   }
 }
 
@@ -545,6 +507,30 @@ module RollbackTargetBlockNumber = {
   }
 }
 
+module ProcessingBlockNumber = {
+  let gauge = SafeGauge.makeOrThrow(
+    ~name="envio_processing_block_number",
+    ~help="The latest item block number included in the currently processing batch for the chain.",
+    ~labelSchema=chainIdLabelsSchema,
+  )
+
+  let set = (~blockNumber, ~chainId) => {
+    gauge->SafeGauge.handleInt(~labels=chainId, ~value=blockNumber)
+  }
+}
+
+module ProcessingBatchSize = {
+  let gauge = SafeGauge.makeOrThrow(
+    ~name="envio_processing_batch_size",
+    ~help="The number of items included in the currently processing batch for the chain.",
+    ~labelSchema=chainIdLabelsSchema,
+  )
+
+  let set = (~batchSize, ~chainId) => {
+    gauge->SafeGauge.handleInt(~labels=chainId, ~value=batchSize)
+  }
+}
+
 module ProcessingMaxBatchSize = {
   let gauge = PromClient.Gauge.makeGauge({
     "name": "envio_processing_max_batch_size",
@@ -555,3 +541,52 @@ module ProcessingMaxBatchSize = {
     gauge->PromClient.Gauge.set(maxBatchSize)
   }
 }
+
+module ProgressBlockNumber = {
+  let gauge = SafeGauge.makeOrThrow(
+    ~name="envio_progress_block_number",
+    ~help="The block number of the latest block processed and stored in the database.",
+    ~labelSchema=chainIdLabelsSchema,
+  )
+
+  let set = (~blockNumber, ~chainId) => {
+    gauge->SafeGauge.handleInt(~labels=chainId, ~value=blockNumber)
+  }
+}
+
+module ProgressEventsCount = {
+  let deprecatedGauge = PromClient.Gauge.makeGauge({
+    "name": "events_processed",
+    "help": "Total number of events processed",
+    "labelNames": ["chainId"],
+  })
+
+  let gauge = SafeGauge.makeOrThrow(
+    ~name="envio_progress_events_count",
+    ~help="The number of events processed and reflected in the database.",
+    ~labelSchema=chainIdLabelsSchema,
+  )
+
+  let set = (~processedCount, ~chainId) => {
+    deprecatedGauge
+    ->PromClient.Gauge.labels({"chainId": chainId})
+    ->PromClient.Gauge.set(processedCount)
+    gauge->SafeGauge.handleInt(~labels=chainId, ~value=processedCount)
+  }
+}
+
+let effectLabelsSchema = S.object(s => {
+  s.field("effect", S.string)
+})
+
+module EffectCallsCount = {
+  let gauge = SafeGauge.makeOrThrow(
+    ~name="envio_effect_calls_count",
+    ~help="The number of calls to the effect. Including both handler execution and cache hits.",
+    ~labelSchema=effectLabelsSchema,
+  )
+
+  let set = (~callsCount, ~effectName) => {
+    gauge->SafeGauge.handleInt(~labels=effectName, ~value=callsCount)
+  }
+}