envio 2.21.5 → 2.22.0

package/index.js

@@ -2,7 +2,7 @@
 // Some parts like Sury reexport are impossible to implement
 // on the JS side, so we need to do it here
 
-const envioGen = require("./src/Envio.bs.js");
+const envioGen = require("./src/Envio.res.js");
 Object.assign(exports, envioGen);
 
 const Sury = require("rescript-schema");
@@ -30,7 +30,7 @@ exports.S = {
   merge: Sury.merge,
   optional: Sury.optional,
   nullable: Sury.nullable,
-  bigDecimal: require("./src/bindings/BigDecimal.bs.js").schema,
+  bigDecimal: require("./src/bindings/BigDecimal.res.js").schema,
   // Nullish type will change in "sury@10"
   // nullish: Sury.nullish,
   assertOrThrow: Sury.assertOrThrow,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "envio",
-  "version": "v2.21.5",
+  "version": "v2.22.0",
   "description": "A latency and sync speed optimized, developer friendly blockchain data indexer.",
   "bin": "./bin.js",
   "main": "./index.js",
@@ -25,16 +25,19 @@
   },
   "homepage": "https://envio.dev",
   "optionalDependencies": {
-    "envio-linux-x64": "v2.21.5",
-    "envio-linux-arm64": "v2.21.5",
-    "envio-darwin-x64": "v2.21.5",
-    "envio-darwin-arm64": "v2.21.5"
+    "envio-linux-x64": "v2.22.0",
+    "envio-linux-arm64": "v2.22.0",
+    "envio-darwin-x64": "v2.22.0",
+    "envio-darwin-arm64": "v2.22.0"
   },
   "dependencies": {
     "@envio-dev/hypersync-client": "0.6.5",
     "rescript": "11.1.3",
     "rescript-schema": "9.3.0",
-    "viem": "2.21.0"
+    "viem": "2.21.0",
+    "bignumber.js": "9.1.2",
+    "pino": "8.16.1",
+    "pino-pretty": "10.2.3"
   },
   "files": [
     "bin.js",
@@ -43,7 +46,6 @@
     "rescript.json",
     "index.d.ts",
     "index.js",
-    "src",
-    "!src/**/*.bs.js"
+    "src"
   ]
 }

package/rescript.json

@@ -7,7 +7,7 @@
       "subdirs": true
     }
   ],
-  "suffix": ".bs.js",
+  "suffix": ".res.js",
   "package-specs": {
     "module": "commonjs",
     "in-source": true

package/src/Envio.res

@@ -39,6 +39,7 @@ let experimental_createEffect = (
   options: effectOptions<'input, 'output>,
   handler: effectArgs<'input> => promise<'output>,
 ) => {
+  Prometheus.EffectCallsCount.set(~callsCount=0, ~effectName=options.name)
   {
     name: options.name,
     handler: handler->(
@@ -46,5 +47,6 @@ let experimental_createEffect = (
         Internal.effectOutput,
       >
     ),
+    callsCount: 0,
   }->(Utils.magic: Internal.effect => effect<'input, 'output>)
 }

package/src/ErrorHandling.res

@@ -33,29 +33,3 @@ let logAndRaise = self => {
   self->log
   self->raiseExn
 }
-
-/**
-An environment to manage control flow propogating results 
-with Error that contain ErrorHandling.t in async
-contexts and avoid nested switch statements on awaited promises
-Similar to rust result propogation
-*/
-module ResultPropogateEnv = {
-  exception ErrorHandlingEarlyReturn(t)
-
-  type resultWithErrorHandle<'a> = result<'a, t>
-  type asyncBody<'a> = unit => promise<resultWithErrorHandle<'a>>
-
-  let runAsyncEnv = async (body: asyncBody<'a>) => {
-    switch await body() {
-    | exception ErrorHandlingEarlyReturn(e) => Error(e)
-    | endReturn => endReturn
-    }
-  }
-
-  let propogate = (res: resultWithErrorHandle<'a>) =>
-    switch res {
-    | Ok(v) => v
-    | Error(e) => raise(ErrorHandlingEarlyReturn(e))
-    }
-}

package/src/Hasura.res

@@ -0,0 +1,297 @@
+type auth = {
+  role: string,
+  secret: string,
+}
+
+type validHasuraResponse = QuerySucceeded | AlreadyDone
+
+let auth = (s: Rest.s) => {
+  role: s.header("X-Hasura-Role", S.string),
+  secret: s.header("X-Hasura-Admin-Secret", S.string),
+}
+
+let responses = [
+  (s: Rest.Response.s) => {
+    s.status(200)
+    let _ = s.data(S.unknown)
+    QuerySucceeded
+  },
+  s => {
+    let _ = s.field("code", S.enum(["already-exists", "already-tracked"]))
+    AlreadyDone
+  },
+]
+
+let clearMetadataRoute = Rest.route(() => {
+  method: Post,
+  path: "",
+  input: s => {
+    let _ = s.field("type", S.literal("clear_metadata"))
+    let _ = s.field("args", S.literal(Js.Obj.empty()))
+    s->auth
+  },
+  responses,
+})
+
+let trackTablesRoute = Rest.route(() => {
+  method: Post,
+  path: "",
+  input: s => {
+    let _ = s.field("type", S.literal("pg_track_tables"))
+    {
+      "args": s.field("args", S.json(~validate=false)),
+      "auth": s->auth,
+    }
+  },
+  responses,
+})
+
+let createSelectPermissionRoute = Rest.route(() => {
+  method: Post,
+  path: "",
+  input: s => {
+    let _ = s.field("type", S.literal("pg_create_select_permission"))
+    {
+      "args": s.field("args", S.json(~validate=false)),
+      "auth": s->auth,
+    }
+  },
+  responses,
+})
+
+let rawBodyRoute = Rest.route(() => {
+  method: Post,
+  path: "",
+  input: s => {
+    {
+      "bodyString": s.rawBody(S.string),
+      "auth": s->auth,
+    }
+  },
+  responses,
+})
+
+let clearHasuraMetadata = async (~endpoint, ~auth) => {
+  try {
+    let result = await clearMetadataRoute->Rest.fetch(auth, ~client=Rest.client(endpoint))
+    let msg = switch result {
+    | QuerySucceeded => "Metadata Cleared"
+    | AlreadyDone => "Metadata Already Cleared"
+    }
+    Logging.trace(msg)
+  } catch {
+  | exn =>
+    Logging.error({
+      "msg": `EE806: There was an issue clearing metadata in hasura - indexing may still work - but you may have issues querying the data in hasura.`,
+      "err": exn->Internal.prettifyExn,
+    })
+  }
+}
+
+let trackTables = async (~endpoint, ~auth, ~pgSchema, ~tableNames: array<string>) => {
+  try {
+    let result = await trackTablesRoute->Rest.fetch(
+      {
+        "auth": auth,
+        "args": {
+          // If set to false, any warnings will cause the API call to fail and no new tables to be tracked. Otherwise tables that fail to track will be raised as warnings. (default: true)
+          "allow_warnings": false,
+          "tables": tableNames->Js.Array2.map(tableName =>
+            {
+              "table": {
+                "name": tableName,
+                "schema": pgSchema,
+              },
+              "configuration": {
+                // Otherwise the entity in gql will be prefixed with the schema name (when it's not public)
+                "custom_name": tableName,
+              },
+            }
+          ),
+        }->(Utils.magic: 'a => Js.Json.t),
+      },
+      ~client=Rest.client(endpoint),
+    )
+    let msg = switch result {
+    | QuerySucceeded => "Tables Tracked"
+    | AlreadyDone => "Table Already Tracked"
+    }
+    Logging.trace({
+      "msg": msg,
+      "tableNames": tableNames,
+    })
+  } catch {
+  | exn =>
+    Logging.error({
+      "msg": `EE807: There was an issue tracking tables in hasura - indexing may still work - but you may have issues querying the data in hasura.`,
+      "tableNames": tableNames,
+      "err": exn->Internal.prettifyExn,
+    })
+  }
+}
+
+let createSelectPermissions = async (
+  ~auth,
+  ~endpoint,
+  ~tableName: string,
+  ~pgSchema,
+  ~responseLimit,
+  ~aggregateEntities,
+) => {
+  try {
+    let result = await createSelectPermissionRoute->Rest.fetch(
+      {
+        "auth": auth,
+        "args": {
+          "table": {
+            "schema": pgSchema,
+            "name": tableName,
+          },
+          "role": "public",
+          "source": "default",
+          "permission": {
+            "columns": "*",
+            "filter": Js.Obj.empty(),
+            "limit": responseLimit,
+            "allow_aggregations": aggregateEntities->Js.Array2.includes(tableName),
+          },
+        }->(Utils.magic: 'a => Js.Json.t),
+      },
+      ~client=Rest.client(endpoint),
+    )
+    let msg = switch result {
+    | QuerySucceeded => "Hasura select permissions created"
+    | AlreadyDone => "Hasura select permissions already created"
+    }
+    Logging.trace({
+      "msg": msg,
+      "tableName": tableName,
+    })
+  } catch {
+  | exn =>
+    Logging.error({
+      "msg": `EE808: There was an issue setting up view permissions for the ${tableName} table in hasura - indexing may still work - but you may have issues querying the data in hasura.`,
+      "tableName": tableName,
+      "err": exn->Internal.prettifyExn,
+    })
+  }
+}
+
+let createEntityRelationship = async (
+  ~pgSchema,
+  ~endpoint,
+  ~auth,
+  ~tableName: string,
+  ~relationshipType: string,
+  ~relationalKey: string,
+  ~objectName: string,
+  ~mappedEntity: string,
+  ~isDerivedFrom: bool,
+) => {
+  let derivedFromTo = isDerivedFrom ? `"id": "${relationalKey}"` : `"${relationalKey}_id" : "id"`
+
+  let bodyString = `{"type": "pg_create_${relationshipType}_relationship","args": {"table": {"schema": "${pgSchema}", "name": "${tableName}"},"name": "${objectName}","source": "default","using": {"manual_configuration": {"remote_table": {"schema": "${pgSchema}", "name": "${mappedEntity}"},"column_mapping": {${derivedFromTo}}}}}}`
+
+  try {
+    let result = await rawBodyRoute->Rest.fetch(
+      {
+        "auth": auth,
+        "bodyString": bodyString,
+      },
+      ~client=Rest.client(endpoint),
+    )
+    let msg = switch result {
+    | QuerySucceeded => `Hasura ${relationshipType} relationship created`
+    | AlreadyDone => `Hasura ${relationshipType} relationship already created`
+    }
+    Logging.trace({
+      "msg": msg,
+      "tableName": tableName,
+    })
+  } catch {
+  | exn =>
+    Logging.error({
+      "msg": `EE808: There was an issue setting up ${relationshipType} relationship for the ${tableName} table in hasura - indexing may still work - but you may have issues querying the data in hasura.`,
+      "tableName": tableName,
+      "err": exn->Internal.prettifyExn,
+    })
+  }
+}
+
+let trackDatabase = async (
+  ~endpoint,
+  ~auth,
+  ~pgSchema,
+  ~allStaticTables,
+  ~allEntityTables,
+  ~aggregateEntities,
+  ~responseLimit,
+  ~schema,
+) => {
+  Logging.info("Tracking tables in Hasura")
+
+  let _ = await clearHasuraMetadata(~endpoint, ~auth)
+  let tableNames =
+    [allStaticTables, allEntityTables]
+    ->Belt.Array.concatMany
+    ->Js.Array2.map(({tableName}: Table.table) => tableName)
+
+  await trackTables(~endpoint, ~auth, ~pgSchema, ~tableNames)
+
+  let _ =
+    await tableNames
+    ->Js.Array2.map(tableName =>
+      createSelectPermissions(
+        ~endpoint,
+        ~auth,
+        ~tableName,
+        ~pgSchema,
+        ~responseLimit,
+        ~aggregateEntities,
+      )
+    )
+    ->Js.Array2.concatMany(
+      allEntityTables->Js.Array2.map(table => {
+        let {tableName} = table
+        [
+          //Set array relationships
+          table
+          ->Table.getDerivedFromFields
+          ->Js.Array2.map(derivedFromField => {
+            //determines the actual name of the underlying relational field (if it's an entity mapping then suffixes _id for eg.)
+            let relationalFieldName =
+              schema->Schema.getDerivedFromFieldName(derivedFromField)->Utils.unwrapResultExn
+
+            createEntityRelationship(
+              ~endpoint,
+              ~auth,
+              ~pgSchema,
+              ~tableName,
+              ~relationshipType="array",
+              ~isDerivedFrom=true,
+              ~objectName=derivedFromField.fieldName,
+              ~relationalKey=relationalFieldName,
+              ~mappedEntity=derivedFromField.derivedFromEntity,
+            )
+          }),
+          //Set object relationships
+          table
+          ->Table.getLinkedEntityFields
+          ->Js.Array2.map(((field, linkedEntityName)) => {
+            createEntityRelationship(
+              ~endpoint,
+              ~auth,
+              ~pgSchema,
+              ~tableName,
+              ~relationshipType="object",
+              ~isDerivedFrom=false,
+              ~objectName=field.fieldName,
+              ~relationalKey=field.fieldName,
+              ~mappedEntity=linkedEntityName,
+            )
+          }),
+        ]->Utils.Array.flatten
+      }),
+    )
+    ->Promise.all
+}

package/src/Internal.res

@@ -160,6 +160,31 @@ let fuelTransferParamsSchema = S.schema(s => {
 })
 
 type entity = private {id: string}
+type entityConfig = {
+  name: string,
+  schema: S.t<entity>,
+  rowsSchema: S.t<array<entity>>,
+  table: Table.table,
+  entityHistory: EntityHistory.t<entity>,
+}
+type enum
+type enumConfig<'enum> = {
+  name: string,
+  variants: array<'enum>,
+  schema: S.t<'enum>,
+  default: 'enum,
+}
+external fromGenericEnumConfig: enumConfig<'enum> => enumConfig<enum> = "%identity"
+
+let makeEnumConfig = (~name, ~variants) => {
+  name,
+  variants,
+  schema: S.enum(variants),
+  default: switch variants->Belt.Array.get(0) {
+  | Some(v) => v
+  | None => Js.Exn.raiseError("No variants defined for enum " ++ name)
+  },
+}
 
 type effectInput
 type effectOutput
@@ -172,6 +197,7 @@ type effectArgs = {
 type effect = {
   name: string,
   handler: effectArgs => promise<effectOutput>,
+  mutable callsCount: int,
 }
 
 @genType.import(("./Types.ts", "Invalid"))

package/src/Logging.res

@@ -26,7 +26,7 @@ let logLevels = [
 
 %%private(let logger = ref(None))
 
-let setLogger = (~logStrategy, ~logFilePath, ~defaultFileLogLevel, ~userLogLevel) => {
+let makeLogger = (~logStrategy, ~logFilePath, ~defaultFileLogLevel, ~userLogLevel) => {
   // Currently unused - useful if using multiple transports.
   // let pinoRaw = {"target": "pino/file", "level": Config.userLogLevel}
   let pinoFile: Transport.transportTarget = {
@@ -42,38 +42,38 @@ let setLogger = (~logStrategy, ~logFilePath, ~defaultFileLogLevel, ~userLogLevel
   let makeMultiStreamLogger =
     MultiStreamLogger.make(~userLogLevel, ~defaultFileLogLevel, ~customLevels=logLevels, ...)
 
-  logger :=
-    Some(
-      switch logStrategy {
-      | EcsFile =>
-        makeWithOptionsAndTransport(
-          {
-            ...Pino.ECS.make(),
-            customLevels: logLevels,
-          },
-          Transport.make(pinoFile),
-        )
-      | EcsConsoleMultistream =>
-        makeMultiStreamLogger(~logFile=None, ~options=Some(Pino.ECS.make()))
-      | EcsConsole =>
-        make({
-          ...Pino.ECS.make(),
-          level: userLogLevel,
-          customLevels: logLevels,
-        })
-      | FileOnly =>
-        makeWithOptionsAndTransport(
-          {
-            customLevels: logLevels,
-            level: defaultFileLogLevel,
-          },
-          Transport.make(pinoFile),
-        )
-      | ConsoleRaw => makeMultiStreamLogger(~logFile=None, ~options=None)
-      | ConsolePretty => makeMultiStreamLogger(~logFile=None, ~options=None)
-      | Both => makeMultiStreamLogger(~logFile=Some(logFilePath), ~options=None)
+  switch logStrategy {
+  | EcsFile =>
+    makeWithOptionsAndTransport(
+      {
+        ...Pino.ECS.make(),
+        customLevels: logLevels,
       },
+      Transport.make(pinoFile),
     )
+  | EcsConsoleMultistream => makeMultiStreamLogger(~logFile=None, ~options=Some(Pino.ECS.make()))
+  | EcsConsole =>
+    make({
+      ...Pino.ECS.make(),
+      level: userLogLevel,
+      customLevels: logLevels,
+    })
+  | FileOnly =>
+    makeWithOptionsAndTransport(
+      {
+        customLevels: logLevels,
+        level: defaultFileLogLevel,
+      },
+      Transport.make(pinoFile),
+    )
+  | ConsoleRaw => makeMultiStreamLogger(~logFile=None, ~options=None)
+  | ConsolePretty => makeMultiStreamLogger(~logFile=None, ~options=None)
+  | Both => makeMultiStreamLogger(~logFile=Some(logFilePath), ~options=None)
+  }
+}
+
+let setLogger = l => {
+  logger := Some(l)
 }
 
 let getLogger = () => {

package/src/Persistence.res

@@ -0,0 +1,111 @@
+// A module for the persistence layer
+// This is currently in a WIP state
+// but in the future we should make all DB and in-memory state
+// interactions to this layer with DI and easy for testing.
+// Currently there are quite many code spread across
+// DbFunctions, Db, Migrations, InMemoryStore modules which use codegen code directly.
+
+type storage = {
+  // Should return true if we already have persisted data
+  // and we can skip initialization
+  isInitialized: unit => promise<bool>,
+  // Should initialize the storage so we can start interacting with it
+  // Eg create connection, schema, tables, etc.
+  initialize: (
+    ~entities: array<Internal.entityConfig>,
+    ~staticTables: array<Table.table>,
+    ~enums: array<Internal.enumConfig<Internal.enum>>,
+    // If true, the storage should clear existing data
+    ~cleanRun: bool,
+  ) => promise<unit>,
+}
+
+type storageStatus =
+  | Unknown
+  | Initializing(promise<unit>)
+  | Ready({cleanRun: bool})
+
+type t = {
+  userEntities: array<Internal.entityConfig>,
+  staticTables: array<Table.table>,
+  allEntities: array<Internal.entityConfig>,
+  allEnums: array<Internal.enumConfig<Internal.enum>>,
+  mutable storageStatus: storageStatus,
+  storage: storage,
+  onStorageInitialize: option<unit => promise<unit>>,
+}
+
+let entityHistoryActionEnumConfig: Internal.enumConfig<EntityHistory.RowAction.t> = {
+  name: EntityHistory.RowAction.name,
+  variants: EntityHistory.RowAction.variants,
+  schema: EntityHistory.RowAction.schema,
+  default: SET,
+}
+
+let make = (
+  ~userEntities,
+  ~dcRegistryEntityConfig,
+  // TODO: Should only pass userEnums and create internal config in runtime
+  ~allEnums,
+  ~staticTables,
+  ~storage,
+  ~onStorageInitialize=?,
+) => {
+  let allEntities = userEntities->Js.Array2.concat([dcRegistryEntityConfig])
+  let allEnums =
+    allEnums->Js.Array2.concat([entityHistoryActionEnumConfig->Internal.fromGenericEnumConfig])
+  {
+    userEntities,
+    staticTables,
+    allEntities,
+    allEnums,
+    storageStatus: Unknown,
+    storage,
+    onStorageInitialize,
+  }
+}
+
+let init = async (
+  persistence,
+  // There are not much sense in the option,
+  // but this is how the runUpMigration used to work
+  // and we want to keep the upsert behavior without breaking changes.
+  ~skipIsInitializedCheck=false,
+  ~reset=false,
+) => {
+  try {
+    let shouldRun = switch persistence.storageStatus {
+    | Unknown => true
+    | Initializing(promise) => {
+        await promise
+        reset
+      }
+    | Ready(_) => reset
+    }
+    if shouldRun {
+      let resolveRef = ref(%raw(`null`))
+      let promise = Promise.make((resolve, _) => {
+        resolveRef := resolve
+      })
+      persistence.storageStatus = Initializing(promise)
+      if !(reset || skipIsInitializedCheck) && (await persistence.storage.isInitialized()) {
+        persistence.storageStatus = Ready({cleanRun: false})
+      } else {
+        let _ = await persistence.storage.initialize(
+          ~entities=persistence.allEntities,
+          ~staticTables=persistence.staticTables,
+          ~enums=persistence.allEnums,
+          ~cleanRun=reset || !skipIsInitializedCheck,
+        )
+        persistence.storageStatus = Ready({cleanRun: true})
+        switch persistence.onStorageInitialize {
+        | Some(onStorageInitialize) => await onStorageInitialize()
+        | None => ()
+        }
+      }
+      resolveRef.contents()
+    }
+  } catch {
+  | exn => exn->ErrorHandling.mkLogAndRaise(~msg=`EE800: Failed to initialize the indexer storage.`)
+  }
+}

package/src/PgStorage.res

@@ -0,0 +1,165 @@
+let makeCreateIndexSqlUnsafe = (~tableName, ~indexFields, ~pgSchema) => {
+  let indexName = tableName ++ "_" ++ indexFields->Js.Array2.joinWith("_")
+  let index = indexFields->Belt.Array.map(idx => `"${idx}"`)->Js.Array2.joinWith(", ")
+  `CREATE INDEX IF NOT EXISTS "${indexName}" ON "${pgSchema}"."${tableName}"(${index});`
+}
+
+let makeCreateTableIndicesSqlUnsafe = (table: Table.table, ~pgSchema) => {
+  open Belt
+  let tableName = table.tableName
+  let createIndex = indexField =>
+    makeCreateIndexSqlUnsafe(~tableName, ~indexFields=[indexField], ~pgSchema)
+  let createCompositeIndex = indexFields => {
+    makeCreateIndexSqlUnsafe(~tableName, ~indexFields, ~pgSchema)
+  }
+
+  let singleIndices = table->Table.getSingleIndices
+  let compositeIndices = table->Table.getCompositeIndices
+
+  singleIndices->Array.map(createIndex)->Js.Array2.joinWith("\n") ++
+    compositeIndices->Array.map(createCompositeIndex)->Js.Array2.joinWith("\n")
+}
+
+let makeCreateTableSqlUnsafe = (table: Table.table, ~pgSchema) => {
+  open Belt
+  let fieldsMapped =
+    table
+    ->Table.getFields
+    ->Array.map(field => {
+      let {fieldType, isNullable, isArray, defaultValue} = field
+      let fieldName = field->Table.getDbFieldName
+
+      {
+        `"${fieldName}" ${switch fieldType {
+          | Custom(name) if !(name->Js.String2.startsWith("NUMERIC(")) => `"${pgSchema}".${name}`
+          | _ => (fieldType :> string)
+          }}${isArray ? "[]" : ""}${switch defaultValue {
+          | Some(defaultValue) => ` DEFAULT ${defaultValue}`
+          | None => isNullable ? `` : ` NOT NULL`
+          }}`
+      }
+    })
+    ->Js.Array2.joinWith(", ")
+
+  let primaryKeyFieldNames = table->Table.getPrimaryKeyFieldNames
+  let primaryKey =
+    primaryKeyFieldNames
+    ->Array.map(field => `"${field}"`)
+    ->Js.Array2.joinWith(", ")
+
+  `CREATE TABLE IF NOT EXISTS "${pgSchema}"."${table.tableName}"(${fieldsMapped}${primaryKeyFieldNames->Array.length > 0
+      ? `, PRIMARY KEY(${primaryKey})`
+      : ""});`
+}
+
+let makeInitializeTransaction = (~pgSchema, ~staticTables, ~entities, ~enums, ~cleanRun) => {
+  let allTables = staticTables->Array.copy
+  let allEntityTables = []
+  entities->Js.Array2.forEach((entity: Internal.entityConfig) => {
+    allEntityTables->Js.Array2.push(entity.table)->ignore
+    allTables->Js.Array2.push(entity.table)->ignore
+    allTables->Js.Array2.push(entity.entityHistory.table)->ignore
+  })
+  let derivedSchema = Schema.make(allEntityTables)
+
+  let query = ref(
+    (
+      cleanRun
+        ? `DROP SCHEMA IF EXISTS "${pgSchema}" CASCADE;
+CREATE SCHEMA "${pgSchema}";`
+        : `CREATE SCHEMA IF NOT EXISTS "${pgSchema}";`
+    ) ++
+    `GRANT ALL ON SCHEMA "${pgSchema}" TO postgres;
+GRANT ALL ON SCHEMA "${pgSchema}" TO public;`,
+  )
+
+  // Optimized enum creation - direct when cleanRun, conditional otherwise
+  enums->Js.Array2.forEach((enumConfig: Internal.enumConfig<Internal.enum>) => {
+    // Create base enum creation query once
+    let enumCreateQuery = `CREATE TYPE "${pgSchema}".${enumConfig.name} AS ENUM(${enumConfig.variants
+      ->Js.Array2.map(v => `'${v->(Utils.magic: Internal.enum => string)}'`)
+      ->Js.Array2.joinWith(", ")});`
+
+    query :=
+      query.contents ++
+      "\n" ++ if cleanRun {
+        // Direct creation when cleanRunting (faster)
+        enumCreateQuery
+      } else {
+        // Wrap with conditional check only when not cleanRunting
+        `IF NOT EXISTS (
+  SELECT 1 FROM pg_type 
+  WHERE typname = '${enumConfig.name->Js.String2.toLowerCase}' 
+  AND typnamespace = (SELECT oid FROM pg_namespace WHERE nspname = '${pgSchema}')
+) THEN 
+    ${enumCreateQuery}
+END IF;`
+      }
+  })
+
+  // Batch all table creation first (optimal for PostgreSQL)
+  allTables->Js.Array2.forEach((table: Table.table) => {
+    query := query.contents ++ "\n" ++ makeCreateTableSqlUnsafe(table, ~pgSchema)
+  })
+
+  // Then batch all indices (better performance when tables exist)
+  allTables->Js.Array2.forEach((table: Table.table) => {
+    let indices = makeCreateTableIndicesSqlUnsafe(table, ~pgSchema)
+    if indices !== "" {
+      query := query.contents ++ "\n" ++ indices
+    }
+  })
+
+  let functionsQuery = ref("")
+
+  // Add derived indices
+  entities->Js.Array2.forEach((entity: Internal.entityConfig) => {
+    functionsQuery := functionsQuery.contents ++ "\n" ++ entity.entityHistory.createInsertFnQuery
+
+    entity.table
+    ->Table.getDerivedFromFields
+    ->Js.Array2.forEach(derivedFromField => {
+      let indexField =
+        derivedSchema->Schema.getDerivedFromFieldName(derivedFromField)->Utils.unwrapResultExn
+      query :=
+        query.contents ++
+        "\n" ++
+        makeCreateIndexSqlUnsafe(
+          ~tableName=derivedFromField.derivedFromEntity,
+          ~indexFields=[indexField],
+          ~pgSchema,
+        )
+    })
+  })
+
+  [
+    // Return optimized queries - main DDL in DO block, functions separate
+    // Note: DO $$ BEGIN wrapper is only needed for PL/pgSQL conditionals (IF NOT EXISTS)
+    // Reset case uses direct DDL (faster), non-cleanRun case uses conditionals (safer)
+    cleanRun ? query.contents : `DO $$ BEGIN ${query.contents} END $$;`,
+    // Functions query (separate as they can't be in DO block)
+  ]->Js.Array2.concat(functionsQuery.contents !== "" ? [functionsQuery.contents] : [])
+}
+
+let make = (~sql: Postgres.sql, ~pgSchema): Persistence.storage => {
+  let isInitialized = async () => {
+    let schemas =
+      await sql->Postgres.unsafe(
+        `SELECT schema_name FROM information_schema.schemata WHERE schema_name = '${pgSchema}';`,
+      )
+    schemas->Utils.Array.notEmpty
+  }
+
+  let initialize = async (~entities, ~staticTables, ~enums, ~cleanRun) => {
+    let queries = makeInitializeTransaction(~pgSchema, ~staticTables, ~entities, ~enums, ~cleanRun)
+    // Execute all queries within a single transaction for integrity
+    let _ = await sql->Postgres.beginSql(sql => {
+      queries->Js.Array2.map(query => sql->Postgres.unsafe(query))
+    })
+  }
+
+  {
+    isInitialized,
+    initialize,
+  }
+}

package/src/Prometheus.res

@@ -16,12 +16,6 @@ let executeBatchDurationCounter = PromClient.Counter.makeCounter({
   "labelNames": [],
 })
 
-let eventsProcessedCounter = PromClient.Gauge.makeGauge({
-  "name": "events_processed",
-  "help": "Total number of events processed",
-  "labelNames": ["chainId"],
-})
-
 let allChainsSyncedToHead = PromClient.Gauge.makeGauge({
   "name": "hyperindex_synced_to_head",
   "help": "All chains fully synced",
@@ -207,12 +201,6 @@ module BenchmarkSummaryData = {
   }
 }
 
-let processedUntilHeight = PromClient.Gauge.makeGauge({
-  "name": "chain_block_height_processed",
-  "help": "Block height processed by indexer",
-  "labelNames": ["chainId"],
-})
-
 let incrementLoadEntityDurationCounter = (~duration) => {
   loadEntitiesDurationCounter->PromClient.Counter.incMany(duration)
 }
@@ -225,12 +213,6 @@ let incrementExecuteBatchDurationCounter = (~duration) => {
   executeBatchDurationCounter->PromClient.Counter.incMany(duration)
 }
 
-let setEventsProcessedGuage = (~number, ~chainId) => {
-  eventsProcessedCounter
-  ->PromClient.Gauge.labels({"chainId": chainId})
-  ->PromClient.Gauge.set(number)
-}
-
 let setSourceChainHeight = (~blockNumber, ~chain) => {
   sourceChainHeight
   ->PromClient.Gauge.labels({"chainId": chain->ChainMap.Chain.toString})
@@ -241,12 +223,6 @@ let setAllChainsSyncedToHead = () => {
   allChainsSyncedToHead->PromClient.Gauge.set(1)
 }
 
-let setProcessedUntilHeight = (~blockNumber, ~chain) => {
-  processedUntilHeight
-  ->PromClient.Gauge.labels({"chainId": chain->ChainMap.Chain.toString})
-  ->PromClient.Gauge.set(blockNumber)
-}
-
 module BenchmarkCounters = {
   type labels = {label: string}
   let labelSchema = S.schema(s => {
@@ -304,18 +280,6 @@ module Info = {
   }
 }
 
-module ProgressBlockNumber = {
-  let gauge = SafeGauge.makeOrThrow(
-    ~name="envio_progress_block_number",
-    ~help="The block number to track the progress of indexing at. Currently uses the fully fetched block number. In the future will be changed to block number processed and stored in the database.",
-    ~labelSchema=chainIdLabelsSchema,
-  )
-
-  let set = (~blockNumber, ~chainId) => {
-    gauge->SafeGauge.handleInt(~labels=chainId, ~value=blockNumber)
-  }
-}
-
 module IndexingAddresses = {
   let gauge = SafeGauge.makeOrThrow(
     ~name="envio_indexing_addresses",
@@ -428,8 +392,6 @@ module IndexingBufferBlockNumber = {
     deprecatedGauge
     ->PromClient.Gauge.labels({"chainId": chainId})
     ->PromClient.Gauge.set(blockNumber)
-    // TODO: Use the block number stored in the database instead
-    ProgressBlockNumber.set(~blockNumber, ~chainId)
     gauge->SafeGauge.handleInt(~labels=chainId, ~value=blockNumber)
   }
 }
@@ -484,7 +446,7 @@ module ReorgCount = {
     "labelNames": ["chainId"],
   })
 
-  let counter = SafeGauge.makeOrThrow(
+  let gauge = SafeGauge.makeOrThrow(
     ~name="envio_reorg_count",
     ~help="Total number of reorgs detected",
     ~labelSchema=chainIdLabelsSchema,
@@ -494,7 +456,7 @@ module ReorgCount = {
     deprecatedCounter
     ->PromClient.Counter.labels({"chainId": chain->ChainMap.Chain.toString})
     ->PromClient.Counter.inc
-    counter->SafeGauge.increment(~labels=chain->ChainMap.Chain.toChainId)
+    gauge->SafeGauge.increment(~labels=chain->ChainMap.Chain.toChainId)
   }
 }
 
@@ -545,6 +507,30 @@ module RollbackTargetBlockNumber = {
   }
 }
 
+module ProcessingBlockNumber = {
+  let gauge = SafeGauge.makeOrThrow(
+    ~name="envio_processing_block_number",
+    ~help="The latest item block number included in the currently processing batch for the chain.",
+    ~labelSchema=chainIdLabelsSchema,
+  )
+
+  let set = (~blockNumber, ~chainId) => {
+    gauge->SafeGauge.handleInt(~labels=chainId, ~value=blockNumber)
+  }
+}
+
+module ProcessingBatchSize = {
+  let gauge = SafeGauge.makeOrThrow(
+    ~name="envio_processing_batch_size",
+    ~help="The number of items included in the currently processing batch for the chain.",
+    ~labelSchema=chainIdLabelsSchema,
+  )
+
+  let set = (~batchSize, ~chainId) => {
+    gauge->SafeGauge.handleInt(~labels=chainId, ~value=batchSize)
+  }
+}
+
 module ProcessingMaxBatchSize = {
   let gauge = PromClient.Gauge.makeGauge({
     "name": "envio_processing_max_batch_size",
@@ -555,3 +541,52 @@ module ProcessingMaxBatchSize = {
     gauge->PromClient.Gauge.set(maxBatchSize)
   }
 }
+
+module ProgressBlockNumber = {
+  let gauge = SafeGauge.makeOrThrow(
+    ~name="envio_progress_block_number",
+    ~help="The block number of the latest block processed and stored in the database.",
+    ~labelSchema=chainIdLabelsSchema,
+  )
+
+  let set = (~blockNumber, ~chainId) => {
+    gauge->SafeGauge.handleInt(~labels=chainId, ~value=blockNumber)
+  }
+}
+
+module ProgressEventsCount = {
+  let deprecatedGauge = PromClient.Gauge.makeGauge({
+    "name": "events_processed",
+    "help": "Total number of events processed",
+    "labelNames": ["chainId"],
+  })
+
+  let gauge = SafeGauge.makeOrThrow(
+    ~name="envio_progress_events_count",
+    ~help="The number of events processed and reflected in the database.",
+    ~labelSchema=chainIdLabelsSchema,
+  )
+
+  let set = (~processedCount, ~chainId) => {
+    deprecatedGauge
+    ->PromClient.Gauge.labels({"chainId": chainId})
+    ->PromClient.Gauge.set(processedCount)
+    gauge->SafeGauge.handleInt(~labels=chainId, ~value=processedCount)
+  }
+}
+
+let effectLabelsSchema = S.object(s => {
+  s.field("effect", S.string)
+})
+
+module EffectCallsCount = {
+  let gauge = SafeGauge.makeOrThrow(
+    ~name="envio_effect_calls_count",
+    ~help="The number of calls to the effect. Including both handler execution and cache hits.",
+    ~labelSchema=effectLabelsSchema,
+  )
+
+  let set = (~callsCount, ~effectName) => {
+    gauge->SafeGauge.handleInt(~labels=effectName, ~value=callsCount)
+  }
+}

package/src/bindings/BigDecimal.gen.ts

@@ -3,7 +3,7 @@
 /* eslint-disable */
 /* tslint:disable */
 
-const BigDecimalJS = require('./BigDecimal.bs.js');
+const BigDecimalJS = require('./BigDecimal.res.js');
 
 import type {S_t as RescriptSchema_S_t} from 'rescript-schema/RescriptSchema.gen';
 

package/src/bindings/Ethers.gen.ts

@@ -3,7 +3,7 @@
 /* eslint-disable */
 /* tslint:disable */
 
-const EthersJS = require('./Ethers.bs.js');
+const EthersJS = require('./Ethers.res.js');
 
 import type {t as Address_t} from '../../src/Address.gen';
 

package/src/bindings/Promise.res

@@ -37,6 +37,11 @@ external all6: ((t<'a>, t<'b>, t<'c>, t<'d>, t<'e>, t<'f>)) => t<('a, 'b, 'c, 'd
 @send
 external catch: (t<'a>, @uncurry exn => t<'a>) => t<'a> = "catch"
 
+%%private(let noop = (() => ())->Obj.magic)
+let silentCatch = (promise: promise<'a>): promise<'a> => {
+  catch(promise, noop)
+}
+
 let catch = (promise: promise<'a>, callback: exn => promise<'a>): promise<'a> => {
   catch(promise, err => {
     callback(Js.Exn.anyToExnInternal(err))

package/src/db/EntityHistory.res

@@ -4,7 +4,7 @@ module RowAction = {
   type t = SET | DELETE
   let variants = [SET, DELETE]
   let name = "ENTITY_HISTORY_ROW_ACTION"
-  let enum = Enum.make(~name, ~variants)
+  let schema = S.enum(variants)
 }
 
 type historyFieldsGeneral<'a> = {
@@ -88,7 +88,7 @@ let makeHistoryRowSchema: S.t<'entity> => S.t<historyRow<'entity>> = entitySchem
       "current": s.flatten(currentHistoryFieldsSchema),
       "previous": s.flatten(previousHistoryFieldsSchema),
       "entityData": s.flatten(nullableEntitySchema),
-      "action": s.field("action", RowAction.enum.schema),
+      "action": s.field("action", RowAction.schema),
     }
   })->S.transform(s => {
     parser: v => {
@@ -228,7 +228,7 @@ let fromTable = (table: table, ~schema: S.t<'entity>): t<'entity> => {
 
   let actionFieldName = "action"
 
-  let actionField = mkField(actionFieldName, Custom(RowAction.enum.name), ~fieldSchema=S.never)
+  let actionField = mkField(actionFieldName, Custom(RowAction.name), ~fieldSchema=S.never)
 
   let serialField = mkField("serial", Serial, ~fieldSchema=S.never, ~isNullable=true, ~isIndex=true)
 

package/src/Enum.res

@@ -1,22 +0,0 @@
-// Graphql Enum Type Variants
-type enum<'a> = {
-  name: string,
-  variants: array<'a>,
-  schema: S.t<'a>,
-  default: 'a,
-}
-
-let make = (~name, ~variants) => {
-  name,
-  variants,
-  schema: S.enum(variants),
-  default: switch variants->Belt.Array.get(0) {
-  | Some(v) => v
-  | None => Js.Exn.raiseError("No variants defined for enum " ++ name)
-  },
-}
-
-module type S = {
-  type t
-  let enum: enum<t>
-}