envio 3.0.0-alpha.2 → 3.0.0-alpha.20

package/src/PgStorage.res

@@ -1,18 +1,73 @@
 let getCacheRowCountFnName = "get_cache_row_count"
 
+// Only needed for some old tests
+// Remove @genType in the future
+@genType
+let makeClient = () => {
+  Postgres.makeSql(
+    ~config={
+      host: Env.Db.host,
+      port: Env.Db.port,
+      username: Env.Db.user,
+      password: Env.Db.password,
+      database: Env.Db.database,
+      ssl: Env.Db.ssl,
+      // TODO: think how we want to pipe these logs to pino.
+      onnotice: ?(
+        Env.userLogLevel == Some(#warn) || Env.userLogLevel == Some(#error)
+          ? None
+          : Some(_str => ())
+      ),
+      transform: {undefined: Null},
+      max: Env.Db.maxConnections,
+      // debug: (~connection, ~query, ~params as _, ~types as _) => Js.log2(connection, query),
+    },
+  )
+}
+
 let makeCreateIndexQuery = (~tableName, ~indexFields, ~pgSchema) => {
   let indexName = tableName ++ "_" ++ indexFields->Js.Array2.joinWith("_")
   let index = indexFields->Belt.Array.map(idx => `"${idx}"`)->Js.Array2.joinWith(", ")
   `CREATE INDEX IF NOT EXISTS "${indexName}" ON "${pgSchema}"."${tableName}"(${index});`
 }
 
+let directionToSql = (direction: Table.indexFieldDirection) =>
+  switch direction {
+  | Asc => ""
+  | Desc => " DESC"
+  }
+
+let directionToIndexName = (direction: Table.indexFieldDirection) =>
+  switch direction {
+  | Asc => ""
+  | Desc => "_desc"
+  }
+
+let makeCreateCompositeIndexQuery = (
+  ~tableName,
+  ~indexFields: array<Table.compositeIndexField>,
+  ~pgSchema,
+) => {
+  let indexName =
+    tableName ++
+    "_" ++
+    indexFields
+    ->Js.Array2.map(f => f.fieldName ++ directionToIndexName(f.direction))
+    ->Js.Array2.joinWith("_")
+  let index =
+    indexFields
+    ->Belt.Array.map(f => `"${f.fieldName}"${directionToSql(f.direction)}`)
+    ->Js.Array2.joinWith(", ")
+  `CREATE INDEX IF NOT EXISTS "${indexName}" ON "${pgSchema}"."${tableName}"(${index});`
+}
+
 let makeCreateTableIndicesQuery = (table: Table.table, ~pgSchema) => {
   open Belt
   let tableName = table.tableName
   let createIndex = indexField =>
     makeCreateIndexQuery(~tableName, ~indexFields=[indexField], ~pgSchema)
   let createCompositeIndex = indexFields => {
-    makeCreateIndexQuery(~tableName, ~indexFields, ~pgSchema)
+    makeCreateCompositeIndexQuery(~tableName, ~indexFields, ~pgSchema)
   }
 
   let singleIndices = table->Table.getSingleIndices
@@ -57,10 +112,11 @@ let makeCreateTableQuery = (table: Table.table, ~pgSchema, ~isNumericArrayAsText
       : ""});`
 }
 
+let entityHistoryCache = Utils.WeakMap.make()
 let getEntityHistory = (~entityConfig: Internal.entityConfig): EntityHistory.pgEntityHistory<
   'entity,
 > => {
-  switch entityConfig.pgEntityHistoryCache {
+  switch entityHistoryCache->Utils.WeakMap.get(entityConfig) {
   | Some(cache) => cache
   | None =>
     let cache = {
@@ -119,7 +175,7 @@ let getEntityHistory = (~entityConfig: Internal.entityConfig): EntityHistory.pgE
       }
     }
 
-    entityConfig.pgEntityHistoryCache = Some(cache)
+    entityHistoryCache->Utils.WeakMap.set(entityConfig, cache)->ignore
     cache
   }
 }
@@ -154,9 +210,9 @@ let makeInitializeTransaction = (
       isEmptyPgSchema && pgSchema === "public"
       // Hosted Service already have a DB with the created public schema
       // It also doesn't allow to simply drop it,
-      // so we reuse the existing schema when it's empty
-      // (but only for public, since it's usually always exists)
-        ? ""
+      // so we reuse the existing schema when it's empty.
+      // IF NOT EXISTS handles the case where public was previously dropped.
+        ? `CREATE SCHEMA IF NOT EXISTS "${pgSchema}";\n`
         : `DROP SCHEMA IF EXISTS "${pgSchema}" CASCADE;
 CREATE SCHEMA "${pgSchema}";\n`
     ) ++
@@ -376,7 +432,9 @@ let makeTableBatchSetQuery = (~pgSchema, ~table: Table.table, ~itemSchema: S.t<'
       ),
       "convertOrThrow": S.compile(
         S.unnest(itemSchema)->S.preprocess(_ => {
-          serializer: Utils.Array.flatten->Utils.magic,
+          serializer: Utils.Array.flatten->(
+            Utils.magic: (array<array<'a>> => array<'a>) => unknown => unknown
+          ),
         }),
         ~input=Value,
         ~output=Unknown,
@@ -454,14 +512,17 @@ let setOrThrow = async (sql, ~items, ~table: Table.table, ~itemSchema, ~pgSchema
           let chunkSize = chunk->Array.length
           let isFullChunk = chunkSize === maxItemsPerQuery
 
-          let response = sql->Postgres.preparedUnsafe(
-            // Either use the sql query for full chunks from cache
-            // or create a new one for partial chunks on the fly.
-            isFullChunk
-              ? data["query"]
-              : makeInsertValuesSetQuery(~pgSchema, ~table, ~itemSchema, ~itemsCount=chunkSize),
-            data["convertOrThrow"](chunk->(Utils.magic: array<'item> => array<unknown>)),
+          let params = data["convertOrThrow"](
+            chunk->(Utils.magic: array<'item> => array<unknown>),
           )
+          // Use prepared query only for full batches where the cached query is reused.
+          // Partial chunks generate unique SQL each time, so preparation has no benefit.
+          let response = isFullChunk
+            ? sql->Postgres.preparedUnsafe(data["query"], params)
+            : sql->Postgres.unpreparedUnsafe(
+                makeInsertValuesSetQuery(~pgSchema, ~table, ~itemSchema, ~itemsCount=chunkSize),
+                params,
+              )
           responses->Js.Array2.push(response)->ignore
         })
         let _ = await Promise.all(responses)
@@ -522,18 +583,17 @@ type psqlExecState =
   Unknown | Pending(promise<result<string, string>>) | Resolved(result<string, string>)
 
 let getConnectedPsqlExec = {
-  let pgDockerServiceName = "envio-postgres"
   // Should use the default port, since we're executing the command
   // from the postgres container's network.
   let pgDockerServicePort = 5432
 
   // For development: We run the indexer process locally,
   //   and there might not be psql installed on the user's machine.
-  //   So we use docker-compose to run psql existing in the postgres container.
+  //   So we use docker exec to run psql inside the postgres container.
   // For production: We expect indexer to be running in a container,
   //   with psql installed. So we can call it directly.
   let psqlExecState = ref(Unknown)
-  async (~pgUser, ~pgHost, ~pgDatabase, ~pgPort) => {
+  async (~pgUser, ~pgHost, ~pgDatabase, ~pgPort, ~containerName) => {
     switch psqlExecState.contents {
     | Unknown => {
         let promise = Promise.make((resolve, _reject) => {
@@ -541,14 +601,16 @@ let getConnectedPsqlExec = {
           NodeJs.ChildProcess.exec(`${binary} --version`, (~error, ~stdout as _, ~stderr as _) => {
             switch error {
             | Value(_) => {
-                let binary = `docker-compose exec -T -u ${pgUser} ${pgDockerServiceName} psql`
+                let binary = `docker exec -i -u ${pgUser} ${containerName} psql`
                 NodeJs.ChildProcess.exec(
                   `${binary} --version`,
                   (~error, ~stdout as _, ~stderr as _) => {
                     switch error {
                     | Value(_) =>
                       resolve(
-                        Error(`Please check if "psql" binary is installed or docker-compose is running for the local indexer.`),
+                        Error(
+                          `Please check if "psql" binary is installed or Docker container "${containerName}" is running.`,
+                        ),
                       )
                     | Null =>
                       resolve(
@@ -766,7 +828,7 @@ let rec writeBatch = async (
                 sql
                 ->Postgres.preparedUnsafe(
                   makeInsertDeleteUpdatesQuery(~entityConfig, ~pgSchema),
-                  (batchDeleteEntityIds, batchDeleteCheckpointIds)->Obj.magic,
+                  (batchDeleteEntityIds, batchDeleteCheckpointIds->BigInt.arrayToStringArray)->Obj.magic,
                 )
                 ->Promise.ignoreValue,
               )
@@ -908,6 +970,7 @@ let rec writeBatch = async (
                 ): InternalTable.Chains.progressedChain => {
                   chainId: chainAfterBatch.fetchState.chainId,
                   progressBlockNumber: chainAfterBatch.progressBlockNumber,
+                  sourceBlockNumber: chainAfterBatch.sourceBlockNumber,
                   totalEventsProcessed: chainAfterBatch.totalEventsProcessed,
                 }),
               ),
@@ -1051,13 +1114,14 @@ let make = (
   ~onInitialize=?,
   ~onNewTables=?,
 ): Persistence.storage => {
+  // Must match PG_CONTAINER in packages/cli/src/docker_env.rs
+  let containerName = "envio-postgres"
   let psqlExecOptions: NodeJs.ChildProcess.execOptions = {
     env: Js.Dict.fromArray([("PGPASSWORD", pgPassword), ("PATH", %raw(`process.env.PATH`))]),
   }
 
   let cacheDirPath = NodeJs.Path.resolve([
-    // Right outside of the generated directory
-    "..",
+    // Right at the project root
     ".envio",
     "cache",
   ])
@@ -1079,7 +1143,7 @@ let make = (
         NodeJs.Fs.Promises.readdir(cacheDirPath)
         ->Promise.thenResolve(e => Ok(e))
         ->Promise.catch(_ => Promise.resolve(Error(nothingToUploadErrorMessage))),
-        getConnectedPsqlExec(~pgUser, ~pgHost, ~pgDatabase, ~pgPort),
+        getConnectedPsqlExec(~pgUser, ~pgHost, ~pgDatabase, ~pgPort, ~containerName),
       )) {
       | (Ok(entries), Ok(psqlExec)) => {
           let cacheFiles = entries->Js.Array2.filter(entry => {
@@ -1175,7 +1239,7 @@ let make = (
         )
     ) {
       Js.Exn.raiseError(
-        `Cannot run Envio migrations on PostgreSQL schema "${pgSchema}" because it contains non-Envio tables. Running migrations would delete all data in this schema.\n\nTo resolve this:\n1. If you want to use this schema, first backup any important data, then drop it with: "pnpm envio local db-migrate down"\n2. Or specify a different schema name by setting the "ENVIO_PG_PUBLIC_SCHEMA" environment variable\n3. Or manually drop the schema in your database if you're certain the data is not needed.`,
+        `Cannot run Envio migrations on PostgreSQL schema "${pgSchema}" because it contains non-Envio tables. Running migrations would delete all data in this schema.\n\nTo resolve this:\n1. If you want to use this schema, first backup any important data, then drop it with: "pnpm envio local db-migrate down"\n2. Or specify a different schema name by setting the "ENVIO_PG_SCHEMA" environment variable\n3. Or manually drop the schema in your database if you're certain the data is not needed.`,
       )
     }
 
@@ -1219,10 +1283,11 @@ let make = (
         endBlock: chainConfig.endBlock,
         maxReorgDepth: chainConfig.maxReorgDepth,
         progressBlockNumber: -1,
-        numEventsProcessed: 0,
+        numEventsProcessed: 0.,
         firstEventBlockNumber: None,
         timestampCaughtUpToHeadOrEndblock: None,
         dynamicContracts: [],
+        sourceBlockNumber: 0,
       }),
       checkpointId: InternalTable.Checkpoints.initialCheckpointId,
     }
@@ -1363,10 +1428,10 @@ let make = (
           await NodeJs.Fs.Promises.mkdir(~path=cacheDirPath, ~options={recursive: true})
         }
 
-        // Command for testing. Run from generated
-        // docker-compose exec -T -u postgres envio-postgres psql -d envio-dev -c 'COPY "public"."envio_effect_getTokenMetadata" TO STDOUT (FORMAT text, HEADER);' > ../.envio/cache/getTokenMetadata.tsv
+        // Command for testing. Run from project root:
+        // docker exec -i -u postgres envio-{indexerName}-postgres psql -d envio-dev -c 'COPY "public"."envio_effect_getTokenMetadata" TO STDOUT (FORMAT text, HEADER);' > ../.envio/cache/getTokenMetadata.tsv
 
-        switch await getConnectedPsqlExec(~pgUser, ~pgHost, ~pgDatabase, ~pgPort) {
+        switch await getConnectedPsqlExec(~pgUser, ~pgHost, ~pgDatabase, ~pgPort, ~containerName) {
         | Ok(psqlExec) => {
             Logging.info(
               `Dumping cache: ${cacheTableInfo
@@ -1425,17 +1490,30 @@ let make = (
           numEventsProcessed: rawInitialState.numEventsProcessed,
           progressBlockNumber: rawInitialState.progressBlockNumber,
           dynamicContracts: rawInitialState.dynamicContracts,
+          sourceBlockNumber: rawInitialState.sourceBlockNumber,
         })
       }),
       sql
       ->Postgres.unsafe(InternalTable.Checkpoints.makeCommitedCheckpointIdQuery(~pgSchema))
-      ->(Utils.magic: promise<array<unknown>> => promise<array<{"id": float}>>),
+      ->(Utils.magic: promise<array<unknown>> => promise<array<{"id": string}>>),
       sql
       ->Postgres.unsafe(InternalTable.Checkpoints.makeGetReorgCheckpointsQuery(~pgSchema))
-      ->(Utils.magic: promise<array<unknown>> => promise<array<Internal.reorgCheckpoint>>),
+      ->(
+        Utils.magic: promise<array<unknown>> => promise<
+          array<{"id": string, "chain_id": int, "block_number": int, "block_hash": string}>,
+        >
+      ),
     ))
 
-    let checkpointId = (checkpointIdResult->Belt.Array.getUnsafe(0))["id"]
+    let checkpointId = (checkpointIdResult->Belt.Array.getUnsafe(0))["id"]->BigInt.fromStringUnsafe
+
+    // Convert string checkpoint IDs from DB to bigint
+    let reorgCheckpoints = Belt.Array.map(reorgCheckpoints, (raw): Internal.reorgCheckpoint => {
+      checkpointId: raw["id"]->BigInt.fromStringUnsafe,
+      chainId: raw["chain_id"],
+      blockNumber: raw["block_number"],
+      blockHash: raw["block_hash"],
+    })
 
     // Resume sink if present - needed to rollback any reorg changes
     switch sink {
@@ -1452,37 +1530,9 @@ let make = (
     }
   }
 
-  let executeUnsafe = query => sql->Postgres.unsafe(query)
-
-  let hasEntityHistoryRows = async () => {
-    // Query for all entity history tables (they have the prefix "envio_history_")
-    let historyTables = await sql->Postgres.unsafe(
-      `SELECT table_name FROM information_schema.tables 
-       WHERE table_schema = '${pgSchema}' 
-       AND table_name LIKE 'envio_history_%';`,
-    )
-
-    if historyTables->Utils.Array.isEmpty {
-      false
-    } else {
-      // Check if any of these tables have rows
-      let checks =
-        await historyTables
-        ->Belt.Array.map(async (table: {"table_name": string}) => {
-          try {
-            let query = `SELECT EXISTS(SELECT 1 FROM "${pgSchema}"."${table["table_name"]}" LIMIT 1);`
-            let result: array<{"exists": bool}> = (await sql->Postgres.unsafe(query))->Utils.magic
-            switch result {
-            | [row] => row["exists"]
-            | _ => false
-            }
-          } catch {
-          | _ => false
-          }
-        })
-        ->Promise.all
-      checks->Belt.Array.some(v => v)
-    }
+  let reset = async () => {
+    let query = `DROP SCHEMA IF EXISTS "${pgSchema}" CASCADE;`
+    await sql->Postgres.unsafe(query)->Promise.ignoreValue
   }
 
   let setChainMeta = chainsData =>
@@ -1522,14 +1572,14 @@ let make = (
       sql
       ->Postgres.preparedUnsafe(
         makeGetRollbackRemovedIdsQuery(~entityConfig, ~pgSchema),
-        [rollbackTargetCheckpointId]->Utils.magic,
+        [rollbackTargetCheckpointId->BigInt.toString]->(Utils.magic: array<string> => unknown),
       )
       ->(Utils.magic: promise<unknown> => promise<array<{"id": string}>>),
       // Get entities that should be restored to their state at or before rollback target
       sql
       ->Postgres.preparedUnsafe(
         makeGetRollbackRestoredEntitiesQuery(~entityConfig, ~pgSchema),
-        [rollbackTargetCheckpointId]->Utils.magic,
+        [rollbackTargetCheckpointId->BigInt.toString]->(Utils.magic: array<string> => unknown),
       )
       ->(Utils.magic: promise<unknown> => promise<array<unknown>>),
     ))
@@ -1554,7 +1604,7 @@ let make = (
           ->Promise.thenResolve(_ => {
             Prometheus.SinkWrite.increment(
               ~sinkName=sink.name,
-              ~timeMillis=timerRef->Hrtime.timeSince->Hrtime.toMillis->Hrtime.intFromMillis,
+              ~timeSeconds=timerRef->Hrtime.timeSince->Hrtime.toSecondsFloat,
             )
             None
           })
@@ -1587,11 +1637,8 @@ let make = (
     resumeInitialState,
     loadByFieldOrThrow,
     loadByIdsOrThrow,
-    setOrThrow,
-    setEffectCacheOrThrow,
     dumpEffectCache,
-    executeUnsafe,
-    hasEntityHistoryRows,
+    reset,
     setChainMeta,
     pruneStaleCheckpoints,
     pruneStaleEntityHistory,

package/src/PgStorage.res.d.mts

@@ -0,0 +1,5 @@
+// This is to prevent tsc --noEmit from failing
+// when importing code from .res.mjs files in genType .ts files
+// After we upgrade GenType and it starts to include ts-ignore,
+// the line can be removed.
+export const makeClient: any;