envio 3.0.0-alpha.2 → 3.0.0-alpha.20

package/src/PgStorage.res.mjs

@@ -1,10 +1,12 @@
 // Generated by ReScript, PLEASE EDIT WITH CARE
 
 import * as Fs from "fs";
+import * as Env from "./Env.res.mjs";
 import * as Path from "path";
 import * as $$Array from "rescript/lib/es6/array.js";
 import * as Table from "./db/Table.res.mjs";
 import * as Utils from "./Utils.res.mjs";
+import * as $$BigInt from "./bindings/BigInt.res.mjs";
 import * as Config from "./Config.res.mjs";
 import * as Hrtime from "./bindings/Hrtime.res.mjs";
 import * as Js_exn from "rescript/lib/es6/js_exn.js";
@@ -12,7 +14,9 @@ import * as Schema from "./db/Schema.res.mjs";
 import * as Js_dict from "rescript/lib/es6/js_dict.js";
 import * as Logging from "./Logging.res.mjs";
 import * as $$Promise from "./bindings/Promise.res.mjs";
+import * as Caml_obj from "rescript/lib/es6/caml_obj.js";
 import * as Internal from "./Internal.res.mjs";
+import Postgres from "postgres";
 import * as Belt_Array from "rescript/lib/es6/belt_Array.js";
 import * as Prometheus from "./Prometheus.res.mjs";
 import * as Caml_option from "rescript/lib/es6/caml_option.js";
@@ -26,6 +30,24 @@ import * as Caml_js_exceptions from "rescript/lib/es6/caml_js_exceptions.js";
 
 var getCacheRowCountFnName = "get_cache_row_count";
 
+function makeClient() {
+  return Postgres({
+              host: Env.Db.host,
+              port: Env.Db.port,
+              database: Env.Db.database,
+              username: Env.Db.user,
+              password: Env.Db.password,
+              ssl: Env.Db.ssl,
+              max: Env.Db.maxConnections,
+              onnotice: Caml_obj.equal(Env.userLogLevel, "warn") || Caml_obj.equal(Env.userLogLevel, "error") ? undefined : (function (_str) {
+                    
+                  }),
+              transform: {
+                undefined: null
+              }
+            });
+}
+
 function makeCreateIndexQuery(tableName, indexFields, pgSchema) {
   var indexName = tableName + "_" + indexFields.join("_");
   var index = Belt_Array.map(indexFields, (function (idx) {
@@ -34,13 +56,39 @@ function makeCreateIndexQuery(tableName, indexFields, pgSchema) {
   return "CREATE INDEX IF NOT EXISTS \"" + indexName + "\" ON \"" + pgSchema + "\".\"" + tableName + "\"(" + index + ");";
 }
 
+function directionToSql(direction) {
+  if (direction === "Asc") {
+    return "";
+  } else {
+    return " DESC";
+  }
+}
+
+function directionToIndexName(direction) {
+  if (direction === "Asc") {
+    return "";
+  } else {
+    return "_desc";
+  }
+}
+
+function makeCreateCompositeIndexQuery(tableName, indexFields, pgSchema) {
+  var indexName = tableName + "_" + indexFields.map(function (f) {
+          return f.fieldName + directionToIndexName(f.direction);
+        }).join("_");
+  var index = Belt_Array.map(indexFields, (function (f) {
+            return "\"" + f.fieldName + "\"" + directionToSql(f.direction);
+          })).join(", ");
+  return "CREATE INDEX IF NOT EXISTS \"" + indexName + "\" ON \"" + pgSchema + "\".\"" + tableName + "\"(" + index + ");";
+}
+
 function makeCreateTableIndicesQuery(table, pgSchema) {
   var tableName = table.tableName;
   var createIndex = function (indexField) {
     return makeCreateIndexQuery(tableName, [indexField], pgSchema);
   };
   var createCompositeIndex = function (indexFields) {
-    return makeCreateIndexQuery(tableName, indexFields, pgSchema);
+    return makeCreateCompositeIndexQuery(tableName, indexFields, pgSchema);
   };
   var singleIndices = Table.getSingleIndices(table);
   var compositeIndices = Table.getCompositeIndices(table);
@@ -67,8 +115,10 @@ function makeCreateTableQuery(table, pgSchema, isNumericArrayAsText) {
         ) + ");";
 }
 
+var entityHistoryCache = new WeakMap();
+
 function getEntityHistory(entityConfig) {
-  var cache = entityConfig.pgEntityHistoryCache;
+  var cache = entityHistoryCache.get(entityConfig);
   if (cache !== undefined) {
     return cache;
   }
@@ -125,7 +175,7 @@ function getEntityHistory(entityConfig) {
     setChangeSchema: setChangeSchema,
     setChangeSchemaRows: cache_setChangeSchemaRows
   };
-  entityConfig.pgEntityHistoryCache = cache$1;
+  entityHistoryCache.set(entityConfig, cache$1);
   return cache$1;
 }
 
@@ -150,7 +200,7 @@ function makeInitializeTransaction(pgSchema, pgUser, isHasuraEnabled, chainConfi
   var derivedSchema = Schema.make(allEntityTables);
   var query = {
     contents: (
-      isEmptyPgSchema && pgSchema === "public" ? "" : "DROP SCHEMA IF EXISTS \"" + pgSchema + "\" CASCADE;\nCREATE SCHEMA \"" + pgSchema + "\";\n"
+      isEmptyPgSchema && pgSchema === "public" ? "CREATE SCHEMA IF NOT EXISTS \"" + pgSchema + "\";\n" : "DROP SCHEMA IF EXISTS \"" + pgSchema + "\" CASCADE;\nCREATE SCHEMA \"" + pgSchema + "\";\n"
     ) + ("GRANT ALL ON SCHEMA \"" + pgSchema + "\" TO \"" + pgUser + "\";\nGRANT ALL ON SCHEMA \"" + pgSchema + "\" TO public;")
   };
   enums.forEach(function (enumConfig) {
@@ -338,7 +388,8 @@ async function setOrThrow(sql, items, table, itemSchema, pgSchema) {
     chunks.forEach(function (chunk) {
           var chunkSize = chunk.length;
           var isFullChunk = chunkSize === 500;
-          var response = sql.unsafe(isFullChunk ? data.query : makeInsertValuesSetQuery(pgSchema, table, itemSchema, chunkSize), data.convertOrThrow(chunk), {prepare: true});
+          var params = data.convertOrThrow(chunk);
+          var response = isFullChunk ? sql.unsafe(data.query, params, {prepare: true}) : sql.unsafe(makeInsertValuesSetQuery(pgSchema, table, itemSchema, chunkSize), params);
           responses.push(response);
         });
     await Promise.all(responses);
@@ -377,7 +428,7 @@ var psqlExecState = {
   contents: "Unknown"
 };
 
-async function getConnectedPsqlExec(pgUser, pgHost, pgDatabase, pgPort) {
+async function getConnectedPsqlExec(pgUser, pgHost, pgDatabase, pgPort, containerName) {
   var promise = psqlExecState.contents;
   if (typeof promise === "object") {
     if (promise.TAG === "Pending") {
@@ -395,7 +446,7 @@ async function getConnectedPsqlExec(pgUser, pgHost, pgDatabase, pgPort) {
                                 _0: binary + " -h " + pgHost + " -p " + pgPort.toString() + " -U " + pgUser + " -d " + pgDatabase
                               });
                   }
-                  var binary$1 = "docker-compose exec -T -u " + pgUser + " envio-postgres psql";
+                  var binary$1 = "docker exec -i -u " + pgUser + " " + containerName + " psql";
                   Child_process.exec(binary$1 + " --version", (function (error, param, param$1) {
                           if (error === null) {
                             return resolve({
@@ -405,7 +456,7 @@ async function getConnectedPsqlExec(pgUser, pgHost, pgDatabase, pgPort) {
                           } else {
                             return resolve({
                                         TAG: "Error",
-                                        _0: "Please check if \"psql\" binary is installed or docker-compose is running for the local indexer."
+                                        _0: "Please check if \"psql\" binary is installed or Docker container \"" + containerName + "\" is running."
                                       });
                           }
                         }));
@@ -531,7 +582,7 @@ async function writeBatch(sql, batch, rawEvents, pgSchema, rollbackTargetCheckpo
                   if (Utils.$$Array.notEmpty(batchDeleteCheckpointIds)) {
                     promises.push(sql.unsafe(makeInsertDeleteUpdatesQuery(entityConfig, pgSchema), [
                               batchDeleteEntityIds,
-                              batchDeleteCheckpointIds
+                              $$BigInt.arrayToStringArray(batchDeleteCheckpointIds)
                             ], {prepare: true}));
                   }
                   if (Utils.$$Array.notEmpty(batchSetUpdates)) {
@@ -614,6 +665,7 @@ async function writeBatch(sql, batch, rawEvents, pgSchema, rollbackTargetCheckpo
                                               return {
                                                       chainId: chainAfterBatch.fetchState.chainId,
                                                       progressBlockNumber: chainAfterBatch.progressBlockNumber,
+                                                      sourceBlockNumber: chainAfterBatch.sourceBlockNumber,
                                                       totalEventsProcessed: chainAfterBatch.totalEventsProcessed
                                                     };
                                             })));
@@ -684,6 +736,7 @@ function makeGetRollbackRemovedIdsQuery(entityConfig, pgSchema) {
 }
 
 function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isHasuraEnabled, sink, onInitialize, onNewTables) {
+  var containerName = "envio-postgres";
   var psqlExecOptions_env = Js_dict.fromArray([
         [
           "PGPASSWORD",
@@ -697,7 +750,7 @@ function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isH
   var psqlExecOptions = {
     env: psqlExecOptions_env
   };
-  var cacheDirPath = Path.resolve("..", ".envio", "cache");
+  var cacheDirPath = Path.resolve(".envio", "cache");
   var isInitialized = async function () {
     var envioTables = await sql.unsafe("SELECT table_schema FROM information_schema.tables WHERE table_schema = '" + pgSchema + "' AND (table_name = 'event_sync_state' OR table_name = '" + InternalTable.Chains.table.tableName + "');");
     return Utils.$$Array.notEmpty(envioTables);
@@ -717,7 +770,7 @@ function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isH
                                 _0: nothingToUploadErrorMessage
                               });
                   })),
-            getConnectedPsqlExec(pgUser, pgHost, pgDatabase, pgPort)
+            getConnectedPsqlExec(pgUser, pgHost, pgDatabase, pgPort, containerName)
           ]);
       var exit = 0;
       var message;
@@ -788,7 +841,7 @@ function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isH
     if (Utils.$$Array.notEmpty(schemaTableNames) && !schemaTableNames.some(function (table) {
             return table.table_name === InternalTable.Chains.table.tableName ? true : table.table_name === "event_sync_state";
           })) {
-      Js_exn.raiseError("Cannot run Envio migrations on PostgreSQL schema \"" + pgSchema + "\" because it contains non-Envio tables. Running migrations would delete all data in this schema.\n\nTo resolve this:\n1. If you want to use this schema, first backup any important data, then drop it with: \"pnpm envio local db-migrate down\"\n2. Or specify a different schema name by setting the \"ENVIO_PG_PUBLIC_SCHEMA\" environment variable\n3. Or manually drop the schema in your database if you're certain the data is not needed.");
+      Js_exn.raiseError("Cannot run Envio migrations on PostgreSQL schema \"" + pgSchema + "\" because it contains non-Envio tables. Running migrations would delete all data in this schema.\n\nTo resolve this:\n1. If you want to use this schema, first backup any important data, then drop it with: \"pnpm envio local db-migrate down\"\n2. Or specify a different schema name by setting the \"ENVIO_PG_SCHEMA\" environment variable\n3. Or manually drop the schema in your database if you're certain the data is not needed.");
     }
     if (sink !== undefined) {
       await sink.initialize(chainConfigs, entities, enums);
@@ -816,7 +869,8 @@ function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isH
                           numEventsProcessed: 0,
                           firstEventBlockNumber: undefined,
                           timestampCaughtUpToHeadOrEndblock: undefined,
-                          dynamicContracts: []
+                          dynamicContracts: [],
+                          sourceBlockNumber: 0
                         };
                 }),
             checkpointId: InternalTable.Checkpoints.initialCheckpointId,
@@ -923,7 +977,7 @@ function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isH
               recursive: true
             });
       }
-      var psqlExec = await getConnectedPsqlExec(pgUser, pgHost, pgDatabase, pgPort);
+      var psqlExec = await getConnectedPsqlExec(pgUser, pgHost, pgDatabase, pgPort, containerName);
       if (psqlExec.TAG !== "Ok") {
         return Logging.error("Failed to dump cache. " + psqlExec._0);
       }
@@ -968,14 +1022,23 @@ function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isH
                                       numEventsProcessed: rawInitialState.numEventsProcessed,
                                       firstEventBlockNumber: Caml_option.null_to_opt(rawInitialState.firstEventBlockNumber),
                                       timestampCaughtUpToHeadOrEndblock: Caml_option.null_to_opt(rawInitialState.timestampCaughtUpToHeadOrEndblock),
-                                      dynamicContracts: rawInitialState.dynamicContracts
+                                      dynamicContracts: rawInitialState.dynamicContracts,
+                                      sourceBlockNumber: rawInitialState.sourceBlockNumber
                                     };
                             }));
               }),
           sql.unsafe(InternalTable.Checkpoints.makeCommitedCheckpointIdQuery(pgSchema)),
           sql.unsafe(InternalTable.Checkpoints.makeGetReorgCheckpointsQuery(pgSchema))
         ]);
-    var checkpointId = match[2][0].id;
+    var checkpointId = BigInt(match[2][0].id);
+    var reorgCheckpoints = Belt_Array.map(match[3], (function (raw) {
+            return {
+                    id: BigInt(raw.id),
+                    chain_id: raw.chain_id,
+                    block_number: raw.block_number,
+                    block_hash: raw.block_hash
+                  };
+          }));
     if (sink !== undefined) {
       await sink.resume(checkpointId);
     }
@@ -984,34 +1047,12 @@ function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isH
             cache: match[0],
             chains: match[1],
             checkpointId: checkpointId,
-            reorgCheckpoints: match[3]
+            reorgCheckpoints: reorgCheckpoints
           };
   };
-  var executeUnsafe = function (query) {
-    return sql.unsafe(query);
-  };
-  var hasEntityHistoryRows = async function () {
-    var historyTables = await sql.unsafe("SELECT table_name FROM information_schema.tables \n       WHERE table_schema = '" + pgSchema + "' \n       AND table_name LIKE 'envio_history_%';");
-    if (Utils.$$Array.isEmpty(historyTables)) {
-      return false;
-    }
-    var checks = await Promise.all(Belt_Array.map(historyTables, (async function (table) {
-                try {
-                  var query = "SELECT EXISTS(SELECT 1 FROM \"" + pgSchema + "\".\"" + table.table_name + "\" LIMIT 1);";
-                  var result = await sql.unsafe(query);
-                  if (result.length !== 1) {
-                    return false;
-                  } else {
-                    return result[0].exists;
-                  }
-                }
-                catch (exn){
-                  return false;
-                }
-              })));
-    return Belt_Array.some(checks, (function (v) {
-                  return v;
-                }));
+  var reset = async function () {
+    var query = "DROP SCHEMA IF EXISTS \"" + pgSchema + "\" CASCADE;";
+    return await sql.unsafe(query);
   };
   var setChainMeta = function (chainsData) {
     return InternalTable.Chains.setMeta(sql, pgSchema, chainsData).then(function (param) {
@@ -1032,8 +1073,8 @@ function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isH
   };
   var getRollbackData = async function (entityConfig, rollbackTargetCheckpointId) {
     return await Promise.all([
-                sql.unsafe(makeGetRollbackRemovedIdsQuery(entityConfig, pgSchema), [rollbackTargetCheckpointId], {prepare: true}),
-                sql.unsafe(makeGetRollbackRestoredEntitiesQuery(entityConfig, pgSchema), [rollbackTargetCheckpointId], {prepare: true})
+                sql.unsafe(makeGetRollbackRemovedIdsQuery(entityConfig, pgSchema), [rollbackTargetCheckpointId.toString()], {prepare: true}),
+                sql.unsafe(makeGetRollbackRestoredEntitiesQuery(entityConfig, pgSchema), [rollbackTargetCheckpointId.toString()], {prepare: true})
               ]);
   };
   var writeBatchMethod = async function (batch, rawEvents, rollbackTargetCheckpointId, isInReorgThreshold, config, allEntities, updatedEffectsCache, updatedEntities) {
@@ -1041,7 +1082,7 @@ function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isH
     if (sink !== undefined) {
       var timerRef = Hrtime.makeTimer();
       sinkPromise = Caml_option.some(sink.writeBatch(batch, updatedEntities).then(function () {
-                  Prometheus.SinkWrite.increment(sink.name, Hrtime.intFromMillis(Hrtime.toMillis(Hrtime.timeSince(timerRef))));
+                  Prometheus.SinkWrite.increment(sink.name, Hrtime.toSecondsFloat(Hrtime.timeSince(timerRef)));
                 }).catch(function (exn) {
                 return exn;
               }));
@@ -1056,11 +1097,8 @@ function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isH
           resumeInitialState: resumeInitialState,
           loadByIdsOrThrow: loadByIdsOrThrow,
           loadByFieldOrThrow: loadByFieldOrThrow,
-          setOrThrow: setOrThrow$1,
-          setEffectCacheOrThrow: setEffectCacheOrThrow,
           dumpEffectCache: dumpEffectCache,
-          executeUnsafe: executeUnsafe,
-          hasEntityHistoryRows: hasEntityHistoryRows,
+          reset: reset,
           setChainMeta: setChainMeta,
           pruneStaleCheckpoints: pruneStaleCheckpoints,
           pruneStaleEntityHistory: pruneStaleEntityHistory,
@@ -1075,9 +1113,14 @@ var maxItemsPerQuery = 500;
 
 export {
   getCacheRowCountFnName ,
+  makeClient ,
   makeCreateIndexQuery ,
+  directionToSql ,
+  directionToIndexName ,
+  makeCreateCompositeIndexQuery ,
   makeCreateTableIndicesQuery ,
   makeCreateTableQuery ,
+  entityHistoryCache ,
   getEntityHistory ,
   makeInitializeTransaction ,
   makeLoadByIdQuery ,
@@ -1108,4 +1151,4 @@ export {
   makeGetRollbackRemovedIdsQuery ,
   make ,
 }
-/* pgErrorMessageSchema Not a pure module */
+/* entityHistoryCache Not a pure module */