envio 2.26.0-rc.0 → 2.26.0-rc.1

package/src/PgStorage.res.js

@@ -1,19 +1,28 @@
 // Generated by ReScript, PLEASE EDIT WITH CARE
 'use strict';
 
+var Fs = require("fs");
+var Path = require("path");
 var $$Array = require("rescript/lib/js/array.js");
 var Table = require("./db/Table.res.js");
 var Utils = require("./Utils.res.js");
 var Js_exn = require("rescript/lib/js/js_exn.js");
 var Schema = require("./db/Schema.res.js");
+var Js_dict = require("rescript/lib/js/js_dict.js");
+var Logging = require("./Logging.res.js");
+var $$Promise = require("./bindings/Promise.res.js");
+var Internal = require("./Internal.res.js");
 var Belt_Array = require("rescript/lib/js/belt_Array.js");
-var Caml_int32 = require("rescript/lib/js/caml_int32.js");
+var Caml_array = require("rescript/lib/js/caml_array.js");
+var Belt_Option = require("rescript/lib/js/belt_Option.js");
 var Caml_option = require("rescript/lib/js/caml_option.js");
 var Persistence = require("./Persistence.res.js");
+var Child_process = require("child_process");
+var Caml_exceptions = require("rescript/lib/js/caml_exceptions.js");
 var S$RescriptSchema = require("rescript-schema/src/S.res.js");
 var Caml_js_exceptions = require("rescript/lib/js/caml_js_exceptions.js");
 
-function makeCreateIndexSql(tableName, indexFields, pgSchema) {
+function makeCreateIndexQuery(tableName, indexFields, pgSchema) {
   var indexName = tableName + "_" + indexFields.join("_");
   var index = Belt_Array.map(indexFields, (function (idx) {
             return "\"" + idx + "\"";
@@ -21,20 +30,20 @@ function makeCreateIndexSql(tableName, indexFields, pgSchema) {
   return "CREATE INDEX IF NOT EXISTS \"" + indexName + "\" ON \"" + pgSchema + "\".\"" + tableName + "\"(" + index + ");";
 }
 
-function makeCreateTableIndicesSql(table, pgSchema) {
+function makeCreateTableIndicesQuery(table, pgSchema) {
   var tableName = table.tableName;
   var createIndex = function (indexField) {
-    return makeCreateIndexSql(tableName, [indexField], pgSchema);
+    return makeCreateIndexQuery(tableName, [indexField], pgSchema);
   };
   var createCompositeIndex = function (indexFields) {
-    return makeCreateIndexSql(tableName, indexFields, pgSchema);
+    return makeCreateIndexQuery(tableName, indexFields, pgSchema);
   };
   var singleIndices = Table.getSingleIndices(table);
   var compositeIndices = Table.getCompositeIndices(table);
   return Belt_Array.map(singleIndices, createIndex).join("\n") + Belt_Array.map(compositeIndices, createCompositeIndex).join("\n");
 }
 
-function makeCreateTableSql(table, pgSchema) {
+function makeCreateTableQuery(table, pgSchema) {
   var fieldsMapped = Belt_Array.map(Table.getFields(table), (function (field) {
             var defaultValue = field.defaultValue;
             var fieldType = field.fieldType;
@@ -58,11 +67,11 @@ function makeCreateTableSql(table, pgSchema) {
         ) + ");";
 }
 
-function makeInitializeTransaction(pgSchema, pgUser, generalTablesOpt, entitiesOpt, enumsOpt, cleanRunOpt) {
+function makeInitializeTransaction(pgSchema, pgUser, generalTablesOpt, entitiesOpt, enumsOpt, isEmptyPgSchemaOpt) {
   var generalTables = generalTablesOpt !== undefined ? generalTablesOpt : [];
   var entities = entitiesOpt !== undefined ? entitiesOpt : [];
   var enums = enumsOpt !== undefined ? enumsOpt : [];
-  var cleanRun = cleanRunOpt !== undefined ? cleanRunOpt : false;
+  var isEmptyPgSchema = isEmptyPgSchemaOpt !== undefined ? isEmptyPgSchemaOpt : false;
   var allTables = $$Array.copy(generalTables);
   var allEntityTables = [];
   entities.forEach(function (entity) {
@@ -73,22 +82,20 @@ function makeInitializeTransaction(pgSchema, pgUser, generalTablesOpt, entitiesO
   var derivedSchema = Schema.make(allEntityTables);
   var query = {
     contents: (
-      cleanRun ? "DROP SCHEMA IF EXISTS \"" + pgSchema + "\" CASCADE;\nCREATE SCHEMA \"" + pgSchema + "\";" : "CREATE SCHEMA IF NOT EXISTS \"" + pgSchema + "\";"
-    ) + ("GRANT ALL ON SCHEMA \"" + pgSchema + "\" TO " + pgUser + ";\nGRANT ALL ON SCHEMA \"" + pgSchema + "\" TO public;")
+      isEmptyPgSchema && pgSchema === "public" ? "" : "DROP SCHEMA IF EXISTS \"" + pgSchema + "\" CASCADE;\nCREATE SCHEMA \"" + pgSchema + "\";\n"
+    ) + ("GRANT ALL ON SCHEMA \"" + pgSchema + "\" TO \"" + pgUser + "\";\nGRANT ALL ON SCHEMA \"" + pgSchema + "\" TO public;")
   };
   enums.forEach(function (enumConfig) {
         var enumCreateQuery = "CREATE TYPE \"" + pgSchema + "\"." + enumConfig.name + " AS ENUM(" + enumConfig.variants.map(function (v) {
                 return "'" + v + "'";
               }).join(", ") + ");";
-        query.contents = query.contents + "\n" + (
-          cleanRun ? enumCreateQuery : "IF NOT EXISTS (\n  SELECT 1 FROM pg_type \n  WHERE typname = '" + enumConfig.name.toLowerCase() + "' \n  AND typnamespace = (SELECT oid FROM pg_namespace WHERE nspname = '" + pgSchema + "')\n) THEN \n    " + enumCreateQuery + "\nEND IF;"
-        );
+        query.contents = query.contents + "\n" + enumCreateQuery;
       });
   allTables.forEach(function (table) {
-        query.contents = query.contents + "\n" + makeCreateTableSql(table, pgSchema);
+        query.contents = query.contents + "\n" + makeCreateTableQuery(table, pgSchema);
       });
   allTables.forEach(function (table) {
-        var indices = makeCreateTableIndicesSql(table, pgSchema);
+        var indices = makeCreateTableIndicesQuery(table, pgSchema);
         if (indices !== "") {
           query.contents = query.contents + "\n" + indices;
           return ;
@@ -102,22 +109,27 @@ function makeInitializeTransaction(pgSchema, pgUser, generalTablesOpt, entitiesO
         functionsQuery.contents = functionsQuery.contents + "\n" + entity.entityHistory.createInsertFnQuery;
         Table.getDerivedFromFields(entity.table).forEach(function (derivedFromField) {
               var indexField = Utils.unwrapResultExn(Schema.getDerivedFromFieldName(derivedSchema, derivedFromField));
-              query.contents = query.contents + "\n" + makeCreateIndexSql(derivedFromField.derivedFromEntity, [indexField], pgSchema);
+              query.contents = query.contents + "\n" + makeCreateIndexQuery(derivedFromField.derivedFromEntity, [indexField], pgSchema);
             });
       });
-  return [cleanRun || Utils.$$Array.isEmpty(enums) ? query.contents : "DO $$ BEGIN " + query.contents + " END $$;"].concat(functionsQuery.contents !== "" ? [functionsQuery.contents] : []);
+  functionsQuery.contents = functionsQuery.contents + "\n" + ("CREATE OR REPLACE FUNCTION get_cache_row_count(table_name text) \n   RETURNS integer AS $$\n   DECLARE\n     result integer;\n   BEGIN\n     EXECUTE format('SELECT COUNT(*) FROM \"" + pgSchema + "\".%I', table_name) INTO result;\n     RETURN result;\n   END;\n   $$ LANGUAGE plpgsql;");
+  return [query.contents].concat(functionsQuery.contents !== "" ? [functionsQuery.contents] : []);
 }
 
-function makeLoadByIdSql(pgSchema, tableName) {
+function makeLoadByIdQuery(pgSchema, tableName) {
   return "SELECT * FROM \"" + pgSchema + "\".\"" + tableName + "\" WHERE id = $1 LIMIT 1;";
 }
 
-function makeLoadByIdsSql(pgSchema, tableName) {
+function makeLoadByFieldQuery(pgSchema, tableName, fieldName, operator) {
+  return "SELECT * FROM \"" + pgSchema + "\".\"" + tableName + "\" WHERE \"" + fieldName + "\" " + operator + " $1;";
+}
+
+function makeLoadByIdsQuery(pgSchema, tableName) {
   return "SELECT * FROM \"" + pgSchema + "\".\"" + tableName + "\" WHERE id = ANY($1::text[]);";
 }
 
-function makeInsertUnnestSetSql(pgSchema, table, itemSchema, isRawEvents) {
-  var match = Table.toSqlParams(table, itemSchema);
+function makeInsertUnnestSetQuery(pgSchema, table, itemSchema, isRawEvents) {
+  var match = Table.toSqlParams(table, itemSchema, pgSchema);
   var quotedNonPrimaryFieldNames = match.quotedNonPrimaryFieldNames;
   var primaryKeyFieldNames = Table.getPrimaryKeyFieldNames(table);
   return "INSERT INTO \"" + pgSchema + "\".\"" + table.tableName + "\" (" + match.quotedFieldNames.join(", ") + ")\nSELECT * FROM unnest(" + match.arrayFieldTypes.map(function (arrayFieldType, idx) {
@@ -133,8 +145,8 @@ function makeInsertUnnestSetSql(pgSchema, table, itemSchema, isRawEvents) {
         ) + ";";
 }
 
-function makeInsertValuesSetSql(pgSchema, table, itemSchema, itemsCount) {
-  var match = Table.toSqlParams(table, itemSchema);
+function makeInsertValuesSetQuery(pgSchema, table, itemSchema, itemsCount) {
+  var match = Table.toSqlParams(table, itemSchema, pgSchema);
   var quotedNonPrimaryFieldNames = match.quotedNonPrimaryFieldNames;
   var quotedFieldNames = match.quotedFieldNames;
   var primaryKeyFieldNames = Table.getPrimaryKeyFieldNames(table);
@@ -166,18 +178,20 @@ function makeInsertValuesSetSql(pgSchema, table, itemSchema, itemsCount) {
 
 var rawEventsTableName = "raw_events";
 
+var eventSyncStateTableName = "event_sync_state";
+
 function makeTableBatchSetQuery(pgSchema, table, itemSchema) {
-  var match = Table.toSqlParams(table, itemSchema);
+  var match = Table.toSqlParams(table, itemSchema, pgSchema);
   var isRawEvents = table.tableName === rawEventsTableName;
   if (isRawEvents || !match.hasArrayField) {
     return {
-            sql: makeInsertUnnestSetSql(pgSchema, table, itemSchema, isRawEvents),
+            query: makeInsertUnnestSetQuery(pgSchema, table, itemSchema, isRawEvents),
             convertOrThrow: S$RescriptSchema.compile(S$RescriptSchema.unnest(match.dbSchema), "Output", "Input", "Sync", false),
             isInsertValues: false
           };
   } else {
     return {
-            sql: makeInsertValuesSetSql(pgSchema, table, itemSchema, 500),
+            query: makeInsertValuesSetQuery(pgSchema, table, itemSchema, 500),
             convertOrThrow: S$RescriptSchema.compile(S$RescriptSchema.preprocess(S$RescriptSchema.unnest(itemSchema), (function (param) {
                         return {
                                 s: (function (prim) {
@@ -201,6 +215,24 @@ function chunkArray(arr, chunkSize) {
   return chunks;
 }
 
+function removeInvalidUtf8InPlace(entities) {
+  entities.forEach(function (item) {
+        Utils.Dict.forEachWithKey(item, (function (key, value) {
+                if (typeof value === "string") {
+                  item[key] = value.replaceAll("\x00", "");
+                  return ;
+                }
+                
+              }));
+      });
+}
+
+var pgEncodingErrorSchema = S$RescriptSchema.object(function (s) {
+      s.tag("message", "invalid byte sequence for encoding \"UTF8\": 0x00");
+    });
+
+var PgEncodingError = /* @__PURE__ */Caml_exceptions.create("PgStorage.PgEncodingError");
+
 var setQueryCache = new WeakMap();
 
 async function setOrThrow(sql, items, table, itemSchema, pgSchema) {
@@ -208,30 +240,24 @@ async function setOrThrow(sql, items, table, itemSchema, pgSchema) {
     return ;
   }
   var cached = setQueryCache.get(table);
-  var query;
+  var data;
   if (cached !== undefined) {
-    query = Caml_option.valFromOption(cached);
+    data = Caml_option.valFromOption(cached);
   } else {
     var newQuery = makeTableBatchSetQuery(pgSchema, table, itemSchema);
     setQueryCache.set(table, newQuery);
-    query = newQuery;
+    data = newQuery;
   }
-  var sqlQuery = query.sql;
   try {
-    var payload = query.convertOrThrow(items);
-    if (!query.isInsertValues) {
-      return await sql.unsafe(sqlQuery, payload, {prepare: true});
+    if (!data.isInsertValues) {
+      return await sql.unsafe(data.query, data.convertOrThrow(items), {prepare: true});
     }
-    var match = itemSchema.t;
-    var fieldsCount;
-    fieldsCount = typeof match !== "object" || match.TAG !== "object" ? Js_exn.raiseError("Expected an object schema for table") : match.items.length;
-    var maxChunkSize = Math.imul(500, fieldsCount);
-    var chunks = chunkArray(payload, maxChunkSize);
+    var chunks = chunkArray(items, 500);
     var responses = [];
     chunks.forEach(function (chunk) {
           var chunkSize = chunk.length;
-          var isFullChunk = chunkSize === maxChunkSize;
-          var response = sql.unsafe(isFullChunk ? sqlQuery : makeInsertValuesSetSql(pgSchema, table, itemSchema, Caml_int32.div(chunkSize, fieldsCount)), chunk, {prepare: true});
+          var isFullChunk = chunkSize === 500;
+          var response = sql.unsafe(isFullChunk ? data.query : makeInsertValuesSetQuery(pgSchema, table, itemSchema, chunkSize), data.convertOrThrow(chunk), {prepare: true});
           responses.push(response);
         });
     await Promise.all(responses);
@@ -250,34 +276,130 @@ async function setOrThrow(sql, items, table, itemSchema, pgSchema) {
     throw {
           RE_EXN_ID: Persistence.StorageError,
           message: "Failed to insert items into table \"" + table.tableName + "\"",
-          reason: exn,
+          reason: Internal.prettifyExn(exn),
           Error: new Error()
         };
   }
 }
 
-function make(sql, pgSchema, pgUser) {
+function setEntityHistoryOrThrow(sql, entityHistory, rows, shouldCopyCurrentEntity, shouldRemoveInvalidUtf8Opt) {
+  var shouldRemoveInvalidUtf8 = shouldRemoveInvalidUtf8Opt !== undefined ? shouldRemoveInvalidUtf8Opt : false;
+  return Promise.all(Belt_Array.map(rows, (function (historyRow) {
+                    var row = S$RescriptSchema.reverseConvertToJsonOrThrow(historyRow, entityHistory.schema);
+                    if (shouldRemoveInvalidUtf8) {
+                      removeInvalidUtf8InPlace([row]);
+                    }
+                    return entityHistory.insertFn(sql, row, shouldCopyCurrentEntity !== undefined ? shouldCopyCurrentEntity : !Belt_Option.getWithDefault(historyRow.containsRollbackDiffChange, false));
+                  })));
+}
+
+function makeSchemaTableNamesQuery(pgSchema) {
+  return "SELECT table_name FROM information_schema.tables WHERE table_schema = '" + pgSchema + "';";
+}
+
+var cacheTablePrefix = "envio_effect_";
+
+var cacheTablePrefixLength = cacheTablePrefix.length;
+
+function makeSchemaCacheTableInfoQuery(pgSchema) {
+  return "SELECT \n    t.table_name,\n    get_cache_row_count(t.table_name) as count\n   FROM information_schema.tables t\n   WHERE t.table_schema = '" + pgSchema + "' \n   AND t.table_name LIKE '" + cacheTablePrefix + "%';";
+}
+
+var psqlExecState = {
+  contents: "Unknown"
+};
+
+async function getConnectedPsqlExec(pgUser, pgHost, pgDatabase, pgPort) {
+  var promise = psqlExecState.contents;
+  if (typeof promise === "object") {
+    if (promise.TAG === "Pending") {
+      return await promise._0;
+    } else {
+      return promise._0;
+    }
+  }
+  var promise$1 = new Promise((function (resolve, _reject) {
+          var binary = "psql";
+          Child_process.exec(binary + " --version", (function (error, param, param$1) {
+                  if (error === null) {
+                    return resolve({
+                                TAG: "Ok",
+                                _0: binary + " -h " + pgHost + " -p " + pgPort.toString() + " -U " + pgUser + " -d " + pgDatabase
+                              });
+                  }
+                  var binary$1 = "docker-compose exec -T -u " + pgUser + " envio-postgres psql";
+                  Child_process.exec(binary$1 + " --version", (function (error, param, param$1) {
+                          if (error === null) {
+                            return resolve({
+                                        TAG: "Ok",
+                                        _0: binary$1 + " -h " + pgHost + " -p " + (5432).toString() + " -U " + pgUser + " -d " + pgDatabase
+                                      });
+                          } else {
+                            return resolve({
+                                        TAG: "Error",
+                                        _0: "Please check if \"psql\" binary is installed or docker-compose is running for the local indexer."
+                                      });
+                          }
+                        }));
+                }));
+        }));
+  psqlExecState.contents = {
+    TAG: "Pending",
+    _0: promise$1
+  };
+  var result = await promise$1;
+  psqlExecState.contents = {
+    TAG: "Resolved",
+    _0: result
+  };
+  return result;
+}
+
+function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, onInitialize, onNewTables) {
+  var psqlExecOptions_env = Js_dict.fromArray([
+        [
+          "PGPASSWORD",
+          pgPassword
+        ],
+        [
+          "PATH",
+          process.env.PATH
+        ]
+      ]);
+  var psqlExecOptions = {
+    env: psqlExecOptions_env
+  };
+  var cacheDirPath = Path.resolve("..", ".envio", "cache");
   var isInitialized = async function () {
-    var schemas = await sql.unsafe("SELECT schema_name FROM information_schema.schemata WHERE schema_name = '" + pgSchema + "';");
-    return Utils.$$Array.notEmpty(schemas);
+    var envioTables = await sql.unsafe("SELECT table_schema FROM information_schema.tables WHERE table_schema = '" + pgSchema + "' AND table_name = '" + eventSyncStateTableName + "';");
+    return Utils.$$Array.notEmpty(envioTables);
   };
-  var initialize = async function (entitiesOpt, generalTablesOpt, enumsOpt, cleanRunOpt) {
+  var initialize = async function (entitiesOpt, generalTablesOpt, enumsOpt) {
     var entities = entitiesOpt !== undefined ? entitiesOpt : [];
     var generalTables = generalTablesOpt !== undefined ? generalTablesOpt : [];
     var enums = enumsOpt !== undefined ? enumsOpt : [];
-    var cleanRun = cleanRunOpt !== undefined ? cleanRunOpt : false;
-    var queries = makeInitializeTransaction(pgSchema, pgUser, generalTables, entities, enums, cleanRun);
+    var schemaTableNames = await sql.unsafe(makeSchemaTableNamesQuery(pgSchema));
+    if (Utils.$$Array.notEmpty(schemaTableNames) && !schemaTableNames.some(function (table) {
+            return table.table_name === eventSyncStateTableName;
+          })) {
+      Js_exn.raiseError("Cannot run Envio migrations on PostgreSQL schema \"" + pgSchema + "\" because it contains non-Envio tables. Running migrations would delete all data in this schema.\n\nTo resolve this:\n1. If you want to use this schema, first backup any important data, then drop it with: \"pnpm envio local db-migrate down\"\n2. Or specify a different schema name by setting the \"ENVIO_PG_PUBLIC_SCHEMA\" environment variable\n3. Or manually drop the schema in your database if you're certain the data is not needed.");
+    }
+    var queries = makeInitializeTransaction(pgSchema, pgUser, generalTables, entities, enums, Utils.$$Array.isEmpty(schemaTableNames));
     await sql.begin(function (sql) {
           return queries.map(function (query) {
                       return sql.unsafe(query);
                     });
         });
+    if (onInitialize !== undefined) {
+      return await onInitialize();
+    }
+    
   };
   var loadByIdsOrThrow = async function (ids, table, rowsSchema) {
     var rows;
     try {
       rows = await (
-        ids.length !== 1 ? sql.unsafe(makeLoadByIdsSql(pgSchema, table.tableName), [ids], {prepare: true}) : sql.unsafe(makeLoadByIdSql(pgSchema, table.tableName), ids, {prepare: true})
+        ids.length !== 1 ? sql.unsafe(makeLoadByIdsQuery(pgSchema, table.tableName), [ids], {prepare: true}) : sql.unsafe(makeLoadByIdQuery(pgSchema, table.tableName), ids, {prepare: true})
       );
     }
     catch (raw_exn){
@@ -302,32 +424,239 @@ function make(sql, pgSchema, pgUser) {
           };
     }
   };
+  var loadByFieldOrThrow = async function (fieldName, fieldSchema, fieldValue, operator, table, rowsSchema) {
+    var params;
+    try {
+      params = [S$RescriptSchema.reverseConvertToJsonOrThrow(fieldValue, fieldSchema)];
+    }
+    catch (raw_exn){
+      var exn = Caml_js_exceptions.internalToOCamlException(raw_exn);
+      throw {
+            RE_EXN_ID: Persistence.StorageError,
+            message: "Failed loading \"" + table.tableName + "\" from storage by field \"" + fieldName + "\". Couldn't serialize provided value.",
+            reason: exn,
+            Error: new Error()
+          };
+    }
+    var rows;
+    try {
+      rows = await sql.unsafe(makeLoadByFieldQuery(pgSchema, table.tableName, fieldName, operator), params, {prepare: true});
+    }
+    catch (raw_exn$1){
+      var exn$1 = Caml_js_exceptions.internalToOCamlException(raw_exn$1);
+      throw {
+            RE_EXN_ID: Persistence.StorageError,
+            message: "Failed loading \"" + table.tableName + "\" from storage by field \"" + fieldName + "\"",
+            reason: exn$1,
+            Error: new Error()
+          };
+    }
+    try {
+      return S$RescriptSchema.parseOrThrow(rows, rowsSchema);
+    }
+    catch (raw_exn$2){
+      var exn$2 = Caml_js_exceptions.internalToOCamlException(raw_exn$2);
+      throw {
+            RE_EXN_ID: Persistence.StorageError,
+            message: "Failed to parse \"" + table.tableName + "\" loaded from storage by ids",
+            reason: exn$2,
+            Error: new Error()
+          };
+    }
+  };
   var setOrThrow$1 = function (items, table, itemSchema) {
     return setOrThrow(sql, items, table, itemSchema, pgSchema);
   };
+  var setEffectCacheOrThrow = async function (effectName, ids, outputs, outputSchema, initialize) {
+    var table = Table.mkTable(cacheTablePrefix + effectName, [], [
+          Table.mkField("id", "TEXT", S$RescriptSchema.string, undefined, undefined, undefined, true, undefined, undefined),
+          Table.mkField("output", "JSONB", S$RescriptSchema.json(false), undefined, undefined, undefined, undefined, undefined, undefined)
+        ]);
+    if (initialize) {
+      await sql.unsafe(makeCreateTableQuery(table, pgSchema));
+      if (onNewTables !== undefined) {
+        await onNewTables([table.tableName]);
+      }
+      
+    }
+    var items = [];
+    for(var idx = 0 ,idx_finish = outputs.length; idx < idx_finish; ++idx){
+      items.push({
+            id: Caml_array.get(ids, idx),
+            output: Caml_array.get(outputs, idx)
+          });
+    }
+    return await setOrThrow$1(items, table, S$RescriptSchema.schema(function (s) {
+                    return {
+                            id: s.m(S$RescriptSchema.string),
+                            output: s.m(outputSchema)
+                          };
+                  }));
+  };
+  var dumpEffectCache = async function () {
+    try {
+      var cacheTableInfo = (await sql.unsafe(makeSchemaCacheTableInfoQuery(pgSchema))).filter(function (i) {
+            return i.count > 0;
+          });
+      if (!Utils.$$Array.notEmpty(cacheTableInfo)) {
+        return ;
+      }
+      try {
+        await Fs.promises.access(cacheDirPath);
+      }
+      catch (exn){
+        await Fs.promises.mkdir(cacheDirPath, {
+              recursive: true
+            });
+      }
+      var psqlExec = await getConnectedPsqlExec(pgUser, pgHost, pgDatabase, pgPort);
+      if (psqlExec.TAG !== "Ok") {
+        return Logging.error("Failed to dump cache. " + psqlExec._0);
+      }
+      var psqlExec$1 = psqlExec._0;
+      Logging.info("Dumping cache: " + cacheTableInfo.map(function (param) {
+                  return param.table_name + " (" + String(param.count) + " rows)";
+                }).join(", "));
+      var promises = cacheTableInfo.map(async function (param) {
+            var tableName = param.table_name;
+            var cacheName = tableName.slice(cacheTablePrefixLength);
+            var outputFile = Path.join(cacheDirPath, cacheName + ".tsv");
+            var command = psqlExec$1 + " -c 'COPY \"" + pgSchema + "\".\"" + tableName + "\" TO STDOUT WITH (FORMAT text, HEADER);' > " + outputFile;
+            return new Promise((function (resolve, reject) {
+                          Child_process.exec(command, psqlExecOptions, (function (error, stdout, param) {
+                                  if (error === null) {
+                                    return resolve(stdout);
+                                  } else {
+                                    return reject(error);
+                                  }
+                                }));
+                        }));
+          });
+      await Promise.all(promises);
+      return Logging.info("Successfully dumped cache to " + cacheDirPath);
+    }
+    catch (raw_exn){
+      var exn$1 = Caml_js_exceptions.internalToOCamlException(raw_exn);
+      return Logging.errorWithExn(Internal.prettifyExn(exn$1), "Failed to dump cache.");
+    }
+  };
+  var restoreEffectCache = async function (withUpload) {
+    if (withUpload) {
+      var nothingToUploadErrorMessage = "Nothing to upload.";
+      var match = await Promise.all([
+            $$Promise.$$catch(Fs.promises.readdir(cacheDirPath).then(function (e) {
+                      return {
+                              TAG: "Ok",
+                              _0: e
+                            };
+                    }), (function (param) {
+                    return Promise.resolve({
+                                TAG: "Error",
+                                _0: nothingToUploadErrorMessage
+                              });
+                  })),
+            getConnectedPsqlExec(pgUser, pgHost, pgDatabase, pgPort)
+          ]);
+      var exit = 0;
+      var message;
+      var entries = match[0];
+      if (entries.TAG === "Ok") {
+        var psqlExec = match[1];
+        if (psqlExec.TAG === "Ok") {
+          var psqlExec$1 = psqlExec._0;
+          var cacheFiles = entries._0.filter(function (entry) {
+                return entry.endsWith(".tsv");
+              });
+          await Promise.all(cacheFiles.map(function (entry) {
+                    var cacheName = entry.slice(0, -4);
+                    var tableName = cacheTablePrefix + cacheName;
+                    var table = Table.mkTable(tableName, [], [
+                          Table.mkField("id", "TEXT", S$RescriptSchema.string, undefined, undefined, undefined, true, undefined, undefined),
+                          Table.mkField("output", "JSONB", S$RescriptSchema.json(false), undefined, undefined, undefined, undefined, undefined, undefined)
+                        ]);
+                    return sql.unsafe(makeCreateTableQuery(table, pgSchema)).then(function () {
+                                var inputFile = Path.join(cacheDirPath, entry);
+                                var command = psqlExec$1 + " -c 'COPY \"" + pgSchema + "\".\"" + tableName + "\" FROM STDIN WITH (FORMAT text, HEADER);' < " + inputFile;
+                                return new Promise((function (resolve, reject) {
+                                              Child_process.exec(command, psqlExecOptions, (function (error, stdout, param) {
+                                                      if (error === null) {
+                                                        return resolve(stdout);
+                                                      } else {
+                                                        return reject(error);
+                                                      }
+                                                    }));
+                                            }));
+                              });
+                  }));
+          Logging.info("Successfully uploaded cache.");
+        } else {
+          message = match[1]._0;
+          exit = 1;
+        }
+      } else {
+        message = entries._0;
+        exit = 1;
+      }
+      if (exit === 1) {
+        if (message === nothingToUploadErrorMessage) {
+          Logging.info("No cache found to upload.");
+        } else {
+          Logging.error("Failed to upload cache, continuing without it. " + message);
+        }
+      }
+      
+    }
+    var cacheTableInfo = await sql.unsafe(makeSchemaCacheTableInfoQuery(pgSchema));
+    if (withUpload && Utils.$$Array.notEmpty(cacheTableInfo) && onNewTables !== undefined) {
+      await onNewTables(cacheTableInfo.map(function (info) {
+                return info.table_name;
+              }));
+    }
+    return cacheTableInfo.map(function (info) {
+                return {
+                        effectName: info.table_name.slice(cacheTablePrefixLength),
+                        count: info.count
+                      };
+              });
+  };
   return {
           isInitialized: isInitialized,
           initialize: initialize,
           loadByIdsOrThrow: loadByIdsOrThrow,
-          setOrThrow: setOrThrow$1
+          loadByFieldOrThrow: loadByFieldOrThrow,
+          setOrThrow: setOrThrow$1,
+          setEffectCacheOrThrow: setEffectCacheOrThrow,
+          dumpEffectCache: dumpEffectCache,
+          restoreEffectCache: restoreEffectCache
         };
 }
 
 var maxItemsPerQuery = 500;
 
-exports.makeCreateIndexSql = makeCreateIndexSql;
-exports.makeCreateTableIndicesSql = makeCreateTableIndicesSql;
-exports.makeCreateTableSql = makeCreateTableSql;
+exports.makeCreateIndexQuery = makeCreateIndexQuery;
+exports.makeCreateTableIndicesQuery = makeCreateTableIndicesQuery;
+exports.makeCreateTableQuery = makeCreateTableQuery;
 exports.makeInitializeTransaction = makeInitializeTransaction;
-exports.makeLoadByIdSql = makeLoadByIdSql;
-exports.makeLoadByIdsSql = makeLoadByIdsSql;
-exports.makeInsertUnnestSetSql = makeInsertUnnestSetSql;
-exports.makeInsertValuesSetSql = makeInsertValuesSetSql;
+exports.makeLoadByIdQuery = makeLoadByIdQuery;
+exports.makeLoadByFieldQuery = makeLoadByFieldQuery;
+exports.makeLoadByIdsQuery = makeLoadByIdsQuery;
+exports.makeInsertUnnestSetQuery = makeInsertUnnestSetQuery;
+exports.makeInsertValuesSetQuery = makeInsertValuesSetQuery;
 exports.rawEventsTableName = rawEventsTableName;
+exports.eventSyncStateTableName = eventSyncStateTableName;
 exports.maxItemsPerQuery = maxItemsPerQuery;
 exports.makeTableBatchSetQuery = makeTableBatchSetQuery;
 exports.chunkArray = chunkArray;
+exports.removeInvalidUtf8InPlace = removeInvalidUtf8InPlace;
+exports.pgEncodingErrorSchema = pgEncodingErrorSchema;
+exports.PgEncodingError = PgEncodingError;
 exports.setQueryCache = setQueryCache;
 exports.setOrThrow = setOrThrow;
+exports.setEntityHistoryOrThrow = setEntityHistoryOrThrow;
+exports.makeSchemaTableNamesQuery = makeSchemaTableNamesQuery;
+exports.cacheTablePrefix = cacheTablePrefix;
+exports.cacheTablePrefixLength = cacheTablePrefixLength;
+exports.makeSchemaCacheTableInfoQuery = makeSchemaCacheTableInfoQuery;
+exports.getConnectedPsqlExec = getConnectedPsqlExec;
 exports.make = make;
-/* setQueryCache Not a pure module */
+/* pgEncodingErrorSchema Not a pure module */

package/src/Prometheus.res

@@ -590,3 +590,15 @@ module EffectCallsCount = {
     gauge->SafeGauge.handleInt(~labels=effectName, ~value=callsCount)
   }
 }
+
+module EffectCacheCount = {
+  let gauge = SafeGauge.makeOrThrow(
+    ~name="envio_effect_cache_count",
+    ~help="The number of items in the effect cache.",
+    ~labelSchema=effectLabelsSchema,
+  )
+
+  let set = (~count, ~effectName) => {
+    gauge->SafeGauge.handleInt(~labels=effectName, ~value=count)
+  }
+}

package/src/Prometheus.res.js

@@ -700,6 +700,17 @@ var EffectCallsCount = {
   set: set$21
 };
 
+var gauge$22 = makeOrThrow$1("envio_effect_cache_count", "The number of items in the effect cache.", effectLabelsSchema);
+
+function set$22(count, effectName) {
+  handleInt$1(gauge$22, effectName, count);
+}
+
+var EffectCacheCount = {
+  gauge: gauge$22,
+  set: set$22
+};
+
 exports.loadEntitiesDurationCounter = loadEntitiesDurationCounter;
 exports.eventRouterDurationCounter = eventRouterDurationCounter;
 exports.executeBatchDurationCounter = executeBatchDurationCounter;
@@ -747,4 +758,5 @@ exports.ProgressBlockNumber = ProgressBlockNumber;
 exports.ProgressEventsCount = ProgressEventsCount;
 exports.effectLabelsSchema = effectLabelsSchema;
 exports.EffectCallsCount = EffectCallsCount;
+exports.EffectCacheCount = EffectCacheCount;
 /* loadEntitiesDurationCounter Not a pure module */