envio 2.25.2 → 2.26.0-alpha.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "envio",
-  "version": "v2.25.2",
+  "version": "v2.26.0-alpha.2",
   "description": "A latency and sync speed optimized, developer friendly blockchain data indexer.",
   "bin": "./bin.js",
   "main": "./index.js",
@@ -25,10 +25,10 @@
   },
   "homepage": "https://envio.dev",
   "optionalDependencies": {
-    "envio-linux-x64": "v2.25.2",
-    "envio-linux-arm64": "v2.25.2",
-    "envio-darwin-x64": "v2.25.2",
-    "envio-darwin-arm64": "v2.25.2"
+    "envio-linux-x64": "v2.26.0-alpha.2",
+    "envio-linux-arm64": "v2.26.0-alpha.2",
+    "envio-darwin-x64": "v2.26.0-alpha.2",
+    "envio-darwin-arm64": "v2.26.0-alpha.2"
   },
   "dependencies": {
     "@envio-dev/hypersync-client": "0.6.5",

package/src/Envio.res

@@ -50,6 +50,7 @@ let experimental_createEffect = (
       >
     ),
     callsCount: 0,
+    output: options.output,
     cache: options.cache->Belt.Option.getWithDefault(false),
   }->(Utils.magic: Internal.effect => effect<'input, 'output>)
 }

package/src/Envio.res.js

@@ -10,6 +10,7 @@ function experimental_createEffect(options, handler) {
           name: options.name,
           handler: handler,
           cache: Belt_Option.getWithDefault(options.cache, false),
+          output: options.output,
           callsCount: 0
         };
 }

package/src/Internal.res

@@ -198,6 +198,7 @@ type effect = {
   name: string,
   handler: effectArgs => promise<effectOutput>,
   cache: bool,
+  output: S.t<effectOutput>,
   mutable callsCount: int,
 }
 

package/src/Persistence.res

@@ -5,14 +5,12 @@
 // Currently there are quite many code spread across
 // DbFunctions, Db, Migrations, InMemoryStore modules which use codegen code directly.
 
-// The type reflects an effect cache table in the db
+// The type reflects an cache table in the db
 // It might be present even if the effect is not used in the application
-type effectCache = {
-  name: string,
+type effectCacheRecord = {
+  effectName: string,
   // Number of rows in the table
-  mutable size: int,
-  // Lazily attached table definition when effect is used in the application
-  mutable table: option<Table.table>,
+  mutable count: int,
 }
 
 type operator = [#">" | #"="]
@@ -28,7 +26,6 @@ type storage = {
     ~generalTables: array<Table.table>=?,
     ~enums: array<Internal.enumConfig<Internal.enum>>=?,
   ) => promise<unit>,
-  loadEffectCaches: unit => promise<array<effectCache>>,
   @raises("StorageError")
   loadByIdsOrThrow: 'item. (
     ~ids: array<string>,
@@ -50,6 +47,16 @@ type storage = {
     ~table: Table.table,
     ~itemSchema: S.t<'item>,
   ) => promise<unit>,
+  @raises("StorageError")
+  setEffectCacheOrThrow: (
+    ~effectName: string,
+    ~ids: array<string>,
+    ~outputs: array<Internal.effectOutput>,
+    ~outputSchema: S.t<Internal.effectOutput>,
+    ~initialize: bool,
+  ) => promise<unit>,
+  dumpEffectCache: unit => promise<unit>,
+  restoreEffectCache: (~withUpload: bool) => promise<array<effectCacheRecord>>,
 }
 
 exception StorageError({message: string, reason: exn})
@@ -57,7 +64,7 @@ exception StorageError({message: string, reason: exn})
 type storageStatus =
   | Unknown
   | Initializing(promise<unit>)
-  | Ready({cleanRun: bool, effectCaches: dict<effectCache>})
+  | Ready({cleanRun: bool, cache: dict<effectCacheRecord>})
 
 type t = {
   userEntities: array<Internal.entityConfig>,
@@ -66,7 +73,6 @@ type t = {
   allEnums: array<Internal.enumConfig<Internal.enum>>,
   mutable storageStatus: storageStatus,
   storage: storage,
-  onStorageInitialize: option<unit => promise<unit>>,
 }
 
 let entityHistoryActionEnumConfig: Internal.enumConfig<EntityHistory.RowAction.t> = {
@@ -83,7 +89,6 @@ let make = (
   ~allEnums,
   ~staticTables,
   ~storage,
-  ~onStorageInitialize=?,
 ) => {
   let allEntities = userEntities->Js.Array2.concat([dcRegistryEntityConfig])
   let allEnums =
@@ -95,62 +100,70 @@ let make = (
     allEnums,
     storageStatus: Unknown,
     storage,
-    onStorageInitialize,
   }
 }
 
-let init = async (persistence, ~reset=false) => {
-  try {
-    let shouldRun = switch persistence.storageStatus {
-    | Unknown => true
-    | Initializing(promise) => {
-        await promise
-        reset
-      }
-    | Ready(_) => reset
-    }
-    if shouldRun {
-      let resolveRef = ref(%raw(`null`))
-      let promise = Promise.make((resolve, _) => {
-        resolveRef := resolve
-      })
-      persistence.storageStatus = Initializing(promise)
-      if reset || !(await persistence.storage.isInitialized()) {
-        let _ = await persistence.storage.initialize(
-          ~entities=persistence.allEntities,
-          ~generalTables=persistence.staticTables,
-          ~enums=persistence.allEnums,
-        )
+let init = {
+  let loadInitialCache = async (persistence, ~withUpload) => {
+    let effectCacheRecords = await persistence.storage.restoreEffectCache(~withUpload)
+    let cache = Js.Dict.empty()
+    effectCacheRecords->Js.Array2.forEach(record => {
+      Prometheus.EffectCacheCount.set(~count=record.count, ~effectName=record.effectName)
+      cache->Js.Dict.set(record.effectName, record)
+    })
+    cache
+  }
 
-        persistence.storageStatus = Ready({
-          cleanRun: true,
-          effectCaches: Js.Dict.empty(),
-        })
-        switch persistence.onStorageInitialize {
-        | Some(onStorageInitialize) => await onStorageInitialize()
-        | None => ()
-        }
-      } else if (
-        // In case of a race condition,
-        // we want to set the initial status to Ready only once.
-        switch persistence.storageStatus {
-        | Initializing(_) => true
-        | _ => false
+  async (persistence, ~reset=false) => {
+    try {
+      let shouldRun = switch persistence.storageStatus {
+      | Unknown => true
+      | Initializing(promise) => {
+          await promise
+          reset
         }
-      ) {
-        let effectCaches = Js.Dict.empty()
-        (await persistence.storage.loadEffectCaches())->Js.Array2.forEach(effectCache => {
-          effectCaches->Js.Dict.set(effectCache.name, effectCache)
-        })
-        persistence.storageStatus = Ready({
-          cleanRun: false,
-          effectCaches,
+      | Ready(_) => reset
+      }
+      if shouldRun {
+        let resolveRef = ref(%raw(`null`))
+        let promise = Promise.make((resolve, _) => {
+          resolveRef := resolve
         })
+        persistence.storageStatus = Initializing(promise)
+        if reset || !(await persistence.storage.isInitialized()) {
+          Logging.info(`Initializing the indexer storage...`)
+
+          await persistence.storage.initialize(
+            ~entities=persistence.allEntities,
+            ~generalTables=persistence.staticTables,
+            ~enums=persistence.allEnums,
+          )
+
+          Logging.info(`The indexer storage is ready. Restoring cache...`)
+          persistence.storageStatus = Ready({
+            cleanRun: true,
+            cache: await loadInitialCache(persistence, ~withUpload=true),
+          })
+        } else if (
+          // In case of a race condition,
+          // we want to set the initial status to Ready only once.
+          switch persistence.storageStatus {
+          | Initializing(_) => true
+          | _ => false
+          }
+        ) {
+          Logging.info(`The indexer storage is initialized. Restoring cache...`)
+          persistence.storageStatus = Ready({
+            cleanRun: false,
+            cache: await loadInitialCache(persistence, ~withUpload=false),
+          })
+        }
+        resolveRef.contents()
       }
-      resolveRef.contents()
+    } catch {
+    | exn =>
+      exn->ErrorHandling.mkLogAndRaise(~msg=`EE800: Failed to initialize the indexer storage.`)
     }
-  } catch {
-  | exn => exn->ErrorHandling.mkLogAndRaise(~msg=`EE800: Failed to initialize the indexer storage.`)
   }
 }
 
@@ -162,3 +175,26 @@ let getInitializedStorageOrThrow = persistence => {
   | Ready(_) => persistence.storage
   }
 }
+
+let setEffectCacheOrThrow = async (persistence, ~effectName, ~ids, ~outputs, ~outputSchema) => {
+  switch persistence.storageStatus {
+  | Unknown
+  | Initializing(_) =>
+    Js.Exn.raiseError(`Failed to access the indexer storage. The Persistence layer is not initialized.`)
+  | Ready({cache}) => {
+      let storage = persistence.storage
+      let effectCacheRecord = switch cache->Utils.Dict.dangerouslyGetNonOption(effectName) {
+      | Some(c) => c
+      | None => {
+          let c = {effectName, count: 0}
+          cache->Js.Dict.set(effectName, c)
+          c
+        }
+      }
+      let initialize = effectCacheRecord.count === 0
+      await storage.setEffectCacheOrThrow(~effectName, ~ids, ~outputs, ~outputSchema, ~initialize)
+      effectCacheRecord.count = effectCacheRecord.count + ids->Js.Array2.length
+      Prometheus.EffectCacheCount.set(~count=effectCacheRecord.count, ~effectName)
+    }
+  }
+}

package/src/Persistence.res.js

@@ -2,6 +2,8 @@
 'use strict';
 
 var Js_exn = require("rescript/lib/js/js_exn.js");
+var Logging = require("./Logging.res.js");
+var Prometheus = require("./Prometheus.res.js");
 var EntityHistory = require("./db/EntityHistory.res.js");
 var ErrorHandling = require("./ErrorHandling.res.js");
 var Caml_exceptions = require("rescript/lib/js/caml_exceptions.js");
@@ -22,7 +24,7 @@ var entityHistoryActionEnumConfig = {
   default: "SET"
 };
 
-function make(userEntities, dcRegistryEntityConfig, allEnums, staticTables, storage, onStorageInitialize) {
+function make(userEntities, dcRegistryEntityConfig, allEnums, staticTables, storage) {
   var allEntities = userEntities.concat([dcRegistryEntityConfig]);
   var allEnums$1 = allEnums.concat([entityHistoryActionEnumConfig]);
   return {
@@ -31,11 +33,20 @@ function make(userEntities, dcRegistryEntityConfig, allEnums, staticTables, stor
           allEntities: allEntities,
           allEnums: allEnums$1,
           storageStatus: "Unknown",
-          storage: storage,
-          onStorageInitialize: onStorageInitialize
+          storage: storage
         };
 }
 
+async function loadInitialCache(persistence, withUpload) {
+  var effectCacheRecords = await persistence.storage.restoreEffectCache(withUpload);
+  var cache = {};
+  effectCacheRecords.forEach(function (record) {
+        Prometheus.EffectCacheCount.set(record.count, record.effectName);
+        cache[record.effectName] = record;
+      });
+  return cache;
+}
+
 async function init(persistence, resetOpt) {
   var reset = resetOpt !== undefined ? resetOpt : false;
   try {
@@ -63,30 +74,24 @@ async function init(persistence, resetOpt) {
       _0: promise$1
     };
     if (reset || !await persistence.storage.isInitialized()) {
+      Logging.info("Initializing the indexer storage...");
       await persistence.storage.initialize(persistence.allEntities, persistence.staticTables, persistence.allEnums);
+      Logging.info("The indexer storage is ready. Restoring cache...");
       persistence.storageStatus = {
         TAG: "Ready",
         cleanRun: true,
-        effectCaches: {}
+        cache: await loadInitialCache(persistence, true)
       };
-      var onStorageInitialize = persistence.onStorageInitialize;
-      if (onStorageInitialize !== undefined) {
-        await onStorageInitialize();
-      }
-      
     } else {
       var match = persistence.storageStatus;
       var tmp;
       tmp = typeof match !== "object" || match.TAG !== "Initializing" ? false : true;
       if (tmp) {
-        var effectCaches = {};
-        (await persistence.storage.loadEffectCaches()).forEach(function (effectCache) {
-              effectCaches[effectCache.name] = effectCache;
-            });
+        Logging.info("The indexer storage is initialized. Restoring cache...");
         persistence.storageStatus = {
           TAG: "Ready",
           cleanRun: false,
-          effectCaches: effectCaches
+          cache: await loadInitialCache(persistence, false)
         };
       }
       
@@ -108,9 +113,38 @@ function getInitializedStorageOrThrow(persistence) {
   }
 }
 
+async function setEffectCacheOrThrow(persistence, effectName, ids, outputs, outputSchema) {
+  var match = persistence.storageStatus;
+  if (typeof match !== "object") {
+    return Js_exn.raiseError("Failed to access the indexer storage. The Persistence layer is not initialized.");
+  }
+  if (match.TAG === "Initializing") {
+    return Js_exn.raiseError("Failed to access the indexer storage. The Persistence layer is not initialized.");
+  }
+  var cache = match.cache;
+  var storage = persistence.storage;
+  var c = cache[effectName];
+  var effectCacheRecord;
+  if (c !== undefined) {
+    effectCacheRecord = c;
+  } else {
+    var c$1 = {
+      effectName: effectName,
+      count: 0
+    };
+    cache[effectName] = c$1;
+    effectCacheRecord = c$1;
+  }
+  var initialize = effectCacheRecord.count === 0;
+  await storage.setEffectCacheOrThrow(effectName, ids, outputs, outputSchema, initialize);
+  effectCacheRecord.count = effectCacheRecord.count + ids.length | 0;
+  return Prometheus.EffectCacheCount.set(effectCacheRecord.count, effectName);
+}
+
 exports.StorageError = StorageError;
 exports.entityHistoryActionEnumConfig = entityHistoryActionEnumConfig;
 exports.make = make;
 exports.init = init;
 exports.getInitializedStorageOrThrow = getInitializedStorageOrThrow;
-/* EntityHistory Not a pure module */
+exports.setEffectCacheOrThrow = setEffectCacheOrThrow;
+/* Logging Not a pure module */

package/src/PgStorage.res

@@ -128,6 +128,20 @@ GRANT ALL ON SCHEMA "${pgSchema}" TO public;`,
     })
   })
 
+  // Add cache row count function
+  functionsQuery :=
+    functionsQuery.contents ++
+    "\n" ++
+    `CREATE OR REPLACE FUNCTION get_cache_row_count(table_name text) 
+   RETURNS integer AS $$
+   DECLARE
+     result integer;
+   BEGIN
+     EXECUTE format('SELECT COUNT(*) FROM "${pgSchema}".%I', table_name) INTO result;
+     RETURN result;
+   END;
+   $$ LANGUAGE plpgsql;`
+
   [query.contents]->Js.Array2.concat(
     functionsQuery.contents !== "" ? [functionsQuery.contents] : [],
   )
@@ -418,7 +432,86 @@ let makeSchemaTableNamesQuery = (~pgSchema) => {
   `SELECT table_name FROM information_schema.tables WHERE table_schema = '${pgSchema}';`
 }
 
-let make = (~sql: Postgres.sql, ~pgSchema, ~pgUser): Persistence.storage => {
+let cacheTablePrefix = "envio_effect_"
+let cacheTablePrefixLength = cacheTablePrefix->String.length
+
+type schemaCacheTableInfo = {
+  @as("table_name")
+  tableName: string,
+  @as("count")
+  count: int,
+}
+
+let makeSchemaCacheTableInfoQuery = (~pgSchema) => {
+  `SELECT 
+    t.table_name,
+    get_cache_row_count(t.table_name) as count
+   FROM information_schema.tables t
+   WHERE t.table_schema = '${pgSchema}' 
+   AND t.table_name LIKE '${cacheTablePrefix}%';`
+}
+
+type psqlExecState =
+  Unknown | Pending(promise<result<string, string>>) | Resolved(result<string, string>)
+
+let getPsqlExec = {
+  // For development: We run the indexer process locally,
+  //   and there might not be psql installed on the user's machine.
+  //   So we use docker-compose to run psql existing in the postgres container.
+  // For production: We expect indexer to be running in a container,
+  //   with psql installed. So we can call it directly.
+  let psqlExecState = ref(Unknown)
+  async () => {
+    switch psqlExecState.contents {
+    | Unknown => {
+        let promise = Promise.make((resolve, _reject) => {
+          let binary = "psql"
+          NodeJs.ChildProcess.exec(`${binary} --version`, (~error, ~stdout as _, ~stderr as _) => {
+            switch error {
+            | Value(_) => {
+                let binary = "docker-compose exec -T -u postgres envio-postgres psql"
+                NodeJs.ChildProcess.exec(
+                  `${binary} --version`,
+                  (~error, ~stdout as _, ~stderr as _) => {
+                    switch error {
+                    | Value(_) => resolve(Error("Failed to find psql binary"))
+                    | Null => resolve(Ok(binary))
+                    }
+                  },
+                )
+              }
+            | Null => resolve(Ok(binary))
+            }
+          })
+        })
+
+        psqlExecState := Pending(promise)
+        let result = await promise
+        psqlExecState := Resolved(result)
+        result
+      }
+    | Pending(promise) => await promise
+    | Resolved(result) => result
+    }
+  }
+}
+let psqlExecMissingErrorMessage = `Please check if "psql" binary is installed or docker-compose is running for the local indexer.`
+
+let make = (
+  ~sql: Postgres.sql,
+  ~pgSchema,
+  ~pgUser,
+  ~pgDatabase,
+  ~onInitialize=?,
+  ~onNewTables=?,
+): Persistence.storage => {
+  let cacheDirPath = NodeJs.Path.resolve([
+    // Right outside of the generated directory
+    "..",
+    ".envio",
+    "cache",
+  ])
+
   let isInitialized = async () => {
     let envioTables =
       await sql->Postgres.unsafe(
@@ -459,26 +552,11 @@ let make = (~sql: Postgres.sql, ~pgSchema, ~pgUser): Persistence.storage => {
     let _ = await sql->Postgres.beginSql(sql => {
       queries->Js.Array2.map(query => sql->Postgres.unsafe(query))
     })
-  }
 
-  let loadEffectCaches = async () => {
-    let schemaTableNames: array<schemaTableName> =
-      await sql->Postgres.unsafe(makeSchemaTableNamesQuery(~pgSchema))
-    schemaTableNames->Belt.Array.keepMapU(schemaTableName => {
-      if schemaTableName.tableName->Js.String2.startsWith("effect_cache_") {
-        Some(
-          (
-            {
-              name: schemaTableName.tableName,
-              size: 0,
-              table: None,
-            }: Persistence.effectCache
-          ),
-        )
-      } else {
-        None
-      }
-    })
+    switch onInitialize {
+    | Some(onInitialize) => await onInitialize()
+    | None => ()
+    }
   }
 
   let loadByIdsOrThrow = async (~ids, ~table: Table.table, ~rowsSchema) => {
@@ -577,12 +655,199 @@ let make = (~sql: Postgres.sql, ~pgSchema, ~pgUser): Persistence.storage => {
     )
   }
 
+  let setEffectCacheOrThrow = async (
+    ~effectName: string,
+    ~ids: array<string>,
+    ~outputs: array<Internal.effectOutput>,
+    ~outputSchema: S.t<Internal.effectOutput>,
+    ~initialize: bool,
+  ) => {
+    let table = Table.mkTable(
+      cacheTablePrefix ++ effectName,
+      ~fields=[
+        Table.mkField("id", Text, ~fieldSchema=S.string, ~isPrimaryKey=true),
+        Table.mkField("output", JsonB, ~fieldSchema=outputSchema),
+      ],
+      ~compositeIndices=[],
+    )
+
+    if initialize {
+      let _ = await sql->Postgres.unsafe(makeCreateTableQuery(table, ~pgSchema))
+      switch onNewTables {
+      | Some(onNewTables) => await onNewTables(~tableNames=[table.tableName])
+      | None => ()
+      }
+    }
+
+    let items = []
+    for idx in 0 to outputs->Array.length - 1 {
+      items
+      ->Js.Array2.push({
+        "id": ids[idx],
+        "output": outputs[idx],
+      })
+      ->ignore
+    }
+
+    await setOrThrow(
+      ~items,
+      ~table,
+      ~itemSchema=S.schema(s =>
+        {
+          "id": s.matches(S.string),
+          "output": s.matches(outputSchema),
+        }
+      ),
+    )
+  }
+
+  let dumpEffectCache = async () => {
+    let cacheTableInfo: array<schemaCacheTableInfo> =
+      (await sql
+      ->Postgres.unsafe(makeSchemaCacheTableInfoQuery(~pgSchema)))
+      ->Js.Array2.filter(i => i.count > 0)
+
+    if cacheTableInfo->Utils.Array.notEmpty {
+      // Create .envio/cache directory if it doesn't exist
+      try {
+        await NodeJs.Fs.Promises.access(cacheDirPath)
+      } catch {
+      | _ =>
+        // Create directory if it doesn't exist
+        await NodeJs.Fs.Promises.mkdir(~path=cacheDirPath, ~options={recursive: true})
+      }
+
+      // Command for testing. Run from generated
+      // docker-compose exec -T -u postgres envio-postgres psql -d envio-dev -c 'COPY "public"."envio_effect_getTokenMetadata" TO STDOUT (FORMAT text, HEADER);' > ../.envio/cache/getTokenMetadata.tsv
+
+      switch await getPsqlExec() {
+      | Ok(psqlExec) => {
+          Logging.info(
+            `Dumping cache: ${cacheTableInfo
+              ->Js.Array2.map(({tableName, count}) =>
+                tableName ++ " (" ++ count->Belt.Int.toString ++ " rows)"
+              )
+              ->Js.Array2.joinWith(", ")}`,
+          )
+
+          let promises = cacheTableInfo->Js.Array2.map(async ({tableName}) => {
+            let cacheName = tableName->Js.String2.sliceToEnd(~from=cacheTablePrefixLength)
+            let outputFile =
+              NodeJs.Path.join(cacheDirPath, cacheName ++ ".tsv")->NodeJs.Path.toString
+
+            let command = `${psqlExec} -d ${pgDatabase} -c 'COPY "${pgSchema}"."${tableName}" TO STDOUT WITH (FORMAT text, HEADER);' > ${outputFile}`
+
+            Promise.make((resolve, reject) => {
+              NodeJs.ChildProcess.exec(
+                command,
+                (~error, ~stdout, ~stderr as _) => {
+                  switch error {
+                  | Value(error) => reject(error)
+                  | Null => resolve(stdout)
+                  }
+                },
+              )
+            })
+          })
+
+          let _ = await promises->Promise.all
+          Logging.info(`Successfully dumped cache to ${cacheDirPath->NodeJs.Path.toString}`)
+        }
+      | Error(_) => Logging.error(`Failed to dump cache. ${psqlExecMissingErrorMessage}`)
+      }
+    }
+  }
+
+  let restoreEffectCache = async (~withUpload) => {
+    if withUpload {
+      // Try to restore cache tables from binary files
+      let (entries, psqlExecResult) = await Promise.all2((
+        NodeJs.Fs.Promises.readdir(cacheDirPath),
+        getPsqlExec(),
+      ))
+
+      switch psqlExecResult {
+      | Ok(psqlExec) => {
+          let cacheFiles = entries->Js.Array2.filter(entry => {
+            entry->Js.String2.endsWith(".tsv")
+          })
+
+          let _ =
+            await cacheFiles
+            ->Js.Array2.map(entry => {
+              let cacheName = entry->Js.String2.slice(~from=0, ~to_=-4) // Remove .tsv extension
+              let tableName = cacheTablePrefix ++ cacheName
+              let table = Table.mkTable(
+                tableName,
+                ~fields=[
+                  Table.mkField("id", Text, ~fieldSchema=S.string, ~isPrimaryKey=true),
+                  Table.mkField("output", JsonB, ~fieldSchema=S.json(~validate=false)),
+                ],
+                ~compositeIndices=[],
+              )
+
+              sql
+              ->Postgres.unsafe(makeCreateTableQuery(table, ~pgSchema))
+              ->Promise.then(() => {
+                let inputFile = NodeJs.Path.join(cacheDirPath, entry)->NodeJs.Path.toString
+
+                let command = `${psqlExec} -d ${pgDatabase} -c 'COPY "${pgSchema}"."${tableName}" FROM STDIN WITH (FORMAT text, HEADER);' < ${inputFile}`
+
+                Promise.make(
+                  (resolve, reject) => {
+                    NodeJs.ChildProcess.exec(
+                      command,
+                      (~error, ~stdout, ~stderr as _) => {
+                        switch error {
+                        | Value(error) => reject(error)
+                        | Null => resolve(stdout)
+                        }
+                      },
+                    )
+                  },
+                )
+              })
+            })
+            ->Promise.all
+        }
+      | Error(_) =>
+        Logging.error(
+          `Failed to restore cache, continuing without it. ${psqlExecMissingErrorMessage}`,
+        )
+      }
+    }
+
+    let cacheTableInfo: array<schemaCacheTableInfo> =
+      await sql->Postgres.unsafe(makeSchemaCacheTableInfoQuery(~pgSchema))
+
+    if withUpload {
+      switch onNewTables {
+      | Some(onNewTables) =>
+        await onNewTables(
+          ~tableNames=cacheTableInfo->Js.Array2.map(info => {
+            info.tableName
+          }),
+        )
+      | None => ()
+      }
+    }
+
+    cacheTableInfo->Js.Array2.map((info): Persistence.effectCacheRecord => {
+      {
+        effectName: info.tableName->Js.String2.sliceToEnd(~from=cacheTablePrefixLength),
+        count: info.count,
+      }
+    })
+  }
+
   {
     isInitialized,
     initialize,
     loadByFieldOrThrow,
-    loadEffectCaches,
     loadByIdsOrThrow,
     setOrThrow,
+    setEffectCacheOrThrow,
+    dumpEffectCache,
+    restoreEffectCache,
   }
 }

package/src/PgStorage.res.js

@@ -1,15 +1,20 @@
 // Generated by ReScript, PLEASE EDIT WITH CARE
 'use strict';
 
+var Fs = require("fs");
+var Path = require("path");
 var $$Array = require("rescript/lib/js/array.js");
 var Table = require("./db/Table.res.js");
 var Utils = require("./Utils.res.js");
 var Js_exn = require("rescript/lib/js/js_exn.js");
 var Schema = require("./db/Schema.res.js");
+var Logging = require("./Logging.res.js");
 var Belt_Array = require("rescript/lib/js/belt_Array.js");
+var Caml_array = require("rescript/lib/js/caml_array.js");
 var Belt_Option = require("rescript/lib/js/belt_Option.js");
 var Caml_option = require("rescript/lib/js/caml_option.js");
 var Persistence = require("./Persistence.res.js");
+var Child_process = require("child_process");
 var Caml_exceptions = require("rescript/lib/js/caml_exceptions.js");
 var S$RescriptSchema = require("rescript-schema/src/S.res.js");
 var Caml_js_exceptions = require("rescript/lib/js/caml_js_exceptions.js");
@@ -104,6 +109,7 @@ function makeInitializeTransaction(pgSchema, pgUser, generalTablesOpt, entitiesO
               query.contents = query.contents + "\n" + makeCreateIndexQuery(derivedFromField.derivedFromEntity, [indexField], pgSchema);
             });
       });
+  functionsQuery.contents = functionsQuery.contents + "\n" + ("CREATE OR REPLACE FUNCTION get_cache_row_count(table_name text) \n   RETURNS integer AS $$\n   DECLARE\n     result integer;\n   BEGIN\n     EXECUTE format('SELECT COUNT(*) FROM \"" + pgSchema + "\".%I', table_name) INTO result;\n     RETURN result;\n   END;\n   $$ LANGUAGE plpgsql;");
   return [query.contents].concat(functionsQuery.contents !== "" ? [functionsQuery.contents] : []);
 }
 
@@ -288,7 +294,68 @@ function makeSchemaTableNamesQuery(pgSchema) {
   return "SELECT table_name FROM information_schema.tables WHERE table_schema = '" + pgSchema + "';";
 }
 
-function make(sql, pgSchema, pgUser) {
+var cacheTablePrefix = "envio_effect_";
+
+var cacheTablePrefixLength = cacheTablePrefix.length;
+
+function makeSchemaCacheTableInfoQuery(pgSchema) {
+  return "SELECT \n    t.table_name,\n    get_cache_row_count(t.table_name) as count\n   FROM information_schema.tables t\n   WHERE t.table_schema = '" + pgSchema + "' \n   AND t.table_name LIKE '" + cacheTablePrefix + "%';";
+}
+
+var psqlExecState = {
+  contents: "Unknown"
+};
+
+async function getPsqlExec() {
+  var promise = psqlExecState.contents;
+  if (typeof promise === "object") {
+    if (promise.TAG === "Pending") {
+      return await promise._0;
+    } else {
+      return promise._0;
+    }
+  }
+  var promise$1 = new Promise((function (resolve, _reject) {
+          var binary = "psql";
+          Child_process.exec(binary + " --version", (function (error, param, param$1) {
+                  if (error === null) {
+                    return resolve({
+                                TAG: "Ok",
+                                _0: binary
+                              });
+                  }
+                  var binary$1 = "docker-compose exec -T -u postgres envio-postgres psql";
+                  Child_process.exec(binary$1 + " --version", (function (error, param, param$1) {
+                          if (error === null) {
+                            return resolve({
+                                        TAG: "Ok",
+                                        _0: binary$1
+                                      });
+                          } else {
+                            return resolve({
+                                        TAG: "Error",
+                                        _0: "Failed to find psql binary"
+                                      });
+                          }
+                        }));
+                }));
+        }));
+  psqlExecState.contents = {
+    TAG: "Pending",
+    _0: promise$1
+  };
+  var result = await promise$1;
+  psqlExecState.contents = {
+    TAG: "Resolved",
+    _0: result
+  };
+  return result;
+}
+
+var psqlExecMissingErrorMessage = "Please check if \"psql\" binary is installed or docker-compose is running for the local indexer.";
+
+function make(sql, pgSchema, pgUser, pgDatabase, onInitialize, onNewTables) {
+  var cacheDirPath = Path.resolve("..", ".envio", "cache");
   var isInitialized = async function () {
     var envioTables = await sql.unsafe("SELECT table_schema FROM information_schema.tables WHERE table_schema = '" + pgSchema + "' AND table_name = '" + eventSyncStateTableName + "';");
     return Utils.$$Array.notEmpty(envioTables);
@@ -309,19 +376,10 @@ function make(sql, pgSchema, pgUser) {
                       return sql.unsafe(query);
                     });
         });
-  };
-  var loadEffectCaches = async function () {
-    var schemaTableNames = await sql.unsafe(makeSchemaTableNamesQuery(pgSchema));
-    return Belt_Array.keepMapU(schemaTableNames, (function (schemaTableName) {
-                  if (schemaTableName.table_name.startsWith("effect_cache_")) {
-                    return {
-                            name: schemaTableName.table_name,
-                            size: 0,
-                            table: undefined
-                          };
-                  }
-                  
-                }));
+    if (onInitialize !== undefined) {
+      return await onInitialize();
+    }
+    
   };
   var loadByIdsOrThrow = async function (ids, table, rowsSchema) {
     var rows;
@@ -395,13 +453,132 @@ function make(sql, pgSchema, pgUser) {
   var setOrThrow$1 = function (items, table, itemSchema) {
     return setOrThrow(sql, items, table, itemSchema, pgSchema);
   };
+  var setEffectCacheOrThrow = async function (effectName, ids, outputs, outputSchema, initialize) {
+    var table = Table.mkTable(cacheTablePrefix + effectName, [], [
+          Table.mkField("id", "TEXT", S$RescriptSchema.string, undefined, undefined, undefined, true, undefined, undefined),
+          Table.mkField("output", "JSONB", outputSchema, undefined, undefined, undefined, undefined, undefined, undefined)
+        ]);
+    if (initialize) {
+      await sql.unsafe(makeCreateTableQuery(table, pgSchema));
+      if (onNewTables !== undefined) {
+        await onNewTables([table.tableName]);
+      }
+      
+    }
+    var items = [];
+    for(var idx = 0 ,idx_finish = outputs.length; idx < idx_finish; ++idx){
+      items.push({
+            id: Caml_array.get(ids, idx),
+            output: Caml_array.get(outputs, idx)
+          });
+    }
+    return await setOrThrow$1(items, table, S$RescriptSchema.schema(function (s) {
+                    return {
+                            id: s.m(S$RescriptSchema.string),
+                            output: s.m(outputSchema)
+                          };
+                  }));
+  };
+  var dumpEffectCache = async function () {
+    var cacheTableInfo = (await sql.unsafe(makeSchemaCacheTableInfoQuery(pgSchema))).filter(function (i) {
+          return i.count > 0;
+        });
+    if (!Utils.$$Array.notEmpty(cacheTableInfo)) {
+      return ;
+    }
+    try {
+      await Fs.promises.access(cacheDirPath);
+    }
+    catch (exn){
+      await Fs.promises.mkdir(cacheDirPath, {
+            recursive: true
+          });
+    }
+    var psqlExec = await getPsqlExec();
+    if (psqlExec.TAG !== "Ok") {
+      return Logging.error("Failed to dump cache. " + psqlExecMissingErrorMessage);
+    }
+    var psqlExec$1 = psqlExec._0;
+    Logging.info("Dumping cache: " + cacheTableInfo.map(function (param) {
+                return param.table_name + " (" + String(param.count) + " rows)";
+              }).join(", "));
+    var promises = cacheTableInfo.map(async function (param) {
+          var tableName = param.table_name;
+          var cacheName = tableName.slice(cacheTablePrefixLength);
+          var outputFile = Path.join(cacheDirPath, cacheName + ".tsv");
+          var command = psqlExec$1 + " -d " + pgDatabase + " -c 'COPY \"" + pgSchema + "\".\"" + tableName + "\" TO STDOUT WITH (FORMAT text, HEADER);' > " + outputFile;
+          return new Promise((function (resolve, reject) {
+                        Child_process.exec(command, (function (error, stdout, param) {
+                                if (error === null) {
+                                  return resolve(stdout);
+                                } else {
+                                  return reject(error);
+                                }
+                              }));
+                      }));
+        });
+    await Promise.all(promises);
+    return Logging.info("Successfully dumped cache to " + cacheDirPath);
+  };
+  var restoreEffectCache = async function (withUpload) {
+    if (withUpload) {
+      var match = await Promise.all([
+            Fs.promises.readdir(cacheDirPath),
+            getPsqlExec()
+          ]);
+      var psqlExecResult = match[1];
+      if (psqlExecResult.TAG === "Ok") {
+        var psqlExec = psqlExecResult._0;
+        var cacheFiles = match[0].filter(function (entry) {
+              return entry.endsWith(".tsv");
+            });
+        await Promise.all(cacheFiles.map(function (entry) {
+                  var cacheName = entry.slice(0, -4);
+                  var tableName = cacheTablePrefix + cacheName;
+                  var table = Table.mkTable(tableName, [], [
+                        Table.mkField("id", "TEXT", S$RescriptSchema.string, undefined, undefined, undefined, true, undefined, undefined),
+                        Table.mkField("output", "JSONB", S$RescriptSchema.json(false), undefined, undefined, undefined, undefined, undefined, undefined)
+                      ]);
+                  return sql.unsafe(makeCreateTableQuery(table, pgSchema)).then(function () {
+                              var inputFile = Path.join(cacheDirPath, entry);
+                              var command = psqlExec + " -d " + pgDatabase + " -c 'COPY \"" + pgSchema + "\".\"" + tableName + "\" FROM STDIN WITH (FORMAT text, HEADER);' < " + inputFile;
+                              return new Promise((function (resolve, reject) {
+                                            Child_process.exec(command, (function (error, stdout, param) {
+                                                    if (error === null) {
+                                                      return resolve(stdout);
+                                                    } else {
+                                                      return reject(error);
+                                                    }
+                                                  }));
+                                          }));
+                            });
+                }));
+      } else {
+        Logging.error("Failed to restore cache, continuing without it. " + psqlExecMissingErrorMessage);
+      }
+    }
+    var cacheTableInfo = await sql.unsafe(makeSchemaCacheTableInfoQuery(pgSchema));
+    if (withUpload && onNewTables !== undefined) {
+      await onNewTables(cacheTableInfo.map(function (info) {
+                return info.table_name;
+              }));
+    }
+    return cacheTableInfo.map(function (info) {
+                return {
+                        effectName: info.table_name.slice(cacheTablePrefixLength),
+                        count: info.count
+                      };
+              });
+  };
   return {
           isInitialized: isInitialized,
           initialize: initialize,
-          loadEffectCaches: loadEffectCaches,
           loadByIdsOrThrow: loadByIdsOrThrow,
           loadByFieldOrThrow: loadByFieldOrThrow,
-          setOrThrow: setOrThrow$1
+          setOrThrow: setOrThrow$1,
+          setEffectCacheOrThrow: setEffectCacheOrThrow,
+          dumpEffectCache: dumpEffectCache,
+          restoreEffectCache: restoreEffectCache
         };
 }
 
@@ -428,5 +605,10 @@ exports.setQueryCache = setQueryCache;
 exports.setOrThrow = setOrThrow;
 exports.setEntityHistoryOrThrow = setEntityHistoryOrThrow;
 exports.makeSchemaTableNamesQuery = makeSchemaTableNamesQuery;
+exports.cacheTablePrefix = cacheTablePrefix;
+exports.cacheTablePrefixLength = cacheTablePrefixLength;
+exports.makeSchemaCacheTableInfoQuery = makeSchemaCacheTableInfoQuery;
+exports.getPsqlExec = getPsqlExec;
+exports.psqlExecMissingErrorMessage = psqlExecMissingErrorMessage;
 exports.make = make;
 /* pgEncodingErrorSchema Not a pure module */

package/src/Prometheus.res

@@ -590,3 +590,15 @@ module EffectCallsCount = {
     gauge->SafeGauge.handleInt(~labels=effectName, ~value=callsCount)
   }
 }
+
+module EffectCacheCount = {
+  let gauge = SafeGauge.makeOrThrow(
+    ~name="envio_effect_cache_count",
+    ~help="The number of items in the effect cache.",
+    ~labelSchema=effectLabelsSchema,
+  )
+
+  let set = (~count, ~effectName) => {
+    gauge->SafeGauge.handleInt(~labels=effectName, ~value=count)
+  }
+}

package/src/Prometheus.res.js

@@ -700,6 +700,17 @@ var EffectCallsCount = {
   set: set$21
 };
 
+var gauge$22 = makeOrThrow$1("envio_effect_cache_count", "The number of items in the effect cache.", effectLabelsSchema);
+
+function set$22(count, effectName) {
+  handleInt$1(gauge$22, effectName, count);
+}
+
+var EffectCacheCount = {
+  gauge: gauge$22,
+  set: set$22
+};
+
 exports.loadEntitiesDurationCounter = loadEntitiesDurationCounter;
 exports.eventRouterDurationCounter = eventRouterDurationCounter;
 exports.executeBatchDurationCounter = executeBatchDurationCounter;
@@ -747,4 +758,5 @@ exports.ProgressBlockNumber = ProgressBlockNumber;
 exports.ProgressEventsCount = ProgressEventsCount;
 exports.effectLabelsSchema = effectLabelsSchema;
 exports.EffectCallsCount = EffectCallsCount;
+exports.EffectCacheCount = EffectCacheCount;
 /* loadEntitiesDurationCounter Not a pure module */

package/src/Utils.res

@@ -516,13 +516,17 @@ module Proxy = {
 }
 
 module Hash = {
+  // Hash to JSON string. No specific reason for this,
+  // just to stick to at least some sort of spec.
+  // After Sury v11 is out we'll be able to do it with schema
   let rec makeOrThrow = (any: 'a): string => {
     switch any->Js.typeof {
-    | "string" => any->magic
+    | "string" => `"${any->magic}"` // Ideally should escape here,
+    // but since we don't parse it back, it's fine to keep it super simple
     | "number" => any->magic->Js.Int.toString
-    | "bigint" => any->magic->BigInt.toString
+    | "bigint" => `"${any->magic->BigInt.toString}"`
     | "boolean" => any->magic ? "true" : "false"
-    | "undefined" => "undefined"
+    | "undefined" => "null"
     | "object" =>
       if any === %raw(`null`) {
         "null"
@@ -530,7 +534,10 @@ module Hash = {
         let any: array<'a> = any->magic
         let hash = ref("[")
         for i in 0 to any->Js.Array2.length - 1 {
-          hash := hash.contents ++ any->Js.Array2.unsafe_get(i)->makeOrThrow ++ ","
+          if i !== 0 {
+            hash := hash.contents ++ ","
+          }
+          hash := hash.contents ++ any->Js.Array2.unsafe_get(i)->makeOrThrow
         }
         hash.contents ++ "]"
       } else {
@@ -541,10 +548,16 @@ module Hash = {
           let keys = any->Js.Dict.keys->Js.Array2.sortInPlace
           for i in 0 to keys->Js.Array2.length - 1 {
             let key = keys->Js.Array2.unsafe_get(i)
-            // Ideally should escape and wrap the key in double quotes
-            // but since we don't need to decode the hash,
-            // it's fine to keep it super simple
-            hash := hash.contents ++ key ++ ":" ++ any->Js.Dict.unsafeGet(key)->makeOrThrow ++ ","
+            let value = any->Js.Dict.unsafeGet(key)
+            if i !== 0 {
+              hash := hash.contents ++ ","
+            }
+            if value !== %raw(`undefined`) {
+              // Ideally should escape and wrap the key in double quotes
+              // but since we don't need to decode the hash,
+              // it's fine to keep it super simple
+              hash := hash.contents ++ `"${key}":${any->Js.Dict.unsafeGet(key)->makeOrThrow}`
+            }
           }
           hash.contents ++ "}"
         } else if constructor["name"] === "BigNumber" {

package/src/Utils.res.js

@@ -455,13 +455,14 @@ var $$Proxy = {};
 function makeOrThrow(any) {
   var $$typeof = typeof any;
   switch ($$typeof) {
+    case "bigint" :
+        return "\"" + any.toString() + "\"";
     case "boolean" :
         if (any) {
           return "true";
         } else {
           return "false";
         }
-    case "bigint" :
     case "number" :
         return any.toString();
     case "object" :
@@ -471,7 +472,10 @@ function makeOrThrow(any) {
         if (Array.isArray(any)) {
           var hash = "[";
           for(var i = 0 ,i_finish = any.length; i < i_finish; ++i){
-            hash = hash + makeOrThrow(any[i]) + ",";
+            if (i !== 0) {
+              hash = hash + ",";
+            }
+            hash = hash + makeOrThrow(any[i]);
           }
           return hash + "]";
         }
@@ -487,16 +491,23 @@ function makeOrThrow(any) {
         var keys = Object.keys(any).sort();
         for(var i$1 = 0 ,i_finish$1 = keys.length; i$1 < i_finish$1; ++i$1){
           var key = keys[i$1];
-          hash$1 = hash$1 + key + ":" + makeOrThrow(any[key]) + ",";
+          var value = any[key];
+          if (i$1 !== 0) {
+            hash$1 = hash$1 + ",";
+          }
+          if (value !== undefined) {
+            hash$1 = hash$1 + ("\"" + key + "\":" + makeOrThrow(any[key]));
+          }
+          
         }
         return hash$1 + "}";
     case "string" :
-        return any;
+        return "\"" + any + "\"";
     case "function" :
     case "symbol" :
         return any.toString();
     case "undefined" :
-        return "undefined";
+        return "null";
     default:
       return Js_exn.raiseError("Don't know how to serialize " + $$typeof);
   }

package/src/bindings/NodeJs.res

@@ -4,35 +4,23 @@ type t
 type exitCode = | @as(0) Success | @as(1) Failure
 @send external exitWithCode: (t, exitCode) => unit = "exit"
 
-module Util = {
-  @unboxed
-  type depth = Int(int) | @as(null) Null
-  @unboxed
-  type compact = Bool(bool) | Int(int)
-  @unboxed
-  type sorted = Bool(bool) | Fn((string, string) => int)
-  @unboxed
-  type getters = | @as(true) True | @as(false) False | @as("get") Get | @as("set") Set
-
-  @unbox
-  type inspectOptions = {
-    showHidden?: bool,
-    depth?: depth,
-    colors?: bool,
-    customInspect?: bool,
-    showProxy?: bool,
-    maxArrayLength?: int,
-    maxStringLength?: int,
-    breakLength?: int,
-    @as("compact") compact?: compact,
-    sorted?: sorted,
-    getters?: string,
-    numericSeparator?: bool,
-  }
+module Process = {
+  type t = {env: Js.Dict.t<string>}
+  @module external process: t = "process"
+}
 
-  @module("util") external inspect: ('a, inspectOptions) => string = "inspect"
+module ChildProcess = {
+  type execOptions = {
+    cwd?: string,
+    env?: Js.Dict.t<string>,
+    shell?: string,
+  }
 
-  let inspectObj = a => inspect(a, {showHidden: false, depth: Null, colors: true})
+  @module("child_process")
+  external exec: (
+    string,
+    (~error: Js.null<exn>, ~stdout: string, ~stderr: string) => unit,
+  ) => unit = "exec"
 }
 
 module Path = {
@@ -55,6 +43,11 @@ module Fs = {
     encoding?: string,
   }
 
+  type mkdirOptions = {
+    recursive?: bool,
+    mode?: int,
+  }
+
   module Promises = {
     @module("fs") @scope("promises")
     external writeFile: (
@@ -77,5 +70,11 @@ module Fs = {
 
     @module("fs") @scope("promises")
     external readFile: (~filepath: Path.t, ~encoding: encoding) => promise<string> = "readFile"
+
+    @module("fs") @scope("promises")
+    external mkdir: (~path: Path.t, ~options: mkdirOptions=?) => Js.Promise.t<unit> = "mkdir"
+
+    @module("fs") @scope("promises")
+    external readdir: Path.t => Js.Promise.t<array<string>> = "readdir"
   }
 }

package/src/bindings/NodeJs.res.js

@@ -1,19 +1,10 @@
 // Generated by ReScript, PLEASE EDIT WITH CARE
 'use strict';
 
-var Util = require("util");
 
-function inspectObj(a) {
-  return Util.inspect(a, {
-              showHidden: false,
-              depth: null,
-              colors: true
-            });
-}
+var Process = {};
 
-var Util$1 = {
-  inspectObj: inspectObj
-};
+var ChildProcess = {};
 
 var Path = {};
 
@@ -23,7 +14,8 @@ var Fs = {
   Promises: Promises
 };
 
-exports.Util = Util$1;
+exports.Process = Process;
+exports.ChildProcess = ChildProcess;
 exports.Path = Path;
 exports.Fs = Fs;
-/* util Not a pure module */
+/* No side effect */