envio 2.26.0-rc.1 → 2.26.0-rc.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "envio",
-  "version": "v2.26.0-rc.1",
+  "version": "v2.26.0-rc.2",
   "description": "A latency and sync speed optimized, developer friendly blockchain data indexer.",
   "bin": "./bin.js",
   "main": "./index.js",
@@ -25,10 +25,10 @@
   },
   "homepage": "https://envio.dev",
   "optionalDependencies": {
-    "envio-linux-x64": "v2.26.0-rc.1",
-    "envio-linux-arm64": "v2.26.0-rc.1",
-    "envio-darwin-x64": "v2.26.0-rc.1",
-    "envio-darwin-arm64": "v2.26.0-rc.1"
+    "envio-linux-x64": "v2.26.0-rc.2",
+    "envio-linux-arm64": "v2.26.0-rc.2",
+    "envio-darwin-x64": "v2.26.0-rc.2",
+    "envio-darwin-arm64": "v2.26.0-rc.2"
   },
   "dependencies": {
     "@envio-dev/hypersync-client": "0.6.5",

package/src/Envio.res

@@ -42,6 +42,8 @@ let experimental_createEffect = (
   handler: effectArgs<'input> => promise<'output>,
 ) => {
   Prometheus.EffectCallsCount.set(~callsCount=0, ~effectName=options.name)
+  let outputSchema =
+    S.schema(_ => options.output)->(Utils.magic: S.t<S.t<'output>> => S.t<Internal.effectOutput>)
   {
     name: options.name,
     handler: handler->(
@@ -57,9 +59,21 @@ let experimental_createEffect = (
     input: S.schema(_ => options.input)->(
       Utils.magic: S.t<S.t<'input>> => S.t<Internal.effectInput>
     ),
-    output: S.schema(_ => options.output)->(
-      Utils.magic: S.t<S.t<'output>> => S.t<Internal.effectOutput>
-    ),
-    cache: options.cache->Belt.Option.getWithDefault(false),
+    output: outputSchema,
+    cache: switch options.cache {
+    | Some(true) =>
+      let itemSchema = S.schema((s): Internal.effectCacheItem => {
+        id: s.matches(S.string),
+        output: s.matches(outputSchema),
+      })
+      Some({
+        table: Internal.makeCacheTable(~effectName=options.name),
+        rowsSchema: S.array(itemSchema),
+        itemSchema,
+      })
+    | None
+    | Some(false) =>
+      None
+    },
   }->(Utils.magic: Internal.effect => effect<'input, 'output>)
 }

package/src/Envio.res.js

@@ -1,19 +1,37 @@
 // Generated by ReScript, PLEASE EDIT WITH CARE
 'use strict';
 
+var Internal = require("./Internal.res.js");
 var Prometheus = require("./Prometheus.res.js");
-var Belt_Option = require("rescript/lib/js/belt_Option.js");
 var S$RescriptSchema = require("rescript-schema/src/S.res.js");
 
 function experimental_createEffect(options, handler) {
   Prometheus.EffectCallsCount.set(0, options.name);
+  var outputSchema = S$RescriptSchema.schema(function (param) {
+        return options.output;
+      });
+  var match = options.cache;
+  var tmp;
+  if (match !== undefined && match) {
+    var itemSchema = S$RescriptSchema.schema(function (s) {
+          return {
+                  id: s.m(S$RescriptSchema.string),
+                  output: s.m(outputSchema)
+                };
+        });
+    tmp = {
+      itemSchema: itemSchema,
+      rowsSchema: S$RescriptSchema.array(itemSchema),
+      table: Internal.makeCacheTable(options.name)
+    };
+  } else {
+    tmp = undefined;
+  }
   return {
           name: options.name,
           handler: handler,
-          cache: Belt_Option.getWithDefault(options.cache, false),
-          output: S$RescriptSchema.schema(function (param) {
-                return options.output;
-              }),
+          cache: tmp,
+          output: outputSchema,
           input: S$RescriptSchema.schema(function (param) {
                 return options.input;
               }),
@@ -22,4 +40,4 @@ function experimental_createEffect(options, handler) {
 }
 
 exports.experimental_createEffect = experimental_createEffect;
-/* Prometheus Not a pure module */
+/* Internal Not a pure module */

package/src/Internal.res

@@ -194,14 +194,31 @@ type effectArgs = {
   context: effectContext,
   cacheKey: string,
 }
+type effectCacheItem = {id: string, output: effectOutput}
+type effectCacheMeta = {
+  itemSchema: S.t<effectCacheItem>,
+  rowsSchema: S.t<array<effectCacheItem>>,
+  table: Table.table,
+}
 type effect = {
   name: string,
   handler: effectArgs => promise<effectOutput>,
-  cache: bool,
+  cache: option<effectCacheMeta>,
   output: S.t<effectOutput>,
   input: S.t<effectInput>,
   mutable callsCount: int,
 }
+let cacheTablePrefix = "envio_effect_"
+let makeCacheTable = (~effectName) => {
+  Table.mkTable(
+    cacheTablePrefix ++ effectName,
+    ~fields=[
+      Table.mkField("id", Text, ~fieldSchema=S.string, ~isPrimaryKey=true),
+      Table.mkField("output", JsonB, ~fieldSchema=S.json(~validate=false)),
+    ],
+    ~compositeIndices=[],
+  )
+}
 
 @genType.import(("./Types.ts", "Invalid"))
 type noEventFilters

package/src/Internal.res.js

@@ -1,6 +1,7 @@
 // Generated by ReScript, PLEASE EDIT WITH CARE
 'use strict';
 
+var Table = require("./db/Table.res.js");
 var $$BigInt = require("./bindings/BigInt.res.js");
 var Js_exn = require("rescript/lib/js/js_exn.js");
 var Address = require("./Address.res.js");
@@ -34,6 +35,15 @@ function makeEnumConfig(name, variants) {
         };
 }
 
+var cacheTablePrefix = "envio_effect_";
+
+function makeCacheTable(effectName) {
+  return Table.mkTable(cacheTablePrefix + effectName, [], [
+              Table.mkField("id", "TEXT", S$RescriptSchema.string, undefined, undefined, undefined, true, undefined, undefined),
+              Table.mkField("output", "JSONB", S$RescriptSchema.json(false), undefined, undefined, undefined, undefined, undefined, undefined)
+            ]);
+}
+
 function prettifyExn(exn) {
   var e = Caml_js_exceptions.internalToOCamlException(exn);
   if (e.RE_EXN_ID === Js_exn.$$Error) {
@@ -46,5 +56,7 @@ function prettifyExn(exn) {
 exports.fuelSupplyParamsSchema = fuelSupplyParamsSchema;
 exports.fuelTransferParamsSchema = fuelTransferParamsSchema;
 exports.makeEnumConfig = makeEnumConfig;
+exports.cacheTablePrefix = cacheTablePrefix;
+exports.makeCacheTable = makeCacheTable;
 exports.prettifyExn = prettifyExn;
 /* fuelSupplyParamsSchema Not a pure module */

package/src/Persistence.res

@@ -49,10 +49,8 @@ type storage = {
   ) => promise<unit>,
   @raises("StorageError")
   setEffectCacheOrThrow: (
-    ~effectName: string,
-    ~ids: array<string>,
-    ~outputs: array<Internal.effectOutput>,
-    ~outputSchema: S.t<Internal.effectOutput>,
+    ~effect: Internal.effect,
+    ~items: array<Internal.effectCacheItem>,
     ~initialize: bool,
   ) => promise<unit>,
   dumpEffectCache: unit => promise<unit>,
@@ -176,13 +174,14 @@ let getInitializedStorageOrThrow = persistence => {
   }
 }
 
-let setEffectCacheOrThrow = async (persistence, ~effectName, ~ids, ~outputs, ~outputSchema) => {
+let setEffectCacheOrThrow = async (persistence, ~effect: Internal.effect, ~items) => {
   switch persistence.storageStatus {
   | Unknown
   | Initializing(_) =>
     Js.Exn.raiseError(`Failed to access the indexer storage. The Persistence layer is not initialized.`)
   | Ready({cache}) => {
       let storage = persistence.storage
+      let effectName = effect.name
       let effectCacheRecord = switch cache->Utils.Dict.dangerouslyGetNonOption(effectName) {
       | Some(c) => c
       | None => {
@@ -192,8 +191,8 @@ let setEffectCacheOrThrow = async (persistence, ~effectName, ~ids, ~outputs, ~ou
         }
       }
       let initialize = effectCacheRecord.count === 0
-      await storage.setEffectCacheOrThrow(~effectName, ~ids, ~outputs, ~outputSchema, ~initialize)
-      effectCacheRecord.count = effectCacheRecord.count + ids->Js.Array2.length
+      await storage.setEffectCacheOrThrow(~effect, ~items, ~initialize)
+      effectCacheRecord.count = effectCacheRecord.count + items->Js.Array2.length
       Prometheus.EffectCacheCount.set(~count=effectCacheRecord.count, ~effectName)
     }
   }

package/src/Persistence.res.js

@@ -113,7 +113,7 @@ function getInitializedStorageOrThrow(persistence) {
   }
 }
 
-async function setEffectCacheOrThrow(persistence, effectName, ids, outputs, outputSchema) {
+async function setEffectCacheOrThrow(persistence, effect, items) {
   var match = persistence.storageStatus;
   if (typeof match !== "object") {
     return Js_exn.raiseError("Failed to access the indexer storage. The Persistence layer is not initialized.");
@@ -123,6 +123,7 @@ async function setEffectCacheOrThrow(persistence, effectName, ids, outputs, outp
   }
   var cache = match.cache;
   var storage = persistence.storage;
+  var effectName = effect.name;
   var c = cache[effectName];
   var effectCacheRecord;
   if (c !== undefined) {
@@ -136,8 +137,8 @@ async function setEffectCacheOrThrow(persistence, effectName, ids, outputs, outp
     effectCacheRecord = c$1;
   }
   var initialize = effectCacheRecord.count === 0;
-  await storage.setEffectCacheOrThrow(effectName, ids, outputs, outputSchema, initialize);
-  effectCacheRecord.count = effectCacheRecord.count + ids.length | 0;
+  await storage.setEffectCacheOrThrow(effect, items, initialize);
+  effectCacheRecord.count = effectCacheRecord.count + items.length | 0;
   return Prometheus.EffectCacheCount.set(effectCacheRecord.count, effectName);
 }
 

package/src/PgStorage.res

@@ -433,8 +433,7 @@ let makeSchemaTableNamesQuery = (~pgSchema) => {
   `SELECT table_name FROM information_schema.tables WHERE table_schema = '${pgSchema}';`
 }
 
-let cacheTablePrefix = "envio_effect_"
-let cacheTablePrefixLength = cacheTablePrefix->String.length
+let cacheTablePrefixLength = Internal.cacheTablePrefix->String.length
 
 type schemaCacheTableInfo = {
   @as("table_name")
@@ -449,7 +448,7 @@ let makeSchemaCacheTableInfoQuery = (~pgSchema) => {
     get_cache_row_count(t.table_name) as count
    FROM information_schema.tables t
    WHERE t.table_schema = '${pgSchema}' 
-   AND t.table_name LIKE '${cacheTablePrefix}%';`
+   AND t.table_name LIKE '${Internal.cacheTablePrefix}%';`
 }
 
 type psqlExecState =
@@ -681,20 +680,17 @@ let make = (
   }
 
   let setEffectCacheOrThrow = async (
-    ~effectName: string,
-    ~ids: array<string>,
-    ~outputs: array<Internal.effectOutput>,
-    ~outputSchema: S.t<Internal.effectOutput>,
+    ~effect: Internal.effect,
+    ~items: array<Internal.effectCacheItem>,
     ~initialize: bool,
   ) => {
-    let table = Table.mkTable(
-      cacheTablePrefix ++ effectName,
-      ~fields=[
-        Table.mkField("id", Text, ~fieldSchema=S.string, ~isPrimaryKey=true),
-        Table.mkField("output", JsonB, ~fieldSchema=S.json(~validate=false)),
-      ],
-      ~compositeIndices=[],
-    )
+    let {table, itemSchema} = switch effect.cache {
+    | Some(cacheMeta) => cacheMeta
+    | None =>
+      Js.Exn.raiseError(
+        `Failed to set effect cache for "${effect.name}". Effect has no cache enabled.`,
+      )
+    }
 
     if initialize {
       let _ = await sql->Postgres.unsafe(makeCreateTableQuery(table, ~pgSchema))
@@ -704,26 +700,7 @@ let make = (
       }
     }
 
-    let items = []
-    for idx in 0 to outputs->Array.length - 1 {
-      items
-      ->Js.Array2.push({
-        "id": ids[idx],
-        "output": outputs[idx],
-      })
-      ->ignore
-    }
-
-    await setOrThrow(
-      ~items,
-      ~table,
-      ~itemSchema=S.schema(s =>
-        {
-          "id": s.matches(S.string),
-          "output": s.matches(outputSchema),
-        }
-      ),
-    )
+    await setOrThrow(~items, ~table, ~itemSchema)
   }
 
   let dumpEffectCache = async () => {
@@ -807,23 +784,15 @@ let make = (
           let _ =
             await cacheFiles
             ->Js.Array2.map(entry => {
-              let cacheName = entry->Js.String2.slice(~from=0, ~to_=-4) // Remove .tsv extension
-              let tableName = cacheTablePrefix ++ cacheName
-              let table = Table.mkTable(
-                tableName,
-                ~fields=[
-                  Table.mkField("id", Text, ~fieldSchema=S.string, ~isPrimaryKey=true),
-                  Table.mkField("output", JsonB, ~fieldSchema=S.json(~validate=false)),
-                ],
-                ~compositeIndices=[],
-              )
+              let effectName = entry->Js.String2.slice(~from=0, ~to_=-4) // Remove .tsv extension
+              let table = Internal.makeCacheTable(~effectName)
 
               sql
               ->Postgres.unsafe(makeCreateTableQuery(table, ~pgSchema))
               ->Promise.then(() => {
                 let inputFile = NodeJs.Path.join(cacheDirPath, entry)->NodeJs.Path.toString
 
-                let command = `${psqlExec} -c 'COPY "${pgSchema}"."${tableName}" FROM STDIN WITH (FORMAT text, HEADER);' < ${inputFile}`
+                let command = `${psqlExec} -c 'COPY "${pgSchema}"."${table.tableName}" FROM STDIN WITH (FORMAT text, HEADER);' < ${inputFile}`
 
                 Promise.make(
                   (resolve, reject) => {

package/src/PgStorage.res.js

@@ -13,7 +13,6 @@ var Logging = require("./Logging.res.js");
 var $$Promise = require("./bindings/Promise.res.js");
 var Internal = require("./Internal.res.js");
 var Belt_Array = require("rescript/lib/js/belt_Array.js");
-var Caml_array = require("rescript/lib/js/caml_array.js");
 var Belt_Option = require("rescript/lib/js/belt_Option.js");
 var Caml_option = require("rescript/lib/js/caml_option.js");
 var Persistence = require("./Persistence.res.js");
@@ -297,12 +296,10 @@ function makeSchemaTableNamesQuery(pgSchema) {
   return "SELECT table_name FROM information_schema.tables WHERE table_schema = '" + pgSchema + "';";
 }
 
-var cacheTablePrefix = "envio_effect_";
-
-var cacheTablePrefixLength = cacheTablePrefix.length;
+var cacheTablePrefixLength = Internal.cacheTablePrefix.length;
 
 function makeSchemaCacheTableInfoQuery(pgSchema) {
-  return "SELECT \n    t.table_name,\n    get_cache_row_count(t.table_name) as count\n   FROM information_schema.tables t\n   WHERE t.table_schema = '" + pgSchema + "' \n   AND t.table_name LIKE '" + cacheTablePrefix + "%';";
+  return "SELECT \n    t.table_name,\n    get_cache_row_count(t.table_name) as count\n   FROM information_schema.tables t\n   WHERE t.table_schema = '" + pgSchema + "' \n   AND t.table_name LIKE '" + Internal.cacheTablePrefix + "%';";
 }
 
 var psqlExecState = {
@@ -467,11 +464,10 @@ function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, onI
   var setOrThrow$1 = function (items, table, itemSchema) {
     return setOrThrow(sql, items, table, itemSchema, pgSchema);
   };
-  var setEffectCacheOrThrow = async function (effectName, ids, outputs, outputSchema, initialize) {
-    var table = Table.mkTable(cacheTablePrefix + effectName, [], [
-          Table.mkField("id", "TEXT", S$RescriptSchema.string, undefined, undefined, undefined, true, undefined, undefined),
-          Table.mkField("output", "JSONB", S$RescriptSchema.json(false), undefined, undefined, undefined, undefined, undefined, undefined)
-        ]);
+  var setEffectCacheOrThrow = async function (effect, items, initialize) {
+    var cacheMeta = effect.cache;
+    var match = cacheMeta !== undefined ? cacheMeta : Js_exn.raiseError("Failed to set effect cache for \"" + effect.name + "\". Effect has no cache enabled.");
+    var table = match.table;
     if (initialize) {
       await sql.unsafe(makeCreateTableQuery(table, pgSchema));
       if (onNewTables !== undefined) {
@@ -479,19 +475,7 @@ function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, onI
       }
       
     }
-    var items = [];
-    for(var idx = 0 ,idx_finish = outputs.length; idx < idx_finish; ++idx){
-      items.push({
-            id: Caml_array.get(ids, idx),
-            output: Caml_array.get(outputs, idx)
-          });
-    }
-    return await setOrThrow$1(items, table, S$RescriptSchema.schema(function (s) {
-                    return {
-                            id: s.m(S$RescriptSchema.string),
-                            output: s.m(outputSchema)
-                          };
-                  }));
+    return await setOrThrow$1(items, table, match.itemSchema);
   };
   var dumpEffectCache = async function () {
     try {
@@ -568,15 +552,11 @@ function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, onI
                 return entry.endsWith(".tsv");
               });
           await Promise.all(cacheFiles.map(function (entry) {
-                    var cacheName = entry.slice(0, -4);
-                    var tableName = cacheTablePrefix + cacheName;
-                    var table = Table.mkTable(tableName, [], [
-                          Table.mkField("id", "TEXT", S$RescriptSchema.string, undefined, undefined, undefined, true, undefined, undefined),
-                          Table.mkField("output", "JSONB", S$RescriptSchema.json(false), undefined, undefined, undefined, undefined, undefined, undefined)
-                        ]);
+                    var effectName = entry.slice(0, -4);
+                    var table = Internal.makeCacheTable(effectName);
                     return sql.unsafe(makeCreateTableQuery(table, pgSchema)).then(function () {
                                 var inputFile = Path.join(cacheDirPath, entry);
-                                var command = psqlExec$1 + " -c 'COPY \"" + pgSchema + "\".\"" + tableName + "\" FROM STDIN WITH (FORMAT text, HEADER);' < " + inputFile;
+                                var command = psqlExec$1 + " -c 'COPY \"" + pgSchema + "\".\"" + table.tableName + "\" FROM STDIN WITH (FORMAT text, HEADER);' < " + inputFile;
                                 return new Promise((function (resolve, reject) {
                                               Child_process.exec(command, psqlExecOptions, (function (error, stdout, param) {
                                                       if (error === null) {
@@ -654,7 +634,6 @@ exports.setQueryCache = setQueryCache;
 exports.setOrThrow = setOrThrow;
 exports.setEntityHistoryOrThrow = setEntityHistoryOrThrow;
 exports.makeSchemaTableNamesQuery = makeSchemaTableNamesQuery;
-exports.cacheTablePrefix = cacheTablePrefix;
 exports.cacheTablePrefixLength = cacheTablePrefixLength;
 exports.makeSchemaCacheTableInfoQuery = makeSchemaCacheTableInfoQuery;
 exports.getConnectedPsqlExec = getConnectedPsqlExec;