envio 2.26.0-alpha.7 → 2.26.0-alpha.9

package/index.d.ts

@@ -35,6 +35,26 @@ type UnknownToOutput<T> = T extends Sury.Schema<unknown>
     >
   : T;
 
+type UnknownToInput<T> = T extends Sury.Schema<unknown>
+  ? Sury.Input<T>
+  : T extends (...args: any[]) => any
+  ? T
+  : T extends unknown[]
+  ? { [K in keyof T]: UnknownToInput<T[K]> }
+  : T extends { [k in keyof T]: unknown }
+  ? Flatten<
+      {
+        [k in keyof T as HasUndefined<UnknownToInput<T[k]>> extends true
+          ? k
+          : never]?: UnknownToInput<T[k]>;
+      } & {
+        [k in keyof T as HasUndefined<UnknownToInput<T[k]>> extends true
+          ? never
+          : k]: UnknownToInput<T[k]>;
+      }
+    >
+  : T;
+
 type HasUndefined<T> = [T] extends [undefined]
   ? true
   : undefined extends T
@@ -72,7 +92,7 @@ type Flatten<T> = T extends object
 export function experimental_createEffect<
   IS,
   OS,
-  I = UnknownToOutput<IS>,
+  I = UnknownToInput<IS>,
   O = UnknownToOutput<OS>,
   // A hack to enforce that the inferred return type
   // matches the output schema type
@@ -110,7 +130,6 @@ export declare namespace S {
   // Don't expose recursive for now, since it's too advanced
   // export const recursive: typeof Sury.recursive;
   export const transform: typeof Sury.transform;
-  export const shape: typeof Sury.shape;
   export const refine: typeof Sury.refine;
   export const schema: typeof Sury.schema;
   export const record: typeof Sury.record;

package/index.js

@@ -22,7 +22,6 @@ exports.S = {
   // Don't expose recursive for now, since it's too advanced
   // recursive: Sury.recursive,
   transform: Sury.transform,
-  shape: Sury.shape,
   refine: Sury.refine,
   schema: Sury.schema,
   record: Sury.record,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "envio",
-  "version": "v2.26.0-alpha.7",
+  "version": "v2.26.0-alpha.9",
   "description": "A latency and sync speed optimized, developer friendly blockchain data indexer.",
   "bin": "./bin.js",
   "main": "./index.js",
@@ -25,10 +25,10 @@
   },
   "homepage": "https://envio.dev",
   "optionalDependencies": {
-    "envio-linux-x64": "v2.26.0-alpha.7",
-    "envio-linux-arm64": "v2.26.0-alpha.7",
-    "envio-darwin-x64": "v2.26.0-alpha.7",
-    "envio-darwin-arm64": "v2.26.0-alpha.7"
+    "envio-linux-x64": "v2.26.0-alpha.9",
+    "envio-linux-arm64": "v2.26.0-alpha.9",
+    "envio-darwin-x64": "v2.26.0-alpha.9",
+    "envio-darwin-arm64": "v2.26.0-alpha.9"
   },
   "dependencies": {
     "@envio-dev/hypersync-client": "0.6.5",

package/src/Envio.res

@@ -50,16 +50,6 @@ let experimental_createEffect = (
       >
     ),
     callsCount: 0,
-    // This is the way to make the createEffect API
-    // work without the need for users to call S.schema themselves,
-    // but simply pass the desired object/tuple/etc.
-    // If they pass a schem, it'll also work.
-    input: S.schema(_ => options.input)->(
-      Utils.magic: S.t<S.t<'input>> => S.t<Internal.effectInput>
-    ),
-    output: S.schema(_ => options.output)->(
-      Utils.magic: S.t<S.t<'output>> => S.t<Internal.effectOutput>
-    ),
     cache: options.cache->Belt.Option.getWithDefault(false),
   }->(Utils.magic: Internal.effect => effect<'input, 'output>)
 }

package/src/Envio.res.js

@@ -3,7 +3,6 @@
 
 var Prometheus = require("./Prometheus.res.js");
 var Belt_Option = require("rescript/lib/js/belt_Option.js");
-var S$RescriptSchema = require("rescript-schema/src/S.res.js");
 
 function experimental_createEffect(options, handler) {
   Prometheus.EffectCallsCount.set(0, options.name);
@@ -11,12 +10,6 @@ function experimental_createEffect(options, handler) {
           name: options.name,
           handler: handler,
           cache: Belt_Option.getWithDefault(options.cache, false),
-          output: S$RescriptSchema.schema(function (param) {
-                return options.output;
-              }),
-          input: S$RescriptSchema.schema(function (param) {
-                return options.input;
-              }),
           callsCount: 0
         };
 }

package/src/Internal.res

@@ -198,8 +198,6 @@ type effect = {
   name: string,
   handler: effectArgs => promise<effectOutput>,
   cache: bool,
-  output: S.t<effectOutput>,
-  input: S.t<effectInput>,
   mutable callsCount: int,
 }
 

package/src/Persistence.res

@@ -5,12 +5,14 @@
 // Currently there are quite many code spread across
 // DbFunctions, Db, Migrations, InMemoryStore modules which use codegen code directly.
 
-// The type reflects an cache table in the db
+// The type reflects an effect cache table in the db
 // It might be present even if the effect is not used in the application
-type effectCacheRecord = {
-  effectName: string,
+type effectCache = {
+  name: string,
   // Number of rows in the table
-  mutable count: int,
+  mutable size: int,
+  // Lazily attached table definition when effect is used in the application
+  mutable table: option<Table.table>,
 }
 
 type operator = [#">" | #"="]
@@ -26,6 +28,7 @@ type storage = {
     ~generalTables: array<Table.table>=?,
     ~enums: array<Internal.enumConfig<Internal.enum>>=?,
   ) => promise<unit>,
+  loadEffectCaches: unit => promise<array<effectCache>>,
   @raises("StorageError")
   loadByIdsOrThrow: 'item. (
     ~ids: array<string>,
@@ -47,16 +50,6 @@ type storage = {
     ~table: Table.table,
     ~itemSchema: S.t<'item>,
   ) => promise<unit>,
-  @raises("StorageError")
-  setEffectCacheOrThrow: (
-    ~effectName: string,
-    ~ids: array<string>,
-    ~outputs: array<Internal.effectOutput>,
-    ~outputSchema: S.t<Internal.effectOutput>,
-    ~initialize: bool,
-  ) => promise<unit>,
-  dumpEffectCache: unit => promise<unit>,
-  restoreEffectCache: (~withUpload: bool) => promise<array<effectCacheRecord>>,
 }
 
 exception StorageError({message: string, reason: exn})
@@ -64,7 +57,7 @@ exception StorageError({message: string, reason: exn})
 type storageStatus =
   | Unknown
   | Initializing(promise<unit>)
-  | Ready({cleanRun: bool, cache: dict<effectCacheRecord>})
+  | Ready({cleanRun: bool, effectCaches: dict<effectCache>})
 
 type t = {
   userEntities: array<Internal.entityConfig>,
@@ -73,6 +66,7 @@ type t = {
   allEnums: array<Internal.enumConfig<Internal.enum>>,
   mutable storageStatus: storageStatus,
   storage: storage,
+  onStorageInitialize: option<unit => promise<unit>>,
 }
 
 let entityHistoryActionEnumConfig: Internal.enumConfig<EntityHistory.RowAction.t> = {
@@ -89,6 +83,7 @@ let make = (
   ~allEnums,
   ~staticTables,
   ~storage,
+  ~onStorageInitialize=?,
 ) => {
   let allEntities = userEntities->Js.Array2.concat([dcRegistryEntityConfig])
   let allEnums =
@@ -100,70 +95,62 @@ let make = (
     allEnums,
     storageStatus: Unknown,
     storage,
+    onStorageInitialize,
   }
 }
 
-let init = {
-  let loadInitialCache = async (persistence, ~withUpload) => {
-    let effectCacheRecords = await persistence.storage.restoreEffectCache(~withUpload)
-    let cache = Js.Dict.empty()
-    effectCacheRecords->Js.Array2.forEach(record => {
-      Prometheus.EffectCacheCount.set(~count=record.count, ~effectName=record.effectName)
-      cache->Js.Dict.set(record.effectName, record)
-    })
-    cache
-  }
-
-  async (persistence, ~reset=false) => {
-    try {
-      let shouldRun = switch persistence.storageStatus {
-      | Unknown => true
-      | Initializing(promise) => {
-          await promise
-          reset
-        }
-      | Ready(_) => reset
+let init = async (persistence, ~reset=false) => {
+  try {
+    let shouldRun = switch persistence.storageStatus {
+    | Unknown => true
+    | Initializing(promise) => {
+        await promise
+        reset
       }
-      if shouldRun {
-        let resolveRef = ref(%raw(`null`))
-        let promise = Promise.make((resolve, _) => {
-          resolveRef := resolve
-        })
-        persistence.storageStatus = Initializing(promise)
-        if reset || !(await persistence.storage.isInitialized()) {
-          Logging.info(`Initializing the indexer storage...`)
-
-          await persistence.storage.initialize(
-            ~entities=persistence.allEntities,
-            ~generalTables=persistence.staticTables,
-            ~enums=persistence.allEnums,
-          )
+    | Ready(_) => reset
+    }
+    if shouldRun {
+      let resolveRef = ref(%raw(`null`))
+      let promise = Promise.make((resolve, _) => {
+        resolveRef := resolve
+      })
+      persistence.storageStatus = Initializing(promise)
+      if reset || !(await persistence.storage.isInitialized()) {
+        let _ = await persistence.storage.initialize(
+          ~entities=persistence.allEntities,
+          ~generalTables=persistence.staticTables,
+          ~enums=persistence.allEnums,
+        )
 
-          Logging.info(`The indexer storage is ready. Uploading cache...`)
-          persistence.storageStatus = Ready({
-            cleanRun: true,
-            cache: await loadInitialCache(persistence, ~withUpload=true),
-          })
-        } else if (
-          // In case of a race condition,
-          // we want to set the initial status to Ready only once.
-          switch persistence.storageStatus {
-          | Initializing(_) => true
-          | _ => false
-          }
-        ) {
-          Logging.info(`The indexer storage is ready.`)
-          persistence.storageStatus = Ready({
-            cleanRun: false,
-            cache: await loadInitialCache(persistence, ~withUpload=false),
-          })
+        persistence.storageStatus = Ready({
+          cleanRun: true,
+          effectCaches: Js.Dict.empty(),
+        })
+        switch persistence.onStorageInitialize {
+        | Some(onStorageInitialize) => await onStorageInitialize()
+        | None => ()
+        }
+      } else if (
+        // In case of a race condition,
+        // we want to set the initial status to Ready only once.
+        switch persistence.storageStatus {
+        | Initializing(_) => true
+        | _ => false
         }
-        resolveRef.contents()
+      ) {
+        let effectCaches = Js.Dict.empty()
+        (await persistence.storage.loadEffectCaches())->Js.Array2.forEach(effectCache => {
+          effectCaches->Js.Dict.set(effectCache.name, effectCache)
+        })
+        persistence.storageStatus = Ready({
+          cleanRun: false,
+          effectCaches,
+        })
       }
-    } catch {
-    | exn =>
-      exn->ErrorHandling.mkLogAndRaise(~msg=`EE800: Failed to initialize the indexer storage.`)
+      resolveRef.contents()
     }
+  } catch {
+  | exn => exn->ErrorHandling.mkLogAndRaise(~msg=`EE800: Failed to initialize the indexer storage.`)
   }
 }
 
@@ -175,26 +162,3 @@ let getInitializedStorageOrThrow = persistence => {
   | Ready(_) => persistence.storage
   }
 }
-
-let setEffectCacheOrThrow = async (persistence, ~effectName, ~ids, ~outputs, ~outputSchema) => {
-  switch persistence.storageStatus {
-  | Unknown
-  | Initializing(_) =>
-    Js.Exn.raiseError(`Failed to access the indexer storage. The Persistence layer is not initialized.`)
-  | Ready({cache}) => {
-      let storage = persistence.storage
-      let effectCacheRecord = switch cache->Utils.Dict.dangerouslyGetNonOption(effectName) {
-      | Some(c) => c
-      | None => {
-          let c = {effectName, count: 0}
-          cache->Js.Dict.set(effectName, c)
-          c
-        }
-      }
-      let initialize = effectCacheRecord.count === 0
-      await storage.setEffectCacheOrThrow(~effectName, ~ids, ~outputs, ~outputSchema, ~initialize)
-      effectCacheRecord.count = effectCacheRecord.count + ids->Js.Array2.length
-      Prometheus.EffectCacheCount.set(~count=effectCacheRecord.count, ~effectName)
-    }
-  }
-}

package/src/Persistence.res.js

@@ -2,8 +2,6 @@
 'use strict';
 
 var Js_exn = require("rescript/lib/js/js_exn.js");
-var Logging = require("./Logging.res.js");
-var Prometheus = require("./Prometheus.res.js");
 var EntityHistory = require("./db/EntityHistory.res.js");
 var ErrorHandling = require("./ErrorHandling.res.js");
 var Caml_exceptions = require("rescript/lib/js/caml_exceptions.js");
@@ -24,7 +22,7 @@ var entityHistoryActionEnumConfig = {
   default: "SET"
 };
 
-function make(userEntities, dcRegistryEntityConfig, allEnums, staticTables, storage) {
+function make(userEntities, dcRegistryEntityConfig, allEnums, staticTables, storage, onStorageInitialize) {
   var allEntities = userEntities.concat([dcRegistryEntityConfig]);
   var allEnums$1 = allEnums.concat([entityHistoryActionEnumConfig]);
   return {
@@ -33,20 +31,11 @@ function make(userEntities, dcRegistryEntityConfig, allEnums, staticTables, stor
           allEntities: allEntities,
           allEnums: allEnums$1,
           storageStatus: "Unknown",
-          storage: storage
+          storage: storage,
+          onStorageInitialize: onStorageInitialize
         };
 }
 
-async function loadInitialCache(persistence, withUpload) {
-  var effectCacheRecords = await persistence.storage.restoreEffectCache(withUpload);
-  var cache = {};
-  effectCacheRecords.forEach(function (record) {
-        Prometheus.EffectCacheCount.set(record.count, record.effectName);
-        cache[record.effectName] = record;
-      });
-  return cache;
-}
-
 async function init(persistence, resetOpt) {
   var reset = resetOpt !== undefined ? resetOpt : false;
   try {
@@ -74,24 +63,30 @@ async function init(persistence, resetOpt) {
       _0: promise$1
     };
     if (reset || !await persistence.storage.isInitialized()) {
-      Logging.info("Initializing the indexer storage...");
       await persistence.storage.initialize(persistence.allEntities, persistence.staticTables, persistence.allEnums);
-      Logging.info("The indexer storage is ready. Uploading cache...");
       persistence.storageStatus = {
         TAG: "Ready",
         cleanRun: true,
-        cache: await loadInitialCache(persistence, true)
+        effectCaches: {}
       };
+      var onStorageInitialize = persistence.onStorageInitialize;
+      if (onStorageInitialize !== undefined) {
+        await onStorageInitialize();
+      }
+      
     } else {
       var match = persistence.storageStatus;
       var tmp;
       tmp = typeof match !== "object" || match.TAG !== "Initializing" ? false : true;
       if (tmp) {
-        Logging.info("The indexer storage is ready.");
+        var effectCaches = {};
+        (await persistence.storage.loadEffectCaches()).forEach(function (effectCache) {
+              effectCaches[effectCache.name] = effectCache;
+            });
         persistence.storageStatus = {
           TAG: "Ready",
           cleanRun: false,
-          cache: await loadInitialCache(persistence, false)
+          effectCaches: effectCaches
         };
       }
       
@@ -113,38 +108,9 @@ function getInitializedStorageOrThrow(persistence) {
   }
 }
 
-async function setEffectCacheOrThrow(persistence, effectName, ids, outputs, outputSchema) {
-  var match = persistence.storageStatus;
-  if (typeof match !== "object") {
-    return Js_exn.raiseError("Failed to access the indexer storage. The Persistence layer is not initialized.");
-  }
-  if (match.TAG === "Initializing") {
-    return Js_exn.raiseError("Failed to access the indexer storage. The Persistence layer is not initialized.");
-  }
-  var cache = match.cache;
-  var storage = persistence.storage;
-  var c = cache[effectName];
-  var effectCacheRecord;
-  if (c !== undefined) {
-    effectCacheRecord = c;
-  } else {
-    var c$1 = {
-      effectName: effectName,
-      count: 0
-    };
-    cache[effectName] = c$1;
-    effectCacheRecord = c$1;
-  }
-  var initialize = effectCacheRecord.count === 0;
-  await storage.setEffectCacheOrThrow(effectName, ids, outputs, outputSchema, initialize);
-  effectCacheRecord.count = effectCacheRecord.count + ids.length | 0;
-  return Prometheus.EffectCacheCount.set(effectCacheRecord.count, effectName);
-}
-
 exports.StorageError = StorageError;
 exports.entityHistoryActionEnumConfig = entityHistoryActionEnumConfig;
 exports.make = make;
 exports.init = init;
 exports.getInitializedStorageOrThrow = getInitializedStorageOrThrow;
-exports.setEffectCacheOrThrow = setEffectCacheOrThrow;
-/* Logging Not a pure module */
+/* EntityHistory Not a pure module */