envio 2.26.0-alpha.9 → 2.26.0-rc.1

package/index.d.ts

@@ -13,6 +13,7 @@ import type {
 } from "./src/Envio.gen.ts";
 
 import { schema as bigDecimalSchema } from "./src/bindings/BigDecimal.gen.ts";
+import { schema as bigintSchema } from "./src/bindings/BigInt.gen.ts";
 import * as Sury from "rescript-schema";
 
 type UnknownToOutput<T> = T extends Sury.Schema<unknown>
@@ -35,26 +36,6 @@ type UnknownToOutput<T> = T extends Sury.Schema<unknown>
     >
   : T;
 
-type UnknownToInput<T> = T extends Sury.Schema<unknown>
-  ? Sury.Input<T>
-  : T extends (...args: any[]) => any
-  ? T
-  : T extends unknown[]
-  ? { [K in keyof T]: UnknownToInput<T[K]> }
-  : T extends { [k in keyof T]: unknown }
-  ? Flatten<
-      {
-        [k in keyof T as HasUndefined<UnknownToInput<T[k]>> extends true
-          ? k
-          : never]?: UnknownToInput<T[k]>;
-      } & {
-        [k in keyof T as HasUndefined<UnknownToInput<T[k]>> extends true
-          ? never
-          : k]: UnknownToInput<T[k]>;
-      }
-    >
-  : T;
-
 type HasUndefined<T> = [T] extends [undefined]
   ? true
   : undefined extends T
@@ -92,7 +73,7 @@ type Flatten<T> = T extends object
 export function experimental_createEffect<
   IS,
   OS,
-  I = UnknownToInput<IS>,
+  I = UnknownToOutput<IS>,
   O = UnknownToOutput<OS>,
   // A hack to enforce that the inferred return type
   // matches the output schema type
@@ -121,7 +102,7 @@ export declare namespace S {
   export const boolean: typeof Sury.boolean;
   export const int32: typeof Sury.int32;
   export const number: typeof Sury.number;
-  export const bigint: typeof Sury.bigint;
+  export const bigint: typeof bigintSchema;
   export const never: typeof Sury.never;
   export const union: typeof Sury.union;
   export const object: typeof Sury.object;
@@ -130,6 +111,7 @@ export declare namespace S {
   // Don't expose recursive for now, since it's too advanced
   // export const recursive: typeof Sury.recursive;
   export const transform: typeof Sury.transform;
+  export const shape: typeof Sury.shape;
   export const refine: typeof Sury.refine;
   export const schema: typeof Sury.schema;
   export const record: typeof Sury.record;

package/index.js

@@ -13,7 +13,7 @@ exports.S = {
   boolean: Sury.boolean,
   int32: Sury.int32,
   number: Sury.number,
-  bigint: Sury.bigint,
+  bigint: require("./src/bindings/BigInt.res.js").schema,
   never: Sury.never,
   union: Sury.union,
   object: Sury.object,
@@ -22,6 +22,7 @@ exports.S = {
   // Don't expose recursive for now, since it's too advanced
   // recursive: Sury.recursive,
   transform: Sury.transform,
+  shape: Sury.shape,
   refine: Sury.refine,
   schema: Sury.schema,
   record: Sury.record,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "envio",
-  "version": "v2.26.0-alpha.9",
+  "version": "v2.26.0-rc.1",
   "description": "A latency and sync speed optimized, developer friendly blockchain data indexer.",
   "bin": "./bin.js",
   "main": "./index.js",
@@ -25,10 +25,10 @@
   },
   "homepage": "https://envio.dev",
   "optionalDependencies": {
-    "envio-linux-x64": "v2.26.0-alpha.9",
-    "envio-linux-arm64": "v2.26.0-alpha.9",
-    "envio-darwin-x64": "v2.26.0-alpha.9",
-    "envio-darwin-arm64": "v2.26.0-alpha.9"
+    "envio-linux-x64": "v2.26.0-rc.1",
+    "envio-linux-arm64": "v2.26.0-rc.1",
+    "envio-darwin-x64": "v2.26.0-rc.1",
+    "envio-darwin-arm64": "v2.26.0-rc.1"
   },
   "dependencies": {
     "@envio-dev/hypersync-client": "0.6.5",

package/src/Envio.res

@@ -50,6 +50,16 @@ let experimental_createEffect = (
       >
     ),
     callsCount: 0,
+    // This is the way to make the createEffect API
+    // work without the need for users to call S.schema themselves,
+    // but simply pass the desired object/tuple/etc.
+    // If they pass a schem, it'll also work.
+    input: S.schema(_ => options.input)->(
+      Utils.magic: S.t<S.t<'input>> => S.t<Internal.effectInput>
+    ),
+    output: S.schema(_ => options.output)->(
+      Utils.magic: S.t<S.t<'output>> => S.t<Internal.effectOutput>
+    ),
     cache: options.cache->Belt.Option.getWithDefault(false),
   }->(Utils.magic: Internal.effect => effect<'input, 'output>)
 }

package/src/Envio.res.js

@@ -3,6 +3,7 @@
 
 var Prometheus = require("./Prometheus.res.js");
 var Belt_Option = require("rescript/lib/js/belt_Option.js");
+var S$RescriptSchema = require("rescript-schema/src/S.res.js");
 
 function experimental_createEffect(options, handler) {
   Prometheus.EffectCallsCount.set(0, options.name);
@@ -10,6 +11,12 @@ function experimental_createEffect(options, handler) {
           name: options.name,
           handler: handler,
           cache: Belt_Option.getWithDefault(options.cache, false),
+          output: S$RescriptSchema.schema(function (param) {
+                return options.output;
+              }),
+          input: S$RescriptSchema.schema(function (param) {
+                return options.input;
+              }),
           callsCount: 0
         };
 }

package/src/Internal.res

@@ -198,6 +198,8 @@ type effect = {
   name: string,
   handler: effectArgs => promise<effectOutput>,
   cache: bool,
+  output: S.t<effectOutput>,
+  input: S.t<effectInput>,
   mutable callsCount: int,
 }
 

package/src/Persistence.res

@@ -5,14 +5,12 @@
 // Currently there are quite many code spread across
 // DbFunctions, Db, Migrations, InMemoryStore modules which use codegen code directly.
 
-// The type reflects an effect cache table in the db
+// The type reflects an cache table in the db
 // It might be present even if the effect is not used in the application
-type effectCache = {
-  name: string,
+type effectCacheRecord = {
+  effectName: string,
   // Number of rows in the table
-  mutable size: int,
-  // Lazily attached table definition when effect is used in the application
-  mutable table: option<Table.table>,
+  mutable count: int,
 }
 
 type operator = [#">" | #"="]
@@ -28,7 +26,6 @@ type storage = {
     ~generalTables: array<Table.table>=?,
     ~enums: array<Internal.enumConfig<Internal.enum>>=?,
   ) => promise<unit>,
-  loadEffectCaches: unit => promise<array<effectCache>>,
   @raises("StorageError")
   loadByIdsOrThrow: 'item. (
     ~ids: array<string>,
@@ -50,6 +47,16 @@ type storage = {
     ~table: Table.table,
     ~itemSchema: S.t<'item>,
   ) => promise<unit>,
+  @raises("StorageError")
+  setEffectCacheOrThrow: (
+    ~effectName: string,
+    ~ids: array<string>,
+    ~outputs: array<Internal.effectOutput>,
+    ~outputSchema: S.t<Internal.effectOutput>,
+    ~initialize: bool,
+  ) => promise<unit>,
+  dumpEffectCache: unit => promise<unit>,
+  restoreEffectCache: (~withUpload: bool) => promise<array<effectCacheRecord>>,
 }
 
 exception StorageError({message: string, reason: exn})
@@ -57,7 +64,7 @@ exception StorageError({message: string, reason: exn})
 type storageStatus =
   | Unknown
   | Initializing(promise<unit>)
-  | Ready({cleanRun: bool, effectCaches: dict<effectCache>})
+  | Ready({cleanRun: bool, cache: dict<effectCacheRecord>})
 
 type t = {
   userEntities: array<Internal.entityConfig>,
@@ -66,7 +73,6 @@ type t = {
   allEnums: array<Internal.enumConfig<Internal.enum>>,
   mutable storageStatus: storageStatus,
   storage: storage,
-  onStorageInitialize: option<unit => promise<unit>>,
 }
 
 let entityHistoryActionEnumConfig: Internal.enumConfig<EntityHistory.RowAction.t> = {
@@ -83,7 +89,6 @@ let make = (
   ~allEnums,
   ~staticTables,
   ~storage,
-  ~onStorageInitialize=?,
 ) => {
   let allEntities = userEntities->Js.Array2.concat([dcRegistryEntityConfig])
   let allEnums =
@@ -95,62 +100,70 @@ let make = (
     allEnums,
     storageStatus: Unknown,
     storage,
-    onStorageInitialize,
   }
 }
 
-let init = async (persistence, ~reset=false) => {
-  try {
-    let shouldRun = switch persistence.storageStatus {
-    | Unknown => true
-    | Initializing(promise) => {
-        await promise
-        reset
-      }
-    | Ready(_) => reset
-    }
-    if shouldRun {
-      let resolveRef = ref(%raw(`null`))
-      let promise = Promise.make((resolve, _) => {
-        resolveRef := resolve
-      })
-      persistence.storageStatus = Initializing(promise)
-      if reset || !(await persistence.storage.isInitialized()) {
-        let _ = await persistence.storage.initialize(
-          ~entities=persistence.allEntities,
-          ~generalTables=persistence.staticTables,
-          ~enums=persistence.allEnums,
-        )
+let init = {
+  let loadInitialCache = async (persistence, ~withUpload) => {
+    let effectCacheRecords = await persistence.storage.restoreEffectCache(~withUpload)
+    let cache = Js.Dict.empty()
+    effectCacheRecords->Js.Array2.forEach(record => {
+      Prometheus.EffectCacheCount.set(~count=record.count, ~effectName=record.effectName)
+      cache->Js.Dict.set(record.effectName, record)
+    })
+    cache
+  }
 
-        persistence.storageStatus = Ready({
-          cleanRun: true,
-          effectCaches: Js.Dict.empty(),
-        })
-        switch persistence.onStorageInitialize {
-        | Some(onStorageInitialize) => await onStorageInitialize()
-        | None => ()
-        }
-      } else if (
-        // In case of a race condition,
-        // we want to set the initial status to Ready only once.
-        switch persistence.storageStatus {
-        | Initializing(_) => true
-        | _ => false
+  async (persistence, ~reset=false) => {
+    try {
+      let shouldRun = switch persistence.storageStatus {
+      | Unknown => true
+      | Initializing(promise) => {
+          await promise
+          reset
         }
-      ) {
-        let effectCaches = Js.Dict.empty()
-        (await persistence.storage.loadEffectCaches())->Js.Array2.forEach(effectCache => {
-          effectCaches->Js.Dict.set(effectCache.name, effectCache)
-        })
-        persistence.storageStatus = Ready({
-          cleanRun: false,
-          effectCaches,
+      | Ready(_) => reset
+      }
+      if shouldRun {
+        let resolveRef = ref(%raw(`null`))
+        let promise = Promise.make((resolve, _) => {
+          resolveRef := resolve
         })
+        persistence.storageStatus = Initializing(promise)
+        if reset || !(await persistence.storage.isInitialized()) {
+          Logging.info(`Initializing the indexer storage...`)
+
+          await persistence.storage.initialize(
+            ~entities=persistence.allEntities,
+            ~generalTables=persistence.staticTables,
+            ~enums=persistence.allEnums,
+          )
+
+          Logging.info(`The indexer storage is ready. Uploading cache...`)
+          persistence.storageStatus = Ready({
+            cleanRun: true,
+            cache: await loadInitialCache(persistence, ~withUpload=true),
+          })
+        } else if (
+          // In case of a race condition,
+          // we want to set the initial status to Ready only once.
+          switch persistence.storageStatus {
+          | Initializing(_) => true
+          | _ => false
+          }
+        ) {
+          Logging.info(`The indexer storage is ready.`)
+          persistence.storageStatus = Ready({
+            cleanRun: false,
+            cache: await loadInitialCache(persistence, ~withUpload=false),
+          })
+        }
+        resolveRef.contents()
       }
-      resolveRef.contents()
+    } catch {
+    | exn =>
+      exn->ErrorHandling.mkLogAndRaise(~msg=`EE800: Failed to initialize the indexer storage.`)
     }
-  } catch {
-  | exn => exn->ErrorHandling.mkLogAndRaise(~msg=`EE800: Failed to initialize the indexer storage.`)
   }
 }
 
@@ -162,3 +175,26 @@ let getInitializedStorageOrThrow = persistence => {
   | Ready(_) => persistence.storage
   }
 }
+
+let setEffectCacheOrThrow = async (persistence, ~effectName, ~ids, ~outputs, ~outputSchema) => {
+  switch persistence.storageStatus {
+  | Unknown
+  | Initializing(_) =>
+    Js.Exn.raiseError(`Failed to access the indexer storage. The Persistence layer is not initialized.`)
+  | Ready({cache}) => {
+      let storage = persistence.storage
+      let effectCacheRecord = switch cache->Utils.Dict.dangerouslyGetNonOption(effectName) {
+      | Some(c) => c
+      | None => {
+          let c = {effectName, count: 0}
+          cache->Js.Dict.set(effectName, c)
+          c
+        }
+      }
+      let initialize = effectCacheRecord.count === 0
+      await storage.setEffectCacheOrThrow(~effectName, ~ids, ~outputs, ~outputSchema, ~initialize)
+      effectCacheRecord.count = effectCacheRecord.count + ids->Js.Array2.length
+      Prometheus.EffectCacheCount.set(~count=effectCacheRecord.count, ~effectName)
+    }
+  }
+}

package/src/Persistence.res.js

@@ -2,6 +2,8 @@
 'use strict';
 
 var Js_exn = require("rescript/lib/js/js_exn.js");
+var Logging = require("./Logging.res.js");
+var Prometheus = require("./Prometheus.res.js");
 var EntityHistory = require("./db/EntityHistory.res.js");
 var ErrorHandling = require("./ErrorHandling.res.js");
 var Caml_exceptions = require("rescript/lib/js/caml_exceptions.js");
@@ -22,7 +24,7 @@ var entityHistoryActionEnumConfig = {
   default: "SET"
 };
 
-function make(userEntities, dcRegistryEntityConfig, allEnums, staticTables, storage, onStorageInitialize) {
+function make(userEntities, dcRegistryEntityConfig, allEnums, staticTables, storage) {
   var allEntities = userEntities.concat([dcRegistryEntityConfig]);
   var allEnums$1 = allEnums.concat([entityHistoryActionEnumConfig]);
   return {
@@ -31,11 +33,20 @@ function make(userEntities, dcRegistryEntityConfig, allEnums, staticTables, stor
           allEntities: allEntities,
           allEnums: allEnums$1,
           storageStatus: "Unknown",
-          storage: storage,
-          onStorageInitialize: onStorageInitialize
+          storage: storage
         };
 }
 
+async function loadInitialCache(persistence, withUpload) {
+  var effectCacheRecords = await persistence.storage.restoreEffectCache(withUpload);
+  var cache = {};
+  effectCacheRecords.forEach(function (record) {
+        Prometheus.EffectCacheCount.set(record.count, record.effectName);
+        cache[record.effectName] = record;
+      });
+  return cache;
+}
+
 async function init(persistence, resetOpt) {
   var reset = resetOpt !== undefined ? resetOpt : false;
   try {
@@ -63,30 +74,24 @@ async function init(persistence, resetOpt) {
       _0: promise$1
     };
     if (reset || !await persistence.storage.isInitialized()) {
+      Logging.info("Initializing the indexer storage...");
       await persistence.storage.initialize(persistence.allEntities, persistence.staticTables, persistence.allEnums);
+      Logging.info("The indexer storage is ready. Uploading cache...");
       persistence.storageStatus = {
         TAG: "Ready",
         cleanRun: true,
-        effectCaches: {}
+        cache: await loadInitialCache(persistence, true)
       };
-      var onStorageInitialize = persistence.onStorageInitialize;
-      if (onStorageInitialize !== undefined) {
-        await onStorageInitialize();
-      }
-      
     } else {
       var match = persistence.storageStatus;
       var tmp;
       tmp = typeof match !== "object" || match.TAG !== "Initializing" ? false : true;
       if (tmp) {
-        var effectCaches = {};
-        (await persistence.storage.loadEffectCaches()).forEach(function (effectCache) {
-              effectCaches[effectCache.name] = effectCache;
-            });
+        Logging.info("The indexer storage is ready.");
         persistence.storageStatus = {
           TAG: "Ready",
           cleanRun: false,
-          effectCaches: effectCaches
+          cache: await loadInitialCache(persistence, false)
         };
       }
       
@@ -108,9 +113,38 @@ function getInitializedStorageOrThrow(persistence) {
   }
 }
 
+async function setEffectCacheOrThrow(persistence, effectName, ids, outputs, outputSchema) {
+  var match = persistence.storageStatus;
+  if (typeof match !== "object") {
+    return Js_exn.raiseError("Failed to access the indexer storage. The Persistence layer is not initialized.");
+  }
+  if (match.TAG === "Initializing") {
+    return Js_exn.raiseError("Failed to access the indexer storage. The Persistence layer is not initialized.");
+  }
+  var cache = match.cache;
+  var storage = persistence.storage;
+  var c = cache[effectName];
+  var effectCacheRecord;
+  if (c !== undefined) {
+    effectCacheRecord = c;
+  } else {
+    var c$1 = {
+      effectName: effectName,
+      count: 0
+    };
+    cache[effectName] = c$1;
+    effectCacheRecord = c$1;
+  }
+  var initialize = effectCacheRecord.count === 0;
+  await storage.setEffectCacheOrThrow(effectName, ids, outputs, outputSchema, initialize);
+  effectCacheRecord.count = effectCacheRecord.count + ids.length | 0;
+  return Prometheus.EffectCacheCount.set(effectCacheRecord.count, effectName);
+}
+
 exports.StorageError = StorageError;
 exports.entityHistoryActionEnumConfig = entityHistoryActionEnumConfig;
 exports.make = make;
 exports.init = init;
 exports.getInitializedStorageOrThrow = getInitializedStorageOrThrow;
-/* EntityHistory Not a pure module */
+exports.setEffectCacheOrThrow = setEffectCacheOrThrow;
+/* Logging Not a pure module */