envio 2.26.0-rc.0 → 2.26.0-rc.1

package/evm.schema.json

@@ -33,6 +33,13 @@
         "null"
       ]
     },
+    "output": {
+      "description": "Path where the generated directory will be placed. By default it's 'generated' relative to the current working directory. If set, it'll be a path relative to the config file location.",
+      "type": [
+        "string",
+        "null"
+      ]
+    },
     "contracts": {
       "description": "Global contract definitions that must contain all definitions except addresses. You can share a single handler/abi/event definitions for contracts across multiple chains.",
       "type": [

package/fuel.schema.json

@@ -26,6 +26,13 @@
         "null"
       ]
     },
+    "output": {
+      "description": "Path where the generated directory will be placed. By default it's 'generated' relative to the current working directory. If set, it'll be a path relative to the config file location.",
+      "type": [
+        "string",
+        "null"
+      ]
+    },
     "contracts": {
       "description": "Global contract definitions that must contain all definitions except addresses. You can share a single handler/abi/event definitions for contracts across multiple chains.",
       "type": [

package/index.d.ts

@@ -13,6 +13,7 @@ import type {
 } from "./src/Envio.gen.ts";
 
 import { schema as bigDecimalSchema } from "./src/bindings/BigDecimal.gen.ts";
+import { schema as bigintSchema } from "./src/bindings/BigInt.gen.ts";
 import * as Sury from "rescript-schema";
 
 type UnknownToOutput<T> = T extends Sury.Schema<unknown>
@@ -35,26 +36,6 @@ type UnknownToOutput<T> = T extends Sury.Schema<unknown>
     >
   : T;
 
-type UnknownToInput<T> = T extends Sury.Schema<unknown>
-  ? Sury.Input<T>
-  : T extends (...args: any[]) => any
-  ? T
-  : T extends unknown[]
-  ? { [K in keyof T]: UnknownToInput<T[K]> }
-  : T extends { [k in keyof T]: unknown }
-  ? Flatten<
-      {
-        [k in keyof T as HasUndefined<UnknownToInput<T[k]>> extends true
-          ? k
-          : never]?: UnknownToInput<T[k]>;
-      } & {
-        [k in keyof T as HasUndefined<UnknownToInput<T[k]>> extends true
-          ? never
-          : k]: UnknownToInput<T[k]>;
-      }
-    >
-  : T;
-
 type HasUndefined<T> = [T] extends [undefined]
   ? true
   : undefined extends T
@@ -92,7 +73,7 @@ type Flatten<T> = T extends object
 export function experimental_createEffect<
   IS,
   OS,
-  I = UnknownToInput<IS>,
+  I = UnknownToOutput<IS>,
   O = UnknownToOutput<OS>,
   // A hack to enforce that the inferred return type
   // matches the output schema type
@@ -105,6 +86,8 @@ export function experimental_createEffect<
     readonly input: IS;
     /** The output schema of the effect. */
     readonly output: OS;
+    /** Whether the effect should be cached. */
+    readonly cache?: boolean;
   },
   handler: (args: EffectArgs<I>) => Promise<R>
 ): Effect<I, O>;
@@ -119,7 +102,7 @@ export declare namespace S {
   export const boolean: typeof Sury.boolean;
   export const int32: typeof Sury.int32;
   export const number: typeof Sury.number;
-  export const bigint: typeof Sury.bigint;
+  export const bigint: typeof bigintSchema;
   export const never: typeof Sury.never;
   export const union: typeof Sury.union;
   export const object: typeof Sury.object;
@@ -128,6 +111,7 @@ export declare namespace S {
   // Don't expose recursive for now, since it's too advanced
   // export const recursive: typeof Sury.recursive;
   export const transform: typeof Sury.transform;
+  export const shape: typeof Sury.shape;
   export const refine: typeof Sury.refine;
   export const schema: typeof Sury.schema;
   export const record: typeof Sury.record;

package/index.js

@@ -13,7 +13,7 @@ exports.S = {
   boolean: Sury.boolean,
   int32: Sury.int32,
   number: Sury.number,
-  bigint: Sury.bigint,
+  bigint: require("./src/bindings/BigInt.res.js").schema,
   never: Sury.never,
   union: Sury.union,
   object: Sury.object,
@@ -22,6 +22,7 @@ exports.S = {
   // Don't expose recursive for now, since it's too advanced
   // recursive: Sury.recursive,
   transform: Sury.transform,
+  shape: Sury.shape,
   refine: Sury.refine,
   schema: Sury.schema,
   record: Sury.record,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "envio",
-  "version": "v2.26.0-rc.0",
+  "version": "v2.26.0-rc.1",
   "description": "A latency and sync speed optimized, developer friendly blockchain data indexer.",
   "bin": "./bin.js",
   "main": "./index.js",
@@ -25,10 +25,10 @@
   },
   "homepage": "https://envio.dev",
   "optionalDependencies": {
-    "envio-linux-x64": "v2.26.0-rc.0",
-    "envio-linux-arm64": "v2.26.0-rc.0",
-    "envio-darwin-x64": "v2.26.0-rc.0",
-    "envio-darwin-arm64": "v2.26.0-rc.0"
+    "envio-linux-x64": "v2.26.0-rc.1",
+    "envio-linux-arm64": "v2.26.0-rc.1",
+    "envio-darwin-x64": "v2.26.0-rc.1",
+    "envio-darwin-arm64": "v2.26.0-rc.1"
   },
   "dependencies": {
     "@envio-dev/hypersync-client": "0.6.5",

package/src/Envio.gen.ts

@@ -21,7 +21,9 @@ export type effectOptions<input,output> = {
   /** The input schema of the effect. */
   readonly input: RescriptSchema_S_t<input>; 
   /** The output schema of the effect. */
-  readonly output: RescriptSchema_S_t<output>
+  readonly output: RescriptSchema_S_t<output>; 
+  /** Whether the effect should be cached. */
+  readonly cache?: boolean
 };
 
 export type effectContext = $$effectContext;

package/src/Envio.res

@@ -22,6 +22,8 @@ and effectOptions<'input, 'output> = {
   input: S.t<'input>,
   /** The output schema of the effect. */
   output: S.t<'output>,
+  /** Whether the effect should be cached. */
+  cache?: bool,
 }
 @genType.import(("./Types.ts", "EffectContext"))
 and effectContext = {
@@ -48,5 +50,16 @@ let experimental_createEffect = (
       >
     ),
     callsCount: 0,
+    // This is the way to make the createEffect API
+    // work without the need for users to call S.schema themselves,
+    // but simply pass the desired object/tuple/etc.
+    // If they pass a schem, it'll also work.
+    input: S.schema(_ => options.input)->(
+      Utils.magic: S.t<S.t<'input>> => S.t<Internal.effectInput>
+    ),
+    output: S.schema(_ => options.output)->(
+      Utils.magic: S.t<S.t<'output>> => S.t<Internal.effectOutput>
+    ),
+    cache: options.cache->Belt.Option.getWithDefault(false),
   }->(Utils.magic: Internal.effect => effect<'input, 'output>)
 }

package/src/Envio.res.js

@@ -2,12 +2,21 @@
 'use strict';
 
 var Prometheus = require("./Prometheus.res.js");
+var Belt_Option = require("rescript/lib/js/belt_Option.js");
+var S$RescriptSchema = require("rescript-schema/src/S.res.js");
 
 function experimental_createEffect(options, handler) {
   Prometheus.EffectCallsCount.set(0, options.name);
   return {
           name: options.name,
           handler: handler,
+          cache: Belt_Option.getWithDefault(options.cache, false),
+          output: S$RescriptSchema.schema(function (param) {
+                return options.output;
+              }),
+          input: S$RescriptSchema.schema(function (param) {
+                return options.input;
+              }),
           callsCount: 0
         };
 }

package/src/FetchState.res

@@ -901,10 +901,7 @@ let queueItemIsInReorgThreshold = (
   if currentBlockHeight === 0 {
     false
   } else {
-    switch queueItem {
-    | Item(_) => queueItem->queueItemBlockNumber > highestBlockBelowThreshold
-    | NoItem(_) => queueItem->queueItemBlockNumber > highestBlockBelowThreshold
-    }
+    queueItem->queueItemBlockNumber > highestBlockBelowThreshold
   }
 }
 

package/src/Internal.res

@@ -197,6 +197,9 @@ type effectArgs = {
 type effect = {
   name: string,
   handler: effectArgs => promise<effectOutput>,
+  cache: bool,
+  output: S.t<effectOutput>,
+  input: S.t<effectInput>,
   mutable callsCount: int,
 }
 

package/src/Logging.res

@@ -168,6 +168,14 @@ let logForItem = (eventItem, level: Pino.logLevel, message: string, ~params=?) =
   (eventItem->getEventLogger->Utils.magic->Js.Dict.unsafeGet((level :> string)))(params, message)
 }
 
+let noopLogger: Envio.logger = {
+  info: (_message: string, ~params as _=?) => (),
+  debug: (_message: string, ~params as _=?) => (),
+  warn: (_message: string, ~params as _=?) => (),
+  error: (_message: string, ~params as _=?) => (),
+  errorWithExn: (_message: string, _exn) => (),
+}
+
 let getUserLogger = (eventItem): Envio.logger => {
   info: (message: string, ~params=?) => eventItem->logForItem(#uinfo, message, ~params?),
   debug: (message: string, ~params=?) => eventItem->logForItem(#udebug, message, ~params?),

package/src/Logging.res.js

@@ -197,6 +197,34 @@ function logForItem(eventItem, level, message, params) {
   return getEventLogger(eventItem)[level](params, message);
 }
 
+function noopLogger_debug(_message, param) {
+  
+}
+
+function noopLogger_info(_message, param) {
+  
+}
+
+function noopLogger_warn(_message, param) {
+  
+}
+
+function noopLogger_error(_message, param) {
+  
+}
+
+function noopLogger_errorWithExn(_message, _exn) {
+  
+}
+
+var noopLogger = {
+  debug: noopLogger_debug,
+  info: noopLogger_info,
+  warn: noopLogger_warn,
+  error: noopLogger_error,
+  errorWithExn: noopLogger_errorWithExn
+};
+
 function getUserLogger(eventItem) {
   return {
           debug: (function (message, params) {
@@ -243,5 +271,6 @@ exports.createChild = createChild;
 exports.createChildFrom = createChildFrom;
 exports.getEventLogger = getEventLogger;
 exports.logForItem = logForItem;
+exports.noopLogger = noopLogger;
 exports.getUserLogger = getUserLogger;
 /* logLevels Not a pure module */

package/src/Persistence.res

@@ -5,6 +5,16 @@
 // Currently there are quite many code spread across
 // DbFunctions, Db, Migrations, InMemoryStore modules which use codegen code directly.
 
+// The type reflects an cache table in the db
+// It might be present even if the effect is not used in the application
+type effectCacheRecord = {
+  effectName: string,
+  // Number of rows in the table
+  mutable count: int,
+}
+
+type operator = [#">" | #"="]
+
 type storage = {
   // Should return true if we already have persisted data
   // and we can skip initialization
@@ -15,8 +25,6 @@ type storage = {
     ~entities: array<Internal.entityConfig>=?,
     ~generalTables: array<Table.table>=?,
     ~enums: array<Internal.enumConfig<Internal.enum>>=?,
-    // If true, the storage should clear existing data
-    ~cleanRun: bool=?,
   ) => promise<unit>,
   @raises("StorageError")
   loadByIdsOrThrow: 'item. (
@@ -25,11 +33,30 @@ type storage = {
     ~rowsSchema: S.t<array<'item>>,
   ) => promise<array<'item>>,
   @raises("StorageError")
+  loadByFieldOrThrow: 'item 'value. (
+    ~fieldName: string,
+    ~fieldSchema: S.t<'value>,
+    ~fieldValue: 'value,
+    ~operator: operator,
+    ~table: Table.table,
+    ~rowsSchema: S.t<array<'item>>,
+  ) => promise<array<'item>>,
+  @raises("StorageError")
   setOrThrow: 'item. (
     ~items: array<'item>,
     ~table: Table.table,
     ~itemSchema: S.t<'item>,
   ) => promise<unit>,
+  @raises("StorageError")
+  setEffectCacheOrThrow: (
+    ~effectName: string,
+    ~ids: array<string>,
+    ~outputs: array<Internal.effectOutput>,
+    ~outputSchema: S.t<Internal.effectOutput>,
+    ~initialize: bool,
+  ) => promise<unit>,
+  dumpEffectCache: unit => promise<unit>,
+  restoreEffectCache: (~withUpload: bool) => promise<array<effectCacheRecord>>,
 }
 
 exception StorageError({message: string, reason: exn})
@@ -37,7 +64,7 @@ exception StorageError({message: string, reason: exn})
 type storageStatus =
   | Unknown
   | Initializing(promise<unit>)
-  | Ready({cleanRun: bool})
+  | Ready({cleanRun: bool, cache: dict<effectCacheRecord>})
 
 type t = {
   userEntities: array<Internal.entityConfig>,
@@ -46,8 +73,6 @@ type t = {
   allEnums: array<Internal.enumConfig<Internal.enum>>,
   mutable storageStatus: storageStatus,
   storage: storage,
-  onStorageInitialize: option<unit => promise<unit>>,
-  cacheStorage: storage,
 }
 
 let entityHistoryActionEnumConfig: Internal.enumConfig<EntityHistory.RowAction.t> = {
@@ -64,8 +89,6 @@ let make = (
   ~allEnums,
   ~staticTables,
   ~storage,
-  ~cacheStorage,
-  ~onStorageInitialize=?,
 ) => {
   let allEntities = userEntities->Js.Array2.concat([dcRegistryEntityConfig])
   let allEnums =
@@ -77,53 +100,70 @@ let make = (
     allEnums,
     storageStatus: Unknown,
     storage,
-    onStorageInitialize,
-    cacheStorage,
   }
 }
 
-let init = async (
-  persistence,
-  // There are not much sense in the option,
-  // but this is how the runUpMigration used to work
-  // and we want to keep the upsert behavior without breaking changes.
-  ~skipIsInitializedCheck=false,
-  ~reset=false,
-) => {
-  try {
-    let shouldRun = switch persistence.storageStatus {
-    | Unknown => true
-    | Initializing(promise) => {
-        await promise
-        reset
+let init = {
+  let loadInitialCache = async (persistence, ~withUpload) => {
+    let effectCacheRecords = await persistence.storage.restoreEffectCache(~withUpload)
+    let cache = Js.Dict.empty()
+    effectCacheRecords->Js.Array2.forEach(record => {
+      Prometheus.EffectCacheCount.set(~count=record.count, ~effectName=record.effectName)
+      cache->Js.Dict.set(record.effectName, record)
+    })
+    cache
+  }
+
+  async (persistence, ~reset=false) => {
+    try {
+      let shouldRun = switch persistence.storageStatus {
+      | Unknown => true
+      | Initializing(promise) => {
+          await promise
+          reset
+        }
+      | Ready(_) => reset
       }
-    | Ready(_) => reset
-    }
-    if shouldRun {
-      let resolveRef = ref(%raw(`null`))
-      let promise = Promise.make((resolve, _) => {
-        resolveRef := resolve
-      })
-      persistence.storageStatus = Initializing(promise)
-      if !(reset || skipIsInitializedCheck) && (await persistence.storage.isInitialized()) {
-        persistence.storageStatus = Ready({cleanRun: false})
-      } else {
-        let _ = await persistence.storage.initialize(
-          ~entities=persistence.allEntities,
-          ~generalTables=persistence.staticTables,
-          ~enums=persistence.allEnums,
-          ~cleanRun=reset || !skipIsInitializedCheck,
-        )
-        persistence.storageStatus = Ready({cleanRun: true})
-        switch persistence.onStorageInitialize {
-        | Some(onStorageInitialize) => await onStorageInitialize()
-        | None => ()
+      if shouldRun {
+        let resolveRef = ref(%raw(`null`))
+        let promise = Promise.make((resolve, _) => {
+          resolveRef := resolve
+        })
+        persistence.storageStatus = Initializing(promise)
+        if reset || !(await persistence.storage.isInitialized()) {
+          Logging.info(`Initializing the indexer storage...`)
+
+          await persistence.storage.initialize(
+            ~entities=persistence.allEntities,
+            ~generalTables=persistence.staticTables,
+            ~enums=persistence.allEnums,
+          )
+
+          Logging.info(`The indexer storage is ready. Uploading cache...`)
+          persistence.storageStatus = Ready({
+            cleanRun: true,
+            cache: await loadInitialCache(persistence, ~withUpload=true),
+          })
+        } else if (
+          // In case of a race condition,
+          // we want to set the initial status to Ready only once.
+          switch persistence.storageStatus {
+          | Initializing(_) => true
+          | _ => false
+          }
+        ) {
+          Logging.info(`The indexer storage is ready.`)
+          persistence.storageStatus = Ready({
+            cleanRun: false,
+            cache: await loadInitialCache(persistence, ~withUpload=false),
+          })
         }
+        resolveRef.contents()
       }
-      resolveRef.contents()
+    } catch {
+    | exn =>
+      exn->ErrorHandling.mkLogAndRaise(~msg=`EE800: Failed to initialize the indexer storage.`)
     }
-  } catch {
-  | exn => exn->ErrorHandling.mkLogAndRaise(~msg=`EE800: Failed to initialize the indexer storage.`)
   }
 }
 
@@ -135,3 +175,26 @@ let getInitializedStorageOrThrow = persistence => {
   | Ready(_) => persistence.storage
   }
 }
+
+let setEffectCacheOrThrow = async (persistence, ~effectName, ~ids, ~outputs, ~outputSchema) => {
+  switch persistence.storageStatus {
+  | Unknown
+  | Initializing(_) =>
+    Js.Exn.raiseError(`Failed to access the indexer storage. The Persistence layer is not initialized.`)
+  | Ready({cache}) => {
+      let storage = persistence.storage
+      let effectCacheRecord = switch cache->Utils.Dict.dangerouslyGetNonOption(effectName) {
+      | Some(c) => c
+      | None => {
+          let c = {effectName, count: 0}
+          cache->Js.Dict.set(effectName, c)
+          c
+        }
+      }
+      let initialize = effectCacheRecord.count === 0
+      await storage.setEffectCacheOrThrow(~effectName, ~ids, ~outputs, ~outputSchema, ~initialize)
+      effectCacheRecord.count = effectCacheRecord.count + ids->Js.Array2.length
+      Prometheus.EffectCacheCount.set(~count=effectCacheRecord.count, ~effectName)
+    }
+  }
+}

package/src/Persistence.res.js

@@ -2,6 +2,8 @@
 'use strict';
 
 var Js_exn = require("rescript/lib/js/js_exn.js");
+var Logging = require("./Logging.res.js");
+var Prometheus = require("./Prometheus.res.js");
 var EntityHistory = require("./db/EntityHistory.res.js");
 var ErrorHandling = require("./ErrorHandling.res.js");
 var Caml_exceptions = require("rescript/lib/js/caml_exceptions.js");
@@ -22,7 +24,7 @@ var entityHistoryActionEnumConfig = {
   default: "SET"
 };
 
-function make(userEntities, dcRegistryEntityConfig, allEnums, staticTables, storage, cacheStorage, onStorageInitialize) {
+function make(userEntities, dcRegistryEntityConfig, allEnums, staticTables, storage) {
   var allEntities = userEntities.concat([dcRegistryEntityConfig]);
   var allEnums$1 = allEnums.concat([entityHistoryActionEnumConfig]);
   return {
@@ -31,14 +33,21 @@ function make(userEntities, dcRegistryEntityConfig, allEnums, staticTables, stor
           allEntities: allEntities,
           allEnums: allEnums$1,
           storageStatus: "Unknown",
-          storage: storage,
-          onStorageInitialize: onStorageInitialize,
-          cacheStorage: cacheStorage
+          storage: storage
         };
 }
 
-async function init(persistence, skipIsInitializedCheckOpt, resetOpt) {
-  var skipIsInitializedCheck = skipIsInitializedCheckOpt !== undefined ? skipIsInitializedCheckOpt : false;
+async function loadInitialCache(persistence, withUpload) {
+  var effectCacheRecords = await persistence.storage.restoreEffectCache(withUpload);
+  var cache = {};
+  effectCacheRecords.forEach(function (record) {
+        Prometheus.EffectCacheCount.set(record.count, record.effectName);
+        cache[record.effectName] = record;
+      });
+  return cache;
+}
+
+async function init(persistence, resetOpt) {
   var reset = resetOpt !== undefined ? resetOpt : false;
   try {
     var promise = persistence.storageStatus;
@@ -64,20 +73,26 @@ async function init(persistence, skipIsInitializedCheckOpt, resetOpt) {
       TAG: "Initializing",
       _0: promise$1
     };
-    if (!(reset || skipIsInitializedCheck) && await persistence.storage.isInitialized()) {
+    if (reset || !await persistence.storage.isInitialized()) {
+      Logging.info("Initializing the indexer storage...");
+      await persistence.storage.initialize(persistence.allEntities, persistence.staticTables, persistence.allEnums);
+      Logging.info("The indexer storage is ready. Uploading cache...");
       persistence.storageStatus = {
         TAG: "Ready",
-        cleanRun: false
+        cleanRun: true,
+        cache: await loadInitialCache(persistence, true)
       };
     } else {
-      await persistence.storage.initialize(persistence.allEntities, persistence.staticTables, persistence.allEnums, reset || !skipIsInitializedCheck);
-      persistence.storageStatus = {
-        TAG: "Ready",
-        cleanRun: true
-      };
-      var onStorageInitialize = persistence.onStorageInitialize;
-      if (onStorageInitialize !== undefined) {
-        await onStorageInitialize();
+      var match = persistence.storageStatus;
+      var tmp;
+      tmp = typeof match !== "object" || match.TAG !== "Initializing" ? false : true;
+      if (tmp) {
+        Logging.info("The indexer storage is ready.");
+        persistence.storageStatus = {
+          TAG: "Ready",
+          cleanRun: false,
+          cache: await loadInitialCache(persistence, false)
+        };
       }
       
     }
@@ -98,9 +113,38 @@ function getInitializedStorageOrThrow(persistence) {
   }
 }
 
+async function setEffectCacheOrThrow(persistence, effectName, ids, outputs, outputSchema) {
+  var match = persistence.storageStatus;
+  if (typeof match !== "object") {
+    return Js_exn.raiseError("Failed to access the indexer storage. The Persistence layer is not initialized.");
+  }
+  if (match.TAG === "Initializing") {
+    return Js_exn.raiseError("Failed to access the indexer storage. The Persistence layer is not initialized.");
+  }
+  var cache = match.cache;
+  var storage = persistence.storage;
+  var c = cache[effectName];
+  var effectCacheRecord;
+  if (c !== undefined) {
+    effectCacheRecord = c;
+  } else {
+    var c$1 = {
+      effectName: effectName,
+      count: 0
+    };
+    cache[effectName] = c$1;
+    effectCacheRecord = c$1;
+  }
+  var initialize = effectCacheRecord.count === 0;
+  await storage.setEffectCacheOrThrow(effectName, ids, outputs, outputSchema, initialize);
+  effectCacheRecord.count = effectCacheRecord.count + ids.length | 0;
+  return Prometheus.EffectCacheCount.set(effectCacheRecord.count, effectName);
+}
+
 exports.StorageError = StorageError;
 exports.entityHistoryActionEnumConfig = entityHistoryActionEnumConfig;
 exports.make = make;
 exports.init = init;
 exports.getInitializedStorageOrThrow = getInitializedStorageOrThrow;
-/* EntityHistory Not a pure module */
+exports.setEffectCacheOrThrow = setEffectCacheOrThrow;
+/* Logging Not a pure module */