envio 2.26.0-rc.0 → 2.26.0-rc.2

package/evm.schema.json

@@ -33,6 +33,13 @@
         "null"
       ]
     },
+    "output": {
+      "description": "Path where the generated directory will be placed. By default it's 'generated' relative to the current working directory. If set, it'll be a path relative to the config file location.",
+      "type": [
+        "string",
+        "null"
+      ]
+    },
     "contracts": {
       "description": "Global contract definitions that must contain all definitions except addresses. You can share a single handler/abi/event definitions for contracts across multiple chains.",
       "type": [

package/fuel.schema.json

@@ -26,6 +26,13 @@
         "null"
       ]
     },
+    "output": {
+      "description": "Path where the generated directory will be placed. By default it's 'generated' relative to the current working directory. If set, it'll be a path relative to the config file location.",
+      "type": [
+        "string",
+        "null"
+      ]
+    },
     "contracts": {
       "description": "Global contract definitions that must contain all definitions except addresses. You can share a single handler/abi/event definitions for contracts across multiple chains.",
       "type": [

package/index.d.ts

@@ -13,6 +13,7 @@ import type {
 } from "./src/Envio.gen.ts";
 
 import { schema as bigDecimalSchema } from "./src/bindings/BigDecimal.gen.ts";
+import { schema as bigintSchema } from "./src/bindings/BigInt.gen.ts";
 import * as Sury from "rescript-schema";
 
 type UnknownToOutput<T> = T extends Sury.Schema<unknown>
@@ -35,26 +36,6 @@ type UnknownToOutput<T> = T extends Sury.Schema<unknown>
     >
   : T;
 
-type UnknownToInput<T> = T extends Sury.Schema<unknown>
-  ? Sury.Input<T>
-  : T extends (...args: any[]) => any
-  ? T
-  : T extends unknown[]
-  ? { [K in keyof T]: UnknownToInput<T[K]> }
-  : T extends { [k in keyof T]: unknown }
-  ? Flatten<
-      {
-        [k in keyof T as HasUndefined<UnknownToInput<T[k]>> extends true
-          ? k
-          : never]?: UnknownToInput<T[k]>;
-      } & {
-        [k in keyof T as HasUndefined<UnknownToInput<T[k]>> extends true
-          ? never
-          : k]: UnknownToInput<T[k]>;
-      }
-    >
-  : T;
-
 type HasUndefined<T> = [T] extends [undefined]
   ? true
   : undefined extends T
@@ -92,7 +73,7 @@ type Flatten<T> = T extends object
 export function experimental_createEffect<
   IS,
   OS,
-  I = UnknownToInput<IS>,
+  I = UnknownToOutput<IS>,
   O = UnknownToOutput<OS>,
   // A hack to enforce that the inferred return type
   // matches the output schema type
@@ -105,6 +86,8 @@ export function experimental_createEffect<
     readonly input: IS;
     /** The output schema of the effect. */
     readonly output: OS;
+    /** Whether the effect should be cached. */
+    readonly cache?: boolean;
   },
   handler: (args: EffectArgs<I>) => Promise<R>
 ): Effect<I, O>;
@@ -119,7 +102,7 @@ export declare namespace S {
   export const boolean: typeof Sury.boolean;
   export const int32: typeof Sury.int32;
   export const number: typeof Sury.number;
-  export const bigint: typeof Sury.bigint;
+  export const bigint: typeof bigintSchema;
   export const never: typeof Sury.never;
   export const union: typeof Sury.union;
   export const object: typeof Sury.object;
@@ -128,6 +111,7 @@ export declare namespace S {
   // Don't expose recursive for now, since it's too advanced
   // export const recursive: typeof Sury.recursive;
   export const transform: typeof Sury.transform;
+  export const shape: typeof Sury.shape;
   export const refine: typeof Sury.refine;
   export const schema: typeof Sury.schema;
   export const record: typeof Sury.record;

package/index.js

@@ -13,7 +13,7 @@ exports.S = {
   boolean: Sury.boolean,
   int32: Sury.int32,
   number: Sury.number,
-  bigint: Sury.bigint,
+  bigint: require("./src/bindings/BigInt.res.js").schema,
   never: Sury.never,
   union: Sury.union,
   object: Sury.object,
@@ -22,6 +22,7 @@ exports.S = {
   // Don't expose recursive for now, since it's too advanced
   // recursive: Sury.recursive,
   transform: Sury.transform,
+  shape: Sury.shape,
   refine: Sury.refine,
   schema: Sury.schema,
   record: Sury.record,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "envio",
-  "version": "v2.26.0-rc.0",
+  "version": "v2.26.0-rc.2",
   "description": "A latency and sync speed optimized, developer friendly blockchain data indexer.",
   "bin": "./bin.js",
   "main": "./index.js",
@@ -25,10 +25,10 @@
   },
   "homepage": "https://envio.dev",
   "optionalDependencies": {
-    "envio-linux-x64": "v2.26.0-rc.0",
-    "envio-linux-arm64": "v2.26.0-rc.0",
-    "envio-darwin-x64": "v2.26.0-rc.0",
-    "envio-darwin-arm64": "v2.26.0-rc.0"
+    "envio-linux-x64": "v2.26.0-rc.2",
+    "envio-linux-arm64": "v2.26.0-rc.2",
+    "envio-darwin-x64": "v2.26.0-rc.2",
+    "envio-darwin-arm64": "v2.26.0-rc.2"
   },
   "dependencies": {
     "@envio-dev/hypersync-client": "0.6.5",

package/src/Envio.gen.ts

@@ -21,7 +21,9 @@ export type effectOptions<input,output> = {
   /** The input schema of the effect. */
   readonly input: RescriptSchema_S_t<input>; 
   /** The output schema of the effect. */
-  readonly output: RescriptSchema_S_t<output>
+  readonly output: RescriptSchema_S_t<output>; 
+  /** Whether the effect should be cached. */
+  readonly cache?: boolean
 };
 
 export type effectContext = $$effectContext;

package/src/Envio.res

@@ -22,6 +22,8 @@ and effectOptions<'input, 'output> = {
   input: S.t<'input>,
   /** The output schema of the effect. */
   output: S.t<'output>,
+  /** Whether the effect should be cached. */
+  cache?: bool,
 }
 @genType.import(("./Types.ts", "EffectContext"))
 and effectContext = {
@@ -40,6 +42,8 @@ let experimental_createEffect = (
   handler: effectArgs<'input> => promise<'output>,
 ) => {
   Prometheus.EffectCallsCount.set(~callsCount=0, ~effectName=options.name)
+  let outputSchema =
+    S.schema(_ => options.output)->(Utils.magic: S.t<S.t<'output>> => S.t<Internal.effectOutput>)
   {
     name: options.name,
     handler: handler->(
@@ -48,5 +52,28 @@ let experimental_createEffect = (
       >
     ),
     callsCount: 0,
+    // This is the way to make the createEffect API
+    // work without the need for users to call S.schema themselves,
+    // but simply pass the desired object/tuple/etc.
+    // If they pass a schem, it'll also work.
+    input: S.schema(_ => options.input)->(
+      Utils.magic: S.t<S.t<'input>> => S.t<Internal.effectInput>
+    ),
+    output: outputSchema,
+    cache: switch options.cache {
+    | Some(true) =>
+      let itemSchema = S.schema((s): Internal.effectCacheItem => {
+        id: s.matches(S.string),
+        output: s.matches(outputSchema),
+      })
+      Some({
+        table: Internal.makeCacheTable(~effectName=options.name),
+        rowsSchema: S.array(itemSchema),
+        itemSchema,
+      })
+    | None
+    | Some(false) =>
+      None
+    },
   }->(Utils.magic: Internal.effect => effect<'input, 'output>)
 }

package/src/Envio.res.js

@@ -1,16 +1,43 @@
 // Generated by ReScript, PLEASE EDIT WITH CARE
 'use strict';
 
+var Internal = require("./Internal.res.js");
 var Prometheus = require("./Prometheus.res.js");
+var S$RescriptSchema = require("rescript-schema/src/S.res.js");
 
 function experimental_createEffect(options, handler) {
   Prometheus.EffectCallsCount.set(0, options.name);
+  var outputSchema = S$RescriptSchema.schema(function (param) {
+        return options.output;
+      });
+  var match = options.cache;
+  var tmp;
+  if (match !== undefined && match) {
+    var itemSchema = S$RescriptSchema.schema(function (s) {
+          return {
+                  id: s.m(S$RescriptSchema.string),
+                  output: s.m(outputSchema)
+                };
+        });
+    tmp = {
+      itemSchema: itemSchema,
+      rowsSchema: S$RescriptSchema.array(itemSchema),
+      table: Internal.makeCacheTable(options.name)
+    };
+  } else {
+    tmp = undefined;
+  }
   return {
           name: options.name,
           handler: handler,
+          cache: tmp,
+          output: outputSchema,
+          input: S$RescriptSchema.schema(function (param) {
+                return options.input;
+              }),
           callsCount: 0
         };
 }
 
 exports.experimental_createEffect = experimental_createEffect;
-/* Prometheus Not a pure module */
+/* Internal Not a pure module */

package/src/FetchState.res

@@ -901,10 +901,7 @@ let queueItemIsInReorgThreshold = (
   if currentBlockHeight === 0 {
     false
   } else {
-    switch queueItem {
-    | Item(_) => queueItem->queueItemBlockNumber > highestBlockBelowThreshold
-    | NoItem(_) => queueItem->queueItemBlockNumber > highestBlockBelowThreshold
-    }
+    queueItem->queueItemBlockNumber > highestBlockBelowThreshold
   }
 }
 

package/src/Internal.res

@@ -194,11 +194,31 @@ type effectArgs = {
   context: effectContext,
   cacheKey: string,
 }
+type effectCacheItem = {id: string, output: effectOutput}
+type effectCacheMeta = {
+  itemSchema: S.t<effectCacheItem>,
+  rowsSchema: S.t<array<effectCacheItem>>,
+  table: Table.table,
+}
 type effect = {
   name: string,
   handler: effectArgs => promise<effectOutput>,
+  cache: option<effectCacheMeta>,
+  output: S.t<effectOutput>,
+  input: S.t<effectInput>,
   mutable callsCount: int,
 }
+let cacheTablePrefix = "envio_effect_"
+let makeCacheTable = (~effectName) => {
+  Table.mkTable(
+    cacheTablePrefix ++ effectName,
+    ~fields=[
+      Table.mkField("id", Text, ~fieldSchema=S.string, ~isPrimaryKey=true),
+      Table.mkField("output", JsonB, ~fieldSchema=S.json(~validate=false)),
+    ],
+    ~compositeIndices=[],
+  )
+}
 
 @genType.import(("./Types.ts", "Invalid"))
 type noEventFilters

package/src/Internal.res.js

@@ -1,6 +1,7 @@
 // Generated by ReScript, PLEASE EDIT WITH CARE
 'use strict';
 
+var Table = require("./db/Table.res.js");
 var $$BigInt = require("./bindings/BigInt.res.js");
 var Js_exn = require("rescript/lib/js/js_exn.js");
 var Address = require("./Address.res.js");
@@ -34,6 +35,15 @@ function makeEnumConfig(name, variants) {
         };
 }
 
+var cacheTablePrefix = "envio_effect_";
+
+function makeCacheTable(effectName) {
+  return Table.mkTable(cacheTablePrefix + effectName, [], [
+              Table.mkField("id", "TEXT", S$RescriptSchema.string, undefined, undefined, undefined, true, undefined, undefined),
+              Table.mkField("output", "JSONB", S$RescriptSchema.json(false), undefined, undefined, undefined, undefined, undefined, undefined)
+            ]);
+}
+
 function prettifyExn(exn) {
   var e = Caml_js_exceptions.internalToOCamlException(exn);
   if (e.RE_EXN_ID === Js_exn.$$Error) {
@@ -46,5 +56,7 @@ function prettifyExn(exn) {
 exports.fuelSupplyParamsSchema = fuelSupplyParamsSchema;
 exports.fuelTransferParamsSchema = fuelTransferParamsSchema;
 exports.makeEnumConfig = makeEnumConfig;
+exports.cacheTablePrefix = cacheTablePrefix;
+exports.makeCacheTable = makeCacheTable;
 exports.prettifyExn = prettifyExn;
 /* fuelSupplyParamsSchema Not a pure module */

package/src/Logging.res

@@ -168,6 +168,14 @@ let logForItem = (eventItem, level: Pino.logLevel, message: string, ~params=?) =
   (eventItem->getEventLogger->Utils.magic->Js.Dict.unsafeGet((level :> string)))(params, message)
 }
 
+let noopLogger: Envio.logger = {
+  info: (_message: string, ~params as _=?) => (),
+  debug: (_message: string, ~params as _=?) => (),
+  warn: (_message: string, ~params as _=?) => (),
+  error: (_message: string, ~params as _=?) => (),
+  errorWithExn: (_message: string, _exn) => (),
+}
+
 let getUserLogger = (eventItem): Envio.logger => {
   info: (message: string, ~params=?) => eventItem->logForItem(#uinfo, message, ~params?),
   debug: (message: string, ~params=?) => eventItem->logForItem(#udebug, message, ~params?),

package/src/Logging.res.js

@@ -197,6 +197,34 @@ function logForItem(eventItem, level, message, params) {
   return getEventLogger(eventItem)[level](params, message);
 }
 
+function noopLogger_debug(_message, param) {
+  
+}
+
+function noopLogger_info(_message, param) {
+  
+}
+
+function noopLogger_warn(_message, param) {
+  
+}
+
+function noopLogger_error(_message, param) {
+  
+}
+
+function noopLogger_errorWithExn(_message, _exn) {
+  
+}
+
+var noopLogger = {
+  debug: noopLogger_debug,
+  info: noopLogger_info,
+  warn: noopLogger_warn,
+  error: noopLogger_error,
+  errorWithExn: noopLogger_errorWithExn
+};
+
 function getUserLogger(eventItem) {
   return {
           debug: (function (message, params) {
@@ -243,5 +271,6 @@ exports.createChild = createChild;
 exports.createChildFrom = createChildFrom;
 exports.getEventLogger = getEventLogger;
 exports.logForItem = logForItem;
+exports.noopLogger = noopLogger;
 exports.getUserLogger = getUserLogger;
 /* logLevels Not a pure module */

package/src/Persistence.res

@@ -5,6 +5,16 @@
 // Currently there are quite many code spread across
 // DbFunctions, Db, Migrations, InMemoryStore modules which use codegen code directly.
 
+// The type reflects an cache table in the db
+// It might be present even if the effect is not used in the application
+type effectCacheRecord = {
+  effectName: string,
+  // Number of rows in the table
+  mutable count: int,
+}
+
+type operator = [#">" | #"="]
+
 type storage = {
   // Should return true if we already have persisted data
   // and we can skip initialization
@@ -15,8 +25,6 @@ type storage = {
     ~entities: array<Internal.entityConfig>=?,
     ~generalTables: array<Table.table>=?,
     ~enums: array<Internal.enumConfig<Internal.enum>>=?,
-    // If true, the storage should clear existing data
-    ~cleanRun: bool=?,
   ) => promise<unit>,
   @raises("StorageError")
   loadByIdsOrThrow: 'item. (
@@ -25,11 +33,28 @@ type storage = {
     ~rowsSchema: S.t<array<'item>>,
   ) => promise<array<'item>>,
   @raises("StorageError")
+  loadByFieldOrThrow: 'item 'value. (
+    ~fieldName: string,
+    ~fieldSchema: S.t<'value>,
+    ~fieldValue: 'value,
+    ~operator: operator,
+    ~table: Table.table,
+    ~rowsSchema: S.t<array<'item>>,
+  ) => promise<array<'item>>,
+  @raises("StorageError")
   setOrThrow: 'item. (
     ~items: array<'item>,
     ~table: Table.table,
     ~itemSchema: S.t<'item>,
   ) => promise<unit>,
+  @raises("StorageError")
+  setEffectCacheOrThrow: (
+    ~effect: Internal.effect,
+    ~items: array<Internal.effectCacheItem>,
+    ~initialize: bool,
+  ) => promise<unit>,
+  dumpEffectCache: unit => promise<unit>,
+  restoreEffectCache: (~withUpload: bool) => promise<array<effectCacheRecord>>,
 }
 
 exception StorageError({message: string, reason: exn})
@@ -37,7 +62,7 @@ exception StorageError({message: string, reason: exn})
 type storageStatus =
   | Unknown
   | Initializing(promise<unit>)
-  | Ready({cleanRun: bool})
+  | Ready({cleanRun: bool, cache: dict<effectCacheRecord>})
 
 type t = {
   userEntities: array<Internal.entityConfig>,
@@ -46,8 +71,6 @@ type t = {
   allEnums: array<Internal.enumConfig<Internal.enum>>,
   mutable storageStatus: storageStatus,
   storage: storage,
-  onStorageInitialize: option<unit => promise<unit>>,
-  cacheStorage: storage,
 }
 
 let entityHistoryActionEnumConfig: Internal.enumConfig<EntityHistory.RowAction.t> = {
@@ -64,8 +87,6 @@ let make = (
   ~allEnums,
   ~staticTables,
   ~storage,
-  ~cacheStorage,
-  ~onStorageInitialize=?,
 ) => {
   let allEntities = userEntities->Js.Array2.concat([dcRegistryEntityConfig])
   let allEnums =
@@ -77,53 +98,70 @@ let make = (
     allEnums,
     storageStatus: Unknown,
     storage,
-    onStorageInitialize,
-    cacheStorage,
   }
 }
 
-let init = async (
-  persistence,
-  // There are not much sense in the option,
-  // but this is how the runUpMigration used to work
-  // and we want to keep the upsert behavior without breaking changes.
-  ~skipIsInitializedCheck=false,
-  ~reset=false,
-) => {
-  try {
-    let shouldRun = switch persistence.storageStatus {
-    | Unknown => true
-    | Initializing(promise) => {
-        await promise
-        reset
+let init = {
+  let loadInitialCache = async (persistence, ~withUpload) => {
+    let effectCacheRecords = await persistence.storage.restoreEffectCache(~withUpload)
+    let cache = Js.Dict.empty()
+    effectCacheRecords->Js.Array2.forEach(record => {
+      Prometheus.EffectCacheCount.set(~count=record.count, ~effectName=record.effectName)
+      cache->Js.Dict.set(record.effectName, record)
+    })
+    cache
+  }
+
+  async (persistence, ~reset=false) => {
+    try {
+      let shouldRun = switch persistence.storageStatus {
+      | Unknown => true
+      | Initializing(promise) => {
+          await promise
+          reset
+        }
+      | Ready(_) => reset
       }
-    | Ready(_) => reset
-    }
-    if shouldRun {
-      let resolveRef = ref(%raw(`null`))
-      let promise = Promise.make((resolve, _) => {
-        resolveRef := resolve
-      })
-      persistence.storageStatus = Initializing(promise)
-      if !(reset || skipIsInitializedCheck) && (await persistence.storage.isInitialized()) {
-        persistence.storageStatus = Ready({cleanRun: false})
-      } else {
-        let _ = await persistence.storage.initialize(
-          ~entities=persistence.allEntities,
-          ~generalTables=persistence.staticTables,
-          ~enums=persistence.allEnums,
-          ~cleanRun=reset || !skipIsInitializedCheck,
-        )
-        persistence.storageStatus = Ready({cleanRun: true})
-        switch persistence.onStorageInitialize {
-        | Some(onStorageInitialize) => await onStorageInitialize()
-        | None => ()
+      if shouldRun {
+        let resolveRef = ref(%raw(`null`))
+        let promise = Promise.make((resolve, _) => {
+          resolveRef := resolve
+        })
+        persistence.storageStatus = Initializing(promise)
+        if reset || !(await persistence.storage.isInitialized()) {
+          Logging.info(`Initializing the indexer storage...`)
+
+          await persistence.storage.initialize(
+            ~entities=persistence.allEntities,
+            ~generalTables=persistence.staticTables,
+            ~enums=persistence.allEnums,
+          )
+
+          Logging.info(`The indexer storage is ready. Uploading cache...`)
+          persistence.storageStatus = Ready({
+            cleanRun: true,
+            cache: await loadInitialCache(persistence, ~withUpload=true),
+          })
+        } else if (
+          // In case of a race condition,
+          // we want to set the initial status to Ready only once.
+          switch persistence.storageStatus {
+          | Initializing(_) => true
+          | _ => false
+          }
+        ) {
+          Logging.info(`The indexer storage is ready.`)
+          persistence.storageStatus = Ready({
+            cleanRun: false,
+            cache: await loadInitialCache(persistence, ~withUpload=false),
+          })
         }
+        resolveRef.contents()
       }
-      resolveRef.contents()
+    } catch {
+    | exn =>
+      exn->ErrorHandling.mkLogAndRaise(~msg=`EE800: Failed to initialize the indexer storage.`)
     }
-  } catch {
-  | exn => exn->ErrorHandling.mkLogAndRaise(~msg=`EE800: Failed to initialize the indexer storage.`)
   }
 }
 
@@ -135,3 +173,27 @@ let getInitializedStorageOrThrow = persistence => {
   | Ready(_) => persistence.storage
   }
 }
+
+let setEffectCacheOrThrow = async (persistence, ~effect: Internal.effect, ~items) => {
+  switch persistence.storageStatus {
+  | Unknown
+  | Initializing(_) =>
+    Js.Exn.raiseError(`Failed to access the indexer storage. The Persistence layer is not initialized.`)
+  | Ready({cache}) => {
+      let storage = persistence.storage
+      let effectName = effect.name
+      let effectCacheRecord = switch cache->Utils.Dict.dangerouslyGetNonOption(effectName) {
+      | Some(c) => c
+      | None => {
+          let c = {effectName, count: 0}
+          cache->Js.Dict.set(effectName, c)
+          c
+        }
+      }
+      let initialize = effectCacheRecord.count === 0
+      await storage.setEffectCacheOrThrow(~effect, ~items, ~initialize)
+      effectCacheRecord.count = effectCacheRecord.count + items->Js.Array2.length
+      Prometheus.EffectCacheCount.set(~count=effectCacheRecord.count, ~effectName)
+    }
+  }
+}