envio 2.26.0-alpha.8 → 2.26.0-rc.0

package/evm.schema.json

@@ -33,13 +33,6 @@
         "null"
       ]
     },
-    "output": {
-      "description": "Path where the generated directory will be placed. By default it's 'generated' relative to the current working directory. If set, it'll be a path relative to the config file location.",
-      "type": [
-        "string",
-        "null"
-      ]
-    },
     "contracts": {
       "description": "Global contract definitions that must contain all definitions except addresses. You can share a single handler/abi/event definitions for contracts across multiple chains.",
       "type": [

package/fuel.schema.json

@@ -26,13 +26,6 @@
         "null"
       ]
     },
-    "output": {
-      "description": "Path where the generated directory will be placed. By default it's 'generated' relative to the current working directory. If set, it'll be a path relative to the config file location.",
-      "type": [
-        "string",
-        "null"
-      ]
-    },
     "contracts": {
       "description": "Global contract definitions that must contain all definitions except addresses. You can share a single handler/abi/event definitions for contracts across multiple chains.",
       "type": [

package/index.d.ts

@@ -13,7 +13,6 @@ import type {
 } from "./src/Envio.gen.ts";
 
 import { schema as bigDecimalSchema } from "./src/bindings/BigDecimal.gen.ts";
-import { schema as bigintSchema } from "./src/bindings/BigInt.gen.ts";
 import * as Sury from "rescript-schema";
 
 type UnknownToOutput<T> = T extends Sury.Schema<unknown>
@@ -36,6 +35,26 @@ type UnknownToOutput<T> = T extends Sury.Schema<unknown>
     >
   : T;
 
+type UnknownToInput<T> = T extends Sury.Schema<unknown>
+  ? Sury.Input<T>
+  : T extends (...args: any[]) => any
+  ? T
+  : T extends unknown[]
+  ? { [K in keyof T]: UnknownToInput<T[K]> }
+  : T extends { [k in keyof T]: unknown }
+  ? Flatten<
+      {
+        [k in keyof T as HasUndefined<UnknownToInput<T[k]>> extends true
+          ? k
+          : never]?: UnknownToInput<T[k]>;
+      } & {
+        [k in keyof T as HasUndefined<UnknownToInput<T[k]>> extends true
+          ? never
+          : k]: UnknownToInput<T[k]>;
+      }
+    >
+  : T;
+
 type HasUndefined<T> = [T] extends [undefined]
   ? true
   : undefined extends T
@@ -73,7 +92,7 @@ type Flatten<T> = T extends object
 export function experimental_createEffect<
   IS,
   OS,
-  I = UnknownToOutput<IS>,
+  I = UnknownToInput<IS>,
   O = UnknownToOutput<OS>,
   // A hack to enforce that the inferred return type
   // matches the output schema type
@@ -86,8 +105,6 @@ export function experimental_createEffect<
     readonly input: IS;
     /** The output schema of the effect. */
     readonly output: OS;
-    /** Whether the effect should be cached. */
-    readonly cache?: boolean;
   },
   handler: (args: EffectArgs<I>) => Promise<R>
 ): Effect<I, O>;
@@ -102,7 +119,7 @@ export declare namespace S {
   export const boolean: typeof Sury.boolean;
   export const int32: typeof Sury.int32;
   export const number: typeof Sury.number;
-  export const bigint: typeof bigintSchema;
+  export const bigint: typeof Sury.bigint;
   export const never: typeof Sury.never;
   export const union: typeof Sury.union;
   export const object: typeof Sury.object;
@@ -111,7 +128,6 @@ export declare namespace S {
   // Don't expose recursive for now, since it's too advanced
   // export const recursive: typeof Sury.recursive;
   export const transform: typeof Sury.transform;
-  export const shape: typeof Sury.shape;
   export const refine: typeof Sury.refine;
   export const schema: typeof Sury.schema;
   export const record: typeof Sury.record;

package/index.js

@@ -13,7 +13,7 @@ exports.S = {
   boolean: Sury.boolean,
   int32: Sury.int32,
   number: Sury.number,
-  bigint: require("./src/bindings/BigInt.res.js").schema,
+  bigint: Sury.bigint,
   never: Sury.never,
   union: Sury.union,
   object: Sury.object,
@@ -22,7 +22,6 @@ exports.S = {
   // Don't expose recursive for now, since it's too advanced
   // recursive: Sury.recursive,
   transform: Sury.transform,
-  shape: Sury.shape,
   refine: Sury.refine,
   schema: Sury.schema,
   record: Sury.record,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "envio",
-  "version": "v2.26.0-alpha.8",
+  "version": "v2.26.0-rc.0",
   "description": "A latency and sync speed optimized, developer friendly blockchain data indexer.",
   "bin": "./bin.js",
   "main": "./index.js",
@@ -25,10 +25,10 @@
   },
   "homepage": "https://envio.dev",
   "optionalDependencies": {
-    "envio-linux-x64": "v2.26.0-alpha.8",
-    "envio-linux-arm64": "v2.26.0-alpha.8",
-    "envio-darwin-x64": "v2.26.0-alpha.8",
-    "envio-darwin-arm64": "v2.26.0-alpha.8"
+    "envio-linux-x64": "v2.26.0-rc.0",
+    "envio-linux-arm64": "v2.26.0-rc.0",
+    "envio-darwin-x64": "v2.26.0-rc.0",
+    "envio-darwin-arm64": "v2.26.0-rc.0"
   },
   "dependencies": {
     "@envio-dev/hypersync-client": "0.6.5",

package/src/Envio.gen.ts

@@ -21,9 +21,7 @@ export type effectOptions<input,output> = {
   /** The input schema of the effect. */
   readonly input: RescriptSchema_S_t<input>; 
   /** The output schema of the effect. */
-  readonly output: RescriptSchema_S_t<output>; 
-  /** Whether the effect should be cached. */
-  readonly cache?: boolean
+  readonly output: RescriptSchema_S_t<output>
 };
 
 export type effectContext = $$effectContext;

package/src/Envio.res

@@ -22,8 +22,6 @@ and effectOptions<'input, 'output> = {
   input: S.t<'input>,
   /** The output schema of the effect. */
   output: S.t<'output>,
-  /** Whether the effect should be cached. */
-  cache?: bool,
 }
 @genType.import(("./Types.ts", "EffectContext"))
 and effectContext = {
@@ -50,16 +48,5 @@ let experimental_createEffect = (
       >
     ),
     callsCount: 0,
-    // This is the way to make the createEffect API
-    // work without the need for users to call S.schema themselves,
-    // but simply pass the desired object/tuple/etc.
-    // If they pass a schem, it'll also work.
-    input: S.schema(_ => options.input)->(
-      Utils.magic: S.t<S.t<'input>> => S.t<Internal.effectInput>
-    ),
-    output: S.schema(_ => options.output)->(
-      Utils.magic: S.t<S.t<'output>> => S.t<Internal.effectOutput>
-    ),
-    cache: options.cache->Belt.Option.getWithDefault(false),
   }->(Utils.magic: Internal.effect => effect<'input, 'output>)
 }

package/src/Envio.res.js

@@ -2,21 +2,12 @@
 'use strict';
 
 var Prometheus = require("./Prometheus.res.js");
-var Belt_Option = require("rescript/lib/js/belt_Option.js");
-var S$RescriptSchema = require("rescript-schema/src/S.res.js");
 
 function experimental_createEffect(options, handler) {
   Prometheus.EffectCallsCount.set(0, options.name);
   return {
           name: options.name,
           handler: handler,
-          cache: Belt_Option.getWithDefault(options.cache, false),
-          output: S$RescriptSchema.schema(function (param) {
-                return options.output;
-              }),
-          input: S$RescriptSchema.schema(function (param) {
-                return options.input;
-              }),
           callsCount: 0
         };
 }

package/src/FetchState.res

@@ -901,7 +901,10 @@ let queueItemIsInReorgThreshold = (
   if currentBlockHeight === 0 {
     false
   } else {
-    queueItem->queueItemBlockNumber > highestBlockBelowThreshold
+    switch queueItem {
+    | Item(_) => queueItem->queueItemBlockNumber > highestBlockBelowThreshold
+    | NoItem(_) => queueItem->queueItemBlockNumber > highestBlockBelowThreshold
+    }
   }
 }
 

package/src/Internal.res

@@ -197,9 +197,6 @@ type effectArgs = {
 type effect = {
   name: string,
   handler: effectArgs => promise<effectOutput>,
-  cache: bool,
-  output: S.t<effectOutput>,
-  input: S.t<effectInput>,
   mutable callsCount: int,
 }
 

package/src/Logging.res

@@ -168,14 +168,6 @@ let logForItem = (eventItem, level: Pino.logLevel, message: string, ~params=?) =
   (eventItem->getEventLogger->Utils.magic->Js.Dict.unsafeGet((level :> string)))(params, message)
 }
 
-let noopLogger: Envio.logger = {
-  info: (_message: string, ~params as _=?) => (),
-  debug: (_message: string, ~params as _=?) => (),
-  warn: (_message: string, ~params as _=?) => (),
-  error: (_message: string, ~params as _=?) => (),
-  errorWithExn: (_message: string, _exn) => (),
-}
-
 let getUserLogger = (eventItem): Envio.logger => {
   info: (message: string, ~params=?) => eventItem->logForItem(#uinfo, message, ~params?),
   debug: (message: string, ~params=?) => eventItem->logForItem(#udebug, message, ~params?),

package/src/Logging.res.js

@@ -197,34 +197,6 @@ function logForItem(eventItem, level, message, params) {
   return getEventLogger(eventItem)[level](params, message);
 }
 
-function noopLogger_debug(_message, param) {
-  
-}
-
-function noopLogger_info(_message, param) {
-  
-}
-
-function noopLogger_warn(_message, param) {
-  
-}
-
-function noopLogger_error(_message, param) {
-  
-}
-
-function noopLogger_errorWithExn(_message, _exn) {
-  
-}
-
-var noopLogger = {
-  debug: noopLogger_debug,
-  info: noopLogger_info,
-  warn: noopLogger_warn,
-  error: noopLogger_error,
-  errorWithExn: noopLogger_errorWithExn
-};
-
 function getUserLogger(eventItem) {
   return {
           debug: (function (message, params) {
@@ -271,6 +243,5 @@ exports.createChild = createChild;
 exports.createChildFrom = createChildFrom;
 exports.getEventLogger = getEventLogger;
 exports.logForItem = logForItem;
-exports.noopLogger = noopLogger;
 exports.getUserLogger = getUserLogger;
 /* logLevels Not a pure module */

package/src/Persistence.res

@@ -5,16 +5,6 @@
 // Currently there are quite many code spread across
 // DbFunctions, Db, Migrations, InMemoryStore modules which use codegen code directly.
 
-// The type reflects an cache table in the db
-// It might be present even if the effect is not used in the application
-type effectCacheRecord = {
-  effectName: string,
-  // Number of rows in the table
-  mutable count: int,
-}
-
-type operator = [#">" | #"="]
-
 type storage = {
   // Should return true if we already have persisted data
   // and we can skip initialization
@@ -25,6 +15,8 @@ type storage = {
     ~entities: array<Internal.entityConfig>=?,
     ~generalTables: array<Table.table>=?,
     ~enums: array<Internal.enumConfig<Internal.enum>>=?,
+    // If true, the storage should clear existing data
+    ~cleanRun: bool=?,
   ) => promise<unit>,
   @raises("StorageError")
   loadByIdsOrThrow: 'item. (
@@ -33,30 +25,11 @@ type storage = {
     ~rowsSchema: S.t<array<'item>>,
   ) => promise<array<'item>>,
   @raises("StorageError")
-  loadByFieldOrThrow: 'item 'value. (
-    ~fieldName: string,
-    ~fieldSchema: S.t<'value>,
-    ~fieldValue: 'value,
-    ~operator: operator,
-    ~table: Table.table,
-    ~rowsSchema: S.t<array<'item>>,
-  ) => promise<array<'item>>,
-  @raises("StorageError")
   setOrThrow: 'item. (
     ~items: array<'item>,
     ~table: Table.table,
     ~itemSchema: S.t<'item>,
   ) => promise<unit>,
-  @raises("StorageError")
-  setEffectCacheOrThrow: (
-    ~effectName: string,
-    ~ids: array<string>,
-    ~outputs: array<Internal.effectOutput>,
-    ~outputSchema: S.t<Internal.effectOutput>,
-    ~initialize: bool,
-  ) => promise<unit>,
-  dumpEffectCache: unit => promise<unit>,
-  restoreEffectCache: (~withUpload: bool) => promise<array<effectCacheRecord>>,
 }
 
 exception StorageError({message: string, reason: exn})
@@ -64,7 +37,7 @@ exception StorageError({message: string, reason: exn})
 type storageStatus =
   | Unknown
   | Initializing(promise<unit>)
-  | Ready({cleanRun: bool, cache: dict<effectCacheRecord>})
+  | Ready({cleanRun: bool})
 
 type t = {
   userEntities: array<Internal.entityConfig>,
@@ -73,6 +46,8 @@ type t = {
   allEnums: array<Internal.enumConfig<Internal.enum>>,
   mutable storageStatus: storageStatus,
   storage: storage,
+  onStorageInitialize: option<unit => promise<unit>>,
+  cacheStorage: storage,
 }
 
 let entityHistoryActionEnumConfig: Internal.enumConfig<EntityHistory.RowAction.t> = {
@@ -89,6 +64,8 @@ let make = (
   ~allEnums,
   ~staticTables,
   ~storage,
+  ~cacheStorage,
+  ~onStorageInitialize=?,
 ) => {
   let allEntities = userEntities->Js.Array2.concat([dcRegistryEntityConfig])
   let allEnums =
@@ -100,70 +77,53 @@ let make = (
     allEnums,
     storageStatus: Unknown,
     storage,
+    onStorageInitialize,
+    cacheStorage,
   }
 }
 
-let init = {
-  let loadInitialCache = async (persistence, ~withUpload) => {
-    let effectCacheRecords = await persistence.storage.restoreEffectCache(~withUpload)
-    let cache = Js.Dict.empty()
-    effectCacheRecords->Js.Array2.forEach(record => {
-      Prometheus.EffectCacheCount.set(~count=record.count, ~effectName=record.effectName)
-      cache->Js.Dict.set(record.effectName, record)
-    })
-    cache
-  }
-
-  async (persistence, ~reset=false) => {
-    try {
-      let shouldRun = switch persistence.storageStatus {
-      | Unknown => true
-      | Initializing(promise) => {
-          await promise
-          reset
-        }
-      | Ready(_) => reset
+let init = async (
+  persistence,
+  // There are not much sense in the option,
+  // but this is how the runUpMigration used to work
+  // and we want to keep the upsert behavior without breaking changes.
+  ~skipIsInitializedCheck=false,
+  ~reset=false,
+) => {
+  try {
+    let shouldRun = switch persistence.storageStatus {
+    | Unknown => true
+    | Initializing(promise) => {
+        await promise
+        reset
       }
-      if shouldRun {
-        let resolveRef = ref(%raw(`null`))
-        let promise = Promise.make((resolve, _) => {
-          resolveRef := resolve
-        })
-        persistence.storageStatus = Initializing(promise)
-        if reset || !(await persistence.storage.isInitialized()) {
-          Logging.info(`Initializing the indexer storage...`)
-
-          await persistence.storage.initialize(
-            ~entities=persistence.allEntities,
-            ~generalTables=persistence.staticTables,
-            ~enums=persistence.allEnums,
-          )
-
-          Logging.info(`The indexer storage is ready. Uploading cache...`)
-          persistence.storageStatus = Ready({
-            cleanRun: true,
-            cache: await loadInitialCache(persistence, ~withUpload=true),
-          })
-        } else if (
-          // In case of a race condition,
-          // we want to set the initial status to Ready only once.
-          switch persistence.storageStatus {
-          | Initializing(_) => true
-          | _ => false
-          }
-        ) {
-          Logging.info(`The indexer storage is ready.`)
-          persistence.storageStatus = Ready({
-            cleanRun: false,
-            cache: await loadInitialCache(persistence, ~withUpload=false),
-          })
+    | Ready(_) => reset
+    }
+    if shouldRun {
+      let resolveRef = ref(%raw(`null`))
+      let promise = Promise.make((resolve, _) => {
+        resolveRef := resolve
+      })
+      persistence.storageStatus = Initializing(promise)
+      if !(reset || skipIsInitializedCheck) && (await persistence.storage.isInitialized()) {
+        persistence.storageStatus = Ready({cleanRun: false})
+      } else {
+        let _ = await persistence.storage.initialize(
+          ~entities=persistence.allEntities,
+          ~generalTables=persistence.staticTables,
+          ~enums=persistence.allEnums,
+          ~cleanRun=reset || !skipIsInitializedCheck,
+        )
+        persistence.storageStatus = Ready({cleanRun: true})
+        switch persistence.onStorageInitialize {
+        | Some(onStorageInitialize) => await onStorageInitialize()
+        | None => ()
         }
-        resolveRef.contents()
       }
-    } catch {
-    | exn =>
-      exn->ErrorHandling.mkLogAndRaise(~msg=`EE800: Failed to initialize the indexer storage.`)
+      resolveRef.contents()
     }
+  } catch {
+  | exn => exn->ErrorHandling.mkLogAndRaise(~msg=`EE800: Failed to initialize the indexer storage.`)
   }
 }
 
@@ -175,26 +135,3 @@ let getInitializedStorageOrThrow = persistence => {
   | Ready(_) => persistence.storage
   }
 }
-
-let setEffectCacheOrThrow = async (persistence, ~effectName, ~ids, ~outputs, ~outputSchema) => {
-  switch persistence.storageStatus {
-  | Unknown
-  | Initializing(_) =>
-    Js.Exn.raiseError(`Failed to access the indexer storage. The Persistence layer is not initialized.`)
-  | Ready({cache}) => {
-      let storage = persistence.storage
-      let effectCacheRecord = switch cache->Utils.Dict.dangerouslyGetNonOption(effectName) {
-      | Some(c) => c
-      | None => {
-          let c = {effectName, count: 0}
-          cache->Js.Dict.set(effectName, c)
-          c
-        }
-      }
-      let initialize = effectCacheRecord.count === 0
-      await storage.setEffectCacheOrThrow(~effectName, ~ids, ~outputs, ~outputSchema, ~initialize)
-      effectCacheRecord.count = effectCacheRecord.count + ids->Js.Array2.length
-      Prometheus.EffectCacheCount.set(~count=effectCacheRecord.count, ~effectName)
-    }
-  }
-}

package/src/Persistence.res.js

@@ -2,8 +2,6 @@
 'use strict';
 
 var Js_exn = require("rescript/lib/js/js_exn.js");
-var Logging = require("./Logging.res.js");
-var Prometheus = require("./Prometheus.res.js");
 var EntityHistory = require("./db/EntityHistory.res.js");
 var ErrorHandling = require("./ErrorHandling.res.js");
 var Caml_exceptions = require("rescript/lib/js/caml_exceptions.js");
@@ -24,7 +22,7 @@ var entityHistoryActionEnumConfig = {
   default: "SET"
 };
 
-function make(userEntities, dcRegistryEntityConfig, allEnums, staticTables, storage) {
+function make(userEntities, dcRegistryEntityConfig, allEnums, staticTables, storage, cacheStorage, onStorageInitialize) {
   var allEntities = userEntities.concat([dcRegistryEntityConfig]);
   var allEnums$1 = allEnums.concat([entityHistoryActionEnumConfig]);
   return {
@@ -33,21 +31,14 @@ function make(userEntities, dcRegistryEntityConfig, allEnums, staticTables, stor
           allEntities: allEntities,
           allEnums: allEnums$1,
           storageStatus: "Unknown",
-          storage: storage
+          storage: storage,
+          onStorageInitialize: onStorageInitialize,
+          cacheStorage: cacheStorage
         };
 }
 
-async function loadInitialCache(persistence, withUpload) {
-  var effectCacheRecords = await persistence.storage.restoreEffectCache(withUpload);
-  var cache = {};
-  effectCacheRecords.forEach(function (record) {
-        Prometheus.EffectCacheCount.set(record.count, record.effectName);
-        cache[record.effectName] = record;
-      });
-  return cache;
-}
-
-async function init(persistence, resetOpt) {
+async function init(persistence, skipIsInitializedCheckOpt, resetOpt) {
+  var skipIsInitializedCheck = skipIsInitializedCheckOpt !== undefined ? skipIsInitializedCheckOpt : false;
   var reset = resetOpt !== undefined ? resetOpt : false;
   try {
     var promise = persistence.storageStatus;
@@ -73,26 +64,20 @@ async function init(persistence, resetOpt) {
       TAG: "Initializing",
       _0: promise$1
     };
-    if (reset || !await persistence.storage.isInitialized()) {
-      Logging.info("Initializing the indexer storage...");
-      await persistence.storage.initialize(persistence.allEntities, persistence.staticTables, persistence.allEnums);
-      Logging.info("The indexer storage is ready. Uploading cache...");
+    if (!(reset || skipIsInitializedCheck) && await persistence.storage.isInitialized()) {
       persistence.storageStatus = {
         TAG: "Ready",
-        cleanRun: true,
-        cache: await loadInitialCache(persistence, true)
+        cleanRun: false
       };
     } else {
-      var match = persistence.storageStatus;
-      var tmp;
-      tmp = typeof match !== "object" || match.TAG !== "Initializing" ? false : true;
-      if (tmp) {
-        Logging.info("The indexer storage is ready.");
-        persistence.storageStatus = {
-          TAG: "Ready",
-          cleanRun: false,
-          cache: await loadInitialCache(persistence, false)
-        };
+      await persistence.storage.initialize(persistence.allEntities, persistence.staticTables, persistence.allEnums, reset || !skipIsInitializedCheck);
+      persistence.storageStatus = {
+        TAG: "Ready",
+        cleanRun: true
+      };
+      var onStorageInitialize = persistence.onStorageInitialize;
+      if (onStorageInitialize !== undefined) {
+        await onStorageInitialize();
       }
       
     }
@@ -113,38 +98,9 @@ function getInitializedStorageOrThrow(persistence) {
   }
 }
 
-async function setEffectCacheOrThrow(persistence, effectName, ids, outputs, outputSchema) {
-  var match = persistence.storageStatus;
-  if (typeof match !== "object") {
-    return Js_exn.raiseError("Failed to access the indexer storage. The Persistence layer is not initialized.");
-  }
-  if (match.TAG === "Initializing") {
-    return Js_exn.raiseError("Failed to access the indexer storage. The Persistence layer is not initialized.");
-  }
-  var cache = match.cache;
-  var storage = persistence.storage;
-  var c = cache[effectName];
-  var effectCacheRecord;
-  if (c !== undefined) {
-    effectCacheRecord = c;
-  } else {
-    var c$1 = {
-      effectName: effectName,
-      count: 0
-    };
-    cache[effectName] = c$1;
-    effectCacheRecord = c$1;
-  }
-  var initialize = effectCacheRecord.count === 0;
-  await storage.setEffectCacheOrThrow(effectName, ids, outputs, outputSchema, initialize);
-  effectCacheRecord.count = effectCacheRecord.count + ids.length | 0;
-  return Prometheus.EffectCacheCount.set(effectCacheRecord.count, effectName);
-}
-
 exports.StorageError = StorageError;
 exports.entityHistoryActionEnumConfig = entityHistoryActionEnumConfig;
 exports.make = make;
 exports.init = init;
 exports.getInitializedStorageOrThrow = getInitializedStorageOrThrow;
-exports.setEffectCacheOrThrow = setEffectCacheOrThrow;
-/* Logging Not a pure module */
+/* EntityHistory Not a pure module */