envio 3.1.1 → 3.2.0

package/src/LoadLayer.res.mjs

@@ -1,14 +1,14 @@
 // Generated by ReScript, PLEASE EDIT WITH CARE
 
+import * as Table from "./db/Table.res.mjs";
 import * as Utils from "./Utils.res.mjs";
 import * as Hrtime from "./bindings/Hrtime.res.mjs";
 import * as Logging from "./Logging.res.mjs";
-import * as Internal from "./Internal.res.mjs";
-import * as Belt_Array from "@rescript/runtime/lib/es6/Belt_Array.js";
 import * as Prometheus from "./Prometheus.res.mjs";
 import * as LoadManager from "./LoadManager.res.mjs";
 import * as Persistence from "./Persistence.res.mjs";
-import * as TableIndices from "./TableIndices.res.mjs";
+import * as EntityFilter from "./db/EntityFilter.res.mjs";
+import * as Stdlib_Array from "@rescript/runtime/lib/es6/Stdlib_Array.js";
 import * as ErrorHandling from "./ErrorHandling.res.mjs";
 import * as InMemoryStore from "./InMemoryStore.res.mjs";
 import * as InMemoryTable from "./InMemoryTable.res.mjs";
@@ -23,7 +23,11 @@ function loadById(loadManager, persistence, entityConfig, inMemoryStore, shouldG
     let timerRef = Prometheus.StorageLoad.startOperation(storage.name, key);
     let dbEntities;
     try {
-      dbEntities = await storage.loadByIdsOrThrow(idsToLoad, entityConfig.table, entityConfig.rowsSchema);
+      dbEntities = await storage.loadOrThrow({
+        operator: "in",
+        fieldName: Table.idFieldName,
+        fieldValue: idsToLoad
+      }, entityConfig.table);
     } catch (raw_exn) {
       let exn = Primitive_exceptions.internalToException(raw_exn);
       if (exn.RE_EXN_ID === Persistence.StorageError) {
@@ -79,8 +83,8 @@ function executeWithRateLimit(effect, effectArgs, inMemTable, onError, isFromQue
       state.nextWindowPromise = undefined;
     }
     let immediateCount = Math.min(state.availableCalls, effectArgs.length);
-    let immediateArgs = Belt_Array.slice(effectArgs, 0, immediateCount);
-    let queuedArgs = Belt_Array.sliceToEnd(effectArgs, immediateCount);
+    let immediateArgs = effectArgs.slice(0, immediateCount);
+    let queuedArgs = effectArgs.slice(immediateCount);
     state.availableCalls = state.availableCalls - immediateCount | 0;
     for (let idx = 0, idx_finish = immediateArgs.length; idx < idx_finish; ++idx) {
       promises.push(callEffect(effect, immediateArgs[idx], inMemTable, timerRef, onError));
@@ -139,7 +143,11 @@ function loadEffect(loadManager, persistence, effect, effectArgs, inMemoryStore,
       let outputSchema = match$1.outputSchema;
       let dbEntities;
       try {
-        dbEntities = await storage.loadByIdsOrThrow(idsToLoad, match$1.table, Internal.effectCacheItemRowsSchema);
+        dbEntities = await storage.loadOrThrow({
+          operator: "in",
+          fieldName: Table.idFieldName,
+          fieldValue: idsToLoad
+        }, match$1.table);
       } catch (raw_exn) {
         let exn = Primitive_exceptions.internalToException(raw_exn);
         Logging.childWarn(Logging.getItemLogger(item), {
@@ -189,48 +197,20 @@ function loadEffect(loadManager, persistence, effect, effectArgs, inMemoryStore,
   return LoadManager.call(loadManager, effectArgs, key, load, args => args.cacheKey, shouldGroup, hash => InMemoryStore.hasEffectOutput(inMemTable, hash), hash => InMemoryStore.getEffectOutputUnsafe(inMemTable, hash));
 }
 
-function loadByField(loadManager, persistence, operator, entityConfig, inMemoryStore, fieldName, fieldValueSchema, shouldGroup, item, fieldValue) {
-  let operatorCallName;
-  switch (operator) {
-    case "Eq" :
-      operatorCallName = "eq";
-      break;
-    case "Gt" :
-      operatorCallName = "gt";
-      break;
-    case "Lt" :
-      operatorCallName = "lt";
-      break;
-  }
-  let key = entityConfig.name + `.getWhere.` + fieldName + `.` + operatorCallName;
+function loadByFilter(loadManager, persistence, entityConfig, inMemoryStore, shouldGroup, item, filter) {
+  let key = EntityFilter.toOperationKey(filter, entityConfig.name);
   let inMemTable = InMemoryStore.getInMemTable(inMemoryStore, entityConfig);
-  let load = async (fieldValues, param) => {
+  let load = async (filters, param) => {
     let storage = Persistence.getInitializedStorageOrThrow(persistence);
     let timerRef = Prometheus.StorageLoad.startOperation(storage.name, key);
     let size = {
       contents: 0
     };
-    let indiciesToLoad = fieldValues.map(fieldValue => ({
-      fieldName: fieldName,
-      fieldValue: fieldValue,
-      operator: operator
-    }));
-    await Promise.all(indiciesToLoad.map(async index => {
-      InMemoryTable.Entity.addEmptyIndex(inMemTable, index);
+    filters.forEach(filter => InMemoryTable.Entity.addEmptyIndex(inMemTable, filter));
+    let queries = EntityFilter.merge(filters);
+    await Promise.all(queries.map(async filter => {
       try {
-        let tmp;
-        switch (index.operator) {
-          case "Eq" :
-            tmp = "=";
-            break;
-          case "Gt" :
-            tmp = ">";
-            break;
-          case "Lt" :
-            tmp = "<";
-            break;
-        }
-        let entities = await storage.loadByFieldOrThrow(TableIndices.Index.getFieldName(index), fieldValueSchema, index.fieldValue, tmp, entityConfig.table, entityConfig.rowsSchema);
+        let entities = await storage.loadOrThrow(filter, entityConfig.table);
         entities.forEach(entity => InMemoryTable.Entity.initValue(inMemTable, inMemoryStore.committedCheckpointId, entity.id, entity));
         size.contents = size.contents + entities.length | 0;
         return;
@@ -238,23 +218,21 @@ function loadByField(loadManager, persistence, operator, entityConfig, inMemoryS
         let exn = Primitive_exceptions.internalToException(raw_exn);
         if (exn.RE_EXN_ID === Persistence.StorageError) {
           return ErrorHandling.mkLogAndRaise(Logging.createChildFrom(Logging.getItemLogger(item), {
-            operator: operatorCallName,
-            tableName: entityConfig.table.tableName,
-            fieldName: fieldName,
-            fieldValue: fieldValue
+            operation: key,
+            params: EntityFilter.getParams(filter)
           }), exn.message, exn.reason);
         }
         throw exn;
       }
     }));
-    return Prometheus.StorageLoad.endOperation(timerRef, storage.name, key, fieldValues.length, size.contents);
+    return Prometheus.StorageLoad.endOperation(timerRef, storage.name, key, Stdlib_Array.reduce(queries, 0, (acc, query) => acc + EntityFilter.valuesCount(query) | 0), size.contents);
   };
-  return LoadManager.call(loadManager, fieldValue, key, load, fieldValue => TableIndices.FieldValue.toString(fieldValue), shouldGroup, InMemoryTable.Entity.hasIndex(inMemTable, fieldName, operator), InMemoryTable.Entity.getUnsafeOnIndex(inMemTable, fieldName, operator));
+  return LoadManager.call(loadManager, filter, key, load, EntityFilter.toString, shouldGroup, InMemoryTable.Entity.hasIndex(inMemTable), InMemoryTable.Entity.getUnsafeOnIndex(inMemTable));
 }
 
 export {
   loadById,
-  loadByField,
+  loadByFilter,
   loadEffect,
 }
-/* Utils Not a pure module */
+/* Table Not a pure module */

package/src/LoadLayer.resi

@@ -8,17 +8,14 @@ let loadById: (
   ~entityId: string,
 ) => promise<option<Internal.entity>>
 
-let loadByField: (
+let loadByFilter: (
   ~loadManager: LoadManager.t,
   ~persistence: Persistence.t,
-  ~operator: TableIndices.Operator.t,
   ~entityConfig: Internal.entityConfig,
   ~inMemoryStore: InMemoryStore.t,
-  ~fieldName: string,
-  ~fieldValueSchema: RescriptSchema.S.t<'fieldValue>,
   ~shouldGroup: bool,
   ~item: Internal.item,
-  ~fieldValue: 'fieldValue,
+  ~filter: EntityFilter.t,
 ) => promise<array<Internal.entity>>
 
 let loadEffect: (

package/src/LogSelection.res

@@ -12,7 +12,7 @@ let makeTopicSelection = (~topic0, ~topic1=[], ~topic2=[], ~topic3=[]) =>
   }
 
 let hasFilters = ({topic1, topic2, topic3}: Internal.topicSelection) => {
-  [topic1, topic2, topic3]->Array.find(topic => !Utils.Array.isEmpty(topic))->Belt.Option.isSome
+  [topic1, topic2, topic3]->Array.find(topic => !Utils.Array.isEmpty(topic))->Option.isSome
 }
 
 /**
@@ -24,11 +24,11 @@ let compressTopicSelections = (topicSelections: array<Internal.topicSelection>)
 
   let selectionsWithFilters = []
 
-  topicSelections->Belt.Array.forEach(selection => {
+  topicSelections->Array.forEach(selection => {
     if selection->hasFilters {
       selectionsWithFilters->Array.push(selection)->ignore
     } else {
-      selection.topic0->Belt.Array.forEach(topic0 => {
+      selection.topic0->Array.forEach(topic0 => {
         topic0sOfSelectionsWithoutFilters->Array.push(topic0)->ignore
       })
     }
@@ -43,7 +43,7 @@ let compressTopicSelections = (topicSelections: array<Internal.topicSelection>)
       topic2: [],
       topic3: [],
     }
-    Belt.Array.concat([selectionWithoutFilters], selectionsWithFilters)
+    Array.concat([selectionWithoutFilters], selectionsWithFilters)
   }
 }
 

package/src/LogSelection.res.mjs

@@ -1,8 +1,7 @@
 // Generated by ReScript, PLEASE EDIT WITH CARE
 
 import * as Utils from "./Utils.res.mjs";
-import * as Belt_Array from "@rescript/runtime/lib/es6/Belt_Array.js";
-import * as Belt_Option from "@rescript/runtime/lib/es6/Belt_Option.js";
+import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
 import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
 import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js";
 import * as S$RescriptSchema from "rescript-schema/src/S.res.mjs";
@@ -35,7 +34,7 @@ function makeTopicSelection(topic0, topic1Opt, topic2Opt, topic3Opt) {
 }
 
 function hasFilters(param) {
-  return Belt_Option.isSome([
+  return Stdlib_Option.isSome([
     param.topic1,
     param.topic2,
     param.topic3
@@ -45,12 +44,11 @@ function hasFilters(param) {
 function compressTopicSelections(topicSelections) {
   let topic0sOfSelectionsWithoutFilters = [];
   let selectionsWithFilters = [];
-  Belt_Array.forEach(topicSelections, selection => {
+  topicSelections.forEach(selection => {
     if (hasFilters(selection)) {
       selectionsWithFilters.push(selection);
-      return;
     } else {
-      return Belt_Array.forEach(selection.topic0, topic0 => {
+      selection.topic0.forEach(topic0 => {
         topic0sOfSelectionsWithoutFilters.push(topic0);
       });
     }
@@ -67,7 +65,7 @@ function compressTopicSelections(topicSelections) {
     topic2: selectionWithoutFilters_topic2,
     topic3: selectionWithoutFilters_topic3
   };
-  return Belt_Array.concat([selectionWithoutFilters], selectionsWithFilters);
+  return [selectionWithoutFilters].concat(selectionsWithFilters);
 }
 
 function make(addresses, topicSelections) {

package/src/Logging.res

@@ -172,7 +172,7 @@ let getItemLogger = {
               "chainId": chain->ChainMap.Chain.toChainId,
               "block": blockNumber,
               "logIndex": logIndex,
-              "address": event.srcAddress,
+              "address": (event->Internal.toGenericEvent).srcAddress,
             }->createChildParams
           | Block({blockNumber, onBlockConfig}) =>
             {

package/src/Main.res

@@ -296,6 +296,59 @@ let getGlobalIndexer = (): 'indexer => {
     )
   }
 
+  // SVM identity: `{program, instruction}` from TS or
+  // `{instruction: GADT{contract, _0}}` from ReScript. Same two-format dance
+  // as the EVM `parseIdentityConfig`, but reading the SVM-native field names.
+  let parseSvmIdentityConfig = (identityConfig: 'a) => {
+    let raw =
+      identityConfig->(
+        Utils.magic: 'a => {"program": unknown, "instruction": unknown, "where": option<JSON.t>}
+      )
+    let (programName, instructionName) = if typeof(raw["program"]) === #string {
+      (
+        raw["program"]->(Utils.magic: unknown => string),
+        raw["instruction"]->(Utils.magic: unknown => string),
+      )
+    } else {
+      let inst = raw["instruction"]->(Utils.magic: unknown => {"contract": string, "_0": string})
+      (inst["contract"], inst["_0"])
+    }
+    let where = raw["where"]
+    let eventOptions: option<Internal.eventOptions<_>> = switch where {
+    | None => None
+    | Some(_) =>
+      Some({
+        where: ?(where->(Utils.magic: option<JSON.t> => option<_>)),
+      })
+    }
+    (programName, instructionName, eventOptions)
+  }
+
+  // onInstruction: delegates to HandlerRegister.setHandler. The SVM analog of
+  // onEvent; the registration store keys on `(contractName, eventName)` which
+  // for SVM is `(programName, instructionName)`.
+  let onInstructionFn = (identityConfig: 'a, handler: 'b) => {
+    HandlerRegister.throwIfFinishedRegistration(~methodName="onInstruction")
+    let (programName, instructionName, eventOptions) = parseSvmIdentityConfig(identityConfig)
+    // The generic dispatch hands every handler `{event, context}`. SVM handlers
+    // receive the instruction under `instruction`, so remap the field here; the
+    // payload object itself is the `svmInstruction` built in SvmHyperSyncSource.
+    let userHandler =
+      handler->(
+        Utils.magic: 'b => Envio.svmOnInstructionArgs<Internal.handlerContext> => promise<unit>
+      )
+    HandlerRegister.setHandler(
+      ~contractName=programName,
+      ~eventName=instructionName,
+      (args: Internal.genericHandlerArgs<Internal.event, Internal.handlerContext>) =>
+        userHandler({
+          instruction: args.event->(Utils.magic: Internal.event => Envio.svmInstruction),
+          context: args.context,
+        }),
+      ~eventOptions,
+    )
+  }
+
   // contractRegister: delegates to HandlerRegister.setContractRegister
   let contractRegisterFn = (identityConfig: 'a, handler: 'b) => {
     HandlerRegister.throwIfFinishedRegistration(~methodName="contractRegister")
@@ -467,6 +520,7 @@ let getGlobalIndexer = (): 'indexer => {
             "description",
             "chainIds",
             "chains",
+            "onInstruction",
             "onSlot",
             "~internalAndWillBeRemovedSoon_onRollbackCommit",
           ]
@@ -490,6 +544,7 @@ let getGlobalIndexer = (): 'indexer => {
         chains->(Utils.magic: {..} => unknown)
       }
     | "onEvent" => onEventFn->Utils.magic
+    | "onInstruction" => onInstructionFn->Utils.magic
     | "contractRegister" => contractRegisterFn->Utils.magic
     | "onBlock" | "onSlot" => onBlockFn->Utils.magic
     | "~internalAndWillBeRemovedSoon_onRollbackCommit" => onRollbackCommitFn->Utils.magic
@@ -791,8 +846,12 @@ let start = async (
     ~onError,
   )
   let gsManager =
-    globalState->GlobalStateManager.make(~onError=exn =>
-      onError(exn->ErrorHandling.make(~msg="Indexer has failed with an unexpected error"))
+    globalState->GlobalStateManager.make(
+      ~reducers=GlobalState.makeReducers(
+        ~markBatchProcessed=MarkBatchProcessedAdapter.make(~inMemoryStore=ctx.inMemoryStore),
+      ),
+      ~onError=exn =>
+        onError(exn->ErrorHandling.make(~msg="Indexer has failed with an unexpected error")),
     )
   if shouldUseTui {
     let _rerender = Tui.start(~getState=() => gsManager->GlobalStateManager.getState)

package/src/Main.res.mjs

@@ -32,6 +32,7 @@ import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js
 import * as S$RescriptSchema from "rescript-schema/src/S.res.mjs";
 import * as GlobalStateManager from "./GlobalStateManager.res.mjs";
 import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
+import * as MarkBatchProcessedAdapter from "./adapters/MarkBatchProcessedAdapter.res.mjs";
 
 let chainDataSchema = S$RescriptSchema.schema(s => ({
   chainId: s.m(S$RescriptSchema.float),
@@ -251,6 +252,38 @@ function getGlobalIndexer() {
     let match = parseIdentityConfig(identityConfig);
     HandlerRegister.setHandler(match[0], match[1], handler, match[2], undefined);
   };
+  let parseSvmIdentityConfig = identityConfig => {
+    let match;
+    if (typeof identityConfig.program === "string") {
+      match = [
+        identityConfig.program,
+        identityConfig.instruction
+      ];
+    } else {
+      let inst = identityConfig.instruction;
+      match = [
+        inst.contract,
+        inst._0
+      ];
+    }
+    let where = identityConfig.where;
+    let eventOptions = where !== undefined ? ({
+        where: where
+      }) : undefined;
+    return [
+      match[0],
+      match[1],
+      eventOptions
+    ];
+  };
+  let onInstructionFn = (identityConfig, handler) => {
+    HandlerRegister.throwIfFinishedRegistration("onInstruction");
+    let match = parseSvmIdentityConfig(identityConfig);
+    HandlerRegister.setHandler(match[0], match[1], args => handler({
+      instruction: args.event,
+      context: args.context
+    }), match[2], undefined);
+  };
   let contractRegisterFn = (identityConfig, handler) => {
     HandlerRegister.throwIfFinishedRegistration("contractRegister");
     let match = parseIdentityConfig(identityConfig);
@@ -346,6 +379,7 @@ function getGlobalIndexer() {
           "description",
           "chainIds",
           "chains",
+          "onInstruction",
           "onSlot",
           "~internalAndWillBeRemovedSoon_onRollbackCommit"
         ];
@@ -380,6 +414,8 @@ function getGlobalIndexer() {
         return Config.loadWithoutRegistrations().name;
       case "onEvent" :
         return onEventFn;
+      case "onInstruction" :
+        return onInstructionFn;
       case "onBlock" :
       case "onSlot" :
         return onBlockFn;
@@ -587,7 +623,7 @@ async function start(persistence, resetOpt, isTestOpt, exitAfterFirstEventBlockO
   }
   let chainManager = ChainManager.makeFromDbState(Persistence.getInitializedState(persistence$1), config$2, registrations, undefined);
   let globalState = GlobalState.make(ctx, chainManager, isDevelopmentMode, shouldUseTui, exitAfterFirstEventBlock, onError);
-  let gsManager = GlobalStateManager.make(globalState, exn => onError(ErrorHandling.make(exn, undefined, "Indexer has failed with an unexpected error")));
+  let gsManager = GlobalStateManager.make(globalState, GlobalState.makeReducers(undefined, undefined, undefined, MarkBatchProcessedAdapter.make(ctx_inMemoryStore)), exn => onError(ErrorHandling.make(exn, undefined, "Indexer has failed with an unexpected error")));
   if (shouldUseTui) {
     Tui.start(() => GlobalStateManager.getState(gsManager));
   }

package/src/Persistence.res

@@ -40,8 +40,6 @@ type initialState = {
   envioInfo: option<JSON.t>,
 }
 
-type operator = [#">" | #"=" | #"<"]
-
 type updatedEffectCache = {
   effect: Internal.effect,
   items: array<Internal.effectCacheItem>,
@@ -78,21 +76,10 @@ type storage = {
     ~envioInfo: JSON.t,
   ) => promise<initialState>,
   resumeInitialState: unit => promise<initialState>,
+  // Returns rows matching the filter.
+  // Field values are serialized and rows parsed with the table's field schemas.
   @raises("StorageError")
-  loadByIdsOrThrow: 'item. (
-    ~ids: array<string>,
-    ~table: Table.table,
-    ~rowsSchema: S.t<array<'item>>,
-  ) => promise<array<'item>>,
-  @raises("StorageError")
-  loadByFieldOrThrow: 'item 'value. (
-    ~fieldName: string,
-    ~fieldSchema: S.t<'value>,
-    ~fieldValue: 'value,
-    ~operator: operator,
-    ~table: Table.table,
-    ~rowsSchema: S.t<array<'item>>,
-  ) => promise<array<'item>>,
+  loadOrThrow: (~filter: EntityFilter.t, ~table: Table.table) => promise<array<unknown>>,
   // This is to download cache from the database to .envio/cache
   dumpEffectCache: unit => promise<unit>,
   reset: unit => promise<unit>,