envio 3.3.0-alpha.9 → 3.3.0-rc.0

package/src/SimulateItems.res

@@ -240,9 +240,12 @@ let deriveSrcAddress = (
   }
 }
 
-let parse = (~simulateItems: array<JSON.t>, ~config: Config.t, ~chainConfig: Config.chain): array<
-  Internal.item,
-> => {
+let parse = (
+  ~simulateItems: array<JSON.t>,
+  ~config: Config.t,
+  ~chainConfig: Config.chain,
+  ~onEventRegistrations: array<Internal.onEventRegistration>,
+): array<Internal.item> => {
   let chain = ChainMap.Chain.makeUnsafe(~chainId=chainConfig.id)
   let chainId = chainConfig.id
   let startBlock = chainConfig.startBlock
@@ -345,6 +348,11 @@ let parse = (~simulateItems: array<JSON.t>, ~config: Config.t, ~chainConfig: Con
         ~chainId,
         ~eventConfig,
       )
+      // Append into the registration array that the chain state will own and
+      // put that same registration object directly on the simulated item.
+      let onEventRegistrationIndex = onEventRegistrations->Array.length
+      let onEventRegistration = {...onEventRegistration, index: onEventRegistrationIndex}
+      onEventRegistrations->Array.push(onEventRegistration)->ignore
 
       items
       ->Array.push(
@@ -382,7 +390,11 @@ let parse = (~simulateItems: array<JSON.t>, ~config: Config.t, ~chainConfig: Con
 
 // Apply simulate source config from processConfig JSON to a Config.t
 // This patches chainMap entries that have simulate items with CustomSources
-let patchConfig = (~config: Config.t, ~processConfig: JSON.t): Config.t => {
+let patchConfig = (
+  ~config: Config.t,
+  ~processConfig: JSON.t,
+  ~registrationsByChainId: HandlerRegister.registrationsByChainId,
+): Config.t => {
   let processChains: option<dict<JSON.t>> =
     (processConfig->(Utils.magic: JSON.t => {..}))["chains"]->Nullable.toOption
   switch processChains {
@@ -395,11 +407,31 @@ let patchConfig = (~config: Config.t, ~processConfig: JSON.t): Config.t => {
         let simulateRaw: option<array<JSON.t>> = raw["simulate"]->Nullable.toOption
         switch simulateRaw {
         | Some(simulateItems) =>
-          let items = parse(~simulateItems, ~config, ~chainConfig)
+          let chainRegistrations = switch registrationsByChainId->Utils.Dict.dangerouslyGetNonOption(
+            chainIdStr,
+          ) {
+          | Some(registrations) => registrations
+          | None =>
+            let registrations: HandlerRegister.chainRegistrations = {
+              onEventRegistrations: [],
+              onBlockRegistrations: [],
+            }
+            registrationsByChainId->Dict.set(chainIdStr, registrations)
+            registrations
+          }
           let startBlock: int = raw["startBlock"]->(Utils.magic: 'a => int)
           let endBlock: int = raw["endBlock"]->(Utils.magic: 'a => int)
+          // Parse with the process's startBlock so items default into the range
+          // the source will be queried over; the source now filters by range.
+          let chainConfig = {...chainConfig, startBlock, endBlock}
+          let items = parse(
+            ~simulateItems,
+            ~config,
+            ~chainConfig,
+            ~onEventRegistrations=chainRegistrations.onEventRegistrations,
+          )
           let source = SimulateSource.make(~items, ~endBlock, ~chain)
-          {...chainConfig, startBlock, endBlock, sourceConfig: Config.CustomSources([source])}
+          {...chainConfig, sourceConfig: Config.CustomSources([source])}
         | None => chainConfig
         }
       | None => chainConfig

package/src/SimulateItems.res.mjs

@@ -167,7 +167,7 @@ function deriveSrcAddress(providedSrcAddress, eventConfig, chainConfig, config)
   }
 }
 
-function parse(simulateItems, config, chainConfig) {
+function parse(simulateItems, config, chainConfig, onEventRegistrations) {
   let chain = ChainMap.Chain.makeUnsafe(chainConfig.id);
   let chainId = chainConfig.id;
   let startBlock = chainConfig.startBlock;
@@ -254,9 +254,13 @@ function parse(simulateItems, config, chainConfig) {
       seenCoordinates[coordinate] = itemIndex;
     }
     let onEventRegistration = HandlerRegister.buildOnEventRegistration(config, chainId, eventConfig, undefined);
+    let onEventRegistrationIndex = onEventRegistrations.length;
+    let newrecord = {...onEventRegistration};
+    newrecord.index = onEventRegistrationIndex;
+    onEventRegistrations.push(newrecord);
     items.push({
       kind: 0,
-      onEventRegistration: onEventRegistration,
+      onEventRegistration: newrecord,
       chain: chain,
       blockNumber: blockNumber,
       logIndex: logIndex,
@@ -276,7 +280,7 @@ function parse(simulateItems, config, chainConfig) {
   return items;
 }
 
-function patchConfig(config, processConfig) {
+function patchConfig(config, processConfig, registrationsByChainId) {
   let processChains = processConfig.chains;
   if (processChains == null) {
     return config;
@@ -291,18 +295,33 @@ function patchConfig(config, processConfig) {
     if (simulateRaw == null) {
       return chainConfig;
     }
-    let items = parse(simulateRaw, config, chainConfig);
+    let registrations = registrationsByChainId[chainIdStr];
+    let chainRegistrations;
+    if (registrations !== undefined) {
+      chainRegistrations = registrations;
+    } else {
+      let registrations_onEventRegistrations = [];
+      let registrations_onBlockRegistrations = [];
+      let registrations$1 = {
+        onEventRegistrations: registrations_onEventRegistrations,
+        onBlockRegistrations: registrations_onBlockRegistrations
+      };
+      registrationsByChainId[chainIdStr] = registrations$1;
+      chainRegistrations = registrations$1;
+    }
     let startBlock = processChainJson.startBlock;
     let endBlock = processChainJson.endBlock;
-    let source = SimulateSource.make(items, endBlock, chain);
     let newrecord = {...chainConfig};
-    newrecord.sourceConfig = {
+    newrecord.endBlock = endBlock;
+    newrecord.startBlock = startBlock;
+    let items = parse(simulateRaw, config, newrecord, chainRegistrations.onEventRegistrations);
+    let source = SimulateSource.make(items, endBlock, chain);
+    let newrecord$1 = {...newrecord};
+    newrecord$1.sourceConfig = {
       TAG: "CustomSources",
       _0: [source]
     };
-    newrecord.endBlock = endBlock;
-    newrecord.startBlock = startBlock;
-    return newrecord;
+    return newrecord$1;
   });
   return {
     name: config.name,

package/src/TestIndexer.res

@@ -863,8 +863,8 @@ let initTestWorker = () => {
     | Some(_) => ()
     }
 
-    let patchConfig = (config: Config.t, _registrations) => {
-      let config = SimulateItems.patchConfig(~config, ~processConfig)
+    let patchConfig = (config: Config.t, registrationsByChainId) => {
+      let config = SimulateItems.patchConfig(~config, ~processConfig, ~registrationsByChainId)
 
       // In auto-exit mode, set batchSize=1 to process one block checkpoint at a time
       if exitAfterFirstEventBlock {

package/src/TestIndexer.res.mjs

@@ -612,8 +612,8 @@ function initTestWorker() {
   } else {
     Logging.setLogLevel("silent");
   }
-  let patchConfig = (config, _registrations) => {
-    let config$1 = SimulateItems.patchConfig(config, processConfig);
+  let patchConfig = (config, registrationsByChainId) => {
+    let config$1 = SimulateItems.patchConfig(config, processConfig, registrationsByChainId);
     if (exitAfterFirstEventBlock) {
       return {
         name: config$1.name,

package/src/TopicFilter.res

@@ -1,27 +1,3 @@
-let toTwosComplement = (num: bigint, ~bytesLen: int) => {
-  let maxValue = 1n->BigInt.shiftLeft(BigInt.fromInt(bytesLen * 8))
-  let mask = maxValue->BigInt.sub(1n)
-  num->BigInt.add(maxValue)->BigInt.bitwiseAnd(mask)
-}
-
-let fromSignedBigInt = val => {
-  let bytesLen = 32
-  let val = val >= 0n ? val : val->toTwosComplement(~bytesLen)
-  val->Viem.bigintToHex(~options={size: bytesLen})
-}
-
 type hex = EvmTypes.Hex.t
-//bytes currently does not work with genType and we also currently generate bytes as a string type
-type bytesHex = string
-let keccak256 = Viem.keccak256
-let bytesToHex = Viem.bytesToHex
-let concat = Viem.concat
-let castToHexUnsafe: 'a => hex = val => val->(Utils.magic: 'a => hex)
-let fromBigInt: bigint => hex = val => val->Viem.bigintToHex(~options={size: 32})
-let fromDynamicString: string => hex = val => val->(Utils.magic: string => hex)->keccak256
-let fromString: string => hex = val => val->Viem.stringToHex(~options={size: 32})
+
 let fromAddress: Address.t => hex = addr => addr->(Utils.magic: Address.t => hex)->Viem.pad
-let fromDynamicBytes: bytesHex => hex = bytes => bytes->(Utils.magic: bytesHex => hex)->keccak256
-let fromBytes: bytesHex => hex = bytes =>
-  bytes->(Utils.magic: bytesHex => Uint8Array.t)->Viem.bytesToHex(~options={size: 32})
-let fromBool: bool => hex = b => b->Viem.boolToHex(~options={size: 32})

package/src/TopicFilter.res.mjs

@@ -1,86 +1,12 @@
 // Generated by ReScript, PLEASE EDIT WITH CARE
 
 import * as Viem from "viem";
-import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js";
-
-function toTwosComplement(num, bytesLen) {
-  let maxValue = (1n << BigInt((bytesLen << 3)));
-  let mask = maxValue - 1n;
-  return num + maxValue & mask;
-}
-
-function fromSignedBigInt(val) {
-  let val$1 = val >= 0n ? val : toTwosComplement(val, 32);
-  return Viem.numberToHex(val$1, {
-    size: 32
-  });
-}
-
-function keccak256(prim) {
-  return Viem.keccak256(prim);
-}
-
-function bytesToHex(prim0, prim1) {
-  return Viem.bytesToHex(prim0, prim1 !== undefined ? Primitive_option.valFromOption(prim1) : undefined);
-}
-
-function concat(prim) {
-  return Viem.concat(prim);
-}
-
-function castToHexUnsafe(val) {
-  return val;
-}
-
-function fromBigInt(val) {
-  return Viem.numberToHex(val, {
-    size: 32
-  });
-}
-
-function fromDynamicString(val) {
-  return Viem.keccak256(val);
-}
-
-function fromString(val) {
-  return Viem.stringToHex(val, {
-    size: 32
-  });
-}
 
 function fromAddress(addr) {
   return Viem.pad(addr);
 }
 
-function fromDynamicBytes(bytes) {
-  return Viem.keccak256(bytes);
-}
-
-function fromBytes(bytes) {
-  return Viem.bytesToHex(bytes, {
-    size: 32
-  });
-}
-
-function fromBool(b) {
-  return Viem.boolToHex(b, {
-    size: 32
-  });
-}
-
 export {
-  toTwosComplement,
-  fromSignedBigInt,
-  keccak256,
-  bytesToHex,
-  concat,
-  castToHexUnsafe,
-  fromBigInt,
-  fromDynamicString,
-  fromString,
   fromAddress,
-  fromDynamicBytes,
-  fromBytes,
-  fromBool,
 }
 /* viem Not a pure module */

package/src/UserContext.res

@@ -28,10 +28,23 @@ Utils.Object.defineProperty(
   },
 )
 %%raw(`
-var EffectContext = function(params, defaultShouldCache, callEffect) {
+// Top-level so the getter is created once, not per context. \`this\` is the
+// EffectContext instance; only cross-chain contexts install it.
+function throwCrossChainChainAccess() {
+  throw new Error('context.chain is not available on the cross-chain effect "' + this._effectName + '". Set \`crossChain: false\` in its options to scope the effect to a single chain, then read context.chain.id.');
+}
+var crossChainChainDescriptor = { get: throwCrossChainChainAccess, enumerable: true };
+var EffectContext = function(params, chainId, effectName, defaultShouldCache, callEffect) {
   paramsByThis.set(this, params);
   this.effect = callEffect;
   this.cache = defaultShouldCache;
+  if (chainId === undefined) {
+    // Cross-chain: reading context.chain throws, via the shared getter.
+    Object.defineProperty(this, "_effectName", { value: effectName });
+    Object.defineProperty(this, "chain", crossChainChainDescriptor);
+  } else {
+    this.chain = { id: chainId };
+  }
 };
 EffectContext.prototype = effectContextPrototype;
 `)
@@ -39,35 +52,61 @@ EffectContext.prototype = effectContextPrototype;
 @new
 external makeEffectContext: (
   contextParams,
+  ~chainId: option<int>,
+  ~effectName: string,
   ~defaultShouldCache: bool,
   ~callEffect: (Internal.effect, Internal.effectInput) => promise<Internal.effectOutput>,
 ) => Internal.effectContext = "EffectContext"
 
 let initEffect = (params: contextParams) => {
-  let rec callEffect = (effect: Internal.effect, input: Internal.effectInput) => {
-    let effectContext = makeEffectContext(
-      params,
-      ~defaultShouldCache=effect.defaultShouldCache,
-      ~callEffect,
-    )
-    let effectArgs: Internal.effectArgs = {
-      input,
-      context: effectContext,
-      cacheKey: input->S.reverseConvertOrThrow(effect.input)->Utils.Hash.makeOrThrow,
-      checkpointId: params.checkpointId,
+  let handlerChainId = params.item->Internal.getItemChainId
+  // A chain-scoped effect always resolves against the chain of the handler that
+  // triggered the call, even several effects deep, so the chain id is captured
+  // once from the item and reused for the whole nested-call tree.
+  let rec makeCaller = (~caller: option<Internal.effect>) => {
+    (effect: Internal.effect, input: Internal.effectInput) => {
+      let scope = effect.crossChain ? Internal.CrossChain : Internal.Chain(handlerChainId)
+
+      switch caller {
+      | Some(callerEffect) if callerEffect.crossChain && !effect.crossChain =>
+        // A cross-chain effect isn't tied to a single chain, so it has no chain
+        // to resolve a chain-scoped child against. Reject before any cache work.
+        JsError.throwWithMessage(
+          `The cross-chain effect "${callerEffect.name}" cannot call the chain-scoped effect "${effect.name}", because a cross-chain effect isn't tied to a single chain. Make "${effect.name}" cross-chain (\`crossChain: true\`), or make "${callerEffect.name}" chain-scoped (\`crossChain: false\`).`,
+        )
+      | _ => ()
+      }
+
+      let effectContext = makeEffectContext(
+        params,
+        ~chainId=switch scope {
+        | Internal.Chain(chainId) => Some(chainId)
+        | Internal.CrossChain => None
+        },
+        ~effectName=effect.name,
+        ~defaultShouldCache=effect.defaultShouldCache,
+        ~callEffect=makeCaller(~caller=Some(effect)),
+      )
+      let effectArgs: Internal.effectArgs = {
+        input,
+        context: effectContext,
+        cacheKey: input->S.reverseConvertOrThrow(effect.input)->Utils.Hash.makeOrThrow,
+        checkpointId: params.checkpointId,
+      }
+      LoadLayer.loadEffect(
+        ~loadManager=params.loadManager,
+        ~persistence=params.persistence,
+        ~effect,
+        ~effectArgs,
+        ~scope,
+        ~indexerState=params.indexerState,
+        ~shouldGroup=params.isPreload,
+        ~item=params.item,
+        ~ecosystem=params.config.ecosystem,
+      )
     }
-    LoadLayer.loadEffect(
-      ~loadManager=params.loadManager,
-      ~persistence=params.persistence,
-      ~effect,
-      ~effectArgs,
-      ~indexerState=params.indexerState,
-      ~shouldGroup=params.isPreload,
-      ~item=params.item,
-      ~ecosystem=params.config.ecosystem,
-    )
   }
-  callEffect
+  makeCaller(~caller=None)
 }
 
 type entityContextParams = {

package/src/UserContext.res.mjs

@@ -25,17 +25,37 @@ Object.defineProperty(effectContextPrototype, "log", {
   }
 });
 
-var EffectContext = function(params, defaultShouldCache, callEffect) {
+// Top-level so the getter is created once, not per context. \`this\` is the
+// EffectContext instance; only cross-chain contexts install it.
+function throwCrossChainChainAccess() {
+  throw new Error('context.chain is not available on the cross-chain effect "' + this._effectName + '". Set \`crossChain: false\` in its options to scope the effect to a single chain, then read context.chain.id.');
+}
+var crossChainChainDescriptor = { get: throwCrossChainChainAccess, enumerable: true };
+var EffectContext = function(params, chainId, effectName, defaultShouldCache, callEffect) {
   paramsByThis.set(this, params);
   this.effect = callEffect;
   this.cache = defaultShouldCache;
+  if (chainId === undefined) {
+    // Cross-chain: reading context.chain throws, via the shared getter.
+    Object.defineProperty(this, "_effectName", { value: effectName });
+    Object.defineProperty(this, "chain", crossChainChainDescriptor);
+  } else {
+    this.chain = { id: chainId };
+  }
 };
 EffectContext.prototype = effectContextPrototype;
 ;
 
 function initEffect(params) {
-  let callEffect = (effect, input) => {
-    let effectContext = new EffectContext(params, effect.defaultShouldCache, callEffect);
+  let handlerChainId = Internal.getItemChainId(params.item);
+  let makeCaller = caller => ((effect, input) => {
+    let scope = effect.crossChain ? "crossChain" : handlerChainId;
+    if (caller !== undefined && caller.crossChain && !effect.crossChain) {
+      Stdlib_JsError.throwWithMessage(`The cross-chain effect "` + caller.name + `" cannot call the chain-scoped effect "` + effect.name + `", because a cross-chain effect isn't tied to a single chain. Make "` + effect.name + `" cross-chain (\`crossChain: true\`), or make "` + caller.name + `" chain-scoped (\`crossChain: false\`).`);
+    }
+    let tmp;
+    tmp = scope === "crossChain" ? undefined : scope;
+    let effectContext = new EffectContext(params, tmp, effect.name, effect.defaultShouldCache, makeCaller(effect));
     let effectArgs_cacheKey = Utils.Hash.makeOrThrow(S$RescriptSchema.reverseConvertOrThrow(input, effect.input));
     let effectArgs_checkpointId = params.checkpointId;
     let effectArgs = {
@@ -44,9 +64,9 @@ function initEffect(params) {
       cacheKey: effectArgs_cacheKey,
       checkpointId: effectArgs_checkpointId
     };
-    return LoadLayer.loadEffect(params.loadManager, params.persistence, effect, effectArgs, params.indexerState, params.isPreload, params.item, params.config.ecosystem);
-  };
-  return callEffect;
+    return LoadLayer.loadEffect(params.loadManager, params.persistence, effect, effectArgs, scope, params.indexerState, params.isPreload, params.item, params.config.ecosystem);
+  });
+  return makeCaller(undefined);
 }
 
 function getWhereHandler(params, filter) {

package/src/Writing.res

@@ -16,8 +16,8 @@ let getChangesCount = (state: IndexerState.t) => {
     total := total.contents +. (state->InMemoryStore.getInMemTable(~entityConfig)).changesCount
   })
   state
-  ->IndexerState.effects
-  ->Utils.Dict.forEach(inMemTable => {
+  ->IndexerState.effectState
+  ->EffectState.forEach(inMemTable => {
     total := total.contents +. inMemTable.changesCount
   })
   state
@@ -38,9 +38,9 @@ let waitForCommit = (state: IndexerState.t): promise<unit> =>
 let snapshotEffects = (state: IndexerState.t, ~cache): array<Persistence.updatedEffectCache> => {
   let acc = []
   state
-  ->IndexerState.effects
-  ->Utils.Dict.forEach(inMemTable => {
-    let {idsToStore, dict, effect, invalidationsCount} = inMemTable
+  ->IndexerState.effectState
+  ->EffectState.forEach(inMemTable => {
+    let {idsToStore, dict, effect, invalidationsCount, scope, table} = inMemTable
     switch idsToStore {
     | [] => ()
     | ids =>
@@ -51,17 +51,33 @@ let snapshotEffects = (state: IndexerState.t, ~cache): array<Persistence.updated
         }
       )
       let effectName = effect.name
-      let effectCacheRecord = switch cache->Utils.Dict.dangerouslyGetNonOption(effectName) {
+      let tableName = table.tableName
+      let effectCacheRecord = switch cache->Utils.Dict.dangerouslyGetNonOption(tableName) {
       | Some(c) => c
       | None =>
-        let c: Persistence.effectCacheRecord = {effectName, count: 0}
-        cache->Dict.set(effectName, c)
+        let c: Persistence.effectCacheRecord = {effectName, scope, tableName, count: 0}
+        cache->Dict.set(tableName, c)
         c
       }
       let shouldInitialize = effectCacheRecord.count === 0
       effectCacheRecord.count = effectCacheRecord.count + items->Array.length - invalidationsCount
-      Prometheus.EffectCacheCount.set(~count=effectCacheRecord.count, ~effectName)
-      acc->Array.push(({effect, items, shouldInitialize}: Persistence.updatedEffectCache))->ignore
+      Prometheus.EffectCacheCount.set(
+        ~count=effectCacheRecord.count,
+        ~effectName,
+        ~scope=scope->Internal.EffectCache.scopeToString,
+      )
+      acc
+      ->Array.push(
+        (
+          {
+            table,
+            itemSchema: effect.storageMeta.itemSchema,
+            items,
+            shouldInitialize,
+          }: Persistence.updatedEffectCache
+        ),
+      )
+      ->ignore
     }
     inMemTable.idsToStore = []
     inMemTable.invalidationsCount = 0
@@ -113,17 +129,36 @@ let runOneWrite = async (state: IndexerState.t) => {
     })
     let updatedEffectsCache = snapshotEffects(state, ~cache)
 
-    await persistence.storage.writeBatch(
-      ~batch,
-      ~rollback,
-      ~isInReorgThreshold=batch.isInReorgThreshold,
-      ~config,
-      ~allEntities=persistence.allEntities,
-      ~updatedEntities,
-      ~updatedEffectsCache,
-      ~chainMetaData,
+    let writtenEntityNames = Utils.Set.make()
+    updatedEntities->Array.forEach(({entityConfig}) =>
+      writtenEntityNames->Utils.Set.add(entityConfig.name)->ignore
+    )
+    let pruneTargets = PruneStaleHistory.select(
+      state,
+      ~writtenEntityNames,
+      ~isRollback=rollback->Option.isSome,
     )
 
+    // The prune runs concurrently with the batch write, but only for entities
+    // absent from it, so they never touch the same history table. Both must be
+    // awaited before the next write starts, otherwise a still-running prune
+    // could overlap the next batch's history writes and lose an anchor (which
+    // breaks a later rollback). Rollback writes touch every history table, so
+    // they get no concurrent prune at all.
+    let _ = await Promise.all2((
+      persistence.storage.writeBatch(
+        ~batch,
+        ~rollback,
+        ~isInReorgThreshold=batch.isInReorgThreshold,
+        ~config,
+        ~allEntities=persistence.allEntities,
+        ~updatedEntities,
+        ~updatedEffectsCache,
+        ~chainMetaData,
+      ),
+      PruneStaleHistory.runConcurrent(state, ~targets=pruneTargets),
+    ))
+
     state->IndexerState.markCommitted(~upToCheckpointId)
 
     switch rollback {
@@ -131,6 +166,10 @@ let runOneWrite = async (state: IndexerState.t) => {
       await RollbackCommit.fire(~progressBlockNumberByChainId)
     | _ => ()
     }
+
+    // Entities starved of the concurrent prune (eg written in every batch) are
+    // pruned here with no other pg writer running.
+    await PruneStaleHistory.runForced(state, ~targets=pruneTargets)
   }
 }
 
@@ -192,8 +231,8 @@ let dropCommitted = (state: IndexerState.t, ~keepLoadedFromDb) => {
     ->InMemoryTable.Entity.dropCommittedChanges(~committedCheckpointId, ~keepLoadedFromDb)
   )
   state
-  ->IndexerState.effects
-  ->Utils.Dict.forEach(inMemTable =>
+  ->IndexerState.effectState
+  ->EffectState.forEach(inMemTable =>
     inMemTable->InMemoryStore.dropCommittedEffects(~committedCheckpointId, ~keepLoadedFromDb)
   )
 }