envio 3.3.0-alpha.12 → 3.3.0-alpha.13

package/src/Prometheus.res

@@ -606,29 +606,39 @@ let effectLabelsSchema = S.object(s => {
   s.field("effect", S.string)
 })
 
+// For metrics whose backing state lives per scope ("crossChain" or a chain
+// id) — without the label, scopes of the same effect would clobber each
+// other's gauge value.
+let effectScopeLabelsSchema = S.schema(s =>
+  {
+    "effect": s.matches(S.string),
+    "scope": s.matches(S.string),
+  }
+)
+
 module EffectCalls = {
   let timeCounter = SafeCounter.makeOrThrow(
     ~name="envio_effect_call_seconds",
     ~help="Processing time taken to call the Effect function.",
-    ~labelSchema=effectLabelsSchema,
+    ~labelSchema=effectScopeLabelsSchema,
   )
 
   let sumTimeCounter = SafeCounter.makeOrThrow(
     ~name="envio_effect_call_seconds_total",
     ~help="Cumulative time spent calling the Effect function during the indexing process.",
-    ~labelSchema=effectLabelsSchema,
+    ~labelSchema=effectScopeLabelsSchema,
   )
 
   let totalCallsCount = SafeCounter.makeOrThrow(
     ~name="envio_effect_call_total",
     ~help="Cumulative number of resolved Effect function calls during the indexing process.",
-    ~labelSchema=effectLabelsSchema,
+    ~labelSchema=effectScopeLabelsSchema,
   )
 
   let activeCallsCount = SafeGauge.makeOrThrow(
     ~name="envio_effect_active_calls",
     ~help="The number of Effect function calls that are currently running.",
-    ~labelSchema=effectLabelsSchema,
+    ~labelSchema=effectScopeLabelsSchema,
   )
 }
 
@@ -636,11 +646,11 @@ module EffectCacheCount = {
   let gauge = SafeGauge.makeOrThrow(
     ~name="envio_effect_cache",
     ~help="The number of items in the effect cache.",
-    ~labelSchema=effectLabelsSchema,
+    ~labelSchema=effectScopeLabelsSchema,
   )
 
-  let set = (~count, ~effectName) => {
-    gauge->SafeGauge.handleInt(~labels=effectName, ~value=count)
+  let set = (~count, ~effectName, ~scope) => {
+    gauge->SafeGauge.handleInt(~labels={"effect": effectName, "scope": scope}, ~value=count)
   }
 }
 
@@ -660,7 +670,7 @@ module EffectQueueCount = {
   let gauge = SafeGauge.makeOrThrow(
     ~name="envio_effect_queue",
     ~help="The number of effect calls waiting in the rate limit queue.",
-    ~labelSchema=effectLabelsSchema,
+    ~labelSchema=effectScopeLabelsSchema,
   )
 
   let timeCounter = SafeCounter.makeOrThrow(
@@ -669,8 +679,8 @@ module EffectQueueCount = {
     ~labelSchema=effectLabelsSchema,
   )
 
-  let set = (~count, ~effectName) => {
-    gauge->SafeGauge.handleInt(~labels=effectName, ~value=count)
+  let set = (~count, ~effectName, ~scope) => {
+    gauge->SafeGauge.handleInt(~labels={"effect": effectName, "scope": scope}, ~value=count)
   }
 }
 

package/src/Prometheus.res.mjs

@@ -651,13 +651,18 @@ let ProgressLatency = {
 
 let effectLabelsSchema = S$RescriptSchema.object(s => s.f("effect", S$RescriptSchema.string));
 
-let timeCounter$5 = makeOrThrow("envio_effect_call_seconds", "Processing time taken to call the Effect function.", effectLabelsSchema);
+let effectScopeLabelsSchema = S$RescriptSchema.schema(s => ({
+  effect: s.m(S$RescriptSchema.string),
+  scope: s.m(S$RescriptSchema.string)
+}));
+
+let timeCounter$5 = makeOrThrow("envio_effect_call_seconds", "Processing time taken to call the Effect function.", effectScopeLabelsSchema);
 
-let sumTimeCounter$1 = makeOrThrow("envio_effect_call_seconds_total", "Cumulative time spent calling the Effect function during the indexing process.", effectLabelsSchema);
+let sumTimeCounter$1 = makeOrThrow("envio_effect_call_seconds_total", "Cumulative time spent calling the Effect function during the indexing process.", effectScopeLabelsSchema);
 
-let totalCallsCount = makeOrThrow("envio_effect_call_total", "Cumulative number of resolved Effect function calls during the indexing process.", effectLabelsSchema);
+let totalCallsCount = makeOrThrow("envio_effect_call_total", "Cumulative number of resolved Effect function calls during the indexing process.", effectScopeLabelsSchema);
 
-let activeCallsCount = makeOrThrow$1("envio_effect_active_calls", "The number of Effect function calls that are currently running.", effectLabelsSchema);
+let activeCallsCount = makeOrThrow$1("envio_effect_active_calls", "The number of Effect function calls that are currently running.", effectScopeLabelsSchema);
 
 let EffectCalls = {
   timeCounter: timeCounter$5,
@@ -666,10 +671,13 @@ let EffectCalls = {
   activeCallsCount: activeCallsCount
 };
 
-let gauge$19 = makeOrThrow$1("envio_effect_cache", "The number of items in the effect cache.", effectLabelsSchema);
+let gauge$19 = makeOrThrow$1("envio_effect_cache", "The number of items in the effect cache.", effectScopeLabelsSchema);
 
-function set$19(count, effectName) {
-  handleInt$1(gauge$19, effectName, count);
+function set$19(count, effectName, scope) {
+  handleInt$1(gauge$19, {
+    effect: effectName,
+    scope: scope
+  }, count);
 }
 
 let EffectCacheCount = {
@@ -688,12 +696,15 @@ let EffectCacheInvalidationsCount = {
   increment: increment$7
 };
 
-let gauge$20 = makeOrThrow$1("envio_effect_queue", "The number of effect calls waiting in the rate limit queue.", effectLabelsSchema);
+let gauge$20 = makeOrThrow$1("envio_effect_queue", "The number of effect calls waiting in the rate limit queue.", effectScopeLabelsSchema);
 
 let timeCounter$6 = makeOrThrow("envio_effect_queue_wait_seconds", "The time spent waiting in the rate limit queue.", effectLabelsSchema);
 
-function set$20(count, effectName) {
-  handleInt$1(gauge$20, effectName, count);
+function set$20(count, effectName, scope) {
+  handleInt$1(gauge$20, {
+    effect: effectName,
+    scope: scope
+  }, count);
 }
 
 let EffectQueueCount = {
@@ -825,6 +836,7 @@ export {
   ProgressEventsCount,
   ProgressLatency,
   effectLabelsSchema,
+  effectScopeLabelsSchema,
   EffectCalls,
   EffectCacheCount,
   EffectCacheInvalidationsCount,

package/src/UserContext.res

@@ -28,10 +28,23 @@ Utils.Object.defineProperty(
   },
 )
 %%raw(`
-var EffectContext = function(params, defaultShouldCache, callEffect) {
+// Top-level so the getter is created once, not per context. \`this\` is the
+// EffectContext instance; only cross-chain contexts install it.
+function throwCrossChainChainAccess() {
+  throw new Error('context.chain is not available on the cross-chain effect "' + this._effectName + '". Set \`crossChain: false\` in its options to scope the effect to a single chain, then read context.chain.id.');
+}
+var crossChainChainDescriptor = { get: throwCrossChainChainAccess, enumerable: true };
+var EffectContext = function(params, chainId, effectName, defaultShouldCache, callEffect) {
   paramsByThis.set(this, params);
   this.effect = callEffect;
   this.cache = defaultShouldCache;
+  if (chainId === undefined) {
+    // Cross-chain: reading context.chain throws, via the shared getter.
+    Object.defineProperty(this, "_effectName", { value: effectName });
+    Object.defineProperty(this, "chain", crossChainChainDescriptor);
+  } else {
+    this.chain = { id: chainId };
+  }
 };
 EffectContext.prototype = effectContextPrototype;
 `)
@@ -39,35 +52,61 @@ EffectContext.prototype = effectContextPrototype;
 @new
 external makeEffectContext: (
   contextParams,
+  ~chainId: option<int>,
+  ~effectName: string,
   ~defaultShouldCache: bool,
   ~callEffect: (Internal.effect, Internal.effectInput) => promise<Internal.effectOutput>,
 ) => Internal.effectContext = "EffectContext"
 
 let initEffect = (params: contextParams) => {
-  let rec callEffect = (effect: Internal.effect, input: Internal.effectInput) => {
-    let effectContext = makeEffectContext(
-      params,
-      ~defaultShouldCache=effect.defaultShouldCache,
-      ~callEffect,
-    )
-    let effectArgs: Internal.effectArgs = {
-      input,
-      context: effectContext,
-      cacheKey: input->S.reverseConvertOrThrow(effect.input)->Utils.Hash.makeOrThrow,
-      checkpointId: params.checkpointId,
+  let handlerChainId = params.item->Internal.getItemChainId
+  // A chain-scoped effect always resolves against the chain of the handler that
+  // triggered the call, even several effects deep, so the chain id is captured
+  // once from the item and reused for the whole nested-call tree.
+  let rec makeCaller = (~caller: option<Internal.effect>) => {
+    (effect: Internal.effect, input: Internal.effectInput) => {
+      let scope = effect.crossChain ? Internal.CrossChain : Internal.Chain(handlerChainId)
+
+      switch caller {
+      | Some(callerEffect) if callerEffect.crossChain && !effect.crossChain =>
+        // A cross-chain effect isn't tied to a single chain, so it has no chain
+        // to resolve a chain-scoped child against. Reject before any cache work.
+        JsError.throwWithMessage(
+          `The cross-chain effect "${callerEffect.name}" cannot call the chain-scoped effect "${effect.name}", because a cross-chain effect isn't tied to a single chain. Make "${effect.name}" cross-chain (\`crossChain: true\`), or make "${callerEffect.name}" chain-scoped (\`crossChain: false\`).`,
+        )
+      | _ => ()
+      }
+
+      let effectContext = makeEffectContext(
+        params,
+        ~chainId=switch scope {
+        | Internal.Chain(chainId) => Some(chainId)
+        | Internal.CrossChain => None
+        },
+        ~effectName=effect.name,
+        ~defaultShouldCache=effect.defaultShouldCache,
+        ~callEffect=makeCaller(~caller=Some(effect)),
+      )
+      let effectArgs: Internal.effectArgs = {
+        input,
+        context: effectContext,
+        cacheKey: input->S.reverseConvertOrThrow(effect.input)->Utils.Hash.makeOrThrow,
+        checkpointId: params.checkpointId,
+      }
+      LoadLayer.loadEffect(
+        ~loadManager=params.loadManager,
+        ~persistence=params.persistence,
+        ~effect,
+        ~effectArgs,
+        ~scope,
+        ~indexerState=params.indexerState,
+        ~shouldGroup=params.isPreload,
+        ~item=params.item,
+        ~ecosystem=params.config.ecosystem,
+      )
     }
-    LoadLayer.loadEffect(
-      ~loadManager=params.loadManager,
-      ~persistence=params.persistence,
-      ~effect,
-      ~effectArgs,
-      ~indexerState=params.indexerState,
-      ~shouldGroup=params.isPreload,
-      ~item=params.item,
-      ~ecosystem=params.config.ecosystem,
-    )
   }
-  callEffect
+  makeCaller(~caller=None)
 }
 
 type entityContextParams = {

package/src/UserContext.res.mjs

@@ -25,17 +25,37 @@ Object.defineProperty(effectContextPrototype, "log", {
   }
 });
 
-var EffectContext = function(params, defaultShouldCache, callEffect) {
+// Top-level so the getter is created once, not per context. \`this\` is the
+// EffectContext instance; only cross-chain contexts install it.
+function throwCrossChainChainAccess() {
+  throw new Error('context.chain is not available on the cross-chain effect "' + this._effectName + '". Set \`crossChain: false\` in its options to scope the effect to a single chain, then read context.chain.id.');
+}
+var crossChainChainDescriptor = { get: throwCrossChainChainAccess, enumerable: true };
+var EffectContext = function(params, chainId, effectName, defaultShouldCache, callEffect) {
   paramsByThis.set(this, params);
   this.effect = callEffect;
   this.cache = defaultShouldCache;
+  if (chainId === undefined) {
+    // Cross-chain: reading context.chain throws, via the shared getter.
+    Object.defineProperty(this, "_effectName", { value: effectName });
+    Object.defineProperty(this, "chain", crossChainChainDescriptor);
+  } else {
+    this.chain = { id: chainId };
+  }
 };
 EffectContext.prototype = effectContextPrototype;
 ;
 
 function initEffect(params) {
-  let callEffect = (effect, input) => {
-    let effectContext = new EffectContext(params, effect.defaultShouldCache, callEffect);
+  let handlerChainId = Internal.getItemChainId(params.item);
+  let makeCaller = caller => ((effect, input) => {
+    let scope = effect.crossChain ? "crossChain" : handlerChainId;
+    if (caller !== undefined && caller.crossChain && !effect.crossChain) {
+      Stdlib_JsError.throwWithMessage(`The cross-chain effect "` + caller.name + `" cannot call the chain-scoped effect "` + effect.name + `", because a cross-chain effect isn't tied to a single chain. Make "` + effect.name + `" cross-chain (\`crossChain: true\`), or make "` + caller.name + `" chain-scoped (\`crossChain: false\`).`);
+    }
+    let tmp;
+    tmp = scope === "crossChain" ? undefined : scope;
+    let effectContext = new EffectContext(params, tmp, effect.name, effect.defaultShouldCache, makeCaller(effect));
     let effectArgs_cacheKey = Utils.Hash.makeOrThrow(S$RescriptSchema.reverseConvertOrThrow(input, effect.input));
     let effectArgs_checkpointId = params.checkpointId;
     let effectArgs = {
@@ -44,9 +64,9 @@ function initEffect(params) {
       cacheKey: effectArgs_cacheKey,
       checkpointId: effectArgs_checkpointId
     };
-    return LoadLayer.loadEffect(params.loadManager, params.persistence, effect, effectArgs, params.indexerState, params.isPreload, params.item, params.config.ecosystem);
-  };
-  return callEffect;
+    return LoadLayer.loadEffect(params.loadManager, params.persistence, effect, effectArgs, scope, params.indexerState, params.isPreload, params.item, params.config.ecosystem);
+  });
+  return makeCaller(undefined);
 }
 
 function getWhereHandler(params, filter) {

package/src/Writing.res

@@ -16,8 +16,8 @@ let getChangesCount = (state: IndexerState.t) => {
     total := total.contents +. (state->InMemoryStore.getInMemTable(~entityConfig)).changesCount
   })
   state
-  ->IndexerState.effects
-  ->Utils.Dict.forEach(inMemTable => {
+  ->IndexerState.effectState
+  ->EffectState.forEach(inMemTable => {
     total := total.contents +. inMemTable.changesCount
   })
   state
@@ -38,9 +38,9 @@ let waitForCommit = (state: IndexerState.t): promise<unit> =>
 let snapshotEffects = (state: IndexerState.t, ~cache): array<Persistence.updatedEffectCache> => {
   let acc = []
   state
-  ->IndexerState.effects
-  ->Utils.Dict.forEach(inMemTable => {
-    let {idsToStore, dict, effect, invalidationsCount} = inMemTable
+  ->IndexerState.effectState
+  ->EffectState.forEach(inMemTable => {
+    let {idsToStore, dict, effect, invalidationsCount, scope, table} = inMemTable
     switch idsToStore {
     | [] => ()
     | ids =>
@@ -51,17 +51,33 @@ let snapshotEffects = (state: IndexerState.t, ~cache): array<Persistence.updated
         }
       )
       let effectName = effect.name
-      let effectCacheRecord = switch cache->Utils.Dict.dangerouslyGetNonOption(effectName) {
+      let tableName = table.tableName
+      let effectCacheRecord = switch cache->Utils.Dict.dangerouslyGetNonOption(tableName) {
       | Some(c) => c
       | None =>
-        let c: Persistence.effectCacheRecord = {effectName, count: 0}
-        cache->Dict.set(effectName, c)
+        let c: Persistence.effectCacheRecord = {effectName, scope, tableName, count: 0}
+        cache->Dict.set(tableName, c)
         c
       }
       let shouldInitialize = effectCacheRecord.count === 0
       effectCacheRecord.count = effectCacheRecord.count + items->Array.length - invalidationsCount
-      Prometheus.EffectCacheCount.set(~count=effectCacheRecord.count, ~effectName)
-      acc->Array.push(({effect, items, shouldInitialize}: Persistence.updatedEffectCache))->ignore
+      Prometheus.EffectCacheCount.set(
+        ~count=effectCacheRecord.count,
+        ~effectName,
+        ~scope=scope->Internal.EffectCache.scopeToString,
+      )
+      acc
+      ->Array.push(
+        (
+          {
+            table,
+            itemSchema: effect.storageMeta.itemSchema,
+            items,
+            shouldInitialize,
+          }: Persistence.updatedEffectCache
+        ),
+      )
+      ->ignore
     }
     inMemTable.idsToStore = []
     inMemTable.invalidationsCount = 0
@@ -192,8 +208,8 @@ let dropCommitted = (state: IndexerState.t, ~keepLoadedFromDb) => {
     ->InMemoryTable.Entity.dropCommittedChanges(~committedCheckpointId, ~keepLoadedFromDb)
   )
   state
-  ->IndexerState.effects
-  ->Utils.Dict.forEach(inMemTable =>
+  ->IndexerState.effectState
+  ->EffectState.forEach(inMemTable =>
     inMemTable->InMemoryStore.dropCommittedEffects(~committedCheckpointId, ~keepLoadedFromDb)
   )
 }

package/src/Writing.res.mjs

@@ -2,8 +2,10 @@
 
 import * as Env from "./Env.res.mjs";
 import * as Utils from "./Utils.res.mjs";
+import * as Internal from "./Internal.res.mjs";
 import * as Throttler from "./Throttler.res.mjs";
 import * as Prometheus from "./Prometheus.res.mjs";
+import * as EffectState from "./EffectState.res.mjs";
 import * as IndexerState from "./IndexerState.res.mjs";
 import * as Stdlib_Array from "@rescript/runtime/lib/es6/Stdlib_Array.js";
 import * as InMemoryStore from "./InMemoryStore.res.mjs";
@@ -20,7 +22,7 @@ function getChangesCount(state) {
   IndexerState.allEntities(state).forEach(entityConfig => {
     total.contents = total.contents + InMemoryStore.getInMemTable(state, entityConfig).changesCount;
   });
-  Utils.Dict.forEach(IndexerState.effects(state), inMemTable => {
+  EffectState.forEach(IndexerState.effectState(state), inMemTable => {
     total.contents = total.contents + inMemTable.changesCount;
   });
   IndexerState.processedBatches(state).forEach(batch => {
@@ -35,10 +37,12 @@ function waitForCommit(state) {
 
 function snapshotEffects(state, cache) {
   let acc = [];
-  Utils.Dict.forEach(IndexerState.effects(state), inMemTable => {
+  EffectState.forEach(IndexerState.effectState(state), inMemTable => {
     let idsToStore = inMemTable.idsToStore;
     let invalidationsCount = inMemTable.invalidationsCount;
     let dict = inMemTable.dict;
+    let table = inMemTable.table;
+    let scope = inMemTable.scope;
     let effect = inMemTable.effect;
     if (idsToStore.length !== 0) {
       let items = Stdlib_Array.filterMap(idsToStore, id => {
@@ -51,23 +55,27 @@ function snapshotEffects(state, cache) {
         }
       });
       let effectName = effect.name;
-      let c = cache[effectName];
+      let tableName = table.tableName;
+      let c = cache[tableName];
       let effectCacheRecord;
       if (c !== undefined) {
         effectCacheRecord = c;
       } else {
         let c$1 = {
           effectName: effectName,
+          scope: scope,
+          tableName: tableName,
           count: 0
         };
-        cache[effectName] = c$1;
+        cache[tableName] = c$1;
         effectCacheRecord = c$1;
       }
       let shouldInitialize = effectCacheRecord.count === 0;
       effectCacheRecord.count = (effectCacheRecord.count + items.length | 0) - invalidationsCount | 0;
-      Prometheus.EffectCacheCount.set(effectCacheRecord.count, effectName);
+      Prometheus.EffectCacheCount.set(effectCacheRecord.count, effectName, Internal.EffectCache.scopeToString(scope));
       acc.push({
-        effect: effect,
+        table: table,
+        itemSchema: effect.storageMeta.itemSchema,
         items: items,
         shouldInitialize: shouldInitialize
       });
@@ -164,7 +172,7 @@ function commitBatch(state, batch) {
 function dropCommitted(state, keepLoadedFromDb) {
   let committedCheckpointId = IndexerState.committedCheckpointId(state);
   IndexerState.allEntities(state).forEach(entityConfig => InMemoryTable.Entity.dropCommittedChanges(InMemoryStore.getInMemTable(state, entityConfig), committedCheckpointId, keepLoadedFromDb));
-  Utils.Dict.forEach(IndexerState.effects(state), inMemTable => InMemoryStore.dropCommittedEffects(inMemTable, committedCheckpointId, keepLoadedFromDb));
+  EffectState.forEach(IndexerState.effectState(state), inMemTable => InMemoryStore.dropCommittedEffects(inMemTable, committedCheckpointId, keepLoadedFromDb));
 }
 
 async function awaitCapacity(state) {

package/src/bindings/NodeJs.res

@@ -173,5 +173,10 @@ module Fs = {
 
     @module("fs") @scope("promises")
     external readdir: Path.t => promise<array<string>> = "readdir"
+
+    type stats
+    @module("fs") @scope("promises")
+    external stat: Path.t => promise<stats> = "stat"
+    @send external statsIsDirectory: stats => bool = "isDirectory"
   }
 }