envio 3.15.0 → 3.16.0-subgraph

package/src/EffectCacheLookup.res

@@ -0,0 +1,98 @@
+// A cached effect's stored outputs, read straight from storage. For the
+// subgraph register pass: it runs at fetch time, outside any handler context,
+// so its contract calls can't go through the effect machinery, and after
+// `envio start -r` (which keeps the subgraph call caches) it would otherwise ask
+// the RPC again for every call it makes.
+//
+// Only a chain-scoped effect's table that held rows when the run started is
+// read: rows this run writes are already in the register pass's own cache.
+// The calls of one turn share one storage query.
+
+type batch = {
+  effect: Internal.effect,
+  table: Table.table,
+  keys: array<string>,
+  resolvers: array<option<Internal.effectOutput> => unit>,
+}
+
+let startupTables: ref<option<Utils.Set.t<string>>> = ref(None)
+
+// Reset between test indexers, which each bring their own storage.
+let resetStartupTables = () => startupTables := None
+
+let tablesWithRowsAtStart = (cache: dict<Persistence.effectCacheRecord>) =>
+  switch startupTables.contents {
+  | Some(tables) => tables
+  | None =>
+    let tables = Utils.Set.make()
+    cache
+    ->Dict.valuesToArray
+    ->Array.forEach(record =>
+      if record.count > 0 {
+        tables->Utils.Set.add(record.tableName)->ignore
+      }
+    )
+    startupTables := Some(tables)
+    tables
+  }
+
+let pending: dict<batch> = Dict.make()
+
+let load = async (storage: Persistence.storage, batch: batch) => {
+  let rows =
+    (await storage.loadOrThrow(~table=batch.table, ~filter=EntityFilter.byIds(batch.keys)))->(
+      Utils.magic: array<unknown> => array<Internal.effectCacheItem>
+    )
+  let byId = Dict.make()
+  rows->Array.forEach(row =>
+    switch row.output->S.parseOrThrow(batch.effect.storageMeta.outputSchema) {
+    | output => byId->Dict.set(row.id, output)
+    | exception _ => ()
+    }
+  )
+  batch.keys->Array.map(key => byId->Utils.Dict.dangerouslyGetNonOption(key))
+}
+
+let flush = async (storage: Persistence.storage, tableName) => {
+  let batch = pending->Dict.getUnsafe(tableName)
+  pending->Utils.Dict.deleteInPlace(tableName)
+  let outputs = switch await load(storage, batch) {
+  | outputs => outputs
+  | exception _ => batch.keys->Array.map(_ => None)
+  }
+  batch.resolvers->Array.forEachWithIndex((resolve, idx) => resolve(outputs->Array.getUnsafe(idx)))
+}
+
+let cached = (effect: Internal.effect, chainId: int, input: Internal.effectInput): promise<
+  option<Internal.effectOutput>,
+> =>
+  switch (
+    effect.crossChain,
+    EnvioGlobal.value.persistence->(Utils.magic: option<unknown> => option<Persistence.t>),
+  ) {
+  | (Some(false), Some({storageStatus: Ready({cache}), storage})) =>
+    let table = Internal.makeCacheTable(
+      ~effectName=effect.name,
+      ~scope=Chain(chainId->ChainId.fromInt),
+    )
+    if cache->tablesWithRowsAtStart->Utils.Set.has(table.tableName) {
+      let key =
+        input
+        ->S.reverseConvertOrThrow(effect.input)
+        ->Utils.Hash.makeOrThrow
+        ->Utils.Hash.boundCacheKey
+      Promise.make((resolve, _) => {
+        switch pending->Utils.Dict.dangerouslyGetNonOption(table.tableName) {
+        | Some(batch) =>
+          batch.keys->Array.push(key)
+          batch.resolvers->Array.push(resolve)
+        | None =>
+          pending->Dict.set(table.tableName, {effect, table, keys: [key], resolvers: [resolve]})
+          let _ = setTimeout(() => storage->flush(table.tableName)->ignore, 0)
+        }
+      })
+    } else {
+      Promise.resolve(None)
+    }
+  | _ => Promise.resolve(None)
+  }

package/src/EffectCacheLookup.res.mjs

@@ -0,0 +1,116 @@
+// Generated by ReScript, PLEASE EDIT WITH CARE
+
+import * as Utils from "./Utils.res.mjs";
+import * as Internal from "./Internal.res.mjs";
+import * as EnvioGlobal from "./EnvioGlobal.res.mjs";
+import * as EntityFilter from "./db/EntityFilter.res.mjs";
+import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js";
+import * as S$RescriptSchema from "rescript-schema/src/S.res.mjs";
+
+let startupTables = {
+  contents: undefined
+};
+
+function resetStartupTables() {
+  startupTables.contents = undefined;
+}
+
+function tablesWithRowsAtStart(cache) {
+  let tables = startupTables.contents;
+  if (tables !== undefined) {
+    return Primitive_option.valFromOption(tables);
+  }
+  let tables$1 = new Set();
+  Object.values(cache).forEach(record => {
+    if (record.count > 0) {
+      tables$1.add(record.tableName);
+      return;
+    }
+  });
+  startupTables.contents = Primitive_option.some(tables$1);
+  return tables$1;
+}
+
+let pending = {};
+
+async function load(storage, batch) {
+  let rows = await storage.loadOrThrow(EntityFilter.byIds(batch.keys), batch.table);
+  let byId = {};
+  rows.forEach(row => {
+    let output;
+    try {
+      output = S$RescriptSchema.parseOrThrow(row.output, batch.effect.storageMeta.outputSchema);
+    } catch (exn) {
+      return;
+    }
+    byId[row.id] = output;
+  });
+  return batch.keys.map(key => byId[key]);
+}
+
+async function flush(storage, tableName) {
+  let batch = pending[tableName];
+  Utils.Dict.deleteInPlace(pending, tableName);
+  let outputs;
+  try {
+    outputs = await load(storage, batch);
+  } catch (exn) {
+    outputs = batch.keys.map(param => {});
+  }
+  batch.resolvers.forEach((resolve, idx) => resolve(outputs[idx]));
+}
+
+function cached(effect, chainId, input) {
+  let match = effect.crossChain;
+  let match$1 = EnvioGlobal.value.persistence;
+  if (match === undefined) {
+    return Promise.resolve(undefined);
+  }
+  if (match) {
+    return Promise.resolve(undefined);
+  }
+  if (match$1 === undefined) {
+    return Promise.resolve(undefined);
+  }
+  let match$2 = match$1.storageStatus;
+  if (typeof match$2 !== "object") {
+    return Promise.resolve(undefined);
+  }
+  if (match$2.TAG === "Initializing") {
+    return Promise.resolve(undefined);
+  }
+  let storage = match$1.storage;
+  let table = Internal.makeCacheTable(effect.name, chainId);
+  if (!tablesWithRowsAtStart(match$2._0.cache).has(table.tableName)) {
+    return Promise.resolve(undefined);
+  }
+  let key = Utils.Hash.boundCacheKey(Utils.Hash.makeOrThrow(S$RescriptSchema.reverseConvertOrThrow(input, effect.input)));
+  return new Promise((resolve, param) => {
+    let batch = pending[table.tableName];
+    if (batch !== undefined) {
+      batch.keys.push(key);
+      batch.resolvers.push(resolve);
+    } else {
+      pending[table.tableName] = {
+        effect: effect,
+        table: table,
+        keys: [key],
+        resolvers: [resolve]
+      };
+      setTimeout(() => {
+        flush(storage, table.tableName);
+      }, 0);
+    }
+  });
+}
+
+export {
+  startupTables,
+  resetStartupTables,
+  tablesWithRowsAtStart,
+  pending,
+  load,
+  flush,
+  cached,
+}
+/* Utils Not a pure module */

package/src/EffectState.res

@@ -11,6 +11,8 @@ type effectRateLimitState = {
   mutable availableCalls: int,
   mutable windowStartTime: float,
   mutable nextWindowPromise: option<promise<unit>>,
+  // The run's clock: a test on virtual time opens the next window by advancing.
+  clock: Clock.t,
 }
 
 // Per-(effect, scope) counters rendered into the envio_effect_* metrics. Live
@@ -80,9 +82,14 @@ type t = {
   // rollback; stats and the rate-limit window survive because the table does.
   tables: dict<effectCacheInMemTable>,
   unregisteredCacheCounts: dict<unregisteredCacheCount>,
+  clock: Clock.t,
 }
 
-let make = (): t => {tables: Dict.make(), unregisteredCacheCounts: Dict.make()}
+let make = (~clock=Clock.real, ()): t => {
+  tables: Dict.make(),
+  unregisteredCacheCounts: Dict.make(),
+  clock,
+}
 
 let setUnregisteredCacheCount = (self: t, ~effectName, ~scope, ~count) => {
   let tableName = Internal.EffectCache.toTableName(~effectName, ~scope)
@@ -214,8 +221,9 @@ let getTable = (self: t, ~effect: Internal.effect, ~scope: Internal.chainScope)
         callsPerDuration,
         durationMs,
         availableCalls: callsPerDuration,
-        windowStartTime: Date.now(),
+        windowStartTime: self.clock.now(),
         nextWindowPromise: None,
+        clock: self.clock,
       })
     }
     let inMemTable: effectCacheInMemTable = {

package/src/EffectState.res.mjs

@@ -1,13 +1,16 @@
 // Generated by ReScript, PLEASE EDIT WITH CARE
 
+import * as Clock from "./Clock.res.mjs";
 import * as Utils from "./Utils.res.mjs";
 import * as Internal from "./Internal.res.mjs";
 import * as Performance from "./bindings/Performance.res.mjs";
 
-function make() {
+function make(clockOpt, param) {
+  let clock = clockOpt !== undefined ? clockOpt : Clock.real;
   return {
     tables: {},
-    unregisteredCacheCounts: {}
+    unregisteredCacheCounts: {},
+    clock: clock
   };
 }
 
@@ -130,8 +133,9 @@ function getTable(self, effect, scope) {
       callsPerDuration: callsPerDuration,
       durationMs: match$1.durationMs,
       availableCalls: callsPerDuration,
-      windowStartTime: Date.now(),
-      nextWindowPromise: undefined
+      windowStartTime: self.clock.now(),
+      nextWindowPromise: undefined,
+      clock: self.clock
     };
   } else {
     rateLimitState = undefined;

package/src/EffectState.resi

@@ -4,6 +4,8 @@ type effectRateLimitState = {
   mutable availableCalls: int,
   mutable windowStartTime: float,
   mutable nextWindowPromise: option<promise<unit>>,
+  // The run's clock: a test on virtual time opens the next window by advancing.
+  clock: Clock.t,
 }
 
 // Per-(effect, scope) metric counters. Survive rollback so they stay monotonic.
@@ -32,7 +34,7 @@ type effectCacheInMemTable = {
 // Opaque store of all per-(effect, scope) runtime state.
 type t
 
-let make: unit => t
+let make: (~clock: Clock.t=?, unit) => t
 // Get, or lazily create, the in-mem table for an (effect, scope). Reuses the
 // surviving rate-limit window when the table is recreated after a rollback.
 let getTable: (t, ~effect: Internal.effect, ~scope: Internal.chainScope) => effectCacheInMemTable

package/src/EntityId.res

@@ -1,15 +1,10 @@
 // Opaque representation of an entity id. The generated per-entity types expose
-// the real scalar (string / int / bigint); the generic storage and in-memory
+// the real scalar (string / int / bigint / Uint8Array); the generic storage and in-memory
 // layers work with any entity's id through this type without knowing which
 // scalar backs it. The runtime value is always the real id, never a stringified
-// form — `toKey` derives the string only where a JS object/dict key is needed.
+// form — a table's `entityIdToKey` derives the string only where a dict or Set key is needed.
 type t
 
 external unsafeOfAny: 'a => t = "%identity"
 external unsafeToAny: t => 'a = "%identity"
 external unsafeOfString: string => t = "%identity"
-
-// Stringified id used as a JS object/dict key. `String` matches how JS coerces
-// a value used as an object key, so an id indexes the same whether the raw
-// value or its key form is used for lookup, across string/int/bigint.
-let toKey: t => string = %raw(`String`)

package/src/EntityId.res.mjs

@@ -1,9 +1,2 @@
 // Generated by ReScript, PLEASE EDIT WITH CARE
-
-
-let toKey = String;
-
-export {
-  toKey,
-}
-/* toKey Not a pure module */
+/* This output is empty. Its source's type definitions, externals and/or unused code got optimized away. */

package/src/EntityTables.res

@@ -6,7 +6,10 @@ type t = dict<InMemoryTable.Entity.t>
 let make = (entities: array<Internal.entityConfig>): t => {
   let init = Dict.make()
   entities->Array.forEach(entityConfig => {
-    init->Dict.set((entityConfig.name :> string), InMemoryTable.Entity.make())
+    init->Dict.set(
+      (entityConfig.name :> string),
+      InMemoryTable.Entity.make(~entityIdToKey=entityConfig.table.entityIdToKey),
+    )
   })
   init
 }

package/src/EntityTables.res.mjs

@@ -6,7 +6,7 @@ import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
 function make(entities) {
   let init = {};
   entities.forEach(entityConfig => {
-    init[entityConfig.name] = InMemoryTable.Entity.make();
+    init[entityConfig.name] = InMemoryTable.Entity.make(entityConfig.table.entityIdToKey);
   });
   return init;
 }

package/src/Env.res

@@ -4,6 +4,26 @@
 %%private(let envSafe = EnvSafe.make())
 
 let targetBufferSize = envSafe->EnvSafe.get("ENVIO_INDEXING_MAX_BUFFER_SIZE", S.option(S.int))
+
+// Set once a subgraph-mode config is loaded. A subgraph's mappings run per
+// event, and every fetched event that can create a data source also holds a
+// live register pass and its contract calls until the response is merged, so
+// the pool of events ready to process is sized for that (CrossChainState).
+let subgraphMode = ref(false)
+
+// Set with subgraphMode when the manifest declares templates: without one no
+// event can create a data source, so none holds a register pass.
+let subgraphHasTemplates = ref(false)
+
+// Most events with a register pass one subgraph response may hold before it is
+// cut after a block and the rest fetched again. 0 turns the cut off; unset, it
+// follows what the register passes cost (ChainFetching).
+let subgraphRegisterChunk = envSafe->EnvSafe.get("ENVIO_SUBGRAPH_REGISTER_CHUNK", S.option(S.int))
+
+// How many items ahead of the one executing a subgraph handler is preloaded
+// again once an entity it read as absent is created (PreloadLookahead). 0 = off.
+let subgraphPreloadLookahead =
+  envSafe->EnvSafe.get("ENVIO_SUBGRAPH_PRELOAD_LOOKAHEAD", S.int, ~fallback=256)
 let maxAddrInPartition = envSafe->EnvSafe.get("MAX_PARTITION_SIZE", S.int, ~fallback=5_000)
 
 // Most parallel in-flight queries a single chain may have at once, across all

package/src/Env.res.mjs

@@ -13,6 +13,18 @@ let envSafe = EnvSafe.make(undefined);
 
 let targetBufferSize = EnvSafe.get(envSafe, "ENVIO_INDEXING_MAX_BUFFER_SIZE", S$RescriptSchema.option(S$RescriptSchema.int), undefined, undefined, undefined, undefined);
 
+let subgraphMode = {
+  contents: false
+};
+
+let subgraphHasTemplates = {
+  contents: false
+};
+
+let subgraphRegisterChunk = EnvSafe.get(envSafe, "ENVIO_SUBGRAPH_REGISTER_CHUNK", S$RescriptSchema.option(S$RescriptSchema.int), undefined, undefined, undefined, undefined);
+
+let subgraphPreloadLookahead = EnvSafe.get(envSafe, "ENVIO_SUBGRAPH_PRELOAD_LOOKAHEAD", S$RescriptSchema.int, undefined, 256, undefined, undefined);
+
 let maxAddrInPartition = EnvSafe.get(envSafe, "MAX_PARTITION_SIZE", S$RescriptSchema.int, undefined, 5000, undefined, undefined);
 
 let clientFilterAddressThreshold = EnvSafe.get(envSafe, "ENVIO_CLIENT_FILTER_ADDRESS_THRESHOLD", S$RescriptSchema.int, undefined, (maxAddrInPartition * 100 | 0) / 5 | 0, undefined, undefined);
@@ -216,6 +228,10 @@ let maxChainConcurrency = 100;
 
 export {
   targetBufferSize,
+  subgraphMode,
+  subgraphHasTemplates,
+  subgraphRegisterChunk,
+  subgraphPreloadLookahead,
   maxAddrInPartition,
   maxChainConcurrency,
   clientFilterAddressThreshold,

package/src/EventProcessing.res

@@ -125,17 +125,52 @@ let preloadBatchOrThrow = async (batch: Batch.t, ~runtime) => {
 
 let runBatchHandlersOrThrow = async (batch: Batch.t, ~runtime) => {
   let itemIdx = ref(0)
+  let lookahead = PreloadLookahead.active.contents
+  // Settled before the batch ends: one finishing after a rollback would put a
+  // value read before it back into memory.
+  let lookaheadPreloads = []
+  // Only read by the lookahead, which preloads items out of checkpoint order.
+  let checkpointIdOf = switch lookahead {
+  | Some(_) =>
+    let ids = []
+    batch.checkpointIds->Array.forEachWithIndex((checkpointId, checkpointIdx) => {
+      for _ in 1 to batch.checkpointItemsCount->Array.getUnsafe(checkpointIdx) {
+        ids->Array.push(checkpointId)
+      }
+    })
+    ids
+  | None => []
+  }
 
   for checkpointIdx in 0 to batch.checkpointIds->Array.length - 1 {
     let checkpointId = batch.checkpointIds->Array.getUnsafe(checkpointIdx)
     let checkpointItemsCount = batch.checkpointItemsCount->Array.getUnsafe(checkpointIdx)
 
     for idx in 0 to checkpointItemsCount - 1 {
-      let item = batch.items->Array.getUnsafe(itemIdx.contents + idx)
+      let index = itemIdx.contents + idx
+      let item = batch.items->Array.getUnsafe(index)
       await runHandlerOrThrow(item, ~checkpointId, ~runtime)
+
+      switch lookahead {
+      | Some(lookahead) =>
+        lookahead->PreloadLookahead.afterItem(~index, ~preload=(ahead, aheadIdx) =>
+          if ahead->hasHandler {
+            lookaheadPreloads
+            ->Array.push(
+              runtime->preload(
+                ~item=ahead,
+                ~checkpointId=checkpointIdOf->Array.getUnsafe(aheadIdx),
+              ),
+            )
+            ->ignore
+          }
+        )
+      | None => ()
+      }
     }
     itemIdx := itemIdx.contents + checkpointItemsCount
   }
+  let _ = await Promise.all(lookaheadPreloads)
 }
 
 let registerProcessEventBatchMetrics = (
@@ -219,6 +254,7 @@ let processEventBatch = async (~batch: Batch.t, ~runtime: HandlerRuntime.t) => {
       // Materialise store-backed transactions onto payloads before any handler
       // (preload or execute) reads them.
       await materializeBatchEvents(batch, ~chainStates)
+      PreloadLookahead.start(~items=batch.items)
       await batch->preloadBatchOrThrow(~runtime)
     }
 
@@ -227,6 +263,7 @@ let processEventBatch = async (~batch: Batch.t, ~runtime: HandlerRuntime.t) => {
     if batch.items->Utils.Array.notEmpty {
       await batch->runBatchHandlersOrThrow(~runtime)
     }
+    PreloadLookahead.stop()
 
     let elapsedTimeAfterProcessing = timeRef->Performance.secondsSince
 

package/src/EventProcessing.res.mjs

@@ -14,6 +14,7 @@ import * as Stdlib_Dict from "@rescript/runtime/lib/es6/Stdlib_Dict.js";
 import * as IndexerState from "./IndexerState.res.mjs";
 import * as ErrorHandling from "./ErrorHandling.res.mjs";
 import * as HandlerRuntime from "./HandlerRuntime.res.mjs";
+import * as PreloadLookahead from "./PreloadLookahead.res.mjs";
 import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js";
 import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
 
@@ -138,15 +139,39 @@ async function preloadBatchOrThrow(batch, runtime) {
 
 async function runBatchHandlersOrThrow(batch, runtime) {
   let itemIdx = 0;
+  let lookahead = PreloadLookahead.active.contents;
+  let lookaheadPreloads = [];
+  let checkpointIdOf;
+  if (lookahead !== undefined) {
+    let ids = [];
+    batch.checkpointIds.forEach((checkpointId, checkpointIdx) => {
+      for (let _for = 1, _for_finish = batch.checkpointItemsCount[checkpointIdx]; _for <= _for_finish; ++_for) {
+        ids.push(checkpointId);
+      }
+    });
+    checkpointIdOf = ids;
+  } else {
+    checkpointIdOf = [];
+  }
   for (let checkpointIdx = 0, checkpointIdx_finish = batch.checkpointIds.length; checkpointIdx < checkpointIdx_finish; ++checkpointIdx) {
     let checkpointId = batch.checkpointIds[checkpointIdx];
     let checkpointItemsCount = batch.checkpointItemsCount[checkpointIdx];
     for (let idx = 0; idx < checkpointItemsCount; ++idx) {
-      let item = batch.items[itemIdx + idx | 0];
+      let index = itemIdx + idx | 0;
+      let item = batch.items[index];
       await runHandlerOrThrow(item, checkpointId, runtime);
+      if (lookahead !== undefined) {
+        PreloadLookahead.afterItem(lookahead, index, (ahead, aheadIdx) => {
+          if (hasHandler(ahead)) {
+            lookaheadPreloads.push(preload(runtime, ahead, checkpointIdOf[aheadIdx]));
+            return;
+          }
+        });
+      }
     }
     itemIdx = itemIdx + checkpointItemsCount | 0;
   }
+  await Promise.all(lookaheadPreloads);
 }
 
 function registerProcessEventBatchMetrics(logger, batch, indexerState, loadDuration, handlerDuration) {
@@ -195,12 +220,14 @@ async function processEventBatch(batch, runtime) {
     let timeRef = Performance.now();
     if (Utils.$$Array.notEmpty(batch.items)) {
       await materializeBatchEvents(batch, chainStates);
+      PreloadLookahead.start(batch.items);
       await preloadBatchOrThrow(batch, runtime);
     }
     let elapsedTimeAfterLoaders = Performance.secondsSince(timeRef);
     if (Utils.$$Array.notEmpty(batch.items)) {
       await runBatchHandlersOrThrow(batch, runtime);
     }
+    PreloadLookahead.stop();
     let elapsedTimeAfterProcessing = Performance.secondsSince(timeRef);
     Writing.commitBatch(indexerState, batch);
     let handlerDuration = elapsedTimeAfterProcessing - elapsedTimeAfterLoaders;