envio 3.5.1 → 3.6.1-subgraph

package/src/Internal.res.mjs

@@ -166,21 +166,36 @@ let fuelTransferParamsSchema = S$RescriptSchema.schema(s => ({
   amount: s.m(Utils.$$BigInt.schema)
 }));
 
-let cacheTablePrefix = "envio_effect_";
+function chainScopeToString(scope) {
+  if (scope === "crossChain") {
+    return "crossChain";
+  } else {
+    return ChainId.toString(scope);
+  }
+}
 
-function toTableName(effectName, scope) {
+function chainScopeChainId(scope) {
   if (scope === "crossChain") {
-    return cacheTablePrefix + effectName;
+    return;
   } else {
-    return `envio_` + ChainId.toString(scope) + `_effect_` + effectName;
+    return Primitive_option.some(scope);
   }
 }
 
-function scopeToString(scope) {
+function stampChainId(entity, fieldName, chainId) {
+  return Utils.Dict.merge(entity, Object.fromEntries([[
+      fieldName,
+      chainId
+    ]]));
+}
+
+let cacheTablePrefix = "envio_effect_";
+
+function toTableName(effectName, scope) {
   if (scope === "crossChain") {
-    return "crossChain";
+    return cacheTablePrefix + effectName;
   } else {
-    return ChainId.toString(scope);
+    return `envio_` + ChainId.toString(scope) + `_effect_` + effectName;
   }
 }
 
@@ -260,7 +275,6 @@ function toCachePath(effectName, scope) {
 
 let EffectCache = {
   toTableName: toTableName,
-  scopeToString: scopeToString,
   parseChainId: parseChainId,
   chainScopedRe: chainScopedRe,
   crossChainRe: crossChainRe,
@@ -272,8 +286,8 @@ let cacheOutputSchema = S$RescriptSchema.json(false);
 
 function makeCacheTable(effectName, scope) {
   return Table.mkTable(toTableName(effectName, scope), undefined, [
-    Table.mkField("id", "String", S$RescriptSchema.string, undefined, undefined, undefined, true, undefined, undefined, undefined, undefined, undefined),
-    Table.mkField("output", "Json", cacheOutputSchema, undefined, undefined, true, undefined, undefined, undefined, undefined, undefined, undefined)
+    Table.mkField("id", "String", S$RescriptSchema.string, undefined, undefined, undefined, true, undefined, undefined, undefined, undefined, undefined, undefined),
+    Table.mkField("output", "Json", cacheOutputSchema, undefined, undefined, true, undefined, undefined, undefined, undefined, undefined, undefined, undefined)
   ], undefined);
 }
 
@@ -296,6 +310,9 @@ export {
   getItemChainId,
   fuelSupplyParamsSchema,
   fuelTransferParamsSchema,
+  chainScopeToString,
+  chainScopeChainId,
+  stampChainId,
   cacheTablePrefix,
   EffectCache,
   cacheOutputSchema,

package/src/LoadLayer.res

@@ -1,15 +1,41 @@
+// A per-chain entity's rows must never dedup or merge across chains, so both
+// the load-group key and the storage query carry the scope.
+let scopeKeySuffix = (scope: Internal.chainScope) =>
+  switch scope {
+  | CrossChain => ""
+  | Chain(chainId) => `.${chainId->ChainId.toString}`
+  }
+
+// Narrows a query to the scope's chain. Cross-chain entities have no chain-id
+// column, so their filter is left untouched.
+let scopeFilter = (filter: EntityFilter.t, ~table: Table.table, ~scope: Internal.chainScope) =>
+  switch (scope, table->Table.getChainIdField) {
+  | (Chain(chainId), Some(field)) =>
+    EntityFilter.And({
+      filters: [
+        filter,
+        Eq({
+          fieldName: field.fieldName,
+          fieldValue: chainId->(Utils.magic: ChainId.t => unknown),
+        }),
+      ],
+    })
+  | _ => filter
+  }
+
 let loadById = (
   ~loadManager,
   ~persistence: Persistence.t,
   ~entityConfig: Internal.entityConfig,
+  ~scope: Internal.chainScope,
   ~indexerState,
   ~shouldGroup,
   ~item,
   ~ecosystem,
   ~entityId,
 ) => {
-  let key = `${entityConfig.name}.get`
-  let inMemTable = indexerState->InMemoryStore.getInMemTable(~entityConfig)
+  let key = `${entityConfig.name}.get${scope->scopeKeySuffix}`
+  let inMemTable = indexerState->InMemoryStore.getInMemTable(~entityConfig, ~scope)
 
   let load = async (idsToLoad, ~onError as _) => {
     let storage = persistence->Persistence.getInitializedStorageOrThrow
@@ -25,7 +51,7 @@ let loadById = (
           ~filter=EntityFilter.In({
             fieldName: Table.idFieldName,
             fieldValue: idsToLoad->(Utils.magic: array<string> => array<unknown>),
-          }),
+          })->scopeFilter(~table=entityConfig.table, ~scope),
         )
       )->(Utils.magic: array<unknown> => array<Internal.entity>)
     } catch {
@@ -327,14 +353,16 @@ let loadByFilter = (
   ~loadManager,
   ~persistence: Persistence.t,
   ~entityConfig: Internal.entityConfig,
+  ~scope: Internal.chainScope,
   ~indexerState,
   ~shouldGroup,
   ~item,
   ~ecosystem,
   ~filter: EntityFilter.t,
 ) => {
-  let key = filter->EntityFilter.toOperationKey(~entityName=entityConfig.name)
-  let inMemTable = indexerState->InMemoryStore.getInMemTable(~entityConfig)
+  let key =
+    filter->EntityFilter.toOperationKey(~entityName=entityConfig.name) ++ scope->scopeKeySuffix
+  let inMemTable = indexerState->InMemoryStore.getInMemTable(~entityConfig, ~scope)
 
   let load = async (filters: array<EntityFilter.t>, ~onError as _) => {
     let storage = persistence->Persistence.getInitializedStorageOrThrow
@@ -362,9 +390,12 @@ let loadByFilter = (
     ->Array.map(async filter => {
       try {
         let entities =
-          (await storage.loadOrThrow(~table=entityConfig.table, ~filter))->(
-            Utils.magic: array<unknown> => array<Internal.entity>
-          )
+          (
+            await storage.loadOrThrow(
+              ~table=entityConfig.table,
+              ~filter=filter->scopeFilter(~table=entityConfig.table, ~scope),
+            )
+          )->(Utils.magic: array<unknown> => array<Internal.entity>)
 
         entities->Array.forEach(entity => {
           inMemTable->InMemoryTable.Entity.initValue(

package/src/LoadLayer.res.mjs

@@ -18,19 +18,46 @@ import * as InMemoryTable from "./InMemoryTable.res.mjs";
 import * as S$RescriptSchema from "rescript-schema/src/S.res.mjs";
 import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
 
-function loadById(loadManager, persistence, entityConfig, indexerState, shouldGroup, item, ecosystem, entityId) {
-  let key = entityConfig.name + `.get`;
-  let inMemTable = InMemoryStore.getInMemTable(indexerState, entityConfig);
+function scopeKeySuffix(scope) {
+  if (scope === "crossChain") {
+    return "";
+  } else {
+    return `.` + ChainId.toString(scope);
+  }
+}
+
+function scopeFilter(filter, table, scope) {
+  let match = Table.getChainIdField(table);
+  if (scope === "crossChain" || match === undefined) {
+    return filter;
+  } else {
+    return {
+      operator: "and",
+      filters: [
+        filter,
+        {
+          operator: "=",
+          fieldName: match.fieldName,
+          fieldValue: scope
+        }
+      ]
+    };
+  }
+}
+
+function loadById(loadManager, persistence, entityConfig, scope, indexerState, shouldGroup, item, ecosystem, entityId) {
+  let key = entityConfig.name + `.get` + scopeKeySuffix(scope);
+  let inMemTable = InMemoryStore.getInMemTable(indexerState, entityConfig, scope);
   let load = async (idsToLoad, param) => {
     let storage = Persistence.getInitializedStorageOrThrow(persistence);
     let timerRef = IndexerState.startStorageLoad(indexerState, storage.name, key);
     let dbEntities;
     try {
-      dbEntities = await storage.loadOrThrow({
+      dbEntities = await storage.loadOrThrow(scopeFilter({
         operator: "in",
         fieldName: Table.idFieldName,
         fieldValue: idsToLoad
-      }, entityConfig.table);
+      }, entityConfig.table, scope), entityConfig.table);
     } catch (raw_exn) {
       let exn = Primitive_exceptions.internalToException(raw_exn);
       if (exn.RE_EXN_ID === Persistence.StorageError) {
@@ -181,9 +208,9 @@ function loadEffect(loadManager, persistence, effect, effectArgs, scope, indexer
   return LoadManager.call(loadManager, effectArgs, key, load, args => args.cacheKey, shouldGroup, hash => InMemoryStore.hasEffectOutput(inMemTable, hash), hash => InMemoryStore.getEffectOutputUnsafe(inMemTable, hash));
 }
 
-function loadByFilter(loadManager, persistence, entityConfig, indexerState, shouldGroup, item, ecosystem, filter) {
-  let key = EntityFilter.toOperationKey(filter, entityConfig.name);
-  let inMemTable = InMemoryStore.getInMemTable(indexerState, entityConfig);
+function loadByFilter(loadManager, persistence, entityConfig, scope, indexerState, shouldGroup, item, ecosystem, filter) {
+  let key = EntityFilter.toOperationKey(filter, entityConfig.name) + scopeKeySuffix(scope);
+  let inMemTable = InMemoryStore.getInMemTable(indexerState, entityConfig, scope);
   let load = async (filters, param) => {
     let storage = Persistence.getInitializedStorageOrThrow(persistence);
     let timerRef = IndexerState.startStorageLoad(indexerState, storage.name, key);
@@ -195,7 +222,7 @@ function loadByFilter(loadManager, persistence, entityConfig, indexerState, shou
     let queries = EntityFilter.merge(filters);
     await Promise.all(queries.map(async filter => {
       try {
-        let entities = await storage.loadOrThrow(filter, entityConfig.table);
+        let entities = await storage.loadOrThrow(scopeFilter(filter, entityConfig.table, scope), entityConfig.table);
         entities.forEach(entity => InMemoryTable.Entity.initValue(inMemTable, IndexerState.committedCheckpointId(indexerState), entity.id, entity));
         size.contents = size.contents + entities.length | 0;
         return;

package/src/LoadLayer.resi

@@ -2,6 +2,7 @@ let loadById: (
   ~loadManager: LoadManager.t,
   ~persistence: Persistence.t,
   ~entityConfig: Internal.entityConfig,
+  ~scope: Internal.chainScope,
   ~indexerState: IndexerState.t,
   ~shouldGroup: bool,
   ~item: Internal.item,
@@ -13,6 +14,7 @@ let loadByFilter: (
   ~loadManager: LoadManager.t,
   ~persistence: Persistence.t,
   ~entityConfig: Internal.entityConfig,
+  ~scope: Internal.chainScope,
   ~indexerState: IndexerState.t,
   ~shouldGroup: bool,
   ~item: Internal.item,

package/src/Metrics.res

@@ -445,15 +445,15 @@ let renderMetrics = (b: builder, metrics: t) => {
     ~entries=chains,
     ~value=m => m.endBlock->Option.map(Int.toFloat),
   )
-  b->series(
+  b->seriesOpt(
     ~name="envio_source_request_total",
-    ~help="The number of requests made to data sources.",
+    ~help="The number of requests made to data sources. Heights pushed by a subscription stream are counted here too, under the heightPush and heightPushIgnored methods.",
     ~kind="counter",
     ~entries=sourceRequests,
-    ~value=s => s.count->Int.toFloat,
+    ~value=s => s.count > 0 ? Some(s.count->Int.toFloat) : None,
   )
-  // Skips a method's seconds line when it has no timing (e.g. heightSubscription,
-  // which only ever records a count).
+  // Skips a method's seconds line when it has no timing (e.g. heightPush, which
+  // only ever records a count).
   b->seriesOpt(
     ~name="envio_source_request_seconds_total",
     ~help="Cumulative time spent on data source requests.",

package/src/Metrics.res.mjs

@@ -144,7 +144,11 @@ function renderMetrics(b, metrics) {
   single(b, "envio_indexing_target_buffer_size", "The indexer-wide target buffer size shared across all chains. The actual number of items in the queue may exceed this value, but the indexer always tries to keep the buffer filled up to this target.", "gauge", metrics.targetBufferSize);
   series(b, "envio_indexing_buffer_block", "The highest block number that has been fully fetched by the indexer.", "gauge", chains, m => m.bufferBlockNumber);
   seriesOpt(b, "envio_indexing_end_block", "The block number to stop indexing at. (inclusive)", "gauge", chains, m => Stdlib_Option.map(m.endBlock, prim => prim));
-  series(b, "envio_source_request_total", "The number of requests made to data sources.", "counter", sourceRequests, s => s.count);
+  seriesOpt(b, "envio_source_request_total", "The number of requests made to data sources. Heights pushed by a subscription stream are counted here too, under the heightPush and heightPushIgnored methods.", "counter", sourceRequests, s => {
+    if (s.count > 0) {
+      return s.count;
+    }
+  });
   seriesOpt(b, "envio_source_request_seconds_total", "Cumulative time spent on data source requests.", "counter", sourceRequests, s => {
     if (s.seconds !== 0) {
       return s.seconds;

package/src/Persistence.res

@@ -63,11 +63,21 @@ type rollback = {
   progressBlockNumberByChainId: dict<int>,
 }
 
+// One flush group: the changes an entity accumulated within a single chain
+// scope. A per-chain entity contributes one group per chain, and the scope is
+// what stamps the chain id onto the rows — it's never re-derived downstream.
 type updatedEntity = {
   entityConfig: Internal.entityConfig,
+  scope: Internal.chainScope,
   changes: array<Change.t<Internal.entity>>,
 }
 
+// An id the rollback must delete, together with the scope its row lives in.
+type rollbackRemoval = {
+  entityId: EntityId.t,
+  scope: Internal.chainScope,
+}
+
 type storage = {
   // Identifier used as the `storage` label on Prometheus metrics.
   name: string,
@@ -119,6 +129,7 @@ type storage = {
   pruneStaleEntityHistory: (
     ~entityName: string,
     ~entityIndex: int,
+    ~chainIdColumn: option<string>,
     ~safeCheckpointId: Internal.checkpointId,
   ) => promise<unit>,
   // Get rollback target checkpoint
@@ -140,7 +151,7 @@ type storage = {
   getRollbackData: (
     ~entityConfig: Internal.entityConfig,
     ~rollbackTargetCheckpointId: Internal.checkpointId,
-  ) => promise<(array<EntityId.t>, array<unknown>)>,
+  ) => promise<(array<rollbackRemoval>, array<unknown>)>,
   // Write batch to storage
   writeBatch: (
     ~batch: Batch.t,