envio 3.5.1 → 3.6.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. package/evm.schema.json +7 -0
  2. package/fuel.schema.json +7 -0
  3. package/index.d.ts +33 -10
  4. package/package.json +6 -6
  5. package/src/ChainState.res +22 -4
  6. package/src/ChainState.res.mjs +20 -3
  7. package/src/ChainState.resi +3 -0
  8. package/src/Config.res +60 -2
  9. package/src/Config.res.mjs +49 -13
  10. package/src/EffectState.res +2 -2
  11. package/src/EffectState.res.mjs +2 -2
  12. package/src/EntityTables.res +32 -0
  13. package/src/EntityTables.res.mjs +44 -0
  14. package/src/Envio.res +7 -7
  15. package/src/Envio.res.mjs +5 -6
  16. package/src/Hasura.res +37 -3
  17. package/src/Hasura.res.mjs +23 -5
  18. package/src/InMemoryStore.res +48 -8
  19. package/src/InMemoryStore.res.mjs +37 -7
  20. package/src/IndexerState.res +30 -24
  21. package/src/IndexerState.res.mjs +20 -35
  22. package/src/IndexerState.resi +8 -6
  23. package/src/Internal.res +29 -11
  24. package/src/Internal.res.mjs +27 -10
  25. package/src/LoadLayer.res +39 -8
  26. package/src/LoadLayer.res.mjs +36 -9
  27. package/src/LoadLayer.resi +2 -0
  28. package/src/Metrics.res +5 -5
  29. package/src/Metrics.res.mjs +5 -1
  30. package/src/Persistence.res +12 -1
  31. package/src/PgStorage.res +183 -37
  32. package/src/PgStorage.res.mjs +149 -34
  33. package/src/PruneStaleHistory.res +1 -0
  34. package/src/PruneStaleHistory.res.mjs +2 -1
  35. package/src/Sink.res +3 -3
  36. package/src/Sink.res.mjs +2 -2
  37. package/src/TestIndexer.res +128 -17
  38. package/src/TestIndexer.res.mjs +75 -9
  39. package/src/UserContext.res +19 -4
  40. package/src/UserContext.res.mjs +15 -9
  41. package/src/Writing.res +8 -16
  42. package/src/Writing.res.mjs +10 -10
  43. package/src/bindings/ClickHouse.res +40 -4
  44. package/src/bindings/ClickHouse.res.mjs +37 -5
  45. package/src/db/EntityHistory.res +64 -21
  46. package/src/db/EntityHistory.res.mjs +43 -22
  47. package/src/db/InternalTable.res.mjs +31 -31
  48. package/src/db/Table.res +24 -1
  49. package/src/db/Table.res.mjs +26 -4
  50. package/src/sources/SourceManager.res +6 -12
  51. package/src/sources/SourceManager.res.mjs +8 -5
  52. package/svm.schema.json +7 -0
@@ -12,40 +12,17 @@ import * as ChainState from "./ChainState.res.mjs";
12
12
  import * as EffectState from "./EffectState.res.mjs";
13
13
  import * as LoadManager from "./LoadManager.res.mjs";
14
14
  import * as Performance from "./bindings/Performance.res.mjs";
15
+ import * as EntityTables from "./EntityTables.res.mjs";
15
16
  import * as ErrorHandling from "./ErrorHandling.res.mjs";
16
- import * as InMemoryTable from "./InMemoryTable.res.mjs";
17
17
  import * as SourceManager from "./sources/SourceManager.res.mjs";
18
18
  import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
19
19
  import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
20
20
  import * as CrossChainState from "./CrossChainState.res.mjs";
21
21
  import * as Primitive_object from "@rescript/runtime/lib/es6/Primitive_object.js";
22
22
  import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js";
23
- import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
24
23
  import * as SimulateDeadInputTracker from "./SimulateDeadInputTracker.res.mjs";
25
24
 
26
- let UndefinedEntity = /* @__PURE__ */Primitive_exceptions.create("IndexerState.EntityTables.UndefinedEntity");
27
-
28
- function make(entities) {
29
- let init = {};
30
- entities.forEach(entityConfig => {
31
- init[entityConfig.name] = InMemoryTable.Entity.make();
32
- });
33
- return init;
34
- }
35
-
36
- function get(self, entityName) {
37
- let table = self[entityName];
38
- if (table !== undefined) {
39
- return table;
40
- } else {
41
- return ErrorHandling.mkLogAndRaise(undefined, "Unexpected, entity InMemoryTable is undefined", {
42
- RE_EXN_ID: UndefinedEntity,
43
- entityName: entityName
44
- });
45
- }
46
- }
47
-
48
- function make$1(config, persistence, chainStates, isInReorgThreshold, isRealtime, targetBufferSizeOpt, committedCheckpointIdOpt, isDevelopmentModeOpt, shouldUseTuiOpt, exitAfterFirstEventBlockOpt, onError, onExit) {
25
+ function make(config, persistence, chainStates, isInReorgThreshold, isRealtime, targetBufferSizeOpt, committedCheckpointIdOpt, isDevelopmentModeOpt, shouldUseTuiOpt, exitAfterFirstEventBlockOpt, onError, onExit) {
49
26
  let targetBufferSize = targetBufferSizeOpt !== undefined ? targetBufferSizeOpt : CrossChainState.calculateTargetBufferSize();
50
27
  let committedCheckpointId = committedCheckpointIdOpt !== undefined ? committedCheckpointIdOpt : Internal.initialCheckpointId;
51
28
  let isDevelopmentMode = isDevelopmentModeOpt !== undefined ? isDevelopmentModeOpt : false;
@@ -60,7 +37,7 @@ function make$1(config, persistence, chainStates, isInReorgThreshold, isRealtime
60
37
  config: config,
61
38
  persistence: persistence,
62
39
  allEntities: persistence.allEntities,
63
- entities: make(persistence.allEntities),
40
+ entities: EntityTables.make(EntityTables.crossChain(persistence.allEntities)),
64
41
  effectState: EffectState.make(),
65
42
  rollback: undefined,
66
43
  committedCheckpointId: committedCheckpointId,
@@ -125,7 +102,7 @@ function makeFromDbState(config, persistence, initialState, registrationsByChain
125
102
  let chainConfig = ChainMap.get(config.chainMap, chainId);
126
103
  chainStates[resumedChainState.id] = ChainState.makeFromDbState(chainConfig, resumedChainState, initialState.reorgCheckpoints, isInReorgThreshold, isRealtime, config, registrationsByChainId, reducedPollingInterval);
127
104
  });
128
- let state = make$1(config, persistence, chainStates, isInReorgThreshold, isRealtime, targetBufferSize, initialState.checkpointId, isDevelopmentMode, shouldUseTui, exitAfterFirstEventBlock, onError, onExit);
105
+ let state = make(config, persistence, chainStates, isInReorgThreshold, isRealtime, targetBufferSize, initialState.checkpointId, isDevelopmentMode, shouldUseTui, exitAfterFirstEventBlock, onError, onExit);
129
106
  CrossChainState.markCaughtUpOnResume(state.crossChainState);
130
107
  Utils.Dict.forEach(initialState.cache, param => {
131
108
  let count = param.count;
@@ -288,6 +265,17 @@ function entities(state) {
288
265
  return state.entities;
289
266
  }
290
267
 
268
+ function eachEntityTable(state, fn) {
269
+ let chainStates = CrossChainState.chainStates(state.crossChainState);
270
+ state.allEntities.forEach(entityConfig => {
271
+ if (entityConfig.crossChain) {
272
+ return fn(entityConfig, "crossChain", EntityTables.get(state.entities, entityConfig.name));
273
+ } else {
274
+ return Utils.Dict.forEach(chainStates, chainState => fn(entityConfig, ChainState.chainConfig(chainState).id, EntityTables.get(ChainState.entities(chainState), entityConfig.name)));
275
+ }
276
+ });
277
+ }
278
+
291
279
  function effectState(state) {
292
280
  return state.effectState;
293
281
  }
@@ -678,7 +666,9 @@ function markCommitted(state, upToCheckpointId) {
678
666
  }
679
667
 
680
668
  function beginRollbackDiff(state, targetCheckpointId, diffCheckpointId, progressBlockNumberByChainId) {
681
- state.entities = make(state.allEntities);
669
+ let perChainEntities = EntityTables.perChain(state.allEntities);
670
+ state.entities = EntityTables.make(EntityTables.crossChain(state.allEntities));
671
+ Utils.Dict.forEach(CrossChainState.chainStates(state.crossChainState), cs => ChainState.resetEntities(cs, perChainEntities));
682
672
  EffectState.resetForRollback(state.effectState);
683
673
  state.rollback = {
684
674
  targetCheckpointId: targetCheckpointId,
@@ -745,16 +735,10 @@ function takeChainMetaSnapshot(state) {
745
735
  }
746
736
  }
747
737
 
748
- let EntityTables = {
749
- make: make,
750
- get: get
751
- };
752
-
753
738
  let unexpectedErrorMsg = "Indexer has failed with an unexpected error";
754
739
 
755
740
  export {
756
- EntityTables,
757
- make$1 as make,
741
+ make,
758
742
  makeFromDbState,
759
743
  unexpectedErrorMsg,
760
744
  isStale,
@@ -781,6 +765,7 @@ export {
781
765
  persistence,
782
766
  allEntities,
783
767
  entities,
768
+ eachEntityTable,
784
769
  effectState,
785
770
  committedCheckpointId,
786
771
  processedCheckpointId,
@@ -9,12 +9,6 @@ type rollbackState =
9
9
  | FoundReorgDepth({chainId: ChainId.t, rollbackTargetBlockNumber: int})
10
10
  | RollbackReady({eventsProcessedDiffByChain: dict<float>})
11
11
 
12
- module EntityTables: {
13
- type t = dict<InMemoryTable.Entity.t>
14
- let make: array<Internal.entityConfig> => t
15
- let get: (t, ~entityName: string) => InMemoryTable.Entity.t
16
- }
17
-
18
12
  type t
19
13
 
20
14
  let make: (
@@ -80,6 +74,14 @@ let config: t => Config.t
80
74
  let persistence: t => Persistence.t
81
75
  let allEntities: t => array<Internal.entityConfig>
82
76
  let entities: t => EntityTables.t
77
+ let eachEntityTable: (
78
+ t,
79
+ (
80
+ ~entityConfig: Internal.entityConfig,
81
+ ~scope: Internal.chainScope,
82
+ ~table: InMemoryTable.Entity.t,
83
+ ) => unit,
84
+ ) => unit
83
85
  let effectState: t => EffectState.t
84
86
  let committedCheckpointId: t => Internal.checkpointId
85
87
  let processedCheckpointId: t => Internal.checkpointId
package/src/Internal.res CHANGED
@@ -744,6 +744,10 @@ type genericEntityConfig<'entity> = {
744
744
  schema: S.t<'entity>,
745
745
  table: Table.table,
746
746
  storage: entityStorage,
747
+ // Resolved by the CLI against the config's `defaultCrossChain` and the
748
+ // entity's `@crossChain`. When false the table carries a chain-id column in
749
+ // its primary key and every row belongs to exactly one chain.
750
+ crossChain: bool,
747
751
  }
748
752
  type entityConfig = genericEntityConfig<entity>
749
753
  external fromGenericEntityConfig: genericEntityConfig<'entity> => entityConfig = "%identity"
@@ -773,10 +777,11 @@ type effect = {
773
777
  handler: effectArgs => promise<effectOutput>,
774
778
  storageMeta: effectCacheStorageMeta,
775
779
  defaultShouldCache: bool,
776
- // When true (the default) a single cache is shared across every chain and the
777
- // handler must not read context.chain. When false the cache is isolated per
778
- // chain and context.chain.id is available.
779
- crossChain: bool,
780
+ // When true a single cache is shared across every chain and the handler must
781
+ // not read context.chain. When false the cache is isolated per chain and
782
+ // context.chain.id is available. None means the effect didn't say, and the
783
+ // config's `defaultCrossChain` decides.
784
+ crossChain: option<bool>,
780
785
  output: S.t<effectOutput>,
781
786
  input: S.t<effectInput>,
782
787
  rateLimit: option<rateLimitOptions>,
@@ -791,6 +796,26 @@ type chainScope =
791
796
  | @as("crossChain") CrossChain
792
797
  | Chain(ChainId.t)
793
798
 
799
+ let chainScopeToString = scope =>
800
+ switch scope {
801
+ | CrossChain => "crossChain"
802
+ | Chain(chainId) => chainId->ChainId.toString
803
+ }
804
+
805
+ let chainScopeChainId = scope =>
806
+ switch scope {
807
+ | CrossChain => None
808
+ | Chain(chainId) => Some(chainId)
809
+ }
810
+
811
+ // A copy of the entity carrying the chain-id column a per-chain entity's row
812
+ // needs. Never mutates the handler's object, which stays in the in-memory table.
813
+ let stampChainId = (entity: entity, ~fieldName, ~chainId: ChainId.t): entity =>
814
+ Utils.Dict.merge(
815
+ entity->(Utils.magic: entity => dict<unknown>),
816
+ Dict.fromArray([(fieldName, chainId->(Utils.magic: ChainId.t => unknown))]),
817
+ )->(Utils.magic: dict<unknown> => entity)
818
+
794
819
  let cacheTablePrefix = "envio_effect_"
795
820
 
796
821
  // The single reversible mapping between an effect's (name, scope) and its
@@ -806,13 +831,6 @@ module EffectCache = {
806
831
  | Chain(chainId) => `envio_${chainId->ChainId.toString}_effect_${effectName}`
807
832
  }
808
833
 
809
- // "crossChain" or the decimal chain id. Used as the `scope` Prometheus label.
810
- let scopeToString = scope =>
811
- switch scope {
812
- | CrossChain => "crossChain"
813
- | Chain(chainId) => chainId->ChainId.toString
814
- }
815
-
816
834
  // Only accepts a canonical decimal chain id ("7", not "007" or "1foo") —
817
835
  // the schema's parser follows parseFloat semantics and accepts both.
818
836
  let parseChainId = str =>
@@ -166,21 +166,36 @@ let fuelTransferParamsSchema = S$RescriptSchema.schema(s => ({
166
166
  amount: s.m(Utils.$$BigInt.schema)
167
167
  }));
168
168
 
169
- let cacheTablePrefix = "envio_effect_";
169
+ function chainScopeToString(scope) {
170
+ if (scope === "crossChain") {
171
+ return "crossChain";
172
+ } else {
173
+ return ChainId.toString(scope);
174
+ }
175
+ }
170
176
 
171
- function toTableName(effectName, scope) {
177
+ function chainScopeChainId(scope) {
172
178
  if (scope === "crossChain") {
173
- return cacheTablePrefix + effectName;
179
+ return;
174
180
  } else {
175
- return `envio_` + ChainId.toString(scope) + `_effect_` + effectName;
181
+ return Primitive_option.some(scope);
176
182
  }
177
183
  }
178
184
 
179
- function scopeToString(scope) {
185
+ function stampChainId(entity, fieldName, chainId) {
186
+ return Utils.Dict.merge(entity, Object.fromEntries([[
187
+ fieldName,
188
+ chainId
189
+ ]]));
190
+ }
191
+
192
+ let cacheTablePrefix = "envio_effect_";
193
+
194
+ function toTableName(effectName, scope) {
180
195
  if (scope === "crossChain") {
181
- return "crossChain";
196
+ return cacheTablePrefix + effectName;
182
197
  } else {
183
- return ChainId.toString(scope);
198
+ return `envio_` + ChainId.toString(scope) + `_effect_` + effectName;
184
199
  }
185
200
  }
186
201
 
@@ -260,7 +275,6 @@ function toCachePath(effectName, scope) {
260
275
 
261
276
  let EffectCache = {
262
277
  toTableName: toTableName,
263
- scopeToString: scopeToString,
264
278
  parseChainId: parseChainId,
265
279
  chainScopedRe: chainScopedRe,
266
280
  crossChainRe: crossChainRe,
@@ -272,8 +286,8 @@ let cacheOutputSchema = S$RescriptSchema.json(false);
272
286
 
273
287
  function makeCacheTable(effectName, scope) {
274
288
  return Table.mkTable(toTableName(effectName, scope), undefined, [
275
- Table.mkField("id", "String", S$RescriptSchema.string, undefined, undefined, undefined, true, undefined, undefined, undefined, undefined, undefined),
276
- Table.mkField("output", "Json", cacheOutputSchema, undefined, undefined, true, undefined, undefined, undefined, undefined, undefined, undefined)
289
+ Table.mkField("id", "String", S$RescriptSchema.string, undefined, undefined, undefined, true, undefined, undefined, undefined, undefined, undefined, undefined),
290
+ Table.mkField("output", "Json", cacheOutputSchema, undefined, undefined, true, undefined, undefined, undefined, undefined, undefined, undefined, undefined)
277
291
  ], undefined);
278
292
  }
279
293
 
@@ -296,6 +310,9 @@ export {
296
310
  getItemChainId,
297
311
  fuelSupplyParamsSchema,
298
312
  fuelTransferParamsSchema,
313
+ chainScopeToString,
314
+ chainScopeChainId,
315
+ stampChainId,
299
316
  cacheTablePrefix,
300
317
  EffectCache,
301
318
  cacheOutputSchema,
package/src/LoadLayer.res CHANGED
@@ -1,15 +1,41 @@
1
+ // A per-chain entity's rows must never dedup or merge across chains, so both
2
+ // the load-group key and the storage query carry the scope.
3
+ let scopeKeySuffix = (scope: Internal.chainScope) =>
4
+ switch scope {
5
+ | CrossChain => ""
6
+ | Chain(chainId) => `.${chainId->ChainId.toString}`
7
+ }
8
+
9
+ // Narrows a query to the scope's chain. Cross-chain entities have no chain-id
10
+ // column, so their filter is left untouched.
11
+ let scopeFilter = (filter: EntityFilter.t, ~table: Table.table, ~scope: Internal.chainScope) =>
12
+ switch (scope, table->Table.getChainIdField) {
13
+ | (Chain(chainId), Some(field)) =>
14
+ EntityFilter.And({
15
+ filters: [
16
+ filter,
17
+ Eq({
18
+ fieldName: field.fieldName,
19
+ fieldValue: chainId->(Utils.magic: ChainId.t => unknown),
20
+ }),
21
+ ],
22
+ })
23
+ | _ => filter
24
+ }
25
+
1
26
  let loadById = (
2
27
  ~loadManager,
3
28
  ~persistence: Persistence.t,
4
29
  ~entityConfig: Internal.entityConfig,
30
+ ~scope: Internal.chainScope,
5
31
  ~indexerState,
6
32
  ~shouldGroup,
7
33
  ~item,
8
34
  ~ecosystem,
9
35
  ~entityId,
10
36
  ) => {
11
- let key = `${entityConfig.name}.get`
12
- let inMemTable = indexerState->InMemoryStore.getInMemTable(~entityConfig)
37
+ let key = `${entityConfig.name}.get${scope->scopeKeySuffix}`
38
+ let inMemTable = indexerState->InMemoryStore.getInMemTable(~entityConfig, ~scope)
13
39
 
14
40
  let load = async (idsToLoad, ~onError as _) => {
15
41
  let storage = persistence->Persistence.getInitializedStorageOrThrow
@@ -25,7 +51,7 @@ let loadById = (
25
51
  ~filter=EntityFilter.In({
26
52
  fieldName: Table.idFieldName,
27
53
  fieldValue: idsToLoad->(Utils.magic: array<string> => array<unknown>),
28
- }),
54
+ })->scopeFilter(~table=entityConfig.table, ~scope),
29
55
  )
30
56
  )->(Utils.magic: array<unknown> => array<Internal.entity>)
31
57
  } catch {
@@ -327,14 +353,16 @@ let loadByFilter = (
327
353
  ~loadManager,
328
354
  ~persistence: Persistence.t,
329
355
  ~entityConfig: Internal.entityConfig,
356
+ ~scope: Internal.chainScope,
330
357
  ~indexerState,
331
358
  ~shouldGroup,
332
359
  ~item,
333
360
  ~ecosystem,
334
361
  ~filter: EntityFilter.t,
335
362
  ) => {
336
- let key = filter->EntityFilter.toOperationKey(~entityName=entityConfig.name)
337
- let inMemTable = indexerState->InMemoryStore.getInMemTable(~entityConfig)
363
+ let key =
364
+ filter->EntityFilter.toOperationKey(~entityName=entityConfig.name) ++ scope->scopeKeySuffix
365
+ let inMemTable = indexerState->InMemoryStore.getInMemTable(~entityConfig, ~scope)
338
366
 
339
367
  let load = async (filters: array<EntityFilter.t>, ~onError as _) => {
340
368
  let storage = persistence->Persistence.getInitializedStorageOrThrow
@@ -362,9 +390,12 @@ let loadByFilter = (
362
390
  ->Array.map(async filter => {
363
391
  try {
364
392
  let entities =
365
- (await storage.loadOrThrow(~table=entityConfig.table, ~filter))->(
366
- Utils.magic: array<unknown> => array<Internal.entity>
367
- )
393
+ (
394
+ await storage.loadOrThrow(
395
+ ~table=entityConfig.table,
396
+ ~filter=filter->scopeFilter(~table=entityConfig.table, ~scope),
397
+ )
398
+ )->(Utils.magic: array<unknown> => array<Internal.entity>)
368
399
 
369
400
  entities->Array.forEach(entity => {
370
401
  inMemTable->InMemoryTable.Entity.initValue(
@@ -18,19 +18,46 @@ import * as InMemoryTable from "./InMemoryTable.res.mjs";
18
18
  import * as S$RescriptSchema from "rescript-schema/src/S.res.mjs";
19
19
  import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
20
20
 
21
- function loadById(loadManager, persistence, entityConfig, indexerState, shouldGroup, item, ecosystem, entityId) {
22
- let key = entityConfig.name + `.get`;
23
- let inMemTable = InMemoryStore.getInMemTable(indexerState, entityConfig);
21
+ function scopeKeySuffix(scope) {
22
+ if (scope === "crossChain") {
23
+ return "";
24
+ } else {
25
+ return `.` + ChainId.toString(scope);
26
+ }
27
+ }
28
+
29
+ function scopeFilter(filter, table, scope) {
30
+ let match = Table.getChainIdField(table);
31
+ if (scope === "crossChain" || match === undefined) {
32
+ return filter;
33
+ } else {
34
+ return {
35
+ operator: "and",
36
+ filters: [
37
+ filter,
38
+ {
39
+ operator: "=",
40
+ fieldName: match.fieldName,
41
+ fieldValue: scope
42
+ }
43
+ ]
44
+ };
45
+ }
46
+ }
47
+
48
+ function loadById(loadManager, persistence, entityConfig, scope, indexerState, shouldGroup, item, ecosystem, entityId) {
49
+ let key = entityConfig.name + `.get` + scopeKeySuffix(scope);
50
+ let inMemTable = InMemoryStore.getInMemTable(indexerState, entityConfig, scope);
24
51
  let load = async (idsToLoad, param) => {
25
52
  let storage = Persistence.getInitializedStorageOrThrow(persistence);
26
53
  let timerRef = IndexerState.startStorageLoad(indexerState, storage.name, key);
27
54
  let dbEntities;
28
55
  try {
29
- dbEntities = await storage.loadOrThrow({
56
+ dbEntities = await storage.loadOrThrow(scopeFilter({
30
57
  operator: "in",
31
58
  fieldName: Table.idFieldName,
32
59
  fieldValue: idsToLoad
33
- }, entityConfig.table);
60
+ }, entityConfig.table, scope), entityConfig.table);
34
61
  } catch (raw_exn) {
35
62
  let exn = Primitive_exceptions.internalToException(raw_exn);
36
63
  if (exn.RE_EXN_ID === Persistence.StorageError) {
@@ -181,9 +208,9 @@ function loadEffect(loadManager, persistence, effect, effectArgs, scope, indexer
181
208
  return LoadManager.call(loadManager, effectArgs, key, load, args => args.cacheKey, shouldGroup, hash => InMemoryStore.hasEffectOutput(inMemTable, hash), hash => InMemoryStore.getEffectOutputUnsafe(inMemTable, hash));
182
209
  }
183
210
 
184
- function loadByFilter(loadManager, persistence, entityConfig, indexerState, shouldGroup, item, ecosystem, filter) {
185
- let key = EntityFilter.toOperationKey(filter, entityConfig.name);
186
- let inMemTable = InMemoryStore.getInMemTable(indexerState, entityConfig);
211
+ function loadByFilter(loadManager, persistence, entityConfig, scope, indexerState, shouldGroup, item, ecosystem, filter) {
212
+ let key = EntityFilter.toOperationKey(filter, entityConfig.name) + scopeKeySuffix(scope);
213
+ let inMemTable = InMemoryStore.getInMemTable(indexerState, entityConfig, scope);
187
214
  let load = async (filters, param) => {
188
215
  let storage = Persistence.getInitializedStorageOrThrow(persistence);
189
216
  let timerRef = IndexerState.startStorageLoad(indexerState, storage.name, key);
@@ -195,7 +222,7 @@ function loadByFilter(loadManager, persistence, entityConfig, indexerState, shou
195
222
  let queries = EntityFilter.merge(filters);
196
223
  await Promise.all(queries.map(async filter => {
197
224
  try {
198
- let entities = await storage.loadOrThrow(filter, entityConfig.table);
225
+ let entities = await storage.loadOrThrow(scopeFilter(filter, entityConfig.table, scope), entityConfig.table);
199
226
  entities.forEach(entity => InMemoryTable.Entity.initValue(inMemTable, IndexerState.committedCheckpointId(indexerState), entity.id, entity));
200
227
  size.contents = size.contents + entities.length | 0;
201
228
  return;
@@ -2,6 +2,7 @@ let loadById: (
2
2
  ~loadManager: LoadManager.t,
3
3
  ~persistence: Persistence.t,
4
4
  ~entityConfig: Internal.entityConfig,
5
+ ~scope: Internal.chainScope,
5
6
  ~indexerState: IndexerState.t,
6
7
  ~shouldGroup: bool,
7
8
  ~item: Internal.item,
@@ -13,6 +14,7 @@ let loadByFilter: (
13
14
  ~loadManager: LoadManager.t,
14
15
  ~persistence: Persistence.t,
15
16
  ~entityConfig: Internal.entityConfig,
17
+ ~scope: Internal.chainScope,
16
18
  ~indexerState: IndexerState.t,
17
19
  ~shouldGroup: bool,
18
20
  ~item: Internal.item,
package/src/Metrics.res CHANGED
@@ -445,15 +445,15 @@ let renderMetrics = (b: builder, metrics: t) => {
445
445
  ~entries=chains,
446
446
  ~value=m => m.endBlock->Option.map(Int.toFloat),
447
447
  )
448
- b->series(
448
+ b->seriesOpt(
449
449
  ~name="envio_source_request_total",
450
- ~help="The number of requests made to data sources.",
450
+ ~help="The number of requests made to data sources. Heights pushed by a subscription stream are counted here too, under the heightPush and heightPushIgnored methods.",
451
451
  ~kind="counter",
452
452
  ~entries=sourceRequests,
453
- ~value=s => s.count->Int.toFloat,
453
+ ~value=s => s.count > 0 ? Some(s.count->Int.toFloat) : None,
454
454
  )
455
- // Skips a method's seconds line when it has no timing (e.g. heightSubscription,
456
- // which only ever records a count).
455
+ // Skips a method's seconds line when it has no timing (e.g. heightPush, which
456
+ // only ever records a count).
457
457
  b->seriesOpt(
458
458
  ~name="envio_source_request_seconds_total",
459
459
  ~help="Cumulative time spent on data source requests.",
@@ -144,7 +144,11 @@ function renderMetrics(b, metrics) {
144
144
  single(b, "envio_indexing_target_buffer_size", "The indexer-wide target buffer size shared across all chains. The actual number of items in the queue may exceed this value, but the indexer always tries to keep the buffer filled up to this target.", "gauge", metrics.targetBufferSize);
145
145
  series(b, "envio_indexing_buffer_block", "The highest block number that has been fully fetched by the indexer.", "gauge", chains, m => m.bufferBlockNumber);
146
146
  seriesOpt(b, "envio_indexing_end_block", "The block number to stop indexing at. (inclusive)", "gauge", chains, m => Stdlib_Option.map(m.endBlock, prim => prim));
147
- series(b, "envio_source_request_total", "The number of requests made to data sources.", "counter", sourceRequests, s => s.count);
147
+ seriesOpt(b, "envio_source_request_total", "The number of requests made to data sources. Heights pushed by a subscription stream are counted here too, under the heightPush and heightPushIgnored methods.", "counter", sourceRequests, s => {
148
+ if (s.count > 0) {
149
+ return s.count;
150
+ }
151
+ });
148
152
  seriesOpt(b, "envio_source_request_seconds_total", "Cumulative time spent on data source requests.", "counter", sourceRequests, s => {
149
153
  if (s.seconds !== 0) {
150
154
  return s.seconds;
@@ -63,11 +63,21 @@ type rollback = {
63
63
  progressBlockNumberByChainId: dict<int>,
64
64
  }
65
65
 
66
+ // One flush group: the changes an entity accumulated within a single chain
67
+ // scope. A per-chain entity contributes one group per chain, and the scope is
68
+ // what stamps the chain id onto the rows — it's never re-derived downstream.
66
69
  type updatedEntity = {
67
70
  entityConfig: Internal.entityConfig,
71
+ scope: Internal.chainScope,
68
72
  changes: array<Change.t<Internal.entity>>,
69
73
  }
70
74
 
75
+ // An id the rollback must delete, together with the scope its row lives in.
76
+ type rollbackRemoval = {
77
+ entityId: EntityId.t,
78
+ scope: Internal.chainScope,
79
+ }
80
+
71
81
  type storage = {
72
82
  // Identifier used as the `storage` label on Prometheus metrics.
73
83
  name: string,
@@ -119,6 +129,7 @@ type storage = {
119
129
  pruneStaleEntityHistory: (
120
130
  ~entityName: string,
121
131
  ~entityIndex: int,
132
+ ~chainIdColumn: option<string>,
122
133
  ~safeCheckpointId: Internal.checkpointId,
123
134
  ) => promise<unit>,
124
135
  // Get rollback target checkpoint
@@ -140,7 +151,7 @@ type storage = {
140
151
  getRollbackData: (
141
152
  ~entityConfig: Internal.entityConfig,
142
153
  ~rollbackTargetCheckpointId: Internal.checkpointId,
143
- ) => promise<(array<EntityId.t>, array<unknown>)>,
154
+ ) => promise<(array<rollbackRemoval>, array<unknown>)>,
144
155
  // Write batch to storage
145
156
  writeBatch: (
146
157
  ~batch: Batch.t,