envio 3.5.1 → 3.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/evm.schema.json +7 -0
- package/fuel.schema.json +7 -0
- package/index.d.ts +33 -10
- package/package.json +6 -6
- package/src/ChainState.res +22 -4
- package/src/ChainState.res.mjs +20 -3
- package/src/ChainState.resi +3 -0
- package/src/Config.res +60 -2
- package/src/Config.res.mjs +49 -13
- package/src/EffectState.res +2 -2
- package/src/EffectState.res.mjs +2 -2
- package/src/EntityTables.res +32 -0
- package/src/EntityTables.res.mjs +44 -0
- package/src/Envio.res +7 -7
- package/src/Envio.res.mjs +5 -6
- package/src/Hasura.res +37 -3
- package/src/Hasura.res.mjs +23 -5
- package/src/InMemoryStore.res +48 -8
- package/src/InMemoryStore.res.mjs +37 -7
- package/src/IndexerState.res +30 -24
- package/src/IndexerState.res.mjs +20 -35
- package/src/IndexerState.resi +8 -6
- package/src/Internal.res +29 -11
- package/src/Internal.res.mjs +27 -10
- package/src/LoadLayer.res +39 -8
- package/src/LoadLayer.res.mjs +36 -9
- package/src/LoadLayer.resi +2 -0
- package/src/Metrics.res +5 -5
- package/src/Metrics.res.mjs +5 -1
- package/src/Persistence.res +12 -1
- package/src/PgStorage.res +183 -37
- package/src/PgStorage.res.mjs +149 -34
- package/src/PruneStaleHistory.res +1 -0
- package/src/PruneStaleHistory.res.mjs +2 -1
- package/src/Sink.res +3 -3
- package/src/Sink.res.mjs +2 -2
- package/src/TestIndexer.res +128 -17
- package/src/TestIndexer.res.mjs +75 -9
- package/src/UserContext.res +19 -4
- package/src/UserContext.res.mjs +15 -9
- package/src/Writing.res +8 -16
- package/src/Writing.res.mjs +10 -10
- package/src/bindings/ClickHouse.res +40 -4
- package/src/bindings/ClickHouse.res.mjs +37 -5
- package/src/db/EntityHistory.res +64 -21
- package/src/db/EntityHistory.res.mjs +43 -22
- package/src/db/InternalTable.res.mjs +31 -31
- package/src/db/Table.res +24 -1
- package/src/db/Table.res.mjs +26 -4
- package/src/sources/SourceManager.res +6 -12
- package/src/sources/SourceManager.res.mjs +8 -5
- package/svm.schema.json +7 -0
package/src/IndexerState.res.mjs
CHANGED
|
@@ -12,40 +12,17 @@ import * as ChainState from "./ChainState.res.mjs";
|
|
|
12
12
|
import * as EffectState from "./EffectState.res.mjs";
|
|
13
13
|
import * as LoadManager from "./LoadManager.res.mjs";
|
|
14
14
|
import * as Performance from "./bindings/Performance.res.mjs";
|
|
15
|
+
import * as EntityTables from "./EntityTables.res.mjs";
|
|
15
16
|
import * as ErrorHandling from "./ErrorHandling.res.mjs";
|
|
16
|
-
import * as InMemoryTable from "./InMemoryTable.res.mjs";
|
|
17
17
|
import * as SourceManager from "./sources/SourceManager.res.mjs";
|
|
18
18
|
import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
|
|
19
19
|
import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
|
|
20
20
|
import * as CrossChainState from "./CrossChainState.res.mjs";
|
|
21
21
|
import * as Primitive_object from "@rescript/runtime/lib/es6/Primitive_object.js";
|
|
22
22
|
import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js";
|
|
23
|
-
import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
|
|
24
23
|
import * as SimulateDeadInputTracker from "./SimulateDeadInputTracker.res.mjs";
|
|
25
24
|
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
function make(entities) {
|
|
29
|
-
let init = {};
|
|
30
|
-
entities.forEach(entityConfig => {
|
|
31
|
-
init[entityConfig.name] = InMemoryTable.Entity.make();
|
|
32
|
-
});
|
|
33
|
-
return init;
|
|
34
|
-
}
|
|
35
|
-
|
|
36
|
-
function get(self, entityName) {
|
|
37
|
-
let table = self[entityName];
|
|
38
|
-
if (table !== undefined) {
|
|
39
|
-
return table;
|
|
40
|
-
} else {
|
|
41
|
-
return ErrorHandling.mkLogAndRaise(undefined, "Unexpected, entity InMemoryTable is undefined", {
|
|
42
|
-
RE_EXN_ID: UndefinedEntity,
|
|
43
|
-
entityName: entityName
|
|
44
|
-
});
|
|
45
|
-
}
|
|
46
|
-
}
|
|
47
|
-
|
|
48
|
-
function make$1(config, persistence, chainStates, isInReorgThreshold, isRealtime, targetBufferSizeOpt, committedCheckpointIdOpt, isDevelopmentModeOpt, shouldUseTuiOpt, exitAfterFirstEventBlockOpt, onError, onExit) {
|
|
25
|
+
function make(config, persistence, chainStates, isInReorgThreshold, isRealtime, targetBufferSizeOpt, committedCheckpointIdOpt, isDevelopmentModeOpt, shouldUseTuiOpt, exitAfterFirstEventBlockOpt, onError, onExit) {
|
|
49
26
|
let targetBufferSize = targetBufferSizeOpt !== undefined ? targetBufferSizeOpt : CrossChainState.calculateTargetBufferSize();
|
|
50
27
|
let committedCheckpointId = committedCheckpointIdOpt !== undefined ? committedCheckpointIdOpt : Internal.initialCheckpointId;
|
|
51
28
|
let isDevelopmentMode = isDevelopmentModeOpt !== undefined ? isDevelopmentModeOpt : false;
|
|
@@ -60,7 +37,7 @@ function make$1(config, persistence, chainStates, isInReorgThreshold, isRealtime
|
|
|
60
37
|
config: config,
|
|
61
38
|
persistence: persistence,
|
|
62
39
|
allEntities: persistence.allEntities,
|
|
63
|
-
entities: make(persistence.allEntities),
|
|
40
|
+
entities: EntityTables.make(EntityTables.crossChain(persistence.allEntities)),
|
|
64
41
|
effectState: EffectState.make(),
|
|
65
42
|
rollback: undefined,
|
|
66
43
|
committedCheckpointId: committedCheckpointId,
|
|
@@ -125,7 +102,7 @@ function makeFromDbState(config, persistence, initialState, registrationsByChain
|
|
|
125
102
|
let chainConfig = ChainMap.get(config.chainMap, chainId);
|
|
126
103
|
chainStates[resumedChainState.id] = ChainState.makeFromDbState(chainConfig, resumedChainState, initialState.reorgCheckpoints, isInReorgThreshold, isRealtime, config, registrationsByChainId, reducedPollingInterval);
|
|
127
104
|
});
|
|
128
|
-
let state = make
|
|
105
|
+
let state = make(config, persistence, chainStates, isInReorgThreshold, isRealtime, targetBufferSize, initialState.checkpointId, isDevelopmentMode, shouldUseTui, exitAfterFirstEventBlock, onError, onExit);
|
|
129
106
|
CrossChainState.markCaughtUpOnResume(state.crossChainState);
|
|
130
107
|
Utils.Dict.forEach(initialState.cache, param => {
|
|
131
108
|
let count = param.count;
|
|
@@ -288,6 +265,17 @@ function entities(state) {
|
|
|
288
265
|
return state.entities;
|
|
289
266
|
}
|
|
290
267
|
|
|
268
|
+
function eachEntityTable(state, fn) {
|
|
269
|
+
let chainStates = CrossChainState.chainStates(state.crossChainState);
|
|
270
|
+
state.allEntities.forEach(entityConfig => {
|
|
271
|
+
if (entityConfig.crossChain) {
|
|
272
|
+
return fn(entityConfig, "crossChain", EntityTables.get(state.entities, entityConfig.name));
|
|
273
|
+
} else {
|
|
274
|
+
return Utils.Dict.forEach(chainStates, chainState => fn(entityConfig, ChainState.chainConfig(chainState).id, EntityTables.get(ChainState.entities(chainState), entityConfig.name)));
|
|
275
|
+
}
|
|
276
|
+
});
|
|
277
|
+
}
|
|
278
|
+
|
|
291
279
|
function effectState(state) {
|
|
292
280
|
return state.effectState;
|
|
293
281
|
}
|
|
@@ -678,7 +666,9 @@ function markCommitted(state, upToCheckpointId) {
|
|
|
678
666
|
}
|
|
679
667
|
|
|
680
668
|
function beginRollbackDiff(state, targetCheckpointId, diffCheckpointId, progressBlockNumberByChainId) {
|
|
681
|
-
|
|
669
|
+
let perChainEntities = EntityTables.perChain(state.allEntities);
|
|
670
|
+
state.entities = EntityTables.make(EntityTables.crossChain(state.allEntities));
|
|
671
|
+
Utils.Dict.forEach(CrossChainState.chainStates(state.crossChainState), cs => ChainState.resetEntities(cs, perChainEntities));
|
|
682
672
|
EffectState.resetForRollback(state.effectState);
|
|
683
673
|
state.rollback = {
|
|
684
674
|
targetCheckpointId: targetCheckpointId,
|
|
@@ -745,16 +735,10 @@ function takeChainMetaSnapshot(state) {
|
|
|
745
735
|
}
|
|
746
736
|
}
|
|
747
737
|
|
|
748
|
-
let EntityTables = {
|
|
749
|
-
make: make,
|
|
750
|
-
get: get
|
|
751
|
-
};
|
|
752
|
-
|
|
753
738
|
let unexpectedErrorMsg = "Indexer has failed with an unexpected error";
|
|
754
739
|
|
|
755
740
|
export {
|
|
756
|
-
|
|
757
|
-
make$1 as make,
|
|
741
|
+
make,
|
|
758
742
|
makeFromDbState,
|
|
759
743
|
unexpectedErrorMsg,
|
|
760
744
|
isStale,
|
|
@@ -781,6 +765,7 @@ export {
|
|
|
781
765
|
persistence,
|
|
782
766
|
allEntities,
|
|
783
767
|
entities,
|
|
768
|
+
eachEntityTable,
|
|
784
769
|
effectState,
|
|
785
770
|
committedCheckpointId,
|
|
786
771
|
processedCheckpointId,
|
package/src/IndexerState.resi
CHANGED
|
@@ -9,12 +9,6 @@ type rollbackState =
|
|
|
9
9
|
| FoundReorgDepth({chainId: ChainId.t, rollbackTargetBlockNumber: int})
|
|
10
10
|
| RollbackReady({eventsProcessedDiffByChain: dict<float>})
|
|
11
11
|
|
|
12
|
-
module EntityTables: {
|
|
13
|
-
type t = dict<InMemoryTable.Entity.t>
|
|
14
|
-
let make: array<Internal.entityConfig> => t
|
|
15
|
-
let get: (t, ~entityName: string) => InMemoryTable.Entity.t
|
|
16
|
-
}
|
|
17
|
-
|
|
18
12
|
type t
|
|
19
13
|
|
|
20
14
|
let make: (
|
|
@@ -80,6 +74,14 @@ let config: t => Config.t
|
|
|
80
74
|
let persistence: t => Persistence.t
|
|
81
75
|
let allEntities: t => array<Internal.entityConfig>
|
|
82
76
|
let entities: t => EntityTables.t
|
|
77
|
+
let eachEntityTable: (
|
|
78
|
+
t,
|
|
79
|
+
(
|
|
80
|
+
~entityConfig: Internal.entityConfig,
|
|
81
|
+
~scope: Internal.chainScope,
|
|
82
|
+
~table: InMemoryTable.Entity.t,
|
|
83
|
+
) => unit,
|
|
84
|
+
) => unit
|
|
83
85
|
let effectState: t => EffectState.t
|
|
84
86
|
let committedCheckpointId: t => Internal.checkpointId
|
|
85
87
|
let processedCheckpointId: t => Internal.checkpointId
|
package/src/Internal.res
CHANGED
|
@@ -744,6 +744,10 @@ type genericEntityConfig<'entity> = {
|
|
|
744
744
|
schema: S.t<'entity>,
|
|
745
745
|
table: Table.table,
|
|
746
746
|
storage: entityStorage,
|
|
747
|
+
// Resolved by the CLI against the config's `defaultCrossChain` and the
|
|
748
|
+
// entity's `@crossChain`. When false the table carries a chain-id column in
|
|
749
|
+
// its primary key and every row belongs to exactly one chain.
|
|
750
|
+
crossChain: bool,
|
|
747
751
|
}
|
|
748
752
|
type entityConfig = genericEntityConfig<entity>
|
|
749
753
|
external fromGenericEntityConfig: genericEntityConfig<'entity> => entityConfig = "%identity"
|
|
@@ -773,10 +777,11 @@ type effect = {
|
|
|
773
777
|
handler: effectArgs => promise<effectOutput>,
|
|
774
778
|
storageMeta: effectCacheStorageMeta,
|
|
775
779
|
defaultShouldCache: bool,
|
|
776
|
-
// When true
|
|
777
|
-
//
|
|
778
|
-
//
|
|
779
|
-
|
|
780
|
+
// When true a single cache is shared across every chain and the handler must
|
|
781
|
+
// not read context.chain. When false the cache is isolated per chain and
|
|
782
|
+
// context.chain.id is available. None means the effect didn't say, and the
|
|
783
|
+
// config's `defaultCrossChain` decides.
|
|
784
|
+
crossChain: option<bool>,
|
|
780
785
|
output: S.t<effectOutput>,
|
|
781
786
|
input: S.t<effectInput>,
|
|
782
787
|
rateLimit: option<rateLimitOptions>,
|
|
@@ -791,6 +796,26 @@ type chainScope =
|
|
|
791
796
|
| @as("crossChain") CrossChain
|
|
792
797
|
| Chain(ChainId.t)
|
|
793
798
|
|
|
799
|
+
let chainScopeToString = scope =>
|
|
800
|
+
switch scope {
|
|
801
|
+
| CrossChain => "crossChain"
|
|
802
|
+
| Chain(chainId) => chainId->ChainId.toString
|
|
803
|
+
}
|
|
804
|
+
|
|
805
|
+
let chainScopeChainId = scope =>
|
|
806
|
+
switch scope {
|
|
807
|
+
| CrossChain => None
|
|
808
|
+
| Chain(chainId) => Some(chainId)
|
|
809
|
+
}
|
|
810
|
+
|
|
811
|
+
// A copy of the entity carrying the chain-id column a per-chain entity's row
|
|
812
|
+
// needs. Never mutates the handler's object, which stays in the in-memory table.
|
|
813
|
+
let stampChainId = (entity: entity, ~fieldName, ~chainId: ChainId.t): entity =>
|
|
814
|
+
Utils.Dict.merge(
|
|
815
|
+
entity->(Utils.magic: entity => dict<unknown>),
|
|
816
|
+
Dict.fromArray([(fieldName, chainId->(Utils.magic: ChainId.t => unknown))]),
|
|
817
|
+
)->(Utils.magic: dict<unknown> => entity)
|
|
818
|
+
|
|
794
819
|
let cacheTablePrefix = "envio_effect_"
|
|
795
820
|
|
|
796
821
|
// The single reversible mapping between an effect's (name, scope) and its
|
|
@@ -806,13 +831,6 @@ module EffectCache = {
|
|
|
806
831
|
| Chain(chainId) => `envio_${chainId->ChainId.toString}_effect_${effectName}`
|
|
807
832
|
}
|
|
808
833
|
|
|
809
|
-
// "crossChain" or the decimal chain id. Used as the `scope` Prometheus label.
|
|
810
|
-
let scopeToString = scope =>
|
|
811
|
-
switch scope {
|
|
812
|
-
| CrossChain => "crossChain"
|
|
813
|
-
| Chain(chainId) => chainId->ChainId.toString
|
|
814
|
-
}
|
|
815
|
-
|
|
816
834
|
// Only accepts a canonical decimal chain id ("7", not "007" or "1foo") —
|
|
817
835
|
// the schema's parser follows parseFloat semantics and accepts both.
|
|
818
836
|
let parseChainId = str =>
|
package/src/Internal.res.mjs
CHANGED
|
@@ -166,21 +166,36 @@ let fuelTransferParamsSchema = S$RescriptSchema.schema(s => ({
|
|
|
166
166
|
amount: s.m(Utils.$$BigInt.schema)
|
|
167
167
|
}));
|
|
168
168
|
|
|
169
|
-
|
|
169
|
+
function chainScopeToString(scope) {
|
|
170
|
+
if (scope === "crossChain") {
|
|
171
|
+
return "crossChain";
|
|
172
|
+
} else {
|
|
173
|
+
return ChainId.toString(scope);
|
|
174
|
+
}
|
|
175
|
+
}
|
|
170
176
|
|
|
171
|
-
function
|
|
177
|
+
function chainScopeChainId(scope) {
|
|
172
178
|
if (scope === "crossChain") {
|
|
173
|
-
return
|
|
179
|
+
return;
|
|
174
180
|
} else {
|
|
175
|
-
return
|
|
181
|
+
return Primitive_option.some(scope);
|
|
176
182
|
}
|
|
177
183
|
}
|
|
178
184
|
|
|
179
|
-
function
|
|
185
|
+
function stampChainId(entity, fieldName, chainId) {
|
|
186
|
+
return Utils.Dict.merge(entity, Object.fromEntries([[
|
|
187
|
+
fieldName,
|
|
188
|
+
chainId
|
|
189
|
+
]]));
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
let cacheTablePrefix = "envio_effect_";
|
|
193
|
+
|
|
194
|
+
function toTableName(effectName, scope) {
|
|
180
195
|
if (scope === "crossChain") {
|
|
181
|
-
return
|
|
196
|
+
return cacheTablePrefix + effectName;
|
|
182
197
|
} else {
|
|
183
|
-
return ChainId.toString(scope);
|
|
198
|
+
return `envio_` + ChainId.toString(scope) + `_effect_` + effectName;
|
|
184
199
|
}
|
|
185
200
|
}
|
|
186
201
|
|
|
@@ -260,7 +275,6 @@ function toCachePath(effectName, scope) {
|
|
|
260
275
|
|
|
261
276
|
let EffectCache = {
|
|
262
277
|
toTableName: toTableName,
|
|
263
|
-
scopeToString: scopeToString,
|
|
264
278
|
parseChainId: parseChainId,
|
|
265
279
|
chainScopedRe: chainScopedRe,
|
|
266
280
|
crossChainRe: crossChainRe,
|
|
@@ -272,8 +286,8 @@ let cacheOutputSchema = S$RescriptSchema.json(false);
|
|
|
272
286
|
|
|
273
287
|
function makeCacheTable(effectName, scope) {
|
|
274
288
|
return Table.mkTable(toTableName(effectName, scope), undefined, [
|
|
275
|
-
Table.mkField("id", "String", S$RescriptSchema.string, undefined, undefined, undefined, true, undefined, undefined, undefined, undefined, undefined),
|
|
276
|
-
Table.mkField("output", "Json", cacheOutputSchema, undefined, undefined, true, undefined, undefined, undefined, undefined, undefined, undefined)
|
|
289
|
+
Table.mkField("id", "String", S$RescriptSchema.string, undefined, undefined, undefined, true, undefined, undefined, undefined, undefined, undefined, undefined),
|
|
290
|
+
Table.mkField("output", "Json", cacheOutputSchema, undefined, undefined, true, undefined, undefined, undefined, undefined, undefined, undefined, undefined)
|
|
277
291
|
], undefined);
|
|
278
292
|
}
|
|
279
293
|
|
|
@@ -296,6 +310,9 @@ export {
|
|
|
296
310
|
getItemChainId,
|
|
297
311
|
fuelSupplyParamsSchema,
|
|
298
312
|
fuelTransferParamsSchema,
|
|
313
|
+
chainScopeToString,
|
|
314
|
+
chainScopeChainId,
|
|
315
|
+
stampChainId,
|
|
299
316
|
cacheTablePrefix,
|
|
300
317
|
EffectCache,
|
|
301
318
|
cacheOutputSchema,
|
package/src/LoadLayer.res
CHANGED
|
@@ -1,15 +1,41 @@
|
|
|
1
|
+
// A per-chain entity's rows must never dedup or merge across chains, so both
|
|
2
|
+
// the load-group key and the storage query carry the scope.
|
|
3
|
+
let scopeKeySuffix = (scope: Internal.chainScope) =>
|
|
4
|
+
switch scope {
|
|
5
|
+
| CrossChain => ""
|
|
6
|
+
| Chain(chainId) => `.${chainId->ChainId.toString}`
|
|
7
|
+
}
|
|
8
|
+
|
|
9
|
+
// Narrows a query to the scope's chain. Cross-chain entities have no chain-id
|
|
10
|
+
// column, so their filter is left untouched.
|
|
11
|
+
let scopeFilter = (filter: EntityFilter.t, ~table: Table.table, ~scope: Internal.chainScope) =>
|
|
12
|
+
switch (scope, table->Table.getChainIdField) {
|
|
13
|
+
| (Chain(chainId), Some(field)) =>
|
|
14
|
+
EntityFilter.And({
|
|
15
|
+
filters: [
|
|
16
|
+
filter,
|
|
17
|
+
Eq({
|
|
18
|
+
fieldName: field.fieldName,
|
|
19
|
+
fieldValue: chainId->(Utils.magic: ChainId.t => unknown),
|
|
20
|
+
}),
|
|
21
|
+
],
|
|
22
|
+
})
|
|
23
|
+
| _ => filter
|
|
24
|
+
}
|
|
25
|
+
|
|
1
26
|
let loadById = (
|
|
2
27
|
~loadManager,
|
|
3
28
|
~persistence: Persistence.t,
|
|
4
29
|
~entityConfig: Internal.entityConfig,
|
|
30
|
+
~scope: Internal.chainScope,
|
|
5
31
|
~indexerState,
|
|
6
32
|
~shouldGroup,
|
|
7
33
|
~item,
|
|
8
34
|
~ecosystem,
|
|
9
35
|
~entityId,
|
|
10
36
|
) => {
|
|
11
|
-
let key = `${entityConfig.name}.get`
|
|
12
|
-
let inMemTable = indexerState->InMemoryStore.getInMemTable(~entityConfig)
|
|
37
|
+
let key = `${entityConfig.name}.get${scope->scopeKeySuffix}`
|
|
38
|
+
let inMemTable = indexerState->InMemoryStore.getInMemTable(~entityConfig, ~scope)
|
|
13
39
|
|
|
14
40
|
let load = async (idsToLoad, ~onError as _) => {
|
|
15
41
|
let storage = persistence->Persistence.getInitializedStorageOrThrow
|
|
@@ -25,7 +51,7 @@ let loadById = (
|
|
|
25
51
|
~filter=EntityFilter.In({
|
|
26
52
|
fieldName: Table.idFieldName,
|
|
27
53
|
fieldValue: idsToLoad->(Utils.magic: array<string> => array<unknown>),
|
|
28
|
-
}),
|
|
54
|
+
})->scopeFilter(~table=entityConfig.table, ~scope),
|
|
29
55
|
)
|
|
30
56
|
)->(Utils.magic: array<unknown> => array<Internal.entity>)
|
|
31
57
|
} catch {
|
|
@@ -327,14 +353,16 @@ let loadByFilter = (
|
|
|
327
353
|
~loadManager,
|
|
328
354
|
~persistence: Persistence.t,
|
|
329
355
|
~entityConfig: Internal.entityConfig,
|
|
356
|
+
~scope: Internal.chainScope,
|
|
330
357
|
~indexerState,
|
|
331
358
|
~shouldGroup,
|
|
332
359
|
~item,
|
|
333
360
|
~ecosystem,
|
|
334
361
|
~filter: EntityFilter.t,
|
|
335
362
|
) => {
|
|
336
|
-
let key =
|
|
337
|
-
|
|
363
|
+
let key =
|
|
364
|
+
filter->EntityFilter.toOperationKey(~entityName=entityConfig.name) ++ scope->scopeKeySuffix
|
|
365
|
+
let inMemTable = indexerState->InMemoryStore.getInMemTable(~entityConfig, ~scope)
|
|
338
366
|
|
|
339
367
|
let load = async (filters: array<EntityFilter.t>, ~onError as _) => {
|
|
340
368
|
let storage = persistence->Persistence.getInitializedStorageOrThrow
|
|
@@ -362,9 +390,12 @@ let loadByFilter = (
|
|
|
362
390
|
->Array.map(async filter => {
|
|
363
391
|
try {
|
|
364
392
|
let entities =
|
|
365
|
-
(
|
|
366
|
-
|
|
367
|
-
|
|
393
|
+
(
|
|
394
|
+
await storage.loadOrThrow(
|
|
395
|
+
~table=entityConfig.table,
|
|
396
|
+
~filter=filter->scopeFilter(~table=entityConfig.table, ~scope),
|
|
397
|
+
)
|
|
398
|
+
)->(Utils.magic: array<unknown> => array<Internal.entity>)
|
|
368
399
|
|
|
369
400
|
entities->Array.forEach(entity => {
|
|
370
401
|
inMemTable->InMemoryTable.Entity.initValue(
|
package/src/LoadLayer.res.mjs
CHANGED
|
@@ -18,19 +18,46 @@ import * as InMemoryTable from "./InMemoryTable.res.mjs";
|
|
|
18
18
|
import * as S$RescriptSchema from "rescript-schema/src/S.res.mjs";
|
|
19
19
|
import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
|
|
20
20
|
|
|
21
|
-
function
|
|
22
|
-
|
|
23
|
-
|
|
21
|
+
function scopeKeySuffix(scope) {
|
|
22
|
+
if (scope === "crossChain") {
|
|
23
|
+
return "";
|
|
24
|
+
} else {
|
|
25
|
+
return `.` + ChainId.toString(scope);
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
function scopeFilter(filter, table, scope) {
|
|
30
|
+
let match = Table.getChainIdField(table);
|
|
31
|
+
if (scope === "crossChain" || match === undefined) {
|
|
32
|
+
return filter;
|
|
33
|
+
} else {
|
|
34
|
+
return {
|
|
35
|
+
operator: "and",
|
|
36
|
+
filters: [
|
|
37
|
+
filter,
|
|
38
|
+
{
|
|
39
|
+
operator: "=",
|
|
40
|
+
fieldName: match.fieldName,
|
|
41
|
+
fieldValue: scope
|
|
42
|
+
}
|
|
43
|
+
]
|
|
44
|
+
};
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
function loadById(loadManager, persistence, entityConfig, scope, indexerState, shouldGroup, item, ecosystem, entityId) {
|
|
49
|
+
let key = entityConfig.name + `.get` + scopeKeySuffix(scope);
|
|
50
|
+
let inMemTable = InMemoryStore.getInMemTable(indexerState, entityConfig, scope);
|
|
24
51
|
let load = async (idsToLoad, param) => {
|
|
25
52
|
let storage = Persistence.getInitializedStorageOrThrow(persistence);
|
|
26
53
|
let timerRef = IndexerState.startStorageLoad(indexerState, storage.name, key);
|
|
27
54
|
let dbEntities;
|
|
28
55
|
try {
|
|
29
|
-
dbEntities = await storage.loadOrThrow({
|
|
56
|
+
dbEntities = await storage.loadOrThrow(scopeFilter({
|
|
30
57
|
operator: "in",
|
|
31
58
|
fieldName: Table.idFieldName,
|
|
32
59
|
fieldValue: idsToLoad
|
|
33
|
-
}, entityConfig.table);
|
|
60
|
+
}, entityConfig.table, scope), entityConfig.table);
|
|
34
61
|
} catch (raw_exn) {
|
|
35
62
|
let exn = Primitive_exceptions.internalToException(raw_exn);
|
|
36
63
|
if (exn.RE_EXN_ID === Persistence.StorageError) {
|
|
@@ -181,9 +208,9 @@ function loadEffect(loadManager, persistence, effect, effectArgs, scope, indexer
|
|
|
181
208
|
return LoadManager.call(loadManager, effectArgs, key, load, args => args.cacheKey, shouldGroup, hash => InMemoryStore.hasEffectOutput(inMemTable, hash), hash => InMemoryStore.getEffectOutputUnsafe(inMemTable, hash));
|
|
182
209
|
}
|
|
183
210
|
|
|
184
|
-
function loadByFilter(loadManager, persistence, entityConfig, indexerState, shouldGroup, item, ecosystem, filter) {
|
|
185
|
-
let key = EntityFilter.toOperationKey(filter, entityConfig.name);
|
|
186
|
-
let inMemTable = InMemoryStore.getInMemTable(indexerState, entityConfig);
|
|
211
|
+
function loadByFilter(loadManager, persistence, entityConfig, scope, indexerState, shouldGroup, item, ecosystem, filter) {
|
|
212
|
+
let key = EntityFilter.toOperationKey(filter, entityConfig.name) + scopeKeySuffix(scope);
|
|
213
|
+
let inMemTable = InMemoryStore.getInMemTable(indexerState, entityConfig, scope);
|
|
187
214
|
let load = async (filters, param) => {
|
|
188
215
|
let storage = Persistence.getInitializedStorageOrThrow(persistence);
|
|
189
216
|
let timerRef = IndexerState.startStorageLoad(indexerState, storage.name, key);
|
|
@@ -195,7 +222,7 @@ function loadByFilter(loadManager, persistence, entityConfig, indexerState, shou
|
|
|
195
222
|
let queries = EntityFilter.merge(filters);
|
|
196
223
|
await Promise.all(queries.map(async filter => {
|
|
197
224
|
try {
|
|
198
|
-
let entities = await storage.loadOrThrow(filter, entityConfig.table);
|
|
225
|
+
let entities = await storage.loadOrThrow(scopeFilter(filter, entityConfig.table, scope), entityConfig.table);
|
|
199
226
|
entities.forEach(entity => InMemoryTable.Entity.initValue(inMemTable, IndexerState.committedCheckpointId(indexerState), entity.id, entity));
|
|
200
227
|
size.contents = size.contents + entities.length | 0;
|
|
201
228
|
return;
|
package/src/LoadLayer.resi
CHANGED
|
@@ -2,6 +2,7 @@ let loadById: (
|
|
|
2
2
|
~loadManager: LoadManager.t,
|
|
3
3
|
~persistence: Persistence.t,
|
|
4
4
|
~entityConfig: Internal.entityConfig,
|
|
5
|
+
~scope: Internal.chainScope,
|
|
5
6
|
~indexerState: IndexerState.t,
|
|
6
7
|
~shouldGroup: bool,
|
|
7
8
|
~item: Internal.item,
|
|
@@ -13,6 +14,7 @@ let loadByFilter: (
|
|
|
13
14
|
~loadManager: LoadManager.t,
|
|
14
15
|
~persistence: Persistence.t,
|
|
15
16
|
~entityConfig: Internal.entityConfig,
|
|
17
|
+
~scope: Internal.chainScope,
|
|
16
18
|
~indexerState: IndexerState.t,
|
|
17
19
|
~shouldGroup: bool,
|
|
18
20
|
~item: Internal.item,
|
package/src/Metrics.res
CHANGED
|
@@ -445,15 +445,15 @@ let renderMetrics = (b: builder, metrics: t) => {
|
|
|
445
445
|
~entries=chains,
|
|
446
446
|
~value=m => m.endBlock->Option.map(Int.toFloat),
|
|
447
447
|
)
|
|
448
|
-
b->
|
|
448
|
+
b->seriesOpt(
|
|
449
449
|
~name="envio_source_request_total",
|
|
450
|
-
~help="The number of requests made to data sources.",
|
|
450
|
+
~help="The number of requests made to data sources. Heights pushed by a subscription stream are counted here too, under the heightPush and heightPushIgnored methods.",
|
|
451
451
|
~kind="counter",
|
|
452
452
|
~entries=sourceRequests,
|
|
453
|
-
~value=s => s.count->Int.toFloat,
|
|
453
|
+
~value=s => s.count > 0 ? Some(s.count->Int.toFloat) : None,
|
|
454
454
|
)
|
|
455
|
-
// Skips a method's seconds line when it has no timing (e.g.
|
|
456
|
-
//
|
|
455
|
+
// Skips a method's seconds line when it has no timing (e.g. heightPush, which
|
|
456
|
+
// only ever records a count).
|
|
457
457
|
b->seriesOpt(
|
|
458
458
|
~name="envio_source_request_seconds_total",
|
|
459
459
|
~help="Cumulative time spent on data source requests.",
|
package/src/Metrics.res.mjs
CHANGED
|
@@ -144,7 +144,11 @@ function renderMetrics(b, metrics) {
|
|
|
144
144
|
single(b, "envio_indexing_target_buffer_size", "The indexer-wide target buffer size shared across all chains. The actual number of items in the queue may exceed this value, but the indexer always tries to keep the buffer filled up to this target.", "gauge", metrics.targetBufferSize);
|
|
145
145
|
series(b, "envio_indexing_buffer_block", "The highest block number that has been fully fetched by the indexer.", "gauge", chains, m => m.bufferBlockNumber);
|
|
146
146
|
seriesOpt(b, "envio_indexing_end_block", "The block number to stop indexing at. (inclusive)", "gauge", chains, m => Stdlib_Option.map(m.endBlock, prim => prim));
|
|
147
|
-
|
|
147
|
+
seriesOpt(b, "envio_source_request_total", "The number of requests made to data sources. Heights pushed by a subscription stream are counted here too, under the heightPush and heightPushIgnored methods.", "counter", sourceRequests, s => {
|
|
148
|
+
if (s.count > 0) {
|
|
149
|
+
return s.count;
|
|
150
|
+
}
|
|
151
|
+
});
|
|
148
152
|
seriesOpt(b, "envio_source_request_seconds_total", "Cumulative time spent on data source requests.", "counter", sourceRequests, s => {
|
|
149
153
|
if (s.seconds !== 0) {
|
|
150
154
|
return s.seconds;
|
package/src/Persistence.res
CHANGED
|
@@ -63,11 +63,21 @@ type rollback = {
|
|
|
63
63
|
progressBlockNumberByChainId: dict<int>,
|
|
64
64
|
}
|
|
65
65
|
|
|
66
|
+
// One flush group: the changes an entity accumulated within a single chain
|
|
67
|
+
// scope. A per-chain entity contributes one group per chain, and the scope is
|
|
68
|
+
// what stamps the chain id onto the rows — it's never re-derived downstream.
|
|
66
69
|
type updatedEntity = {
|
|
67
70
|
entityConfig: Internal.entityConfig,
|
|
71
|
+
scope: Internal.chainScope,
|
|
68
72
|
changes: array<Change.t<Internal.entity>>,
|
|
69
73
|
}
|
|
70
74
|
|
|
75
|
+
// An id the rollback must delete, together with the scope its row lives in.
|
|
76
|
+
type rollbackRemoval = {
|
|
77
|
+
entityId: EntityId.t,
|
|
78
|
+
scope: Internal.chainScope,
|
|
79
|
+
}
|
|
80
|
+
|
|
71
81
|
type storage = {
|
|
72
82
|
// Identifier used as the `storage` label on Prometheus metrics.
|
|
73
83
|
name: string,
|
|
@@ -119,6 +129,7 @@ type storage = {
|
|
|
119
129
|
pruneStaleEntityHistory: (
|
|
120
130
|
~entityName: string,
|
|
121
131
|
~entityIndex: int,
|
|
132
|
+
~chainIdColumn: option<string>,
|
|
122
133
|
~safeCheckpointId: Internal.checkpointId,
|
|
123
134
|
) => promise<unit>,
|
|
124
135
|
// Get rollback target checkpoint
|
|
@@ -140,7 +151,7 @@ type storage = {
|
|
|
140
151
|
getRollbackData: (
|
|
141
152
|
~entityConfig: Internal.entityConfig,
|
|
142
153
|
~rollbackTargetCheckpointId: Internal.checkpointId,
|
|
143
|
-
) => promise<(array<
|
|
154
|
+
) => promise<(array<rollbackRemoval>, array<unknown>)>,
|
|
144
155
|
// Write batch to storage
|
|
145
156
|
writeBatch: (
|
|
146
157
|
~batch: Batch.t,
|