envio 3.5.0-alpha.0 → 3.5.0-alpha.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.d.ts +10 -6
- package/package.json +6 -6
- package/src/BatchProcessing.res +13 -0
- package/src/BatchProcessing.res.mjs +4 -0
- package/src/Bin.res +2 -2
- package/src/ChainFetching.res +5 -5
- package/src/ChainFetching.res.mjs +3 -4
- package/src/ChainState.res +44 -36
- package/src/ChainState.res.mjs +36 -31
- package/src/ChainState.resi +2 -12
- package/src/Change.res +3 -3
- package/src/Config.res +4 -4
- package/src/Config.res.mjs +1 -10
- package/src/Core.res +4 -35
- package/src/Core.res.mjs +1 -31
- package/src/CrossChainState.res +2 -20
- package/src/CrossChainState.res.mjs +2 -6
- package/src/EntityId.res +15 -0
- package/src/EntityId.res.mjs +9 -0
- package/src/Env.res +16 -0
- package/src/Env.res.mjs +6 -0
- package/src/EventConfigBuilder.res +1 -58
- package/src/EventConfigBuilder.res.mjs +1 -33
- package/src/FetchState.res +663 -484
- package/src/FetchState.res.mjs +491 -440
- package/src/HandlerRegister.res +45 -0
- package/src/HandlerRegister.res.mjs +43 -0
- package/src/InMemoryStore.res +11 -4
- package/src/InMemoryTable.res +12 -10
- package/src/InMemoryTable.res.mjs +6 -5
- package/src/IndexerState.res +59 -2
- package/src/IndexerState.res.mjs +48 -3
- package/src/IndexerState.resi +2 -0
- package/src/Internal.res +17 -8
- package/src/LogSelection.res +2 -1
- package/src/Metrics.res +37 -6
- package/src/Metrics.res.mjs +5 -1
- package/src/Persistence.res +1 -1
- package/src/PgStorage.res +55 -22
- package/src/PgStorage.res.mjs +25 -17
- package/src/SimulateItems.res +2 -2
- package/src/TestIndexer.res +6 -5
- package/src/TestIndexer.res.mjs +3 -2
- package/src/UserContext.res +3 -3
- package/src/Writing.res +12 -2
- package/src/Writing.res.mjs +13 -2
- package/src/bindings/ClickHouse.res +5 -3
- package/src/bindings/ClickHouse.res.mjs +2 -5
- package/src/bindings/Vitest.res +22 -1
- package/src/bindings/Vitest.res.mjs +15 -0
- package/src/db/EntityHistory.res +16 -7
- package/src/db/EntityHistory.res.mjs +7 -6
- package/src/db/Table.res +40 -9
- package/src/db/Table.res.mjs +44 -6
- package/src/sources/AddressSet.res +48 -0
- package/src/sources/AddressSet.res.mjs +11 -0
- package/src/sources/AddressStore.res +152 -0
- package/src/sources/AddressStore.res.mjs +97 -0
- package/src/sources/EvmChain.res +3 -0
- package/src/sources/EvmChain.res.mjs +4 -2
- package/src/sources/EvmHyperSyncSource.res +9 -9
- package/src/sources/EvmHyperSyncSource.res.mjs +3 -3
- package/src/sources/EvmRpcClient.res +10 -3
- package/src/sources/EvmRpcClient.res.mjs +3 -3
- package/src/sources/FuelHyperSync.res +4 -3
- package/src/sources/FuelHyperSync.res.mjs +3 -3
- package/src/sources/FuelHyperSync.resi +2 -1
- package/src/sources/FuelHyperSyncClient.res +12 -6
- package/src/sources/FuelHyperSyncClient.res.mjs +2 -2
- package/src/sources/FuelHyperSyncSource.res +7 -4
- package/src/sources/FuelHyperSyncSource.res.mjs +3 -3
- package/src/sources/HyperSync.res +9 -5
- package/src/sources/HyperSync.res.mjs +3 -3
- package/src/sources/HyperSync.resi +3 -2
- package/src/sources/HyperSyncClient.res +24 -10
- package/src/sources/HyperSyncClient.res.mjs +4 -4
- package/src/sources/RpcSource.res +71 -67
- package/src/sources/RpcSource.res.mjs +4 -4
- package/src/sources/SimulateSource.res +26 -11
- package/src/sources/SimulateSource.res.mjs +13 -5
- package/src/sources/Source.res +8 -4
- package/src/sources/SourceManager.res +2 -3
- package/src/sources/SourceManager.res.mjs +2 -3
- package/src/sources/Svm.res +1 -2
- package/src/sources/Svm.res.mjs +1 -1
- package/src/sources/SvmHyperSyncClient.res +14 -4
- package/src/sources/SvmHyperSyncClient.res.mjs +2 -2
- package/src/sources/SvmHyperSyncSource.res +11 -6
- package/src/sources/SvmHyperSyncSource.res.mjs +5 -4
- package/src/IndexingAddresses.res +0 -108
- package/src/IndexingAddresses.res.mjs +0 -101
- package/src/IndexingAddresses.resi +0 -34
package/src/HandlerRegister.res
CHANGED
|
@@ -215,6 +215,25 @@ let mergeRegistrations = (resolved: array<Internal.onEventRegistration>, ~config
|
|
|
215
215
|
let isDroppedByWhere = (~config: Config.t, reg: Internal.onEventRegistration) =>
|
|
216
216
|
config.ecosystem.name === Evm && (reg->getResolvedWhere).topicSelections->Utils.Array.isEmpty
|
|
217
217
|
|
|
218
|
+
// Names configured across every chain, used to explain a registration that
|
|
219
|
+
// matched nothing. Only built on the failure path.
|
|
220
|
+
let describeConfigured = (registration: activeRegistration, ~contractName) => {
|
|
221
|
+
let contractNames = Utils.Set.make()
|
|
222
|
+
let eventNames = Utils.Set.make()
|
|
223
|
+
registration.config.chainMap
|
|
224
|
+
->ChainMap.values
|
|
225
|
+
->Array.forEach(chainConfig =>
|
|
226
|
+
chainConfig.contracts->Array.forEach(contract => {
|
|
227
|
+
contractNames->Utils.Set.add(contract.name)->ignore
|
|
228
|
+
if contract.name === contractName {
|
|
229
|
+
contract.events->Array.forEach(e => eventNames->Utils.Set.add(e.name)->ignore)
|
|
230
|
+
}
|
|
231
|
+
})
|
|
232
|
+
)
|
|
233
|
+
let sorted = set => set->Utils.Set.toArray->Array.toSorted(String.compare)
|
|
234
|
+
(sorted(contractNames), sorted(eventNames))
|
|
235
|
+
}
|
|
236
|
+
|
|
218
237
|
// Resolve one `onEvent`/`contractRegister` call into a registration for every
|
|
219
238
|
// chain in the config (the full chain set) and store it in registration order.
|
|
220
239
|
// Building runs the user's `where` callback here — once per chain — so a broken
|
|
@@ -230,6 +249,7 @@ let addOnEventRegistration = (
|
|
|
230
249
|
) => {
|
|
231
250
|
let isWildcard = eventOptions->Option.flatMap(v => v.wildcard)->Option.getOr(false)
|
|
232
251
|
let where = eventOptions->Option.flatMap(v => v.where)
|
|
252
|
+
let matched = ref(false)
|
|
233
253
|
registration.config.chainMap
|
|
234
254
|
->ChainMap.values
|
|
235
255
|
->Array.forEach(chainConfig =>
|
|
@@ -239,6 +259,7 @@ let addOnEventRegistration = (
|
|
|
239
259
|
switch contract.events->Array.find(e => e.name === eventName) {
|
|
240
260
|
| None => ()
|
|
241
261
|
| Some(eventConfig) =>
|
|
262
|
+
matched := true
|
|
242
263
|
let reg = buildOnEventRegistrationWith(
|
|
243
264
|
~config=registration.config,
|
|
244
265
|
~chainId=chainConfig.id,
|
|
@@ -255,6 +276,30 @@ let addOnEventRegistration = (
|
|
|
255
276
|
}
|
|
256
277
|
}
|
|
257
278
|
)
|
|
279
|
+
|
|
280
|
+
// Nothing matched on any chain, so the callback could never be dispatched.
|
|
281
|
+
// Reported at the registration call site, where the stack still points at the
|
|
282
|
+
// offending `onEvent`/`contractRegister`.
|
|
283
|
+
if !matched.contents {
|
|
284
|
+
let (contractNames, eventNames) = registration->describeConfigured(~contractName)
|
|
285
|
+
let listOr = (names, empty) =>
|
|
286
|
+
names->Utils.Array.isEmpty ? empty : names->Array.joinUnsafe(", ")
|
|
287
|
+
if eventNames->Utils.Array.isEmpty {
|
|
288
|
+
JsError.throwWithMessage(
|
|
289
|
+
`Contract "${contractName}" is not configured on any chain, so its handler for "${eventName}" would never run. Add it to your config, or remove the registration. Configured contracts: ${listOr(
|
|
290
|
+
contractNames,
|
|
291
|
+
"none",
|
|
292
|
+
)}.`,
|
|
293
|
+
)
|
|
294
|
+
} else {
|
|
295
|
+
JsError.throwWithMessage(
|
|
296
|
+
`Event "${eventName}" is not configured on contract "${contractName}", so its handler would never run. Add it to your config, or remove the registration. Configured events on "${contractName}": ${listOr(
|
|
297
|
+
eventNames,
|
|
298
|
+
"none",
|
|
299
|
+
)}.`,
|
|
300
|
+
)
|
|
301
|
+
}
|
|
302
|
+
}
|
|
258
303
|
}
|
|
259
304
|
|
|
260
305
|
let setHandler = (~contractName, ~eventName, handler, ~eventOptions) => {
|
|
@@ -8,6 +8,7 @@ import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
|
|
|
8
8
|
import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
|
|
9
9
|
import * as Primitive_object from "@rescript/runtime/lib/es6/Primitive_object.js";
|
|
10
10
|
import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js";
|
|
11
|
+
import * as Primitive_string from "@rescript/runtime/lib/es6/Primitive_string.js";
|
|
11
12
|
import * as S$RescriptSchema from "rescript-schema/src/S.res.mjs";
|
|
12
13
|
import * as EventConfigBuilder from "./EventConfigBuilder.res.mjs";
|
|
13
14
|
import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
|
|
@@ -145,9 +146,33 @@ function isDroppedByWhere(config, reg) {
|
|
|
145
146
|
}
|
|
146
147
|
}
|
|
147
148
|
|
|
149
|
+
function describeConfigured(registration, contractName) {
|
|
150
|
+
let contractNames = new Set();
|
|
151
|
+
let eventNames = new Set();
|
|
152
|
+
ChainMap.values(registration.config.chainMap).forEach(chainConfig => {
|
|
153
|
+
chainConfig.contracts.forEach(contract => {
|
|
154
|
+
contractNames.add(contract.name);
|
|
155
|
+
if (contract.name === contractName) {
|
|
156
|
+
contract.events.forEach(e => {
|
|
157
|
+
eventNames.add(e.name);
|
|
158
|
+
});
|
|
159
|
+
return;
|
|
160
|
+
}
|
|
161
|
+
});
|
|
162
|
+
});
|
|
163
|
+
let sorted = set => Array.from(set).toSorted(Primitive_string.compare);
|
|
164
|
+
return [
|
|
165
|
+
sorted(contractNames),
|
|
166
|
+
sorted(eventNames)
|
|
167
|
+
];
|
|
168
|
+
}
|
|
169
|
+
|
|
148
170
|
function addOnEventRegistration(registration, contractName, eventName, handler, contractRegister, eventOptions) {
|
|
149
171
|
let isWildcard = Stdlib_Option.getOr(Stdlib_Option.flatMap(eventOptions, v => v.wildcard), false);
|
|
150
172
|
let where = Stdlib_Option.flatMap(eventOptions, v => v.where);
|
|
173
|
+
let matched = {
|
|
174
|
+
contents: false
|
|
175
|
+
};
|
|
151
176
|
ChainMap.values(registration.config.chainMap).forEach(chainConfig => {
|
|
152
177
|
let contract = chainConfig.contracts.find(c => c.name === contractName);
|
|
153
178
|
if (contract === undefined) {
|
|
@@ -157,9 +182,27 @@ function addOnEventRegistration(registration, contractName, eventName, handler,
|
|
|
157
182
|
if (eventConfig === undefined) {
|
|
158
183
|
return;
|
|
159
184
|
}
|
|
185
|
+
matched.contents = true;
|
|
160
186
|
let reg = buildOnEventRegistrationWith(registration.config, chainConfig.id, eventConfig, isWildcard, handler, contractRegister, where, contract.startBlock);
|
|
161
187
|
getChainRegistrations(registration, chainConfig.id).onEventRegistrations.push(reg);
|
|
162
188
|
});
|
|
189
|
+
if (matched.contents) {
|
|
190
|
+
return;
|
|
191
|
+
}
|
|
192
|
+
let match = describeConfigured(registration, contractName);
|
|
193
|
+
let eventNames = match[1];
|
|
194
|
+
let listOr = (names, empty) => {
|
|
195
|
+
if (Utils.$$Array.isEmpty(names)) {
|
|
196
|
+
return empty;
|
|
197
|
+
} else {
|
|
198
|
+
return names.join(", ");
|
|
199
|
+
}
|
|
200
|
+
};
|
|
201
|
+
if (Utils.$$Array.isEmpty(eventNames)) {
|
|
202
|
+
return Stdlib_JsError.throwWithMessage(`Contract "` + contractName + `" is not configured on any chain, so its handler for "` + eventName + `" would never run. Add it to your config, or remove the registration. Configured contracts: ` + listOr(match[0], "none") + `.`);
|
|
203
|
+
} else {
|
|
204
|
+
return Stdlib_JsError.throwWithMessage(`Event "` + eventName + `" is not configured on contract "` + contractName + `", so its handler would never run. Add it to your config, or remove the registration. Configured events on "` + contractName + `": ` + listOr(eventNames, "none") + `.`);
|
|
205
|
+
}
|
|
163
206
|
}
|
|
164
207
|
|
|
165
208
|
function setHandler(contractName, eventName, handler, eventOptions) {
|
package/src/InMemoryStore.res
CHANGED
|
@@ -45,7 +45,10 @@ let setEffectOutput = (
|
|
|
45
45
|
| Some(_) => ()
|
|
46
46
|
| None => inMemTable.changesCount = inMemTable.changesCount +. 1.
|
|
47
47
|
}
|
|
48
|
-
inMemTable.dict->Dict.set(
|
|
48
|
+
inMemTable.dict->Dict.set(
|
|
49
|
+
cacheKey,
|
|
50
|
+
Set({entityId: cacheKey->EntityId.unsafeOfString, entity: output, checkpointId}),
|
|
51
|
+
)
|
|
49
52
|
if shouldCache {
|
|
50
53
|
inMemTable.idsToStore->Array.push(cacheKey)->ignore
|
|
51
54
|
}
|
|
@@ -58,7 +61,11 @@ let initEffectOutputFromDb = (inMemTable: EffectState.effectCacheInMemTable, ~ca
|
|
|
58
61
|
inMemTable.changesCount = inMemTable.changesCount +. 1.
|
|
59
62
|
inMemTable.dict->Dict.set(
|
|
60
63
|
cacheKey,
|
|
61
|
-
Set({
|
|
64
|
+
Set({
|
|
65
|
+
entityId: cacheKey->EntityId.unsafeOfString,
|
|
66
|
+
entity: output,
|
|
67
|
+
checkpointId: Internal.loadedFromDbCheckpointId,
|
|
68
|
+
}),
|
|
62
69
|
)
|
|
63
70
|
}
|
|
64
71
|
|
|
@@ -131,7 +138,7 @@ let prepareRollbackDiff = async (
|
|
|
131
138
|
entityTable->InMemoryTable.Entity.set(
|
|
132
139
|
~committedCheckpointId,
|
|
133
140
|
Set({
|
|
134
|
-
entityId: entity.id,
|
|
141
|
+
entityId: entity.id->EntityId.unsafeOfString,
|
|
135
142
|
checkpointId: rollbackDiffCheckpointId,
|
|
136
143
|
entity,
|
|
137
144
|
}),
|
|
@@ -177,7 +184,7 @@ let setBatchDcs = (state: IndexerState.t, ~batch: Batch.t) => {
|
|
|
177
184
|
inMemTable->InMemoryTable.Entity.set(
|
|
178
185
|
~committedCheckpointId,
|
|
179
186
|
Set({
|
|
180
|
-
entityId: entity.id,
|
|
187
|
+
entityId: entity.id->EntityId.unsafeOfString,
|
|
181
188
|
checkpointId,
|
|
182
189
|
entity: entity->InternalTable.EnvioAddresses.castToInternal,
|
|
183
190
|
}),
|
package/src/InMemoryTable.res
CHANGED
|
@@ -17,11 +17,13 @@ module Entity = {
|
|
|
17
17
|
mutable filterIndices: filterIndices,
|
|
18
18
|
}
|
|
19
19
|
|
|
20
|
-
// Helper to extract entity
|
|
20
|
+
// Helper to extract an entity's id as a dict key. The raw id may be a
|
|
21
|
+
// string/int/bigint, so it's stringified to a stable key for in-memory
|
|
22
|
+
// indexing.
|
|
21
23
|
exception UnexpectedIdNotDefinedOnEntity
|
|
22
24
|
let getEntityIdUnsafe = (entity: Internal.entity): string =>
|
|
23
|
-
switch (entity->(Utils.magic: Internal.entity => {"id": option<
|
|
24
|
-
| Some(id) => id
|
|
25
|
+
switch (entity->(Utils.magic: Internal.entity => {"id": option<EntityId.t>}))["id"] {
|
|
26
|
+
| Some(id) => id->EntityId.toKey
|
|
25
27
|
| None =>
|
|
26
28
|
UnexpectedIdNotDefinedOnEntity->ErrorHandling.mkLogAndRaise(
|
|
27
29
|
~msg="Property 'id' does not exist on expected entity object",
|
|
@@ -142,8 +144,8 @@ module Entity = {
|
|
|
142
144
|
}
|
|
143
145
|
|
|
144
146
|
let set = (inMemTable: t, ~committedCheckpointId, change: Change.t<Internal.entity>) => {
|
|
145
|
-
let
|
|
146
|
-
switch inMemTable.latestEntityChangeById->Utils.Dict.dangerouslyGetNonOption(
|
|
147
|
+
let entityKey = change->Change.getEntityId->EntityId.toKey
|
|
148
|
+
switch inMemTable.latestEntityChangeById->Utils.Dict.dangerouslyGetNonOption(entityKey) {
|
|
147
149
|
| Some(prev) =>
|
|
148
150
|
let prevCheckpointId = prev->Change.getCheckpointId
|
|
149
151
|
if (
|
|
@@ -158,9 +160,9 @@ module Entity = {
|
|
|
158
160
|
|
|
159
161
|
switch change {
|
|
160
162
|
| Set({entity}) => inMemTable->updateIndices(~entity)
|
|
161
|
-
| Delete({entityId}) => inMemTable->deleteEntityFromIndices(~entityId)
|
|
163
|
+
| Delete({entityId}) => inMemTable->deleteEntityFromIndices(~entityId=entityId->EntityId.toKey)
|
|
162
164
|
}
|
|
163
|
-
inMemTable.latestEntityChangeById->Dict.set(
|
|
165
|
+
inMemTable.latestEntityChangeById->Dict.set(entityKey, change)
|
|
164
166
|
}
|
|
165
167
|
|
|
166
168
|
// Only writes when the id isn't already present, so set always takes its
|
|
@@ -172,10 +174,10 @@ module Entity = {
|
|
|
172
174
|
~entity: option<Internal.entity>,
|
|
173
175
|
) =>
|
|
174
176
|
if inMemTable.latestEntityChangeById->Utils.Dict.dangerouslyGetNonOption(key)->Option.isNone {
|
|
177
|
+
let entityId = key->EntityId.unsafeOfString
|
|
175
178
|
let change: Change.t<Internal.entity> = switch entity {
|
|
176
|
-
| Some(entity) =>
|
|
177
|
-
|
|
178
|
-
| None => Delete({entityId: key, checkpointId: Internal.loadedFromDbCheckpointId})
|
|
179
|
+
| Some(entity) => Set({entityId, entity, checkpointId: Internal.loadedFromDbCheckpointId})
|
|
180
|
+
| None => Delete({entityId, checkpointId: Internal.loadedFromDbCheckpointId})
|
|
179
181
|
}
|
|
180
182
|
inMemTable->set(~committedCheckpointId, change)
|
|
181
183
|
}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
// Generated by ReScript, PLEASE EDIT WITH CARE
|
|
2
2
|
|
|
3
3
|
import * as Utils from "./Utils.res.mjs";
|
|
4
|
+
import * as EntityId from "./EntityId.res.mjs";
|
|
4
5
|
import * as Internal from "./Internal.res.mjs";
|
|
5
6
|
import * as EntityFilter from "./db/EntityFilter.res.mjs";
|
|
6
7
|
import * as Stdlib_Array from "@rescript/runtime/lib/es6/Stdlib_Array.js";
|
|
@@ -15,7 +16,7 @@ let UnexpectedIdNotDefinedOnEntity = /* @__PURE__ */Primitive_exceptions.create(
|
|
|
15
16
|
function getEntityIdUnsafe(entity) {
|
|
16
17
|
let id = entity.id;
|
|
17
18
|
if (id !== undefined) {
|
|
18
|
-
return id;
|
|
19
|
+
return EntityId.toKey(Primitive_option.valFromOption(id));
|
|
19
20
|
} else {
|
|
20
21
|
return ErrorHandling.mkLogAndRaise(undefined, "Property 'id' does not exist on expected entity object", {
|
|
21
22
|
RE_EXN_ID: UnexpectedIdNotDefinedOnEntity
|
|
@@ -126,8 +127,8 @@ function deleteEntityFromIndices(self, entityId) {
|
|
|
126
127
|
}
|
|
127
128
|
|
|
128
129
|
function set(inMemTable, committedCheckpointId, change) {
|
|
129
|
-
let
|
|
130
|
-
let prev = inMemTable.latestEntityChangeById[
|
|
130
|
+
let entityKey = EntityId.toKey(change.entityId);
|
|
131
|
+
let prev = inMemTable.latestEntityChangeById[entityKey];
|
|
131
132
|
if (prev !== undefined) {
|
|
132
133
|
let prevCheckpointId = prev.checkpointId;
|
|
133
134
|
if (prevCheckpointId > committedCheckpointId && prevCheckpointId < change.checkpointId) {
|
|
@@ -140,9 +141,9 @@ function set(inMemTable, committedCheckpointId, change) {
|
|
|
140
141
|
if (change.type === "SET") {
|
|
141
142
|
updateIndices(inMemTable, change.entity);
|
|
142
143
|
} else {
|
|
143
|
-
deleteEntityFromIndices(inMemTable, change.entityId);
|
|
144
|
+
deleteEntityFromIndices(inMemTable, EntityId.toKey(change.entityId));
|
|
144
145
|
}
|
|
145
|
-
inMemTable.latestEntityChangeById[
|
|
146
|
+
inMemTable.latestEntityChangeById[entityKey] = change;
|
|
146
147
|
}
|
|
147
148
|
|
|
148
149
|
function initValue(inMemTable, committedCheckpointId, key, entity) {
|
package/src/IndexerState.res
CHANGED
|
@@ -107,6 +107,10 @@ type t = {
|
|
|
107
107
|
crossChainState: CrossChainState.t,
|
|
108
108
|
mutable rollbackState: rollbackState,
|
|
109
109
|
indexerStartTime: Date.t,
|
|
110
|
+
// Monotonic peer of indexerStartTime. Elapsed run time is derived from this so
|
|
111
|
+
// an NTP correction can't move it out of step with the Performance-based
|
|
112
|
+
// counters that get divided by it.
|
|
113
|
+
indexerStartTimeRef: Performance.timeRef,
|
|
110
114
|
// When an entity's history was last pruned. No key = never pruned yet,
|
|
111
115
|
// which counts as overdue.
|
|
112
116
|
lastPrunedAtMillis: dict<float>,
|
|
@@ -133,6 +137,12 @@ type t = {
|
|
|
133
137
|
// --- Metric counters, rendered by Metrics at scrape time. ---
|
|
134
138
|
mutable preloadSeconds: float,
|
|
135
139
|
mutable processingSeconds: float,
|
|
140
|
+
mutable processingStalledOnFetchSeconds: float,
|
|
141
|
+
// Set while the processing loop is idle after starving on an empty buffer;
|
|
142
|
+
// None while processing or when the loop stopped for a reorg/shutdown. The
|
|
143
|
+
// getter folds the in-progress interval in so a scrape mid-stall isn't stale.
|
|
144
|
+
mutable processingStalledOnFetchSince: option<Performance.timeRef>,
|
|
145
|
+
mutable processingStalledOnStorageWriteSeconds: float,
|
|
136
146
|
handlerStats: dict<handlerStat>,
|
|
137
147
|
storageLoadStats: dict<storageLoadStat>,
|
|
138
148
|
storageWriteStats: dict<storageWriteStat>,
|
|
@@ -194,6 +204,7 @@ let make = (
|
|
|
194
204
|
~targetBufferSize,
|
|
195
205
|
),
|
|
196
206
|
indexerStartTime: Date.make(),
|
|
207
|
+
indexerStartTimeRef: Performance.now(),
|
|
197
208
|
rollbackState: NoRollback,
|
|
198
209
|
lastPrunedAtMillis: Dict.make(),
|
|
199
210
|
loadManager: LoadManager.make(),
|
|
@@ -206,6 +217,9 @@ let make = (
|
|
|
206
217
|
simulateDeadInputTracker: SimulateDeadInputTracker.makeFromConfig(config),
|
|
207
218
|
preloadSeconds: 0.,
|
|
208
219
|
processingSeconds: 0.,
|
|
220
|
+
processingStalledOnFetchSeconds: 0.,
|
|
221
|
+
processingStalledOnFetchSince: None,
|
|
222
|
+
processingStalledOnStorageWriteSeconds: 0.,
|
|
209
223
|
handlerStats: Dict.make(),
|
|
210
224
|
storageLoadStats: Dict.make(),
|
|
211
225
|
storageWriteStats: Dict.make(),
|
|
@@ -315,8 +329,22 @@ let isResolvingReorg = (state: t) =>
|
|
|
315
329
|
// reports the first error so redundant handlers (eg an error caught in two
|
|
316
330
|
// nested scopes) don't double-report.
|
|
317
331
|
@inline
|
|
332
|
+
let // Close an open fetch-stall interval, accruing it into the counter. Called
|
|
333
|
+
// whenever the reason for the idle changes, so the interval never spans into
|
|
334
|
+
// time another counter owns — or, at shutdown, past the point where the loops
|
|
335
|
+
// stop and nothing would ever close it.
|
|
336
|
+
settleStalledOnFetch = (state: t) =>
|
|
337
|
+
switch state.processingStalledOnFetchSince {
|
|
338
|
+
| Some(since) =>
|
|
339
|
+
state.processingStalledOnFetchSeconds =
|
|
340
|
+
state.processingStalledOnFetchSeconds +. since->Performance.secondsSince
|
|
341
|
+
state.processingStalledOnFetchSince = None
|
|
342
|
+
| None => ()
|
|
343
|
+
}
|
|
344
|
+
|
|
318
345
|
let errorExit = (state: t, errHandler) =>
|
|
319
346
|
if !state.isStopped {
|
|
347
|
+
state->settleStalledOnFetch
|
|
320
348
|
state.isStopped = true
|
|
321
349
|
state.onError(errHandler)
|
|
322
350
|
}
|
|
@@ -325,7 +353,10 @@ let unexpectedErrorMsg = "Indexer has failed with an unexpected error"
|
|
|
325
353
|
|
|
326
354
|
// Halt the loops without reporting an error, eg to hand the shared db over to a
|
|
327
355
|
// resumed indexer in tests.
|
|
328
|
-
let stop = (state: t) =>
|
|
356
|
+
let stop = (state: t) => {
|
|
357
|
+
state->settleStalledOnFetch
|
|
358
|
+
state.isStopped = true
|
|
359
|
+
}
|
|
329
360
|
|
|
330
361
|
let getChainState = (state: t, ~chain: chain): ChainState.t =>
|
|
331
362
|
switch state.crossChainState
|
|
@@ -360,6 +391,9 @@ let enterReorgThreshold = (state: t) => state.crossChainState->CrossChainState.e
|
|
|
360
391
|
// caller has already mutated the chain states (restored counters, reset pending
|
|
361
392
|
// queries). isResolvingReorg derives from rollbackState.
|
|
362
393
|
let beginReorg = (state: t, ~chain, ~blockNumber) => {
|
|
394
|
+
// Settle here, or the rollback that follows would be folded into the stall on
|
|
395
|
+
// the next beginProcessing — time envio_rollback_seconds already counts.
|
|
396
|
+
state->settleStalledOnFetch
|
|
363
397
|
state.epoch = state.epoch + 1
|
|
364
398
|
state.rollbackState = ReorgDetected({chain, blockNumber})
|
|
365
399
|
}
|
|
@@ -391,9 +425,24 @@ let applyBatchProgress = (state: t, ~batch: Batch.t) =>
|
|
|
391
425
|
// Processing-loop mutex. Guards ProcessEventBatch re-entry so only one
|
|
392
426
|
// processing loop runs at a time.
|
|
393
427
|
let isProcessing = (state: t) => state.isProcessing
|
|
394
|
-
let beginProcessing = (state: t) =>
|
|
428
|
+
let beginProcessing = (state: t) => {
|
|
429
|
+
state->settleStalledOnFetch
|
|
430
|
+
state.isProcessing = true
|
|
431
|
+
}
|
|
395
432
|
let endProcessing = (state: t) => state.isProcessing = false
|
|
396
433
|
|
|
434
|
+
// Start attributing idle time to fetch starvation. Called when the processing
|
|
435
|
+
// loop exits with no work left; skipped when it exits for a reorg or shutdown,
|
|
436
|
+
// so only genuine buffer starvation is counted.
|
|
437
|
+
let markProcessingStalledOnFetch = (state: t) =>
|
|
438
|
+
if state.processingStalledOnFetchSince->Option.isNone {
|
|
439
|
+
state.processingStalledOnFetchSince = Some(Performance.now())
|
|
440
|
+
}
|
|
441
|
+
|
|
442
|
+
let recordStalledOnStorageWrite = (state: t, ~seconds) =>
|
|
443
|
+
state.processingStalledOnStorageWriteSeconds =
|
|
444
|
+
state.processingStalledOnStorageWriteSeconds +. seconds
|
|
445
|
+
|
|
397
446
|
let recordProcessedBatch = (state: t) =>
|
|
398
447
|
state.processedBatchesCount = state.processedBatchesCount + 1
|
|
399
448
|
|
|
@@ -469,12 +518,20 @@ let toMetrics = (state: t): Metrics.t => {
|
|
|
469
518
|
)
|
|
470
519
|
{
|
|
471
520
|
startTime: state.indexerStartTime,
|
|
521
|
+
metricTime: Date.make(),
|
|
522
|
+
elapsedSeconds: state.indexerStartTimeRef->Performance.secondsSince,
|
|
472
523
|
targetBufferSize: state.crossChainState->CrossChainState.targetBufferSize,
|
|
473
524
|
isInReorgThreshold: state.crossChainState->CrossChainState.isInReorgThreshold,
|
|
474
525
|
rollbackEnabled: state.config.shouldRollbackOnReorg,
|
|
475
526
|
maxBatchSize: state.config.batchSize,
|
|
476
527
|
preloadSeconds: state.preloadSeconds,
|
|
477
528
|
processingSeconds: state.processingSeconds,
|
|
529
|
+
processingStalledOnFetchSeconds: state.processingStalledOnFetchSeconds +.
|
|
530
|
+
switch state.processingStalledOnFetchSince {
|
|
531
|
+
| Some(since) => since->Performance.secondsSince
|
|
532
|
+
| None => 0.
|
|
533
|
+
},
|
|
534
|
+
processingStalledOnStorageWriteSeconds: state.processingStalledOnStorageWriteSeconds,
|
|
478
535
|
rollbackSeconds: state.rollbackSeconds,
|
|
479
536
|
rollbackCount: state.rollbackCount,
|
|
480
537
|
rollbackEventsCount: state.rollbackEventsCount,
|
package/src/IndexerState.res.mjs
CHANGED
|
@@ -76,6 +76,7 @@ function make$1(config, persistence, chainStates, isInReorgThreshold, isRealtime
|
|
|
76
76
|
crossChainState: CrossChainState.make(chainStates, isInReorgThreshold, isRealtime, targetBufferSize),
|
|
77
77
|
rollbackState: "NoRollback",
|
|
78
78
|
indexerStartTime: new Date(),
|
|
79
|
+
indexerStartTimeRef: Performance.now(),
|
|
79
80
|
lastPrunedAtMillis: {},
|
|
80
81
|
loadManager: LoadManager.make(),
|
|
81
82
|
keepProcessAlive: isDevelopmentMode || shouldUseTui,
|
|
@@ -87,6 +88,9 @@ function make$1(config, persistence, chainStates, isInReorgThreshold, isRealtime
|
|
|
87
88
|
simulateDeadInputTracker: SimulateDeadInputTracker.makeFromConfig(config),
|
|
88
89
|
preloadSeconds: 0,
|
|
89
90
|
processingSeconds: 0,
|
|
91
|
+
processingStalledOnFetchSeconds: 0,
|
|
92
|
+
processingStalledOnFetchSince: undefined,
|
|
93
|
+
processingStalledOnStorageWriteSeconds: 0,
|
|
90
94
|
handlerStats: {},
|
|
91
95
|
storageLoadStats: {},
|
|
92
96
|
storageWriteStats: {},
|
|
@@ -145,13 +149,24 @@ function isResolvingReorg(state) {
|
|
|
145
149
|
}
|
|
146
150
|
|
|
147
151
|
function errorExit(state, errHandler) {
|
|
148
|
-
if (
|
|
149
|
-
|
|
150
|
-
return state.onError(errHandler);
|
|
152
|
+
if (state.isStopped) {
|
|
153
|
+
return;
|
|
151
154
|
}
|
|
155
|
+
let since = state.processingStalledOnFetchSince;
|
|
156
|
+
if (since !== undefined) {
|
|
157
|
+
state.processingStalledOnFetchSeconds = state.processingStalledOnFetchSeconds + Performance.secondsSince(since);
|
|
158
|
+
state.processingStalledOnFetchSince = undefined;
|
|
159
|
+
}
|
|
160
|
+
state.isStopped = true;
|
|
161
|
+
state.onError(errHandler);
|
|
152
162
|
}
|
|
153
163
|
|
|
154
164
|
function stop(state) {
|
|
165
|
+
let since = state.processingStalledOnFetchSince;
|
|
166
|
+
if (since !== undefined) {
|
|
167
|
+
state.processingStalledOnFetchSeconds = state.processingStalledOnFetchSeconds + Performance.secondsSince(since);
|
|
168
|
+
state.processingStalledOnFetchSince = undefined;
|
|
169
|
+
}
|
|
155
170
|
state.isStopped = true;
|
|
156
171
|
}
|
|
157
172
|
|
|
@@ -177,6 +192,11 @@ function enterReorgThreshold(state) {
|
|
|
177
192
|
}
|
|
178
193
|
|
|
179
194
|
function beginReorg(state, chain, blockNumber) {
|
|
195
|
+
let since = state.processingStalledOnFetchSince;
|
|
196
|
+
if (since !== undefined) {
|
|
197
|
+
state.processingStalledOnFetchSeconds = state.processingStalledOnFetchSeconds + Performance.secondsSince(since);
|
|
198
|
+
state.processingStalledOnFetchSince = undefined;
|
|
199
|
+
}
|
|
180
200
|
state.epoch = state.epoch + 1 | 0;
|
|
181
201
|
state.rollbackState = {
|
|
182
202
|
TAG: "ReorgDetected",
|
|
@@ -221,6 +241,11 @@ function isProcessing(state) {
|
|
|
221
241
|
}
|
|
222
242
|
|
|
223
243
|
function beginProcessing(state) {
|
|
244
|
+
let since = state.processingStalledOnFetchSince;
|
|
245
|
+
if (since !== undefined) {
|
|
246
|
+
state.processingStalledOnFetchSeconds = state.processingStalledOnFetchSeconds + Performance.secondsSince(since);
|
|
247
|
+
state.processingStalledOnFetchSince = undefined;
|
|
248
|
+
}
|
|
224
249
|
state.isProcessing = true;
|
|
225
250
|
}
|
|
226
251
|
|
|
@@ -228,6 +253,17 @@ function endProcessing(state) {
|
|
|
228
253
|
state.isProcessing = false;
|
|
229
254
|
}
|
|
230
255
|
|
|
256
|
+
function markProcessingStalledOnFetch(state) {
|
|
257
|
+
if (Stdlib_Option.isNone(state.processingStalledOnFetchSince)) {
|
|
258
|
+
state.processingStalledOnFetchSince = Performance.now();
|
|
259
|
+
return;
|
|
260
|
+
}
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
function recordStalledOnStorageWrite(state, seconds) {
|
|
264
|
+
state.processingStalledOnStorageWriteSeconds = state.processingStalledOnStorageWriteSeconds + seconds;
|
|
265
|
+
}
|
|
266
|
+
|
|
231
267
|
function recordProcessedBatch(state) {
|
|
232
268
|
state.processedBatchesCount = state.processedBatchesCount + 1 | 0;
|
|
233
269
|
}
|
|
@@ -371,14 +407,21 @@ function toMetrics(state) {
|
|
|
371
407
|
count: s.count
|
|
372
408
|
});
|
|
373
409
|
});
|
|
410
|
+
let since = state.processingStalledOnFetchSince;
|
|
374
411
|
return {
|
|
375
412
|
startTime: state.indexerStartTime,
|
|
413
|
+
metricTime: new Date(),
|
|
414
|
+
elapsedSeconds: Performance.secondsSince(state.indexerStartTimeRef),
|
|
376
415
|
targetBufferSize: CrossChainState.targetBufferSize(state.crossChainState),
|
|
377
416
|
isInReorgThreshold: CrossChainState.isInReorgThreshold(state.crossChainState),
|
|
378
417
|
rollbackEnabled: state.config.shouldRollbackOnReorg,
|
|
379
418
|
maxBatchSize: state.config.batchSize,
|
|
380
419
|
preloadSeconds: state.preloadSeconds,
|
|
381
420
|
processingSeconds: state.processingSeconds,
|
|
421
|
+
processingStalledOnFetchSeconds: state.processingStalledOnFetchSeconds + (
|
|
422
|
+
since !== undefined ? Performance.secondsSince(since) : 0
|
|
423
|
+
),
|
|
424
|
+
processingStalledOnStorageWriteSeconds: state.processingStalledOnStorageWriteSeconds,
|
|
382
425
|
rollbackSeconds: state.rollbackSeconds,
|
|
383
426
|
rollbackCount: state.rollbackCount,
|
|
384
427
|
rollbackEventsCount: state.rollbackEventsCount,
|
|
@@ -698,6 +741,7 @@ export {
|
|
|
698
741
|
isProcessing,
|
|
699
742
|
beginProcessing,
|
|
700
743
|
endProcessing,
|
|
744
|
+
markProcessingStalledOnFetch,
|
|
701
745
|
recordProcessedBatch,
|
|
702
746
|
createBatch,
|
|
703
747
|
getSafeCheckpointId,
|
|
@@ -730,6 +774,7 @@ export {
|
|
|
730
774
|
simulateDeadInputTracker,
|
|
731
775
|
toMetrics,
|
|
732
776
|
recordBatchDurations,
|
|
777
|
+
recordStalledOnStorageWrite,
|
|
733
778
|
recordHandlerDuration,
|
|
734
779
|
startPreloadHandler,
|
|
735
780
|
endPreloadHandler,
|
package/src/IndexerState.resi
CHANGED
|
@@ -66,6 +66,7 @@ let applyBatchProgress: (t, ~batch: Batch.t) => unit
|
|
|
66
66
|
let isProcessing: t => bool
|
|
67
67
|
let beginProcessing: t => unit
|
|
68
68
|
let endProcessing: t => unit
|
|
69
|
+
let markProcessingStalledOnFetch: t => unit
|
|
69
70
|
let recordProcessedBatch: t => unit
|
|
70
71
|
let createBatch: (
|
|
71
72
|
t,
|
|
@@ -110,6 +111,7 @@ let toMetrics: t => Metrics.t
|
|
|
110
111
|
|
|
111
112
|
// Metric counters.
|
|
112
113
|
let recordBatchDurations: (t, ~loadDuration: float, ~handlerDuration: float) => unit
|
|
114
|
+
let recordStalledOnStorageWrite: (t, ~seconds: float) => unit
|
|
113
115
|
let recordHandlerDuration: (t, ~contract: string, ~event: string, ~duration: float) => unit
|
|
114
116
|
let startPreloadHandler: (t, ~contract: string, ~event: string) => Performance.timeRef
|
|
115
117
|
let endPreloadHandler: (t, Performance.timeRef, ~contract: string, ~event: string) => unit
|
package/src/Internal.res
CHANGED
|
@@ -333,6 +333,11 @@ type eventPayload
|
|
|
333
333
|
// The log's emitting address (EVM/Fuel; the program id carries it for SVM).
|
|
334
334
|
@get external getPayloadSrcAddress: eventPayload => Address.t = "srcAddress"
|
|
335
335
|
|
|
336
|
+
// The decoded params, read by name for the address-valued ones a `where`
|
|
337
|
+
// filters on. Only those names are ever looked up, so the address type is
|
|
338
|
+
// accurate at every use site.
|
|
339
|
+
@get external getPayloadAddressParams: eventPayload => dict<Address.t> = "params"
|
|
340
|
+
|
|
336
341
|
type genericLoaderArgs<'event, 'context> = {
|
|
337
342
|
event: 'event,
|
|
338
343
|
context: 'context,
|
|
@@ -356,11 +361,11 @@ type genericHandlerArgs<'event, 'context> = {
|
|
|
356
361
|
type genericHandler<'args> = 'args => promise<unit>
|
|
357
362
|
|
|
358
363
|
type entityHandlerContext<'entity> = {
|
|
359
|
-
get:
|
|
360
|
-
getOrThrow: (
|
|
364
|
+
get: EntityId.t => promise<option<'entity>>,
|
|
365
|
+
getOrThrow: (EntityId.t, ~message: string=?) => promise<'entity>,
|
|
361
366
|
getOrCreate: 'entity => promise<'entity>,
|
|
362
367
|
set: 'entity => unit,
|
|
363
|
-
deleteUnsafe:
|
|
368
|
+
deleteUnsafe: EntityId.t => unit,
|
|
364
369
|
}
|
|
365
370
|
|
|
366
371
|
type chainInfo = {
|
|
@@ -572,11 +577,15 @@ type onEventRegistration = {
|
|
|
572
577
|
// Usually always false for wildcard events, but might be true for a wildcard
|
|
573
578
|
// event with a dynamic event filter by addresses.
|
|
574
579
|
dependsOnAddresses: bool,
|
|
575
|
-
//
|
|
576
|
-
//
|
|
577
|
-
//
|
|
578
|
-
//
|
|
579
|
-
|
|
580
|
+
// Indexed address params this event filters on, in disjunctive normal form
|
|
581
|
+
// (OR of AND-groups), from `where: {params: {to: chain.C.addresses}}`. Every
|
|
582
|
+
// source applies this natively while routing; it's carried here for the
|
|
583
|
+
// simulate source, which has no native query boundary. Absent otherwise.
|
|
584
|
+
//
|
|
585
|
+
// Keep it optional: with every field required, ReScript compiles a
|
|
586
|
+
// `{...registration, ...}` spread into an explicit field-by-field copy, which
|
|
587
|
+
// drops the ecosystem-only fields an `evmOnEventRegistration` carries.
|
|
588
|
+
addressFilterParamGroups?: array<array<string>>,
|
|
580
589
|
// Final start block: the contract/chain config value, overridden by a
|
|
581
590
|
// `where.block.number._gte` when the registered `where` supplies one.
|
|
582
591
|
startBlock: option<int>,
|
package/src/LogSelection.res
CHANGED
|
@@ -23,7 +23,8 @@ type parsedWhere = {
|
|
|
23
23
|
filterByAddresses: bool,
|
|
24
24
|
// Indexed params filtered by `chain.<Contract>.addresses`, in disjunctive
|
|
25
25
|
// normal form (outer array OR of AND-groups). Empty unless `filterByAddresses`.
|
|
26
|
-
//
|
|
26
|
+
// Applied natively by every source while routing; carried on the registration
|
|
27
|
+
// for the simulate source, which has no native query boundary.
|
|
27
28
|
addressFilterParamGroups: array<array<string>>,
|
|
28
29
|
}
|
|
29
30
|
|