envio 3.5.0-alpha.1 → 3.5.0-alpha.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (86) hide show
  1. package/index.d.ts +10 -6
  2. package/package.json +6 -6
  3. package/src/BatchProcessing.res +13 -0
  4. package/src/BatchProcessing.res.mjs +4 -0
  5. package/src/ChainFetching.res +5 -5
  6. package/src/ChainFetching.res.mjs +3 -4
  7. package/src/ChainState.res +38 -21
  8. package/src/ChainState.res.mjs +17 -24
  9. package/src/ChainState.resi +1 -5
  10. package/src/Change.res +3 -3
  11. package/src/Config.res +0 -4
  12. package/src/Config.res.mjs +0 -10
  13. package/src/Core.res +3 -0
  14. package/src/EntityId.res +15 -0
  15. package/src/EntityId.res.mjs +9 -0
  16. package/src/EventConfigBuilder.res +1 -58
  17. package/src/EventConfigBuilder.res.mjs +1 -33
  18. package/src/FetchState.res +345 -459
  19. package/src/FetchState.res.mjs +269 -430
  20. package/src/HandlerRegister.res +45 -0
  21. package/src/HandlerRegister.res.mjs +43 -0
  22. package/src/InMemoryStore.res +11 -4
  23. package/src/InMemoryTable.res +12 -10
  24. package/src/InMemoryTable.res.mjs +6 -5
  25. package/src/IndexerState.res +59 -2
  26. package/src/IndexerState.res.mjs +48 -3
  27. package/src/IndexerState.resi +2 -0
  28. package/src/Internal.res +17 -8
  29. package/src/LogSelection.res +2 -1
  30. package/src/Metrics.res +37 -6
  31. package/src/Metrics.res.mjs +5 -1
  32. package/src/Persistence.res +1 -1
  33. package/src/PgStorage.res +55 -22
  34. package/src/PgStorage.res.mjs +25 -17
  35. package/src/SimulateItems.res +2 -2
  36. package/src/TestIndexer.res +6 -5
  37. package/src/TestIndexer.res.mjs +3 -2
  38. package/src/UserContext.res +3 -3
  39. package/src/Writing.res +12 -2
  40. package/src/Writing.res.mjs +13 -2
  41. package/src/bindings/ClickHouse.res +5 -3
  42. package/src/bindings/ClickHouse.res.mjs +2 -5
  43. package/src/bindings/Vitest.res +22 -1
  44. package/src/bindings/Vitest.res.mjs +15 -0
  45. package/src/db/EntityHistory.res +16 -7
  46. package/src/db/EntityHistory.res.mjs +7 -6
  47. package/src/db/Table.res +40 -9
  48. package/src/db/Table.res.mjs +44 -6
  49. package/src/sources/AddressSet.res +48 -0
  50. package/src/sources/AddressSet.res.mjs +11 -0
  51. package/src/sources/AddressStore.res +152 -0
  52. package/src/sources/AddressStore.res.mjs +97 -0
  53. package/src/sources/EvmChain.res +3 -0
  54. package/src/sources/EvmChain.res.mjs +4 -2
  55. package/src/sources/EvmHyperSyncSource.res +6 -3
  56. package/src/sources/EvmHyperSyncSource.res.mjs +3 -3
  57. package/src/sources/EvmRpcClient.res +6 -3
  58. package/src/sources/EvmRpcClient.res.mjs +3 -3
  59. package/src/sources/FuelHyperSync.res +4 -3
  60. package/src/sources/FuelHyperSync.res.mjs +3 -3
  61. package/src/sources/FuelHyperSync.resi +2 -1
  62. package/src/sources/FuelHyperSyncClient.res +12 -6
  63. package/src/sources/FuelHyperSyncClient.res.mjs +2 -2
  64. package/src/sources/FuelHyperSyncSource.res +7 -4
  65. package/src/sources/FuelHyperSyncSource.res.mjs +3 -3
  66. package/src/sources/HyperSync.res +6 -4
  67. package/src/sources/HyperSync.res.mjs +2 -3
  68. package/src/sources/HyperSync.resi +1 -1
  69. package/src/sources/HyperSyncClient.res +18 -9
  70. package/src/sources/HyperSyncClient.res.mjs +4 -4
  71. package/src/sources/RpcSource.res +15 -10
  72. package/src/sources/RpcSource.res.mjs +3 -4
  73. package/src/sources/SimulateSource.res +26 -11
  74. package/src/sources/SimulateSource.res.mjs +13 -5
  75. package/src/sources/Source.res +4 -2
  76. package/src/sources/SourceManager.res +2 -3
  77. package/src/sources/SourceManager.res.mjs +2 -3
  78. package/src/sources/Svm.res +1 -2
  79. package/src/sources/Svm.res.mjs +1 -1
  80. package/src/sources/SvmHyperSyncClient.res +12 -3
  81. package/src/sources/SvmHyperSyncClient.res.mjs +2 -2
  82. package/src/sources/SvmHyperSyncSource.res +10 -5
  83. package/src/sources/SvmHyperSyncSource.res.mjs +5 -4
  84. package/src/IndexingAddresses.res +0 -151
  85. package/src/IndexingAddresses.res.mjs +0 -130
  86. package/src/IndexingAddresses.resi +0 -39
@@ -215,6 +215,25 @@ let mergeRegistrations = (resolved: array<Internal.onEventRegistration>, ~config
215
215
  let isDroppedByWhere = (~config: Config.t, reg: Internal.onEventRegistration) =>
216
216
  config.ecosystem.name === Evm && (reg->getResolvedWhere).topicSelections->Utils.Array.isEmpty
217
217
 
218
+ // Names configured across every chain, used to explain a registration that
219
+ // matched nothing. Only built on the failure path.
220
+ let describeConfigured = (registration: activeRegistration, ~contractName) => {
221
+ let contractNames = Utils.Set.make()
222
+ let eventNames = Utils.Set.make()
223
+ registration.config.chainMap
224
+ ->ChainMap.values
225
+ ->Array.forEach(chainConfig =>
226
+ chainConfig.contracts->Array.forEach(contract => {
227
+ contractNames->Utils.Set.add(contract.name)->ignore
228
+ if contract.name === contractName {
229
+ contract.events->Array.forEach(e => eventNames->Utils.Set.add(e.name)->ignore)
230
+ }
231
+ })
232
+ )
233
+ let sorted = set => set->Utils.Set.toArray->Array.toSorted(String.compare)
234
+ (sorted(contractNames), sorted(eventNames))
235
+ }
236
+
218
237
  // Resolve one `onEvent`/`contractRegister` call into a registration for every
219
238
  // chain in the config (the full chain set) and store it in registration order.
220
239
  // Building runs the user's `where` callback here — once per chain — so a broken
@@ -230,6 +249,7 @@ let addOnEventRegistration = (
230
249
  ) => {
231
250
  let isWildcard = eventOptions->Option.flatMap(v => v.wildcard)->Option.getOr(false)
232
251
  let where = eventOptions->Option.flatMap(v => v.where)
252
+ let matched = ref(false)
233
253
  registration.config.chainMap
234
254
  ->ChainMap.values
235
255
  ->Array.forEach(chainConfig =>
@@ -239,6 +259,7 @@ let addOnEventRegistration = (
239
259
  switch contract.events->Array.find(e => e.name === eventName) {
240
260
  | None => ()
241
261
  | Some(eventConfig) =>
262
+ matched := true
242
263
  let reg = buildOnEventRegistrationWith(
243
264
  ~config=registration.config,
244
265
  ~chainId=chainConfig.id,
@@ -255,6 +276,30 @@ let addOnEventRegistration = (
255
276
  }
256
277
  }
257
278
  )
279
+
280
+ // Nothing matched on any chain, so the callback could never be dispatched.
281
+ // Reported at the registration call site, where the stack still points at the
282
+ // offending `onEvent`/`contractRegister`.
283
+ if !matched.contents {
284
+ let (contractNames, eventNames) = registration->describeConfigured(~contractName)
285
+ let listOr = (names, empty) =>
286
+ names->Utils.Array.isEmpty ? empty : names->Array.joinUnsafe(", ")
287
+ if eventNames->Utils.Array.isEmpty {
288
+ JsError.throwWithMessage(
289
+ `Contract "${contractName}" is not configured on any chain, so its handler for "${eventName}" would never run. Add it to your config, or remove the registration. Configured contracts: ${listOr(
290
+ contractNames,
291
+ "none",
292
+ )}.`,
293
+ )
294
+ } else {
295
+ JsError.throwWithMessage(
296
+ `Event "${eventName}" is not configured on contract "${contractName}", so its handler would never run. Add it to your config, or remove the registration. Configured events on "${contractName}": ${listOr(
297
+ eventNames,
298
+ "none",
299
+ )}.`,
300
+ )
301
+ }
302
+ }
258
303
  }
259
304
 
260
305
  let setHandler = (~contractName, ~eventName, handler, ~eventOptions) => {
@@ -8,6 +8,7 @@ import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
8
8
  import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
9
9
  import * as Primitive_object from "@rescript/runtime/lib/es6/Primitive_object.js";
10
10
  import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js";
11
+ import * as Primitive_string from "@rescript/runtime/lib/es6/Primitive_string.js";
11
12
  import * as S$RescriptSchema from "rescript-schema/src/S.res.mjs";
12
13
  import * as EventConfigBuilder from "./EventConfigBuilder.res.mjs";
13
14
  import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
@@ -145,9 +146,33 @@ function isDroppedByWhere(config, reg) {
145
146
  }
146
147
  }
147
148
 
149
+ function describeConfigured(registration, contractName) {
150
+ let contractNames = new Set();
151
+ let eventNames = new Set();
152
+ ChainMap.values(registration.config.chainMap).forEach(chainConfig => {
153
+ chainConfig.contracts.forEach(contract => {
154
+ contractNames.add(contract.name);
155
+ if (contract.name === contractName) {
156
+ contract.events.forEach(e => {
157
+ eventNames.add(e.name);
158
+ });
159
+ return;
160
+ }
161
+ });
162
+ });
163
+ let sorted = set => Array.from(set).toSorted(Primitive_string.compare);
164
+ return [
165
+ sorted(contractNames),
166
+ sorted(eventNames)
167
+ ];
168
+ }
169
+
148
170
  function addOnEventRegistration(registration, contractName, eventName, handler, contractRegister, eventOptions) {
149
171
  let isWildcard = Stdlib_Option.getOr(Stdlib_Option.flatMap(eventOptions, v => v.wildcard), false);
150
172
  let where = Stdlib_Option.flatMap(eventOptions, v => v.where);
173
+ let matched = {
174
+ contents: false
175
+ };
151
176
  ChainMap.values(registration.config.chainMap).forEach(chainConfig => {
152
177
  let contract = chainConfig.contracts.find(c => c.name === contractName);
153
178
  if (contract === undefined) {
@@ -157,9 +182,27 @@ function addOnEventRegistration(registration, contractName, eventName, handler,
157
182
  if (eventConfig === undefined) {
158
183
  return;
159
184
  }
185
+ matched.contents = true;
160
186
  let reg = buildOnEventRegistrationWith(registration.config, chainConfig.id, eventConfig, isWildcard, handler, contractRegister, where, contract.startBlock);
161
187
  getChainRegistrations(registration, chainConfig.id).onEventRegistrations.push(reg);
162
188
  });
189
+ if (matched.contents) {
190
+ return;
191
+ }
192
+ let match = describeConfigured(registration, contractName);
193
+ let eventNames = match[1];
194
+ let listOr = (names, empty) => {
195
+ if (Utils.$$Array.isEmpty(names)) {
196
+ return empty;
197
+ } else {
198
+ return names.join(", ");
199
+ }
200
+ };
201
+ if (Utils.$$Array.isEmpty(eventNames)) {
202
+ return Stdlib_JsError.throwWithMessage(`Contract "` + contractName + `" is not configured on any chain, so its handler for "` + eventName + `" would never run. Add it to your config, or remove the registration. Configured contracts: ` + listOr(match[0], "none") + `.`);
203
+ } else {
204
+ return Stdlib_JsError.throwWithMessage(`Event "` + eventName + `" is not configured on contract "` + contractName + `", so its handler would never run. Add it to your config, or remove the registration. Configured events on "` + contractName + `": ` + listOr(eventNames, "none") + `.`);
205
+ }
163
206
  }
164
207
 
165
208
  function setHandler(contractName, eventName, handler, eventOptions) {
@@ -45,7 +45,10 @@ let setEffectOutput = (
45
45
  | Some(_) => ()
46
46
  | None => inMemTable.changesCount = inMemTable.changesCount +. 1.
47
47
  }
48
- inMemTable.dict->Dict.set(cacheKey, Set({entityId: cacheKey, entity: output, checkpointId}))
48
+ inMemTable.dict->Dict.set(
49
+ cacheKey,
50
+ Set({entityId: cacheKey->EntityId.unsafeOfString, entity: output, checkpointId}),
51
+ )
49
52
  if shouldCache {
50
53
  inMemTable.idsToStore->Array.push(cacheKey)->ignore
51
54
  }
@@ -58,7 +61,11 @@ let initEffectOutputFromDb = (inMemTable: EffectState.effectCacheInMemTable, ~ca
58
61
  inMemTable.changesCount = inMemTable.changesCount +. 1.
59
62
  inMemTable.dict->Dict.set(
60
63
  cacheKey,
61
- Set({entityId: cacheKey, entity: output, checkpointId: Internal.loadedFromDbCheckpointId}),
64
+ Set({
65
+ entityId: cacheKey->EntityId.unsafeOfString,
66
+ entity: output,
67
+ checkpointId: Internal.loadedFromDbCheckpointId,
68
+ }),
62
69
  )
63
70
  }
64
71
 
@@ -131,7 +138,7 @@ let prepareRollbackDiff = async (
131
138
  entityTable->InMemoryTable.Entity.set(
132
139
  ~committedCheckpointId,
133
140
  Set({
134
- entityId: entity.id,
141
+ entityId: entity.id->EntityId.unsafeOfString,
135
142
  checkpointId: rollbackDiffCheckpointId,
136
143
  entity,
137
144
  }),
@@ -177,7 +184,7 @@ let setBatchDcs = (state: IndexerState.t, ~batch: Batch.t) => {
177
184
  inMemTable->InMemoryTable.Entity.set(
178
185
  ~committedCheckpointId,
179
186
  Set({
180
- entityId: entity.id,
187
+ entityId: entity.id->EntityId.unsafeOfString,
181
188
  checkpointId,
182
189
  entity: entity->InternalTable.EnvioAddresses.castToInternal,
183
190
  }),
@@ -17,11 +17,13 @@ module Entity = {
17
17
  mutable filterIndices: filterIndices,
18
18
  }
19
19
 
20
- // Helper to extract entity ID from any entity
20
+ // Helper to extract an entity's id as a dict key. The raw id may be a
21
+ // string/int/bigint, so it's stringified to a stable key for in-memory
22
+ // indexing.
21
23
  exception UnexpectedIdNotDefinedOnEntity
22
24
  let getEntityIdUnsafe = (entity: Internal.entity): string =>
23
- switch (entity->(Utils.magic: Internal.entity => {"id": option<string>}))["id"] {
24
- | Some(id) => id
25
+ switch (entity->(Utils.magic: Internal.entity => {"id": option<EntityId.t>}))["id"] {
26
+ | Some(id) => id->EntityId.toKey
25
27
  | None =>
26
28
  UnexpectedIdNotDefinedOnEntity->ErrorHandling.mkLogAndRaise(
27
29
  ~msg="Property 'id' does not exist on expected entity object",
@@ -142,8 +144,8 @@ module Entity = {
142
144
  }
143
145
 
144
146
  let set = (inMemTable: t, ~committedCheckpointId, change: Change.t<Internal.entity>) => {
145
- let entityId = change->Change.getEntityId
146
- switch inMemTable.latestEntityChangeById->Utils.Dict.dangerouslyGetNonOption(entityId) {
147
+ let entityKey = change->Change.getEntityId->EntityId.toKey
148
+ switch inMemTable.latestEntityChangeById->Utils.Dict.dangerouslyGetNonOption(entityKey) {
147
149
  | Some(prev) =>
148
150
  let prevCheckpointId = prev->Change.getCheckpointId
149
151
  if (
@@ -158,9 +160,9 @@ module Entity = {
158
160
 
159
161
  switch change {
160
162
  | Set({entity}) => inMemTable->updateIndices(~entity)
161
- | Delete({entityId}) => inMemTable->deleteEntityFromIndices(~entityId)
163
+ | Delete({entityId}) => inMemTable->deleteEntityFromIndices(~entityId=entityId->EntityId.toKey)
162
164
  }
163
- inMemTable.latestEntityChangeById->Dict.set(entityId, change)
165
+ inMemTable.latestEntityChangeById->Dict.set(entityKey, change)
164
166
  }
165
167
 
166
168
  // Only writes when the id isn't already present, so set always takes its
@@ -172,10 +174,10 @@ module Entity = {
172
174
  ~entity: option<Internal.entity>,
173
175
  ) =>
174
176
  if inMemTable.latestEntityChangeById->Utils.Dict.dangerouslyGetNonOption(key)->Option.isNone {
177
+ let entityId = key->EntityId.unsafeOfString
175
178
  let change: Change.t<Internal.entity> = switch entity {
176
- | Some(entity) =>
177
- Set({entityId: key, entity, checkpointId: Internal.loadedFromDbCheckpointId})
178
- | None => Delete({entityId: key, checkpointId: Internal.loadedFromDbCheckpointId})
179
+ | Some(entity) => Set({entityId, entity, checkpointId: Internal.loadedFromDbCheckpointId})
180
+ | None => Delete({entityId, checkpointId: Internal.loadedFromDbCheckpointId})
179
181
  }
180
182
  inMemTable->set(~committedCheckpointId, change)
181
183
  }
@@ -1,6 +1,7 @@
1
1
  // Generated by ReScript, PLEASE EDIT WITH CARE
2
2
 
3
3
  import * as Utils from "./Utils.res.mjs";
4
+ import * as EntityId from "./EntityId.res.mjs";
4
5
  import * as Internal from "./Internal.res.mjs";
5
6
  import * as EntityFilter from "./db/EntityFilter.res.mjs";
6
7
  import * as Stdlib_Array from "@rescript/runtime/lib/es6/Stdlib_Array.js";
@@ -15,7 +16,7 @@ let UnexpectedIdNotDefinedOnEntity = /* @__PURE__ */Primitive_exceptions.create(
15
16
  function getEntityIdUnsafe(entity) {
16
17
  let id = entity.id;
17
18
  if (id !== undefined) {
18
- return id;
19
+ return EntityId.toKey(Primitive_option.valFromOption(id));
19
20
  } else {
20
21
  return ErrorHandling.mkLogAndRaise(undefined, "Property 'id' does not exist on expected entity object", {
21
22
  RE_EXN_ID: UnexpectedIdNotDefinedOnEntity
@@ -126,8 +127,8 @@ function deleteEntityFromIndices(self, entityId) {
126
127
  }
127
128
 
128
129
  function set(inMemTable, committedCheckpointId, change) {
129
- let entityId = change.entityId;
130
- let prev = inMemTable.latestEntityChangeById[entityId];
130
+ let entityKey = EntityId.toKey(change.entityId);
131
+ let prev = inMemTable.latestEntityChangeById[entityKey];
131
132
  if (prev !== undefined) {
132
133
  let prevCheckpointId = prev.checkpointId;
133
134
  if (prevCheckpointId > committedCheckpointId && prevCheckpointId < change.checkpointId) {
@@ -140,9 +141,9 @@ function set(inMemTable, committedCheckpointId, change) {
140
141
  if (change.type === "SET") {
141
142
  updateIndices(inMemTable, change.entity);
142
143
  } else {
143
- deleteEntityFromIndices(inMemTable, change.entityId);
144
+ deleteEntityFromIndices(inMemTable, EntityId.toKey(change.entityId));
144
145
  }
145
- inMemTable.latestEntityChangeById[entityId] = change;
146
+ inMemTable.latestEntityChangeById[entityKey] = change;
146
147
  }
147
148
 
148
149
  function initValue(inMemTable, committedCheckpointId, key, entity) {
@@ -107,6 +107,10 @@ type t = {
107
107
  crossChainState: CrossChainState.t,
108
108
  mutable rollbackState: rollbackState,
109
109
  indexerStartTime: Date.t,
110
+ // Monotonic peer of indexerStartTime. Elapsed run time is derived from this so
111
+ // an NTP correction can't move it out of step with the Performance-based
112
+ // counters that get divided by it.
113
+ indexerStartTimeRef: Performance.timeRef,
110
114
  // When an entity's history was last pruned. No key = never pruned yet,
111
115
  // which counts as overdue.
112
116
  lastPrunedAtMillis: dict<float>,
@@ -133,6 +137,12 @@ type t = {
133
137
  // --- Metric counters, rendered by Metrics at scrape time. ---
134
138
  mutable preloadSeconds: float,
135
139
  mutable processingSeconds: float,
140
+ mutable processingStalledOnFetchSeconds: float,
141
+ // Set while the processing loop is idle after starving on an empty buffer;
142
+ // None while processing or when the loop stopped for a reorg/shutdown. The
143
+ // getter folds the in-progress interval in so a scrape mid-stall isn't stale.
144
+ mutable processingStalledOnFetchSince: option<Performance.timeRef>,
145
+ mutable processingStalledOnStorageWriteSeconds: float,
136
146
  handlerStats: dict<handlerStat>,
137
147
  storageLoadStats: dict<storageLoadStat>,
138
148
  storageWriteStats: dict<storageWriteStat>,
@@ -194,6 +204,7 @@ let make = (
194
204
  ~targetBufferSize,
195
205
  ),
196
206
  indexerStartTime: Date.make(),
207
+ indexerStartTimeRef: Performance.now(),
197
208
  rollbackState: NoRollback,
198
209
  lastPrunedAtMillis: Dict.make(),
199
210
  loadManager: LoadManager.make(),
@@ -206,6 +217,9 @@ let make = (
206
217
  simulateDeadInputTracker: SimulateDeadInputTracker.makeFromConfig(config),
207
218
  preloadSeconds: 0.,
208
219
  processingSeconds: 0.,
220
+ processingStalledOnFetchSeconds: 0.,
221
+ processingStalledOnFetchSince: None,
222
+ processingStalledOnStorageWriteSeconds: 0.,
209
223
  handlerStats: Dict.make(),
210
224
  storageLoadStats: Dict.make(),
211
225
  storageWriteStats: Dict.make(),
@@ -315,8 +329,22 @@ let isResolvingReorg = (state: t) =>
315
329
  // reports the first error so redundant handlers (eg an error caught in two
316
330
  // nested scopes) don't double-report.
317
331
  @inline
332
+ let // Close an open fetch-stall interval, accruing it into the counter. Called
333
+ // whenever the reason for the idle changes, so the interval never spans into
334
+ // time another counter owns — or, at shutdown, past the point where the loops
335
+ // stop and nothing would ever close it.
336
+ settleStalledOnFetch = (state: t) =>
337
+ switch state.processingStalledOnFetchSince {
338
+ | Some(since) =>
339
+ state.processingStalledOnFetchSeconds =
340
+ state.processingStalledOnFetchSeconds +. since->Performance.secondsSince
341
+ state.processingStalledOnFetchSince = None
342
+ | None => ()
343
+ }
344
+
318
345
  let errorExit = (state: t, errHandler) =>
319
346
  if !state.isStopped {
347
+ state->settleStalledOnFetch
320
348
  state.isStopped = true
321
349
  state.onError(errHandler)
322
350
  }
@@ -325,7 +353,10 @@ let unexpectedErrorMsg = "Indexer has failed with an unexpected error"
325
353
 
326
354
  // Halt the loops without reporting an error, eg to hand the shared db over to a
327
355
  // resumed indexer in tests.
328
- let stop = (state: t) => state.isStopped = true
356
+ let stop = (state: t) => {
357
+ state->settleStalledOnFetch
358
+ state.isStopped = true
359
+ }
329
360
 
330
361
  let getChainState = (state: t, ~chain: chain): ChainState.t =>
331
362
  switch state.crossChainState
@@ -360,6 +391,9 @@ let enterReorgThreshold = (state: t) => state.crossChainState->CrossChainState.e
360
391
  // caller has already mutated the chain states (restored counters, reset pending
361
392
  // queries). isResolvingReorg derives from rollbackState.
362
393
  let beginReorg = (state: t, ~chain, ~blockNumber) => {
394
+ // Settle here, or the rollback that follows would be folded into the stall on
395
+ // the next beginProcessing — time envio_rollback_seconds already counts.
396
+ state->settleStalledOnFetch
363
397
  state.epoch = state.epoch + 1
364
398
  state.rollbackState = ReorgDetected({chain, blockNumber})
365
399
  }
@@ -391,9 +425,24 @@ let applyBatchProgress = (state: t, ~batch: Batch.t) =>
391
425
  // Processing-loop mutex. Guards ProcessEventBatch re-entry so only one
392
426
  // processing loop runs at a time.
393
427
  let isProcessing = (state: t) => state.isProcessing
394
- let beginProcessing = (state: t) => state.isProcessing = true
428
+ let beginProcessing = (state: t) => {
429
+ state->settleStalledOnFetch
430
+ state.isProcessing = true
431
+ }
395
432
  let endProcessing = (state: t) => state.isProcessing = false
396
433
 
434
+ // Start attributing idle time to fetch starvation. Called when the processing
435
+ // loop exits with no work left; skipped when it exits for a reorg or shutdown,
436
+ // so only genuine buffer starvation is counted.
437
+ let markProcessingStalledOnFetch = (state: t) =>
438
+ if state.processingStalledOnFetchSince->Option.isNone {
439
+ state.processingStalledOnFetchSince = Some(Performance.now())
440
+ }
441
+
442
+ let recordStalledOnStorageWrite = (state: t, ~seconds) =>
443
+ state.processingStalledOnStorageWriteSeconds =
444
+ state.processingStalledOnStorageWriteSeconds +. seconds
445
+
397
446
  let recordProcessedBatch = (state: t) =>
398
447
  state.processedBatchesCount = state.processedBatchesCount + 1
399
448
 
@@ -469,12 +518,20 @@ let toMetrics = (state: t): Metrics.t => {
469
518
  )
470
519
  {
471
520
  startTime: state.indexerStartTime,
521
+ metricTime: Date.make(),
522
+ elapsedSeconds: state.indexerStartTimeRef->Performance.secondsSince,
472
523
  targetBufferSize: state.crossChainState->CrossChainState.targetBufferSize,
473
524
  isInReorgThreshold: state.crossChainState->CrossChainState.isInReorgThreshold,
474
525
  rollbackEnabled: state.config.shouldRollbackOnReorg,
475
526
  maxBatchSize: state.config.batchSize,
476
527
  preloadSeconds: state.preloadSeconds,
477
528
  processingSeconds: state.processingSeconds,
529
+ processingStalledOnFetchSeconds: state.processingStalledOnFetchSeconds +.
530
+ switch state.processingStalledOnFetchSince {
531
+ | Some(since) => since->Performance.secondsSince
532
+ | None => 0.
533
+ },
534
+ processingStalledOnStorageWriteSeconds: state.processingStalledOnStorageWriteSeconds,
478
535
  rollbackSeconds: state.rollbackSeconds,
479
536
  rollbackCount: state.rollbackCount,
480
537
  rollbackEventsCount: state.rollbackEventsCount,
@@ -76,6 +76,7 @@ function make$1(config, persistence, chainStates, isInReorgThreshold, isRealtime
76
76
  crossChainState: CrossChainState.make(chainStates, isInReorgThreshold, isRealtime, targetBufferSize),
77
77
  rollbackState: "NoRollback",
78
78
  indexerStartTime: new Date(),
79
+ indexerStartTimeRef: Performance.now(),
79
80
  lastPrunedAtMillis: {},
80
81
  loadManager: LoadManager.make(),
81
82
  keepProcessAlive: isDevelopmentMode || shouldUseTui,
@@ -87,6 +88,9 @@ function make$1(config, persistence, chainStates, isInReorgThreshold, isRealtime
87
88
  simulateDeadInputTracker: SimulateDeadInputTracker.makeFromConfig(config),
88
89
  preloadSeconds: 0,
89
90
  processingSeconds: 0,
91
+ processingStalledOnFetchSeconds: 0,
92
+ processingStalledOnFetchSince: undefined,
93
+ processingStalledOnStorageWriteSeconds: 0,
90
94
  handlerStats: {},
91
95
  storageLoadStats: {},
92
96
  storageWriteStats: {},
@@ -145,13 +149,24 @@ function isResolvingReorg(state) {
145
149
  }
146
150
 
147
151
  function errorExit(state, errHandler) {
148
- if (!state.isStopped) {
149
- state.isStopped = true;
150
- return state.onError(errHandler);
152
+ if (state.isStopped) {
153
+ return;
151
154
  }
155
+ let since = state.processingStalledOnFetchSince;
156
+ if (since !== undefined) {
157
+ state.processingStalledOnFetchSeconds = state.processingStalledOnFetchSeconds + Performance.secondsSince(since);
158
+ state.processingStalledOnFetchSince = undefined;
159
+ }
160
+ state.isStopped = true;
161
+ state.onError(errHandler);
152
162
  }
153
163
 
154
164
  function stop(state) {
165
+ let since = state.processingStalledOnFetchSince;
166
+ if (since !== undefined) {
167
+ state.processingStalledOnFetchSeconds = state.processingStalledOnFetchSeconds + Performance.secondsSince(since);
168
+ state.processingStalledOnFetchSince = undefined;
169
+ }
155
170
  state.isStopped = true;
156
171
  }
157
172
 
@@ -177,6 +192,11 @@ function enterReorgThreshold(state) {
177
192
  }
178
193
 
179
194
  function beginReorg(state, chain, blockNumber) {
195
+ let since = state.processingStalledOnFetchSince;
196
+ if (since !== undefined) {
197
+ state.processingStalledOnFetchSeconds = state.processingStalledOnFetchSeconds + Performance.secondsSince(since);
198
+ state.processingStalledOnFetchSince = undefined;
199
+ }
180
200
  state.epoch = state.epoch + 1 | 0;
181
201
  state.rollbackState = {
182
202
  TAG: "ReorgDetected",
@@ -221,6 +241,11 @@ function isProcessing(state) {
221
241
  }
222
242
 
223
243
  function beginProcessing(state) {
244
+ let since = state.processingStalledOnFetchSince;
245
+ if (since !== undefined) {
246
+ state.processingStalledOnFetchSeconds = state.processingStalledOnFetchSeconds + Performance.secondsSince(since);
247
+ state.processingStalledOnFetchSince = undefined;
248
+ }
224
249
  state.isProcessing = true;
225
250
  }
226
251
 
@@ -228,6 +253,17 @@ function endProcessing(state) {
228
253
  state.isProcessing = false;
229
254
  }
230
255
 
256
+ function markProcessingStalledOnFetch(state) {
257
+ if (Stdlib_Option.isNone(state.processingStalledOnFetchSince)) {
258
+ state.processingStalledOnFetchSince = Performance.now();
259
+ return;
260
+ }
261
+ }
262
+
263
+ function recordStalledOnStorageWrite(state, seconds) {
264
+ state.processingStalledOnStorageWriteSeconds = state.processingStalledOnStorageWriteSeconds + seconds;
265
+ }
266
+
231
267
  function recordProcessedBatch(state) {
232
268
  state.processedBatchesCount = state.processedBatchesCount + 1 | 0;
233
269
  }
@@ -371,14 +407,21 @@ function toMetrics(state) {
371
407
  count: s.count
372
408
  });
373
409
  });
410
+ let since = state.processingStalledOnFetchSince;
374
411
  return {
375
412
  startTime: state.indexerStartTime,
413
+ metricTime: new Date(),
414
+ elapsedSeconds: Performance.secondsSince(state.indexerStartTimeRef),
376
415
  targetBufferSize: CrossChainState.targetBufferSize(state.crossChainState),
377
416
  isInReorgThreshold: CrossChainState.isInReorgThreshold(state.crossChainState),
378
417
  rollbackEnabled: state.config.shouldRollbackOnReorg,
379
418
  maxBatchSize: state.config.batchSize,
380
419
  preloadSeconds: state.preloadSeconds,
381
420
  processingSeconds: state.processingSeconds,
421
+ processingStalledOnFetchSeconds: state.processingStalledOnFetchSeconds + (
422
+ since !== undefined ? Performance.secondsSince(since) : 0
423
+ ),
424
+ processingStalledOnStorageWriteSeconds: state.processingStalledOnStorageWriteSeconds,
382
425
  rollbackSeconds: state.rollbackSeconds,
383
426
  rollbackCount: state.rollbackCount,
384
427
  rollbackEventsCount: state.rollbackEventsCount,
@@ -698,6 +741,7 @@ export {
698
741
  isProcessing,
699
742
  beginProcessing,
700
743
  endProcessing,
744
+ markProcessingStalledOnFetch,
701
745
  recordProcessedBatch,
702
746
  createBatch,
703
747
  getSafeCheckpointId,
@@ -730,6 +774,7 @@ export {
730
774
  simulateDeadInputTracker,
731
775
  toMetrics,
732
776
  recordBatchDurations,
777
+ recordStalledOnStorageWrite,
733
778
  recordHandlerDuration,
734
779
  startPreloadHandler,
735
780
  endPreloadHandler,
@@ -66,6 +66,7 @@ let applyBatchProgress: (t, ~batch: Batch.t) => unit
66
66
  let isProcessing: t => bool
67
67
  let beginProcessing: t => unit
68
68
  let endProcessing: t => unit
69
+ let markProcessingStalledOnFetch: t => unit
69
70
  let recordProcessedBatch: t => unit
70
71
  let createBatch: (
71
72
  t,
@@ -110,6 +111,7 @@ let toMetrics: t => Metrics.t
110
111
 
111
112
  // Metric counters.
112
113
  let recordBatchDurations: (t, ~loadDuration: float, ~handlerDuration: float) => unit
114
+ let recordStalledOnStorageWrite: (t, ~seconds: float) => unit
113
115
  let recordHandlerDuration: (t, ~contract: string, ~event: string, ~duration: float) => unit
114
116
  let startPreloadHandler: (t, ~contract: string, ~event: string) => Performance.timeRef
115
117
  let endPreloadHandler: (t, Performance.timeRef, ~contract: string, ~event: string) => unit
package/src/Internal.res CHANGED
@@ -333,6 +333,11 @@ type eventPayload
333
333
  // The log's emitting address (EVM/Fuel; the program id carries it for SVM).
334
334
  @get external getPayloadSrcAddress: eventPayload => Address.t = "srcAddress"
335
335
 
336
+ // The decoded params, read by name for the address-valued ones a `where`
337
+ // filters on. Only those names are ever looked up, so the address type is
338
+ // accurate at every use site.
339
+ @get external getPayloadAddressParams: eventPayload => dict<Address.t> = "params"
340
+
336
341
  type genericLoaderArgs<'event, 'context> = {
337
342
  event: 'event,
338
343
  context: 'context,
@@ -356,11 +361,11 @@ type genericHandlerArgs<'event, 'context> = {
356
361
  type genericHandler<'args> = 'args => promise<unit>
357
362
 
358
363
  type entityHandlerContext<'entity> = {
359
- get: string => promise<option<'entity>>,
360
- getOrThrow: (string, ~message: string=?) => promise<'entity>,
364
+ get: EntityId.t => promise<option<'entity>>,
365
+ getOrThrow: (EntityId.t, ~message: string=?) => promise<'entity>,
361
366
  getOrCreate: 'entity => promise<'entity>,
362
367
  set: 'entity => unit,
363
- deleteUnsafe: string => unit,
368
+ deleteUnsafe: EntityId.t => unit,
364
369
  }
365
370
 
366
371
  type chainInfo = {
@@ -572,11 +577,15 @@ type onEventRegistration = {
572
577
  // Usually always false for wildcard events, but might be true for a wildcard
573
578
  // event with a dynamic event filter by addresses.
574
579
  dependsOnAddresses: bool,
575
- // Precompiled predicate for events that filter an indexed address param by
576
- // registered addresses (see `EventConfigBuilder.buildAddressFilter`); drops a
577
- // decoded event whose param-address isn't registered at/before the log's
578
- // block. Absent otherwise.
579
- clientAddressFilter?: (eventPayload, int, dict<indexingContract>) => bool,
580
+ // Indexed address params this event filters on, in disjunctive normal form
581
+ // (OR of AND-groups), from `where: {params: {to: chain.C.addresses}}`. Every
582
+ // source applies this natively while routing; it's carried here for the
583
+ // simulate source, which has no native query boundary. Absent otherwise.
584
+ //
585
+ // Keep it optional: with every field required, ReScript compiles a
586
+ // `{...registration, ...}` spread into an explicit field-by-field copy, which
587
+ // drops the ecosystem-only fields an `evmOnEventRegistration` carries.
588
+ addressFilterParamGroups?: array<array<string>>,
580
589
  // Final start block: the contract/chain config value, overridden by a
581
590
  // `where.block.number._gte` when the registered `where` supplies one.
582
591
  startBlock: option<int>,
@@ -23,7 +23,8 @@ type parsedWhere = {
23
23
  filterByAddresses: bool,
24
24
  // Indexed params filtered by `chain.<Contract>.addresses`, in disjunctive
25
25
  // normal form (outer array OR of AND-groups). Empty unless `filterByAddresses`.
26
- // Consumed by the codegen of the event's `clientAddressFilter`.
26
+ // Applied natively by every source while routing; carried on the registration
27
+ // for the simulate source, which has no native query boundary.
27
28
  addressFilterParamGroups: array<array<string>>,
28
29
  }
29
30