envio 3.3.0-alpha.1 → 3.3.0-alpha.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (84) hide show
  1. package/index.d.ts +23 -21
  2. package/package.json +6 -6
  3. package/src/Batch.res +0 -6
  4. package/src/Batch.res.mjs +0 -11
  5. package/src/BatchProcessing.res +2 -12
  6. package/src/BatchProcessing.res.mjs +2 -7
  7. package/src/ChainFetching.res +42 -6
  8. package/src/ChainFetching.res.mjs +30 -11
  9. package/src/ChainMetadata.res +1 -11
  10. package/src/ChainMetadata.res.mjs +1 -10
  11. package/src/ChainState.res +150 -1
  12. package/src/ChainState.res.mjs +138 -9
  13. package/src/ChainState.resi +51 -1
  14. package/src/Config.res +3 -6
  15. package/src/Config.res.mjs +2 -3
  16. package/src/Core.res +7 -0
  17. package/src/CrossChainState.res +24 -26
  18. package/src/CrossChainState.res.mjs +19 -16
  19. package/src/Ecosystem.res +6 -3
  20. package/src/Envio.res +3 -2
  21. package/src/EventConfigBuilder.res +61 -23
  22. package/src/EventConfigBuilder.res.mjs +27 -13
  23. package/src/EventProcessing.res +63 -21
  24. package/src/EventProcessing.res.mjs +47 -22
  25. package/src/FetchState.res +53 -43
  26. package/src/FetchState.res.mjs +71 -54
  27. package/src/HandlerLoader.res +10 -1
  28. package/src/HandlerLoader.res.mjs +15 -8
  29. package/src/Internal.res +46 -7
  30. package/src/Internal.res.mjs +19 -1
  31. package/src/LoadLayer.res +5 -5
  32. package/src/LoadLayer.res.mjs +6 -6
  33. package/src/Main.res +3 -50
  34. package/src/Main.res.mjs +2 -23
  35. package/src/PgStorage.res +4 -4
  36. package/src/PgStorage.res.mjs +5 -5
  37. package/src/Prometheus.res +10 -10
  38. package/src/Prometheus.res.mjs +9 -9
  39. package/src/PruneStaleHistory.res +2 -2
  40. package/src/PruneStaleHistory.res.mjs +3 -3
  41. package/src/Rollback.res +3 -3
  42. package/src/Rollback.res.mjs +4 -4
  43. package/src/SimulateItems.res +80 -15
  44. package/src/SimulateItems.res.mjs +65 -21
  45. package/src/TestIndexer.res +48 -36
  46. package/src/TestIndexer.res.mjs +35 -28
  47. package/src/bindings/Performance.res +7 -0
  48. package/src/bindings/Performance.res.mjs +21 -0
  49. package/src/bindings/Performance.resi +7 -0
  50. package/src/sources/EventRouter.res +17 -21
  51. package/src/sources/EventRouter.res.mjs +6 -9
  52. package/src/sources/Evm.res +47 -67
  53. package/src/sources/Evm.res.mjs +42 -65
  54. package/src/sources/Fuel.res +3 -3
  55. package/src/sources/Fuel.res.mjs +1 -6
  56. package/src/sources/HyperFuelSource.res +15 -11
  57. package/src/sources/HyperFuelSource.res.mjs +13 -11
  58. package/src/sources/HyperSync.res +7 -1
  59. package/src/sources/HyperSync.res.mjs +6 -3
  60. package/src/sources/HyperSync.resi +2 -0
  61. package/src/sources/HyperSyncClient.res +14 -79
  62. package/src/sources/HyperSyncClient.res.mjs +1 -22
  63. package/src/sources/HyperSyncSource.res +44 -27
  64. package/src/sources/HyperSyncSource.res.mjs +32 -27
  65. package/src/sources/RpcSource.res +14 -10
  66. package/src/sources/RpcSource.res.mjs +11 -9
  67. package/src/sources/SimulateSource.res +4 -2
  68. package/src/sources/SimulateSource.res.mjs +1 -0
  69. package/src/sources/Source.res +5 -1
  70. package/src/sources/SourceManager.res +17 -16
  71. package/src/sources/SourceManager.res.mjs +10 -15
  72. package/src/sources/Svm.res +23 -6
  73. package/src/sources/Svm.res.mjs +24 -6
  74. package/src/sources/SvmHyperSyncClient.res +3 -29
  75. package/src/sources/SvmHyperSyncSource.res +78 -96
  76. package/src/sources/SvmHyperSyncSource.res.mjs +79 -91
  77. package/src/sources/TransactionStore.res +128 -0
  78. package/src/sources/TransactionStore.res.mjs +97 -0
  79. package/src/tui/Tui.res +13 -16
  80. package/src/tui/Tui.res.mjs +12 -14
  81. package/svm.schema.json +30 -37
  82. package/src/bindings/Hrtime.res +0 -58
  83. package/src/bindings/Hrtime.res.mjs +0 -90
  84. package/src/bindings/Hrtime.resi +0 -30
@@ -57,7 +57,7 @@ let totalReadyCount = (crossChainState: t) => {
57
57
  let total = ref(0)
58
58
  for i in 0 to crossChainState.chainIds->Array.length - 1 {
59
59
  let cs = crossChainState->getChainState(crossChainState.chainIds->Array.getUnsafe(i))
60
- total := total.contents + cs->ChainState.fetchState->FetchState.bufferReadyCount
60
+ total := total.contents + cs->ChainState.bufferReadyCount
61
61
  }
62
62
  total.contents
63
63
  }
@@ -65,9 +65,7 @@ let totalReadyCount = (crossChainState: t) => {
65
65
  // --- Derived (pure). ---
66
66
 
67
67
  let nextItemIsNone = (crossChainState: t): bool =>
68
- !Batch.hasReadyItem(
69
- crossChainState.chainStates->Dict.valuesToArray->Array.map(ChainState.fetchState),
70
- )
68
+ !(crossChainState.chainStates->Dict.valuesToArray->Array.some(ChainState.hasReadyItem))
71
69
 
72
70
  let getSafeCheckpointId = (crossChainState: t) => {
73
71
  let result: ref<option<bigint>> = ref(None)
@@ -79,7 +77,7 @@ let getSafeCheckpointId = (crossChainState: t) => {
79
77
  | Some(safeCheckpointTracking) =>
80
78
  let safeCheckpointId =
81
79
  safeCheckpointTracking->SafeCheckpointTracking.getSafeCheckpointId(
82
- ~sourceBlockNumber=(cs->ChainState.fetchState).knownHeight,
80
+ ~sourceBlockNumber=cs->ChainState.knownHeight,
83
81
  )
84
82
  switch result.contents {
85
83
  | None => result := Some(safeCheckpointId)
@@ -111,16 +109,9 @@ let createBatch = (
111
109
  // in an append-only ClickHouse insert.
112
110
  isRollback ? 1n : 0n,
113
111
  ),
114
- ~chainsBeforeBatch=crossChainState.chainStates->Utils.Dict.mapValues((
115
- cs
116
- ): Batch.chainBeforeBatch => {
117
- fetchState: cs->ChainState.fetchState,
118
- progressBlockNumber: cs->ChainState.committedProgressBlockNumber,
119
- totalEventsProcessed: cs->ChainState.numEventsProcessed,
120
- sourceBlockNumber: (cs->ChainState.fetchState).knownHeight,
121
- reorgDetection: cs->ChainState.reorgDetection,
122
- chainConfig: cs->ChainState.chainConfig,
123
- }),
112
+ ~chainsBeforeBatch=crossChainState.chainStates->Utils.Dict.mapValues(
113
+ ChainState.toChainBeforeBatch,
114
+ ),
124
115
  ~batchSizeTarget,
125
116
  )
126
117
  }
@@ -184,10 +175,7 @@ let priorityOrder = (crossChainState: t) =>
184
175
  crossChainState.chainStates
185
176
  ->Dict.valuesToArray
186
177
  ->Array.toSorted((a, b) =>
187
- Float.compare(
188
- a->ChainState.fetchState->FetchState.getProgressPercentage,
189
- b->ChainState.fetchState->FetchState.getProgressPercentage,
190
- )
178
+ Float.compare(a->ChainState.getProgressPercentage, b->ChainState.getProgressPercentage)
191
179
  )
192
180
 
193
181
  // In-flight estimated items across every chain — the live draw against
@@ -235,11 +223,7 @@ let checkAndFetch = async (
235
223
  for i in 0 to chainIds->Array.length - 1 {
236
224
  let chainId = chainIds->Array.getUnsafe(i)
237
225
  let cs = crossChainState->getChainState(chainId)
238
- let fetchState = cs->ChainState.fetchState
239
- switch fetchState->FetchState.getNextQuery(
240
- ~budget=remaining,
241
- ~chainPendingBudget=cs->ChainState.pendingBudget,
242
- ) {
226
+ switch cs->ChainState.getNextQuery(~budget=remaining) {
243
227
  | (WaitingForNewBlock | NothingToQuery) as action =>
244
228
  actionByChain->Utils.Dict.setByInt(chainId, action)
245
229
  | Ready(queries) =>
@@ -247,8 +231,7 @@ let checkAndFetch = async (
247
231
  actionByChain->Utils.Dict.setByInt(chainId, FetchState.NothingToQuery)
248
232
  queries->Array.forEach(query => {
249
233
  query.chainId = chainId
250
- query.progress =
251
- fetchState->FetchState.getProgressPercentageAt(~blockNumber=query.fromBlock)
234
+ query.progress = cs->ChainState.getProgressPercentageAt(~blockNumber=query.fromBlock)
252
235
  candidates->Array.push(query)
253
236
  })
254
237
  }
@@ -271,6 +254,21 @@ let checkAndFetch = async (
271
254
  idx := idx.contents + 1
272
255
  }
273
256
  admittedByChain->Dict.forEachWithKey((queries, chainId) => {
257
+ let partitions = Dict.make()
258
+ queries->Array.forEach((query: FetchState.query) =>
259
+ partitions->Dict.set(
260
+ query.partitionId,
261
+ {
262
+ "fromBlock": query.fromBlock,
263
+ "targetBlock": query.toBlock,
264
+ },
265
+ )
266
+ )
267
+ Logging.trace({
268
+ "msg": "Started querying",
269
+ "chainId": chainId->Int.fromString->Option.getUnsafe,
270
+ "partitions": partitions,
271
+ })
274
272
  actionByChain->Dict.set(chainId, FetchState.Ready(queries))
275
273
  // Mark the admitted queries in flight and reserve their size against the
276
274
  // shared budget; released as each response lands in handleQueryResult.
@@ -6,7 +6,6 @@ import * as Utils from "./Utils.res.mjs";
6
6
  import * as Logging from "./Logging.res.mjs";
7
7
  import * as ChainMap from "./ChainMap.res.mjs";
8
8
  import * as ChainState from "./ChainState.res.mjs";
9
- import * as FetchState from "./FetchState.res.mjs";
10
9
  import * as Prometheus from "./Prometheus.res.mjs";
11
10
  import * as Stdlib_Int from "@rescript/runtime/lib/es6/Stdlib_Int.js";
12
11
  import * as Stdlib_Dict from "@rescript/runtime/lib/es6/Stdlib_Dict.js";
@@ -53,13 +52,13 @@ function totalReadyCount(crossChainState) {
53
52
  let total = 0;
54
53
  for (let i = 0, i_finish = crossChainState.chainIds.length; i < i_finish; ++i) {
55
54
  let cs = crossChainState.chainStates[crossChainState.chainIds[i]];
56
- total = total + FetchState.bufferReadyCount(ChainState.fetchState(cs)) | 0;
55
+ total = total + ChainState.bufferReadyCount(cs) | 0;
57
56
  }
58
57
  return total;
59
58
  }
60
59
 
61
60
  function nextItemIsNone(crossChainState) {
62
- return !Batch.hasReadyItem(Object.values(crossChainState.chainStates).map(ChainState.fetchState));
61
+ return !Object.values(crossChainState.chainStates).some(ChainState.hasReadyItem);
63
62
  }
64
63
 
65
64
  function getSafeCheckpointId(crossChainState) {
@@ -68,7 +67,7 @@ function getSafeCheckpointId(crossChainState) {
68
67
  let cs = crossChainState.chainStates[crossChainState.chainIds[i]];
69
68
  let safeCheckpointTracking = ChainState.safeCheckpointTracking(cs);
70
69
  if (safeCheckpointTracking !== undefined) {
71
- let safeCheckpointId = SafeCheckpointTracking.getSafeCheckpointId(safeCheckpointTracking, ChainState.fetchState(cs).knownHeight);
70
+ let safeCheckpointId = SafeCheckpointTracking.getSafeCheckpointId(safeCheckpointTracking, ChainState.knownHeight(cs));
72
71
  let current = result;
73
72
  if (!(current !== undefined && safeCheckpointId >= current)) {
74
73
  result = safeCheckpointId;
@@ -84,14 +83,7 @@ function getSafeCheckpointId(crossChainState) {
84
83
  function createBatch(crossChainState, processedCheckpointId, batchSizeTarget, isRollback) {
85
84
  return Batch.make(processedCheckpointId + (
86
85
  isRollback ? 1n : 0n
87
- ), Utils.Dict.mapValues(crossChainState.chainStates, cs => ({
88
- fetchState: ChainState.fetchState(cs),
89
- reorgDetection: ChainState.reorgDetection(cs),
90
- progressBlockNumber: ChainState.committedProgressBlockNumber(cs),
91
- sourceBlockNumber: ChainState.fetchState(cs).knownHeight,
92
- totalEventsProcessed: ChainState.numEventsProcessed(cs),
93
- chainConfig: ChainState.chainConfig(cs)
94
- })), batchSizeTarget, crossChainState.isInReorgThreshold);
86
+ ), Utils.Dict.mapValues(crossChainState.chainStates, ChainState.toChainBeforeBatch), batchSizeTarget, crossChainState.isInReorgThreshold);
95
87
  }
96
88
 
97
89
  function enterReorgThreshold(crossChainState) {
@@ -131,7 +123,7 @@ function applyBatchProgress(crossChainState, batch) {
131
123
  }
132
124
 
133
125
  function priorityOrder(crossChainState) {
134
- return Object.values(crossChainState.chainStates).toSorted((a, b) => Primitive_float.compare(FetchState.getProgressPercentage(ChainState.fetchState(a)), FetchState.getProgressPercentage(ChainState.fetchState(b))));
126
+ return Object.values(crossChainState.chainStates).toSorted((a, b) => Primitive_float.compare(ChainState.getProgressPercentage(a), ChainState.getProgressPercentage(b)));
135
127
  }
136
128
 
137
129
  function totalReservedSize(crossChainState) {
@@ -155,15 +147,14 @@ async function checkAndFetch(crossChainState, dispatchChain) {
155
147
  for (let i = 0, i_finish = chainIds.length; i < i_finish; ++i) {
156
148
  let chainId = chainIds[i];
157
149
  let cs = crossChainState.chainStates[chainId];
158
- let fetchState = ChainState.fetchState(cs);
159
- let action = FetchState.getNextQuery(fetchState, remaining, ChainState.pendingBudget(cs));
150
+ let action = ChainState.getNextQuery(cs, remaining);
160
151
  if (typeof action !== "object") {
161
152
  actionByChain[chainId] = action;
162
153
  } else {
163
154
  actionByChain[chainId] = "NothingToQuery";
164
155
  action._0.forEach(query => {
165
156
  query.chainId = chainId;
166
- query.progress = FetchState.getProgressPercentageAt(fetchState, query.fromBlock);
157
+ query.progress = ChainState.getProgressPercentageAt(cs, query.fromBlock);
167
158
  candidates.push(query);
168
159
  });
169
160
  }
@@ -179,6 +170,18 @@ async function checkAndFetch(crossChainState, dispatchChain) {
179
170
  idx = idx + 1 | 0;
180
171
  };
181
172
  Stdlib_Dict.forEachWithKey(admittedByChain, (queries, chainId) => {
173
+ let partitions = {};
174
+ queries.forEach(query => {
175
+ partitions[query.partitionId] = {
176
+ fromBlock: query.fromBlock,
177
+ targetBlock: query.toBlock
178
+ };
179
+ });
180
+ Logging.trace({
181
+ msg: "Started querying",
182
+ chainId: Stdlib_Int.fromString(chainId, undefined),
183
+ partitions: partitions
184
+ });
182
185
  actionByChain[chainId] = {
183
186
  TAG: "Ready",
184
187
  _0: queries
package/src/Ecosystem.res CHANGED
@@ -2,8 +2,6 @@ type name = | @as("evm") Evm | @as("fuel") Fuel | @as("svm") Svm
2
2
 
3
3
  type t = {
4
4
  name: name,
5
- blockFields: array<string>,
6
- transactionFields: array<string>,
7
5
  blockNumberName: string,
8
6
  blockTimestampName: string,
9
7
  blockHashName: string,
@@ -33,8 +31,13 @@ type t = {
33
31
  loggers (see `getItemLogger`). */
34
32
  logger: Pino.t,
35
33
  /** Materialise the user-facing event handed to handlers and contract
36
- registration from an item's opaque payload. */
34
+ registration from an item's opaque payload. `event.transaction` is written
35
+ onto the payload at batch prep (HyperSync) or inline (RPC/simulate). */
37
36
  toEvent: Internal.eventItem => Internal.event,
37
+ /** Bitmask (as a float) of the transaction fields selected across the chain's
38
+ events — the set the store materialises at batch prep. `0.` when the
39
+ ecosystem carries transactions inline. */
40
+ transactionFieldMask: array<Internal.eventConfig> => float,
38
41
  /** Build the per-item child logger for an event item, with
39
42
  ecosystem-specific log fields (EVM/Fuel: contract/event/address; SVM:
40
43
  program/instruction/programId). Closes over the injected logger. */
package/src/Envio.res CHANGED
@@ -48,6 +48,7 @@ type svmTokenBalance = {
48
48
  }
49
49
 
50
50
  type svmTransaction = {
51
+ transactionIndex?: int,
51
52
  signatures: array<string>,
52
53
  feePayer?: SvmTypes.Pubkey.t,
53
54
  success?: bool,
@@ -104,8 +105,8 @@ type svmInstruction = {
104
105
  d8?: string,
105
106
  /** Borsh-decoded params view. See [[svmInstructionParams]]. */
106
107
  params?: svmInstructionParams,
107
- /** Parent transaction. Absent when the per-instruction
108
- `include_transaction` flag is `false`. */
108
+ /** Parent transaction. Carries only the fields selected via
109
+ `field_selection.transaction_fields`; absent when none are selected. */
109
110
  transaction?: svmTransaction,
110
111
  /** Program log entries scoped to this instruction. Absent when the
111
112
  per-instruction `include_logs` flag is `false`. */
@@ -306,7 +306,14 @@ let resolveFieldSelection = (
306
306
  | Some(fields) => Utils.Set.fromArray(fields)
307
307
  | None => globalTransactionFieldsSet
308
308
  }
309
- (selectedBlockFields, selectedTransactionFields)
309
+ // The base eventConfig stores these as a string set (field names match the
310
+ // typed variants at runtime).
311
+ (
312
+ selectedBlockFields,
313
+ selectedTransactionFields->(
314
+ Utils.magic: Utils.Set.t<Internal.evmTransactionField> => Utils.Set.t<string>
315
+ ),
316
+ )
310
317
  }
311
318
 
312
319
  // ============== Client-side address filter ==============
@@ -319,28 +326,50 @@ let compileAddressFilter: string => (
319
326
  return new Function("event", "blockNumber", "indexingAddresses", body);
320
327
  }`)
321
328
 
322
- // Body of the client-side address filter for a DNF of address-filtered param
323
- // names (OR of AND-groups): keep the event only if some group's params are all
324
- // registered at or before the log's block. The DNF is fixed here, so it's
325
- // unrolled into one boolean expression no per-event closure, loop, or array.
326
- // `None` when there's no address-param filter. Exposed for snapshotting.
327
- let buildAddressFilterBody = (groups: array<array<string>>): option<string> => {
328
- switch groups {
329
+ // Body of the client-side address filter. Two analogous registered-at-or-before
330
+ // checks, ANDed: (1) for non-wildcard events, the log's srcAddress must itself be
331
+ // registered (ownership is resolved structurally by partition, but the temporal
332
+ // `effectiveStartBlock` gate lives here now); (2) a DNF of address-filtered param
333
+ // names (OR of AND-groups) for events that filter an indexed address param. The
334
+ // DNF is fixed here, so it's unrolled into one boolean expression — no per-event
335
+ // closure, loop, or array. `None` only for wildcard events without a param
336
+ // filter. Exposed for snapshotting.
337
+ // `srcAddressExpr` is the JS expression for the event's owning address: EVM and
338
+ // Fuel events expose `event.srcAddress`; SVM instructions expose `event.programId`.
339
+ let buildAddressFilterBody = (
340
+ groups: array<array<string>>,
341
+ ~isWildcard: bool,
342
+ ~srcAddressExpr: string="event.srcAddress",
343
+ ): option<string> => {
344
+ let paramLeaf = name =>
345
+ `(ic = indexingAddresses[p[${JSON.stringify(
346
+ JSON.String(name),
347
+ )}]]) !== undefined && ic.effectiveStartBlock <= blockNumber`
348
+ let paramDnf = switch groups {
329
349
  | [] => None
330
350
  | _ =>
331
- let leaf = name =>
332
- `(ic = indexingAddresses[p[${JSON.stringify(
333
- JSON.String(name),
334
- )}]]) !== undefined && ic.effectiveStartBlock <= blockNumber`
335
- let groupExprs =
336
- groups->Array.map(group => "(" ++ group->Array.map(leaf)->Array.join(" && ") ++ ")")
337
- Some("var p = event.params, ic; return " ++ groupExprs->Array.join(" || ") ++ ";")
351
+ Some(
352
+ groups
353
+ ->Array.map(group => "(" ++ group->Array.map(paramLeaf)->Array.join(" && ") ++ ")")
354
+ ->Array.join(" || "),
355
+ )
356
+ }
357
+ let srcLeaf = `(ic = indexingAddresses[${srcAddressExpr}]) !== undefined && ic.effectiveStartBlock <= blockNumber`
358
+ switch (isWildcard, paramDnf) {
359
+ | (true, None) => None
360
+ | (true, Some(dnf)) => Some("var p = event.params, ic; return " ++ dnf ++ ";")
361
+ | (false, None) => Some("var ic; return " ++ srcLeaf ++ ";")
362
+ | (false, Some(dnf)) =>
363
+ Some("var p = event.params, ic; return " ++ srcLeaf ++ " && (" ++ dnf ++ ");")
338
364
  }
339
365
  }
340
366
 
341
- let buildAddressFilter = (groups: array<array<string>>): option<
342
- (Internal.eventPayload, int, dict<Internal.indexingContract>) => bool,
343
- > => buildAddressFilterBody(groups)->Option.map(compileAddressFilter)
367
+ let buildAddressFilter = (
368
+ groups: array<array<string>>,
369
+ ~isWildcard: bool,
370
+ ~srcAddressExpr: string="event.srcAddress",
371
+ ): option<(Internal.eventPayload, int, dict<Internal.indexingContract>) => bool> =>
372
+ buildAddressFilterBody(groups, ~isWildcard, ~srcAddressExpr)->Option.map(compileAddressFilter)
344
373
 
345
374
  // ============== Build complete EVM event config ==============
346
375
 
@@ -408,7 +437,7 @@ let buildEvmEventConfig = (
408
437
  simulateParamsSchema: buildSimulateParamsSchema(params),
409
438
  getEventFiltersOrThrow,
410
439
  filterByAddresses,
411
- clientAddressFilter: ?buildAddressFilter(addressFilterParamGroups),
440
+ clientAddressFilter: ?buildAddressFilter(addressFilterParamGroups, ~isWildcard),
412
441
  dependsOnAddresses: !isWildcard || filterByAddresses,
413
442
  startBlock: resolvedStartBlock,
414
443
  selectedBlockFields,
@@ -427,9 +456,8 @@ let buildSvmInstructionEventConfig = (
427
456
  ~programId: SvmTypes.Pubkey.t,
428
457
  ~discriminator: option<string>,
429
458
  ~discriminatorByteLen: int,
430
- ~includeTransaction: bool,
431
459
  ~includeLogs: bool,
432
- ~includeTokenBalances: bool,
460
+ ~transactionFields: array<Internal.svmTransactionField>=[],
433
461
  ~accountFilters: array<Internal.svmAccountFilterGroup>,
434
462
  ~isInner: option<bool>,
435
463
  ~isWildcard: bool,
@@ -444,6 +472,13 @@ let buildSvmInstructionEventConfig = (
444
472
  S.json(~validate=false)
445
473
  ->Utils.Schema.coerceToJsonPgType
446
474
  ->(Utils.magic: S.t<JSON.t> => S.t<Internal.eventParams>)
475
+
476
+ // The base eventConfig stores these as a string set (field names match the
477
+ // typed variants at runtime).
478
+ let selectedTransactionFields =
479
+ Utils.Set.fromArray(transactionFields)->(
480
+ Utils.magic: Utils.Set.t<Internal.svmTransactionField> => Utils.Set.t<string>
481
+ )
447
482
  {
448
483
  id: switch discriminator {
449
484
  | Some(d) => d
@@ -458,13 +493,13 @@ let buildSvmInstructionEventConfig = (
458
493
  simulateParamsSchema: paramsSchema,
459
494
  filterByAddresses: false,
460
495
  dependsOnAddresses: !isWildcard,
496
+ clientAddressFilter: ?buildAddressFilter([], ~isWildcard, ~srcAddressExpr="event.programId"),
461
497
  startBlock,
462
498
  programId,
463
499
  discriminator,
464
500
  discriminatorByteLen,
465
- includeTransaction,
466
501
  includeLogs,
467
- includeTokenBalances,
502
+ selectedTransactionFields,
468
503
  accountFilters,
469
504
  isInner,
470
505
  accounts,
@@ -529,7 +564,10 @@ let buildFuelEventConfig = (
529
564
  simulateParamsSchema: paramsSchema,
530
565
  filterByAddresses: false,
531
566
  dependsOnAddresses: !isWildcard,
567
+ clientAddressFilter: ?buildAddressFilter([], ~isWildcard),
532
568
  startBlock,
569
+ // Fuel keeps the transaction inline on the payload; nothing to materialise.
570
+ selectedTransactionFields: Utils.Set.make(),
533
571
  kind: fuelKind,
534
572
  }
535
573
  }
@@ -255,17 +255,27 @@ let compileAddressFilter = (function (body) {
255
255
  return new Function("event", "blockNumber", "indexingAddresses", body);
256
256
  });
257
257
 
258
- function buildAddressFilterBody(groups) {
259
- if (groups.length === 0) {
260
- return;
258
+ function buildAddressFilterBody(groups, isWildcard, srcAddressExprOpt) {
259
+ let srcAddressExpr = srcAddressExprOpt !== undefined ? srcAddressExprOpt : "event.srcAddress";
260
+ let paramLeaf = name => `(ic = indexingAddresses[p[` + JSON.stringify(name) + `]]) !== undefined && ic.effectiveStartBlock <= blockNumber`;
261
+ let paramDnf = groups.length !== 0 ? groups.map(group => "(" + group.map(paramLeaf).join(" && ") + ")").join(" || ") : undefined;
262
+ let srcLeaf = `(ic = indexingAddresses[` + srcAddressExpr + `]) !== undefined && ic.effectiveStartBlock <= blockNumber`;
263
+ if (isWildcard) {
264
+ if (paramDnf !== undefined) {
265
+ return "var p = event.params, ic; return " + paramDnf + ";";
266
+ } else {
267
+ return;
268
+ }
269
+ } else if (paramDnf !== undefined) {
270
+ return "var p = event.params, ic; return " + srcLeaf + " && (" + paramDnf + ");";
271
+ } else {
272
+ return "var ic; return " + srcLeaf + ";";
261
273
  }
262
- let leaf = name => `(ic = indexingAddresses[p[` + JSON.stringify(name) + `]]) !== undefined && ic.effectiveStartBlock <= blockNumber`;
263
- let groupExprs = groups.map(group => "(" + group.map(leaf).join(" && ") + ")");
264
- return "var p = event.params, ic; return " + groupExprs.join(" || ") + ";";
265
274
  }
266
275
 
267
- function buildAddressFilter(groups) {
268
- return Stdlib_Option.map(buildAddressFilterBody(groups), compileAddressFilter);
276
+ function buildAddressFilter(groups, isWildcard, srcAddressExprOpt) {
277
+ let srcAddressExpr = srcAddressExprOpt !== undefined ? srcAddressExprOpt : "event.srcAddress";
278
+ return Stdlib_Option.map(buildAddressFilterBody(groups, isWildcard, srcAddressExpr), compileAddressFilter);
269
279
  }
270
280
 
271
281
  function buildEvmEventConfig(contractName, eventName, sighash, params, isWildcard, handler, contractRegister, eventFilters, probeChainId, onEventBlockFilterSchema, blockFields, transactionFields, startBlock, globalBlockFieldsSetOpt, globalTransactionFieldsSetOpt) {
@@ -291,26 +301,28 @@ function buildEvmEventConfig(contractName, eventName, sighash, params, isWildcar
291
301
  isWildcard: isWildcard,
292
302
  filterByAddresses: filterByAddresses,
293
303
  dependsOnAddresses: !isWildcard || filterByAddresses,
294
- clientAddressFilter: Stdlib_Option.map(buildAddressFilterBody(match.addressFilterParamGroups), compileAddressFilter),
304
+ clientAddressFilter: buildAddressFilter(match.addressFilterParamGroups, isWildcard, undefined),
295
305
  handler: handler,
296
306
  contractRegister: contractRegister,
297
307
  paramsRawEventSchema: buildParamsSchema(params),
298
308
  simulateParamsSchema: buildSimulateParamsSchema(params),
299
309
  startBlock: resolvedStartBlock,
310
+ selectedTransactionFields: match$1[1],
300
311
  getEventFiltersOrThrow: match.getEventFiltersOrThrow,
301
312
  selectedBlockFields: match$1[0],
302
- selectedTransactionFields: match$1[1],
303
313
  sighash: sighash,
304
314
  topicCount: topicCount,
305
315
  paramsMetadata: params
306
316
  };
307
317
  }
308
318
 
309
- function buildSvmInstructionEventConfig(contractName, instructionName, programId, discriminator, discriminatorByteLen, includeTransaction, includeLogs, includeTokenBalances, accountFilters, isInner, isWildcard, handler, contractRegister, accountsOpt, argsOpt, definedTypesOpt, startBlock) {
319
+ function buildSvmInstructionEventConfig(contractName, instructionName, programId, discriminator, discriminatorByteLen, includeLogs, transactionFieldsOpt, accountFilters, isInner, isWildcard, handler, contractRegister, accountsOpt, argsOpt, definedTypesOpt, startBlock) {
320
+ let transactionFields = transactionFieldsOpt !== undefined ? transactionFieldsOpt : [];
310
321
  let accounts = accountsOpt !== undefined ? accountsOpt : [];
311
322
  let args = argsOpt !== undefined ? argsOpt : null;
312
323
  let definedTypes = definedTypesOpt !== undefined ? definedTypesOpt : null;
313
324
  let paramsSchema = Utils.Schema.coerceToJsonPgType(S$RescriptSchema.json(false));
325
+ let selectedTransactionFields = new Set(transactionFields);
314
326
  return {
315
327
  id: discriminator !== undefined ? discriminator : "none",
316
328
  name: instructionName,
@@ -318,17 +330,17 @@ function buildSvmInstructionEventConfig(contractName, instructionName, programId
318
330
  isWildcard: isWildcard,
319
331
  filterByAddresses: false,
320
332
  dependsOnAddresses: !isWildcard,
333
+ clientAddressFilter: buildAddressFilter([], isWildcard, "event.programId"),
321
334
  handler: handler,
322
335
  contractRegister: contractRegister,
323
336
  paramsRawEventSchema: paramsSchema,
324
337
  simulateParamsSchema: paramsSchema,
325
338
  startBlock: startBlock,
339
+ selectedTransactionFields: selectedTransactionFields,
326
340
  programId: programId,
327
341
  discriminator: discriminator,
328
342
  discriminatorByteLen: discriminatorByteLen,
329
- includeTransaction: includeTransaction,
330
343
  includeLogs: includeLogs,
331
- includeTokenBalances: includeTokenBalances,
332
344
  accountFilters: accountFilters,
333
345
  isInner: isInner,
334
346
  accounts: accounts,
@@ -387,11 +399,13 @@ function buildFuelEventConfig(contractName, eventName, kind, sighash, rawAbi, is
387
399
  isWildcard: isWildcard,
388
400
  filterByAddresses: false,
389
401
  dependsOnAddresses: !isWildcard,
402
+ clientAddressFilter: buildAddressFilter([], isWildcard, undefined),
390
403
  handler: handler,
391
404
  contractRegister: contractRegister,
392
405
  paramsRawEventSchema: paramsSchema,
393
406
  simulateParamsSchema: paramsSchema,
394
407
  startBlock: startBlock,
408
+ selectedTransactionFields: new Set(),
395
409
  kind: fuelKind
396
410
  };
397
411
  }
@@ -39,7 +39,7 @@ let runEventHandlerOrThrow = async (
39
39
  let eventItem = item->Internal.castUnsafeEventItem
40
40
 
41
41
  //Include the load in time before handler
42
- let timeBeforeHandler = Hrtime.makeTimer()
42
+ let timeBeforeHandler = Performance.now()
43
43
 
44
44
  try {
45
45
  let contextParams: UserContext.contextParams = {
@@ -72,7 +72,7 @@ let runEventHandlerOrThrow = async (
72
72
  }),
73
73
  )
74
74
  }
75
- let handlerDuration = timeBeforeHandler->Hrtime.timeSince->Hrtime.toSecondsFloat
75
+ let handlerDuration = timeBeforeHandler->Performance.secondsSince
76
76
  Prometheus.ProcessingHandler.increment(
77
77
  ~contract=eventItem.eventConfig.contractName,
78
78
  ~event=eventItem.eventConfig.name,
@@ -264,11 +264,19 @@ let runBatchHandlersOrThrow = async (
264
264
  }
265
265
  }
266
266
 
267
- let registerProcessEventBatchMetrics = (~logger, ~loadDuration, ~handlerDuration) => {
268
- logger->Logging.childTrace({
269
- "msg": "Finished processing batch",
270
- "loader_time_elapsed": loadDuration,
271
- "handlers_time_elapsed": handlerDuration,
267
+ let registerProcessEventBatchMetrics = (
268
+ ~logger,
269
+ ~batch: Batch.t,
270
+ ~loadDuration,
271
+ ~handlerDuration,
272
+ ) => {
273
+ batch.progressedChainsById->Dict.forEachWithKey((chainAfterBatch, chainId) => {
274
+ logger->Logging.childTrace({
275
+ "msg": "Finished processing",
276
+ "chainId": chainId->Int.fromString->Option.getUnsafe,
277
+ "batchSize": chainAfterBatch.batchSize,
278
+ "progress": chainAfterBatch.progressBlockNumber,
279
+ })
272
280
  })
273
281
 
274
282
  Prometheus.ProcessingBatch.registerMetrics(~loadDuration, ~handlerDuration)
@@ -281,6 +289,35 @@ type logPartitionInfo = {
281
289
  lastItemBlockNumber?: int,
282
290
  }
283
291
 
292
+ // Off the hot path: bulk-materialise the selected transaction fields for the
293
+ // batch's store-backed (HyperSync) items and write them onto the payloads, so
294
+ // handlers read plain objects. A batch can span chains, each with its own store
295
+ // and field mask, so group items by chain before materialising.
296
+ let materializeBatchTransactions = async (batch: Batch.t, ~chainStates: dict<ChainState.t>) => {
297
+ switch chainStates->Dict.valuesToArray {
298
+ // Single-chain indexers (the common case): every item belongs to the one
299
+ // chain, so skip the per-chain grouping and its allocations.
300
+ | [cs] => await cs->ChainState.materializeBatchItems(~items=batch.items)
301
+ | _ =>
302
+ let itemsByChain: dict<array<Internal.item>> = Dict.make()
303
+ batch.items->Array.forEach(item => {
304
+ let chainId = item->Internal.getItemChainId->Int.toString
305
+ switch itemsByChain->Utils.Dict.dangerouslyGetNonOption(chainId) {
306
+ | Some(items) => items->Array.push(item)
307
+ | None => itemsByChain->Dict.set(chainId, [item])
308
+ }
309
+ })
310
+
311
+ let _ = await itemsByChain
312
+ ->Dict.toArray
313
+ ->Array.map(async ((chainId, items)) => {
314
+ let cs = chainStates->Dict.getUnsafe(chainId)
315
+ await cs->ChainState.materializeBatchItems(~items)
316
+ })
317
+ ->Promise.all
318
+ }
319
+ }
320
+
284
321
  let processEventBatch = async (
285
322
  ~batch: Batch.t,
286
323
  ~indexerState: IndexerState.t,
@@ -289,33 +326,33 @@ let processEventBatch = async (
289
326
  ~config: Config.t,
290
327
  ~chainStates: dict<ChainState.t>,
291
328
  ) => {
292
- let totalBatchSize = batch.totalBatchSize
293
329
  // Compute chains state for this batch
294
330
  let chains: Internal.chains = chainStates->computeChainsState
295
331
 
296
332
  let logger = Logging.getLogger()
297
- logger->Logging.childTrace({
298
- "msg": "Started processing batch",
299
- "totalBatchSize": totalBatchSize,
300
- "chains": batch.progressedChainsById->Utils.Dict.mapValues(chainAfterBatch => {
301
- {
302
- "batchSize": chainAfterBatch.batchSize,
303
- "progress": chainAfterBatch.progressBlockNumber,
304
- }
305
- }),
333
+
334
+ batch.progressedChainsById->Dict.forEachWithKey((chainAfterBatch, chainId) => {
335
+ logger->Logging.childTrace({
336
+ "msg": "Started processing",
337
+ "chainId": chainId->Int.fromString->Option.getUnsafe,
338
+ "batchSize": chainAfterBatch.batchSize,
339
+ })
306
340
  })
307
341
 
308
342
  try {
309
343
  // Backpressure: keep processing within keepLatestChangesLimit of the cycle.
310
344
  await indexerState->Writing.awaitCapacity
311
345
 
312
- let timeRef = Hrtime.makeTimer()
346
+ let timeRef = Performance.now()
313
347
 
314
348
  if batch.items->Utils.Array.notEmpty {
349
+ // Materialise store-backed transactions onto payloads before any handler
350
+ // (preload or execute) reads them.
351
+ await materializeBatchTransactions(batch, ~chainStates)
315
352
  await batch->preloadBatchOrThrow(~loadManager, ~persistence, ~indexerState, ~chains, ~config)
316
353
  }
317
354
 
318
- let elapsedTimeAfterLoaders = timeRef->Hrtime.timeSince->Hrtime.toSecondsFloat
355
+ let elapsedTimeAfterLoaders = timeRef->Performance.secondsSince
319
356
 
320
357
  if batch.items->Utils.Array.notEmpty {
321
358
  await batch->runBatchHandlersOrThrow(
@@ -327,13 +364,18 @@ let processEventBatch = async (
327
364
  )
328
365
  }
329
366
 
330
- let elapsedTimeAfterProcessing = timeRef->Hrtime.timeSince->Hrtime.toSecondsFloat
367
+ let elapsedTimeAfterProcessing = timeRef->Performance.secondsSince
331
368
 
332
369
  indexerState->Writing.commitBatch(~batch)
333
370
 
334
371
  let loaderDuration = elapsedTimeAfterLoaders
335
372
  let handlerDuration = elapsedTimeAfterProcessing -. loaderDuration
336
- registerProcessEventBatchMetrics(~logger, ~loadDuration=loaderDuration, ~handlerDuration)
373
+ registerProcessEventBatchMetrics(
374
+ ~logger,
375
+ ~batch,
376
+ ~loadDuration=loaderDuration,
377
+ ~handlerDuration,
378
+ )
337
379
  Ok()
338
380
  } catch {
339
381
  | Persistence.StorageError({message, reason}) =>