envio 3.3.0-alpha.1 → 3.3.0-alpha.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (136) hide show
  1. package/evm.schema.json +10 -0
  2. package/index.d.ts +45 -25
  3. package/package.json +7 -7
  4. package/src/Address.res +5 -2
  5. package/src/Address.res.mjs +3 -1
  6. package/src/Api.res +1 -1
  7. package/src/Api.res.mjs +1 -1
  8. package/src/Batch.res +0 -6
  9. package/src/Batch.res.mjs +1 -12
  10. package/src/BatchProcessing.res +7 -12
  11. package/src/BatchProcessing.res.mjs +8 -7
  12. package/src/Bin.res +3 -0
  13. package/src/Bin.res.mjs +4 -0
  14. package/src/ChainFetching.res +43 -9
  15. package/src/ChainFetching.res.mjs +34 -15
  16. package/src/ChainMetadata.res +1 -11
  17. package/src/ChainMetadata.res.mjs +1 -10
  18. package/src/ChainState.res +605 -151
  19. package/src/ChainState.res.mjs +427 -104
  20. package/src/ChainState.resi +72 -4
  21. package/src/Config.res +61 -33
  22. package/src/Config.res.mjs +50 -10
  23. package/src/ContractRegisterContext.res +2 -12
  24. package/src/ContractRegisterContext.res.mjs +3 -5
  25. package/src/Core.res +14 -3
  26. package/src/CrossChainState.res +113 -89
  27. package/src/CrossChainState.res.mjs +64 -60
  28. package/src/CrossChainState.resi +1 -1
  29. package/src/Ecosystem.res +5 -6
  30. package/src/Ecosystem.res.mjs +3 -3
  31. package/src/Envio.res +17 -11
  32. package/src/EnvioGlobal.res +53 -0
  33. package/src/EnvioGlobal.res.mjs +31 -0
  34. package/src/EventConfigBuilder.res +177 -88
  35. package/src/EventConfigBuilder.res.mjs +99 -42
  36. package/src/EventProcessing.res +74 -28
  37. package/src/EventProcessing.res.mjs +55 -29
  38. package/src/ExitOnCaughtUp.res +10 -2
  39. package/src/ExitOnCaughtUp.res.mjs +10 -1
  40. package/src/FetchState.res +544 -356
  41. package/src/FetchState.res.mjs +412 -471
  42. package/src/HandlerLoader.res +8 -104
  43. package/src/HandlerLoader.res.mjs +2 -81
  44. package/src/HandlerRegister.res +518 -144
  45. package/src/HandlerRegister.res.mjs +285 -133
  46. package/src/HandlerRegister.resi +24 -8
  47. package/src/IndexerState.res +10 -3
  48. package/src/IndexerState.res.mjs +11 -4
  49. package/src/IndexerState.resi +2 -1
  50. package/src/IndexingAddresses.res +108 -0
  51. package/src/IndexingAddresses.res.mjs +101 -0
  52. package/src/IndexingAddresses.resi +34 -0
  53. package/src/Internal.res +155 -46
  54. package/src/Internal.res.mjs +30 -2
  55. package/src/LoadLayer.res +5 -5
  56. package/src/LoadLayer.res.mjs +6 -6
  57. package/src/LogSelection.res +102 -165
  58. package/src/LogSelection.res.mjs +101 -116
  59. package/src/Main.res +71 -236
  60. package/src/Main.res.mjs +60 -145
  61. package/src/Metrics.res +74 -0
  62. package/src/Metrics.res.mjs +75 -0
  63. package/src/PgStorage.res +4 -4
  64. package/src/PgStorage.res.mjs +5 -5
  65. package/src/Prometheus.res +10 -63
  66. package/src/Prometheus.res.mjs +124 -171
  67. package/src/PruneStaleHistory.res +2 -2
  68. package/src/PruneStaleHistory.res.mjs +3 -3
  69. package/src/RawEvent.res +7 -2
  70. package/src/RawEvent.res.mjs +4 -4
  71. package/src/Rollback.res +3 -3
  72. package/src/Rollback.res.mjs +4 -4
  73. package/src/RollbackCommit.res +4 -1
  74. package/src/RollbackCommit.res.mjs +3 -2
  75. package/src/SimulateDeadInputTracker.res +85 -0
  76. package/src/SimulateDeadInputTracker.res.mjs +73 -0
  77. package/src/SimulateDeadInputTracker.resi +12 -0
  78. package/src/SimulateItems.res +91 -23
  79. package/src/SimulateItems.res.mjs +55 -32
  80. package/src/TestIndexer.res +54 -40
  81. package/src/TestIndexer.res.mjs +37 -30
  82. package/src/bindings/Performance.res +7 -0
  83. package/src/bindings/Performance.res.mjs +21 -0
  84. package/src/bindings/Performance.resi +7 -0
  85. package/src/bindings/Viem.res +0 -41
  86. package/src/bindings/Viem.res.mjs +1 -43
  87. package/src/sources/BlockStore.res +46 -0
  88. package/src/sources/BlockStore.res.mjs +24 -0
  89. package/src/sources/EventRouter.res +36 -33
  90. package/src/sources/EventRouter.res.mjs +13 -14
  91. package/src/sources/Evm.res +83 -72
  92. package/src/sources/Evm.res.mjs +53 -69
  93. package/src/sources/EvmChain.res +18 -20
  94. package/src/sources/EvmChain.res.mjs +15 -15
  95. package/src/sources/EvmRpcClient.res +74 -5
  96. package/src/sources/EvmRpcClient.res.mjs +13 -4
  97. package/src/sources/FieldMask.res +39 -0
  98. package/src/sources/FieldMask.res.mjs +42 -0
  99. package/src/sources/Fuel.res +9 -6
  100. package/src/sources/Fuel.res.mjs +5 -10
  101. package/src/sources/HyperFuelSource.res +46 -50
  102. package/src/sources/HyperFuelSource.res.mjs +64 -54
  103. package/src/sources/HyperSync.res +39 -7
  104. package/src/sources/HyperSync.res.mjs +57 -37
  105. package/src/sources/HyperSync.resi +12 -2
  106. package/src/sources/HyperSyncClient.res +22 -111
  107. package/src/sources/HyperSyncClient.res.mjs +2 -30
  108. package/src/sources/HyperSyncHeightStream.res +1 -8
  109. package/src/sources/HyperSyncHeightStream.res.mjs +0 -2
  110. package/src/sources/HyperSyncSource.res +94 -107
  111. package/src/sources/HyperSyncSource.res.mjs +82 -86
  112. package/src/sources/Rpc.res +15 -47
  113. package/src/sources/Rpc.res.mjs +25 -56
  114. package/src/sources/RpcSource.res +320 -467
  115. package/src/sources/RpcSource.res.mjs +268 -385
  116. package/src/sources/RpcWebSocketHeightStream.res +0 -5
  117. package/src/sources/RpcWebSocketHeightStream.res.mjs +0 -2
  118. package/src/sources/SimulateSource.res +10 -4
  119. package/src/sources/SimulateSource.res.mjs +16 -6
  120. package/src/sources/Source.res +34 -6
  121. package/src/sources/SourceManager.res +77 -19
  122. package/src/sources/SourceManager.res.mjs +63 -21
  123. package/src/sources/SourceManager.resi +10 -0
  124. package/src/sources/Svm.res +31 -16
  125. package/src/sources/Svm.res.mjs +36 -13
  126. package/src/sources/SvmHyperSyncClient.res +5 -32
  127. package/src/sources/SvmHyperSyncSource.res +159 -134
  128. package/src/sources/SvmHyperSyncSource.res.mjs +164 -129
  129. package/src/sources/TransactionStore.res +49 -0
  130. package/src/sources/TransactionStore.res.mjs +30 -0
  131. package/src/tui/Tui.res +13 -16
  132. package/src/tui/Tui.res.mjs +12 -14
  133. package/svm.schema.json +49 -37
  134. package/src/bindings/Hrtime.res +0 -58
  135. package/src/bindings/Hrtime.res.mjs +0 -90
  136. package/src/bindings/Hrtime.resi +0 -30
@@ -108,11 +108,6 @@ let subscribe = (~wsUrl, ~chainId, ~onHeight: int => unit): (unit => unit) => {
108
108
  | NewHead(blockNumber) =>
109
109
  errorCount := 0
110
110
  resetStaleTimeout()
111
- Prometheus.SourceRequestCount.increment(
112
- ~sourceName="WebSocket",
113
- ~chainId,
114
- ~method="eth_subscribe",
115
- )
116
111
  onHeight(blockNumber)
117
112
  | SubscriptionConfirmed(_) => resetStaleTimeout()
118
113
  | ErrorResponse =>
@@ -3,7 +3,6 @@
3
3
  import * as Rpc from "./Rpc.res.mjs";
4
4
  import * as Utils from "../Utils.res.mjs";
5
5
  import * as Logging from "../Logging.res.mjs";
6
- import * as Prometheus from "../Prometheus.res.mjs";
7
6
  import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js";
8
7
  import * as S$RescriptSchema from "rescript-schema/src/S.res.mjs";
9
8
  import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
@@ -105,7 +104,6 @@ function subscribe(wsUrl, chainId, onHeight) {
105
104
  }
106
105
  errorCount.contents = 0;
107
106
  resetStaleTimeout();
108
- Prometheus.SourceRequestCount.increment("WebSocket", chainId, "eth_subscribe");
109
107
  return onHeight(blockNumber._0);
110
108
  } catch (raw_e) {
111
109
  let e = Primitive_exceptions.internalToException(raw_e);
@@ -6,29 +6,31 @@ let make = (~items: array<Internal.item>, ~endBlock: int, ~chain: ChainMap.Chain
6
6
 
7
7
  {
8
8
  name: "SimulateSource",
9
+ simulateItems: items,
9
10
  sourceFor: Sync,
10
11
  chain,
11
12
  poweredByHyperSync: false,
12
13
  pollingInterval: 0,
13
14
  getBlockHashes: (~blockNumbers as _, ~logger as _) => {
14
- Promise.resolve(Ok([]))
15
+ Promise.resolve({Source.result: Ok([]), requestStats: []})
15
16
  },
16
17
  getHeightOrThrow: () => {
17
18
  // Report at least height 1 so the engine doesn't treat 0 as "no blocks available"
18
- Promise.resolve(max(endBlock, 1))
19
+ Promise.resolve({Source.height: max(endBlock, 1), requestStats: []})
19
20
  },
20
21
  getItemsOrThrow: (
21
22
  ~fromBlock as _,
22
23
  ~toBlock as _,
23
24
  ~addressesByContractName as _,
24
- ~indexingAddresses as _,
25
+ ~contractNameByAddress as _,
25
26
  ~knownHeight as _,
26
27
  ~partitionId as _,
27
28
  ~selection as _,
29
+ ~itemsTarget as _,
28
30
  ~retry as _,
29
31
  ~logger as _,
30
32
  ) => {
31
- // Return all items on first call, empty on subsequent calls
33
+ // Return all items on the first call, empty on subsequent.
32
34
  let result = if delivered.contents {
33
35
  []
34
36
  } else {
@@ -41,12 +43,16 @@ let make = (~items: array<Internal.item>, ~endBlock: int, ~chain: ChainMap.Chain
41
43
  Source.knownHeight: reportedHeight,
42
44
  blockHashes: [],
43
45
  parsedQueueItems: result,
46
+ // Simulate keeps the transaction and block inline on the payload; no store pages.
47
+ transactionStore: None,
48
+ blockStore: None,
44
49
  fromBlockQueried: 0,
45
50
  latestFetchedBlockNumber: reportedHeight,
46
51
  latestFetchedBlockTimestamp: 0,
47
52
  stats: {
48
53
  totalTimeElapsed: 0.,
49
54
  },
55
+ requestStats: [],
50
56
  })
51
57
  },
52
58
  }
@@ -13,25 +13,35 @@ function make(items, endBlock, chain) {
13
13
  poweredByHyperSync: false,
14
14
  pollingInterval: 0,
15
15
  getBlockHashes: (param, param$1) => Promise.resolve({
16
- TAG: "Ok",
17
- _0: []
16
+ result: {
17
+ TAG: "Ok",
18
+ _0: []
19
+ },
20
+ requestStats: []
18
21
  }),
19
- getHeightOrThrow: () => Promise.resolve(Primitive_int.max(endBlock, 1)),
20
- getItemsOrThrow: (param, param$1, param$2, param$3, param$4, param$5, param$6, param$7, param$8) => {
22
+ getHeightOrThrow: () => Promise.resolve({
23
+ height: Primitive_int.max(endBlock, 1),
24
+ requestStats: []
25
+ }),
26
+ getItemsOrThrow: (param, param$1, param$2, param$3, param$4, param$5, param$6, param$7, param$8, param$9) => {
21
27
  let result = delivered.contents ? [] : (delivered.contents = true, items);
22
28
  let reportedHeight = Primitive_int.max(endBlock, 1);
23
29
  return Promise.resolve({
24
30
  knownHeight: reportedHeight,
25
31
  blockHashes: [],
26
32
  parsedQueueItems: result,
33
+ transactionStore: undefined,
34
+ blockStore: undefined,
27
35
  fromBlockQueried: 0,
28
36
  latestFetchedBlockNumber: reportedHeight,
29
37
  latestFetchedBlockTimestamp: 0,
30
38
  stats: {
31
39
  "total time elapsed (s)": 0
32
- }
40
+ },
41
+ requestStats: []
33
42
  });
34
- }
43
+ },
44
+ simulateItems: items
35
45
  };
36
46
  }
37
47
 
@@ -7,6 +7,11 @@ type blockRangeFetchStats = {
7
7
  @as("page fetch time (s)") pageFetchTime?: float,
8
8
  }
9
9
 
10
+ // A single backend request a source method actually made (cache/dedup hits
11
+ // aren't requests), with the time it took. SourceManager aggregates these
12
+ // per (source, method) into the envio_source_request_* metrics.
13
+ type requestStat = {method: string, seconds: float}
14
+
10
15
  /**
11
16
  Thes response returned from a block range fetch
12
17
  */
@@ -18,10 +23,26 @@ type blockRangeFetchResponse = {
18
23
  // a within-array hash mismatch on the same block number as a reorg.
19
24
  blockHashes: array<ReorgDetection.blockData>,
20
25
  parsedQueueItems: array<Internal.item>,
26
+ // Page of transactions for this response's items, keyed by (blockNumber,
27
+ // transactionIndex); merged into the chain's store on apply. `None` for
28
+ // sources that keep the transaction inline on the payload (RPC/Fuel/Simulate).
29
+ transactionStore: option<TransactionStore.t>,
30
+ // Page of blocks for this response's items, keyed by block number; merged into
31
+ // the chain's store on apply. `None` for sources that keep the block fully
32
+ // inline on the payload (RPC/Fuel/Simulate).
33
+ blockStore: option<BlockStore.t>,
21
34
  fromBlockQueried: int,
22
35
  latestFetchedBlockNumber: int,
23
36
  latestFetchedBlockTimestamp: int,
24
37
  stats: blockRangeFetchStats,
38
+ requestStats: array<requestStat>,
39
+ }
40
+
41
+ type getHeightResponse = {height: int, requestStats: array<requestStat>}
42
+
43
+ type getBlockHashesResponse = {
44
+ result: result<array<ReorgDetection.blockDataWithTimestamp>, exn>,
45
+ requestStats: array<requestStat>,
25
46
  }
26
47
 
27
48
  type getItemsRetry =
@@ -47,19 +68,22 @@ type t = {
47
68
  poweredByHyperSync: bool,
48
69
  /* Frequency (in ms) used when polling for new events on this network. */
49
70
  pollingInterval: int,
50
- getBlockHashes: (
51
- ~blockNumbers: array<int>,
52
- ~logger: Pino.t,
53
- ) => promise<result<array<ReorgDetection.blockDataWithTimestamp>, exn>>,
54
- getHeightOrThrow: unit => promise<int>,
71
+ getBlockHashes: (~blockNumbers: array<int>, ~logger: Pino.t) => promise<getBlockHashesResponse>,
72
+ getHeightOrThrow: unit => promise<getHeightResponse>,
55
73
  getItemsOrThrow: (
56
74
  ~fromBlock: int,
57
75
  ~toBlock: option<int>,
58
76
  ~addressesByContractName: dict<array<Address.t>>,
59
- ~indexingAddresses: dict<FetchState.indexingAddress>,
77
+ ~contractNameByAddress: dict<string>,
60
78
  ~knownHeight: int,
61
79
  ~partitionId: string,
62
80
  ~selection: FetchState.selection,
81
+ // Soft cap on the number of primary items (logs/instructions/receipts) the
82
+ // source should ask its backend for, from the query's own estResponseSize.
83
+ // A HyperSync-backed source enforces it server-side, so a wrong estimate
84
+ // truncates the response instead of overshooting the shared buffer. Sources
85
+ // without an equivalent lever (RPC, Fuel, Simulate) ignore it.
86
+ ~itemsTarget: int,
63
87
  ~retry: int,
64
88
  ~logger: Pino.t,
65
89
  ) => promise<blockRangeFetchResponse>,
@@ -67,4 +91,8 @@ type t = {
67
91
  // Invoked by SourceManager once a rollback target is known so the source can
68
92
  // drop any state that may now point at an orphaned chain (e.g. RPC block cache).
69
93
  onReorg?: (~rollbackTargetBlock: int) => unit,
94
+ // Present only on the simulate source: the items a test fed in. The chain
95
+ // tracks which of these never reach a handler so the run can report dead
96
+ // simulate inputs on completion.
97
+ simulateItems?: array<Internal.item>,
70
98
  }
@@ -1,5 +1,10 @@
1
1
  type sourceManagerStatus = Idle | WaitingForNewBlock | Querieng
2
2
 
3
+ // Cumulative per-method request count/time for a source, aggregated from the
4
+ // requestStat arrays returned by its methods. Rendered into
5
+ // envio_source_request_* by Metrics.renderSourceRequests.
6
+ type requestStatAgg = {mutable count: int, mutable seconds: float}
7
+
3
8
  type sourceState = {
4
9
  source: Source.t,
5
10
  mutable knownHeight: int,
@@ -9,13 +14,35 @@ type sourceState = {
9
14
  // Timestamp (ms) when this source last failed during executeQuery.
10
15
  // Used to decide when to attempt recovery to this source.
11
16
  mutable lastFailedAt: option<float>,
17
+ requestStats: dict<requestStatAgg>,
18
+ }
19
+
20
+ let recordRequestStats = (sourceState: sourceState, requestStats: array<Source.requestStat>) => {
21
+ requestStats->Array.forEach(({method, seconds}) => {
22
+ switch sourceState.requestStats->Utils.Dict.dangerouslyGetNonOption(method) {
23
+ | Some(agg) =>
24
+ agg.count = agg.count + 1
25
+ agg.seconds = agg.seconds +. seconds
26
+ | None => sourceState.requestStats->Dict.set(method, {count: 1, seconds})
27
+ }
28
+ })
29
+ }
30
+
31
+ // Flattened (source, method) aggregates for Metrics.renderSourceRequests to
32
+ // inline into the /metrics response.
33
+ type requestStatSample = {
34
+ sourceName: string,
35
+ chainId: int,
36
+ method: string,
37
+ count: int,
38
+ seconds: float,
12
39
  }
13
40
 
14
41
  // Encapsulates the fetching logic for a chain's sources.
15
42
  // with a mutable state for easier reasoning and testing.
16
43
  type t = {
17
44
  sourcesState: array<sourceState>,
18
- mutable statusStart: Hrtime.timeRef,
45
+ mutable statusStart: Performance.timeRef,
19
46
  mutable status: sourceManagerStatus,
20
47
  newBlockStallTimeout: int,
21
48
  newBlockStallTimeoutRealtime: int,
@@ -24,6 +51,10 @@ type t = {
24
51
  getHeightRetryInterval: (~retry: int) => int,
25
52
  mutable activeSource: Source.t,
26
53
  mutable waitingForNewBlockStateId: option<int>,
54
+ // Dedupes the "waiting for new blocks" trace so it fires once per contiguous
55
+ // wait period instead of on every epoch that re-enters the wait before any
56
+ // new block is found. Reset when blocks are found.
57
+ mutable waitingLogged: bool,
27
58
  // Should take into consideration partitions fetching for previous states (before rollback)
28
59
  mutable fetchingPartitionsCount: int,
29
60
  recoveryTimeout: float,
@@ -42,6 +73,25 @@ type t = {
42
73
 
43
74
  let getActiveSource = sourceManager => sourceManager.activeSource
44
75
 
76
+ let getRequestStatSamples = (sourceManager: t): array<requestStatSample> => {
77
+ let samples = []
78
+ sourceManager.sourcesState->Array.forEach(sourceState => {
79
+ let chainId = sourceState.source.chain->ChainMap.Chain.toChainId
80
+ sourceState.requestStats->Utils.Dict.forEachWithKey((agg, method) => {
81
+ samples
82
+ ->Array.push({
83
+ sourceName: sourceState.source.name,
84
+ chainId,
85
+ method,
86
+ count: agg.count,
87
+ seconds: agg.seconds,
88
+ })
89
+ ->ignore
90
+ })
91
+ })
92
+ samples
93
+ }
94
+
45
95
  // Partition queries currently in flight on this chain's sources. Summed across
46
96
  // chains by CrossChainState to enforce the indexer-wide concurrency budget.
47
97
  let inFlightCount = sourceManager => sourceManager.fetchingPartitionsCount
@@ -183,9 +233,11 @@ let make = (
183
233
  pendingHeightResolvers: [],
184
234
  disabled: false,
185
235
  lastFailedAt: None,
236
+ requestStats: Dict.make(),
186
237
  }),
187
238
  activeSource: initialActiveSource,
188
239
  waitingForNewBlockStateId: None,
240
+ waitingLogged: false,
189
241
  fetchingPartitionsCount: 0,
190
242
  newBlockStallTimeout,
191
243
  newBlockStallTimeoutRealtime,
@@ -193,7 +245,7 @@ let make = (
193
245
  reducedPollingInterval,
194
246
  getHeightRetryInterval,
195
247
  recoveryTimeout,
196
- statusStart: Hrtime.makeTimer(),
248
+ statusStart: Performance.now(),
197
249
  status: Idle,
198
250
  hasRealtime,
199
251
  committedRateLimitTimeMs: 0.0,
@@ -211,9 +263,9 @@ let trackNewStatus = (sourceManager: t, ~newStatus) => {
211
263
  }
212
264
  promCounter->Prometheus.SafeCounter.handleFloat(
213
265
  ~labels=sourceManager.activeSource.chain->ChainMap.Chain.toChainId,
214
- ~value=sourceManager.statusStart->Hrtime.timeSince->Hrtime.toSecondsFloat,
266
+ ~value=sourceManager.statusStart->Performance.secondsSince,
215
267
  )
216
- sourceManager.statusStart = Hrtime.makeTimer()
268
+ sourceManager.statusStart = Performance.now()
217
269
  sourceManager.status = newStatus
218
270
  }
219
271
 
@@ -339,7 +391,9 @@ let getSourceNewHeight = async (
339
391
  let h = ref(initialHeight)
340
392
  while h.contents <= knownHeight && !(newHeight.contents > initialHeight) {
341
393
  try {
342
- h := (await source.getHeightOrThrow())
394
+ let res = await source.getHeightOrThrow()
395
+ sourceState->recordRequestStats(res.requestStats)
396
+ h := res.height
343
397
  } catch {
344
398
  | _ => ()
345
399
  }
@@ -358,7 +412,9 @@ let getSourceNewHeight = async (
358
412
  | None =>
359
413
  // No subscription, use REST polling
360
414
  try {
361
- let height = await source.getHeightOrThrow()
415
+ let res = await source.getHeightOrThrow()
416
+ sourceState->recordRequestStats(res.requestStats)
417
+ let height = res.height
362
418
 
363
419
  newHeight := height
364
420
  if height <= knownHeight {
@@ -380,6 +436,9 @@ let getSourceNewHeight = async (
380
436
  }
381
437
  })
382
438
  sourceState.unsubscribe = Some(unsubscribe)
439
+ // Count a subscription (re)start rather than every pushed height —
440
+ // there's no request/response to time here.
441
+ sourceState->recordRequestStats([{Source.method: "heightSubscription", seconds: 0.}])
383
442
  | _ =>
384
443
  // Slowdown polling when the chain isn't progressing
385
444
  let pollingInterval = if reducedPolling {
@@ -496,13 +555,14 @@ let waitForNewBlock = async (sourceManager: t, ~knownHeight, ~isRealtime, ~reduc
496
555
  "knownHeight": knownHeight,
497
556
  },
498
557
  )
499
- if reducedPolling {
558
+ if !sourceManager.waitingLogged {
500
559
  logger->Logging.childTrace(
501
- `Waiting for new blocks with reduced polling (${(sourceManager.reducedPollingInterval / 1000)
502
- ->Int.toString}s). Chain is caught up, waiting for other chains to backfill.`,
560
+ reducedPolling
561
+ ? `Waiting for new blocks with reduced polling (${(sourceManager.reducedPollingInterval / 1000)
562
+ ->Int.toString}s). Chain is caught up, waiting for other chains to backfill.`
563
+ : "Initiating check for new blocks.",
503
564
  )
504
- } else {
505
- logger->Logging.childTrace("Initiating check for new blocks.")
565
+ sourceManager.waitingLogged = true
506
566
  }
507
567
 
508
568
  let mainSources = sourceManager->getNextSources(~isRealtime)
@@ -600,6 +660,7 @@ let waitForNewBlock = async (sourceManager: t, ~knownHeight, ~isRealtime, ~reduc
600
660
  "source": source.name,
601
661
  "newBlockHeight": newBlockHeight,
602
662
  })
663
+ sourceManager.waitingLogged = false
603
664
 
604
665
  status := Done
605
666
 
@@ -669,19 +730,15 @@ let executeQuery = async (
669
730
  ~fromBlock=query.fromBlock,
670
731
  ~toBlock,
671
732
  ~addressesByContractName=query.addressesByContractName,
672
- ~indexingAddresses=query.indexingAddresses,
733
+ ~contractNameByAddress=query.addressesByContractName->FetchState.deriveContractNameByAddress,
673
734
  ~partitionId=query.partitionId,
674
735
  ~knownHeight,
675
736
  ~selection=query.selection,
737
+ ~itemsTarget=query.itemsTarget,
676
738
  ~retry,
677
739
  ~logger,
678
740
  )
679
- logger->Logging.childTrace({
680
- "msg": "Fetched block range from server",
681
- "toBlock": response.latestFetchedBlockNumber,
682
- "numEvents": response.parsedQueueItems->Array.length,
683
- "stats": response.stats,
684
- })
741
+ sourceState->recordRequestStats(response.requestStats)
685
742
  sourceState.lastFailedAt = None
686
743
  responseRef := Some(response)
687
744
  } catch {
@@ -821,7 +878,8 @@ let getBlockHashes = async (sourceManager: t, ~blockNumbers: array<int>, ~isReal
821
878
 
822
879
  try {
823
880
  let res = await source.getBlockHashes(~blockNumbers, ~logger)
824
- switch res {
881
+ sourceState->recordRequestStats(res.requestStats)
882
+ switch res.result {
825
883
  | Ok(data) =>
826
884
  sourceState.lastFailedAt = None
827
885
  responseRef := Some(data)
@@ -1,11 +1,11 @@
1
1
  // Generated by ReScript, PLEASE EDIT WITH CARE
2
2
 
3
3
  import * as Utils from "../Utils.res.mjs";
4
- import * as Hrtime from "../bindings/Hrtime.res.mjs";
5
4
  import * as Source from "./Source.res.mjs";
6
5
  import * as Logging from "../Logging.res.mjs";
7
6
  import * as FetchState from "../FetchState.res.mjs";
8
7
  import * as Prometheus from "../Prometheus.res.mjs";
8
+ import * as Performance from "../bindings/Performance.res.mjs";
9
9
  import * as ErrorHandling from "../ErrorHandling.res.mjs";
10
10
  import * as Primitive_int from "@rescript/runtime/lib/es6/Primitive_int.js";
11
11
  import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
@@ -14,10 +14,44 @@ import * as Primitive_float from "@rescript/runtime/lib/es6/Primitive_float.js";
14
14
  import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js";
15
15
  import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
16
16
 
17
+ function recordRequestStats(sourceState, requestStats) {
18
+ requestStats.forEach(param => {
19
+ let seconds = param.seconds;
20
+ let method = param.method;
21
+ let agg = sourceState.requestStats[method];
22
+ if (agg !== undefined) {
23
+ agg.count = agg.count + 1 | 0;
24
+ agg.seconds = agg.seconds + seconds;
25
+ } else {
26
+ sourceState.requestStats[method] = {
27
+ count: 1,
28
+ seconds: seconds
29
+ };
30
+ }
31
+ });
32
+ }
33
+
17
34
  function getActiveSource(sourceManager) {
18
35
  return sourceManager.activeSource;
19
36
  }
20
37
 
38
+ function getRequestStatSamples(sourceManager) {
39
+ let samples = [];
40
+ sourceManager.sourcesState.forEach(sourceState => {
41
+ let chainId = sourceState.source.chain;
42
+ Utils.Dict.forEachWithKey(sourceState.requestStats, (agg, method) => {
43
+ samples.push({
44
+ sourceName: sourceState.source.name,
45
+ chainId: chainId,
46
+ method: method,
47
+ count: agg.count,
48
+ seconds: agg.seconds
49
+ });
50
+ });
51
+ });
52
+ return samples;
53
+ }
54
+
21
55
  function inFlightCount(sourceManager) {
22
56
  return sourceManager.fetchingPartitionsCount;
23
57
  }
@@ -137,9 +171,10 @@ function make(sources, isRealtime, newBlockStallTimeoutOpt, newBlockStallTimeout
137
171
  unsubscribe: undefined,
138
172
  pendingHeightResolvers: [],
139
173
  disabled: false,
140
- lastFailedAt: undefined
174
+ lastFailedAt: undefined,
175
+ requestStats: {}
141
176
  })),
142
- statusStart: Hrtime.makeTimer(),
177
+ statusStart: Performance.now(),
143
178
  status: "Idle",
144
179
  newBlockStallTimeout: newBlockStallTimeout,
145
180
  newBlockStallTimeoutRealtime: newBlockStallTimeoutRealtime,
@@ -148,6 +183,7 @@ function make(sources, isRealtime, newBlockStallTimeoutOpt, newBlockStallTimeout
148
183
  getHeightRetryInterval: getHeightRetryInterval,
149
184
  activeSource: initialActiveSource,
150
185
  waitingForNewBlockStateId: undefined,
186
+ waitingLogged: false,
151
187
  fetchingPartitionsCount: 0,
152
188
  recoveryTimeout: recoveryTimeout,
153
189
  hasRealtime: hasRealtime,
@@ -172,8 +208,8 @@ function trackNewStatus(sourceManager, newStatus) {
172
208
  promCounter = Prometheus.IndexingQueryTime.counter;
173
209
  break;
174
210
  }
175
- Prometheus.SafeCounter.handleFloat(promCounter, sourceManager.activeSource.chain, Hrtime.toSecondsFloat(Hrtime.timeSince(sourceManager.statusStart)));
176
- sourceManager.statusStart = Hrtime.makeTimer();
211
+ Prometheus.SafeCounter.handleFloat(promCounter, sourceManager.activeSource.chain, Performance.secondsSince(sourceManager.statusStart));
212
+ sourceManager.statusStart = Performance.now();
177
213
  sourceManager.status = newStatus;
178
214
  }
179
215
 
@@ -262,7 +298,9 @@ async function getSourceNewHeight(sourceManager, sourceState, knownHeight, stall
262
298
  let h = initialHeight;
263
299
  while (h <= knownHeight && newHeight.contents <= initialHeight) {
264
300
  try {
265
- h = await source.getHeightOrThrow();
301
+ let res = await source.getHeightOrThrow();
302
+ recordRequestStats(sourceState, res.requestStats);
303
+ h = res.height;
266
304
  } catch (exn) {
267
305
 
268
306
  }
@@ -281,7 +319,9 @@ async function getSourceNewHeight(sourceManager, sourceState, knownHeight, stall
281
319
  }
282
320
  } else {
283
321
  try {
284
- let height$1 = await source.getHeightOrThrow();
322
+ let res = await source.getHeightOrThrow();
323
+ recordRequestStats(sourceState, res.requestStats);
324
+ let height$1 = res.height;
285
325
  newHeight.contents = height$1;
286
326
  if (height$1 <= knownHeight) {
287
327
  retry = 0;
@@ -298,6 +338,10 @@ async function getSourceNewHeight(sourceManager, sourceState, knownHeight, stall
298
338
  resolvers.forEach(resolve => resolve(newHeight));
299
339
  });
300
340
  sourceState.unsubscribe = unsubscribe;
341
+ recordRequestStats(sourceState, [{
342
+ method: "heightSubscription",
343
+ seconds: 0
344
+ }]);
301
345
  } else {
302
346
  exit = 1;
303
347
  }
@@ -408,10 +452,9 @@ async function waitForNewBlock(sourceManager, knownHeight, isRealtime, reducedPo
408
452
  chainId: sourceManager.activeSource.chain,
409
453
  knownHeight: knownHeight
410
454
  });
411
- if (reducedPolling) {
412
- Logging.childTrace(logger, `Waiting for new blocks with reduced polling (` + (sourceManager.reducedPollingInterval / 1000 | 0).toString() + `s). Chain is caught up, waiting for other chains to backfill.`);
413
- } else {
414
- Logging.childTrace(logger, "Initiating check for new blocks.");
455
+ if (!sourceManager.waitingLogged) {
456
+ Logging.childTrace(logger, reducedPolling ? `Waiting for new blocks with reduced polling (` + (sourceManager.reducedPollingInterval / 1000 | 0).toString() + `s). Chain is caught up, waiting for other chains to backfill.` : "Initiating check for new blocks.");
457
+ sourceManager.waitingLogged = true;
415
458
  }
416
459
  let mainSources = getNextSources(sourceManager, isRealtime, undefined);
417
460
  let status = {
@@ -453,6 +496,7 @@ async function waitForNewBlock(sourceManager, knownHeight, isRealtime, reducedPo
453
496
  source: source.name,
454
497
  newBlockHeight: newBlockHeight
455
498
  });
499
+ sourceManager.waitingLogged = false;
456
500
  status.contents = "Done";
457
501
  return newBlockHeight;
458
502
  }
@@ -499,13 +543,8 @@ async function executeQuery(sourceManager, query, knownHeight, isRealtime) {
499
543
  retry: retry
500
544
  });
501
545
  try {
502
- let response = await source.getItemsOrThrow(query.fromBlock, toBlock, query.addressesByContractName, query.indexingAddresses, knownHeight, query.partitionId, query.selection, retry, logger$2);
503
- Logging.childTrace(logger$2, {
504
- msg: "Fetched block range from server",
505
- toBlock: response.latestFetchedBlockNumber,
506
- numEvents: response.parsedQueueItems.length,
507
- stats: response.stats
508
- });
546
+ let response = await source.getItemsOrThrow(query.fromBlock, toBlock, query.addressesByContractName, FetchState.deriveContractNameByAddress(query.addressesByContractName), knownHeight, query.partitionId, query.selection, query.itemsTarget, retry, logger$2);
547
+ recordRequestStats(sourceState, response.requestStats);
509
548
  sourceState.lastFailedAt = undefined;
510
549
  responseRef = response;
511
550
  } catch (raw_error) {
@@ -641,11 +680,13 @@ async function getBlockHashes(sourceManager, blockNumbers, isRealtime) {
641
680
  });
642
681
  try {
643
682
  let res = await source.getBlockHashes(blockNumbers, logger$1);
644
- if (res.TAG === "Ok") {
683
+ recordRequestStats(sourceState, res.requestStats);
684
+ let data = res.result;
685
+ if (data.TAG === "Ok") {
645
686
  sourceState.lastFailedAt = undefined;
646
- responseRef = res._0;
687
+ responseRef = data._0;
647
688
  } else {
648
- throw res._0;
689
+ throw data._0;
649
690
  }
650
691
  } catch (raw_exn) {
651
692
  let exn = Primitive_exceptions.internalToException(raw_exn);
@@ -677,6 +718,7 @@ export {
677
718
  getSourceRole,
678
719
  make,
679
720
  getActiveSource,
721
+ getRequestStatSamples,
680
722
  inFlightCount,
681
723
  onReorg,
682
724
  dispatch,
@@ -21,6 +21,16 @@ let make: (
21
21
 
22
22
  let getActiveSource: t => Source.t
23
23
 
24
+ type requestStatSample = {
25
+ sourceName: string,
26
+ chainId: int,
27
+ method: string,
28
+ count: int,
29
+ seconds: float,
30
+ }
31
+
32
+ let getRequestStatSamples: t => array<requestStatSample>
33
+
24
34
  let inFlightCount: t => int
25
35
 
26
36
  let onReorg: (t, ~rollbackTargetBlock: int) => unit