envio 3.3.0-alpha.6 → 3.3.0-alpha.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (93) hide show
  1. package/index.d.ts +23 -5
  2. package/package.json +6 -6
  3. package/src/Address.res +5 -2
  4. package/src/Address.res.mjs +3 -1
  5. package/src/Api.res +1 -1
  6. package/src/Api.res.mjs +1 -1
  7. package/src/ChainFetching.res +34 -34
  8. package/src/ChainFetching.res.mjs +29 -29
  9. package/src/ChainState.res +328 -165
  10. package/src/ChainState.res.mjs +205 -100
  11. package/src/ChainState.resi +16 -7
  12. package/src/Config.res +55 -27
  13. package/src/Config.res.mjs +46 -7
  14. package/src/ContractRegisterContext.res +2 -12
  15. package/src/ContractRegisterContext.res.mjs +3 -5
  16. package/src/Core.res +7 -0
  17. package/src/CrossChainState.res +54 -79
  18. package/src/CrossChainState.res.mjs +45 -60
  19. package/src/CrossChainState.resi +1 -1
  20. package/src/Ecosystem.res +3 -3
  21. package/src/Ecosystem.res.mjs +3 -3
  22. package/src/Envio.res +14 -9
  23. package/src/EventConfigBuilder.res +105 -56
  24. package/src/EventConfigBuilder.res.mjs +69 -33
  25. package/src/EventProcessing.res +19 -15
  26. package/src/EventProcessing.res.mjs +13 -12
  27. package/src/FetchState.res +383 -182
  28. package/src/FetchState.res.mjs +260 -264
  29. package/src/HandlerLoader.res +7 -112
  30. package/src/HandlerLoader.res.mjs +1 -87
  31. package/src/HandlerRegister.res +209 -6
  32. package/src/HandlerRegister.res.mjs +90 -5
  33. package/src/HandlerRegister.resi +13 -2
  34. package/src/IndexerState.res +6 -3
  35. package/src/IndexerState.res.mjs +3 -3
  36. package/src/IndexerState.resi +1 -1
  37. package/src/IndexingAddresses.res +10 -7
  38. package/src/IndexingAddresses.res.mjs +8 -7
  39. package/src/IndexingAddresses.resi +3 -1
  40. package/src/Internal.res +82 -36
  41. package/src/Internal.res.mjs +11 -1
  42. package/src/Main.res +22 -23
  43. package/src/Main.res.mjs +15 -17
  44. package/src/Metrics.res +43 -2
  45. package/src/Metrics.res.mjs +39 -3
  46. package/src/Prometheus.res +0 -35
  47. package/src/Prometheus.res.mjs +33 -68
  48. package/src/RawEvent.res +7 -2
  49. package/src/RawEvent.res.mjs +4 -4
  50. package/src/SimulateItems.res +26 -8
  51. package/src/SimulateItems.res.mjs +11 -14
  52. package/src/TestIndexer.res +1 -1
  53. package/src/TestIndexer.res.mjs +1 -1
  54. package/src/sources/BlockStore.res +46 -0
  55. package/src/sources/BlockStore.res.mjs +24 -0
  56. package/src/sources/EventRouter.res +19 -12
  57. package/src/sources/EventRouter.res.mjs +7 -5
  58. package/src/sources/Evm.res +54 -4
  59. package/src/sources/Evm.res.mjs +43 -4
  60. package/src/sources/EvmChain.res +14 -18
  61. package/src/sources/EvmChain.res.mjs +12 -13
  62. package/src/sources/FieldMask.res +39 -0
  63. package/src/sources/FieldMask.res.mjs +42 -0
  64. package/src/sources/Fuel.res +5 -2
  65. package/src/sources/Fuel.res.mjs +4 -3
  66. package/src/sources/HyperFuelSource.res +32 -40
  67. package/src/sources/HyperFuelSource.res.mjs +53 -45
  68. package/src/sources/HyperSync.res +35 -9
  69. package/src/sources/HyperSync.res.mjs +52 -35
  70. package/src/sources/HyperSync.resi +11 -3
  71. package/src/sources/HyperSyncClient.res +15 -4
  72. package/src/sources/HyperSyncHeightStream.res +1 -8
  73. package/src/sources/HyperSyncHeightStream.res.mjs +0 -2
  74. package/src/sources/HyperSyncSource.res +39 -58
  75. package/src/sources/HyperSyncSource.res.mjs +44 -44
  76. package/src/sources/RpcSource.res +114 -92
  77. package/src/sources/RpcSource.res.mjs +77 -46
  78. package/src/sources/RpcWebSocketHeightStream.res +0 -5
  79. package/src/sources/RpcWebSocketHeightStream.res.mjs +0 -2
  80. package/src/sources/SimulateSource.res +6 -3
  81. package/src/sources/SimulateSource.res.mjs +13 -5
  82. package/src/sources/Source.res +25 -5
  83. package/src/sources/SourceManager.res +66 -3
  84. package/src/sources/SourceManager.res.mjs +55 -7
  85. package/src/sources/SourceManager.resi +10 -0
  86. package/src/sources/Svm.res +15 -10
  87. package/src/sources/Svm.res.mjs +24 -7
  88. package/src/sources/SvmHyperSyncClient.res +5 -6
  89. package/src/sources/SvmHyperSyncSource.res +84 -41
  90. package/src/sources/SvmHyperSyncSource.res.mjs +90 -43
  91. package/src/sources/TransactionStore.res +6 -107
  92. package/src/sources/TransactionStore.res.mjs +5 -86
  93. package/svm.schema.json +19 -0
@@ -3,7 +3,6 @@
3
3
  import * as Source from "./Source.res.mjs";
4
4
  import * as Logging from "../Logging.res.mjs";
5
5
  import * as HyperFuel from "./HyperFuel.res.mjs";
6
- import * as Prometheus from "../Prometheus.res.mjs";
7
6
  import * as EventRouter from "./EventRouter.res.mjs";
8
7
  import * as Performance from "../bindings/Performance.res.mjs";
9
8
  import * as Stdlib_JsExn from "@rescript/runtime/lib/es6/Stdlib_JsExn.js";
@@ -101,41 +100,43 @@ function getSelectionConfig(selection, chain) {
101
100
  }
102
101
  };
103
102
  let contractNames = new Set();
104
- selection.eventConfigs.forEach(eventConfig => {
103
+ selection.onEventRegistrations.forEach(reg => {
104
+ let eventConfig = reg.eventConfig;
105
105
  let contractName = eventConfig.contractName;
106
- if (!eventConfig.isWildcard) {
106
+ let isWildcard = reg.isWildcard;
107
+ if (!isWildcard) {
107
108
  contractNames.add(contractName);
108
109
  }
109
- EventRouter.addOrThrow(eventRouter, eventConfig.id, eventConfig, contractName, eventConfig.isWildcard, eventConfig.name, chain);
110
- let isWildcard = eventConfig.isWildcard;
111
- if (isWildcard) {
112
- let tmp = eventConfig.kind;
113
- if (typeof tmp !== "object") {
114
- switch (tmp) {
115
- case "Mint" :
116
- return addNonLogDataWildcardReceiptTypes(11);
117
- case "Burn" :
118
- return addNonLogDataWildcardReceiptTypes(12);
119
- case "Transfer" :
120
- addNonLogDataWildcardReceiptTypes(7);
121
- return addNonLogDataWildcardReceiptTypes(8);
122
- case "Call" :
123
- return addNonLogDataWildcardReceiptTypes(0);
124
- }
125
- }
126
- }
110
+ EventRouter.addOrThrow(eventRouter, eventConfig.id, reg, contractName, isWildcard, eventConfig.name, chain);
127
111
  let match = eventConfig.kind;
128
112
  if (typeof match !== "object") {
129
113
  switch (match) {
130
114
  case "Mint" :
131
- return addNonLogDataReceiptType(contractName, 11);
115
+ if (isWildcard) {
116
+ return addNonLogDataWildcardReceiptTypes(11);
117
+ } else {
118
+ return addNonLogDataReceiptType(contractName, 11);
119
+ }
132
120
  case "Burn" :
133
- return addNonLogDataReceiptType(contractName, 12);
121
+ if (isWildcard) {
122
+ return addNonLogDataWildcardReceiptTypes(12);
123
+ } else {
124
+ return addNonLogDataReceiptType(contractName, 12);
125
+ }
134
126
  case "Transfer" :
135
- addNonLogDataReceiptType(contractName, 7);
136
- return addNonLogDataReceiptType(contractName, 8);
127
+ if (isWildcard) {
128
+ addNonLogDataWildcardReceiptTypes(7);
129
+ return addNonLogDataWildcardReceiptTypes(8);
130
+ } else {
131
+ addNonLogDataReceiptType(contractName, 7);
132
+ return addNonLogDataReceiptType(contractName, 8);
133
+ }
137
134
  case "Call" :
138
- return Stdlib_JsError.throwWithMessage("Call receipt indexing currently supported only in wildcard mode");
135
+ if (isWildcard) {
136
+ return addNonLogDataWildcardReceiptTypes(0);
137
+ } else {
138
+ return Stdlib_JsError.throwWithMessage("Call receipt indexing currently supported only in wildcard mode");
139
+ }
139
140
  }
140
141
  } else {
141
142
  let rb = BigInt(match.logId);
@@ -181,7 +182,6 @@ function memoGetSelectionConfig(chain) {
181
182
  function make(param) {
182
183
  let apiToken = param.apiToken;
183
184
  let chain = param.chain;
184
- let name = "HyperFuel";
185
185
  let apiToken$1 = apiToken !== undefined ? apiToken : Stdlib_JsError.throwWithMessage(`An Envio API token is required for using HyperFuel as a data-source.
186
186
  Set the ENVIO_API_TOKEN environment variable in your .env file.
187
187
  Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`);
@@ -196,12 +196,11 @@ Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`);
196
196
  client = ErrorHandling.mkLogAndRaise(undefined, "Failed to instantiate the HyperFuel client", exn);
197
197
  }
198
198
  let getSelectionConfig = memoGetSelectionConfig(chain);
199
- let getItemsOrThrow = async (fromBlock, toBlock, addressesByContractName, contractNameByAddress, knownHeight, param, selection, retry, logger) => {
199
+ let getItemsOrThrow = async (fromBlock, toBlock, addressesByContractName, contractNameByAddress, knownHeight, param, selection, param$1, retry, logger) => {
200
200
  let totalTimeRef = Performance.now();
201
201
  let selectionConfig = getSelectionConfig(selection);
202
202
  let recieptsSelection = selectionConfig.getRecieptsSelection(addressesByContractName);
203
203
  let startFetchingBatchTimeRef = Performance.now();
204
- Prometheus.SourceRequestCount.increment(name, chain, "getLogs");
205
204
  let pageUnsafe;
206
205
  try {
207
206
  pageUnsafe = await HyperFuel.GetLogs.query(client, fromBlock, toBlock, recieptsSelection);
@@ -250,6 +249,10 @@ Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`);
250
249
  };
251
250
  }
252
251
  let pageFetchTime = Performance.secondsSince(startFetchingBatchTimeRef);
252
+ let requestStats = [{
253
+ method: "getLogs",
254
+ seconds: pageFetchTime
255
+ }];
253
256
  let knownHeight$1 = pageUnsafe.archiveHeight;
254
257
  let heighestBlockQueried = pageUnsafe.nextBlock - 1 | 0;
255
258
  let parsingTimeRef = Performance.now();
@@ -277,10 +280,10 @@ Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`);
277
280
  eventId = burnEventTag;
278
281
  break;
279
282
  }
280
- let eventConfig = EventRouter.get(selectionConfig.eventRouter, eventId, contractAddress, contractNameByAddress);
281
- let eventConfig$1;
282
- if (eventConfig !== undefined) {
283
- eventConfig$1 = eventConfig;
283
+ let onEventRegistration = EventRouter.get(selectionConfig.eventRouter, eventId, contractAddress, contractNameByAddress);
284
+ let onEventRegistration$1;
285
+ if (onEventRegistration !== undefined) {
286
+ onEventRegistration$1 = onEventRegistration;
284
287
  } else {
285
288
  let logger$1 = Logging.createChildFrom(logger, {
286
289
  chainId: chain,
@@ -289,11 +292,12 @@ Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`);
289
292
  contractAddress: contractAddress,
290
293
  eventId: eventId
291
294
  });
292
- eventConfig$1 = ErrorHandling.mkLogAndRaise(logger$1, "Failed to route registered event", {
295
+ onEventRegistration$1 = ErrorHandling.mkLogAndRaise(logger$1, "Failed to route registered event", {
293
296
  RE_EXN_ID: EventRoutingFailed
294
297
  });
295
298
  }
296
- let match = eventConfig$1.kind;
299
+ let eventConfig = onEventRegistration$1.eventConfig;
300
+ let match = eventConfig.kind;
297
301
  let params;
298
302
  let exit = 0;
299
303
  if (typeof match !== "object" || receipt.receiptType !== 6) {
@@ -349,16 +353,14 @@ Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`);
349
353
  }
350
354
  return {
351
355
  kind: 0,
352
- eventConfig: eventConfig$1,
353
- timestamp: block.time,
356
+ onEventRegistration: onEventRegistration$1,
354
357
  chain: chain,
355
358
  blockNumber: block.height,
356
- blockHash: block.id,
357
359
  logIndex: receiptIndex,
358
360
  transactionIndex: 0,
359
361
  payload: {
360
- contractName: eventConfig$1.contractName,
361
- eventName: eventConfig$1.name,
362
+ contractName: eventConfig.contractName,
363
+ eventName: eventConfig.name,
362
364
  params: params,
363
365
  chainId: chain,
364
366
  srcAddress: contractAddress,
@@ -399,15 +401,17 @@ Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`);
399
401
  blockHashes: blockHashes,
400
402
  parsedQueueItems: parsedQueueItems,
401
403
  transactionStore: undefined,
404
+ blockStore: undefined,
402
405
  fromBlockQueried: fromBlock,
403
406
  latestFetchedBlockNumber: heighestBlockQueried,
404
407
  latestFetchedBlockTimestamp: latestFetchedBlockTimestamp,
405
- stats: stats
408
+ stats: stats,
409
+ requestStats: requestStats
406
410
  };
407
411
  };
408
412
  let getBlockHashes = (param, param$1) => Stdlib_JsError.throwWithMessage("HyperFuel does not support getting block hashes");
409
413
  return {
410
- name: name,
414
+ name: "HyperFuel",
411
415
  sourceFor: "Sync",
412
416
  chain: chain,
413
417
  poweredByHyperSync: true,
@@ -447,9 +451,13 @@ Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`);
447
451
  }
448
452
  }
449
453
  let seconds = Performance.secondsSince(timerRef);
450
- Prometheus.SourceRequestCount.increment(name, chain, "getHeight");
451
- Prometheus.SourceRequestCount.addSeconds(name, chain, "getHeight", seconds);
452
- return height;
454
+ return {
455
+ height: height,
456
+ requestStats: [{
457
+ method: "getHeight",
458
+ seconds: seconds
459
+ }]
460
+ };
453
461
  },
454
462
  getItemsOrThrow: getItemsOrThrow
455
463
  };
@@ -16,13 +16,15 @@ let reraisIfRateLimited = exn =>
16
16
 
17
17
  type logsQueryPage = {
18
18
  items: array<HyperSyncClient.EventItems.item>,
19
- // Blocks referenced by `items`, deduplicated by block number.
20
- blocks: array<HyperSyncClient.ResponseTypes.block>,
19
+ // Block headers referenced by `items`, deduplicated by block number.
20
+ blocks: array<HyperSyncClient.EventItems.blockHeader>,
21
21
  nextBlock: int,
22
22
  archiveHeight: int,
23
23
  rollbackGuard: option<HyperSyncClient.ResponseTypes.rollbackGuard>,
24
24
  // Page store owning this page's raw transactions.
25
25
  transactionStore: TransactionStore.t,
26
+ // Page store owning this page's raw blocks.
27
+ blockStore: BlockStore.t,
26
28
  }
27
29
 
28
30
  type missingParams = {
@@ -64,6 +66,7 @@ module GetLogs = {
64
66
  ~toBlockInclusive,
65
67
  ~addressesWithTopics,
66
68
  ~fieldSelection,
69
+ ~maxNumLogs,
67
70
  ): HyperSyncClient.QueryTypes.query => {
68
71
  fromBlock,
69
72
  toBlockExclusive: ?switch toBlockInclusive {
@@ -72,6 +75,7 @@ module GetLogs = {
72
75
  },
73
76
  logs: addressesWithTopics,
74
77
  fieldSelection,
78
+ maxNumLogs,
75
79
  }
76
80
 
77
81
  // Rust encodes structured failures as a JSON payload in the napi error's
@@ -105,6 +109,7 @@ module GetLogs = {
105
109
  ~toBlock,
106
110
  ~logSelections: array<LogSelection.t>,
107
111
  ~fieldSelection,
112
+ ~maxNumLogs,
108
113
  ): logsQueryPage => {
109
114
  let addressesWithTopics = logSelections->Array.flatMap(({addresses, topicSelections}) =>
110
115
  topicSelections->Array.map(({topic0, topic1, topic2, topic3}) => {
@@ -123,9 +128,10 @@ module GetLogs = {
123
128
  ~toBlockInclusive=toBlock,
124
129
  ~addressesWithTopics,
125
130
  ~fieldSelection,
131
+ ~maxNumLogs,
126
132
  )
127
133
 
128
- let (res, transactionStore) = switch await client.getEventItems(~query) {
134
+ let (res, transactionStore, blockStore) = switch await client.getEventItems(~query) {
129
135
  | res => res
130
136
  | exception exn =>
131
137
  reraisIfRateLimited(exn)
@@ -146,6 +152,7 @@ module GetLogs = {
146
152
  archiveHeight: res.archiveHeight->Option.getOr(0), //Archive Height is only None if height is 0
147
153
  rollbackGuard: res.rollbackGuard,
148
154
  transactionStore,
155
+ blockStore,
149
156
  }
150
157
  }
151
158
  }
@@ -202,6 +209,7 @@ module BlockData = {
202
209
  ~sourceName,
203
210
  ~chainId,
204
211
  ~logger,
212
+ ~requestStats: array<Source.requestStat>,
205
213
  ): queryResponse<array<ReorgDetection.blockDataWithTimestamp>> => {
206
214
  let body = makeRequestBody(~fromBlock, ~toBlock)
207
215
 
@@ -214,7 +222,7 @@ module BlockData = {
214
222
  },
215
223
  )
216
224
 
217
- Prometheus.SourceRequestCount.increment(~sourceName, ~chainId, ~method="getBlockHashes")
225
+ let timerRef = Performance.now()
218
226
  let maybeSuccessfulRes = switch await client.get(~query=body) {
219
227
  | exception exn =>
220
228
  reraisIfRateLimited(exn)
@@ -222,6 +230,9 @@ module BlockData = {
222
230
  | res if res.nextBlock <= fromBlock => None
223
231
  | res => Some(res)
224
232
  }
233
+ requestStats
234
+ ->Array.push({Source.method: "getBlockHashes", seconds: timerRef->Performance.secondsSince})
235
+ ->ignore
225
236
 
226
237
  // If the block is not found, retry the query. This can occur since replicas of hypersync might not have caught up yet
227
238
  switch maybeSuccessfulRes {
@@ -231,7 +242,15 @@ module BlockData = {
231
242
  `Block #${fromBlock->Int.toString} not found in HyperSync. HyperSync has multiple instances and it's possible that they drift independently slightly from the head. Indexing should continue correctly after retrying the query in ${delayMilliseconds->Int.toString}ms.`,
232
243
  )
233
244
  await Time.resolvePromiseAfterDelay(~delayMilliseconds)
234
- await queryBlockData(~client, ~fromBlock, ~toBlock, ~sourceName, ~chainId, ~logger)
245
+ await queryBlockData(
246
+ ~client,
247
+ ~fromBlock,
248
+ ~toBlock,
249
+ ~sourceName,
250
+ ~chainId,
251
+ ~logger,
252
+ ~requestStats,
253
+ )
235
254
  }
236
255
  | Some(res) =>
237
256
  switch res->convertResponse {
@@ -244,6 +263,7 @@ module BlockData = {
244
263
  ~sourceName,
245
264
  ~chainId,
246
265
  ~logger,
266
+ ~requestStats,
247
267
  )
248
268
  restRes->Result.map(rest => datas->Array.concat(rest))
249
269
  }
@@ -258,8 +278,9 @@ module BlockData = {
258
278
  ~sourceName,
259
279
  ~chainId,
260
280
  ~logger,
261
- ) => {
262
- switch blockNumbers->Array.get(0) {
281
+ ): (queryResponse<array<ReorgDetection.blockDataWithTimestamp>>, array<Source.requestStat>) => {
282
+ let requestStats = []
283
+ let result = switch blockNumbers->Array.get(0) {
263
284
  | None => Ok([])
264
285
  | Some(firstBlock) => {
265
286
  let fromBlock = ref(firstBlock)
@@ -287,6 +308,7 @@ module BlockData = {
287
308
  ~sourceName,
288
309
  ~chainId,
289
310
  ~logger,
311
+ ~requestStats,
290
312
  )
291
313
  let filtered = res->Result.map(datas => {
292
314
  datas->Array.filter(data => set->Utils.Set.delete(data.blockNumber))
@@ -301,10 +323,12 @@ module BlockData = {
301
323
  filtered
302
324
  }
303
325
  }
326
+ (result, requestStats)
304
327
  }
305
328
  }
306
329
 
307
- let queryBlockData = (~client, ~blockNumber, ~sourceName, ~chainId, ~logger) =>
330
+ let queryBlockData = (~client, ~blockNumber, ~sourceName, ~chainId, ~logger) => {
331
+ let requestStats = []
308
332
  BlockData.queryBlockData(
309
333
  ~client,
310
334
  ~fromBlock=blockNumber,
@@ -312,5 +336,7 @@ let queryBlockData = (~client, ~blockNumber, ~sourceName, ~chainId, ~logger) =>
312
336
  ~sourceName,
313
337
  ~chainId,
314
338
  ~logger,
315
- )->Promise.thenResolve(res => res->Result.map(res => res->Array.get(0)))
339
+ ~requestStats,
340
+ )->Promise.thenResolve(res => (res->Result.map(res => res->Array.get(0)), requestStats))
341
+ }
316
342
  let queryBlockDataMulti = BlockData.queryBlockDataMulti
@@ -4,8 +4,8 @@ import * as Time from "../Time.res.mjs";
4
4
  import * as Utils from "../Utils.res.mjs";
5
5
  import * as Source from "./Source.res.mjs";
6
6
  import * as Logging from "../Logging.res.mjs";
7
- import * as Prometheus from "../Prometheus.res.mjs";
8
7
  import * as Stdlib_Int from "@rescript/runtime/lib/es6/Stdlib_Int.js";
8
+ import * as Performance from "../bindings/Performance.res.mjs";
9
9
  import * as Stdlib_JSON from "@rescript/runtime/lib/es6/Stdlib_JSON.js";
10
10
  import * as Stdlib_Array from "@rescript/runtime/lib/es6/Stdlib_Array.js";
11
11
  import * as Stdlib_JsExn from "@rescript/runtime/lib/es6/Stdlib_JsExn.js";
@@ -62,12 +62,13 @@ function mapExn(queryResponse) {
62
62
 
63
63
  let $$Error = /* @__PURE__ */Primitive_exceptions.create("HyperSync.GetLogs.Error");
64
64
 
65
- function makeRequestBody(fromBlock, toBlockInclusive, addressesWithTopics, fieldSelection) {
65
+ function makeRequestBody(fromBlock, toBlockInclusive, addressesWithTopics, fieldSelection, maxNumLogs) {
66
66
  return {
67
67
  fromBlock: fromBlock,
68
68
  toBlock: toBlockInclusive !== undefined ? toBlockInclusive + 1 | 0 : undefined,
69
69
  logs: addressesWithTopics,
70
- fieldSelection: fieldSelection
70
+ fieldSelection: fieldSelection,
71
+ maxNumLogs: maxNumLogs
71
72
  };
72
73
  }
73
74
 
@@ -92,7 +93,7 @@ function extractMissingParams(exn) {
92
93
  }
93
94
  }
94
95
 
95
- async function query(client, fromBlock, toBlock, logSelections, fieldSelection) {
96
+ async function query(client, fromBlock, toBlock, logSelections, fieldSelection, maxNumLogs) {
96
97
  let addressesWithTopics = logSelections.flatMap(param => {
97
98
  let addresses = param.addresses;
98
99
  return param.topicSelections.map(param => {
@@ -100,7 +101,7 @@ async function query(client, fromBlock, toBlock, logSelections, fieldSelection)
100
101
  return HyperSyncClient.QueryTypes.makeLogSelection(addresses, topics);
101
102
  });
102
103
  });
103
- let query$1 = makeRequestBody(fromBlock, toBlock, addressesWithTopics, fieldSelection);
104
+ let query$1 = makeRequestBody(fromBlock, toBlock, addressesWithTopics, fieldSelection, maxNumLogs);
104
105
  let match;
105
106
  try {
106
107
  match = await client.getEventItems(query$1);
@@ -134,7 +135,8 @@ async function query(client, fromBlock, toBlock, logSelections, fieldSelection)
134
135
  nextBlock: res.nextBlock,
135
136
  archiveHeight: Stdlib_Option.getOr(res.archiveHeight, 0),
136
137
  rollbackGuard: res.rollbackGuard,
137
- transactionStore: match[1]
138
+ transactionStore: match[1],
139
+ blockStore: match[2]
138
140
  };
139
141
  }
140
142
 
@@ -190,14 +192,14 @@ function convertResponse(res) {
190
192
  }));
191
193
  }
192
194
 
193
- async function queryBlockData(client, fromBlock, toBlock, sourceName, chainId, logger) {
195
+ async function queryBlockData(client, fromBlock, toBlock, sourceName, chainId, logger, requestStats) {
194
196
  let body = makeRequestBody$1(fromBlock, toBlock);
195
197
  let logger$1 = Logging.createChildFrom(logger, {
196
198
  logType: "HyperSync get block hash query",
197
199
  fromBlock: fromBlock,
198
200
  toBlock: toBlock
199
201
  });
200
- Prometheus.SourceRequestCount.increment(sourceName, chainId, "getBlockHashes");
202
+ let timerRef = Performance.now();
201
203
  let maybeSuccessfulRes;
202
204
  let exit = 0;
203
205
  let res;
@@ -212,6 +214,10 @@ async function queryBlockData(client, fromBlock, toBlock, sourceName, chainId, l
212
214
  if (exit === 1) {
213
215
  maybeSuccessfulRes = res.nextBlock <= fromBlock ? undefined : res;
214
216
  }
217
+ requestStats.push({
218
+ method: "getBlockHashes",
219
+ seconds: Performance.secondsSince(timerRef)
220
+ });
215
221
  if (maybeSuccessfulRes !== undefined) {
216
222
  let err = convertResponse(maybeSuccessfulRes);
217
223
  if (err.TAG !== "Ok") {
@@ -221,48 +227,59 @@ async function queryBlockData(client, fromBlock, toBlock, sourceName, chainId, l
221
227
  return err;
222
228
  }
223
229
  let datas = err._0;
224
- let restRes = await queryBlockData(client, maybeSuccessfulRes.nextBlock, toBlock, sourceName, chainId, logger$1);
230
+ let restRes = await queryBlockData(client, maybeSuccessfulRes.nextBlock, toBlock, sourceName, chainId, logger$1, requestStats);
225
231
  return Stdlib_Result.map(restRes, rest => datas.concat(rest));
226
232
  }
227
233
  Logging.childInfo(logger$1, `Block #` + fromBlock.toString() + ` not found in HyperSync. HyperSync has multiple instances and it's possible that they drift independently slightly from the head. Indexing should continue correctly after retrying the query in ` + (100).toString() + `ms.`);
228
234
  await Time.resolvePromiseAfterDelay(100);
229
- return await queryBlockData(client, fromBlock, toBlock, sourceName, chainId, logger$1);
235
+ return await queryBlockData(client, fromBlock, toBlock, sourceName, chainId, logger$1, requestStats);
230
236
  }
231
237
 
232
238
  async function queryBlockDataMulti(client, blockNumbers, sourceName, chainId, logger) {
239
+ let requestStats = [];
233
240
  let firstBlock = blockNumbers[0];
234
- if (firstBlock === undefined) {
235
- return {
241
+ let result;
242
+ if (firstBlock !== undefined) {
243
+ let fromBlock = firstBlock;
244
+ let toBlock = firstBlock;
245
+ let set = new Set();
246
+ for (let idx = 0, idx_finish = blockNumbers.length; idx < idx_finish; ++idx) {
247
+ let blockNumber = blockNumbers[idx];
248
+ if (blockNumber < fromBlock) {
249
+ fromBlock = blockNumber;
250
+ }
251
+ if (blockNumber > toBlock) {
252
+ toBlock = blockNumber;
253
+ }
254
+ set.add(blockNumber);
255
+ }
256
+ if ((toBlock - fromBlock | 0) > 1000) {
257
+ Stdlib_JsError.throwWithMessage(`Invalid block data request. Range of block numbers is too large. Max range is 1000. Requested range: ` + fromBlock.toString() + `-` + toBlock.toString());
258
+ }
259
+ let res = await queryBlockData(client, fromBlock, toBlock, sourceName, chainId, logger, requestStats);
260
+ let filtered = Stdlib_Result.map(res, datas => datas.filter(data => set.delete(data.blockNumber)));
261
+ if (set.size > 0) {
262
+ Stdlib_JsError.throwWithMessage(`Invalid response. Failed to get block data for block numbers: ` + Array.from(set).join(", "));
263
+ }
264
+ result = filtered;
265
+ } else {
266
+ result = {
236
267
  TAG: "Ok",
237
268
  _0: []
238
269
  };
239
270
  }
240
- let fromBlock = firstBlock;
241
- let toBlock = firstBlock;
242
- let set = new Set();
243
- for (let idx = 0, idx_finish = blockNumbers.length; idx < idx_finish; ++idx) {
244
- let blockNumber = blockNumbers[idx];
245
- if (blockNumber < fromBlock) {
246
- fromBlock = blockNumber;
247
- }
248
- if (blockNumber > toBlock) {
249
- toBlock = blockNumber;
250
- }
251
- set.add(blockNumber);
252
- }
253
- if ((toBlock - fromBlock | 0) > 1000) {
254
- Stdlib_JsError.throwWithMessage(`Invalid block data request. Range of block numbers is too large. Max range is 1000. Requested range: ` + fromBlock.toString() + `-` + toBlock.toString());
255
- }
256
- let res = await queryBlockData(client, fromBlock, toBlock, sourceName, chainId, logger);
257
- let filtered = Stdlib_Result.map(res, datas => datas.filter(data => set.delete(data.blockNumber)));
258
- if (set.size > 0) {
259
- Stdlib_JsError.throwWithMessage(`Invalid response. Failed to get block data for block numbers: ` + Array.from(set).join(", "));
260
- }
261
- return filtered;
271
+ return [
272
+ result,
273
+ requestStats
274
+ ];
262
275
  }
263
276
 
264
277
  function queryBlockData$1(client, blockNumber, sourceName, chainId, logger) {
265
- return queryBlockData(client, blockNumber, blockNumber, sourceName, chainId, logger).then(res => Stdlib_Result.map(res, res => res[0]));
278
+ let requestStats = [];
279
+ return queryBlockData(client, blockNumber, blockNumber, sourceName, chainId, logger, requestStats).then(res => [
280
+ Stdlib_Result.map(res, res => res[0]),
281
+ requestStats
282
+ ]);
266
283
  }
267
284
 
268
285
  let GetLogs = {
@@ -1,10 +1,11 @@
1
1
  type logsQueryPage = {
2
2
  items: array<HyperSyncClient.EventItems.item>,
3
- blocks: array<HyperSyncClient.ResponseTypes.block>,
3
+ blocks: array<HyperSyncClient.EventItems.blockHeader>,
4
4
  nextBlock: int,
5
5
  archiveHeight: int,
6
6
  rollbackGuard: option<HyperSyncClient.ResponseTypes.rollbackGuard>,
7
7
  transactionStore: TransactionStore.t,
8
+ blockStore: BlockStore.t,
8
9
  }
9
10
 
10
11
  type missingParams = {
@@ -33,6 +34,7 @@ module GetLogs: {
33
34
  ~toBlock: option<int>,
34
35
  ~logSelections: array<LogSelection.t>,
35
36
  ~fieldSelection: HyperSyncClient.QueryTypes.fieldSelection,
37
+ ~maxNumLogs: int,
36
38
  ) => promise<logsQueryPage>
37
39
  }
38
40
 
@@ -42,7 +44,10 @@ let queryBlockData: (
42
44
  ~sourceName: string,
43
45
  ~chainId: int,
44
46
  ~logger: Pino.t,
45
- ) => promise<queryResponse<option<ReorgDetection.blockDataWithTimestamp>>>
47
+ ) => promise<(
48
+ queryResponse<option<ReorgDetection.blockDataWithTimestamp>>,
49
+ array<Source.requestStat>,
50
+ )>
46
51
 
47
52
  let queryBlockDataMulti: (
48
53
  ~client: HyperSyncClient.t,
@@ -50,6 +55,9 @@ let queryBlockDataMulti: (
50
55
  ~sourceName: string,
51
56
  ~chainId: int,
52
57
  ~logger: Pino.t,
53
- ) => promise<queryResponse<array<ReorgDetection.blockDataWithTimestamp>>>
58
+ ) => promise<(
59
+ queryResponse<array<ReorgDetection.blockDataWithTimestamp>>,
60
+ array<Source.requestStat>,
61
+ )>
54
62
 
55
63
  let mapExn: queryResponse<'a> => result<'a, exn>
@@ -254,11 +254,19 @@ module EventItems = {
254
254
  params: Nullable.t<dict<Internal.eventParams>>,
255
255
  }
256
256
 
257
+ // The always-needed block fields, one per block number. The block's remaining
258
+ // fields live raw in the block store and are materialised on demand.
259
+ type blockHeader = {
260
+ number: int,
261
+ timestamp: int,
262
+ hash: string,
263
+ }
264
+
257
265
  type response = {
258
266
  archiveHeight: option<int>,
259
267
  nextBlock: int,
260
- // One entry per block number referenced by `items`.
261
- blocks: array<ResponseTypes.block>,
268
+ // One header per block number referenced by `items`.
269
+ blocks: array<blockHeader>,
262
270
  items: array<item>,
263
271
  rollbackGuard: option<ResponseTypes.rollbackGuard>,
264
272
  }
@@ -266,8 +274,11 @@ module EventItems = {
266
274
 
267
275
  type t = {
268
276
  get: (~query: query) => promise<queryResponse>,
269
- // Returns the response plus a page store owning this page's raw transactions.
270
- getEventItems: (~query: query) => promise<(EventItems.response, TransactionStore.t)>,
277
+ // Returns the response plus page stores owning this page's raw transactions
278
+ // and blocks.
279
+ getEventItems: (
280
+ ~query: query,
281
+ ) => promise<(EventItems.response, TransactionStore.t, BlockStore.t)>,
271
282
  getHeight: unit => promise<int>,
272
283
  }
273
284
 
@@ -35,8 +35,7 @@ let subscribe = (~hyperSyncUrl, ~apiToken, ~chainId, ~onHeight: int => unit): (u
35
35
  and scheduleReconnect = () => {
36
36
  // Clear any pending stale/reconnect timeout to avoid double reconnect
37
37
  timeoutIdRef.contents->clearTimeout
38
- let delay =
39
- baseDuration * Math.pow(2.0, ~exp=errorCount.contents->Int.toFloat)->Float.toInt
38
+ let delay = baseDuration * Math.pow(2.0, ~exp=errorCount.contents->Int.toFloat)->Float.toInt
40
39
  timeoutIdRef := setTimeout(() => refreshEventSource(), Pervasives.min(delay, 60_000))
41
40
  }
42
41
  and refreshEventSource = () => {
@@ -102,12 +101,6 @@ let subscribe = (~hyperSyncUrl, ~apiToken, ~chainId, ~onHeight: int => unit): (u
102
101
  es->EventSource.addEventListener("height", event => {
103
102
  switch event.data->Int.fromString {
104
103
  | Some(height) =>
105
- // Track the SSE height event
106
- Prometheus.SourceRequestCount.increment(
107
- ~sourceName="HyperSync",
108
- ~chainId,
109
- ~method="heightStream",
110
- )
111
104
  // On a successful height event, update the timeout
112
105
  updateTimeoutId()
113
106
  // Call the callback with the new height
@@ -2,7 +2,6 @@
2
2
 
3
3
  import * as Utils from "../Utils.res.mjs";
4
4
  import * as Logging from "../Logging.res.mjs";
5
- import * as Prometheus from "../Prometheus.res.mjs";
6
5
  import * as Stdlib_Int from "@rescript/runtime/lib/es6/Stdlib_Int.js";
7
6
  import * as Eventsource from "eventsource";
8
7
  import * as Primitive_int from "@rescript/runtime/lib/es6/Primitive_int.js";
@@ -83,7 +82,6 @@ function subscribe(hyperSyncUrl, apiToken, chainId, onHeight) {
83
82
  es$1.addEventListener("height", event => {
84
83
  let height = Stdlib_Int.fromString(event.data, undefined);
85
84
  if (height !== undefined) {
86
- Prometheus.SourceRequestCount.increment("HyperSync", chainId, "heightStream");
87
85
  updateTimeoutId();
88
86
  return onHeight(height);
89
87
  } else {