envio 3.0.0-alpha.2 → 3.0.0-alpha.20

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (175) hide show
  1. package/README.md +164 -30
  2. package/bin.mjs +49 -0
  3. package/evm.schema.json +79 -169
  4. package/fuel.schema.json +50 -21
  5. package/index.d.ts +497 -1
  6. package/index.js +4 -0
  7. package/package.json +42 -31
  8. package/rescript.json +4 -1
  9. package/src/Batch.res +11 -8
  10. package/src/Batch.res.mjs +11 -9
  11. package/src/ChainFetcher.res +531 -0
  12. package/src/ChainFetcher.res.mjs +339 -0
  13. package/src/ChainManager.res +190 -0
  14. package/src/ChainManager.res.mjs +166 -0
  15. package/src/Change.res +3 -3
  16. package/src/Config.gen.ts +19 -0
  17. package/src/Config.res +737 -22
  18. package/src/Config.res.mjs +703 -26
  19. package/src/{Indexer.res → Ctx.res} +1 -1
  20. package/src/Ecosystem.res +9 -124
  21. package/src/Ecosystem.res.mjs +19 -160
  22. package/src/Env.res +30 -74
  23. package/src/Env.res.mjs +25 -87
  24. package/src/Envio.gen.ts +3 -1
  25. package/src/Envio.res +20 -9
  26. package/src/EventProcessing.res +469 -0
  27. package/src/EventProcessing.res.mjs +337 -0
  28. package/src/EvmTypes.gen.ts +6 -0
  29. package/src/EvmTypes.res +1 -0
  30. package/src/FetchState.res +1256 -639
  31. package/src/FetchState.res.mjs +1135 -612
  32. package/src/GlobalState.res +1190 -0
  33. package/src/GlobalState.res.mjs +1183 -0
  34. package/src/GlobalStateManager.res +68 -0
  35. package/src/GlobalStateManager.res.mjs +75 -0
  36. package/src/GlobalStateManager.resi +7 -0
  37. package/src/HandlerLoader.res +89 -0
  38. package/src/HandlerLoader.res.mjs +79 -0
  39. package/src/HandlerRegister.res +357 -0
  40. package/src/HandlerRegister.res.mjs +299 -0
  41. package/src/{EventRegister.resi → HandlerRegister.resi} +13 -13
  42. package/src/Hasura.res +111 -175
  43. package/src/Hasura.res.mjs +88 -150
  44. package/src/InMemoryStore.res +1 -1
  45. package/src/InMemoryStore.res.mjs +3 -3
  46. package/src/InMemoryTable.res +1 -1
  47. package/src/InMemoryTable.res.mjs +1 -1
  48. package/src/Internal.gen.ts +4 -0
  49. package/src/Internal.res +230 -12
  50. package/src/Internal.res.mjs +115 -1
  51. package/src/LoadLayer.res +444 -0
  52. package/src/LoadLayer.res.mjs +296 -0
  53. package/src/LoadLayer.resi +32 -0
  54. package/src/LogSelection.res +33 -27
  55. package/src/LogSelection.res.mjs +6 -0
  56. package/src/Logging.res +21 -7
  57. package/src/Logging.res.mjs +16 -8
  58. package/src/Main.res +377 -0
  59. package/src/Main.res.mjs +339 -0
  60. package/src/Persistence.res +7 -21
  61. package/src/Persistence.res.mjs +3 -3
  62. package/src/PgStorage.gen.ts +10 -0
  63. package/src/PgStorage.res +116 -69
  64. package/src/PgStorage.res.d.mts +5 -0
  65. package/src/PgStorage.res.mjs +93 -50
  66. package/src/Prometheus.res +294 -224
  67. package/src/Prometheus.res.mjs +353 -340
  68. package/src/ReorgDetection.res +6 -10
  69. package/src/ReorgDetection.res.mjs +6 -6
  70. package/src/SafeCheckpointTracking.res +4 -4
  71. package/src/SafeCheckpointTracking.res.mjs +2 -2
  72. package/src/Sink.res +4 -2
  73. package/src/Sink.res.mjs +2 -1
  74. package/src/TableIndices.res +0 -1
  75. package/src/TestIndexer.res +692 -0
  76. package/src/TestIndexer.res.mjs +527 -0
  77. package/src/TestIndexerProxyStorage.res +205 -0
  78. package/src/TestIndexerProxyStorage.res.mjs +151 -0
  79. package/src/TopicFilter.res +1 -1
  80. package/src/Types.ts +1 -1
  81. package/src/UserContext.res +424 -0
  82. package/src/UserContext.res.mjs +279 -0
  83. package/src/Utils.res +97 -26
  84. package/src/Utils.res.mjs +91 -44
  85. package/src/bindings/BigInt.res +10 -0
  86. package/src/bindings/BigInt.res.mjs +15 -0
  87. package/src/bindings/ClickHouse.res +120 -23
  88. package/src/bindings/ClickHouse.res.mjs +118 -28
  89. package/src/bindings/DateFns.res +74 -0
  90. package/src/bindings/DateFns.res.mjs +22 -0
  91. package/src/bindings/EventSource.res +8 -1
  92. package/src/bindings/EventSource.res.mjs +8 -1
  93. package/src/bindings/Express.res +1 -0
  94. package/src/bindings/Hrtime.res +14 -1
  95. package/src/bindings/Hrtime.res.mjs +22 -2
  96. package/src/bindings/Hrtime.resi +4 -0
  97. package/src/bindings/Lodash.res +0 -1
  98. package/src/bindings/NodeJs.res +49 -3
  99. package/src/bindings/NodeJs.res.mjs +11 -3
  100. package/src/bindings/Pino.res +24 -10
  101. package/src/bindings/Pino.res.mjs +14 -8
  102. package/src/bindings/Postgres.gen.ts +8 -0
  103. package/src/bindings/Postgres.res +5 -1
  104. package/src/bindings/Postgres.res.d.mts +5 -0
  105. package/src/bindings/PromClient.res +0 -10
  106. package/src/bindings/PromClient.res.mjs +0 -3
  107. package/src/bindings/Vitest.res +142 -0
  108. package/src/bindings/Vitest.res.mjs +9 -0
  109. package/src/bindings/WebSocket.res +27 -0
  110. package/src/bindings/WebSocket.res.mjs +2 -0
  111. package/src/bindings/Yargs.res +8 -0
  112. package/src/bindings/Yargs.res.mjs +2 -0
  113. package/src/db/EntityHistory.res +7 -7
  114. package/src/db/EntityHistory.res.mjs +9 -9
  115. package/src/db/InternalTable.res +59 -111
  116. package/src/db/InternalTable.res.mjs +73 -104
  117. package/src/db/Table.res +27 -8
  118. package/src/db/Table.res.mjs +25 -14
  119. package/src/sources/Evm.res +84 -0
  120. package/src/sources/Evm.res.mjs +105 -0
  121. package/src/sources/EvmChain.res +94 -0
  122. package/src/sources/EvmChain.res.mjs +60 -0
  123. package/src/sources/Fuel.res +19 -34
  124. package/src/sources/Fuel.res.mjs +34 -16
  125. package/src/sources/FuelSDK.res +38 -0
  126. package/src/sources/FuelSDK.res.mjs +29 -0
  127. package/src/sources/HyperFuel.res +2 -2
  128. package/src/sources/HyperFuel.resi +1 -1
  129. package/src/sources/HyperFuelClient.res +2 -2
  130. package/src/sources/HyperFuelSource.res +33 -13
  131. package/src/sources/HyperFuelSource.res.mjs +24 -16
  132. package/src/sources/HyperSync.res +36 -6
  133. package/src/sources/HyperSync.res.mjs +9 -7
  134. package/src/sources/HyperSync.resi +4 -0
  135. package/src/sources/HyperSyncClient.res +1 -1
  136. package/src/sources/HyperSyncHeightStream.res +47 -116
  137. package/src/sources/HyperSyncHeightStream.res.mjs +46 -73
  138. package/src/sources/HyperSyncSource.res +118 -139
  139. package/src/sources/HyperSyncSource.res.mjs +104 -121
  140. package/src/sources/Rpc.res +86 -14
  141. package/src/sources/Rpc.res.mjs +101 -9
  142. package/src/sources/RpcSource.res +621 -364
  143. package/src/sources/RpcSource.res.mjs +843 -410
  144. package/src/sources/RpcWebSocketHeightStream.res +181 -0
  145. package/src/sources/RpcWebSocketHeightStream.res.mjs +196 -0
  146. package/src/sources/Source.res +7 -5
  147. package/src/sources/SourceManager.res +325 -225
  148. package/src/sources/SourceManager.res.mjs +314 -171
  149. package/src/sources/SourceManager.resi +17 -6
  150. package/src/sources/Svm.res +81 -0
  151. package/src/sources/Svm.res.mjs +90 -0
  152. package/src/tui/Tui.res +247 -0
  153. package/src/tui/Tui.res.mjs +337 -0
  154. package/src/tui/bindings/Ink.res +371 -0
  155. package/src/tui/bindings/Ink.res.mjs +72 -0
  156. package/src/tui/bindings/Style.res +123 -0
  157. package/src/tui/bindings/Style.res.mjs +2 -0
  158. package/src/tui/components/BufferedProgressBar.res +40 -0
  159. package/src/tui/components/BufferedProgressBar.res.mjs +57 -0
  160. package/src/tui/components/CustomHooks.res +122 -0
  161. package/src/tui/components/CustomHooks.res.mjs +179 -0
  162. package/src/tui/components/Messages.res +41 -0
  163. package/src/tui/components/Messages.res.mjs +75 -0
  164. package/src/tui/components/SyncETA.res +174 -0
  165. package/src/tui/components/SyncETA.res.mjs +263 -0
  166. package/src/tui/components/TuiData.res +47 -0
  167. package/src/tui/components/TuiData.res.mjs +34 -0
  168. package/svm.schema.json +112 -0
  169. package/bin.js +0 -48
  170. package/src/EventRegister.res +0 -241
  171. package/src/EventRegister.res.mjs +0 -240
  172. package/src/bindings/Ethers.gen.ts +0 -14
  173. package/src/bindings/Ethers.res +0 -204
  174. package/src/bindings/Ethers.res.mjs +0 -130
  175. /package/src/{Indexer.res.mjs → Ctx.res.mjs} +0 -0
@@ -3,62 +3,67 @@ open Source
3
3
 
4
4
  exception QueryTimout(string)
5
5
 
6
- let getKnownBlock = (provider, blockNumber) =>
7
- provider
8
- ->Ethers.JsonRpcProvider.getBlock(blockNumber)
9
- ->Promise.then(blockNullable =>
10
- switch blockNullable->Js.Nullable.toOption {
11
- | Some(block) => Promise.resolve(block)
12
- | None =>
13
- Promise.reject(
14
- Js.Exn.raiseError(`RPC returned null for blockNumber ${blockNumber->Belt.Int.toString}`),
15
- )
16
- }
17
- )
6
+ // Minimal block data needed for infrastructure (reorg guard, timestamps, etc.)
7
+ type blockInfo = {
8
+ number: int,
9
+ timestamp: int,
10
+ hash: string,
11
+ }
18
12
 
19
- let rec getKnownBlockWithBackoff = async (
20
- ~provider,
13
+ let getKnownRawBlock = async (~client, ~blockNumber) =>
14
+ switch await Rpc.getRawBlock(~client, ~blockNumber) {
15
+ | Some(json) => json
16
+ | None => Js.Exn.raiseError(`RPC returned null for blockNumber ${blockNumber->Belt.Int.toString}`)
17
+ }
18
+
19
+ // Extract infrastructure fields (number, timestamp, hash) from raw block JSON
20
+ let parseBlockInfo = (json: Js.Json.t): blockInfo => {
21
+ let jsonDict = json->(Utils.magic: Js.Json.t => Js.Dict.t<Js.Json.t>)
22
+ {
23
+ number: jsonDict
24
+ ->Js.Dict.unsafeGet("number")
25
+ ->S.parseOrThrow(Rpc.hexIntSchema),
26
+ timestamp: jsonDict
27
+ ->Js.Dict.unsafeGet("timestamp")
28
+ ->S.parseOrThrow(Rpc.hexIntSchema),
29
+ hash: jsonDict
30
+ ->Js.Dict.unsafeGet("hash")
31
+ ->S.parseOrThrow(S.string),
32
+ }
33
+ }
34
+
35
+ let getKnownRawBlockWithBackoff = async (
36
+ ~client,
21
37
  ~sourceName,
22
38
  ~chain,
23
39
  ~blockNumber,
24
40
  ~backoffMsOnFailure,
25
- ~lowercaseAddresses: bool,
26
- ) =>
27
- switch await getKnownBlock(provider, blockNumber) {
28
- | exception err =>
29
- Logging.warn({
30
- "err": err->Utils.prettifyExn,
31
- "msg": `Issue while running fetching batch of events from the RPC. Will wait ${backoffMsOnFailure->Belt.Int.toString}ms and try again.`,
32
- "source": sourceName,
33
- "chainId": chain->ChainMap.Chain.toChainId,
34
- "type": "EXPONENTIAL_BACKOFF",
35
- })
36
- await Time.resolvePromiseAfterDelay(~delayMilliseconds=backoffMsOnFailure)
37
- await getKnownBlockWithBackoff(
38
- ~provider,
41
+ ) => {
42
+ let currentBackoff = ref(backoffMsOnFailure)
43
+ let result = ref(None)
44
+
45
+ while result.contents->Option.isNone {
46
+ Prometheus.SourceRequestCount.increment(
39
47
  ~sourceName,
40
- ~chain,
41
- ~blockNumber,
42
- ~backoffMsOnFailure=backoffMsOnFailure * 2,
43
- ~lowercaseAddresses,
48
+ ~chainId=chain->ChainMap.Chain.toChainId,
49
+ ~method="eth_getBlockByNumber",
44
50
  )
45
- | result =>
46
- if lowercaseAddresses {
47
- // NOTE: this is wasteful if these fields are not selected in the users config.
48
- // There might be a better way to do this based on the block schema.
49
- // However this is not extremely expensive and good enough for now (only on rpc sync also).
50
-
51
- {
52
- ...result,
53
- // Mutation would be cheaper,
54
- // BUT "result" is an Ethers.js Block object,
55
- // which has the fields as readonly.
56
- miner: result.miner->Address.Evm.fromAddressLowercaseOrThrow,
57
- }
58
- } else {
59
- result
51
+ switch await getKnownRawBlock(~client, ~blockNumber) {
52
+ | exception err =>
53
+ Logging.warn({
54
+ "err": err->Utils.prettifyExn,
55
+ "msg": `Issue while running fetching batch of events from the RPC. Will wait ${currentBackoff.contents->Belt.Int.toString}ms and try again.`,
56
+ "source": sourceName,
57
+ "chainId": chain->ChainMap.Chain.toChainId,
58
+ "type": "EXPONENTIAL_BACKOFF",
59
+ })
60
+ await Time.resolvePromiseAfterDelay(~delayMilliseconds=currentBackoff.contents)
61
+ currentBackoff := currentBackoff.contents * 2
62
+ | json => result := Some(json)
60
63
  }
61
64
  }
65
+ result.contents->Option.getExn
66
+ }
62
67
  let getSuggestedBlockIntervalFromExn = {
63
68
  // Unknown provider: "retry with the range 123-456"
64
69
  let suggestedRangeRegExp = %re(`/retry with the range (\d+)-(\d+)/`)
@@ -111,82 +116,68 @@ let getSuggestedBlockIntervalFromExn = {
111
116
  // - Optimism: "backend response too large" or "Block range is too large"
112
117
  // - Arbitrum: "logs matched by query exceeds limit of 10000"
113
118
 
114
- (exn): option<(
115
- // The suggested block range
116
- int,
117
- // Whether it's the max range that the provider allows
118
- bool,
119
- )> =>
120
- switch exn {
121
- | Js.Exn.Error(error) =>
122
- try {
123
- let message: string = (error->Obj.magic)["error"]["message"]
124
- message->S.assertOrThrow(S.string)
125
-
126
- // Helper to extract block range from regex match
127
- let extractBlockRange = (execResult, ~isMaxRange) =>
128
- switch execResult->Js.Re.captures {
129
- | [_, Js.Nullable.Value(blockRangeLimit)] =>
130
- switch blockRangeLimit->Int.fromString {
131
- | Some(blockRangeLimit) if blockRangeLimit > 0 => Some(blockRangeLimit, isMaxRange)
132
- | _ => None
133
- }
134
- | _ => None
135
- }
119
+ let parseMessageForBlockRange = (message: string) => {
120
+ // Helper to extract block range from regex match
121
+ let extractBlockRange = (execResult, ~isMaxRange) =>
122
+ switch execResult->Js.Re.captures {
123
+ | [_, Js.Nullable.Value(blockRangeLimit)] =>
124
+ switch blockRangeLimit->Int.fromString {
125
+ | Some(blockRangeLimit) if blockRangeLimit > 0 => Some(blockRangeLimit, isMaxRange)
126
+ | _ => None
127
+ }
128
+ | _ => None
129
+ }
136
130
 
137
- // Try each regex pattern in order
138
- switch suggestedRangeRegExp->Js.Re.exec_(message) {
139
- | Some(execResult) =>
140
- switch execResult->Js.Re.captures {
141
- | [_, Js.Nullable.Value(fromBlock), Js.Nullable.Value(toBlock)] =>
142
- switch (fromBlock->Int.fromString, toBlock->Int.fromString) {
143
- | (Some(fromBlock), Some(toBlock)) if toBlock >= fromBlock =>
144
- Some(toBlock - fromBlock + 1, false)
145
- | _ => None
146
- }
147
- | _ => None
148
- }
131
+ // Try each regex pattern in order
132
+ switch suggestedRangeRegExp->Js.Re.exec_(message) {
133
+ | Some(execResult) =>
134
+ switch execResult->Js.Re.captures {
135
+ | [_, Js.Nullable.Value(fromBlock), Js.Nullable.Value(toBlock)] =>
136
+ switch (fromBlock->Int.fromString, toBlock->Int.fromString) {
137
+ | (Some(fromBlock), Some(toBlock)) if toBlock >= fromBlock =>
138
+ Some(toBlock - fromBlock + 1, false)
139
+ | _ => None
140
+ }
141
+ | _ => None
142
+ }
143
+ | None =>
144
+ // Try each provider's specific error pattern
145
+ switch blockRangeLimitRegExp->Js.Re.exec_(message) {
146
+ | Some(execResult) => extractBlockRange(execResult, ~isMaxRange=true)
147
+ | None =>
148
+ switch alchemyRangeRegExp->Js.Re.exec_(message) {
149
+ | Some(execResult) => extractBlockRange(execResult, ~isMaxRange=true)
149
150
  | None =>
150
- // Try each provider's specific error pattern
151
- switch blockRangeLimitRegExp->Js.Re.exec_(message) {
151
+ switch cloudflareRangeRegExp->Js.Re.exec_(message) {
152
152
  | Some(execResult) => extractBlockRange(execResult, ~isMaxRange=true)
153
153
  | None =>
154
- switch alchemyRangeRegExp->Js.Re.exec_(message) {
154
+ switch thirdwebRangeRegExp->Js.Re.exec_(message) {
155
155
  | Some(execResult) => extractBlockRange(execResult, ~isMaxRange=true)
156
156
  | None =>
157
- switch cloudflareRangeRegExp->Js.Re.exec_(message) {
157
+ switch blockpiRangeRegExp->Js.Re.exec_(message) {
158
158
  | Some(execResult) => extractBlockRange(execResult, ~isMaxRange=true)
159
159
  | None =>
160
- switch thirdwebRangeRegExp->Js.Re.exec_(message) {
160
+ switch maxAllowedBlocksRegExp->Js.Re.exec_(message) {
161
161
  | Some(execResult) => extractBlockRange(execResult, ~isMaxRange=true)
162
162
  | None =>
163
- switch blockpiRangeRegExp->Js.Re.exec_(message) {
164
- | Some(execResult) => extractBlockRange(execResult, ~isMaxRange=true)
163
+ switch baseRangeRegExp->Js.Re.exec_(message) {
164
+ | Some(_) => Some(2000, true)
165
165
  | None =>
166
- switch maxAllowedBlocksRegExp->Js.Re.exec_(message) {
166
+ switch blastPaidRegExp->Js.Re.exec_(message) {
167
167
  | Some(execResult) => extractBlockRange(execResult, ~isMaxRange=true)
168
168
  | None =>
169
- switch baseRangeRegExp->Js.Re.exec_(message) {
170
- | Some(_) => Some(2000, true)
169
+ switch chainstackRegExp->Js.Re.exec_(message) {
170
+ | Some(_) => Some(10000, true)
171
171
  | None =>
172
- switch blastPaidRegExp->Js.Re.exec_(message) {
172
+ switch coinbaseRegExp->Js.Re.exec_(message) {
173
173
  | Some(execResult) => extractBlockRange(execResult, ~isMaxRange=true)
174
174
  | None =>
175
- switch chainstackRegExp->Js.Re.exec_(message) {
176
- | Some(_) => Some(10000, true)
175
+ switch publicNodeRegExp->Js.Re.exec_(message) {
176
+ | Some(execResult) => extractBlockRange(execResult, ~isMaxRange=true)
177
177
  | None =>
178
- switch coinbaseRegExp->Js.Re.exec_(message) {
178
+ switch hyperliquidRegExp->Js.Re.exec_(message) {
179
179
  | Some(execResult) => extractBlockRange(execResult, ~isMaxRange=true)
180
- | None =>
181
- switch publicNodeRegExp->Js.Re.exec_(message) {
182
- | Some(execResult) => extractBlockRange(execResult, ~isMaxRange=true)
183
- | None =>
184
- switch hyperliquidRegExp->Js.Re.exec_(message) {
185
- | Some(execResult) =>
186
- extractBlockRange(execResult, ~isMaxRange=true)
187
- | None => None
188
- }
189
- }
180
+ | None => None
190
181
  }
191
182
  }
192
183
  }
@@ -198,6 +189,23 @@ let getSuggestedBlockIntervalFromExn = {
198
189
  }
199
190
  }
200
191
  }
192
+ }
193
+ }
194
+ }
195
+
196
+ (exn): option<(
197
+ // The suggested block range
198
+ int,
199
+ // Whether it's the max range that the provider allows
200
+ bool,
201
+ )> =>
202
+ switch exn {
203
+ | Rpc.JsonRpcError({message}) => parseMessageForBlockRange(message)
204
+ | Js.Exn.Error(error) =>
205
+ try {
206
+ let message: string = (error->Obj.magic)["error"]["message"]
207
+ message->S.assertOrThrow(S.string)
208
+ parseMessageForBlockRange(message)
201
209
  } catch {
202
210
  | _ => None
203
211
  }
@@ -206,8 +214,8 @@ let getSuggestedBlockIntervalFromExn = {
206
214
  }
207
215
 
208
216
  type eventBatchQuery = {
209
- logs: array<Ethers.log>,
210
- latestFetchedBlock: Ethers.JsonRpcProvider.block,
217
+ logs: array<Rpc.GetLogs.log>,
218
+ latestFetchedBlockInfo: blockInfo,
211
219
  }
212
220
 
213
221
  let maxSuggestedBlockIntervalKey = "max"
@@ -219,9 +227,11 @@ let getNextPage = (
219
227
  ~topicQuery,
220
228
  ~loadBlock,
221
229
  ~syncConfig as sc: Config.sourceSync,
222
- ~provider,
230
+ ~client,
223
231
  ~mutSuggestedBlockIntervals,
224
232
  ~partitionId,
233
+ ~sourceName,
234
+ ~chainId,
225
235
  ): promise<eventBatchQuery> => {
226
236
  //If the query hangs for longer than this, reject this promise to reduce the block interval
227
237
  let queryTimoutPromise =
@@ -234,22 +244,21 @@ let getNextPage = (
234
244
  )
235
245
 
236
246
  let latestFetchedBlockPromise = loadBlock(toBlock)
237
- let logsPromise =
238
- provider
239
- ->Ethers.JsonRpcProvider.getLogs(
240
- ~filter={
241
- address: ?addresses,
242
- topics: topicQuery,
243
- fromBlock,
244
- toBlock,
245
- }->Ethers.CombinedFilter.toFilter,
246
- )
247
- ->Promise.then(async logs => {
248
- {
249
- logs,
250
- latestFetchedBlock: await latestFetchedBlockPromise,
251
- }
252
- })
247
+ Prometheus.SourceRequestCount.increment(~sourceName, ~chainId, ~method="eth_getLogs")
248
+ let logsPromise = Rpc.getLogs(
249
+ ~client,
250
+ ~param={
251
+ address: ?addresses,
252
+ topics: topicQuery,
253
+ fromBlock,
254
+ toBlock,
255
+ },
256
+ )->Promise.then(async logs => {
257
+ {
258
+ logs,
259
+ latestFetchedBlockInfo: await latestFetchedBlockPromise,
260
+ }
261
+ })
253
262
 
254
263
  [queryTimoutPromise, logsPromise]
255
264
  ->Promise.race
@@ -337,7 +346,7 @@ let getSelectionConfig = (selection: FetchState.selection, ~chain) => {
337
346
  }
338
347
  }
339
348
  | ([], [dynamicEventFilter]) if selection.eventConfigs->Js.Array2.length === 1 =>
340
- let eventConfig = selection.eventConfigs->Js.Array2.unsafe_get(0)
349
+ let eventConfig = selection.eventConfigs->Utils.Array.firstUnsafe
341
350
 
342
351
  (~addressesByContractName) => {
343
352
  let addresses = addressesByContractName->FetchState.addressesByContractNameGetAll
@@ -371,74 +380,343 @@ let getSelectionConfig = (selection: FetchState.selection, ~chain) => {
371
380
  }
372
381
  }
373
382
 
374
- let memoGetSelectionConfig = (~chain) => {
375
- let cache = Utils.WeakMap.make()
376
- selection =>
377
- switch cache->Utils.WeakMap.get(selection) {
378
- | Some(c) => c
379
- | None => {
380
- let c = selection->getSelectionConfig(~chain)
381
- let _ = cache->Utils.WeakMap.set(selection, c)
382
- c
383
- }
383
+ let memoGetSelectionConfig = (~chain) =>
384
+ Utils.WeakMap.memoize(selection => selection->getSelectionConfig(~chain))
385
+
386
+ // Type-erase a schema for storage in the field registry
387
+ external toFieldSchema: S.t<'a> => S.t<Js.Json.t> = "%identity"
388
+
389
+ let lowercaseAddressSchema: S.t<Js.Json.t> =
390
+ S.string
391
+ ->S.transform(_ => {
392
+ parser: str => str->Js.String2.toLowerCase->Address.unsafeFromString,
393
+ })
394
+ ->toFieldSchema
395
+
396
+ let checksumAddressSchema: S.t<Js.Json.t> =
397
+ S.string
398
+ ->S.transform(_ => {
399
+ parser: str => str->Address.Evm.fromStringOrThrow,
400
+ })
401
+ ->toFieldSchema
402
+
403
+ // Block field definition for per-field parsing
404
+ type blockFieldDef = {
405
+ location: Internal.evmBlockField,
406
+ jsonKey: string,
407
+ schema: S.t<Js.Json.t>, // Type-erased schema
408
+ }
409
+
410
+ // Block field registry: maps field location (= JS property name) to parsing info.
411
+ let makeBlockFieldRegistry = (addressSchema: S.t<Js.Json.t>): Utils.Record.t<
412
+ Internal.evmBlockField,
413
+ blockFieldDef,
414
+ > =>
415
+ [
416
+ {location: Number, jsonKey: "number", schema: Rpc.hexIntSchema->toFieldSchema},
417
+ {location: Timestamp, jsonKey: "timestamp", schema: Rpc.hexIntSchema->toFieldSchema},
418
+ {location: Hash, jsonKey: "hash", schema: S.string->toFieldSchema},
419
+ {location: ParentHash, jsonKey: "parentHash", schema: S.string->toFieldSchema},
420
+ {location: Nonce, jsonKey: "nonce", schema: Rpc.hexBigintSchema->toFieldSchema},
421
+ {location: Sha3Uncles, jsonKey: "sha3Uncles", schema: S.string->toFieldSchema},
422
+ {location: LogsBloom, jsonKey: "logsBloom", schema: S.string->toFieldSchema},
423
+ {location: TransactionsRoot, jsonKey: "transactionsRoot", schema: S.string->toFieldSchema},
424
+ {location: StateRoot, jsonKey: "stateRoot", schema: S.string->toFieldSchema},
425
+ {location: ReceiptsRoot, jsonKey: "receiptsRoot", schema: S.string->toFieldSchema},
426
+ {location: Miner, jsonKey: "miner", schema: addressSchema},
427
+ {location: Difficulty, jsonKey: "difficulty", schema: Rpc.hexBigintSchema->toFieldSchema},
428
+ {location: TotalDifficulty, jsonKey: "totalDifficulty", schema: Rpc.hexBigintSchema->toFieldSchema},
429
+ {location: ExtraData, jsonKey: "extraData", schema: S.string->toFieldSchema},
430
+ {location: Size, jsonKey: "size", schema: Rpc.hexBigintSchema->toFieldSchema},
431
+ {location: GasLimit, jsonKey: "gasLimit", schema: Rpc.hexBigintSchema->toFieldSchema},
432
+ {location: GasUsed, jsonKey: "gasUsed", schema: Rpc.hexBigintSchema->toFieldSchema},
433
+ {location: Uncles, jsonKey: "uncles", schema: S.array(S.string)->toFieldSchema},
434
+ {location: BaseFeePerGas, jsonKey: "baseFeePerGas", schema: Rpc.hexBigintSchema->toFieldSchema},
435
+ {location: BlobGasUsed, jsonKey: "blobGasUsed", schema: Rpc.hexBigintSchema->toFieldSchema},
436
+ {location: ExcessBlobGas, jsonKey: "excessBlobGas", schema: Rpc.hexBigintSchema->toFieldSchema},
437
+ {location: ParentBeaconBlockRoot, jsonKey: "parentBeaconBlockRoot", schema: S.string->toFieldSchema},
438
+ {location: WithdrawalsRoot, jsonKey: "withdrawalsRoot", schema: S.string->toFieldSchema},
439
+ {location: L1BlockNumber, jsonKey: "l1BlockNumber", schema: Rpc.hexIntSchema->toFieldSchema},
440
+ {location: SendCount, jsonKey: "sendCount", schema: S.string->toFieldSchema},
441
+ {location: SendRoot, jsonKey: "sendRoot", schema: S.string->toFieldSchema},
442
+ {location: MixHash, jsonKey: "mixHash", schema: S.string->toFieldSchema},
443
+ ]
444
+ ->Array.map(def => (
445
+ def.location,
446
+ if Internal.evmNullableBlockFields->Utils.Set.has(def.location) {
447
+ {...def, schema: S.nullable(def.schema)->toFieldSchema}
448
+ } else {
449
+ def
450
+ },
451
+ ))
452
+ ->Utils.Record.fromArray
453
+
454
+ let blockFieldRegistryLowercase = makeBlockFieldRegistry(lowercaseAddressSchema)
455
+ let blockFieldRegistryChecksum = makeBlockFieldRegistry(checksumAddressSchema)
456
+
457
+ // Parse block fields from raw JSON, similar to parseFieldsFromJson for transactions
458
+ let parseBlockFieldsFromJson = (
459
+ mutBlockAcc: Js.Dict.t<Js.Json.t>,
460
+ fields: array<blockFieldDef>,
461
+ json: Js.Json.t,
462
+ ) => {
463
+ let jsonDict = json->(Utils.magic: Js.Json.t => Js.Dict.t<Js.Json.t>)
464
+ fields->Array.forEach(def => {
465
+ let raw = jsonDict->Js.Dict.unsafeGet(def.jsonKey)
466
+ try {
467
+ let parsed = raw->S.parseOrThrow(def.schema)
468
+ mutBlockAcc->Js.Dict.set((def.location :> string), parsed)
469
+ } catch {
470
+ | S.Raised(error) =>
471
+ Js.Exn.raiseError(
472
+ `Invalid block field "${(def.location :> string)}" found in the RPC response. Error: ${error->S.Error.reason}`,
473
+ )
384
474
  }
475
+ })
385
476
  }
386
477
 
387
- let makeThrowingGetEventBlock = (~getBlock) => {
388
- async (log: Ethers.log) => {
389
- await getBlock(log.blockNumber)
478
+ let makeThrowingGetEventBlock = (
479
+ ~getBlockJson: int => promise<Js.Json.t>,
480
+ ~lowercaseAddresses: bool,
481
+ ) => {
482
+ let blockFieldRegistry = if lowercaseAddresses {
483
+ blockFieldRegistryLowercase
484
+ } else {
485
+ blockFieldRegistryChecksum
390
486
  }
487
+ let fnsCache = Utils.WeakMap.make()
488
+ (log: Rpc.GetLogs.log, ~selectedBlockFields: Utils.Set.t<Internal.evmBlockField>) => {
489
+ (
490
+ switch fnsCache->Utils.WeakMap.get(selectedBlockFields) {
491
+ | Some(fn) => fn
492
+ // Build per-field parser on first call, then cache in WeakMap
493
+ | None => {
494
+ let fields: array<blockFieldDef> = []
495
+ selectedBlockFields->Utils.Set.forEach(fieldName => {
496
+ fields->Js.Array2.push(blockFieldRegistry->Utils.Record.getUnsafe(fieldName))->ignore
497
+ })
498
+
499
+ let fn = if selectedBlockFields->Utils.Set.size == 0 {
500
+ _ => %raw(`{}`)->(Utils.magic: 'a => Internal.eventBlock)->Promise.resolve
501
+ } else {
502
+ (log: Rpc.GetLogs.log) => {
503
+ getBlockJson(log.blockNumber)->Promise.thenResolve(json => {
504
+ let mutBlockAcc = Js.Dict.empty()
505
+ parseBlockFieldsFromJson(mutBlockAcc, fields, json)
506
+ mutBlockAcc->(Utils.magic: Js.Dict.t<Js.Json.t> => Internal.eventBlock)
507
+ })
508
+ }
509
+ }
510
+ let _ = fnsCache->Utils.WeakMap.set(selectedBlockFields, fn)
511
+ fn
512
+ }
513
+ }
514
+ )(log)
515
+ }
516
+ }
517
+
518
+ // Field source classification for RPC calls
519
+ type fieldSource = TransactionOnly | ReceiptOnly | Both
520
+
521
+ type fieldDef = {
522
+ location: Internal.evmTransactionField,
523
+ jsonKey: string,
524
+ schema: S.t<Js.Json.t>, // Type-erased schema (S.nullable for optional fields)
525
+ source: fieldSource,
526
+ }
527
+
528
+ // Field registry: maps field location (= JS property name) to parsing info.
529
+ // Only includes fields that require an RPC call. Log-derived fields (hash, transactionIndex) are special-cased.
530
+ // Nullable fields are wrapped with S.nullable during registry construction based on Internal.evmNullableTransactionFields
531
+ let makeFieldRegistry = (addressSchema: S.t<Js.Json.t>): Utils.Record.t<
532
+ Internal.evmTransactionField,
533
+ fieldDef,
534
+ > =>
535
+ [
536
+ // TransactionOnly fields (only in eth_getTransactionByHash)
537
+ {location: Gas, jsonKey: "gas", schema: Rpc.hexBigintSchema->toFieldSchema, source: TransactionOnly},
538
+ {location: GasPrice, jsonKey: "gasPrice", schema: Rpc.hexBigintSchema->toFieldSchema, source: TransactionOnly},
539
+ {location: Input, jsonKey: "input", schema: S.string->toFieldSchema, source: TransactionOnly},
540
+ {location: Nonce, jsonKey: "nonce", schema: Rpc.hexBigintSchema->toFieldSchema, source: TransactionOnly},
541
+ {location: Value, jsonKey: "value", schema: Rpc.hexBigintSchema->toFieldSchema, source: TransactionOnly},
542
+ {location: V, jsonKey: "v", schema: S.string->toFieldSchema, source: TransactionOnly},
543
+ {location: R, jsonKey: "r", schema: S.string->toFieldSchema, source: TransactionOnly},
544
+ {location: S, jsonKey: "s", schema: S.string->toFieldSchema, source: TransactionOnly},
545
+ {location: YParity, jsonKey: "yParity", schema: S.string->toFieldSchema, source: TransactionOnly},
546
+ {location: MaxPriorityFeePerGas, jsonKey: "maxPriorityFeePerGas", schema: Rpc.hexBigintSchema->toFieldSchema, source: TransactionOnly},
547
+ {location: MaxFeePerGas, jsonKey: "maxFeePerGas", schema: Rpc.hexBigintSchema->toFieldSchema, source: TransactionOnly},
548
+ {location: MaxFeePerBlobGas, jsonKey: "maxFeePerBlobGas", schema: Rpc.hexBigintSchema->toFieldSchema, source: TransactionOnly},
549
+ {location: BlobVersionedHashes, jsonKey: "blobVersionedHashes", schema: S.array(S.string)->toFieldSchema, source: TransactionOnly},
550
+ // ReceiptOnly fields (only in eth_getTransactionReceipt)
551
+ {location: GasUsed, jsonKey: "gasUsed", schema: Rpc.hexBigintSchema->toFieldSchema, source: ReceiptOnly},
552
+ {location: CumulativeGasUsed, jsonKey: "cumulativeGasUsed", schema: Rpc.hexBigintSchema->toFieldSchema, source: ReceiptOnly},
553
+ {location: EffectiveGasPrice, jsonKey: "effectiveGasPrice", schema: Rpc.hexBigintSchema->toFieldSchema, source: ReceiptOnly},
554
+ {location: ContractAddress, jsonKey: "contractAddress", schema: addressSchema, source: ReceiptOnly},
555
+ {location: LogsBloom, jsonKey: "logsBloom", schema: S.string->toFieldSchema, source: ReceiptOnly},
556
+ {location: Root, jsonKey: "root", schema: S.string->toFieldSchema, source: ReceiptOnly},
557
+ {location: Status, jsonKey: "status", schema: Rpc.hexIntSchema->toFieldSchema, source: ReceiptOnly},
558
+ {location: L1Fee, jsonKey: "l1Fee", schema: Rpc.hexBigintSchema->toFieldSchema, source: ReceiptOnly},
559
+ {location: L1GasPrice, jsonKey: "l1GasPrice", schema: Rpc.hexBigintSchema->toFieldSchema, source: ReceiptOnly},
560
+ {location: L1GasUsed, jsonKey: "l1GasUsed", schema: Rpc.hexBigintSchema->toFieldSchema, source: ReceiptOnly},
561
+ {location: L1FeeScalar, jsonKey: "l1FeeScalar", schema: Rpc.decimalFloatSchema->toFieldSchema, source: ReceiptOnly},
562
+ {location: GasUsedForL1, jsonKey: "gasUsedForL1", schema: Rpc.hexBigintSchema->toFieldSchema, source: ReceiptOnly},
563
+ // Both fields (available in both eth_getTransactionByHash and eth_getTransactionReceipt)
564
+ {location: From, jsonKey: "from", schema: addressSchema, source: Both},
565
+ {location: To, jsonKey: "to", schema: addressSchema, source: Both},
566
+ {location: Type, jsonKey: "type", schema: Rpc.hexIntSchema->toFieldSchema, source: Both},
567
+ ]
568
+ ->Array.map(def => (
569
+ def.location,
570
+ if Internal.evmNullableTransactionFields->Utils.Set.has(def.location) {
571
+ {...def, schema: S.nullable(def.schema)->toFieldSchema}
572
+ } else {
573
+ def
574
+ },
575
+ ))
576
+ ->Utils.Record.fromArray
577
+
578
+ let fieldRegistryLowercase = makeFieldRegistry(lowercaseAddressSchema)
579
+ let fieldRegistryChecksum = makeFieldRegistry(checksumAddressSchema)
580
+
581
+ type fetchStrategy = NoRpc | TransactionOnly | ReceiptOnly | TransactionAndReceipt
582
+
583
+ // Parse fields from a raw JSON object into a result dict.
584
+ // Uses unsafeGet so nullable schemas (S.nullable) handle both null and undefined.
585
+ let parseFieldsFromJson = (
586
+ mutTransactionAcc: Js.Dict.t<Js.Json.t>,
587
+ fields: array<fieldDef>,
588
+ json: Js.Json.t,
589
+ ) => {
590
+ let jsonDict = json->(Utils.magic: Js.Json.t => Js.Dict.t<Js.Json.t>)
591
+ fields->Array.forEach(def => {
592
+ let raw = jsonDict->Js.Dict.unsafeGet(def.jsonKey)
593
+ try {
594
+ let parsed = raw->S.parseOrThrow(def.schema)
595
+ mutTransactionAcc->Js.Dict.set((def.location :> string), parsed)
596
+ } catch {
597
+ | S.Raised(error) =>
598
+ Js.Exn.raiseError(
599
+ `Invalid transaction field "${(def.location :> string)}" found in the RPC response. Error: ${error->S.Error.reason}`,
600
+ )
601
+ }
602
+ })
391
603
  }
392
604
 
393
- let makeThrowingGetEventTransaction = (~getTransactionFields) => {
605
+ let makeThrowingGetEventTransaction = (
606
+ ~getTransactionJson: string => promise<Js.Json.t>,
607
+ ~getReceiptJson: string => promise<Js.Json.t>,
608
+ ~lowercaseAddresses: bool,
609
+ ) => {
610
+ let fieldRegistry = if lowercaseAddresses {
611
+ fieldRegistryLowercase
612
+ } else {
613
+ fieldRegistryChecksum
614
+ }
394
615
  let fnsCache = Utils.WeakMap.make()
395
- (log, ~transactionSchema) => {
616
+ (log, ~selectedTransactionFields: Utils.Set.t<Internal.evmTransactionField>) => {
396
617
  (
397
- switch fnsCache->Utils.WeakMap.get(transactionSchema) {
618
+ switch fnsCache->Utils.WeakMap.get(selectedTransactionFields) {
398
619
  | Some(fn) => fn
399
- // This is not super expensive, but don't want to do it on every event
620
+ // Build per-field parser on first call, then cache in WeakMap
400
621
  | None => {
401
- let transactionSchema = transactionSchema->S.removeTypeValidation
402
-
403
- let transactionFieldItems = switch transactionSchema->S.classify {
404
- | Object({items}) => items
405
- | _ => Js.Exn.raiseError("Unexpected internal error: transactionSchema is not an object")
622
+ // Classify fields: log-derived vs RPC fields
623
+ let hasTransactionIndex = ref(false)
624
+ let hasHash = ref(false)
625
+ let txFields: array<fieldDef> = []
626
+ let receiptFields: array<fieldDef> = []
627
+ let bothFields: array<fieldDef> = []
628
+
629
+ selectedTransactionFields->Utils.Set.forEach(fieldName => {
630
+ switch fieldName {
631
+ | TransactionIndex => hasTransactionIndex := true
632
+ | Hash => hasHash := true
633
+ | _ =>
634
+ switch fieldRegistry->Utils.Record.get(fieldName) {
635
+ | Some(def) =>
636
+ switch def.source {
637
+ | TransactionOnly => txFields->Js.Array2.push(def)->ignore
638
+ | ReceiptOnly => receiptFields->Js.Array2.push(def)->ignore
639
+ | Both => bothFields->Js.Array2.push(def)->ignore
640
+ }
641
+ | None => () // Unknown field — skip silently
642
+ }
643
+ }
644
+ })
645
+
646
+ // Determine fetch strategy
647
+ let strategy = switch (txFields->Array.length > 0, receiptFields->Array.length > 0) {
648
+ | (true, true) => TransactionAndReceipt
649
+ | (true, false) => TransactionOnly
650
+ | (false, true) => ReceiptOnly
651
+ | (false, false) if bothFields->Array.length > 0 => TransactionOnly
652
+ | (false, false) => NoRpc
406
653
  }
407
654
 
408
- let parseOrThrowReadableError = data => {
409
- try data->S.parseOrThrow(transactionSchema) catch {
410
- | S.Raised(error) =>
411
- Js.Exn.raiseError(
412
- `Invalid transaction field "${error.path
413
- ->S.Path.toArray
414
- ->Js.Array2.joinWith(
415
- ".",
416
- )}" found in the RPC response. Error: ${error->S.Error.reason}`,
417
- ) // There should always be only one field, but just in case split them with a dot
655
+ // Assign Both fields to whichever source is already being fetched; default to transaction
656
+ let targetForBoth = strategy == ReceiptOnly ? receiptFields : txFields
657
+ bothFields->Array.forEach(f => targetForBoth->Js.Array2.push(f)->ignore)
658
+
659
+ // Set log-derived fields on the mutable accumulator
660
+ let setLogFields = (mutTransactionAcc: Js.Dict.t<Js.Json.t>, log: Rpc.GetLogs.log) => {
661
+ if hasTransactionIndex.contents {
662
+ mutTransactionAcc->Js.Dict.set(
663
+ "transactionIndex",
664
+ log.transactionIndex->(Utils.magic: int => Js.Json.t),
665
+ )
666
+ }
667
+ if hasHash.contents {
668
+ mutTransactionAcc->Js.Dict.set(
669
+ "hash",
670
+ log.transactionHash->(Utils.magic: string => Js.Json.t),
671
+ )
418
672
  }
419
673
  }
420
674
 
421
- let fn = switch transactionFieldItems {
422
- | [] => _ => %raw(`{}`)->Promise.resolve
423
- | [{location: "transactionIndex"}] =>
424
- log => log->parseOrThrowReadableError->Promise.resolve
425
- | [{location: "hash"}]
426
- | [{location: "hash"}, {location: "transactionIndex"}]
427
- | [{location: "transactionIndex"}, {location: "hash"}] =>
428
- (log: Ethers.log) =>
429
- {
430
- "hash": log.transactionHash,
431
- "transactionIndex": log.transactionIndex,
675
+ let fn = if selectedTransactionFields->Utils.Set.size == 0 {
676
+ _ => %raw(`{}`)->Promise.resolve
677
+ } else {
678
+ switch strategy {
679
+ | NoRpc =>
680
+ (log: Rpc.GetLogs.log) => {
681
+ let mutTransactionAcc = Js.Dict.empty()
682
+ setLogFields(mutTransactionAcc, log)
683
+ mutTransactionAcc->(Utils.magic: Js.Dict.t<Js.Json.t> => 'a)->Promise.resolve
432
684
  }
433
- ->parseOrThrowReadableError
434
- ->Promise.resolve
435
- | _ =>
436
- log =>
437
- log
438
- ->getTransactionFields
439
- ->Promise.thenResolve(parseOrThrowReadableError)
685
+ | _ =>
686
+ (log: Rpc.GetLogs.log) => {
687
+ let txJsonPromise = switch strategy {
688
+ | TransactionOnly | TransactionAndReceipt =>
689
+ getTransactionJson(log.transactionHash)->Promise.thenResolve(v => Some(v))
690
+ | _ => Promise.resolve(None)
691
+ }
692
+ let receiptJsonPromise = switch strategy {
693
+ | ReceiptOnly | TransactionAndReceipt =>
694
+ getReceiptJson(log.transactionHash)->Promise.thenResolve(v => Some(v))
695
+ | _ => Promise.resolve(None)
696
+ }
697
+
698
+ Promise.all2((txJsonPromise, receiptJsonPromise))->Promise.thenResolve(((
699
+ txJson,
700
+ receiptJson,
701
+ )) => {
702
+ let mutTransactionAcc = Js.Dict.empty()
703
+ setLogFields(mutTransactionAcc, log)
704
+
705
+ switch txJson {
706
+ | Some(json) => parseFieldsFromJson(mutTransactionAcc, txFields, json)
707
+ | None => ()
708
+ }
709
+ switch receiptJson {
710
+ | Some(json) => parseFieldsFromJson(mutTransactionAcc, receiptFields, json)
711
+ | None => ()
712
+ }
713
+
714
+ mutTransactionAcc->(Utils.magic: Js.Dict.t<Js.Json.t> => 'a)
715
+ })
716
+ }
717
+ }
440
718
  }
441
- let _ = fnsCache->Utils.WeakMap.set(transactionSchema, fn)
719
+ let _ = fnsCache->Utils.WeakMap.set(selectedTransactionFields, fn)
442
720
  fn
443
721
  }
444
722
  }
@@ -446,33 +724,15 @@ let makeThrowingGetEventTransaction = (~getTransactionFields) => {
446
724
  }
447
725
  }
448
726
 
449
- let sanitizeUrl = (url: string) => {
450
- // Regular expression requiring protocol and capturing hostname
451
- // - (https?:\/\/) : Required http:// or https:// (capturing group)
452
- // - ([^\/?]+) : Capture hostname (one or more characters that aren't / or ?)
453
- // - .* : Match rest of the string
454
- let regex = %re("/https?:\/\/([^\/?]+).*/")
455
-
456
- switch Js.Re.exec_(regex, url) {
457
- | Some(result) =>
458
- switch Js.Re.captures(result)->Belt.Array.get(1) {
459
- | Some(host) => host->Js.Nullable.toOption
460
- | None => None
461
- }
462
- | None => None
463
- }
464
- }
465
-
466
727
  type options = {
467
728
  sourceFor: Source.sourceFor,
468
729
  syncConfig: Config.sourceSync,
469
730
  url: string,
470
731
  chain: ChainMap.Chain.t,
471
- contracts: array<Internal.evmContractConfig>,
472
732
  eventRouter: EventRouter.t<Internal.evmEventConfig>,
473
733
  allEventSignatures: array<string>,
474
- shouldUseHypersyncClientDecoder: bool,
475
734
  lowercaseAddresses: bool,
735
+ ws?: string,
476
736
  }
477
737
 
478
738
  let make = (
@@ -481,36 +741,42 @@ let make = (
481
741
  syncConfig,
482
742
  url,
483
743
  chain,
484
- contracts,
485
744
  eventRouter,
486
745
  allEventSignatures,
487
- shouldUseHypersyncClientDecoder,
488
746
  lowercaseAddresses,
747
+ ?ws,
489
748
  }: options,
490
749
  ): t => {
491
- let urlHost = switch sanitizeUrl(url) {
750
+ let chainId = chain->ChainMap.Chain.toChainId
751
+ let urlHost = switch Utils.Url.getHostFromUrl(url) {
492
752
  | None =>
493
753
  Js.Exn.raiseError(
494
- `EE109: The RPC url "${url}" is incorrect format. The RPC url needs to start with either http:// or https://`,
754
+ `The RPC url for chain ${chainId->Belt.Int.toString} is in incorrect format. The RPC url needs to start with either http:// or https://`,
495
755
  )
496
756
  | Some(host) => host
497
757
  }
498
758
  let name = `RPC (${urlHost})`
499
759
 
500
- let provider = Ethers.JsonRpcProvider.make(~rpcUrl=url, ~chainId=chain->ChainMap.Chain.toChainId)
501
-
502
760
  let getSelectionConfig = memoGetSelectionConfig(~chain)
503
761
 
504
762
  let mutSuggestedBlockIntervals = Js.Dict.empty()
505
763
 
764
+ let client = Rpc.makeClient(url)
765
+
506
766
  let makeTransactionLoader = () =>
507
767
  LazyLoader.make(
508
- ~loaderFn=transactionHash =>
509
- provider->Ethers.JsonRpcProvider.getTransaction(~transactionHash),
768
+ ~loaderFn=transactionHash => {
769
+ Prometheus.SourceRequestCount.increment(
770
+ ~sourceName=name,
771
+ ~chainId=chain->ChainMap.Chain.toChainId,
772
+ ~method="eth_getTransactionByHash",
773
+ )
774
+ Rpc.GetTransactionByHash.rawRoute->Rest.fetch(transactionHash, ~client)
775
+ },
510
776
  ~onError=(am, ~exn) => {
511
777
  Logging.error({
512
778
  "err": exn->Utils.prettifyExn,
513
- "msg": `EE1100: Top level promise timeout reached. Please review other errors or warnings in the code. This function will retry in ${(am._retryDelayMillis / 1000)
779
+ "msg": `Top level promise timeout reached. Please review other errors or warnings in the code. This function will retry in ${(am._retryDelayMillis / 1000)
514
780
  ->Belt.Int.toString} seconds. It is highly likely that your indexer isn't syncing on one or more chains currently. Also take a look at the "suggestedFix" in the metadata of this command`,
515
781
  "source": name,
516
782
  "chainId": chain->ChainMap.Chain.toChainId,
@@ -526,19 +792,19 @@ let make = (
526
792
 
527
793
  let makeBlockLoader = () =>
528
794
  LazyLoader.make(
529
- ~loaderFn=blockNumber =>
530
- getKnownBlockWithBackoff(
531
- ~provider,
795
+ ~loaderFn=blockNumber => {
796
+ getKnownRawBlockWithBackoff(
797
+ ~client,
532
798
  ~sourceName=name,
533
799
  ~chain,
534
800
  ~backoffMsOnFailure=1000,
535
801
  ~blockNumber,
536
- ~lowercaseAddresses,
537
- ),
802
+ )
803
+ },
538
804
  ~onError=(am, ~exn) => {
539
805
  Logging.error({
540
806
  "err": exn->Utils.prettifyExn,
541
- "msg": `EE1100: Top level promise timeout reached. Please review other errors or warnings in the code. This function will retry in ${(am._retryDelayMillis / 1000)
807
+ "msg": `Top level promise timeout reached. Please review other errors or warnings in the code. This function will retry in ${(am._retryDelayMillis / 1000)
542
808
  ->Belt.Int.toString} seconds. It is highly likely that your indexer isn't syncing on one or more chains currently. Also take a look at the "suggestedFix" in the metadata of this command`,
543
809
  "source": name,
544
810
  "chainId": chain->ChainMap.Chain.toChainId,
@@ -552,27 +818,60 @@ let make = (
552
818
  },
553
819
  )
554
820
 
821
+ let makeReceiptLoader = () =>
822
+ LazyLoader.make(
823
+ ~loaderFn=transactionHash => {
824
+ Prometheus.SourceRequestCount.increment(
825
+ ~sourceName=name,
826
+ ~chainId=chain->ChainMap.Chain.toChainId,
827
+ ~method="eth_getTransactionReceipt",
828
+ )
829
+ Rpc.GetTransactionReceipt.rawRoute->Rest.fetch(transactionHash, ~client)
830
+ },
831
+ ~onError=(am, ~exn) => {
832
+ Logging.error({
833
+ "err": exn->Utils.prettifyExn,
834
+ "msg": `Top level promise timeout reached. Please review other errors or warnings in the code. This function will retry in ${(am._retryDelayMillis / 1000)
835
+ ->Belt.Int.toString} seconds. It is highly likely that your indexer isn't syncing on one or more chains currently. Also take a look at the "suggestedFix" in the metadata of this command`,
836
+ "source": name,
837
+ "chainId": chain->ChainMap.Chain.toChainId,
838
+ "metadata": {
839
+ {
840
+ "asyncTaskName": "receiptLoader: fetching transaction receipt - `getTransactionReceipt` rpc call",
841
+ "suggestedFix": "This likely means the RPC url you are using is not responding correctly. Please try another RPC endipoint.",
842
+ }
843
+ },
844
+ })
845
+ },
846
+ )
847
+
555
848
  let blockLoader = ref(makeBlockLoader())
556
849
  let transactionLoader = ref(makeTransactionLoader())
850
+ let receiptLoader = ref(makeReceiptLoader())
557
851
 
558
- let getEventBlockOrThrow = makeThrowingGetEventBlock(~getBlock=blockNumber =>
559
- blockLoader.contents->LazyLoader.get(blockNumber)
852
+ let getEventBlockOrThrow = makeThrowingGetEventBlock(
853
+ ~getBlockJson=blockNumber => blockLoader.contents->LazyLoader.get(blockNumber),
854
+ ~lowercaseAddresses,
560
855
  )
561
856
  let getEventTransactionOrThrow = makeThrowingGetEventTransaction(
562
- ~getTransactionFields=Ethers.JsonRpcProvider.makeGetTransactionFields(
563
- ~getTransactionByHash=LazyLoader.get(transactionLoader.contents, _),
564
- ~lowercaseAddresses,
565
- ),
857
+ ~getTransactionJson=async transactionHash => {
858
+ switch await transactionLoader.contents->LazyLoader.get(transactionHash) {
859
+ | Some(json) => json
860
+ | None => Js.Exn.raiseError(`Transaction not found for hash: ${transactionHash}`)
861
+ }
862
+ },
863
+ ~getReceiptJson=async transactionHash => {
864
+ switch await receiptLoader.contents->LazyLoader.get(transactionHash) {
865
+ | Some(json) => json
866
+ | None => Js.Exn.raiseError(`Transaction receipt not found for hash: ${transactionHash}`)
867
+ }
868
+ },
869
+ ~lowercaseAddresses,
566
870
  )
567
871
 
568
- let contractNameAbiMapping = Js.Dict.empty()
569
- contracts->Belt.Array.forEach(contract => {
570
- contractNameAbiMapping->Js.Dict.set(contract.name, contract.abi)
571
- })
572
-
573
- let convertEthersLogToHyperSyncEvent = (log: Ethers.log): HyperSyncClient.ResponseTypes.event => {
872
+ let convertLogToHyperSyncEvent = (log: Rpc.GetLogs.log): HyperSyncClient.ResponseTypes.event => {
574
873
  let hyperSyncLog: HyperSyncClient.ResponseTypes.log = {
575
- removed: log.removed->Option.getWithDefault(false),
874
+ removed: log.removed,
576
875
  index: log.logIndex,
577
876
  transactionIndex: log.transactionIndex,
578
877
  transactionHash: log.transactionHash,
@@ -580,7 +879,7 @@ let make = (
580
879
  blockNumber: log.blockNumber,
581
880
  address: log.address,
582
881
  data: log.data,
583
- topics: log.topics->Array.map(topic => Js.Nullable.return(topic)),
882
+ topics: log.topics->(Utils.magic: array<string> => array<Js.Nullable.t<EvmTypes.Hex.t>>),
584
883
  }
585
884
  {log: hyperSyncLog}
586
885
  }
@@ -601,7 +900,7 @@ let make = (
601
900
  ~toBlock,
602
901
  ~addressesByContractName,
603
902
  ~indexingContracts,
604
- ~currentBlockHeight,
903
+ ~knownHeight,
605
904
  ~partitionId,
606
905
  ~selection: FetchState.selection,
607
906
  ~retry as _,
@@ -621,8 +920,8 @@ let make = (
621
920
 
622
921
  // Always have a toBlock for an RPC worker
623
922
  let toBlock = switch toBlock {
624
- | Some(toBlock) => Pervasives.min(toBlock, currentBlockHeight)
625
- | None => currentBlockHeight
923
+ | Some(toBlock) => Pervasives.min(toBlock, knownHeight)
924
+ | None => knownHeight
626
925
  }
627
926
 
628
927
  let suggestedToBlock = Pervasives.min(fromBlock + suggestedBlockInterval - 1, toBlock)
@@ -631,22 +930,29 @@ let make = (
631
930
 
632
931
  let firstBlockParentPromise =
633
932
  fromBlock > 0
634
- ? blockLoader.contents->LazyLoader.get(fromBlock - 1)->Promise.thenResolve(res => res->Some)
933
+ ? blockLoader.contents
934
+ ->LazyLoader.get(fromBlock - 1)
935
+ ->Promise.thenResolve(json => Some(parseBlockInfo(json)))
635
936
  : Promise.resolve(None)
636
937
 
637
938
  let {getLogSelectionOrThrow} = getSelectionConfig(selection)
638
939
  let {addresses, topicQuery} = getLogSelectionOrThrow(~addressesByContractName)
639
940
 
640
- let {logs, latestFetchedBlock} = await getNextPage(
941
+ let {logs, latestFetchedBlockInfo} = await getNextPage(
641
942
  ~fromBlock,
642
943
  ~toBlock=suggestedToBlock,
643
944
  ~addresses,
644
945
  ~topicQuery,
645
- ~loadBlock=blockNumber => blockLoader.contents->LazyLoader.get(blockNumber),
946
+ ~loadBlock=blockNumber =>
947
+ blockLoader.contents
948
+ ->LazyLoader.get(blockNumber)
949
+ ->Promise.thenResolve(parseBlockInfo),
646
950
  ~syncConfig,
647
- ~provider,
951
+ ~client,
648
952
  ~mutSuggestedBlockIntervals,
649
953
  ~partitionId,
954
+ ~sourceName=name,
955
+ ~chainId=chain->ChainMap.Chain.toChainId,
650
956
  )
651
957
 
652
958
  let executedBlockInterval = suggestedToBlock - fromBlock + 1
@@ -669,44 +975,41 @@ let make = (
669
975
  )
670
976
  }
671
977
 
672
- let parsedQueueItems = if shouldUseHypersyncClientDecoder {
673
- // Convert Ethers logs to HyperSync events
674
- let hyperSyncEvents = logs->Belt.Array.map(convertEthersLogToHyperSyncEvent)
978
+ // Convert RPC logs to HyperSync events
979
+ let hyperSyncEvents = logs->Belt.Array.map(convertLogToHyperSyncEvent)
675
980
 
676
- // Decode using HyperSyncClient decoder
677
- let parsedEvents = try await getHscDecoder().decodeEvents(hyperSyncEvents) catch {
678
- | exn =>
679
- raise(
680
- Source.GetItemsError(
681
- FailedGettingItems({
682
- exn,
683
- attemptedToBlock: toBlock,
684
- retry: ImpossibleForTheQuery({
685
- message: "Failed to parse events using hypersync client decoder. Please double-check your ABI.",
686
- }),
981
+ // Decode using HyperSyncClient decoder
982
+ let parsedEvents = try await getHscDecoder().decodeEvents(hyperSyncEvents) catch {
983
+ | exn =>
984
+ raise(
985
+ Source.GetItemsError(
986
+ FailedGettingItems({
987
+ exn,
988
+ attemptedToBlock: toBlock,
989
+ retry: ImpossibleForTheQuery({
990
+ message: "Failed to parse events using hypersync client decoder. Please double-check your ABI.",
687
991
  }),
688
- ),
689
- )
690
- }
992
+ }),
993
+ ),
994
+ )
995
+ }
691
996
 
997
+ let parsedQueueItems =
692
998
  await logs
693
999
  ->Array.zip(parsedEvents)
694
1000
  ->Array.keepMap(((
695
- log: Ethers.log,
1001
+ log: Rpc.GetLogs.log,
696
1002
  maybeDecodedEvent: Js.Nullable.t<HyperSyncClient.Decoder.decodedEvent>,
697
1003
  )) => {
698
- let topic0 = log.topics[0]->Option.getWithDefault("0x0"->EvmTypes.Hex.fromStringUnsafe)
1004
+ let topic0 = log.topics[0]->Option.getWithDefault("0x0")
699
1005
  let routedAddress = if lowercaseAddresses {
700
1006
  log.address->Address.Evm.fromAddressLowercaseOrThrow
701
1007
  } else {
702
- log.address
1008
+ log.address->Address.Evm.fromAddressOrThrow
703
1009
  }
704
1010
 
705
1011
  switch eventRouter->EventRouter.get(
706
- ~tag=EventRouter.getEvmEventId(
707
- ~sighash=topic0->EvmTypes.Hex.toString,
708
- ~topicCount=log.topics->Array.length,
709
- ),
1012
+ ~tag=EventRouter.getEvmEventId(~sighash=topic0, ~topicCount=log.topics->Array.length),
710
1013
  ~indexingContracts,
711
1014
  ~contractAddress=routedAddress,
712
1015
  ~blockNumber=log.blockNumber,
@@ -719,9 +1022,9 @@ let make = (
719
1022
  (
720
1023
  async () => {
721
1024
  let (block, transaction) = try await Promise.all2((
722
- log->getEventBlockOrThrow,
1025
+ log->getEventBlockOrThrow(~selectedBlockFields=eventConfig.selectedBlockFields),
723
1026
  log->getEventTransactionOrThrow(
724
- ~transactionSchema=eventConfig.transactionSchema,
1027
+ ~selectedTransactionFields=eventConfig.selectedTransactionFields,
725
1028
  ),
726
1029
  )) catch {
727
1030
  | exn =>
@@ -739,17 +1042,15 @@ let make = (
739
1042
 
740
1043
  Internal.Event({
741
1044
  eventConfig: (eventConfig :> Internal.eventConfig),
742
- timestamp: block.timestamp,
743
- blockNumber: block.number,
1045
+ timestamp: block->Evm.getTimestamp,
1046
+ blockNumber: block->Evm.getNumber,
744
1047
  chain,
745
1048
  logIndex: log.logIndex,
746
1049
  event: {
747
1050
  chainId: chain->ChainMap.Chain.toChainId,
748
1051
  params: decoded->eventConfig.convertHyperSyncEventArgs,
749
1052
  transaction,
750
- block: block->(
751
- Utils.magic: Ethers.JsonRpcProvider.block => Internal.eventBlock
752
- ),
1053
+ block,
753
1054
  srcAddress: routedAddress,
754
1055
  logIndex: log.logIndex,
755
1056
  }->Internal.fromGenericEvent,
@@ -764,97 +1065,10 @@ let make = (
764
1065
  }
765
1066
  })
766
1067
  ->Promise.all
767
- } else {
768
- // Decode using Viem
769
- await logs
770
- ->Belt.Array.keepMap(log => {
771
- let topic0 = log.topics->Js.Array2.unsafe_get(0)
772
-
773
- switch eventRouter->EventRouter.get(
774
- ~tag=EventRouter.getEvmEventId(
775
- ~sighash=topic0->EvmTypes.Hex.toString,
776
- ~topicCount=log.topics->Array.length,
777
- ),
778
- ~indexingContracts,
779
- ~contractAddress=log.address,
780
- ~blockNumber=log.blockNumber,
781
- ) {
782
- | None => None //ignore events that aren't registered
783
- | Some(eventConfig) =>
784
- let blockNumber = log.blockNumber
785
- let logIndex = log.logIndex
786
- Some(
787
- (
788
- async () => {
789
- let (block, transaction) = try await Promise.all2((
790
- log->getEventBlockOrThrow,
791
- log->getEventTransactionOrThrow(~transactionSchema=eventConfig.transactionSchema),
792
- )) catch {
793
- // Promise.catch won't work here, because the error
794
- // might be thrown before a microtask is created
795
- | exn =>
796
- raise(
797
- Source.GetItemsError(
798
- FailedGettingFieldSelection({
799
- message: "Failed getting selected fields. Please double-check your RPC provider returns correct data.",
800
- exn,
801
- blockNumber,
802
- logIndex,
803
- }),
804
- ),
805
- )
806
- }
807
-
808
- let decodedEvent = try contractNameAbiMapping->Viem.parseLogOrThrow(
809
- ~contractName=eventConfig.contractName,
810
- ~topics=log.topics,
811
- ~data=log.data,
812
- ) catch {
813
- | exn =>
814
- raise(
815
- Source.GetItemsError(
816
- FailedGettingItems({
817
- exn,
818
- attemptedToBlock: toBlock,
819
- retry: ImpossibleForTheQuery({
820
- message: `Failed to parse event with viem, please double-check your ABI. Block number: ${blockNumber->Int.toString}, log index: ${logIndex->Int.toString}`,
821
- }),
822
- }),
823
- ),
824
- )
825
- }
826
-
827
- Internal.Event({
828
- eventConfig: (eventConfig :> Internal.eventConfig),
829
- timestamp: block.timestamp,
830
- blockNumber: block.number,
831
- chain,
832
- logIndex: log.logIndex,
833
- event: {
834
- chainId: chain->ChainMap.Chain.toChainId,
835
- params: decodedEvent.args,
836
- transaction,
837
- // Unreliably expect that the Ethers block fields match the types in HyperIndex
838
- // I assume this is wrong in some cases, so we need to fix it in the future
839
- block: block->(
840
- Utils.magic: Ethers.JsonRpcProvider.block => Internal.eventBlock
841
- ),
842
- srcAddress: log.address,
843
- logIndex: log.logIndex,
844
- }->Internal.fromGenericEvent,
845
- })
846
- }
847
- )(),
848
- )
849
- }
850
- })
851
- ->Promise.all
852
- }
853
1068
 
854
1069
  let optFirstBlockParent = await firstBlockParentPromise
855
1070
 
856
- let totalTimeElapsed =
857
- startFetchingBatchTimeRef->Hrtime.timeSince->Hrtime.toMillis->Hrtime.intFromMillis
1071
+ let totalTimeElapsed = startFetchingBatchTimeRef->Hrtime.timeSince->Hrtime.toSecondsFloat
858
1072
 
859
1073
  let reorgGuard: ReorgDetection.reorgGuard = {
860
1074
  prevRangeLastBlock: optFirstBlockParent->Option.map(b => {
@@ -862,19 +1076,19 @@ let make = (
862
1076
  blockHash: b.hash,
863
1077
  }),
864
1078
  rangeLastBlock: {
865
- blockNumber: latestFetchedBlock.number,
866
- blockHash: latestFetchedBlock.hash,
1079
+ blockNumber: latestFetchedBlockInfo.number,
1080
+ blockHash: latestFetchedBlockInfo.hash,
867
1081
  },
868
1082
  }
869
1083
 
870
1084
  {
871
- latestFetchedBlockTimestamp: latestFetchedBlock.timestamp,
872
- latestFetchedBlockNumber: latestFetchedBlock.number,
1085
+ latestFetchedBlockTimestamp: latestFetchedBlockInfo.timestamp,
1086
+ latestFetchedBlockNumber: latestFetchedBlockInfo.number,
873
1087
  parsedQueueItems,
874
1088
  stats: {
875
1089
  totalTimeElapsed: totalTimeElapsed,
876
1090
  },
877
- currentBlockHeight,
1091
+ knownHeight,
878
1092
  reorgGuard,
879
1093
  fromBlockQueried: fromBlock,
880
1094
  }
@@ -886,32 +1100,75 @@ let make = (
886
1100
  // function when a reorg is detected
887
1101
  blockLoader := makeBlockLoader()
888
1102
  transactionLoader := makeTransactionLoader()
1103
+ receiptLoader := makeReceiptLoader()
889
1104
 
890
1105
  blockNumbers
891
1106
  ->Array.map(blockNum => blockLoader.contents->LazyLoader.get(blockNum))
892
1107
  ->Promise.all
893
- ->Promise.thenResolve(blocks => {
894
- blocks
895
- ->Array.map((b): ReorgDetection.blockDataWithTimestamp => {
896
- blockNumber: b.number,
897
- blockHash: b.hash,
898
- blockTimestamp: b.timestamp,
1108
+ ->Promise.thenResolve(rawBlocks => {
1109
+ rawBlocks
1110
+ ->Array.map(json => {
1111
+ let b = parseBlockInfo(json)
1112
+
1113
+ (
1114
+ {
1115
+ blockNumber: b.number,
1116
+ blockHash: b.hash,
1117
+ blockTimestamp: b.timestamp,
1118
+ }: ReorgDetection.blockDataWithTimestamp
1119
+ )
899
1120
  })
900
1121
  ->Ok
901
1122
  })
902
1123
  ->Promise.catch(exn => exn->Error->Promise.resolve)
903
1124
  }
904
1125
 
905
- let client = Rest.client(url)
1126
+ let createHeightSubscription =
1127
+ ws->Belt.Option.map(wsUrl => (~onHeight) =>
1128
+ RpcWebSocketHeightStream.subscribe(~wsUrl, ~chainId, ~onHeight))
906
1129
 
907
1130
  {
908
1131
  name,
909
1132
  sourceFor,
910
1133
  chain,
911
1134
  poweredByHyperSync: false,
912
- pollingInterval: 1000,
1135
+ pollingInterval: syncConfig.pollingInterval,
913
1136
  getBlockHashes,
914
- getHeightOrThrow: () => Rpc.GetBlockHeight.route->Rest.fetch((), ~client),
1137
+ getHeightOrThrow: async () => {
1138
+ let timerRef = Hrtime.makeTimer()
1139
+ let height = try {
1140
+ await Rpc.GetBlockHeight.route->Rest.fetch((), ~client)
1141
+ } catch {
1142
+ | exn =>
1143
+ let seconds = timerRef->Hrtime.timeSince->Hrtime.toSecondsFloat
1144
+ Prometheus.SourceRequestCount.increment(
1145
+ ~sourceName=name,
1146
+ ~chainId=chain->ChainMap.Chain.toChainId,
1147
+ ~method="eth_blockNumber",
1148
+ )
1149
+ Prometheus.SourceRequestCount.addSeconds(
1150
+ ~sourceName=name,
1151
+ ~chainId=chain->ChainMap.Chain.toChainId,
1152
+ ~method="eth_blockNumber",
1153
+ ~seconds,
1154
+ )
1155
+ exn->raise
1156
+ }
1157
+ let seconds = timerRef->Hrtime.timeSince->Hrtime.toSecondsFloat
1158
+ Prometheus.SourceRequestCount.increment(
1159
+ ~sourceName=name,
1160
+ ~chainId=chain->ChainMap.Chain.toChainId,
1161
+ ~method="eth_blockNumber",
1162
+ )
1163
+ Prometheus.SourceRequestCount.addSeconds(
1164
+ ~sourceName=name,
1165
+ ~chainId=chain->ChainMap.Chain.toChainId,
1166
+ ~method="eth_blockNumber",
1167
+ ~seconds,
1168
+ )
1169
+ height
1170
+ },
915
1171
  getItemsOrThrow,
1172
+ ?createHeightSubscription,
916
1173
  }
917
1174
  }