envio 3.3.0-alpha.6 → 3.3.0-alpha.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.d.ts +23 -5
- package/package.json +6 -6
- package/src/Address.res +5 -2
- package/src/Address.res.mjs +3 -1
- package/src/Api.res +1 -1
- package/src/Api.res.mjs +1 -1
- package/src/ChainFetching.res +34 -34
- package/src/ChainFetching.res.mjs +29 -29
- package/src/ChainState.res +328 -165
- package/src/ChainState.res.mjs +205 -100
- package/src/ChainState.resi +16 -7
- package/src/Config.res +55 -27
- package/src/Config.res.mjs +46 -7
- package/src/ContractRegisterContext.res +2 -12
- package/src/ContractRegisterContext.res.mjs +3 -5
- package/src/Core.res +7 -0
- package/src/CrossChainState.res +54 -79
- package/src/CrossChainState.res.mjs +45 -60
- package/src/CrossChainState.resi +1 -1
- package/src/Ecosystem.res +3 -3
- package/src/Ecosystem.res.mjs +3 -3
- package/src/Envio.res +14 -9
- package/src/EventConfigBuilder.res +105 -56
- package/src/EventConfigBuilder.res.mjs +69 -33
- package/src/EventProcessing.res +19 -15
- package/src/EventProcessing.res.mjs +13 -12
- package/src/FetchState.res +383 -182
- package/src/FetchState.res.mjs +260 -264
- package/src/HandlerLoader.res +7 -112
- package/src/HandlerLoader.res.mjs +1 -87
- package/src/HandlerRegister.res +209 -6
- package/src/HandlerRegister.res.mjs +90 -5
- package/src/HandlerRegister.resi +13 -2
- package/src/IndexerState.res +6 -3
- package/src/IndexerState.res.mjs +3 -3
- package/src/IndexerState.resi +1 -1
- package/src/IndexingAddresses.res +10 -7
- package/src/IndexingAddresses.res.mjs +8 -7
- package/src/IndexingAddresses.resi +3 -1
- package/src/Internal.res +82 -36
- package/src/Internal.res.mjs +11 -1
- package/src/Main.res +22 -23
- package/src/Main.res.mjs +15 -17
- package/src/Metrics.res +43 -2
- package/src/Metrics.res.mjs +39 -3
- package/src/Prometheus.res +0 -35
- package/src/Prometheus.res.mjs +33 -68
- package/src/RawEvent.res +7 -2
- package/src/RawEvent.res.mjs +4 -4
- package/src/SimulateItems.res +26 -8
- package/src/SimulateItems.res.mjs +11 -14
- package/src/TestIndexer.res +1 -1
- package/src/TestIndexer.res.mjs +1 -1
- package/src/sources/BlockStore.res +46 -0
- package/src/sources/BlockStore.res.mjs +24 -0
- package/src/sources/EventRouter.res +19 -12
- package/src/sources/EventRouter.res.mjs +7 -5
- package/src/sources/Evm.res +54 -4
- package/src/sources/Evm.res.mjs +43 -4
- package/src/sources/EvmChain.res +14 -18
- package/src/sources/EvmChain.res.mjs +12 -13
- package/src/sources/FieldMask.res +39 -0
- package/src/sources/FieldMask.res.mjs +42 -0
- package/src/sources/Fuel.res +5 -2
- package/src/sources/Fuel.res.mjs +4 -3
- package/src/sources/HyperFuelSource.res +32 -40
- package/src/sources/HyperFuelSource.res.mjs +53 -45
- package/src/sources/HyperSync.res +35 -9
- package/src/sources/HyperSync.res.mjs +52 -35
- package/src/sources/HyperSync.resi +11 -3
- package/src/sources/HyperSyncClient.res +15 -4
- package/src/sources/HyperSyncHeightStream.res +1 -8
- package/src/sources/HyperSyncHeightStream.res.mjs +0 -2
- package/src/sources/HyperSyncSource.res +39 -58
- package/src/sources/HyperSyncSource.res.mjs +44 -44
- package/src/sources/RpcSource.res +114 -92
- package/src/sources/RpcSource.res.mjs +77 -46
- package/src/sources/RpcWebSocketHeightStream.res +0 -5
- package/src/sources/RpcWebSocketHeightStream.res.mjs +0 -2
- package/src/sources/SimulateSource.res +6 -3
- package/src/sources/SimulateSource.res.mjs +13 -5
- package/src/sources/Source.res +25 -5
- package/src/sources/SourceManager.res +66 -3
- package/src/sources/SourceManager.res.mjs +55 -7
- package/src/sources/SourceManager.resi +10 -0
- package/src/sources/Svm.res +15 -10
- package/src/sources/Svm.res.mjs +24 -7
- package/src/sources/SvmHyperSyncClient.res +5 -6
- package/src/sources/SvmHyperSyncSource.res +84 -41
- package/src/sources/SvmHyperSyncSource.res.mjs +90 -43
- package/src/sources/TransactionStore.res +6 -107
- package/src/sources/TransactionStore.res.mjs +5 -86
- package/svm.schema.json +19 -0
|
@@ -3,7 +3,6 @@
|
|
|
3
3
|
import * as Source from "./Source.res.mjs";
|
|
4
4
|
import * as Logging from "../Logging.res.mjs";
|
|
5
5
|
import * as HyperFuel from "./HyperFuel.res.mjs";
|
|
6
|
-
import * as Prometheus from "../Prometheus.res.mjs";
|
|
7
6
|
import * as EventRouter from "./EventRouter.res.mjs";
|
|
8
7
|
import * as Performance from "../bindings/Performance.res.mjs";
|
|
9
8
|
import * as Stdlib_JsExn from "@rescript/runtime/lib/es6/Stdlib_JsExn.js";
|
|
@@ -101,41 +100,43 @@ function getSelectionConfig(selection, chain) {
|
|
|
101
100
|
}
|
|
102
101
|
};
|
|
103
102
|
let contractNames = new Set();
|
|
104
|
-
selection.
|
|
103
|
+
selection.onEventRegistrations.forEach(reg => {
|
|
104
|
+
let eventConfig = reg.eventConfig;
|
|
105
105
|
let contractName = eventConfig.contractName;
|
|
106
|
-
|
|
106
|
+
let isWildcard = reg.isWildcard;
|
|
107
|
+
if (!isWildcard) {
|
|
107
108
|
contractNames.add(contractName);
|
|
108
109
|
}
|
|
109
|
-
EventRouter.addOrThrow(eventRouter, eventConfig.id,
|
|
110
|
-
let isWildcard = eventConfig.isWildcard;
|
|
111
|
-
if (isWildcard) {
|
|
112
|
-
let tmp = eventConfig.kind;
|
|
113
|
-
if (typeof tmp !== "object") {
|
|
114
|
-
switch (tmp) {
|
|
115
|
-
case "Mint" :
|
|
116
|
-
return addNonLogDataWildcardReceiptTypes(11);
|
|
117
|
-
case "Burn" :
|
|
118
|
-
return addNonLogDataWildcardReceiptTypes(12);
|
|
119
|
-
case "Transfer" :
|
|
120
|
-
addNonLogDataWildcardReceiptTypes(7);
|
|
121
|
-
return addNonLogDataWildcardReceiptTypes(8);
|
|
122
|
-
case "Call" :
|
|
123
|
-
return addNonLogDataWildcardReceiptTypes(0);
|
|
124
|
-
}
|
|
125
|
-
}
|
|
126
|
-
}
|
|
110
|
+
EventRouter.addOrThrow(eventRouter, eventConfig.id, reg, contractName, isWildcard, eventConfig.name, chain);
|
|
127
111
|
let match = eventConfig.kind;
|
|
128
112
|
if (typeof match !== "object") {
|
|
129
113
|
switch (match) {
|
|
130
114
|
case "Mint" :
|
|
131
|
-
|
|
115
|
+
if (isWildcard) {
|
|
116
|
+
return addNonLogDataWildcardReceiptTypes(11);
|
|
117
|
+
} else {
|
|
118
|
+
return addNonLogDataReceiptType(contractName, 11);
|
|
119
|
+
}
|
|
132
120
|
case "Burn" :
|
|
133
|
-
|
|
121
|
+
if (isWildcard) {
|
|
122
|
+
return addNonLogDataWildcardReceiptTypes(12);
|
|
123
|
+
} else {
|
|
124
|
+
return addNonLogDataReceiptType(contractName, 12);
|
|
125
|
+
}
|
|
134
126
|
case "Transfer" :
|
|
135
|
-
|
|
136
|
-
|
|
127
|
+
if (isWildcard) {
|
|
128
|
+
addNonLogDataWildcardReceiptTypes(7);
|
|
129
|
+
return addNonLogDataWildcardReceiptTypes(8);
|
|
130
|
+
} else {
|
|
131
|
+
addNonLogDataReceiptType(contractName, 7);
|
|
132
|
+
return addNonLogDataReceiptType(contractName, 8);
|
|
133
|
+
}
|
|
137
134
|
case "Call" :
|
|
138
|
-
|
|
135
|
+
if (isWildcard) {
|
|
136
|
+
return addNonLogDataWildcardReceiptTypes(0);
|
|
137
|
+
} else {
|
|
138
|
+
return Stdlib_JsError.throwWithMessage("Call receipt indexing currently supported only in wildcard mode");
|
|
139
|
+
}
|
|
139
140
|
}
|
|
140
141
|
} else {
|
|
141
142
|
let rb = BigInt(match.logId);
|
|
@@ -181,7 +182,6 @@ function memoGetSelectionConfig(chain) {
|
|
|
181
182
|
function make(param) {
|
|
182
183
|
let apiToken = param.apiToken;
|
|
183
184
|
let chain = param.chain;
|
|
184
|
-
let name = "HyperFuel";
|
|
185
185
|
let apiToken$1 = apiToken !== undefined ? apiToken : Stdlib_JsError.throwWithMessage(`An Envio API token is required for using HyperFuel as a data-source.
|
|
186
186
|
Set the ENVIO_API_TOKEN environment variable in your .env file.
|
|
187
187
|
Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`);
|
|
@@ -196,12 +196,11 @@ Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`);
|
|
|
196
196
|
client = ErrorHandling.mkLogAndRaise(undefined, "Failed to instantiate the HyperFuel client", exn);
|
|
197
197
|
}
|
|
198
198
|
let getSelectionConfig = memoGetSelectionConfig(chain);
|
|
199
|
-
let getItemsOrThrow = async (fromBlock, toBlock, addressesByContractName, contractNameByAddress, knownHeight, param, selection, retry, logger) => {
|
|
199
|
+
let getItemsOrThrow = async (fromBlock, toBlock, addressesByContractName, contractNameByAddress, knownHeight, param, selection, param$1, retry, logger) => {
|
|
200
200
|
let totalTimeRef = Performance.now();
|
|
201
201
|
let selectionConfig = getSelectionConfig(selection);
|
|
202
202
|
let recieptsSelection = selectionConfig.getRecieptsSelection(addressesByContractName);
|
|
203
203
|
let startFetchingBatchTimeRef = Performance.now();
|
|
204
|
-
Prometheus.SourceRequestCount.increment(name, chain, "getLogs");
|
|
205
204
|
let pageUnsafe;
|
|
206
205
|
try {
|
|
207
206
|
pageUnsafe = await HyperFuel.GetLogs.query(client, fromBlock, toBlock, recieptsSelection);
|
|
@@ -250,6 +249,10 @@ Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`);
|
|
|
250
249
|
};
|
|
251
250
|
}
|
|
252
251
|
let pageFetchTime = Performance.secondsSince(startFetchingBatchTimeRef);
|
|
252
|
+
let requestStats = [{
|
|
253
|
+
method: "getLogs",
|
|
254
|
+
seconds: pageFetchTime
|
|
255
|
+
}];
|
|
253
256
|
let knownHeight$1 = pageUnsafe.archiveHeight;
|
|
254
257
|
let heighestBlockQueried = pageUnsafe.nextBlock - 1 | 0;
|
|
255
258
|
let parsingTimeRef = Performance.now();
|
|
@@ -277,10 +280,10 @@ Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`);
|
|
|
277
280
|
eventId = burnEventTag;
|
|
278
281
|
break;
|
|
279
282
|
}
|
|
280
|
-
let
|
|
281
|
-
let
|
|
282
|
-
if (
|
|
283
|
-
|
|
283
|
+
let onEventRegistration = EventRouter.get(selectionConfig.eventRouter, eventId, contractAddress, contractNameByAddress);
|
|
284
|
+
let onEventRegistration$1;
|
|
285
|
+
if (onEventRegistration !== undefined) {
|
|
286
|
+
onEventRegistration$1 = onEventRegistration;
|
|
284
287
|
} else {
|
|
285
288
|
let logger$1 = Logging.createChildFrom(logger, {
|
|
286
289
|
chainId: chain,
|
|
@@ -289,11 +292,12 @@ Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`);
|
|
|
289
292
|
contractAddress: contractAddress,
|
|
290
293
|
eventId: eventId
|
|
291
294
|
});
|
|
292
|
-
|
|
295
|
+
onEventRegistration$1 = ErrorHandling.mkLogAndRaise(logger$1, "Failed to route registered event", {
|
|
293
296
|
RE_EXN_ID: EventRoutingFailed
|
|
294
297
|
});
|
|
295
298
|
}
|
|
296
|
-
let
|
|
299
|
+
let eventConfig = onEventRegistration$1.eventConfig;
|
|
300
|
+
let match = eventConfig.kind;
|
|
297
301
|
let params;
|
|
298
302
|
let exit = 0;
|
|
299
303
|
if (typeof match !== "object" || receipt.receiptType !== 6) {
|
|
@@ -349,16 +353,14 @@ Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`);
|
|
|
349
353
|
}
|
|
350
354
|
return {
|
|
351
355
|
kind: 0,
|
|
352
|
-
|
|
353
|
-
timestamp: block.time,
|
|
356
|
+
onEventRegistration: onEventRegistration$1,
|
|
354
357
|
chain: chain,
|
|
355
358
|
blockNumber: block.height,
|
|
356
|
-
blockHash: block.id,
|
|
357
359
|
logIndex: receiptIndex,
|
|
358
360
|
transactionIndex: 0,
|
|
359
361
|
payload: {
|
|
360
|
-
contractName: eventConfig
|
|
361
|
-
eventName: eventConfig
|
|
362
|
+
contractName: eventConfig.contractName,
|
|
363
|
+
eventName: eventConfig.name,
|
|
362
364
|
params: params,
|
|
363
365
|
chainId: chain,
|
|
364
366
|
srcAddress: contractAddress,
|
|
@@ -399,15 +401,17 @@ Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`);
|
|
|
399
401
|
blockHashes: blockHashes,
|
|
400
402
|
parsedQueueItems: parsedQueueItems,
|
|
401
403
|
transactionStore: undefined,
|
|
404
|
+
blockStore: undefined,
|
|
402
405
|
fromBlockQueried: fromBlock,
|
|
403
406
|
latestFetchedBlockNumber: heighestBlockQueried,
|
|
404
407
|
latestFetchedBlockTimestamp: latestFetchedBlockTimestamp,
|
|
405
|
-
stats: stats
|
|
408
|
+
stats: stats,
|
|
409
|
+
requestStats: requestStats
|
|
406
410
|
};
|
|
407
411
|
};
|
|
408
412
|
let getBlockHashes = (param, param$1) => Stdlib_JsError.throwWithMessage("HyperFuel does not support getting block hashes");
|
|
409
413
|
return {
|
|
410
|
-
name:
|
|
414
|
+
name: "HyperFuel",
|
|
411
415
|
sourceFor: "Sync",
|
|
412
416
|
chain: chain,
|
|
413
417
|
poweredByHyperSync: true,
|
|
@@ -447,9 +451,13 @@ Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`);
|
|
|
447
451
|
}
|
|
448
452
|
}
|
|
449
453
|
let seconds = Performance.secondsSince(timerRef);
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
454
|
+
return {
|
|
455
|
+
height: height,
|
|
456
|
+
requestStats: [{
|
|
457
|
+
method: "getHeight",
|
|
458
|
+
seconds: seconds
|
|
459
|
+
}]
|
|
460
|
+
};
|
|
453
461
|
},
|
|
454
462
|
getItemsOrThrow: getItemsOrThrow
|
|
455
463
|
};
|
|
@@ -16,13 +16,15 @@ let reraisIfRateLimited = exn =>
|
|
|
16
16
|
|
|
17
17
|
type logsQueryPage = {
|
|
18
18
|
items: array<HyperSyncClient.EventItems.item>,
|
|
19
|
-
//
|
|
20
|
-
blocks: array<HyperSyncClient.
|
|
19
|
+
// Block headers referenced by `items`, deduplicated by block number.
|
|
20
|
+
blocks: array<HyperSyncClient.EventItems.blockHeader>,
|
|
21
21
|
nextBlock: int,
|
|
22
22
|
archiveHeight: int,
|
|
23
23
|
rollbackGuard: option<HyperSyncClient.ResponseTypes.rollbackGuard>,
|
|
24
24
|
// Page store owning this page's raw transactions.
|
|
25
25
|
transactionStore: TransactionStore.t,
|
|
26
|
+
// Page store owning this page's raw blocks.
|
|
27
|
+
blockStore: BlockStore.t,
|
|
26
28
|
}
|
|
27
29
|
|
|
28
30
|
type missingParams = {
|
|
@@ -64,6 +66,7 @@ module GetLogs = {
|
|
|
64
66
|
~toBlockInclusive,
|
|
65
67
|
~addressesWithTopics,
|
|
66
68
|
~fieldSelection,
|
|
69
|
+
~maxNumLogs,
|
|
67
70
|
): HyperSyncClient.QueryTypes.query => {
|
|
68
71
|
fromBlock,
|
|
69
72
|
toBlockExclusive: ?switch toBlockInclusive {
|
|
@@ -72,6 +75,7 @@ module GetLogs = {
|
|
|
72
75
|
},
|
|
73
76
|
logs: addressesWithTopics,
|
|
74
77
|
fieldSelection,
|
|
78
|
+
maxNumLogs,
|
|
75
79
|
}
|
|
76
80
|
|
|
77
81
|
// Rust encodes structured failures as a JSON payload in the napi error's
|
|
@@ -105,6 +109,7 @@ module GetLogs = {
|
|
|
105
109
|
~toBlock,
|
|
106
110
|
~logSelections: array<LogSelection.t>,
|
|
107
111
|
~fieldSelection,
|
|
112
|
+
~maxNumLogs,
|
|
108
113
|
): logsQueryPage => {
|
|
109
114
|
let addressesWithTopics = logSelections->Array.flatMap(({addresses, topicSelections}) =>
|
|
110
115
|
topicSelections->Array.map(({topic0, topic1, topic2, topic3}) => {
|
|
@@ -123,9 +128,10 @@ module GetLogs = {
|
|
|
123
128
|
~toBlockInclusive=toBlock,
|
|
124
129
|
~addressesWithTopics,
|
|
125
130
|
~fieldSelection,
|
|
131
|
+
~maxNumLogs,
|
|
126
132
|
)
|
|
127
133
|
|
|
128
|
-
let (res, transactionStore) = switch await client.getEventItems(~query) {
|
|
134
|
+
let (res, transactionStore, blockStore) = switch await client.getEventItems(~query) {
|
|
129
135
|
| res => res
|
|
130
136
|
| exception exn =>
|
|
131
137
|
reraisIfRateLimited(exn)
|
|
@@ -146,6 +152,7 @@ module GetLogs = {
|
|
|
146
152
|
archiveHeight: res.archiveHeight->Option.getOr(0), //Archive Height is only None if height is 0
|
|
147
153
|
rollbackGuard: res.rollbackGuard,
|
|
148
154
|
transactionStore,
|
|
155
|
+
blockStore,
|
|
149
156
|
}
|
|
150
157
|
}
|
|
151
158
|
}
|
|
@@ -202,6 +209,7 @@ module BlockData = {
|
|
|
202
209
|
~sourceName,
|
|
203
210
|
~chainId,
|
|
204
211
|
~logger,
|
|
212
|
+
~requestStats: array<Source.requestStat>,
|
|
205
213
|
): queryResponse<array<ReorgDetection.blockDataWithTimestamp>> => {
|
|
206
214
|
let body = makeRequestBody(~fromBlock, ~toBlock)
|
|
207
215
|
|
|
@@ -214,7 +222,7 @@ module BlockData = {
|
|
|
214
222
|
},
|
|
215
223
|
)
|
|
216
224
|
|
|
217
|
-
|
|
225
|
+
let timerRef = Performance.now()
|
|
218
226
|
let maybeSuccessfulRes = switch await client.get(~query=body) {
|
|
219
227
|
| exception exn =>
|
|
220
228
|
reraisIfRateLimited(exn)
|
|
@@ -222,6 +230,9 @@ module BlockData = {
|
|
|
222
230
|
| res if res.nextBlock <= fromBlock => None
|
|
223
231
|
| res => Some(res)
|
|
224
232
|
}
|
|
233
|
+
requestStats
|
|
234
|
+
->Array.push({Source.method: "getBlockHashes", seconds: timerRef->Performance.secondsSince})
|
|
235
|
+
->ignore
|
|
225
236
|
|
|
226
237
|
// If the block is not found, retry the query. This can occur since replicas of hypersync might not have caught up yet
|
|
227
238
|
switch maybeSuccessfulRes {
|
|
@@ -231,7 +242,15 @@ module BlockData = {
|
|
|
231
242
|
`Block #${fromBlock->Int.toString} not found in HyperSync. HyperSync has multiple instances and it's possible that they drift independently slightly from the head. Indexing should continue correctly after retrying the query in ${delayMilliseconds->Int.toString}ms.`,
|
|
232
243
|
)
|
|
233
244
|
await Time.resolvePromiseAfterDelay(~delayMilliseconds)
|
|
234
|
-
await queryBlockData(
|
|
245
|
+
await queryBlockData(
|
|
246
|
+
~client,
|
|
247
|
+
~fromBlock,
|
|
248
|
+
~toBlock,
|
|
249
|
+
~sourceName,
|
|
250
|
+
~chainId,
|
|
251
|
+
~logger,
|
|
252
|
+
~requestStats,
|
|
253
|
+
)
|
|
235
254
|
}
|
|
236
255
|
| Some(res) =>
|
|
237
256
|
switch res->convertResponse {
|
|
@@ -244,6 +263,7 @@ module BlockData = {
|
|
|
244
263
|
~sourceName,
|
|
245
264
|
~chainId,
|
|
246
265
|
~logger,
|
|
266
|
+
~requestStats,
|
|
247
267
|
)
|
|
248
268
|
restRes->Result.map(rest => datas->Array.concat(rest))
|
|
249
269
|
}
|
|
@@ -258,8 +278,9 @@ module BlockData = {
|
|
|
258
278
|
~sourceName,
|
|
259
279
|
~chainId,
|
|
260
280
|
~logger,
|
|
261
|
-
) => {
|
|
262
|
-
|
|
281
|
+
): (queryResponse<array<ReorgDetection.blockDataWithTimestamp>>, array<Source.requestStat>) => {
|
|
282
|
+
let requestStats = []
|
|
283
|
+
let result = switch blockNumbers->Array.get(0) {
|
|
263
284
|
| None => Ok([])
|
|
264
285
|
| Some(firstBlock) => {
|
|
265
286
|
let fromBlock = ref(firstBlock)
|
|
@@ -287,6 +308,7 @@ module BlockData = {
|
|
|
287
308
|
~sourceName,
|
|
288
309
|
~chainId,
|
|
289
310
|
~logger,
|
|
311
|
+
~requestStats,
|
|
290
312
|
)
|
|
291
313
|
let filtered = res->Result.map(datas => {
|
|
292
314
|
datas->Array.filter(data => set->Utils.Set.delete(data.blockNumber))
|
|
@@ -301,10 +323,12 @@ module BlockData = {
|
|
|
301
323
|
filtered
|
|
302
324
|
}
|
|
303
325
|
}
|
|
326
|
+
(result, requestStats)
|
|
304
327
|
}
|
|
305
328
|
}
|
|
306
329
|
|
|
307
|
-
let queryBlockData = (~client, ~blockNumber, ~sourceName, ~chainId, ~logger) =>
|
|
330
|
+
let queryBlockData = (~client, ~blockNumber, ~sourceName, ~chainId, ~logger) => {
|
|
331
|
+
let requestStats = []
|
|
308
332
|
BlockData.queryBlockData(
|
|
309
333
|
~client,
|
|
310
334
|
~fromBlock=blockNumber,
|
|
@@ -312,5 +336,7 @@ let queryBlockData = (~client, ~blockNumber, ~sourceName, ~chainId, ~logger) =>
|
|
|
312
336
|
~sourceName,
|
|
313
337
|
~chainId,
|
|
314
338
|
~logger,
|
|
315
|
-
|
|
339
|
+
~requestStats,
|
|
340
|
+
)->Promise.thenResolve(res => (res->Result.map(res => res->Array.get(0)), requestStats))
|
|
341
|
+
}
|
|
316
342
|
let queryBlockDataMulti = BlockData.queryBlockDataMulti
|
|
@@ -4,8 +4,8 @@ import * as Time from "../Time.res.mjs";
|
|
|
4
4
|
import * as Utils from "../Utils.res.mjs";
|
|
5
5
|
import * as Source from "./Source.res.mjs";
|
|
6
6
|
import * as Logging from "../Logging.res.mjs";
|
|
7
|
-
import * as Prometheus from "../Prometheus.res.mjs";
|
|
8
7
|
import * as Stdlib_Int from "@rescript/runtime/lib/es6/Stdlib_Int.js";
|
|
8
|
+
import * as Performance from "../bindings/Performance.res.mjs";
|
|
9
9
|
import * as Stdlib_JSON from "@rescript/runtime/lib/es6/Stdlib_JSON.js";
|
|
10
10
|
import * as Stdlib_Array from "@rescript/runtime/lib/es6/Stdlib_Array.js";
|
|
11
11
|
import * as Stdlib_JsExn from "@rescript/runtime/lib/es6/Stdlib_JsExn.js";
|
|
@@ -62,12 +62,13 @@ function mapExn(queryResponse) {
|
|
|
62
62
|
|
|
63
63
|
let $$Error = /* @__PURE__ */Primitive_exceptions.create("HyperSync.GetLogs.Error");
|
|
64
64
|
|
|
65
|
-
function makeRequestBody(fromBlock, toBlockInclusive, addressesWithTopics, fieldSelection) {
|
|
65
|
+
function makeRequestBody(fromBlock, toBlockInclusive, addressesWithTopics, fieldSelection, maxNumLogs) {
|
|
66
66
|
return {
|
|
67
67
|
fromBlock: fromBlock,
|
|
68
68
|
toBlock: toBlockInclusive !== undefined ? toBlockInclusive + 1 | 0 : undefined,
|
|
69
69
|
logs: addressesWithTopics,
|
|
70
|
-
fieldSelection: fieldSelection
|
|
70
|
+
fieldSelection: fieldSelection,
|
|
71
|
+
maxNumLogs: maxNumLogs
|
|
71
72
|
};
|
|
72
73
|
}
|
|
73
74
|
|
|
@@ -92,7 +93,7 @@ function extractMissingParams(exn) {
|
|
|
92
93
|
}
|
|
93
94
|
}
|
|
94
95
|
|
|
95
|
-
async function query(client, fromBlock, toBlock, logSelections, fieldSelection) {
|
|
96
|
+
async function query(client, fromBlock, toBlock, logSelections, fieldSelection, maxNumLogs) {
|
|
96
97
|
let addressesWithTopics = logSelections.flatMap(param => {
|
|
97
98
|
let addresses = param.addresses;
|
|
98
99
|
return param.topicSelections.map(param => {
|
|
@@ -100,7 +101,7 @@ async function query(client, fromBlock, toBlock, logSelections, fieldSelection)
|
|
|
100
101
|
return HyperSyncClient.QueryTypes.makeLogSelection(addresses, topics);
|
|
101
102
|
});
|
|
102
103
|
});
|
|
103
|
-
let query$1 = makeRequestBody(fromBlock, toBlock, addressesWithTopics, fieldSelection);
|
|
104
|
+
let query$1 = makeRequestBody(fromBlock, toBlock, addressesWithTopics, fieldSelection, maxNumLogs);
|
|
104
105
|
let match;
|
|
105
106
|
try {
|
|
106
107
|
match = await client.getEventItems(query$1);
|
|
@@ -134,7 +135,8 @@ async function query(client, fromBlock, toBlock, logSelections, fieldSelection)
|
|
|
134
135
|
nextBlock: res.nextBlock,
|
|
135
136
|
archiveHeight: Stdlib_Option.getOr(res.archiveHeight, 0),
|
|
136
137
|
rollbackGuard: res.rollbackGuard,
|
|
137
|
-
transactionStore: match[1]
|
|
138
|
+
transactionStore: match[1],
|
|
139
|
+
blockStore: match[2]
|
|
138
140
|
};
|
|
139
141
|
}
|
|
140
142
|
|
|
@@ -190,14 +192,14 @@ function convertResponse(res) {
|
|
|
190
192
|
}));
|
|
191
193
|
}
|
|
192
194
|
|
|
193
|
-
async function queryBlockData(client, fromBlock, toBlock, sourceName, chainId, logger) {
|
|
195
|
+
async function queryBlockData(client, fromBlock, toBlock, sourceName, chainId, logger, requestStats) {
|
|
194
196
|
let body = makeRequestBody$1(fromBlock, toBlock);
|
|
195
197
|
let logger$1 = Logging.createChildFrom(logger, {
|
|
196
198
|
logType: "HyperSync get block hash query",
|
|
197
199
|
fromBlock: fromBlock,
|
|
198
200
|
toBlock: toBlock
|
|
199
201
|
});
|
|
200
|
-
|
|
202
|
+
let timerRef = Performance.now();
|
|
201
203
|
let maybeSuccessfulRes;
|
|
202
204
|
let exit = 0;
|
|
203
205
|
let res;
|
|
@@ -212,6 +214,10 @@ async function queryBlockData(client, fromBlock, toBlock, sourceName, chainId, l
|
|
|
212
214
|
if (exit === 1) {
|
|
213
215
|
maybeSuccessfulRes = res.nextBlock <= fromBlock ? undefined : res;
|
|
214
216
|
}
|
|
217
|
+
requestStats.push({
|
|
218
|
+
method: "getBlockHashes",
|
|
219
|
+
seconds: Performance.secondsSince(timerRef)
|
|
220
|
+
});
|
|
215
221
|
if (maybeSuccessfulRes !== undefined) {
|
|
216
222
|
let err = convertResponse(maybeSuccessfulRes);
|
|
217
223
|
if (err.TAG !== "Ok") {
|
|
@@ -221,48 +227,59 @@ async function queryBlockData(client, fromBlock, toBlock, sourceName, chainId, l
|
|
|
221
227
|
return err;
|
|
222
228
|
}
|
|
223
229
|
let datas = err._0;
|
|
224
|
-
let restRes = await queryBlockData(client, maybeSuccessfulRes.nextBlock, toBlock, sourceName, chainId, logger$1);
|
|
230
|
+
let restRes = await queryBlockData(client, maybeSuccessfulRes.nextBlock, toBlock, sourceName, chainId, logger$1, requestStats);
|
|
225
231
|
return Stdlib_Result.map(restRes, rest => datas.concat(rest));
|
|
226
232
|
}
|
|
227
233
|
Logging.childInfo(logger$1, `Block #` + fromBlock.toString() + ` not found in HyperSync. HyperSync has multiple instances and it's possible that they drift independently slightly from the head. Indexing should continue correctly after retrying the query in ` + (100).toString() + `ms.`);
|
|
228
234
|
await Time.resolvePromiseAfterDelay(100);
|
|
229
|
-
return await queryBlockData(client, fromBlock, toBlock, sourceName, chainId, logger$1);
|
|
235
|
+
return await queryBlockData(client, fromBlock, toBlock, sourceName, chainId, logger$1, requestStats);
|
|
230
236
|
}
|
|
231
237
|
|
|
232
238
|
async function queryBlockDataMulti(client, blockNumbers, sourceName, chainId, logger) {
|
|
239
|
+
let requestStats = [];
|
|
233
240
|
let firstBlock = blockNumbers[0];
|
|
234
|
-
|
|
235
|
-
|
|
241
|
+
let result;
|
|
242
|
+
if (firstBlock !== undefined) {
|
|
243
|
+
let fromBlock = firstBlock;
|
|
244
|
+
let toBlock = firstBlock;
|
|
245
|
+
let set = new Set();
|
|
246
|
+
for (let idx = 0, idx_finish = blockNumbers.length; idx < idx_finish; ++idx) {
|
|
247
|
+
let blockNumber = blockNumbers[idx];
|
|
248
|
+
if (blockNumber < fromBlock) {
|
|
249
|
+
fromBlock = blockNumber;
|
|
250
|
+
}
|
|
251
|
+
if (blockNumber > toBlock) {
|
|
252
|
+
toBlock = blockNumber;
|
|
253
|
+
}
|
|
254
|
+
set.add(blockNumber);
|
|
255
|
+
}
|
|
256
|
+
if ((toBlock - fromBlock | 0) > 1000) {
|
|
257
|
+
Stdlib_JsError.throwWithMessage(`Invalid block data request. Range of block numbers is too large. Max range is 1000. Requested range: ` + fromBlock.toString() + `-` + toBlock.toString());
|
|
258
|
+
}
|
|
259
|
+
let res = await queryBlockData(client, fromBlock, toBlock, sourceName, chainId, logger, requestStats);
|
|
260
|
+
let filtered = Stdlib_Result.map(res, datas => datas.filter(data => set.delete(data.blockNumber)));
|
|
261
|
+
if (set.size > 0) {
|
|
262
|
+
Stdlib_JsError.throwWithMessage(`Invalid response. Failed to get block data for block numbers: ` + Array.from(set).join(", "));
|
|
263
|
+
}
|
|
264
|
+
result = filtered;
|
|
265
|
+
} else {
|
|
266
|
+
result = {
|
|
236
267
|
TAG: "Ok",
|
|
237
268
|
_0: []
|
|
238
269
|
};
|
|
239
270
|
}
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
let blockNumber = blockNumbers[idx];
|
|
245
|
-
if (blockNumber < fromBlock) {
|
|
246
|
-
fromBlock = blockNumber;
|
|
247
|
-
}
|
|
248
|
-
if (blockNumber > toBlock) {
|
|
249
|
-
toBlock = blockNumber;
|
|
250
|
-
}
|
|
251
|
-
set.add(blockNumber);
|
|
252
|
-
}
|
|
253
|
-
if ((toBlock - fromBlock | 0) > 1000) {
|
|
254
|
-
Stdlib_JsError.throwWithMessage(`Invalid block data request. Range of block numbers is too large. Max range is 1000. Requested range: ` + fromBlock.toString() + `-` + toBlock.toString());
|
|
255
|
-
}
|
|
256
|
-
let res = await queryBlockData(client, fromBlock, toBlock, sourceName, chainId, logger);
|
|
257
|
-
let filtered = Stdlib_Result.map(res, datas => datas.filter(data => set.delete(data.blockNumber)));
|
|
258
|
-
if (set.size > 0) {
|
|
259
|
-
Stdlib_JsError.throwWithMessage(`Invalid response. Failed to get block data for block numbers: ` + Array.from(set).join(", "));
|
|
260
|
-
}
|
|
261
|
-
return filtered;
|
|
271
|
+
return [
|
|
272
|
+
result,
|
|
273
|
+
requestStats
|
|
274
|
+
];
|
|
262
275
|
}
|
|
263
276
|
|
|
264
277
|
function queryBlockData$1(client, blockNumber, sourceName, chainId, logger) {
|
|
265
|
-
|
|
278
|
+
let requestStats = [];
|
|
279
|
+
return queryBlockData(client, blockNumber, blockNumber, sourceName, chainId, logger, requestStats).then(res => [
|
|
280
|
+
Stdlib_Result.map(res, res => res[0]),
|
|
281
|
+
requestStats
|
|
282
|
+
]);
|
|
266
283
|
}
|
|
267
284
|
|
|
268
285
|
let GetLogs = {
|
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
type logsQueryPage = {
|
|
2
2
|
items: array<HyperSyncClient.EventItems.item>,
|
|
3
|
-
blocks: array<HyperSyncClient.
|
|
3
|
+
blocks: array<HyperSyncClient.EventItems.blockHeader>,
|
|
4
4
|
nextBlock: int,
|
|
5
5
|
archiveHeight: int,
|
|
6
6
|
rollbackGuard: option<HyperSyncClient.ResponseTypes.rollbackGuard>,
|
|
7
7
|
transactionStore: TransactionStore.t,
|
|
8
|
+
blockStore: BlockStore.t,
|
|
8
9
|
}
|
|
9
10
|
|
|
10
11
|
type missingParams = {
|
|
@@ -33,6 +34,7 @@ module GetLogs: {
|
|
|
33
34
|
~toBlock: option<int>,
|
|
34
35
|
~logSelections: array<LogSelection.t>,
|
|
35
36
|
~fieldSelection: HyperSyncClient.QueryTypes.fieldSelection,
|
|
37
|
+
~maxNumLogs: int,
|
|
36
38
|
) => promise<logsQueryPage>
|
|
37
39
|
}
|
|
38
40
|
|
|
@@ -42,7 +44,10 @@ let queryBlockData: (
|
|
|
42
44
|
~sourceName: string,
|
|
43
45
|
~chainId: int,
|
|
44
46
|
~logger: Pino.t,
|
|
45
|
-
) => promise<
|
|
47
|
+
) => promise<(
|
|
48
|
+
queryResponse<option<ReorgDetection.blockDataWithTimestamp>>,
|
|
49
|
+
array<Source.requestStat>,
|
|
50
|
+
)>
|
|
46
51
|
|
|
47
52
|
let queryBlockDataMulti: (
|
|
48
53
|
~client: HyperSyncClient.t,
|
|
@@ -50,6 +55,9 @@ let queryBlockDataMulti: (
|
|
|
50
55
|
~sourceName: string,
|
|
51
56
|
~chainId: int,
|
|
52
57
|
~logger: Pino.t,
|
|
53
|
-
) => promise<
|
|
58
|
+
) => promise<(
|
|
59
|
+
queryResponse<array<ReorgDetection.blockDataWithTimestamp>>,
|
|
60
|
+
array<Source.requestStat>,
|
|
61
|
+
)>
|
|
54
62
|
|
|
55
63
|
let mapExn: queryResponse<'a> => result<'a, exn>
|
|
@@ -254,11 +254,19 @@ module EventItems = {
|
|
|
254
254
|
params: Nullable.t<dict<Internal.eventParams>>,
|
|
255
255
|
}
|
|
256
256
|
|
|
257
|
+
// The always-needed block fields, one per block number. The block's remaining
|
|
258
|
+
// fields live raw in the block store and are materialised on demand.
|
|
259
|
+
type blockHeader = {
|
|
260
|
+
number: int,
|
|
261
|
+
timestamp: int,
|
|
262
|
+
hash: string,
|
|
263
|
+
}
|
|
264
|
+
|
|
257
265
|
type response = {
|
|
258
266
|
archiveHeight: option<int>,
|
|
259
267
|
nextBlock: int,
|
|
260
|
-
// One
|
|
261
|
-
blocks: array<
|
|
268
|
+
// One header per block number referenced by `items`.
|
|
269
|
+
blocks: array<blockHeader>,
|
|
262
270
|
items: array<item>,
|
|
263
271
|
rollbackGuard: option<ResponseTypes.rollbackGuard>,
|
|
264
272
|
}
|
|
@@ -266,8 +274,11 @@ module EventItems = {
|
|
|
266
274
|
|
|
267
275
|
type t = {
|
|
268
276
|
get: (~query: query) => promise<queryResponse>,
|
|
269
|
-
// Returns the response plus
|
|
270
|
-
|
|
277
|
+
// Returns the response plus page stores owning this page's raw transactions
|
|
278
|
+
// and blocks.
|
|
279
|
+
getEventItems: (
|
|
280
|
+
~query: query,
|
|
281
|
+
) => promise<(EventItems.response, TransactionStore.t, BlockStore.t)>,
|
|
271
282
|
getHeight: unit => promise<int>,
|
|
272
283
|
}
|
|
273
284
|
|
|
@@ -35,8 +35,7 @@ let subscribe = (~hyperSyncUrl, ~apiToken, ~chainId, ~onHeight: int => unit): (u
|
|
|
35
35
|
and scheduleReconnect = () => {
|
|
36
36
|
// Clear any pending stale/reconnect timeout to avoid double reconnect
|
|
37
37
|
timeoutIdRef.contents->clearTimeout
|
|
38
|
-
let delay =
|
|
39
|
-
baseDuration * Math.pow(2.0, ~exp=errorCount.contents->Int.toFloat)->Float.toInt
|
|
38
|
+
let delay = baseDuration * Math.pow(2.0, ~exp=errorCount.contents->Int.toFloat)->Float.toInt
|
|
40
39
|
timeoutIdRef := setTimeout(() => refreshEventSource(), Pervasives.min(delay, 60_000))
|
|
41
40
|
}
|
|
42
41
|
and refreshEventSource = () => {
|
|
@@ -102,12 +101,6 @@ let subscribe = (~hyperSyncUrl, ~apiToken, ~chainId, ~onHeight: int => unit): (u
|
|
|
102
101
|
es->EventSource.addEventListener("height", event => {
|
|
103
102
|
switch event.data->Int.fromString {
|
|
104
103
|
| Some(height) =>
|
|
105
|
-
// Track the SSE height event
|
|
106
|
-
Prometheus.SourceRequestCount.increment(
|
|
107
|
-
~sourceName="HyperSync",
|
|
108
|
-
~chainId,
|
|
109
|
-
~method="heightStream",
|
|
110
|
-
)
|
|
111
104
|
// On a successful height event, update the timeout
|
|
112
105
|
updateTimeoutId()
|
|
113
106
|
// Call the callback with the new height
|
|
@@ -2,7 +2,6 @@
|
|
|
2
2
|
|
|
3
3
|
import * as Utils from "../Utils.res.mjs";
|
|
4
4
|
import * as Logging from "../Logging.res.mjs";
|
|
5
|
-
import * as Prometheus from "../Prometheus.res.mjs";
|
|
6
5
|
import * as Stdlib_Int from "@rescript/runtime/lib/es6/Stdlib_Int.js";
|
|
7
6
|
import * as Eventsource from "eventsource";
|
|
8
7
|
import * as Primitive_int from "@rescript/runtime/lib/es6/Primitive_int.js";
|
|
@@ -83,7 +82,6 @@ function subscribe(hyperSyncUrl, apiToken, chainId, onHeight) {
|
|
|
83
82
|
es$1.addEventListener("height", event => {
|
|
84
83
|
let height = Stdlib_Int.fromString(event.data, undefined);
|
|
85
84
|
if (height !== undefined) {
|
|
86
|
-
Prometheus.SourceRequestCount.increment("HyperSync", chainId, "heightStream");
|
|
87
85
|
updateTimeoutId();
|
|
88
86
|
return onHeight(height);
|
|
89
87
|
} else {
|