envio 3.2.1 → 3.3.0-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +6 -7
- package/src/Batch.res +12 -15
- package/src/Batch.res.mjs +4 -5
- package/src/BatchProcessing.res +199 -0
- package/src/BatchProcessing.res.mjs +125 -0
- package/src/ChainFetching.res +373 -0
- package/src/ChainFetching.res.mjs +206 -0
- package/src/ChainMetadata.res +27 -0
- package/src/ChainMetadata.res.mjs +27 -0
- package/src/ChainState.res +643 -0
- package/src/ChainState.res.mjs +476 -0
- package/src/ChainState.resi +87 -0
- package/src/Config.res +13 -4
- package/src/Config.res.mjs +8 -3
- package/src/ContractRegisterContext.res +106 -0
- package/src/ContractRegisterContext.res.mjs +76 -0
- package/src/Core.res +3 -0
- package/src/CrossChainState.res +294 -0
- package/src/CrossChainState.res.mjs +221 -0
- package/src/CrossChainState.resi +39 -0
- package/src/Ecosystem.res +58 -0
- package/src/Ecosystem.res.mjs +43 -1
- package/src/Env.res +0 -2
- package/src/Env.res.mjs +0 -3
- package/src/EventConfigBuilder.res +2 -2
- package/src/EventProcessing.res +60 -46
- package/src/EventProcessing.res.mjs +33 -32
- package/src/EventUtils.res +0 -4
- package/src/EventUtils.res.mjs +0 -4
- package/src/ExitOnCaughtUp.res +10 -0
- package/src/ExitOnCaughtUp.res.mjs +22 -0
- package/src/FetchState.res +205 -73
- package/src/FetchState.res.mjs +242 -103
- package/src/InMemoryStore.res +36 -451
- package/src/InMemoryStore.res.mjs +20 -369
- package/src/IndexerLoop.res +43 -0
- package/src/IndexerLoop.res.mjs +46 -0
- package/src/IndexerState.res +536 -0
- package/src/IndexerState.res.mjs +543 -0
- package/src/IndexerState.resi +128 -0
- package/src/Internal.res +29 -8
- package/src/LoadLayer.res +20 -18
- package/src/LoadLayer.res.mjs +14 -12
- package/src/LoadLayer.resi +6 -3
- package/src/Logging.res +11 -44
- package/src/Logging.res.mjs +10 -42
- package/src/Main.res +54 -79
- package/src/Main.res.mjs +44 -56
- package/src/PgStorage.res +8 -71
- package/src/PgStorage.res.mjs +3 -47
- package/src/Prometheus.res +1 -13
- package/src/Prometheus.res.mjs +84 -96
- package/src/PruneStaleHistory.res +45 -0
- package/src/PruneStaleHistory.res.mjs +46 -0
- package/src/RawEvent.res +73 -0
- package/src/RawEvent.res.mjs +51 -0
- package/src/Rollback.res +204 -0
- package/src/Rollback.res.mjs +118 -0
- package/src/SimulateItems.res +12 -10
- package/src/SimulateItems.res.mjs +1 -1
- package/src/UserContext.res +26 -114
- package/src/UserContext.res.mjs +15 -78
- package/src/Writing.res +242 -0
- package/src/Writing.res.mjs +215 -0
- package/src/db/InternalTable.res +14 -27
- package/src/sources/Evm.res +40 -1
- package/src/sources/Evm.res.mjs +94 -84
- package/src/sources/Fuel.res +40 -1
- package/src/sources/Fuel.res.mjs +36 -26
- package/src/sources/HyperFuel.res +41 -120
- package/src/sources/HyperFuel.res.mjs +42 -80
- package/src/sources/HyperFuel.resi +1 -24
- package/src/sources/HyperFuelClient.res +16 -297
- package/src/sources/HyperFuelClient.res.mjs +5 -4
- package/src/sources/HyperFuelSource.res +33 -8
- package/src/sources/HyperFuelSource.res.mjs +56 -10
- package/src/sources/HyperSyncSource.res +5 -3
- package/src/sources/HyperSyncSource.res.mjs +1 -1
- package/src/sources/RpcSource.res +2 -2
- package/src/sources/RpcSource.res.mjs +1 -1
- package/src/sources/SourceManager.res +14 -23
- package/src/sources/SourceManager.res.mjs +27 -29
- package/src/sources/SourceManager.resi +4 -2
- package/src/sources/Svm.res +18 -1
- package/src/sources/Svm.res.mjs +30 -22
- package/src/sources/SvmHyperSyncSource.res +1 -1
- package/src/sources/SvmHyperSyncSource.res.mjs +1 -1
- package/src/tui/Tui.res +38 -36
- package/src/tui/Tui.res.mjs +51 -51
- package/src/ChainFetcher.res +0 -519
- package/src/ChainFetcher.res.mjs +0 -314
- package/src/ChainManager.res +0 -358
- package/src/ChainManager.res.mjs +0 -291
- package/src/Ctx.res +0 -6
- package/src/Ctx.res.mjs +0 -2
- package/src/GlobalState.res +0 -1056
- package/src/GlobalState.res.mjs +0 -1086
- package/src/GlobalStateManager.res +0 -57
- package/src/GlobalStateManager.res.mjs +0 -68
- package/src/GlobalStateManager.resi +0 -7
- package/src/Ports.res +0 -5
- package/src/Ports.res.mjs +0 -9
- package/src/adapters/MarkBatchProcessedAdapter.res +0 -5
- package/src/adapters/MarkBatchProcessedAdapter.res.mjs +0 -14
- package/src/sources/EnvioApiClient.res +0 -15
- package/src/sources/EnvioApiClient.res.mjs +0 -24
|
@@ -2,6 +2,8 @@ open Source
|
|
|
2
2
|
|
|
3
3
|
exception EventRoutingFailed
|
|
4
4
|
|
|
5
|
+
let isUnauthorizedError = (message: string) => message->String.includes("401 Unauthorized")
|
|
6
|
+
|
|
5
7
|
let mintEventTag = "mint"
|
|
6
8
|
let burnEventTag = "burn"
|
|
7
9
|
let transferEventTag = "transfer"
|
|
@@ -207,11 +209,26 @@ let memoGetSelectionConfig = (~chain) => {
|
|
|
207
209
|
type options = {
|
|
208
210
|
chain: ChainMap.Chain.t,
|
|
209
211
|
endpointUrl: string,
|
|
212
|
+
apiToken: option<string>,
|
|
210
213
|
}
|
|
211
214
|
|
|
212
|
-
let make = ({chain, endpointUrl}: options): t => {
|
|
215
|
+
let make = ({chain, endpointUrl, apiToken}: options): t => {
|
|
213
216
|
let name = "HyperFuel"
|
|
214
217
|
|
|
218
|
+
let apiToken = switch apiToken {
|
|
219
|
+
| Some(token) => token
|
|
220
|
+
| None =>
|
|
221
|
+
JsError.throwWithMessage(`An Envio API token is required for using HyperFuel as a data-source.
|
|
222
|
+
Set the ENVIO_API_TOKEN environment variable in your .env file.
|
|
223
|
+
Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`)
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
let client = switch HyperFuelClient.make({url: endpointUrl, apiToken}) {
|
|
227
|
+
| client => client
|
|
228
|
+
| exception exn =>
|
|
229
|
+
exn->ErrorHandling.mkLogAndRaise(~msg="Failed to instantiate the HyperFuel client")
|
|
230
|
+
}
|
|
231
|
+
|
|
215
232
|
let getSelectionConfig = memoGetSelectionConfig(~chain)
|
|
216
233
|
|
|
217
234
|
let getItemsOrThrow = async (
|
|
@@ -239,12 +256,12 @@ let make = ({chain, endpointUrl}: options): t => {
|
|
|
239
256
|
~method="getLogs",
|
|
240
257
|
)
|
|
241
258
|
let pageUnsafe = try await HyperFuel.GetLogs.query(
|
|
242
|
-
~
|
|
259
|
+
~client,
|
|
243
260
|
~fromBlock,
|
|
244
261
|
~toBlock,
|
|
245
262
|
~recieptsSelection,
|
|
246
263
|
) catch {
|
|
247
|
-
|
|
|
264
|
+
| HyperFuel.GetLogs.Error(error) =>
|
|
248
265
|
throw(
|
|
249
266
|
Source.GetItemsError(
|
|
250
267
|
Source.FailedGettingItems({
|
|
@@ -397,7 +414,7 @@ let make = ({chain, endpointUrl}: options): t => {
|
|
|
397
414
|
blockNumber: block.height,
|
|
398
415
|
blockHash: block.id,
|
|
399
416
|
logIndex: receiptIndex,
|
|
400
|
-
|
|
417
|
+
payload: {
|
|
401
418
|
contractName: eventConfig.contractName,
|
|
402
419
|
eventName: eventConfig.name,
|
|
403
420
|
chainId,
|
|
@@ -408,7 +425,7 @@ let make = ({chain, endpointUrl}: options): t => {
|
|
|
408
425
|
block: block->Obj.magic,
|
|
409
426
|
srcAddress: contractAddress,
|
|
410
427
|
logIndex: receiptIndex,
|
|
411
|
-
}->
|
|
428
|
+
}->Fuel.fromPayload,
|
|
412
429
|
})
|
|
413
430
|
})
|
|
414
431
|
|
|
@@ -450,8 +467,6 @@ let make = ({chain, endpointUrl}: options): t => {
|
|
|
450
467
|
let getBlockHashes = (~blockNumbers as _, ~logger as _) =>
|
|
451
468
|
JsError.throwWithMessage("HyperFuel does not support getting block hashes")
|
|
452
469
|
|
|
453
|
-
let jsonApiClient = EnvioApiClient.make(endpointUrl)
|
|
454
|
-
|
|
455
470
|
{
|
|
456
471
|
name,
|
|
457
472
|
sourceFor: Sync,
|
|
@@ -461,7 +476,17 @@ let make = ({chain, endpointUrl}: options): t => {
|
|
|
461
476
|
poweredByHyperSync: true,
|
|
462
477
|
getHeightOrThrow: async () => {
|
|
463
478
|
let timerRef = Hrtime.makeTimer()
|
|
464
|
-
let height = await
|
|
479
|
+
let height = try await client->HyperFuelClient.getHeight catch {
|
|
480
|
+
| JsExn(e) =>
|
|
481
|
+
switch e->JsExn.message {
|
|
482
|
+
| Some(message) if message->isUnauthorizedError =>
|
|
483
|
+
Logging.error(`Your ENVIO_API_TOKEN was rejected by HyperFuel (401 Unauthorized). The indexer will not be able to fetch events. Update the token and try again using 'envio start' or 'envio dev'. For more info: https://docs.envio.dev/docs/HyperSync/api-tokens`)
|
|
484
|
+
// Retrying an unauthorized request can never succeed, so block forever
|
|
485
|
+
let _ = await Promise.make((_, _) => ())
|
|
486
|
+
0
|
|
487
|
+
| _ => throw(JsExn(e))
|
|
488
|
+
}
|
|
489
|
+
}
|
|
465
490
|
let seconds = timerRef->Hrtime.timeSince->Hrtime.toSecondsFloat
|
|
466
491
|
Prometheus.SourceRequestCount.increment(
|
|
467
492
|
~sourceName=name,
|
|
@@ -1,21 +1,24 @@
|
|
|
1
1
|
// Generated by ReScript, PLEASE EDIT WITH CARE
|
|
2
2
|
|
|
3
|
-
import * as Rest from "../vendored/Rest.res.mjs";
|
|
4
3
|
import * as Hrtime from "../bindings/Hrtime.res.mjs";
|
|
5
4
|
import * as Source from "./Source.res.mjs";
|
|
6
5
|
import * as Logging from "../Logging.res.mjs";
|
|
7
6
|
import * as HyperFuel from "./HyperFuel.res.mjs";
|
|
8
|
-
import * as HyperSync from "./HyperSync.res.mjs";
|
|
9
7
|
import * as Prometheus from "../Prometheus.res.mjs";
|
|
10
8
|
import * as EventRouter from "./EventRouter.res.mjs";
|
|
9
|
+
import * as Stdlib_JsExn from "@rescript/runtime/lib/es6/Stdlib_JsExn.js";
|
|
11
10
|
import * as ErrorHandling from "../ErrorHandling.res.mjs";
|
|
12
11
|
import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
|
|
13
|
-
import * as EnvioApiClient from "./EnvioApiClient.res.mjs";
|
|
14
12
|
import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
|
|
13
|
+
import * as HyperFuelClient from "./HyperFuelClient.res.mjs";
|
|
15
14
|
import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
|
|
16
15
|
|
|
17
16
|
let EventRoutingFailed = /* @__PURE__ */Primitive_exceptions.create("HyperFuelSource.EventRoutingFailed");
|
|
18
17
|
|
|
18
|
+
function isUnauthorizedError(message) {
|
|
19
|
+
return message.includes("401 Unauthorized");
|
|
20
|
+
}
|
|
21
|
+
|
|
19
22
|
let mintEventTag = "mint";
|
|
20
23
|
|
|
21
24
|
let burnEventTag = "burn";
|
|
@@ -176,9 +179,22 @@ function memoGetSelectionConfig(chain) {
|
|
|
176
179
|
}
|
|
177
180
|
|
|
178
181
|
function make(param) {
|
|
179
|
-
let
|
|
182
|
+
let apiToken = param.apiToken;
|
|
180
183
|
let chain = param.chain;
|
|
181
184
|
let name = "HyperFuel";
|
|
185
|
+
let apiToken$1 = apiToken !== undefined ? apiToken : Stdlib_JsError.throwWithMessage(`An Envio API token is required for using HyperFuel as a data-source.
|
|
186
|
+
Set the ENVIO_API_TOKEN environment variable in your .env file.
|
|
187
|
+
Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`);
|
|
188
|
+
let client;
|
|
189
|
+
try {
|
|
190
|
+
client = HyperFuelClient.make({
|
|
191
|
+
url: param.endpointUrl,
|
|
192
|
+
apiToken: apiToken$1
|
|
193
|
+
});
|
|
194
|
+
} catch (raw_exn) {
|
|
195
|
+
let exn = Primitive_exceptions.internalToException(raw_exn);
|
|
196
|
+
client = ErrorHandling.mkLogAndRaise(undefined, "Failed to instantiate the HyperFuel client", exn);
|
|
197
|
+
}
|
|
182
198
|
let getSelectionConfig = memoGetSelectionConfig(chain);
|
|
183
199
|
let getItemsOrThrow = async (fromBlock, toBlock, addressesByContractName, indexingAddresses, knownHeight, param, selection, retry, logger) => {
|
|
184
200
|
let totalTimeRef = Hrtime.makeTimer();
|
|
@@ -188,10 +204,10 @@ function make(param) {
|
|
|
188
204
|
Prometheus.SourceRequestCount.increment(name, chain, "getLogs");
|
|
189
205
|
let pageUnsafe;
|
|
190
206
|
try {
|
|
191
|
-
pageUnsafe = await HyperFuel.GetLogs.query(
|
|
207
|
+
pageUnsafe = await HyperFuel.GetLogs.query(client, fromBlock, toBlock, recieptsSelection);
|
|
192
208
|
} catch (raw_error) {
|
|
193
209
|
let error = Primitive_exceptions.internalToException(raw_error);
|
|
194
|
-
if (error.RE_EXN_ID ===
|
|
210
|
+
if (error.RE_EXN_ID === HyperFuel.GetLogs.$$Error) {
|
|
195
211
|
let error$1 = error._1;
|
|
196
212
|
let tmp;
|
|
197
213
|
if (typeof error$1 !== "object") {
|
|
@@ -339,7 +355,7 @@ function make(param) {
|
|
|
339
355
|
blockNumber: block.height,
|
|
340
356
|
blockHash: block.id,
|
|
341
357
|
logIndex: receiptIndex,
|
|
342
|
-
|
|
358
|
+
payload: {
|
|
343
359
|
contractName: eventConfig$1.contractName,
|
|
344
360
|
eventName: eventConfig$1.name,
|
|
345
361
|
params: params,
|
|
@@ -388,7 +404,6 @@ function make(param) {
|
|
|
388
404
|
};
|
|
389
405
|
};
|
|
390
406
|
let getBlockHashes = (param, param$1) => Stdlib_JsError.throwWithMessage("HyperFuel does not support getting block hashes");
|
|
391
|
-
let jsonApiClient = EnvioApiClient.make(endpointUrl);
|
|
392
407
|
return {
|
|
393
408
|
name: name,
|
|
394
409
|
sourceFor: "Sync",
|
|
@@ -398,7 +413,37 @@ function make(param) {
|
|
|
398
413
|
getBlockHashes: getBlockHashes,
|
|
399
414
|
getHeightOrThrow: async () => {
|
|
400
415
|
let timerRef = Hrtime.makeTimer();
|
|
401
|
-
let height
|
|
416
|
+
let height;
|
|
417
|
+
try {
|
|
418
|
+
height = await client.getHeight();
|
|
419
|
+
} catch (raw_e) {
|
|
420
|
+
let e = Primitive_exceptions.internalToException(raw_e);
|
|
421
|
+
if (e.RE_EXN_ID === "JsExn") {
|
|
422
|
+
let e$1 = e._1;
|
|
423
|
+
let message = Stdlib_JsExn.message(e$1);
|
|
424
|
+
if (message !== undefined) {
|
|
425
|
+
if (message.includes("401 Unauthorized")) {
|
|
426
|
+
Logging.error(`Your ENVIO_API_TOKEN was rejected by HyperFuel (401 Unauthorized). The indexer will not be able to fetch events. Update the token and try again using 'envio start' or 'envio dev'. For more info: https://docs.envio.dev/docs/HyperSync/api-tokens`);
|
|
427
|
+
await new Promise((param, param$1) => {});
|
|
428
|
+
height = 0;
|
|
429
|
+
} else {
|
|
430
|
+
throw {
|
|
431
|
+
RE_EXN_ID: "JsExn",
|
|
432
|
+
_1: e$1,
|
|
433
|
+
Error: new Error()
|
|
434
|
+
};
|
|
435
|
+
}
|
|
436
|
+
} else {
|
|
437
|
+
throw {
|
|
438
|
+
RE_EXN_ID: "JsExn",
|
|
439
|
+
_1: e$1,
|
|
440
|
+
Error: new Error()
|
|
441
|
+
};
|
|
442
|
+
}
|
|
443
|
+
} else {
|
|
444
|
+
throw e;
|
|
445
|
+
}
|
|
446
|
+
}
|
|
402
447
|
let seconds = Hrtime.toSecondsFloat(Hrtime.timeSince(timerRef));
|
|
403
448
|
Prometheus.SourceRequestCount.increment(name, chain, "getHeight");
|
|
404
449
|
Prometheus.SourceRequestCount.addSeconds(name, chain, "getHeight", seconds);
|
|
@@ -410,6 +455,7 @@ function make(param) {
|
|
|
410
455
|
|
|
411
456
|
export {
|
|
412
457
|
EventRoutingFailed,
|
|
458
|
+
isUnauthorizedError,
|
|
413
459
|
mintEventTag,
|
|
414
460
|
burnEventTag,
|
|
415
461
|
transferEventTag,
|
|
@@ -422,4 +468,4 @@ export {
|
|
|
422
468
|
memoGetSelectionConfig,
|
|
423
469
|
make,
|
|
424
470
|
}
|
|
425
|
-
/*
|
|
471
|
+
/* Logging Not a pure module */
|
|
@@ -206,16 +206,18 @@ Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`)
|
|
|
206
206
|
blockNumber: block.number->Option.getUnsafe,
|
|
207
207
|
blockHash: block.hash->Option.getUnsafe,
|
|
208
208
|
logIndex,
|
|
209
|
-
|
|
209
|
+
payload: {
|
|
210
210
|
contractName: eventConfig.contractName,
|
|
211
211
|
eventName: eventConfig.name,
|
|
212
212
|
chainId,
|
|
213
213
|
params,
|
|
214
|
-
transaction
|
|
214
|
+
transaction: transaction->(
|
|
215
|
+
Utils.magic: HyperSyncClient.ResponseTypes.transaction => Internal.eventTransaction
|
|
216
|
+
),
|
|
215
217
|
block: block->(Utils.magic: HyperSyncClient.ResponseTypes.block => Internal.eventBlock),
|
|
216
218
|
srcAddress,
|
|
217
219
|
logIndex,
|
|
218
|
-
}->
|
|
220
|
+
}->Evm.fromPayload,
|
|
219
221
|
})
|
|
220
222
|
}
|
|
221
223
|
|
|
@@ -135,7 +135,7 @@ Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`);
|
|
|
135
135
|
blockNumber: block.number,
|
|
136
136
|
blockHash: block.hash,
|
|
137
137
|
logIndex: logIndex,
|
|
138
|
-
|
|
138
|
+
payload: {
|
|
139
139
|
contractName: eventConfig.contractName,
|
|
140
140
|
eventName: eventConfig.name,
|
|
141
141
|
params: params,
|
|
@@ -1200,7 +1200,7 @@ let make = (
|
|
|
1200
1200
|
blockHash: block->getBlockHash,
|
|
1201
1201
|
chain,
|
|
1202
1202
|
logIndex: log.logIndex,
|
|
1203
|
-
|
|
1203
|
+
payload: {
|
|
1204
1204
|
contractName: eventConfig.contractName,
|
|
1205
1205
|
eventName: eventConfig.name,
|
|
1206
1206
|
chainId: chain->ChainMap.Chain.toChainId,
|
|
@@ -1209,7 +1209,7 @@ let make = (
|
|
|
1209
1209
|
block,
|
|
1210
1210
|
srcAddress: routedAddress,
|
|
1211
1211
|
logIndex: log.logIndex,
|
|
1212
|
-
}->
|
|
1212
|
+
}->Evm.fromPayload,
|
|
1213
1213
|
})
|
|
1214
1214
|
}
|
|
1215
1215
|
)(),
|
|
@@ -1131,7 +1131,7 @@ function make(param) {
|
|
|
1131
1131
|
blockNumber: block.number,
|
|
1132
1132
|
blockHash: block.hash,
|
|
1133
1133
|
logIndex: log.logIndex,
|
|
1134
|
-
|
|
1134
|
+
payload: {
|
|
1135
1135
|
contractName: eventConfig.contractName,
|
|
1136
1136
|
eventName: eventConfig.name,
|
|
1137
1137
|
params: decoded$1,
|
|
@@ -11,16 +11,12 @@ type sourceState = {
|
|
|
11
11
|
mutable lastFailedAt: option<float>,
|
|
12
12
|
}
|
|
13
13
|
|
|
14
|
-
//
|
|
15
|
-
// But currently the ChainFetcher module is immutable
|
|
16
|
-
// and handles both processing and fetching.
|
|
17
|
-
// So this module is to encapsulate the fetching logic only
|
|
14
|
+
// Encapsulates the fetching logic for a chain's sources.
|
|
18
15
|
// with a mutable state for easier reasoning and testing.
|
|
19
16
|
type t = {
|
|
20
17
|
sourcesState: array<sourceState>,
|
|
21
18
|
mutable statusStart: Hrtime.timeRef,
|
|
22
19
|
mutable status: sourceManagerStatus,
|
|
23
|
-
maxPartitionConcurrency: int,
|
|
24
20
|
newBlockStallTimeout: int,
|
|
25
21
|
newBlockStallTimeoutRealtime: int,
|
|
26
22
|
stalledPollingInterval: int,
|
|
@@ -46,6 +42,10 @@ type t = {
|
|
|
46
42
|
|
|
47
43
|
let getActiveSource = sourceManager => sourceManager.activeSource
|
|
48
44
|
|
|
45
|
+
// Partition queries currently in flight on this chain's sources. Summed across
|
|
46
|
+
// chains by CrossChainState to enforce the indexer-wide concurrency budget.
|
|
47
|
+
let inFlightCount = sourceManager => sourceManager.fetchingPartitionsCount
|
|
48
|
+
|
|
49
49
|
let getRateLimitTimeMs = sourceManager =>
|
|
50
50
|
sourceManager.committedRateLimitTimeMs +.
|
|
51
51
|
switch sourceManager.activeRateLimitStartMs {
|
|
@@ -151,7 +151,6 @@ let makeGetHeightRetryInterval = (
|
|
|
151
151
|
|
|
152
152
|
let make = (
|
|
153
153
|
~sources: array<Source.t>,
|
|
154
|
-
~maxPartitionConcurrency,
|
|
155
154
|
~isRealtime,
|
|
156
155
|
~newBlockStallTimeout=60_000,
|
|
157
156
|
~newBlockStallTimeoutRealtime=20_000,
|
|
@@ -172,16 +171,11 @@ let make = (
|
|
|
172
171
|
| None =>
|
|
173
172
|
JsError.throwWithMessage("Invalid configuration, no data-source for historical sync provided")
|
|
174
173
|
}
|
|
175
|
-
Prometheus.IndexingMaxConcurrency.set(
|
|
176
|
-
~maxConcurrency=maxPartitionConcurrency,
|
|
177
|
-
~chainId=initialActiveSource.chain->ChainMap.Chain.toChainId,
|
|
178
|
-
)
|
|
179
174
|
Prometheus.IndexingConcurrency.set(
|
|
180
175
|
~concurrency=0,
|
|
181
176
|
~chainId=initialActiveSource.chain->ChainMap.Chain.toChainId,
|
|
182
177
|
)
|
|
183
178
|
{
|
|
184
|
-
maxPartitionConcurrency,
|
|
185
179
|
sourcesState: sources->Array.map(source => {
|
|
186
180
|
source,
|
|
187
181
|
knownHeight: 0,
|
|
@@ -223,24 +217,20 @@ let trackNewStatus = (sourceManager: t, ~newStatus) => {
|
|
|
223
217
|
sourceManager.status = newStatus
|
|
224
218
|
}
|
|
225
219
|
|
|
226
|
-
|
|
220
|
+
// Carry out the fetch decision made by CrossChainState.checkAndFetch: either
|
|
221
|
+
// dispatch the admitted queries or start waiting for a new block. Selection
|
|
222
|
+
// (getNextQuery + cross-chain admission) happens upstream so the budget is split
|
|
223
|
+
// per query across all chains.
|
|
224
|
+
let dispatch = async (
|
|
227
225
|
sourceManager: t,
|
|
228
226
|
~fetchState: FetchState.t,
|
|
229
227
|
~executeQuery,
|
|
230
228
|
~waitForNewBlock,
|
|
231
229
|
~onNewBlock,
|
|
230
|
+
~action: FetchState.nextQuery,
|
|
232
231
|
~stateId,
|
|
233
232
|
) => {
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
let nextQuery = fetchState->FetchState.getNextQuery(
|
|
237
|
-
~concurrencyLimit={
|
|
238
|
-
maxPartitionConcurrency - sourceManager.fetchingPartitionsCount
|
|
239
|
-
},
|
|
240
|
-
)
|
|
241
|
-
|
|
242
|
-
switch nextQuery {
|
|
243
|
-
| ReachedMaxConcurrency
|
|
233
|
+
switch action {
|
|
244
234
|
| NothingToQuery => ()
|
|
245
235
|
| WaitingForNewBlock =>
|
|
246
236
|
switch sourceManager.waitingForNewBlockStateId {
|
|
@@ -261,7 +251,8 @@ let fetchNext = async (
|
|
|
261
251
|
}
|
|
262
252
|
}
|
|
263
253
|
| Ready(queries) => {
|
|
264
|
-
|
|
254
|
+
// Queries are already marked in flight by ChainState.startFetchingQueries
|
|
255
|
+
// when they were admitted; here we just execute them.
|
|
265
256
|
sourceManager.fetchingPartitionsCount =
|
|
266
257
|
sourceManager.fetchingPartitionsCount + queries->Array.length
|
|
267
258
|
Prometheus.IndexingConcurrency.set(
|
|
@@ -18,6 +18,10 @@ function getActiveSource(sourceManager) {
|
|
|
18
18
|
return sourceManager.activeSource;
|
|
19
19
|
}
|
|
20
20
|
|
|
21
|
+
function inFlightCount(sourceManager) {
|
|
22
|
+
return sourceManager.fetchingPartitionsCount;
|
|
23
|
+
}
|
|
24
|
+
|
|
21
25
|
function getRateLimitTimeMs(sourceManager) {
|
|
22
26
|
let startMs = sourceManager.activeRateLimitStartMs;
|
|
23
27
|
return sourceManager.committedRateLimitTimeMs + (
|
|
@@ -115,7 +119,7 @@ function makeGetHeightRetryInterval(initialRetryInterval, backoffMultiplicative,
|
|
|
115
119
|
};
|
|
116
120
|
}
|
|
117
121
|
|
|
118
|
-
function make(sources,
|
|
122
|
+
function make(sources, isRealtime, newBlockStallTimeoutOpt, newBlockStallTimeoutRealtimeOpt, stalledPollingIntervalOpt, reducedPollingIntervalOpt, recoveryTimeoutOpt, getHeightRetryIntervalOpt) {
|
|
119
123
|
let newBlockStallTimeout = newBlockStallTimeoutOpt !== undefined ? newBlockStallTimeoutOpt : 60000;
|
|
120
124
|
let newBlockStallTimeoutRealtime = newBlockStallTimeoutRealtimeOpt !== undefined ? newBlockStallTimeoutRealtimeOpt : 20000;
|
|
121
125
|
let stalledPollingInterval = stalledPollingIntervalOpt !== undefined ? stalledPollingIntervalOpt : 5000;
|
|
@@ -125,7 +129,6 @@ function make(sources, maxPartitionConcurrency, isRealtime, newBlockStallTimeout
|
|
|
125
129
|
let hasRealtime = sources.some(s => s.sourceFor === "Realtime");
|
|
126
130
|
let source = sources.find(source => getSourceRole(source.sourceFor, isRealtime, hasRealtime) === "Primary");
|
|
127
131
|
let initialActiveSource = source !== undefined ? source : Stdlib_JsError.throwWithMessage("Invalid configuration, no data-source for historical sync provided");
|
|
128
|
-
Prometheus.IndexingMaxConcurrency.set(maxPartitionConcurrency, initialActiveSource.chain);
|
|
129
132
|
Prometheus.IndexingConcurrency.set(0, initialActiveSource.chain);
|
|
130
133
|
return {
|
|
131
134
|
sourcesState: sources.map(source => ({
|
|
@@ -138,7 +141,6 @@ function make(sources, maxPartitionConcurrency, isRealtime, newBlockStallTimeout
|
|
|
138
141
|
})),
|
|
139
142
|
statusStart: Hrtime.makeTimer(),
|
|
140
143
|
status: "Idle",
|
|
141
|
-
maxPartitionConcurrency: maxPartitionConcurrency,
|
|
142
144
|
newBlockStallTimeout: newBlockStallTimeout,
|
|
143
145
|
newBlockStallTimeoutRealtime: newBlockStallTimeoutRealtime,
|
|
144
146
|
stalledPollingInterval: stalledPollingInterval,
|
|
@@ -175,33 +177,28 @@ function trackNewStatus(sourceManager, newStatus) {
|
|
|
175
177
|
sourceManager.status = newStatus;
|
|
176
178
|
}
|
|
177
179
|
|
|
178
|
-
async function
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
}
|
|
198
|
-
case "ReachedMaxConcurrency" :
|
|
199
|
-
case "NothingToQuery" :
|
|
200
|
-
return;
|
|
180
|
+
async function dispatch(sourceManager, fetchState, executeQuery, waitForNewBlock, onNewBlock, action, stateId) {
|
|
181
|
+
if (typeof action !== "object") {
|
|
182
|
+
if (action !== "WaitingForNewBlock") {
|
|
183
|
+
return;
|
|
184
|
+
}
|
|
185
|
+
let waitingStateId = sourceManager.waitingForNewBlockStateId;
|
|
186
|
+
if (waitingStateId !== undefined && waitingStateId >= stateId) {
|
|
187
|
+
return;
|
|
188
|
+
}
|
|
189
|
+
trackNewStatus(sourceManager, "WaitingForNewBlock");
|
|
190
|
+
sourceManager.waitingForNewBlockStateId = stateId;
|
|
191
|
+
let knownHeight = await waitForNewBlock(fetchState.knownHeight);
|
|
192
|
+
let waitingStateId$1 = sourceManager.waitingForNewBlockStateId;
|
|
193
|
+
if (waitingStateId$1 !== undefined && waitingStateId$1 === stateId) {
|
|
194
|
+
trackNewStatus(sourceManager, "Idle");
|
|
195
|
+
sourceManager.waitingForNewBlockStateId = undefined;
|
|
196
|
+
return onNewBlock(knownHeight);
|
|
197
|
+
} else {
|
|
198
|
+
return;
|
|
201
199
|
}
|
|
202
200
|
} else {
|
|
203
|
-
let queries =
|
|
204
|
-
FetchState.startFetchingQueries(fetchState, queries);
|
|
201
|
+
let queries = action._0;
|
|
205
202
|
sourceManager.fetchingPartitionsCount = sourceManager.fetchingPartitionsCount + queries.length | 0;
|
|
206
203
|
Prometheus.IndexingConcurrency.set(sourceManager.fetchingPartitionsCount, sourceManager.activeSource.chain);
|
|
207
204
|
trackNewStatus(sourceManager, "Querieng");
|
|
@@ -680,8 +677,9 @@ export {
|
|
|
680
677
|
getSourceRole,
|
|
681
678
|
make,
|
|
682
679
|
getActiveSource,
|
|
680
|
+
inFlightCount,
|
|
683
681
|
onReorg,
|
|
684
|
-
|
|
682
|
+
dispatch,
|
|
685
683
|
waitForNewBlock,
|
|
686
684
|
executeQuery,
|
|
687
685
|
makeGetHeightRetryInterval,
|
|
@@ -10,7 +10,6 @@ let getSourceRole: (
|
|
|
10
10
|
|
|
11
11
|
let make: (
|
|
12
12
|
~sources: array<Source.t>,
|
|
13
|
-
~maxPartitionConcurrency: int,
|
|
14
13
|
~isRealtime: bool,
|
|
15
14
|
~newBlockStallTimeout: int=?,
|
|
16
15
|
~newBlockStallTimeoutRealtime: int=?,
|
|
@@ -22,14 +21,17 @@ let make: (
|
|
|
22
21
|
|
|
23
22
|
let getActiveSource: t => Source.t
|
|
24
23
|
|
|
24
|
+
let inFlightCount: t => int
|
|
25
|
+
|
|
25
26
|
let onReorg: (t, ~rollbackTargetBlock: int) => unit
|
|
26
27
|
|
|
27
|
-
let
|
|
28
|
+
let dispatch: (
|
|
28
29
|
t,
|
|
29
30
|
~fetchState: FetchState.t,
|
|
30
31
|
~executeQuery: FetchState.query => promise<unit>,
|
|
31
32
|
~waitForNewBlock: (~knownHeight: int) => promise<int>,
|
|
32
33
|
~onNewBlock: (~knownHeight: int) => unit,
|
|
34
|
+
~action: FetchState.nextQuery,
|
|
33
35
|
~stateId: int,
|
|
34
36
|
) => promise<unit>
|
|
35
37
|
|
package/src/sources/Svm.res
CHANGED
|
@@ -4,7 +4,7 @@ let cleanUpRawEventFieldsInPlace: JSON.t => unit = %raw(`fields => {
|
|
|
4
4
|
delete fields.time
|
|
5
5
|
}`)
|
|
6
6
|
|
|
7
|
-
let
|
|
7
|
+
let make = (~logger: Pino.t): Ecosystem.t => {
|
|
8
8
|
name: Svm,
|
|
9
9
|
blockFields: ["slot"],
|
|
10
10
|
transactionFields: [],
|
|
@@ -19,6 +19,23 @@ let ecosystem: Ecosystem.t = {
|
|
|
19
19
|
// SVM has no event handlers, so there is no `onEvent` `where` value to
|
|
20
20
|
// parse. The schema is a no-op object that always surfaces `None`.
|
|
21
21
|
onEventBlockFilterSchema: S.object(_ => None),
|
|
22
|
+
logger,
|
|
23
|
+
toEvent: eventItem => eventItem.payload->Internal.payloadToEvent,
|
|
24
|
+
toEventLogger: eventItem => {
|
|
25
|
+
let instruction =
|
|
26
|
+
eventItem.payload->(Utils.magic: Internal.eventPayload => Envio.svmInstruction)
|
|
27
|
+
Logging.createChildFrom(
|
|
28
|
+
~logger,
|
|
29
|
+
~params={
|
|
30
|
+
"program": eventItem.eventConfig.contractName,
|
|
31
|
+
"instruction": eventItem.eventConfig.name,
|
|
32
|
+
"chainId": eventItem.chain->ChainMap.Chain.toChainId,
|
|
33
|
+
"slot": eventItem.blockNumber,
|
|
34
|
+
"programId": instruction.programId,
|
|
35
|
+
},
|
|
36
|
+
)
|
|
37
|
+
},
|
|
38
|
+
toRawEvent: _ => JsError.throwWithMessage("Raw events are not supported for SVM"),
|
|
22
39
|
}
|
|
23
40
|
|
|
24
41
|
module GetFinalizedSlot = {
|
package/src/sources/Svm.res.mjs
CHANGED
|
@@ -4,6 +4,7 @@ import * as Rpc from "./Rpc.res.mjs";
|
|
|
4
4
|
import * as Rest from "../vendored/Rest.res.mjs";
|
|
5
5
|
import * as Utils from "../Utils.res.mjs";
|
|
6
6
|
import * as Hrtime from "../bindings/Hrtime.res.mjs";
|
|
7
|
+
import * as Logging from "../Logging.res.mjs";
|
|
7
8
|
import * as Prometheus from "../Prometheus.res.mjs";
|
|
8
9
|
import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
|
|
9
10
|
import * as S$RescriptSchema from "rescript-schema/src/S.res.mjs";
|
|
@@ -14,26 +15,33 @@ let cleanUpRawEventFieldsInPlace = (fields => {
|
|
|
14
15
|
delete fields.time
|
|
15
16
|
});
|
|
16
17
|
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
18
|
+
function make(logger) {
|
|
19
|
+
return {
|
|
20
|
+
name: "svm",
|
|
21
|
+
blockFields: ["slot"],
|
|
22
|
+
transactionFields: [],
|
|
23
|
+
blockNumberName: "height",
|
|
24
|
+
blockTimestampName: "time",
|
|
25
|
+
blockHashName: "hash",
|
|
26
|
+
cleanUpRawEventFieldsInPlace: cleanUpRawEventFieldsInPlace,
|
|
27
|
+
onBlockMethodName: "onSlot",
|
|
28
|
+
onBlockFilterSchema: S$RescriptSchema.object(s => s.f("slot", S$RescriptSchema.option(S$RescriptSchema.unknown))),
|
|
29
|
+
onEventBlockFilterSchema: S$RescriptSchema.object(param => {}),
|
|
30
|
+
logger: logger,
|
|
31
|
+
toEvent: eventItem => eventItem.payload,
|
|
32
|
+
toEventLogger: eventItem => {
|
|
33
|
+
let instruction = eventItem.payload;
|
|
34
|
+
return Logging.createChildFrom(logger, {
|
|
35
|
+
program: eventItem.eventConfig.contractName,
|
|
36
|
+
instruction: eventItem.eventConfig.name,
|
|
37
|
+
chainId: eventItem.chain,
|
|
38
|
+
slot: eventItem.blockNumber,
|
|
39
|
+
programId: instruction.programId
|
|
40
|
+
});
|
|
41
|
+
},
|
|
42
|
+
toRawEvent: param => Stdlib_JsError.throwWithMessage("Raw events are not supported for SVM")
|
|
43
|
+
};
|
|
44
|
+
}
|
|
37
45
|
|
|
38
46
|
let route = Rpc.makeRpcRoute("getSlot", S$RescriptSchema.tuple(s => {
|
|
39
47
|
s.tag(0, {
|
|
@@ -72,8 +80,8 @@ function makeRPCSource(chain, rpc, sourceForOpt) {
|
|
|
72
80
|
|
|
73
81
|
export {
|
|
74
82
|
cleanUpRawEventFieldsInPlace,
|
|
75
|
-
|
|
83
|
+
make,
|
|
76
84
|
GetFinalizedSlot,
|
|
77
85
|
makeRPCSource,
|
|
78
86
|
}
|
|
79
|
-
/*
|
|
87
|
+
/* route Not a pure module */
|
|
@@ -510,7 +510,7 @@ let make = ({chain, endpointUrl, apiToken, eventConfigs, clientTimeoutMillis}: o
|
|
|
510
510
|
blockNumber: instr.slot,
|
|
511
511
|
blockHash: "",
|
|
512
512
|
logIndex: synthLogIndex(instr),
|
|
513
|
-
|
|
513
|
+
payload: payload->(Utils.magic: Envio.svmInstruction => Internal.eventPayload),
|
|
514
514
|
}),
|
|
515
515
|
)
|
|
516
516
|
->ignore
|