envio 3.2.0 → 3.3.0-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +6 -7
- package/src/Batch.res +12 -15
- package/src/Batch.res.mjs +4 -5
- package/src/BatchProcessing.res +199 -0
- package/src/BatchProcessing.res.mjs +125 -0
- package/src/ChainFetching.res +375 -0
- package/src/ChainFetching.res.mjs +206 -0
- package/src/ChainMetadata.res +27 -0
- package/src/ChainMetadata.res.mjs +27 -0
- package/src/ChainState.res +618 -0
- package/src/ChainState.res.mjs +461 -0
- package/src/ChainState.resi +85 -0
- package/src/Config.res +13 -4
- package/src/Config.res.mjs +8 -3
- package/src/ContractRegisterContext.res +106 -0
- package/src/ContractRegisterContext.res.mjs +76 -0
- package/src/Core.res +3 -0
- package/src/CrossChainState.res +269 -0
- package/src/CrossChainState.res.mjs +197 -0
- package/src/CrossChainState.resi +47 -0
- package/src/Ecosystem.res +58 -0
- package/src/Ecosystem.res.mjs +43 -1
- package/src/Env.res +7 -2
- package/src/Env.res.mjs +5 -2
- package/src/EventConfigBuilder.res +35 -0
- package/src/EventConfigBuilder.res.mjs +21 -0
- package/src/EventProcessing.res +60 -46
- package/src/EventProcessing.res.mjs +33 -32
- package/src/EventUtils.res +0 -4
- package/src/EventUtils.res.mjs +0 -4
- package/src/ExitOnCaughtUp.res +10 -0
- package/src/ExitOnCaughtUp.res.mjs +22 -0
- package/src/FetchState.res +162 -113
- package/src/FetchState.res.mjs +181 -119
- package/src/HandlerLoader.res +6 -1
- package/src/HandlerLoader.res.mjs +13 -36
- package/src/InMemoryStore.res +36 -451
- package/src/InMemoryStore.res.mjs +20 -369
- package/src/IndexerLoop.res +44 -0
- package/src/IndexerLoop.res.mjs +46 -0
- package/src/IndexerState.res +544 -0
- package/src/IndexerState.res.mjs +547 -0
- package/src/IndexerState.resi +132 -0
- package/src/Internal.res +44 -7
- package/src/LoadLayer.res +20 -18
- package/src/LoadLayer.res.mjs +14 -12
- package/src/LoadLayer.resi +6 -3
- package/src/LogSelection.res +86 -17
- package/src/LogSelection.res.mjs +67 -14
- package/src/Logging.res +11 -44
- package/src/Logging.res.mjs +10 -42
- package/src/Main.res +54 -79
- package/src/Main.res.mjs +44 -56
- package/src/PgStorage.res +8 -71
- package/src/PgStorage.res.mjs +3 -47
- package/src/Prometheus.res +7 -8
- package/src/Prometheus.res.mjs +7 -4
- package/src/PruneStaleHistory.res +45 -0
- package/src/PruneStaleHistory.res.mjs +46 -0
- package/src/RawEvent.res +73 -0
- package/src/RawEvent.res.mjs +51 -0
- package/src/Rollback.res +204 -0
- package/src/Rollback.res.mjs +118 -0
- package/src/SimulateItems.res +12 -10
- package/src/SimulateItems.res.mjs +1 -1
- package/src/TestIndexer.res +34 -27
- package/src/TestIndexer.res.mjs +4 -1
- package/src/UserContext.res +26 -114
- package/src/UserContext.res.mjs +15 -78
- package/src/Writing.res +242 -0
- package/src/Writing.res.mjs +215 -0
- package/src/db/InternalTable.res +14 -27
- package/src/sources/Evm.res +40 -1
- package/src/sources/Evm.res.mjs +94 -84
- package/src/sources/Fuel.res +40 -1
- package/src/sources/Fuel.res.mjs +36 -26
- package/src/sources/HyperFuel.res +41 -120
- package/src/sources/HyperFuel.res.mjs +42 -80
- package/src/sources/HyperFuel.resi +1 -24
- package/src/sources/HyperFuelClient.res +16 -297
- package/src/sources/HyperFuelClient.res.mjs +5 -4
- package/src/sources/HyperFuelSource.res +33 -8
- package/src/sources/HyperFuelSource.res.mjs +56 -10
- package/src/sources/HyperSyncSource.res +7 -5
- package/src/sources/HyperSyncSource.res.mjs +3 -3
- package/src/sources/RpcSource.res +2 -2
- package/src/sources/RpcSource.res.mjs +1 -1
- package/src/sources/SourceManager.res +8 -18
- package/src/sources/SourceManager.res.mjs +8 -5
- package/src/sources/SourceManager.resi +4 -1
- package/src/sources/Svm.res +18 -1
- package/src/sources/Svm.res.mjs +30 -22
- package/src/sources/SvmHyperSyncSource.res +1 -1
- package/src/sources/SvmHyperSyncSource.res.mjs +1 -1
- package/src/tui/Tui.res +38 -36
- package/src/tui/Tui.res.mjs +51 -51
- package/src/ChainFetcher.res +0 -519
- package/src/ChainFetcher.res.mjs +0 -314
- package/src/ChainManager.res +0 -358
- package/src/ChainManager.res.mjs +0 -291
- package/src/Ctx.res +0 -6
- package/src/Ctx.res.mjs +0 -2
- package/src/GlobalState.res +0 -1056
- package/src/GlobalState.res.mjs +0 -1086
- package/src/GlobalStateManager.res +0 -57
- package/src/GlobalStateManager.res.mjs +0 -68
- package/src/GlobalStateManager.resi +0 -7
- package/src/Ports.res +0 -5
- package/src/Ports.res.mjs +0 -9
- package/src/adapters/MarkBatchProcessedAdapter.res +0 -5
- package/src/adapters/MarkBatchProcessedAdapter.res.mjs +0 -14
- package/src/sources/EnvioApiClient.res +0 -15
- package/src/sources/EnvioApiClient.res.mjs +0 -24
|
@@ -1,21 +1,24 @@
|
|
|
1
1
|
// Generated by ReScript, PLEASE EDIT WITH CARE
|
|
2
2
|
|
|
3
|
-
import * as Rest from "../vendored/Rest.res.mjs";
|
|
4
3
|
import * as Hrtime from "../bindings/Hrtime.res.mjs";
|
|
5
4
|
import * as Source from "./Source.res.mjs";
|
|
6
5
|
import * as Logging from "../Logging.res.mjs";
|
|
7
6
|
import * as HyperFuel from "./HyperFuel.res.mjs";
|
|
8
|
-
import * as HyperSync from "./HyperSync.res.mjs";
|
|
9
7
|
import * as Prometheus from "../Prometheus.res.mjs";
|
|
10
8
|
import * as EventRouter from "./EventRouter.res.mjs";
|
|
9
|
+
import * as Stdlib_JsExn from "@rescript/runtime/lib/es6/Stdlib_JsExn.js";
|
|
11
10
|
import * as ErrorHandling from "../ErrorHandling.res.mjs";
|
|
12
11
|
import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
|
|
13
|
-
import * as EnvioApiClient from "./EnvioApiClient.res.mjs";
|
|
14
12
|
import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
|
|
13
|
+
import * as HyperFuelClient from "./HyperFuelClient.res.mjs";
|
|
15
14
|
import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
|
|
16
15
|
|
|
17
16
|
let EventRoutingFailed = /* @__PURE__ */Primitive_exceptions.create("HyperFuelSource.EventRoutingFailed");
|
|
18
17
|
|
|
18
|
+
function isUnauthorizedError(message) {
|
|
19
|
+
return message.includes("401 Unauthorized");
|
|
20
|
+
}
|
|
21
|
+
|
|
19
22
|
let mintEventTag = "mint";
|
|
20
23
|
|
|
21
24
|
let burnEventTag = "burn";
|
|
@@ -176,9 +179,22 @@ function memoGetSelectionConfig(chain) {
|
|
|
176
179
|
}
|
|
177
180
|
|
|
178
181
|
function make(param) {
|
|
179
|
-
let
|
|
182
|
+
let apiToken = param.apiToken;
|
|
180
183
|
let chain = param.chain;
|
|
181
184
|
let name = "HyperFuel";
|
|
185
|
+
let apiToken$1 = apiToken !== undefined ? apiToken : Stdlib_JsError.throwWithMessage(`An Envio API token is required for using HyperFuel as a data-source.
|
|
186
|
+
Set the ENVIO_API_TOKEN environment variable in your .env file.
|
|
187
|
+
Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`);
|
|
188
|
+
let client;
|
|
189
|
+
try {
|
|
190
|
+
client = HyperFuelClient.make({
|
|
191
|
+
url: param.endpointUrl,
|
|
192
|
+
apiToken: apiToken$1
|
|
193
|
+
});
|
|
194
|
+
} catch (raw_exn) {
|
|
195
|
+
let exn = Primitive_exceptions.internalToException(raw_exn);
|
|
196
|
+
client = ErrorHandling.mkLogAndRaise(undefined, "Failed to instantiate the HyperFuel client", exn);
|
|
197
|
+
}
|
|
182
198
|
let getSelectionConfig = memoGetSelectionConfig(chain);
|
|
183
199
|
let getItemsOrThrow = async (fromBlock, toBlock, addressesByContractName, indexingAddresses, knownHeight, param, selection, retry, logger) => {
|
|
184
200
|
let totalTimeRef = Hrtime.makeTimer();
|
|
@@ -188,10 +204,10 @@ function make(param) {
|
|
|
188
204
|
Prometheus.SourceRequestCount.increment(name, chain, "getLogs");
|
|
189
205
|
let pageUnsafe;
|
|
190
206
|
try {
|
|
191
|
-
pageUnsafe = await HyperFuel.GetLogs.query(
|
|
207
|
+
pageUnsafe = await HyperFuel.GetLogs.query(client, fromBlock, toBlock, recieptsSelection);
|
|
192
208
|
} catch (raw_error) {
|
|
193
209
|
let error = Primitive_exceptions.internalToException(raw_error);
|
|
194
|
-
if (error.RE_EXN_ID ===
|
|
210
|
+
if (error.RE_EXN_ID === HyperFuel.GetLogs.$$Error) {
|
|
195
211
|
let error$1 = error._1;
|
|
196
212
|
let tmp;
|
|
197
213
|
if (typeof error$1 !== "object") {
|
|
@@ -339,7 +355,7 @@ function make(param) {
|
|
|
339
355
|
blockNumber: block.height,
|
|
340
356
|
blockHash: block.id,
|
|
341
357
|
logIndex: receiptIndex,
|
|
342
|
-
|
|
358
|
+
payload: {
|
|
343
359
|
contractName: eventConfig$1.contractName,
|
|
344
360
|
eventName: eventConfig$1.name,
|
|
345
361
|
params: params,
|
|
@@ -388,7 +404,6 @@ function make(param) {
|
|
|
388
404
|
};
|
|
389
405
|
};
|
|
390
406
|
let getBlockHashes = (param, param$1) => Stdlib_JsError.throwWithMessage("HyperFuel does not support getting block hashes");
|
|
391
|
-
let jsonApiClient = EnvioApiClient.make(endpointUrl);
|
|
392
407
|
return {
|
|
393
408
|
name: name,
|
|
394
409
|
sourceFor: "Sync",
|
|
@@ -398,7 +413,37 @@ function make(param) {
|
|
|
398
413
|
getBlockHashes: getBlockHashes,
|
|
399
414
|
getHeightOrThrow: async () => {
|
|
400
415
|
let timerRef = Hrtime.makeTimer();
|
|
401
|
-
let height
|
|
416
|
+
let height;
|
|
417
|
+
try {
|
|
418
|
+
height = await client.getHeight();
|
|
419
|
+
} catch (raw_e) {
|
|
420
|
+
let e = Primitive_exceptions.internalToException(raw_e);
|
|
421
|
+
if (e.RE_EXN_ID === "JsExn") {
|
|
422
|
+
let e$1 = e._1;
|
|
423
|
+
let message = Stdlib_JsExn.message(e$1);
|
|
424
|
+
if (message !== undefined) {
|
|
425
|
+
if (message.includes("401 Unauthorized")) {
|
|
426
|
+
Logging.error(`Your ENVIO_API_TOKEN was rejected by HyperFuel (401 Unauthorized). The indexer will not be able to fetch events. Update the token and try again using 'envio start' or 'envio dev'. For more info: https://docs.envio.dev/docs/HyperSync/api-tokens`);
|
|
427
|
+
await new Promise((param, param$1) => {});
|
|
428
|
+
height = 0;
|
|
429
|
+
} else {
|
|
430
|
+
throw {
|
|
431
|
+
RE_EXN_ID: "JsExn",
|
|
432
|
+
_1: e$1,
|
|
433
|
+
Error: new Error()
|
|
434
|
+
};
|
|
435
|
+
}
|
|
436
|
+
} else {
|
|
437
|
+
throw {
|
|
438
|
+
RE_EXN_ID: "JsExn",
|
|
439
|
+
_1: e$1,
|
|
440
|
+
Error: new Error()
|
|
441
|
+
};
|
|
442
|
+
}
|
|
443
|
+
} else {
|
|
444
|
+
throw e;
|
|
445
|
+
}
|
|
446
|
+
}
|
|
402
447
|
let seconds = Hrtime.toSecondsFloat(Hrtime.timeSince(timerRef));
|
|
403
448
|
Prometheus.SourceRequestCount.increment(name, chain, "getHeight");
|
|
404
449
|
Prometheus.SourceRequestCount.addSeconds(name, chain, "getHeight", seconds);
|
|
@@ -410,6 +455,7 @@ function make(param) {
|
|
|
410
455
|
|
|
411
456
|
export {
|
|
412
457
|
EventRoutingFailed,
|
|
458
|
+
isUnauthorizedError,
|
|
413
459
|
mintEventTag,
|
|
414
460
|
burnEventTag,
|
|
415
461
|
transferEventTag,
|
|
@@ -422,4 +468,4 @@ export {
|
|
|
422
468
|
memoGetSelectionConfig,
|
|
423
469
|
make,
|
|
424
470
|
}
|
|
425
|
-
/*
|
|
471
|
+
/* Logging Not a pure module */
|
|
@@ -167,9 +167,9 @@ let make = (
|
|
|
167
167
|
let apiToken = switch apiToken {
|
|
168
168
|
| Some(token) => token
|
|
169
169
|
| None =>
|
|
170
|
-
JsError.throwWithMessage(`An API token is required for using HyperSync as a data-source.
|
|
170
|
+
JsError.throwWithMessage(`An Envio API token is required for using HyperSync as a data-source.
|
|
171
171
|
Set the ENVIO_API_TOKEN environment variable in your .env file.
|
|
172
|
-
Learn more or get a free API token at: https://envio.dev/app/api-tokens`)
|
|
172
|
+
Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`)
|
|
173
173
|
}
|
|
174
174
|
|
|
175
175
|
let client = switch HyperSyncClient.make(
|
|
@@ -206,16 +206,18 @@ Learn more or get a free API token at: https://envio.dev/app/api-tokens`)
|
|
|
206
206
|
blockNumber: block.number->Option.getUnsafe,
|
|
207
207
|
blockHash: block.hash->Option.getUnsafe,
|
|
208
208
|
logIndex,
|
|
209
|
-
|
|
209
|
+
payload: {
|
|
210
210
|
contractName: eventConfig.contractName,
|
|
211
211
|
eventName: eventConfig.name,
|
|
212
212
|
chainId,
|
|
213
213
|
params,
|
|
214
|
-
transaction
|
|
214
|
+
transaction: transaction->(
|
|
215
|
+
Utils.magic: HyperSyncClient.ResponseTypes.transaction => Internal.eventTransaction
|
|
216
|
+
),
|
|
215
217
|
block: block->(Utils.magic: HyperSyncClient.ResponseTypes.block => Internal.eventBlock),
|
|
216
218
|
srcAddress,
|
|
217
219
|
logIndex,
|
|
218
|
-
}->
|
|
220
|
+
}->Evm.fromPayload,
|
|
219
221
|
})
|
|
220
222
|
}
|
|
221
223
|
|
|
@@ -113,9 +113,9 @@ function make(param) {
|
|
|
113
113
|
let chain = param.chain;
|
|
114
114
|
let name = "HyperSync";
|
|
115
115
|
let getSelectionConfig = memoGetSelectionConfig(chain);
|
|
116
|
-
let apiToken$1 = apiToken !== undefined ? apiToken : Stdlib_JsError.throwWithMessage(`An API token is required for using HyperSync as a data-source.
|
|
116
|
+
let apiToken$1 = apiToken !== undefined ? apiToken : Stdlib_JsError.throwWithMessage(`An Envio API token is required for using HyperSync as a data-source.
|
|
117
117
|
Set the ENVIO_API_TOKEN environment variable in your .env file.
|
|
118
|
-
Learn more or get a free API token at: https://envio.dev/app/api-tokens`);
|
|
118
|
+
Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`);
|
|
119
119
|
let client;
|
|
120
120
|
try {
|
|
121
121
|
client = HyperSyncClient.make(endpointUrl, apiToken$1, param.clientTimeoutMillis, param.allEventParams, !param.lowercaseAddresses, param.serializationFormat, param.enableQueryCaching, undefined, undefined, undefined, param.logLevel);
|
|
@@ -135,7 +135,7 @@ Learn more or get a free API token at: https://envio.dev/app/api-tokens`);
|
|
|
135
135
|
blockNumber: block.number,
|
|
136
136
|
blockHash: block.hash,
|
|
137
137
|
logIndex: logIndex,
|
|
138
|
-
|
|
138
|
+
payload: {
|
|
139
139
|
contractName: eventConfig.contractName,
|
|
140
140
|
eventName: eventConfig.name,
|
|
141
141
|
params: params,
|
|
@@ -1200,7 +1200,7 @@ let make = (
|
|
|
1200
1200
|
blockHash: block->getBlockHash,
|
|
1201
1201
|
chain,
|
|
1202
1202
|
logIndex: log.logIndex,
|
|
1203
|
-
|
|
1203
|
+
payload: {
|
|
1204
1204
|
contractName: eventConfig.contractName,
|
|
1205
1205
|
eventName: eventConfig.name,
|
|
1206
1206
|
chainId: chain->ChainMap.Chain.toChainId,
|
|
@@ -1209,7 +1209,7 @@ let make = (
|
|
|
1209
1209
|
block,
|
|
1210
1210
|
srcAddress: routedAddress,
|
|
1211
1211
|
logIndex: log.logIndex,
|
|
1212
|
-
}->
|
|
1212
|
+
}->Evm.fromPayload,
|
|
1213
1213
|
})
|
|
1214
1214
|
}
|
|
1215
1215
|
)(),
|
|
@@ -1131,7 +1131,7 @@ function make(param) {
|
|
|
1131
1131
|
blockNumber: block.number,
|
|
1132
1132
|
blockHash: block.hash,
|
|
1133
1133
|
logIndex: log.logIndex,
|
|
1134
|
-
|
|
1134
|
+
payload: {
|
|
1135
1135
|
contractName: eventConfig.contractName,
|
|
1136
1136
|
eventName: eventConfig.name,
|
|
1137
1137
|
params: decoded$1,
|
|
@@ -11,16 +11,12 @@ type sourceState = {
|
|
|
11
11
|
mutable lastFailedAt: option<float>,
|
|
12
12
|
}
|
|
13
13
|
|
|
14
|
-
//
|
|
15
|
-
// But currently the ChainFetcher module is immutable
|
|
16
|
-
// and handles both processing and fetching.
|
|
17
|
-
// So this module is to encapsulate the fetching logic only
|
|
14
|
+
// Encapsulates the fetching logic for a chain's sources.
|
|
18
15
|
// with a mutable state for easier reasoning and testing.
|
|
19
16
|
type t = {
|
|
20
17
|
sourcesState: array<sourceState>,
|
|
21
18
|
mutable statusStart: Hrtime.timeRef,
|
|
22
19
|
mutable status: sourceManagerStatus,
|
|
23
|
-
maxPartitionConcurrency: int,
|
|
24
20
|
newBlockStallTimeout: int,
|
|
25
21
|
newBlockStallTimeoutRealtime: int,
|
|
26
22
|
stalledPollingInterval: int,
|
|
@@ -46,6 +42,10 @@ type t = {
|
|
|
46
42
|
|
|
47
43
|
let getActiveSource = sourceManager => sourceManager.activeSource
|
|
48
44
|
|
|
45
|
+
// Partition queries currently in flight on this chain's sources. Summed across
|
|
46
|
+
// chains by CrossChainState to enforce the indexer-wide concurrency budget.
|
|
47
|
+
let inFlightCount = sourceManager => sourceManager.fetchingPartitionsCount
|
|
48
|
+
|
|
49
49
|
let getRateLimitTimeMs = sourceManager =>
|
|
50
50
|
sourceManager.committedRateLimitTimeMs +.
|
|
51
51
|
switch sourceManager.activeRateLimitStartMs {
|
|
@@ -151,7 +151,6 @@ let makeGetHeightRetryInterval = (
|
|
|
151
151
|
|
|
152
152
|
let make = (
|
|
153
153
|
~sources: array<Source.t>,
|
|
154
|
-
~maxPartitionConcurrency,
|
|
155
154
|
~isRealtime,
|
|
156
155
|
~newBlockStallTimeout=60_000,
|
|
157
156
|
~newBlockStallTimeoutRealtime=20_000,
|
|
@@ -172,16 +171,11 @@ let make = (
|
|
|
172
171
|
| None =>
|
|
173
172
|
JsError.throwWithMessage("Invalid configuration, no data-source for historical sync provided")
|
|
174
173
|
}
|
|
175
|
-
Prometheus.IndexingMaxConcurrency.set(
|
|
176
|
-
~maxConcurrency=maxPartitionConcurrency,
|
|
177
|
-
~chainId=initialActiveSource.chain->ChainMap.Chain.toChainId,
|
|
178
|
-
)
|
|
179
174
|
Prometheus.IndexingConcurrency.set(
|
|
180
175
|
~concurrency=0,
|
|
181
176
|
~chainId=initialActiveSource.chain->ChainMap.Chain.toChainId,
|
|
182
177
|
)
|
|
183
178
|
{
|
|
184
|
-
maxPartitionConcurrency,
|
|
185
179
|
sourcesState: sources->Array.map(source => {
|
|
186
180
|
source,
|
|
187
181
|
knownHeight: 0,
|
|
@@ -229,15 +223,11 @@ let fetchNext = async (
|
|
|
229
223
|
~executeQuery,
|
|
230
224
|
~waitForNewBlock,
|
|
231
225
|
~onNewBlock,
|
|
226
|
+
~concurrencyLimit,
|
|
227
|
+
~bufferLimit,
|
|
232
228
|
~stateId,
|
|
233
229
|
) => {
|
|
234
|
-
let
|
|
235
|
-
|
|
236
|
-
let nextQuery = fetchState->FetchState.getNextQuery(
|
|
237
|
-
~concurrencyLimit={
|
|
238
|
-
maxPartitionConcurrency - sourceManager.fetchingPartitionsCount
|
|
239
|
-
},
|
|
240
|
-
)
|
|
230
|
+
let nextQuery = fetchState->FetchState.getNextQuery(~concurrencyLimit, ~bufferLimit)
|
|
241
231
|
|
|
242
232
|
switch nextQuery {
|
|
243
233
|
| ReachedMaxConcurrency
|
|
@@ -18,6 +18,10 @@ function getActiveSource(sourceManager) {
|
|
|
18
18
|
return sourceManager.activeSource;
|
|
19
19
|
}
|
|
20
20
|
|
|
21
|
+
function inFlightCount(sourceManager) {
|
|
22
|
+
return sourceManager.fetchingPartitionsCount;
|
|
23
|
+
}
|
|
24
|
+
|
|
21
25
|
function getRateLimitTimeMs(sourceManager) {
|
|
22
26
|
let startMs = sourceManager.activeRateLimitStartMs;
|
|
23
27
|
return sourceManager.committedRateLimitTimeMs + (
|
|
@@ -115,7 +119,7 @@ function makeGetHeightRetryInterval(initialRetryInterval, backoffMultiplicative,
|
|
|
115
119
|
};
|
|
116
120
|
}
|
|
117
121
|
|
|
118
|
-
function make(sources,
|
|
122
|
+
function make(sources, isRealtime, newBlockStallTimeoutOpt, newBlockStallTimeoutRealtimeOpt, stalledPollingIntervalOpt, reducedPollingIntervalOpt, recoveryTimeoutOpt, getHeightRetryIntervalOpt) {
|
|
119
123
|
let newBlockStallTimeout = newBlockStallTimeoutOpt !== undefined ? newBlockStallTimeoutOpt : 60000;
|
|
120
124
|
let newBlockStallTimeoutRealtime = newBlockStallTimeoutRealtimeOpt !== undefined ? newBlockStallTimeoutRealtimeOpt : 20000;
|
|
121
125
|
let stalledPollingInterval = stalledPollingIntervalOpt !== undefined ? stalledPollingIntervalOpt : 5000;
|
|
@@ -125,7 +129,6 @@ function make(sources, maxPartitionConcurrency, isRealtime, newBlockStallTimeout
|
|
|
125
129
|
let hasRealtime = sources.some(s => s.sourceFor === "Realtime");
|
|
126
130
|
let source = sources.find(source => getSourceRole(source.sourceFor, isRealtime, hasRealtime) === "Primary");
|
|
127
131
|
let initialActiveSource = source !== undefined ? source : Stdlib_JsError.throwWithMessage("Invalid configuration, no data-source for historical sync provided");
|
|
128
|
-
Prometheus.IndexingMaxConcurrency.set(maxPartitionConcurrency, initialActiveSource.chain);
|
|
129
132
|
Prometheus.IndexingConcurrency.set(0, initialActiveSource.chain);
|
|
130
133
|
return {
|
|
131
134
|
sourcesState: sources.map(source => ({
|
|
@@ -138,7 +141,6 @@ function make(sources, maxPartitionConcurrency, isRealtime, newBlockStallTimeout
|
|
|
138
141
|
})),
|
|
139
142
|
statusStart: Hrtime.makeTimer(),
|
|
140
143
|
status: "Idle",
|
|
141
|
-
maxPartitionConcurrency: maxPartitionConcurrency,
|
|
142
144
|
newBlockStallTimeout: newBlockStallTimeout,
|
|
143
145
|
newBlockStallTimeoutRealtime: newBlockStallTimeoutRealtime,
|
|
144
146
|
stalledPollingInterval: stalledPollingInterval,
|
|
@@ -175,8 +177,8 @@ function trackNewStatus(sourceManager, newStatus) {
|
|
|
175
177
|
sourceManager.status = newStatus;
|
|
176
178
|
}
|
|
177
179
|
|
|
178
|
-
async function fetchNext(sourceManager, fetchState, executeQuery, waitForNewBlock, onNewBlock, stateId) {
|
|
179
|
-
let nextQuery = FetchState.getNextQuery(fetchState,
|
|
180
|
+
async function fetchNext(sourceManager, fetchState, executeQuery, waitForNewBlock, onNewBlock, concurrencyLimit, bufferLimit, stateId) {
|
|
181
|
+
let nextQuery = FetchState.getNextQuery(fetchState, concurrencyLimit, bufferLimit);
|
|
180
182
|
if (typeof nextQuery !== "object") {
|
|
181
183
|
switch (nextQuery) {
|
|
182
184
|
case "WaitingForNewBlock" :
|
|
@@ -680,6 +682,7 @@ export {
|
|
|
680
682
|
getSourceRole,
|
|
681
683
|
make,
|
|
682
684
|
getActiveSource,
|
|
685
|
+
inFlightCount,
|
|
683
686
|
onReorg,
|
|
684
687
|
fetchNext,
|
|
685
688
|
waitForNewBlock,
|
|
@@ -10,7 +10,6 @@ let getSourceRole: (
|
|
|
10
10
|
|
|
11
11
|
let make: (
|
|
12
12
|
~sources: array<Source.t>,
|
|
13
|
-
~maxPartitionConcurrency: int,
|
|
14
13
|
~isRealtime: bool,
|
|
15
14
|
~newBlockStallTimeout: int=?,
|
|
16
15
|
~newBlockStallTimeoutRealtime: int=?,
|
|
@@ -22,6 +21,8 @@ let make: (
|
|
|
22
21
|
|
|
23
22
|
let getActiveSource: t => Source.t
|
|
24
23
|
|
|
24
|
+
let inFlightCount: t => int
|
|
25
|
+
|
|
25
26
|
let onReorg: (t, ~rollbackTargetBlock: int) => unit
|
|
26
27
|
|
|
27
28
|
let fetchNext: (
|
|
@@ -30,6 +31,8 @@ let fetchNext: (
|
|
|
30
31
|
~executeQuery: FetchState.query => promise<unit>,
|
|
31
32
|
~waitForNewBlock: (~knownHeight: int) => promise<int>,
|
|
32
33
|
~onNewBlock: (~knownHeight: int) => unit,
|
|
34
|
+
~concurrencyLimit: int,
|
|
35
|
+
~bufferLimit: int,
|
|
33
36
|
~stateId: int,
|
|
34
37
|
) => promise<unit>
|
|
35
38
|
|
package/src/sources/Svm.res
CHANGED
|
@@ -4,7 +4,7 @@ let cleanUpRawEventFieldsInPlace: JSON.t => unit = %raw(`fields => {
|
|
|
4
4
|
delete fields.time
|
|
5
5
|
}`)
|
|
6
6
|
|
|
7
|
-
let
|
|
7
|
+
let make = (~logger: Pino.t): Ecosystem.t => {
|
|
8
8
|
name: Svm,
|
|
9
9
|
blockFields: ["slot"],
|
|
10
10
|
transactionFields: [],
|
|
@@ -19,6 +19,23 @@ let ecosystem: Ecosystem.t = {
|
|
|
19
19
|
// SVM has no event handlers, so there is no `onEvent` `where` value to
|
|
20
20
|
// parse. The schema is a no-op object that always surfaces `None`.
|
|
21
21
|
onEventBlockFilterSchema: S.object(_ => None),
|
|
22
|
+
logger,
|
|
23
|
+
toEvent: eventItem => eventItem.payload->Internal.payloadToEvent,
|
|
24
|
+
toEventLogger: eventItem => {
|
|
25
|
+
let instruction =
|
|
26
|
+
eventItem.payload->(Utils.magic: Internal.eventPayload => Envio.svmInstruction)
|
|
27
|
+
Logging.createChildFrom(
|
|
28
|
+
~logger,
|
|
29
|
+
~params={
|
|
30
|
+
"program": eventItem.eventConfig.contractName,
|
|
31
|
+
"instruction": eventItem.eventConfig.name,
|
|
32
|
+
"chainId": eventItem.chain->ChainMap.Chain.toChainId,
|
|
33
|
+
"slot": eventItem.blockNumber,
|
|
34
|
+
"programId": instruction.programId,
|
|
35
|
+
},
|
|
36
|
+
)
|
|
37
|
+
},
|
|
38
|
+
toRawEvent: _ => JsError.throwWithMessage("Raw events are not supported for SVM"),
|
|
22
39
|
}
|
|
23
40
|
|
|
24
41
|
module GetFinalizedSlot = {
|
package/src/sources/Svm.res.mjs
CHANGED
|
@@ -4,6 +4,7 @@ import * as Rpc from "./Rpc.res.mjs";
|
|
|
4
4
|
import * as Rest from "../vendored/Rest.res.mjs";
|
|
5
5
|
import * as Utils from "../Utils.res.mjs";
|
|
6
6
|
import * as Hrtime from "../bindings/Hrtime.res.mjs";
|
|
7
|
+
import * as Logging from "../Logging.res.mjs";
|
|
7
8
|
import * as Prometheus from "../Prometheus.res.mjs";
|
|
8
9
|
import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
|
|
9
10
|
import * as S$RescriptSchema from "rescript-schema/src/S.res.mjs";
|
|
@@ -14,26 +15,33 @@ let cleanUpRawEventFieldsInPlace = (fields => {
|
|
|
14
15
|
delete fields.time
|
|
15
16
|
});
|
|
16
17
|
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
18
|
+
function make(logger) {
|
|
19
|
+
return {
|
|
20
|
+
name: "svm",
|
|
21
|
+
blockFields: ["slot"],
|
|
22
|
+
transactionFields: [],
|
|
23
|
+
blockNumberName: "height",
|
|
24
|
+
blockTimestampName: "time",
|
|
25
|
+
blockHashName: "hash",
|
|
26
|
+
cleanUpRawEventFieldsInPlace: cleanUpRawEventFieldsInPlace,
|
|
27
|
+
onBlockMethodName: "onSlot",
|
|
28
|
+
onBlockFilterSchema: S$RescriptSchema.object(s => s.f("slot", S$RescriptSchema.option(S$RescriptSchema.unknown))),
|
|
29
|
+
onEventBlockFilterSchema: S$RescriptSchema.object(param => {}),
|
|
30
|
+
logger: logger,
|
|
31
|
+
toEvent: eventItem => eventItem.payload,
|
|
32
|
+
toEventLogger: eventItem => {
|
|
33
|
+
let instruction = eventItem.payload;
|
|
34
|
+
return Logging.createChildFrom(logger, {
|
|
35
|
+
program: eventItem.eventConfig.contractName,
|
|
36
|
+
instruction: eventItem.eventConfig.name,
|
|
37
|
+
chainId: eventItem.chain,
|
|
38
|
+
slot: eventItem.blockNumber,
|
|
39
|
+
programId: instruction.programId
|
|
40
|
+
});
|
|
41
|
+
},
|
|
42
|
+
toRawEvent: param => Stdlib_JsError.throwWithMessage("Raw events are not supported for SVM")
|
|
43
|
+
};
|
|
44
|
+
}
|
|
37
45
|
|
|
38
46
|
let route = Rpc.makeRpcRoute("getSlot", S$RescriptSchema.tuple(s => {
|
|
39
47
|
s.tag(0, {
|
|
@@ -72,8 +80,8 @@ function makeRPCSource(chain, rpc, sourceForOpt) {
|
|
|
72
80
|
|
|
73
81
|
export {
|
|
74
82
|
cleanUpRawEventFieldsInPlace,
|
|
75
|
-
|
|
83
|
+
make,
|
|
76
84
|
GetFinalizedSlot,
|
|
77
85
|
makeRPCSource,
|
|
78
86
|
}
|
|
79
|
-
/*
|
|
87
|
+
/* route Not a pure module */
|
|
@@ -510,7 +510,7 @@ let make = ({chain, endpointUrl, apiToken, eventConfigs, clientTimeoutMillis}: o
|
|
|
510
510
|
blockNumber: instr.slot,
|
|
511
511
|
blockHash: "",
|
|
512
512
|
logIndex: synthLogIndex(instr),
|
|
513
|
-
|
|
513
|
+
payload: payload->(Utils.magic: Envio.svmInstruction => Internal.eventPayload),
|
|
514
514
|
}),
|
|
515
515
|
)
|
|
516
516
|
->ignore
|
package/src/tui/Tui.res
CHANGED
|
@@ -134,11 +134,11 @@ module App = {
|
|
|
134
134
|
@react.component
|
|
135
135
|
let make = (~getState) => {
|
|
136
136
|
let stdoutColumns = Hooks.useStdoutColumns()
|
|
137
|
-
//
|
|
137
|
+
// IndexerState is mutated in place — passing the same ref to useState
|
|
138
138
|
// would bail out via Object.is and skip the re-render. Tick a counter
|
|
139
139
|
// instead and read state freshly from getState() on every render.
|
|
140
140
|
let (tick, setTick) = React.useState(() => 0)
|
|
141
|
-
let state:
|
|
141
|
+
let state: IndexerState.t = getState()
|
|
142
142
|
|
|
143
143
|
React.useEffect(() => {
|
|
144
144
|
let intervalId = setInterval(() => {
|
|
@@ -153,18 +153,22 @@ module App = {
|
|
|
153
153
|
}, [getState])
|
|
154
154
|
|
|
155
155
|
let chains =
|
|
156
|
-
state
|
|
157
|
-
->
|
|
158
|
-
->
|
|
159
|
-
|
|
156
|
+
state
|
|
157
|
+
->IndexerState.chainStates
|
|
158
|
+
->Dict.valuesToArray
|
|
159
|
+
->Array.map(cs => {
|
|
160
|
+
let numEventsProcessed = cs->ChainState.numEventsProcessed
|
|
161
|
+
let fetchState = cs->ChainState.fetchState
|
|
162
|
+
let committedProgressBlockNumber = cs->ChainState.committedProgressBlockNumber
|
|
163
|
+
let timestampCaughtUpToHeadOrEndblock = cs->ChainState.timestampCaughtUpToHeadOrEndblock
|
|
164
|
+
let sourceManager = cs->ChainState.sourceManager
|
|
160
165
|
let latestFetchedBlockNumber = Pervasives.max(fetchState->FetchState.bufferBlockNumber, 0)
|
|
161
|
-
let hasProcessedToEndblock =
|
|
162
|
-
let knownHeight =
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
: cf.fetchState.knownHeight
|
|
166
|
+
let hasProcessedToEndblock = cs->ChainState.hasProcessedToEndblock
|
|
167
|
+
let knownHeight = hasProcessedToEndblock
|
|
168
|
+
? fetchState.endBlock->Option.getOr(fetchState.knownHeight)
|
|
169
|
+
: fetchState.knownHeight
|
|
166
170
|
|
|
167
|
-
let firstEventBlock =
|
|
171
|
+
let firstEventBlock = fetchState.firstEventBlock
|
|
168
172
|
let progress: TuiData.progress = if hasProcessedToEndblock {
|
|
169
173
|
// If the endblock has been reached then set the progress to synced.
|
|
170
174
|
// if there's chains that have no events in the block range start->end,
|
|
@@ -172,25 +176,25 @@ module App = {
|
|
|
172
176
|
// This ensures TUI still displays synced in this case
|
|
173
177
|
Synced({
|
|
174
178
|
firstEventBlockNumber: firstEventBlock->Option.getOr(0),
|
|
175
|
-
latestProcessedBlock:
|
|
176
|
-
timestampCaughtUpToHeadOrEndblock:
|
|
179
|
+
latestProcessedBlock: committedProgressBlockNumber,
|
|
180
|
+
timestampCaughtUpToHeadOrEndblock: timestampCaughtUpToHeadOrEndblock->Option.getOr(
|
|
177
181
|
Date.now()->Date.fromTime,
|
|
178
182
|
),
|
|
179
183
|
numEventsProcessed,
|
|
180
184
|
})
|
|
181
185
|
} else {
|
|
182
|
-
switch (firstEventBlock,
|
|
186
|
+
switch (firstEventBlock, timestampCaughtUpToHeadOrEndblock) {
|
|
183
187
|
| (Some(firstEventBlockNumber), Some(timestampCaughtUpToHeadOrEndblock)) =>
|
|
184
188
|
Synced({
|
|
185
189
|
firstEventBlockNumber,
|
|
186
|
-
latestProcessedBlock:
|
|
190
|
+
latestProcessedBlock: committedProgressBlockNumber,
|
|
187
191
|
timestampCaughtUpToHeadOrEndblock,
|
|
188
192
|
numEventsProcessed,
|
|
189
193
|
})
|
|
190
194
|
| (Some(firstEventBlockNumber), None) =>
|
|
191
195
|
Syncing({
|
|
192
196
|
firstEventBlockNumber,
|
|
193
|
-
latestProcessedBlock:
|
|
197
|
+
latestProcessedBlock: committedProgressBlockNumber,
|
|
194
198
|
numEventsProcessed,
|
|
195
199
|
})
|
|
196
200
|
| (None, _) => SearchingForEvents
|
|
@@ -203,25 +207,23 @@ module App = {
|
|
|
203
207
|
knownHeight,
|
|
204
208
|
latestFetchedBlockNumber,
|
|
205
209
|
eventsProcessed: numEventsProcessed,
|
|
206
|
-
chainId:
|
|
207
|
-
progressBlock:
|
|
208
|
-
? Some(
|
|
209
|
-
: Some(
|
|
210
|
+
chainId: (cs->ChainState.chainConfig).id->Int.toString,
|
|
211
|
+
progressBlock: committedProgressBlockNumber < fetchState.startBlock
|
|
212
|
+
? Some(fetchState.startBlock)
|
|
213
|
+
: Some(committedProgressBlockNumber),
|
|
210
214
|
bufferBlock: Some(latestFetchedBlockNumber),
|
|
211
|
-
sourceBlock: Some(
|
|
212
|
-
firstEventBlockNumber:
|
|
213
|
-
startBlock:
|
|
214
|
-
endBlock:
|
|
215
|
-
poweredByHyperSync: (
|
|
216
|
-
|
|
217
|
-
).poweredByHyperSync,
|
|
218
|
-
blockUnit: switch state.ctx.config.ecosystem.name {
|
|
215
|
+
sourceBlock: Some(fetchState.knownHeight),
|
|
216
|
+
firstEventBlockNumber: fetchState.firstEventBlock,
|
|
217
|
+
startBlock: fetchState.startBlock,
|
|
218
|
+
endBlock: fetchState.endBlock,
|
|
219
|
+
poweredByHyperSync: (sourceManager->SourceManager.getActiveSource).poweredByHyperSync,
|
|
220
|
+
blockUnit: switch (state->IndexerState.config).ecosystem.name {
|
|
219
221
|
| Svm => "Slot"
|
|
220
222
|
| Evm | Fuel => "Block"
|
|
221
223
|
},
|
|
222
|
-
rateLimitTimeMs:
|
|
223
|
-
isRateLimited:
|
|
224
|
-
rateLimitResetInMs:
|
|
224
|
+
rateLimitTimeMs: sourceManager->SourceManager.getRateLimitTimeMs,
|
|
225
|
+
isRateLimited: sourceManager->SourceManager.isRateLimited,
|
|
226
|
+
rateLimitResetInMs: sourceManager->SourceManager.getRateLimitResetInMs,
|
|
225
227
|
}: TuiData.chain
|
|
226
228
|
)
|
|
227
229
|
})
|
|
@@ -269,7 +271,7 @@ module App = {
|
|
|
269
271
|
totalEventsProcessed
|
|
270
272
|
eventsPerSecond={SyncETA.isIndexerFullySynced(chains) ? None : eventsPerSecond}
|
|
271
273
|
/>
|
|
272
|
-
<SyncETA chains indexerStartTime=state.indexerStartTime />
|
|
274
|
+
<SyncETA chains indexerStartTime={state->IndexerState.indexerStartTime} />
|
|
273
275
|
{
|
|
274
276
|
let maxRateLimitTimeMs =
|
|
275
277
|
chains->Array.reduce(0., (acc, chain) => Pervasives.max(acc, chain.rateLimitTimeMs))
|
|
@@ -315,7 +317,7 @@ module App = {
|
|
|
315
317
|
}
|
|
316
318
|
}
|
|
317
319
|
</Box>
|
|
318
|
-
{if state.
|
|
320
|
+
{if (state->IndexerState.config).isDev {
|
|
319
321
|
<Box flexDirection={Row}>
|
|
320
322
|
<Text> {"Dev Console: "->React.string} </Text>
|
|
321
323
|
<Text color={Info} underline=true> {`${Env.envioAppUrl}/console`->React.string} </Text>
|
|
@@ -323,7 +325,7 @@ module App = {
|
|
|
323
325
|
} else {
|
|
324
326
|
React.null
|
|
325
327
|
}}
|
|
326
|
-
{switch (state.
|
|
328
|
+
{switch ((state->IndexerState.config).storage.clickhouse, Env.ClickHouse.host()) {
|
|
327
329
|
| (true, Some(host)) =>
|
|
328
330
|
<Box flexDirection={Row}>
|
|
329
331
|
<Text> {"ClickHouse: "->React.string} </Text>
|
|
@@ -331,7 +333,7 @@ module App = {
|
|
|
331
333
|
</Box>
|
|
332
334
|
| _ => React.null
|
|
333
335
|
}}
|
|
334
|
-
<Messages config=state.
|
|
336
|
+
<Messages config={state->IndexerState.config} />
|
|
335
337
|
</Box>
|
|
336
338
|
}
|
|
337
339
|
}
|