envio 3.3.0-alpha.1 → 3.3.0-alpha.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.d.ts +23 -21
- package/package.json +6 -6
- package/src/Batch.res +0 -6
- package/src/Batch.res.mjs +0 -11
- package/src/BatchProcessing.res +2 -12
- package/src/BatchProcessing.res.mjs +2 -7
- package/src/ChainFetching.res +42 -6
- package/src/ChainFetching.res.mjs +30 -11
- package/src/ChainMetadata.res +1 -11
- package/src/ChainMetadata.res.mjs +1 -10
- package/src/ChainState.res +150 -1
- package/src/ChainState.res.mjs +138 -9
- package/src/ChainState.resi +51 -1
- package/src/Config.res +3 -6
- package/src/Config.res.mjs +2 -3
- package/src/Core.res +7 -0
- package/src/CrossChainState.res +24 -26
- package/src/CrossChainState.res.mjs +19 -16
- package/src/Ecosystem.res +6 -3
- package/src/Envio.res +3 -2
- package/src/EventConfigBuilder.res +61 -23
- package/src/EventConfigBuilder.res.mjs +27 -13
- package/src/EventProcessing.res +63 -21
- package/src/EventProcessing.res.mjs +47 -22
- package/src/FetchState.res +53 -43
- package/src/FetchState.res.mjs +71 -54
- package/src/HandlerLoader.res +10 -1
- package/src/HandlerLoader.res.mjs +15 -8
- package/src/Internal.res +46 -7
- package/src/Internal.res.mjs +19 -1
- package/src/LoadLayer.res +5 -5
- package/src/LoadLayer.res.mjs +6 -6
- package/src/Main.res +3 -50
- package/src/Main.res.mjs +2 -23
- package/src/PgStorage.res +4 -4
- package/src/PgStorage.res.mjs +5 -5
- package/src/Prometheus.res +10 -10
- package/src/Prometheus.res.mjs +9 -9
- package/src/PruneStaleHistory.res +2 -2
- package/src/PruneStaleHistory.res.mjs +3 -3
- package/src/Rollback.res +3 -3
- package/src/Rollback.res.mjs +4 -4
- package/src/SimulateItems.res +80 -15
- package/src/SimulateItems.res.mjs +65 -21
- package/src/TestIndexer.res +48 -36
- package/src/TestIndexer.res.mjs +35 -28
- package/src/bindings/Performance.res +7 -0
- package/src/bindings/Performance.res.mjs +21 -0
- package/src/bindings/Performance.resi +7 -0
- package/src/sources/EventRouter.res +17 -21
- package/src/sources/EventRouter.res.mjs +6 -9
- package/src/sources/Evm.res +47 -67
- package/src/sources/Evm.res.mjs +42 -65
- package/src/sources/Fuel.res +3 -3
- package/src/sources/Fuel.res.mjs +1 -6
- package/src/sources/HyperFuelSource.res +15 -11
- package/src/sources/HyperFuelSource.res.mjs +13 -11
- package/src/sources/HyperSync.res +7 -1
- package/src/sources/HyperSync.res.mjs +6 -3
- package/src/sources/HyperSync.resi +2 -0
- package/src/sources/HyperSyncClient.res +14 -79
- package/src/sources/HyperSyncClient.res.mjs +1 -22
- package/src/sources/HyperSyncSource.res +44 -27
- package/src/sources/HyperSyncSource.res.mjs +32 -27
- package/src/sources/RpcSource.res +14 -10
- package/src/sources/RpcSource.res.mjs +11 -9
- package/src/sources/SimulateSource.res +4 -2
- package/src/sources/SimulateSource.res.mjs +1 -0
- package/src/sources/Source.res +5 -1
- package/src/sources/SourceManager.res +17 -16
- package/src/sources/SourceManager.res.mjs +10 -15
- package/src/sources/Svm.res +23 -6
- package/src/sources/Svm.res.mjs +24 -6
- package/src/sources/SvmHyperSyncClient.res +3 -29
- package/src/sources/SvmHyperSyncSource.res +78 -96
- package/src/sources/SvmHyperSyncSource.res.mjs +79 -91
- package/src/sources/TransactionStore.res +128 -0
- package/src/sources/TransactionStore.res.mjs +97 -0
- package/src/tui/Tui.res +13 -16
- package/src/tui/Tui.res.mjs +12 -14
- package/svm.schema.json +30 -37
- package/src/bindings/Hrtime.res +0 -58
- package/src/bindings/Hrtime.res.mjs +0 -90
- package/src/bindings/Hrtime.resi +0 -30
package/index.d.ts
CHANGED
|
@@ -997,24 +997,6 @@ export type SvmTokenBalance = {
|
|
|
997
997
|
readonly postAmount?: string;
|
|
998
998
|
};
|
|
999
999
|
|
|
1000
|
-
/** Parent transaction surfaced when an instruction's
|
|
1001
|
-
* `include_transaction` flag is `true`. */
|
|
1002
|
-
export type SvmTransaction = {
|
|
1003
|
-
readonly signatures: readonly string[];
|
|
1004
|
-
readonly feePayer?: string;
|
|
1005
|
-
readonly success?: boolean;
|
|
1006
|
-
readonly err?: string;
|
|
1007
|
-
/** Lamports. */
|
|
1008
|
-
readonly fee?: bigint;
|
|
1009
|
-
readonly computeUnitsConsumed?: bigint;
|
|
1010
|
-
readonly accountKeys: readonly string[];
|
|
1011
|
-
readonly recentBlockhash?: string;
|
|
1012
|
-
readonly version?: string;
|
|
1013
|
-
/** SPL Token / Token-2022 balance snapshots for this transaction.
|
|
1014
|
-
* Present when `include_token_balances` is `true`. */
|
|
1015
|
-
readonly tokenBalances?: readonly SvmTokenBalance[];
|
|
1016
|
-
};
|
|
1017
|
-
|
|
1018
1000
|
export type SvmLog = {
|
|
1019
1001
|
readonly kind: string;
|
|
1020
1002
|
readonly message: string;
|
|
@@ -1033,6 +1015,7 @@ export type SvmLog = {
|
|
|
1033
1015
|
* are base58 strings. */
|
|
1034
1016
|
export type SvmInstruction<
|
|
1035
1017
|
Params extends SvmInstructionParams = SvmInstructionParams,
|
|
1018
|
+
Tx = SvmTransaction,
|
|
1036
1019
|
> = {
|
|
1037
1020
|
/** Program name as declared under `programs[].name` in `config.yaml`. */
|
|
1038
1021
|
readonly programName: string;
|
|
@@ -1050,8 +1033,11 @@ export type SvmInstruction<
|
|
|
1050
1033
|
readonly d8?: string;
|
|
1051
1034
|
/** Borsh-decoded params. Present when a schema is configured and matched. */
|
|
1052
1035
|
readonly params?: Params;
|
|
1053
|
-
/**
|
|
1054
|
-
|
|
1036
|
+
/** Parent transaction. Carries only the fields selected via this
|
|
1037
|
+
* instruction's `field_selection`; unselected fields are typed as
|
|
1038
|
+
* `FieldNotSelected<...>` so reading them is a compile error. Always present
|
|
1039
|
+
* (`{}` when no fields are selected). */
|
|
1040
|
+
readonly transaction: Tx;
|
|
1055
1041
|
/** Present when the instruction's `include_logs` is `true`; only logs
|
|
1056
1042
|
* scoped to this exact instruction (matching `instruction_address`). */
|
|
1057
1043
|
readonly logs?: readonly SvmLog[];
|
|
@@ -1327,7 +1313,10 @@ type SvmEcosystem<Config extends IndexerConfigTypes = GlobalConfig> =
|
|
|
1327
1313
|
handler: (
|
|
1328
1314
|
args: SvmOnInstructionHandlerArgs<
|
|
1329
1315
|
Config,
|
|
1330
|
-
SvmInstruction<
|
|
1316
|
+
SvmInstruction<
|
|
1317
|
+
SvmParamsFromProgramTable<Programs[P][I]>,
|
|
1318
|
+
Programs[P][I]["transaction"]
|
|
1319
|
+
>
|
|
1331
1320
|
>,
|
|
1332
1321
|
) => Promise<void>,
|
|
1333
1322
|
) => void;
|
|
@@ -1695,6 +1684,7 @@ type EvmContractsT = GlobalConfig extends { evm: { contracts: infer X extends
|
|
|
1695
1684
|
type FuelChainsT = GlobalConfig extends { fuel: { chains: infer X extends Record<string, { id: number }> } } ? X : {};
|
|
1696
1685
|
type FuelContractsT = GlobalConfig extends { fuel: { contracts: infer X extends Record<string, Record<string, any>> } } ? X : {};
|
|
1697
1686
|
type SvmChainsT = GlobalConfig extends { svm: { chains: infer X extends Record<string, { id: number }> } } ? X : {};
|
|
1687
|
+
type SvmProgramsT = GlobalConfig extends { svm: { programs: infer X extends Record<string, Record<string, any>> } } ? X : {};
|
|
1698
1688
|
type EntitiesT = GlobalConfig extends { entities: infer X extends Record<string, object> } ? X : {};
|
|
1699
1689
|
type EnumsT = GlobalConfig extends { enums: infer X extends Record<string, any> } ? X : {};
|
|
1700
1690
|
|
|
@@ -1716,6 +1706,18 @@ export type FuelChainId = IsEmptyObject<FuelChainsT> extends true ? NotConfigure
|
|
|
1716
1706
|
/** Union of all configured SVM chain IDs. */
|
|
1717
1707
|
export type SvmChainId = IsEmptyObject<SvmChainsT> extends true ? NotConfigured<"SvmChainId", "Configure SVM chains"> : SvmChainsT [keyof SvmChainsT ]["id"];
|
|
1718
1708
|
|
|
1709
|
+
/** The SVM parent-transaction type generated from this project's
|
|
1710
|
+
* `field_selection`: the union of every instruction's `transaction` shape,
|
|
1711
|
+
* with unselected fields typed as `FieldNotSelected<...>`. Resolves to a
|
|
1712
|
+
* `NotConfigured` hint until `envio codegen` augments {@link Global}. */
|
|
1713
|
+
export type SvmTransaction = IsEmptyObject<SvmProgramsT> extends true
|
|
1714
|
+
? NotConfigured<"SvmTransaction", "Configure SVM programs">
|
|
1715
|
+
: {
|
|
1716
|
+
[P in keyof SvmProgramsT]: {
|
|
1717
|
+
[I in keyof SvmProgramsT[P]]: SvmProgramsT[P][I]["transaction"];
|
|
1718
|
+
}[keyof SvmProgramsT[P]];
|
|
1719
|
+
}[keyof SvmProgramsT];
|
|
1720
|
+
|
|
1719
1721
|
/** Lookup an EVM event type by contract and event name. Without generics,
|
|
1720
1722
|
* resolves to the discriminated union of every EVM event in the project. */
|
|
1721
1723
|
export type EvmEvent<
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "envio",
|
|
3
|
-
"version": "3.3.0-alpha.
|
|
3
|
+
"version": "3.3.0-alpha.3",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "A latency and sync speed optimized, developer friendly blockchain data indexer.",
|
|
6
6
|
"bin": "./bin.mjs",
|
|
@@ -69,10 +69,10 @@
|
|
|
69
69
|
"tsx": "4.21.0"
|
|
70
70
|
},
|
|
71
71
|
"optionalDependencies": {
|
|
72
|
-
"envio-linux-x64": "3.3.0-alpha.
|
|
73
|
-
"envio-linux-x64-musl": "3.3.0-alpha.
|
|
74
|
-
"envio-linux-arm64": "3.3.0-alpha.
|
|
75
|
-
"envio-darwin-x64": "3.3.0-alpha.
|
|
76
|
-
"envio-darwin-arm64": "3.3.0-alpha.
|
|
72
|
+
"envio-linux-x64": "3.3.0-alpha.3",
|
|
73
|
+
"envio-linux-x64-musl": "3.3.0-alpha.3",
|
|
74
|
+
"envio-linux-arm64": "3.3.0-alpha.3",
|
|
75
|
+
"envio-darwin-x64": "3.3.0-alpha.3",
|
|
76
|
+
"envio-darwin-arm64": "3.3.0-alpha.3"
|
|
77
77
|
}
|
|
78
78
|
}
|
package/src/Batch.res
CHANGED
|
@@ -34,12 +34,6 @@ type t = {
|
|
|
34
34
|
checkpointEventsProcessed: array<int>,
|
|
35
35
|
}
|
|
36
36
|
|
|
37
|
-
let hasReadyItem = (fetchStates: array<FetchState.t>) => {
|
|
38
|
-
fetchStates->Array.some(fetchState => {
|
|
39
|
-
fetchState->FetchState.isActivelyIndexing && fetchState->FetchState.hasReadyItem
|
|
40
|
-
})
|
|
41
|
-
}
|
|
42
|
-
|
|
43
37
|
let getProgressedChainsById = {
|
|
44
38
|
let getChainAfterBatchIfProgressed = (
|
|
45
39
|
~chainBeforeBatch: chainBeforeBatch,
|
package/src/Batch.res.mjs
CHANGED
|
@@ -4,16 +4,6 @@ import * as Utils from "./Utils.res.mjs";
|
|
|
4
4
|
import * as FetchState from "./FetchState.res.mjs";
|
|
5
5
|
import * as ReorgDetection from "./ReorgDetection.res.mjs";
|
|
6
6
|
|
|
7
|
-
function hasReadyItem(fetchStates) {
|
|
8
|
-
return fetchStates.some(fetchState => {
|
|
9
|
-
if (FetchState.isActivelyIndexing(fetchState)) {
|
|
10
|
-
return FetchState.hasReadyItem(fetchState);
|
|
11
|
-
} else {
|
|
12
|
-
return false;
|
|
13
|
-
}
|
|
14
|
-
});
|
|
15
|
-
}
|
|
16
|
-
|
|
17
7
|
function getChainAfterBatchIfProgressed(chainBeforeBatch, progressBlockNumberAfterBatch, fetchStateAfterBatch, batchSize) {
|
|
18
8
|
if (chainBeforeBatch.progressBlockNumber < progressBlockNumberAfterBatch) {
|
|
19
9
|
return {
|
|
@@ -208,7 +198,6 @@ function findLastEventItem(batch, chainId) {
|
|
|
208
198
|
}
|
|
209
199
|
|
|
210
200
|
export {
|
|
211
|
-
hasReadyItem,
|
|
212
201
|
getProgressedChainsById,
|
|
213
202
|
addReorgCheckpoints,
|
|
214
203
|
prepareBatch,
|
package/src/BatchProcessing.res
CHANGED
|
@@ -77,15 +77,7 @@ and processNextBatch = async (state: IndexerState.t, ~scheduleFetch): unit => {
|
|
|
77
77
|
state
|
|
78
78
|
->IndexerState.chainStates
|
|
79
79
|
->Dict.valuesToArray
|
|
80
|
-
->Array.every(cs =>
|
|
81
|
-
let fetchState = switch progressedChainsById->Utils.Dict.dangerouslyGetByIntNonOption(
|
|
82
|
-
(cs->ChainState.fetchState).chainId,
|
|
83
|
-
) {
|
|
84
|
-
| Some(chainAfterBatch) => chainAfterBatch.fetchState
|
|
85
|
-
| None => cs->ChainState.fetchState
|
|
86
|
-
}
|
|
87
|
-
fetchState->FetchState.isReadyToEnterReorgThreshold
|
|
88
|
-
})
|
|
80
|
+
->Array.every(cs => cs->ChainState.isReadyToEnterReorgThresholdAfterBatch(~batch))
|
|
89
81
|
|
|
90
82
|
if shouldEnterReorgThreshold {
|
|
91
83
|
IndexerState.enterReorgThreshold(state)
|
|
@@ -171,9 +163,7 @@ and processNextBatch = async (state: IndexerState.t, ~scheduleFetch): unit => {
|
|
|
171
163
|
state
|
|
172
164
|
->IndexerState.chainStates
|
|
173
165
|
->Dict.valuesToArray
|
|
174
|
-
->Array.every(
|
|
175
|
-
cs->ChainState.isProgressAtHead && (cs->ChainState.fetchState).endBlock->Option.isNone
|
|
176
|
-
)
|
|
166
|
+
->Array.every(ChainState.isAtHeadWithoutEndBlock)
|
|
177
167
|
) {
|
|
178
168
|
IndexerState.errorExit(
|
|
179
169
|
state,
|
|
@@ -4,12 +4,10 @@ import * as Utils from "./Utils.res.mjs";
|
|
|
4
4
|
import * as Config from "./Config.res.mjs";
|
|
5
5
|
import * as Logging from "./Logging.res.mjs";
|
|
6
6
|
import * as ChainState from "./ChainState.res.mjs";
|
|
7
|
-
import * as FetchState from "./FetchState.res.mjs";
|
|
8
7
|
import * as IndexerState from "./IndexerState.res.mjs";
|
|
9
8
|
import * as ChainMetadata from "./ChainMetadata.res.mjs";
|
|
10
9
|
import * as ErrorHandling from "./ErrorHandling.res.mjs";
|
|
11
10
|
import * as InMemoryStore from "./InMemoryStore.res.mjs";
|
|
12
|
-
import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
|
|
13
11
|
import * as ExitOnCaughtUp from "./ExitOnCaughtUp.res.mjs";
|
|
14
12
|
import * as EventProcessing from "./EventProcessing.res.mjs";
|
|
15
13
|
import * as PruneStaleHistory from "./PruneStaleHistory.res.mjs";
|
|
@@ -30,10 +28,7 @@ async function processNextBatch(state, scheduleFetch) {
|
|
|
30
28
|
let batch = IndexerState.createBatch(state, IndexerState.processedCheckpointId(state), IndexerState.config(state).batchSize, isRollbackBatch);
|
|
31
29
|
let progressedChainsById = batch.progressedChainsById;
|
|
32
30
|
let isBelowReorgThreshold = !isInReorgThresholdBeforeUpdate && IndexerState.config(state).shouldRollbackOnReorg;
|
|
33
|
-
let shouldEnterReorgThreshold = isBelowReorgThreshold && Object.values(IndexerState.chainStates(state)).every(cs =>
|
|
34
|
-
let chainAfterBatch = progressedChainsById[ChainState.fetchState(cs).chainId];
|
|
35
|
-
return FetchState.isReadyToEnterReorgThreshold(chainAfterBatch !== undefined ? chainAfterBatch.fetchState : ChainState.fetchState(cs));
|
|
36
|
-
});
|
|
31
|
+
let shouldEnterReorgThreshold = isBelowReorgThreshold && Object.values(IndexerState.chainStates(state)).every(cs => ChainState.isReadyToEnterReorgThresholdAfterBatch(cs, batch));
|
|
37
32
|
if (shouldEnterReorgThreshold) {
|
|
38
33
|
IndexerState.enterReorgThreshold(state);
|
|
39
34
|
}
|
|
@@ -75,7 +70,7 @@ async function processNextBatch(state, scheduleFetch) {
|
|
|
75
70
|
}
|
|
76
71
|
if (allCaughtUp && !IndexerState.keepProcessAlive(state)) {
|
|
77
72
|
return await ExitOnCaughtUp.run(state);
|
|
78
|
-
} else if (!allCaughtUp && IndexerState.exitAfterFirstEventBlock(state) && Object.values(IndexerState.chainStates(state)).every(
|
|
73
|
+
} else if (!allCaughtUp && IndexerState.exitAfterFirstEventBlock(state) && Object.values(IndexerState.chainStates(state)).every(ChainState.isAtHeadWithoutEndBlock)) {
|
|
79
74
|
return IndexerState.errorExit(state, ErrorHandling.make(new Error("No events found between startBlock and chain head. Cannot auto-detect endBlock."), undefined, undefined));
|
|
80
75
|
} else {
|
|
81
76
|
ChainMetadata.stage(state);
|
package/src/ChainFetching.res
CHANGED
|
@@ -11,7 +11,14 @@ type partitionQueryResponse = {
|
|
|
11
11
|
let runContractRegistersOrThrow = async (
|
|
12
12
|
~itemsWithContractRegister: array<Internal.item>,
|
|
13
13
|
~config: Config.t,
|
|
14
|
+
~chainState: ChainState.t,
|
|
15
|
+
~page: option<TransactionStore.t>,
|
|
14
16
|
) => {
|
|
17
|
+
// contractRegister handlers can read event.transaction, so materialise the
|
|
18
|
+
// selected fields onto the payloads before running them. All items belong to
|
|
19
|
+
// the chain being fetched, hence its single page store and mask.
|
|
20
|
+
await chainState->ChainState.materializePageItems(~items=itemsWithContractRegister, ~page)
|
|
21
|
+
|
|
15
22
|
let itemsWithDcs = []
|
|
16
23
|
|
|
17
24
|
let onRegister = (~item: Internal.item, ~contractAddress, ~contractName) => {
|
|
@@ -106,6 +113,7 @@ let rec onQueryResponse = async (
|
|
|
106
113
|
let chainState = state->IndexerState.getChainState(~chain)
|
|
107
114
|
let {
|
|
108
115
|
parsedQueueItems,
|
|
116
|
+
transactionStore,
|
|
109
117
|
latestFetchedBlockNumber,
|
|
110
118
|
latestFetchedBlockTimestamp,
|
|
111
119
|
stats,
|
|
@@ -114,7 +122,7 @@ let rec onQueryResponse = async (
|
|
|
114
122
|
fromBlockQueried,
|
|
115
123
|
} = response
|
|
116
124
|
|
|
117
|
-
if knownHeight >
|
|
125
|
+
if knownHeight > chainState->ChainState.knownHeight {
|
|
118
126
|
Prometheus.SourceHeight.set(
|
|
119
127
|
~blockNumber=knownHeight,
|
|
120
128
|
~chainId=(chainState->ChainState.chainConfig).id,
|
|
@@ -212,6 +220,7 @@ let rec onQueryResponse = async (
|
|
|
212
220
|
blockTimestamp: latestFetchedBlockTimestamp,
|
|
213
221
|
},
|
|
214
222
|
~query,
|
|
223
|
+
~transactionStore,
|
|
215
224
|
)
|
|
216
225
|
ChainMetadata.stage(state)
|
|
217
226
|
scheduleFetch()
|
|
@@ -224,6 +233,8 @@ let rec onQueryResponse = async (
|
|
|
224
233
|
switch await runContractRegistersOrThrow(
|
|
225
234
|
~itemsWithContractRegister,
|
|
226
235
|
~config=state->IndexerState.config,
|
|
236
|
+
~chainState,
|
|
237
|
+
~page=transactionStore,
|
|
227
238
|
) {
|
|
228
239
|
| exception exn => IndexerState.errorExit(state, exn->ErrorHandling.make)
|
|
229
240
|
| newItemsWithDcs => proceed(~newItemsWithDcs)
|
|
@@ -240,6 +251,7 @@ and applyQueryResponse = (
|
|
|
240
251
|
~knownHeight,
|
|
241
252
|
~latestFetchedBlock,
|
|
242
253
|
~query,
|
|
254
|
+
~transactionStore,
|
|
243
255
|
) => {
|
|
244
256
|
let chainState = state->IndexerState.getChainState(~chain)
|
|
245
257
|
let wasFetchingAtHead = chainState->ChainState.isFetchingAtHead
|
|
@@ -250,6 +262,7 @@ and applyQueryResponse = (
|
|
|
250
262
|
~newItems,
|
|
251
263
|
~newItemsWithDcs,
|
|
252
264
|
~knownHeight,
|
|
265
|
+
~transactionStore,
|
|
253
266
|
)
|
|
254
267
|
|
|
255
268
|
// In auto-exit mode, set endBlock to the first event's block when events arrive.
|
|
@@ -294,7 +307,7 @@ let finishWaitingForNewBlock = (
|
|
|
294
307
|
->IndexerState.chainStates
|
|
295
308
|
->Dict.valuesToArray
|
|
296
309
|
->Array.every(cs => {
|
|
297
|
-
cs->ChainState.
|
|
310
|
+
cs->ChainState.isReadyToEnterReorgThreshold
|
|
298
311
|
})
|
|
299
312
|
|
|
300
313
|
// Kick processing in case there are block handlers to run.
|
|
@@ -316,7 +329,6 @@ let fetchChain = async (
|
|
|
316
329
|
) => {
|
|
317
330
|
let chainState = state->IndexerState.getChainState(~chain)
|
|
318
331
|
if !(state->IndexerState.isResolvingReorg) && !(state->IndexerState.isStopped) {
|
|
319
|
-
let fetchState = chainState->ChainState.fetchState
|
|
320
332
|
let isRealtime = state->IndexerState.isRealtime
|
|
321
333
|
let sourceManager = chainState->ChainState.sourceManager
|
|
322
334
|
|
|
@@ -324,11 +336,19 @@ let fetchChain = async (
|
|
|
324
336
|
// there's nothing to fetch. During backfill any such chain is idle.
|
|
325
337
|
let reducedPolling = !isRealtime
|
|
326
338
|
|
|
339
|
+
// Accumulated across all queries dispatched in this tick so their per-query
|
|
340
|
+
// results collapse into a single completion log instead of one per query.
|
|
341
|
+
let dispatchedCount = switch action {
|
|
342
|
+
| FetchState.Ready(queries) => queries->Array.length
|
|
343
|
+
| WaitingForNewBlock | NothingToQuery => 0
|
|
344
|
+
}
|
|
345
|
+
let fetchedByPartition = Dict.make()
|
|
346
|
+
let fetchedCount = ref(0)
|
|
347
|
+
|
|
327
348
|
// Owns its error boundary: launch doesn't catch, so any failure here (the
|
|
328
349
|
// query, response handling, or dispatch itself) must stop the indexer.
|
|
329
350
|
try {
|
|
330
|
-
await
|
|
331
|
-
~fetchState,
|
|
351
|
+
await chainState->ChainState.dispatch(
|
|
332
352
|
~waitForNewBlock=(~knownHeight) =>
|
|
333
353
|
sourceManager->SourceManager.waitForNewBlock(~knownHeight, ~isRealtime, ~reducedPolling),
|
|
334
354
|
~onNewBlock=(~knownHeight) =>
|
|
@@ -347,9 +367,25 @@ let fetchChain = async (
|
|
|
347
367
|
try {
|
|
348
368
|
let response = await sourceManager->SourceManager.executeQuery(
|
|
349
369
|
~query,
|
|
350
|
-
~knownHeight=
|
|
370
|
+
~knownHeight=chainState->ChainState.knownHeight,
|
|
351
371
|
~isRealtime,
|
|
352
372
|
)
|
|
373
|
+
fetchedCount := fetchedCount.contents + 1
|
|
374
|
+
fetchedByPartition->Dict.set(
|
|
375
|
+
query.partitionId,
|
|
376
|
+
{
|
|
377
|
+
"fromBlock": response.fromBlockQueried,
|
|
378
|
+
"toBlock": response.latestFetchedBlockNumber,
|
|
379
|
+
"numEvents": response.parsedQueueItems->Array.length,
|
|
380
|
+
},
|
|
381
|
+
)
|
|
382
|
+
if dispatchedCount > 0 && fetchedCount.contents === dispatchedCount {
|
|
383
|
+
Logging.trace({
|
|
384
|
+
"msg": "Finished querying",
|
|
385
|
+
"chainId": chain->ChainMap.Chain.toChainId,
|
|
386
|
+
"partitions": fetchedByPartition,
|
|
387
|
+
})
|
|
388
|
+
}
|
|
353
389
|
await onQueryResponse(
|
|
354
390
|
state,
|
|
355
391
|
{chain, response, query},
|
|
@@ -4,7 +4,6 @@ import * as Utils from "./Utils.res.mjs";
|
|
|
4
4
|
import * as Logging from "./Logging.res.mjs";
|
|
5
5
|
import * as Ecosystem from "./Ecosystem.res.mjs";
|
|
6
6
|
import * as ChainState from "./ChainState.res.mjs";
|
|
7
|
-
import * as FetchState from "./FetchState.res.mjs";
|
|
8
7
|
import * as Prometheus from "./Prometheus.res.mjs";
|
|
9
8
|
import * as IndexerState from "./IndexerState.res.mjs";
|
|
10
9
|
import * as ChainMetadata from "./ChainMetadata.res.mjs";
|
|
@@ -17,7 +16,8 @@ import * as Stdlib_Promise from "@rescript/runtime/lib/es6/Stdlib_Promise.js";
|
|
|
17
16
|
import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
|
|
18
17
|
import * as ContractRegisterContext from "./ContractRegisterContext.res.mjs";
|
|
19
18
|
|
|
20
|
-
async function runContractRegistersOrThrow(itemsWithContractRegister, config) {
|
|
19
|
+
async function runContractRegistersOrThrow(itemsWithContractRegister, config, chainState, page) {
|
|
20
|
+
await ChainState.materializePageItems(chainState, itemsWithContractRegister, page);
|
|
21
21
|
let itemsWithDcs = [];
|
|
22
22
|
let onRegister = (item, contractAddress, contractName) => {
|
|
23
23
|
let dc_registrationBlock = item.blockNumber;
|
|
@@ -71,10 +71,10 @@ async function runContractRegistersOrThrow(itemsWithContractRegister, config) {
|
|
|
71
71
|
return itemsWithDcs;
|
|
72
72
|
}
|
|
73
73
|
|
|
74
|
-
function applyQueryResponse(state, chain, newItems, newItemsWithDcs, knownHeight, latestFetchedBlock, query) {
|
|
74
|
+
function applyQueryResponse(state, chain, newItems, newItemsWithDcs, knownHeight, latestFetchedBlock, query, transactionStore) {
|
|
75
75
|
let chainState = IndexerState.getChainState(state, chain);
|
|
76
76
|
let wasFetchingAtHead = ChainState.isFetchingAtHead(chainState);
|
|
77
|
-
ChainState.handleQueryResult(chainState, query, newItems, newItemsWithDcs, latestFetchedBlock, knownHeight);
|
|
77
|
+
ChainState.handleQueryResult(chainState, query, newItems, newItemsWithDcs, latestFetchedBlock, knownHeight, transactionStore);
|
|
78
78
|
if (IndexerState.exitAfterFirstEventBlock(state) && newItems.length !== 0) {
|
|
79
79
|
ChainState.setEndBlockToFirstEvent(chainState, newItems[0].blockNumber);
|
|
80
80
|
}
|
|
@@ -94,9 +94,10 @@ async function onQueryResponse(state, param, stateId, scheduleFetch, schedulePro
|
|
|
94
94
|
let stats = response.stats;
|
|
95
95
|
let latestFetchedBlockTimestamp = response.latestFetchedBlockTimestamp;
|
|
96
96
|
let latestFetchedBlockNumber = response.latestFetchedBlockNumber;
|
|
97
|
+
let transactionStore = response.transactionStore;
|
|
97
98
|
let parsedQueueItems = response.parsedQueueItems;
|
|
98
99
|
let knownHeight = response.knownHeight;
|
|
99
|
-
if (knownHeight > ChainState.
|
|
100
|
+
if (knownHeight > ChainState.knownHeight(chainState)) {
|
|
100
101
|
Prometheus.SourceHeight.set(SourceManager.getActiveSource(ChainState.sourceManager(chainState)).name, ChainState.chainConfig(chainState).id, knownHeight);
|
|
101
102
|
}
|
|
102
103
|
Prometheus.FetchingBlockRange.increment(chain, stats["total time elapsed (s)"], Stdlib_Option.getOr(stats["parsing time (s)"], 0), parsedQueueItems.length, (latestFetchedBlockNumber - response.fromBlockQueried | 0) + 1 | 0);
|
|
@@ -133,7 +134,7 @@ async function onQueryResponse(state, param, stateId, scheduleFetch, schedulePro
|
|
|
133
134
|
applyQueryResponse(state, chain, newItems, newItemsWithDcs, knownHeight, {
|
|
134
135
|
blockNumber: latestFetchedBlockNumber,
|
|
135
136
|
blockTimestamp: latestFetchedBlockTimestamp
|
|
136
|
-
}, query);
|
|
137
|
+
}, query, transactionStore);
|
|
137
138
|
ChainMetadata.stage(state);
|
|
138
139
|
scheduleFetch();
|
|
139
140
|
return scheduleProcessing();
|
|
@@ -144,7 +145,7 @@ async function onQueryResponse(state, param, stateId, scheduleFetch, schedulePro
|
|
|
144
145
|
}
|
|
145
146
|
let newItemsWithDcs;
|
|
146
147
|
try {
|
|
147
|
-
newItemsWithDcs = await runContractRegistersOrThrow(itemsWithContractRegister, IndexerState.config(state));
|
|
148
|
+
newItemsWithDcs = await runContractRegistersOrThrow(itemsWithContractRegister, IndexerState.config(state), chainState, transactionStore);
|
|
148
149
|
} catch (raw_exn) {
|
|
149
150
|
let exn = Primitive_exceptions.internalToException(raw_exn);
|
|
150
151
|
return IndexerState.errorExit(state, ErrorHandling.make(exn, undefined, undefined));
|
|
@@ -159,7 +160,7 @@ function finishWaitingForNewBlock(state, chain, knownHeight, stateId, scheduleFe
|
|
|
159
160
|
let chainState = IndexerState.getChainState(state, chain);
|
|
160
161
|
ChainState.updateKnownHeight(chainState, knownHeight);
|
|
161
162
|
let isBelowReorgThreshold = !IndexerState.isInReorgThreshold(state) && IndexerState.config(state).shouldRollbackOnReorg;
|
|
162
|
-
let shouldEnterReorgThreshold = isBelowReorgThreshold && Object.values(IndexerState.chainStates(state)).every(
|
|
163
|
+
let shouldEnterReorgThreshold = isBelowReorgThreshold && Object.values(IndexerState.chainStates(state)).every(ChainState.isReadyToEnterReorgThreshold);
|
|
163
164
|
if (shouldEnterReorgThreshold) {
|
|
164
165
|
IndexerState.enterReorgThreshold(state);
|
|
165
166
|
}
|
|
@@ -172,14 +173,32 @@ async function fetchChain(state, chain, action, stateId, scheduleFetch, schedule
|
|
|
172
173
|
if (IndexerState.isResolvingReorg(state) || IndexerState.isStopped(state)) {
|
|
173
174
|
return;
|
|
174
175
|
}
|
|
175
|
-
let fetchState = ChainState.fetchState(chainState);
|
|
176
176
|
let isRealtime = IndexerState.isRealtime(state);
|
|
177
177
|
let sourceManager = ChainState.sourceManager(chainState);
|
|
178
178
|
let reducedPolling = !isRealtime;
|
|
179
|
+
let dispatchedCount;
|
|
180
|
+
dispatchedCount = typeof action !== "object" ? 0 : action._0.length;
|
|
181
|
+
let fetchedByPartition = {};
|
|
182
|
+
let fetchedCount = {
|
|
183
|
+
contents: 0
|
|
184
|
+
};
|
|
179
185
|
try {
|
|
180
|
-
return await
|
|
186
|
+
return await ChainState.dispatch(chainState, async query => {
|
|
181
187
|
try {
|
|
182
|
-
let response = await SourceManager.executeQuery(sourceManager, query,
|
|
188
|
+
let response = await SourceManager.executeQuery(sourceManager, query, ChainState.knownHeight(chainState), isRealtime);
|
|
189
|
+
fetchedCount.contents = fetchedCount.contents + 1 | 0;
|
|
190
|
+
fetchedByPartition[query.partitionId] = {
|
|
191
|
+
fromBlock: response.fromBlockQueried,
|
|
192
|
+
toBlock: response.latestFetchedBlockNumber,
|
|
193
|
+
numEvents: response.parsedQueueItems.length
|
|
194
|
+
};
|
|
195
|
+
if (dispatchedCount > 0 && fetchedCount.contents === dispatchedCount) {
|
|
196
|
+
Logging.trace({
|
|
197
|
+
msg: "Finished querying",
|
|
198
|
+
chainId: chain,
|
|
199
|
+
partitions: fetchedByPartition
|
|
200
|
+
});
|
|
201
|
+
}
|
|
183
202
|
return await onQueryResponse(state, {
|
|
184
203
|
chain: chain,
|
|
185
204
|
response: response,
|
package/src/ChainMetadata.res
CHANGED
|
@@ -7,19 +7,9 @@ let stage = (state: IndexerState.t) => {
|
|
|
7
7
|
->IndexerState.chainStates
|
|
8
8
|
->Dict.valuesToArray
|
|
9
9
|
->Array.forEach(cs => {
|
|
10
|
-
let fetchState = cs->ChainState.fetchState
|
|
11
10
|
chainsData->Dict.set(
|
|
12
11
|
(cs->ChainState.chainConfig).id->Int.toString,
|
|
13
|
-
|
|
14
|
-
firstEventBlockNumber: fetchState.firstEventBlock->Null.fromOption,
|
|
15
|
-
isHyperSync: (
|
|
16
|
-
cs->ChainState.sourceManager->SourceManager.getActiveSource
|
|
17
|
-
).poweredByHyperSync,
|
|
18
|
-
latestFetchedBlockNumber: fetchState->FetchState.bufferBlockNumber,
|
|
19
|
-
timestampCaughtUpToHeadOrEndblock: cs
|
|
20
|
-
->ChainState.timestampCaughtUpToHeadOrEndblock
|
|
21
|
-
->Null.fromOption,
|
|
22
|
-
},
|
|
12
|
+
cs->ChainState.toChainMetadata,
|
|
23
13
|
)
|
|
24
14
|
})
|
|
25
15
|
|
|
@@ -2,21 +2,12 @@
|
|
|
2
2
|
|
|
3
3
|
import * as Writing from "./Writing.res.mjs";
|
|
4
4
|
import * as ChainState from "./ChainState.res.mjs";
|
|
5
|
-
import * as FetchState from "./FetchState.res.mjs";
|
|
6
|
-
import * as Stdlib_Null from "@rescript/runtime/lib/es6/Stdlib_Null.js";
|
|
7
5
|
import * as IndexerState from "./IndexerState.res.mjs";
|
|
8
|
-
import * as SourceManager from "./sources/SourceManager.res.mjs";
|
|
9
6
|
|
|
10
7
|
function stage(state) {
|
|
11
8
|
let chainsData = {};
|
|
12
9
|
Object.values(IndexerState.chainStates(state)).forEach(cs => {
|
|
13
|
-
|
|
14
|
-
chainsData[ChainState.chainConfig(cs).id.toString()] = {
|
|
15
|
-
first_event_block: Stdlib_Null.fromOption(fetchState.firstEventBlock),
|
|
16
|
-
buffer_block: FetchState.bufferBlockNumber(fetchState),
|
|
17
|
-
ready_at: Stdlib_Null.fromOption(ChainState.timestampCaughtUpToHeadOrEndblock(cs)),
|
|
18
|
-
_is_hyper_sync: SourceManager.getActiveSource(ChainState.sourceManager(cs)).poweredByHyperSync
|
|
19
|
-
};
|
|
10
|
+
chainsData[ChainState.chainConfig(cs).id.toString()] = ChainState.toChainMetadata(cs);
|
|
20
11
|
});
|
|
21
12
|
Writing.setChainMeta(state, chainsData);
|
|
22
13
|
}
|