envio 3.5.0-alpha.3 → 3.5.0-rc.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +6 -6
- package/src/Batch.res +8 -8
- package/src/Batch.res.mjs +4 -3
- package/src/BatchProcessing.res +17 -7
- package/src/BatchProcessing.res.mjs +8 -4
- package/src/ChainFetching.res +20 -19
- package/src/ChainFetching.res.mjs +14 -14
- package/src/ChainId.res +58 -0
- package/src/ChainId.res.mjs +49 -0
- package/src/ChainId.resi +27 -0
- package/src/ChainMap.res +12 -26
- package/src/ChainMap.res.mjs +5 -41
- package/src/ChainMap.resi +5 -20
- package/src/ChainMetadata.res +1 -1
- package/src/ChainMetadata.res.mjs +3 -2
- package/src/ChainState.res +35 -20
- package/src/ChainState.res.mjs +34 -11
- package/src/ChainState.resi +2 -0
- package/src/Config.res +41 -24
- package/src/Config.res.mjs +17 -14
- package/src/ContractRegisterContext.res +1 -1
- package/src/ContractRegisterContext.res.mjs +1 -1
- package/src/Core.res +1 -0
- package/src/CrossChainState.res +61 -16
- package/src/CrossChainState.res.mjs +37 -13
- package/src/CrossChainState.resi +3 -2
- package/src/Env.res +0 -3
- package/src/Env.res.mjs +0 -3
- package/src/EventConfigBuilder.res +1 -1
- package/src/EventProcessing.res +4 -4
- package/src/EventProcessing.res.mjs +5 -5
- package/src/FetchState.res +261 -89
- package/src/FetchState.res.mjs +187 -63
- package/src/FinalizeBackfill.res +33 -6
- package/src/FinalizeBackfill.res.mjs +19 -1
- package/src/HandlerRegister.res +8 -8
- package/src/HandlerRegister.res.mjs +5 -4
- package/src/HandlerRegister.resi +1 -1
- package/src/InMemoryStore.res +4 -0
- package/src/InMemoryStore.res.mjs +1 -1
- package/src/IndexerLoop.res +9 -2
- package/src/IndexerLoop.res.mjs +8 -1
- package/src/IndexerState.res +41 -24
- package/src/IndexerState.res.mjs +36 -17
- package/src/IndexerState.resi +9 -7
- package/src/Internal.res +19 -17
- package/src/Internal.res.mjs +18 -9
- package/src/LoadLayer.res +1 -1
- package/src/LoadLayer.res.mjs +2 -1
- package/src/LogSelection.res +2 -2
- package/src/Main.res +7 -11
- package/src/Main.res.mjs +4 -6
- package/src/Metrics.res +6 -6
- package/src/Metrics.res.mjs +4 -3
- package/src/Persistence.res +15 -7
- package/src/PgStorage.res +284 -258
- package/src/PgStorage.res.mjs +181 -208
- package/src/RawEvent.res +1 -2
- package/src/RawEvent.res.mjs +1 -1
- package/src/Rollback.res +18 -20
- package/src/Rollback.res.mjs +9 -9
- package/src/RollbackCommit.res +2 -2
- package/src/RollbackCommit.res.mjs +3 -3
- package/src/SafeCheckpointTracking.res +2 -2
- package/src/SimulateDeadInputTracker.res +17 -21
- package/src/SimulateDeadInputTracker.res.mjs +9 -8
- package/src/SimulateItems.res +8 -10
- package/src/SimulateItems.res.mjs +9 -10
- package/src/Sink.res +2 -2
- package/src/Sink.res.mjs +3 -2
- package/src/TestIndexer.res +11 -14
- package/src/TestIndexer.res.mjs +14 -10
- package/src/UserContext.res +2 -2
- package/src/UserContext.res.mjs +2 -1
- package/src/Utils.res +0 -4
- package/src/Utils.res.mjs +1 -6
- package/src/bindings/ClickHouse.res +20 -2
- package/src/bindings/ClickHouse.res.mjs +25 -14
- package/src/db/EntityHistory.res +1 -1
- package/src/db/IndexCatalog.res +248 -0
- package/src/db/IndexCatalog.res.mjs +222 -0
- package/src/db/IndexDefinition.res +131 -0
- package/src/db/IndexDefinition.res.mjs +138 -0
- package/src/db/IndexManager.res +145 -0
- package/src/db/IndexManager.res.mjs +130 -0
- package/src/db/InternalTable.res +50 -26
- package/src/db/InternalTable.res.mjs +32 -24
- package/src/db/Table.res +11 -1
- package/src/db/Table.res.mjs +9 -4
- package/src/sources/AddressSet.res +5 -4
- package/src/sources/AddressStore.res +25 -14
- package/src/sources/AddressStore.res.mjs +10 -12
- package/src/sources/Evm.res +2 -2
- package/src/sources/Evm.res.mjs +1 -1
- package/src/sources/EvmChain.res +3 -3
- package/src/sources/EvmChain.res.mjs +3 -3
- package/src/sources/EvmHyperSyncSource.res +7 -7
- package/src/sources/EvmHyperSyncSource.res.mjs +6 -6
- package/src/sources/Fuel.res +2 -2
- package/src/sources/Fuel.res.mjs +1 -1
- package/src/sources/FuelHyperSyncClient.res +3 -0
- package/src/sources/FuelHyperSyncClient.res.mjs +1 -0
- package/src/sources/FuelHyperSyncSource.res +5 -7
- package/src/sources/FuelHyperSyncSource.res.mjs +5 -5
- package/src/sources/HyperSync.resi +2 -2
- package/src/sources/HyperSyncClient.res +5 -0
- package/src/sources/HyperSyncClient.res.mjs +1 -0
- package/src/sources/RpcSource.res +13 -14
- package/src/sources/RpcSource.res.mjs +13 -12
- package/src/sources/SimulateSource.res +42 -15
- package/src/sources/SimulateSource.res.mjs +27 -9
- package/src/sources/Source.res +1 -5
- package/src/sources/SourceManager.res +12 -12
- package/src/sources/SourceManager.res.mjs +11 -10
- package/src/sources/SourceManager.resi +2 -2
- package/src/sources/Svm.res +4 -5
- package/src/sources/Svm.res.mjs +5 -4
- package/src/sources/SvmHyperSyncClient.res +3 -0
- package/src/sources/SvmHyperSyncClient.res.mjs +1 -0
- package/src/sources/SvmHyperSyncSource.res +4 -4
- package/src/sources/SvmHyperSyncSource.res.mjs +3 -3
- package/src/tui/Tui.res +1 -1
- package/src/tui/Tui.res.mjs +2 -1
- package/src/tui/components/CustomHooks.res +5 -4
- package/src/tui/components/CustomHooks.res.mjs +4 -2
- package/src/db/IndexRegistry.res +0 -219
- package/src/db/IndexRegistry.res.mjs +0 -195
package/src/Config.res.mjs
CHANGED
|
@@ -8,6 +8,7 @@ import * as Fuel from "./sources/Fuel.res.mjs";
|
|
|
8
8
|
import * as Table from "./db/Table.res.mjs";
|
|
9
9
|
import * as Utils from "./Utils.res.mjs";
|
|
10
10
|
import * as Address from "./Address.res.mjs";
|
|
11
|
+
import * as ChainId from "./ChainId.res.mjs";
|
|
11
12
|
import * as Logging from "./Logging.res.mjs";
|
|
12
13
|
import * as ChainMap from "./ChainMap.res.mjs";
|
|
13
14
|
import * as Internal from "./Internal.res.mjs";
|
|
@@ -24,7 +25,7 @@ import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_excep
|
|
|
24
25
|
let name = "envio_addresses";
|
|
25
26
|
|
|
26
27
|
function makeId(chainId, address) {
|
|
27
|
-
return
|
|
28
|
+
return ChainId.toString(chainId) + "-" + address;
|
|
28
29
|
}
|
|
29
30
|
|
|
30
31
|
function getAddress(entity) {
|
|
@@ -34,7 +35,7 @@ function getAddress(entity) {
|
|
|
34
35
|
|
|
35
36
|
let schema = S$RescriptSchema.schema(s => ({
|
|
36
37
|
id: s.m(S$RescriptSchema.string),
|
|
37
|
-
chain_id: s.m(
|
|
38
|
+
chain_id: s.m(ChainId.schema),
|
|
38
39
|
registration_block: s.m(S$RescriptSchema.int),
|
|
39
40
|
registration_log_index: s.m(S$RescriptSchema.int),
|
|
40
41
|
contract_name: s.m(S$RescriptSchema.string)
|
|
@@ -42,7 +43,7 @@ let schema = S$RescriptSchema.schema(s => ({
|
|
|
42
43
|
|
|
43
44
|
let table = Table.mkTable(name, undefined, [
|
|
44
45
|
Table.mkField("id", "String", S$RescriptSchema.string, undefined, undefined, undefined, true, undefined, undefined, undefined, undefined, undefined),
|
|
45
|
-
Table.mkField("chain_id", "
|
|
46
|
+
Table.mkField("chain_id", "ChainId", ChainId.schema, undefined, undefined, undefined, undefined, undefined, undefined, undefined, undefined, undefined),
|
|
46
47
|
Table.mkField("registration_block", "Int32", S$RescriptSchema.int, undefined, undefined, undefined, undefined, undefined, undefined, undefined, undefined, undefined),
|
|
47
48
|
Table.mkField("registration_log_index", "Int32", S$RescriptSchema.int, undefined, undefined, undefined, undefined, undefined, undefined, undefined, undefined, undefined),
|
|
48
49
|
Table.mkField("contract_name", "String", S$RescriptSchema.string, undefined, undefined, undefined, undefined, undefined, undefined, undefined, undefined, undefined)
|
|
@@ -98,7 +99,7 @@ let chainContractSchema = S$RescriptSchema.schema(s => ({
|
|
|
98
99
|
}));
|
|
99
100
|
|
|
100
101
|
let publicConfigChainSchema = S$RescriptSchema.schema(s => ({
|
|
101
|
-
id: s.m(
|
|
102
|
+
id: s.m(ChainId.schema),
|
|
102
103
|
startBlock: s.m(S$RescriptSchema.int),
|
|
103
104
|
endBlock: s.m(S$RescriptSchema.option(S$RescriptSchema.int)),
|
|
104
105
|
maxReorgDepth: s.m(S$RescriptSchema.option(S$RescriptSchema.int)),
|
|
@@ -400,6 +401,7 @@ let publicConfigSchema = S$RescriptSchema.schema(s => ({
|
|
|
400
401
|
rollbackOnReorg: s.m(S$RescriptSchema.option(S$RescriptSchema.bool)),
|
|
401
402
|
saveFullHistory: s.m(S$RescriptSchema.option(S$RescriptSchema.bool)),
|
|
402
403
|
rawEvents: s.m(S$RescriptSchema.option(S$RescriptSchema.bool)),
|
|
404
|
+
chainIdMode: s.m(S$RescriptSchema.option(ChainId.modeSchema)),
|
|
403
405
|
storage: s.m(publicConfigStorageSchema),
|
|
404
406
|
evm: s.m(S$RescriptSchema.option(publicConfigEvmSchema)),
|
|
405
407
|
fuel: s.m(S$RescriptSchema.option(publicConfigEcosystemSchema)),
|
|
@@ -528,7 +530,7 @@ function fromPublic(publicConfigJson) {
|
|
|
528
530
|
case "svm" :
|
|
529
531
|
let len = addresses.length;
|
|
530
532
|
let programId = len !== 1 ? (
|
|
531
|
-
len !== 0 ? Stdlib_JsError.throwWithMessage(`SVM program ` + contractName + ` on chain ` +
|
|
533
|
+
len !== 0 ? Stdlib_JsError.throwWithMessage(`SVM program ` + contractName + ` on chain ` + ChainId.toString(chainId) + ` has multiple addresses; a program is uniquely identified by a single program_id`) : Stdlib_JsError.throwWithMessage(`SVM program ` + contractName + ` on chain ` + ChainId.toString(chainId) + ` is missing a program_id`)
|
|
532
534
|
) : addresses[0];
|
|
533
535
|
let s = eventItem.svm;
|
|
534
536
|
let svm = s !== undefined ? Primitive_option.valFromOption(s) : Stdlib_JsError.throwWithMessage(`SVM instruction ` + contractName + `.` + eventName + ` is missing the "svm" descriptor in internal config`);
|
|
@@ -591,7 +593,7 @@ function fromPublic(publicConfigJson) {
|
|
|
591
593
|
contract.addresses.forEach(address => {
|
|
592
594
|
let existingContractName = contractNameByAddress[address];
|
|
593
595
|
if (existingContractName !== undefined) {
|
|
594
|
-
return Stdlib_JsError.throwWithMessage(existingContractName === contract.name ? `Address ` + address + ` is listed multiple times for the contract ` + contract.name + ` on chain ` +
|
|
596
|
+
return Stdlib_JsError.throwWithMessage(existingContractName === contract.name ? `Address ` + address + ` is listed multiple times for the contract ` + contract.name + ` on chain ` + ChainId.toString(chainId) + `. Please remove the duplicate from your config.` : `Address ` + address + ` on chain ` + ChainId.toString(chainId) + ` is configured for multiple contracts: ` + existingContractName + ` and ` + contract.name + `. Indexing the same address with multiple contract definitions is not supported. Please define the events on a single contract definition instead.`);
|
|
595
597
|
} else {
|
|
596
598
|
contractNameByAddress[address] = contract.name;
|
|
597
599
|
return;
|
|
@@ -677,7 +679,7 @@ function fromPublic(publicConfigJson) {
|
|
|
677
679
|
};
|
|
678
680
|
});
|
|
679
681
|
let chainMap = ChainMap.fromArrayUnsafe(chains.map(chain => [
|
|
680
|
-
|
|
682
|
+
chain.id,
|
|
681
683
|
chain
|
|
682
684
|
]));
|
|
683
685
|
let allEnums = parseEnumsFromJson(Stdlib_Option.getOr(publicConfig.enums, {}));
|
|
@@ -709,6 +711,7 @@ function fromPublic(publicConfigJson) {
|
|
|
709
711
|
shouldRollbackOnReorg: Stdlib_Option.getOr(publicConfig.rollbackOnReorg, true),
|
|
710
712
|
shouldSaveFullHistory: Stdlib_Option.getOr(publicConfig.saveFullHistory, false),
|
|
711
713
|
storage: globalStorage,
|
|
714
|
+
chainIdMode: Stdlib_Option.getOr(publicConfig.chainIdMode, "int32"),
|
|
712
715
|
chainMap: chainMap,
|
|
713
716
|
defaultChain: chains[0],
|
|
714
717
|
ecosystem: ecosystem,
|
|
@@ -765,14 +768,14 @@ function normalizeSimulateAddress(config, address) {
|
|
|
765
768
|
function getEventConfig(config, contractName, eventName, chainId) {
|
|
766
769
|
let chains;
|
|
767
770
|
if (chainId !== undefined) {
|
|
768
|
-
let
|
|
771
|
+
let chainId$1 = Primitive_option.valFromOption(chainId);
|
|
769
772
|
let exit = 0;
|
|
770
773
|
let chainConfig;
|
|
771
774
|
try {
|
|
772
|
-
chainConfig = ChainMap.get(config.chainMap,
|
|
775
|
+
chainConfig = ChainMap.get(config.chainMap, chainId$1);
|
|
773
776
|
exit = 1;
|
|
774
777
|
} catch (exn) {
|
|
775
|
-
chains = Stdlib_JsError.throwWithMessage(`Chain ` +
|
|
778
|
+
chains = Stdlib_JsError.throwWithMessage(`Chain ` + ChainId.toString(chainId$1) + ` is not configured. Add it to config.yaml or pass a configured chain.`);
|
|
776
779
|
}
|
|
777
780
|
if (exit === 1) {
|
|
778
781
|
chains = [chainConfig];
|
|
@@ -808,11 +811,10 @@ function shouldPruneHistory(config, isInReorgThreshold) {
|
|
|
808
811
|
}
|
|
809
812
|
|
|
810
813
|
function getChain(config, chainId) {
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
return chain;
|
|
814
|
+
if (ChainMap.has(config.chainMap, chainId)) {
|
|
815
|
+
return chainId;
|
|
814
816
|
} else {
|
|
815
|
-
return Stdlib_JsError.throwWithMessage("No chain with id " +
|
|
817
|
+
return Stdlib_JsError.throwWithMessage("No chain with id " + ChainId.toString(chainId) + " found in config.yaml");
|
|
816
818
|
}
|
|
817
819
|
}
|
|
818
820
|
|
|
@@ -983,6 +985,7 @@ function diffPaths(stored, current) {
|
|
|
983
985
|
if (typeof stored === "object" && stored !== null && !Array.isArray(stored) && typeof current === "object" && current !== null && !Array.isArray(current)) {
|
|
984
986
|
let tiers = [
|
|
985
987
|
["version"],
|
|
988
|
+
["chainIdMode"],
|
|
986
989
|
["name"],
|
|
987
990
|
["storage"],
|
|
988
991
|
[
|
|
@@ -36,7 +36,7 @@ let contractRegisterChainTraps: Utils.Proxy.traps<contractRegisterParams> = {
|
|
|
36
36
|
switch prop {
|
|
37
37
|
| "id" =>
|
|
38
38
|
let eventItem = params.item->Internal.castUnsafeEventItem
|
|
39
|
-
eventItem.
|
|
39
|
+
eventItem.chainId->(Utils.magic: ChainId.t => unknown)
|
|
40
40
|
| _ =>
|
|
41
41
|
// Look up the contract name directly in config contracts across all chains.
|
|
42
42
|
let contractName = prop
|
|
@@ -19,7 +19,7 @@ function makeAddFunction(params, contractName) {
|
|
|
19
19
|
|
|
20
20
|
let contractRegisterChainTraps_get = (params, prop) => {
|
|
21
21
|
if (prop === "id") {
|
|
22
|
-
return params.item.
|
|
22
|
+
return params.item.chainId;
|
|
23
23
|
}
|
|
24
24
|
let isValidContract = ChainMap.values(params.config.chainMap).some(chain => chain.contracts.some(c => c.name === prop));
|
|
25
25
|
if (isValidContract) {
|
package/src/Core.res
CHANGED
package/src/CrossChainState.res
CHANGED
|
@@ -7,7 +7,7 @@ type t = {
|
|
|
7
7
|
chainStates: dict<ChainState.t>,
|
|
8
8
|
// Chain ids in a stable order, so the cross-chain loops iterate the chains
|
|
9
9
|
// without allocating a values array on every tick.
|
|
10
|
-
chainIds: array<
|
|
10
|
+
chainIds: array<ChainId.t>,
|
|
11
11
|
// True once every chain has caught up to head/endBlock. Monotonic during a run.
|
|
12
12
|
mutable isRealtime: bool,
|
|
13
13
|
// True once every chain has caught up and there's nothing left to process,
|
|
@@ -45,7 +45,7 @@ let make = (
|
|
|
45
45
|
// Resolve a chain's state by id. The id always comes from `chainIds`, which is
|
|
46
46
|
// derived from `chainStates`, so the entry is guaranteed present.
|
|
47
47
|
let getChainState = (crossChainState: t, chainId) =>
|
|
48
|
-
crossChainState.chainStates->
|
|
48
|
+
crossChainState.chainStates->ChainId.Dict.dangerouslyGetNonOption(chainId)->Option.getUnsafe
|
|
49
49
|
|
|
50
50
|
// --- Accessors. ---
|
|
51
51
|
|
|
@@ -167,10 +167,57 @@ let applyBatchProgress = (crossChainState: t, ~batch: Batch.t, ~blockTimestampNa
|
|
|
167
167
|
crossChainState.isRealtime = crossChainState.isRealtime || allChainsReady.contents
|
|
168
168
|
}
|
|
169
169
|
|
|
170
|
-
//
|
|
171
|
-
//
|
|
172
|
-
//
|
|
173
|
-
|
|
170
|
+
// Every chain has buffered up to its head (or endblock) with nothing
|
|
171
|
+
// processable left. Derived from current state rather than from the flag a
|
|
172
|
+
// progressed batch sets, because a run that reached the head and died before
|
|
173
|
+
// finalizing resumes with no batch to process — nothing would ever set it.
|
|
174
|
+
let isSettledAtHead = (crossChainState: t) => {
|
|
175
|
+
let settled = ref(crossChainState->nextItemIsNone)
|
|
176
|
+
for i in 0 to crossChainState.chainIds->Array.length - 1 {
|
|
177
|
+
if (
|
|
178
|
+
!(
|
|
179
|
+
crossChainState
|
|
180
|
+
->getChainState(crossChainState.chainIds->Array.getUnsafe(i))
|
|
181
|
+
->ChainState.isFetchingAtHead
|
|
182
|
+
)
|
|
183
|
+
) {
|
|
184
|
+
settled := false
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
settled.contents
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
// Enter the FinalizingIndexes phase without a batch, for the resume above.
|
|
191
|
+
let markCaughtUpIfSettled = (crossChainState: t) =>
|
|
192
|
+
if crossChainState->isSettledAtHead {
|
|
193
|
+
crossChainState.isCaughtUp = true
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
// The same resume, decided from persisted values at construction instead of from
|
|
197
|
+
// the fetch frontier. A run that reached the head and died before finalizing has
|
|
198
|
+
// no batch to process on resume, and by the time the first height lands the head
|
|
199
|
+
// may have moved on — at which point no chain looks at head any more and the
|
|
200
|
+
// indexes it still owes would wait out another whole backfill. Deciding here,
|
|
201
|
+
// before any source request, keeps that debt tied to the progress that was
|
|
202
|
+
// actually committed. Skipped once every chain carries `ready_at`: those indexes
|
|
203
|
+
// were committed together with the stamps.
|
|
204
|
+
let markCaughtUpOnResume = (crossChainState: t) => {
|
|
205
|
+
let everyChainCaughtUp = ref(crossChainState.chainIds->Array.length > 0)
|
|
206
|
+
let everyChainReady = ref(true)
|
|
207
|
+
for i in 0 to crossChainState.chainIds->Array.length - 1 {
|
|
208
|
+
let cs = crossChainState->getChainState(crossChainState.chainIds->Array.getUnsafe(i))
|
|
209
|
+
if !(cs->ChainState.isDurablyCaughtUp) {
|
|
210
|
+
everyChainCaughtUp := false
|
|
211
|
+
}
|
|
212
|
+
if !(cs->ChainState.isReady) {
|
|
213
|
+
everyChainReady := false
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
if everyChainCaughtUp.contents && !everyChainReady.contents && crossChainState->nextItemIsNone {
|
|
218
|
+
crossChainState.isCaughtUp = true
|
|
219
|
+
}
|
|
220
|
+
}
|
|
174
221
|
|
|
175
222
|
// Concludes the FinalizingIndexes phase: stamps every chain with the `ready_at`
|
|
176
223
|
// already committed alongside the deferred schema indexes and switches the
|
|
@@ -249,7 +296,7 @@ let idleOrWaitAction = (cs: ChainState.t) =>
|
|
|
249
296
|
// dropped — chains at head only trail each other by real-time block production.
|
|
250
297
|
let checkAndFetch = async (
|
|
251
298
|
crossChainState: t,
|
|
252
|
-
~dispatchChain: (~
|
|
299
|
+
~dispatchChain: (~chainId: ChainId.t, ~action: FetchState.nextQuery) => promise<unit>,
|
|
253
300
|
) => {
|
|
254
301
|
let targetBudget = crossChainState.targetBufferSize->Int.toFloat
|
|
255
302
|
let remaining = ref(
|
|
@@ -297,14 +344,14 @@ let checkAndFetch = async (
|
|
|
297
344
|
// hold the whole pool. (The general can't-fetch-yet rule, including
|
|
298
345
|
// blockLag, lives in FetchState.getNextQuery — this branch only
|
|
299
346
|
// short-circuits the unambiguous no-height case.)
|
|
300
|
-
actionByChain->
|
|
347
|
+
actionByChain->ChainId.Dict.set(chainId, FetchState.WaitingForNewBlock)
|
|
301
348
|
} else if remaining.contents < minimumAdmissionBudget {
|
|
302
349
|
// More than 90% of the target pool is ready or reserved. Don't admit new
|
|
303
350
|
// queries until a full admission unit becomes free. No wake-up poll is
|
|
304
351
|
// needed: a saturated pool means some chain holds ready items or
|
|
305
352
|
// in-flight reservations, so a batch completion or landing response is
|
|
306
353
|
// guaranteed to schedule another tick that revisits this chain.
|
|
307
|
-
actionByChain->
|
|
354
|
+
actionByChain->ChainId.Dict.set(chainId, FetchState.NothingToQuery)
|
|
308
355
|
} else {
|
|
309
356
|
let isCold = cs->ChainState.effectiveDensity === None
|
|
310
357
|
let chainTargetItems =
|
|
@@ -319,12 +366,12 @@ let checkAndFetch = async (
|
|
|
319
366
|
| _ => None
|
|
320
367
|
}
|
|
321
368
|
switch cs->ChainState.getNextQuery(~chainTargetItems, ~maxTargetBlock?) {
|
|
322
|
-
| WaitingForNewBlock as action => actionByChain->
|
|
369
|
+
| WaitingForNewBlock as action => actionByChain->ChainId.Dict.set(chainId, action)
|
|
323
370
|
| NothingToQuery =>
|
|
324
371
|
// A chain below its head can emit no query when its budget went to
|
|
325
372
|
// more-behind chains or the cross-chain alignment clamped its range to
|
|
326
373
|
// nothing — idleOrWaitAction keeps it polling for new blocks.
|
|
327
|
-
actionByChain->
|
|
374
|
+
actionByChain->ChainId.Dict.set(chainId, idleOrWaitAction(cs))
|
|
328
375
|
| Ready(queries) => {
|
|
329
376
|
let consumed =
|
|
330
377
|
queries->Array.reduce(0., (acc, query: FetchState.query) =>
|
|
@@ -348,7 +395,7 @@ let checkAndFetch = async (
|
|
|
348
395
|
"partitions": partitions,
|
|
349
396
|
})
|
|
350
397
|
|
|
351
|
-
actionByChain->
|
|
398
|
+
actionByChain->ChainId.Dict.set(chainId, FetchState.Ready(queries))
|
|
352
399
|
// Mark the queries in flight and reserve their size against the
|
|
353
400
|
// shared budget; released as each response lands in
|
|
354
401
|
// handleQueryResult.
|
|
@@ -362,12 +409,10 @@ let checkAndFetch = async (
|
|
|
362
409
|
let promises = []
|
|
363
410
|
for i in 0 to crossChainState.chainIds->Array.length - 1 {
|
|
364
411
|
let chainId = crossChainState.chainIds->Array.getUnsafe(i)
|
|
365
|
-
switch actionByChain->
|
|
412
|
+
switch actionByChain->ChainId.Dict.dangerouslyGetNonOption(chainId) {
|
|
366
413
|
| Some(NothingToQuery)
|
|
367
414
|
| None => ()
|
|
368
|
-
| Some(action) =>
|
|
369
|
-
let chain = ChainMap.Chain.makeUnsafe(~chainId)
|
|
370
|
-
promises->Array.push(dispatchChain(~chain, ~action))
|
|
415
|
+
| Some(action) => promises->Array.push(dispatchChain(~chainId=chainId, ~action))
|
|
371
416
|
}
|
|
372
417
|
}
|
|
373
418
|
let _ = await promises->Promise.all
|
|
@@ -4,7 +4,6 @@ import * as Env from "./Env.res.mjs";
|
|
|
4
4
|
import * as Batch from "./Batch.res.mjs";
|
|
5
5
|
import * as Utils from "./Utils.res.mjs";
|
|
6
6
|
import * as Logging from "./Logging.res.mjs";
|
|
7
|
-
import * as ChainMap from "./ChainMap.res.mjs";
|
|
8
7
|
import * as ChainState from "./ChainState.res.mjs";
|
|
9
8
|
import * as Stdlib_Array from "@rescript/runtime/lib/es6/Stdlib_Array.js";
|
|
10
9
|
import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
|
|
@@ -119,8 +118,39 @@ function applyBatchProgress(crossChainState, batch, blockTimestampName) {
|
|
|
119
118
|
crossChainState.isRealtime = crossChainState.isRealtime || allChainsReady;
|
|
120
119
|
}
|
|
121
120
|
|
|
122
|
-
function
|
|
123
|
-
|
|
121
|
+
function isSettledAtHead(crossChainState) {
|
|
122
|
+
let settled = nextItemIsNone(crossChainState);
|
|
123
|
+
for (let i = 0, i_finish = crossChainState.chainIds.length; i < i_finish; ++i) {
|
|
124
|
+
if (!ChainState.isFetchingAtHead(crossChainState.chainStates[crossChainState.chainIds[i]])) {
|
|
125
|
+
settled = false;
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
return settled;
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
function markCaughtUpIfSettled(crossChainState) {
|
|
132
|
+
if (isSettledAtHead(crossChainState)) {
|
|
133
|
+
crossChainState.isCaughtUp = true;
|
|
134
|
+
return;
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
function markCaughtUpOnResume(crossChainState) {
|
|
139
|
+
let everyChainCaughtUp = crossChainState.chainIds.length !== 0;
|
|
140
|
+
let everyChainReady = true;
|
|
141
|
+
for (let i = 0, i_finish = crossChainState.chainIds.length; i < i_finish; ++i) {
|
|
142
|
+
let cs = crossChainState.chainStates[crossChainState.chainIds[i]];
|
|
143
|
+
if (!ChainState.isDurablyCaughtUp(cs)) {
|
|
144
|
+
everyChainCaughtUp = false;
|
|
145
|
+
}
|
|
146
|
+
if (!ChainState.isReady(cs)) {
|
|
147
|
+
everyChainReady = false;
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
if (everyChainCaughtUp && !everyChainReady && nextItemIsNone(crossChainState)) {
|
|
151
|
+
crossChainState.isCaughtUp = true;
|
|
152
|
+
return;
|
|
153
|
+
}
|
|
124
154
|
}
|
|
125
155
|
|
|
126
156
|
function markReady(crossChainState, readyAt) {
|
|
@@ -215,15 +245,8 @@ async function checkAndFetch(crossChainState, dispatchChain) {
|
|
|
215
245
|
for (let i = 0, i_finish = crossChainState.chainIds.length; i < i_finish; ++i) {
|
|
216
246
|
let chainId = crossChainState.chainIds[i];
|
|
217
247
|
let action = actionByChain[chainId];
|
|
218
|
-
if (action !== undefined) {
|
|
219
|
-
|
|
220
|
-
if (typeof action === "object" || action === "WaitingForNewBlock") {
|
|
221
|
-
exit = 1;
|
|
222
|
-
}
|
|
223
|
-
if (exit === 1) {
|
|
224
|
-
let chain = ChainMap.Chain.makeUnsafe(chainId);
|
|
225
|
-
promises.push(dispatchChain(chain, action));
|
|
226
|
-
}
|
|
248
|
+
if (action !== undefined && (typeof action === "object" || action === "WaitingForNewBlock")) {
|
|
249
|
+
promises.push(dispatchChain(chainId, action));
|
|
227
250
|
}
|
|
228
251
|
}
|
|
229
252
|
await Promise.all(promises);
|
|
@@ -242,7 +265,8 @@ export {
|
|
|
242
265
|
createBatch,
|
|
243
266
|
enterReorgThreshold,
|
|
244
267
|
applyBatchProgress,
|
|
245
|
-
|
|
268
|
+
markCaughtUpIfSettled,
|
|
269
|
+
markCaughtUpOnResume,
|
|
246
270
|
markReady,
|
|
247
271
|
priorityOrder,
|
|
248
272
|
checkAndFetch,
|
package/src/CrossChainState.resi
CHANGED
|
@@ -32,12 +32,13 @@ let createBatch: (
|
|
|
32
32
|
) => Batch.t
|
|
33
33
|
let enterReorgThreshold: t => unit
|
|
34
34
|
let applyBatchProgress: (t, ~batch: Batch.t, ~blockTimestampName: string) => unit
|
|
35
|
-
let
|
|
35
|
+
let markCaughtUpIfSettled: t => unit
|
|
36
|
+
let markCaughtUpOnResume: t => unit
|
|
36
37
|
let markReady: (t, ~readyAt: Date.t) => unit
|
|
37
38
|
|
|
38
39
|
// Fetch control.
|
|
39
40
|
let priorityOrder: t => array<ChainState.t>
|
|
40
41
|
let checkAndFetch: (
|
|
41
42
|
t,
|
|
42
|
-
~dispatchChain: (~
|
|
43
|
+
~dispatchChain: (~chainId: ChainId.t, ~action: FetchState.nextQuery) => promise<unit>,
|
|
43
44
|
) => promise<unit>
|
package/src/Env.res
CHANGED
|
@@ -3,9 +3,6 @@
|
|
|
3
3
|
|
|
4
4
|
%%private(let envSafe = EnvSafe.make())
|
|
5
5
|
|
|
6
|
-
// resets the timestampCaughtUpToHeadOrEndblock after a restart when true
|
|
7
|
-
let updateSyncTimeOnRestart =
|
|
8
|
-
envSafe->EnvSafe.get("UPDATE_SYNC_TIME_ON_RESTART", S.bool, ~fallback=true)
|
|
9
6
|
let targetBufferSize = envSafe->EnvSafe.get("ENVIO_INDEXING_MAX_BUFFER_SIZE", S.option(S.int))
|
|
10
7
|
let maxAddrInPartition = envSafe->EnvSafe.get("MAX_PARTITION_SIZE", S.int, ~fallback=5_000)
|
|
11
8
|
|
package/src/Env.res.mjs
CHANGED
|
@@ -13,8 +13,6 @@ import 'dotenv/config'
|
|
|
13
13
|
|
|
14
14
|
let envSafe = EnvSafe.make(undefined);
|
|
15
15
|
|
|
16
|
-
let updateSyncTimeOnRestart = EnvSafe.get(envSafe, "UPDATE_SYNC_TIME_ON_RESTART", S$RescriptSchema.bool, undefined, true, undefined, undefined);
|
|
17
|
-
|
|
18
16
|
let targetBufferSize = EnvSafe.get(envSafe, "ENVIO_INDEXING_MAX_BUFFER_SIZE", S$RescriptSchema.option(S$RescriptSchema.int), undefined, undefined, undefined, undefined);
|
|
19
17
|
|
|
20
18
|
let maxAddrInPartition = EnvSafe.get(envSafe, "MAX_PARTITION_SIZE", S$RescriptSchema.int, undefined, 5000, undefined, undefined);
|
|
@@ -221,7 +219,6 @@ EnvSafe.close(envSafe);
|
|
|
221
219
|
let maxChainConcurrency = 100;
|
|
222
220
|
|
|
223
221
|
export {
|
|
224
|
-
updateSyncTimeOnRestart,
|
|
225
222
|
targetBufferSize,
|
|
226
223
|
maxAddrInPartition,
|
|
227
224
|
maxChainConcurrency,
|
|
@@ -342,7 +342,7 @@ let buildEvmOnEventRegistration = (
|
|
|
342
342
|
~handler: option<Internal.handler>,
|
|
343
343
|
~contractRegister: option<Internal.contractRegister>,
|
|
344
344
|
~where: option<JSON.t>,
|
|
345
|
-
~chainId:
|
|
345
|
+
~chainId: ChainId.t,
|
|
346
346
|
~onEventBlockFilterSchema: S.t<option<unknown>>,
|
|
347
347
|
~startBlock: option<int>=?,
|
|
348
348
|
): Internal.evmOnEventRegistration => {
|
package/src/EventProcessing.res
CHANGED
|
@@ -13,7 +13,7 @@ let computeChainsState = (chainStates: dict<ChainState.t>): Internal.chains => {
|
|
|
13
13
|
values->Array.forEach(cs => {
|
|
14
14
|
let chainId = (cs->ChainState.chainConfig).id
|
|
15
15
|
chains->Dict.set(
|
|
16
|
-
chainId->
|
|
16
|
+
chainId->ChainId.toString,
|
|
17
17
|
{
|
|
18
18
|
Internal.id: chainId,
|
|
19
19
|
isRealtime,
|
|
@@ -280,7 +280,7 @@ let registerProcessEventBatchMetrics = (
|
|
|
280
280
|
batch.progressedChainsById->Dict.forEachWithKey((chainAfterBatch, chainId) => {
|
|
281
281
|
logger->Logging.childTrace({
|
|
282
282
|
"msg": "Finished processing",
|
|
283
|
-
"chainId": chainId
|
|
283
|
+
"chainId": chainId,
|
|
284
284
|
"batchSize": chainAfterBatch.batchSize,
|
|
285
285
|
"progress": chainAfterBatch.progressBlockNumber,
|
|
286
286
|
})
|
|
@@ -312,7 +312,7 @@ let materializeBatchEvents = async (
|
|
|
312
312
|
| _ =>
|
|
313
313
|
let itemsByChain: dict<array<Internal.item>> = Dict.make()
|
|
314
314
|
batch.items->Array.forEach(item => {
|
|
315
|
-
let chainId = item->Internal.getItemChainId->
|
|
315
|
+
let chainId = item->Internal.getItemChainId->ChainId.toString
|
|
316
316
|
switch itemsByChain->Utils.Dict.dangerouslyGetNonOption(chainId) {
|
|
317
317
|
| Some(items) => items->Array.push(item)
|
|
318
318
|
| None => itemsByChain->Dict.set(chainId, [item])
|
|
@@ -345,7 +345,7 @@ let processEventBatch = async (
|
|
|
345
345
|
batch.progressedChainsById->Dict.forEachWithKey((chainAfterBatch, chainId) => {
|
|
346
346
|
logger->Logging.childTrace({
|
|
347
347
|
"msg": "Started processing",
|
|
348
|
-
"chainId": chainId
|
|
348
|
+
"chainId": chainId,
|
|
349
349
|
"batchSize": chainAfterBatch.batchSize,
|
|
350
350
|
})
|
|
351
351
|
})
|
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
// Generated by ReScript, PLEASE EDIT WITH CARE
|
|
2
2
|
|
|
3
3
|
import * as Utils from "./Utils.res.mjs";
|
|
4
|
+
import * as ChainId from "./ChainId.res.mjs";
|
|
4
5
|
import * as Logging from "./Logging.res.mjs";
|
|
5
6
|
import * as Writing from "./Writing.res.mjs";
|
|
6
7
|
import * as Internal from "./Internal.res.mjs";
|
|
7
8
|
import * as Ecosystem from "./Ecosystem.res.mjs";
|
|
8
9
|
import * as ChainState from "./ChainState.res.mjs";
|
|
9
|
-
import * as Stdlib_Int from "@rescript/runtime/lib/es6/Stdlib_Int.js";
|
|
10
10
|
import * as Performance from "./bindings/Performance.res.mjs";
|
|
11
11
|
import * as Persistence from "./Persistence.res.mjs";
|
|
12
12
|
import * as Stdlib_Dict from "@rescript/runtime/lib/es6/Stdlib_Dict.js";
|
|
@@ -25,7 +25,7 @@ function computeChainsState(chainStates) {
|
|
|
25
25
|
let isRealtime = values.every(ChainState.isReady);
|
|
26
26
|
values.forEach(cs => {
|
|
27
27
|
let chainId = ChainState.chainConfig(cs).id;
|
|
28
|
-
chains[
|
|
28
|
+
chains[ChainId.toString(chainId)] = {
|
|
29
29
|
id: chainId,
|
|
30
30
|
isRealtime: isRealtime
|
|
31
31
|
};
|
|
@@ -179,7 +179,7 @@ async function runBatchHandlersOrThrow(batch, indexerState, loadManager, persist
|
|
|
179
179
|
function registerProcessEventBatchMetrics(logger, batch, indexerState, loadDuration, handlerDuration) {
|
|
180
180
|
Stdlib_Dict.forEachWithKey(batch.progressedChainsById, (chainAfterBatch, chainId) => Logging.childTrace(logger, {
|
|
181
181
|
msg: "Finished processing",
|
|
182
|
-
chainId:
|
|
182
|
+
chainId: chainId,
|
|
183
183
|
batchSize: chainAfterBatch.batchSize,
|
|
184
184
|
progress: chainAfterBatch.progressBlockNumber
|
|
185
185
|
}));
|
|
@@ -191,7 +191,7 @@ async function materializeBatchEvents(batch, chainStates, ecosystem) {
|
|
|
191
191
|
if (match.length !== 1) {
|
|
192
192
|
let itemsByChain = {};
|
|
193
193
|
batch.items.forEach(item => {
|
|
194
|
-
let chainId = Internal.getItemChainId(item)
|
|
194
|
+
let chainId = ChainId.toString(Internal.getItemChainId(item));
|
|
195
195
|
let items = itemsByChain[chainId];
|
|
196
196
|
if (items !== undefined) {
|
|
197
197
|
items.push(item);
|
|
@@ -214,7 +214,7 @@ async function processEventBatch(batch, indexerState, loadManager, persistence,
|
|
|
214
214
|
let logger = Logging.getLogger();
|
|
215
215
|
Stdlib_Dict.forEachWithKey(batch.progressedChainsById, (chainAfterBatch, chainId) => Logging.childTrace(logger, {
|
|
216
216
|
msg: "Started processing",
|
|
217
|
-
chainId:
|
|
217
|
+
chainId: chainId,
|
|
218
218
|
batchSize: chainAfterBatch.batchSize
|
|
219
219
|
}));
|
|
220
220
|
try {
|