envio 3.11.0 → 3.12.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.d.ts +3 -0
- package/package.json +6 -6
- package/src/Batch.res +111 -78
- package/src/Batch.res.mjs +53 -46
- package/src/BatchProcessing.res +1 -11
- package/src/BatchProcessing.res.mjs +1 -4
- package/src/ChainFetching.res +7 -3
- package/src/ChainFetching.res.mjs +1 -1
- package/src/ChainState.res +82 -20
- package/src/ChainState.res.mjs +51 -13
- package/src/ChainState.resi +6 -1
- package/src/Config.res +58 -37
- package/src/Config.res.mjs +63 -53
- package/src/CrossChainState.res +50 -53
- package/src/CrossChainState.res.mjs +25 -23
- package/src/CrossChainState.resi +6 -13
- package/src/EffectState.res +13 -4
- package/src/EffectState.resi +7 -1
- package/src/EventProcessing.res +6 -6
- package/src/EventProcessing.res.mjs +6 -6
- package/src/FetchState.res +65 -46
- package/src/FetchState.res.mjs +68 -36
- package/src/FinalizeBackfill.res +16 -12
- package/src/FinalizeBackfill.res.mjs +3 -3
- package/src/HistoryPolicy.res +46 -0
- package/src/HistoryPolicy.res.mjs +50 -0
- package/src/InMemoryStore.res +30 -32
- package/src/InMemoryStore.res.mjs +33 -28
- package/src/InMemoryTable.res +155 -67
- package/src/InMemoryTable.res.mjs +151 -60
- package/src/IndexerState.res +66 -45
- package/src/IndexerState.res.mjs +57 -29
- package/src/IndexerState.resi +8 -13
- package/src/Internal.res +4 -2
- package/src/LoadLayer.res +68 -38
- package/src/LoadLayer.res.mjs +50 -64
- package/src/LoadLayer.resi +1 -1
- package/src/Main.res +25 -24
- package/src/Main.res.mjs +24 -25
- package/src/Metrics.res +49 -1
- package/src/Metrics.res.mjs +26 -3
- package/src/Persistence.res +55 -16
- package/src/Persistence.res.mjs +18 -3
- package/src/PgStorage.res +286 -180
- package/src/PgStorage.res.mjs +209 -122
- package/src/PruneStaleHistory.res +9 -15
- package/src/PruneStaleHistory.res.mjs +11 -20
- package/src/Rollback.res +9 -14
- package/src/Rollback.res.mjs +5 -3
- package/src/Sink.res +7 -5
- package/src/Sink.res.mjs +4 -4
- package/src/Staging.res +298 -0
- package/src/Staging.res.mjs +257 -0
- package/src/TestIndexer.res +14 -30
- package/src/TestIndexer.res.mjs +11 -9
- package/src/UserContext.res +20 -36
- package/src/UserContext.res.mjs +6 -9
- package/src/Utils.res +10 -4
- package/src/Utils.res.mjs +16 -16
- package/src/Writing.res +34 -20
- package/src/Writing.res.mjs +15 -20
- package/src/bindings/ClickHouse.res +41 -34
- package/src/bindings/ClickHouse.res.mjs +36 -30
- package/src/bindings/ClickHouseSink.res +45 -186
- package/src/bindings/ClickHouseSink.res.mjs +21 -174
- package/src/db/CheckpointSequence.res +109 -0
- package/src/db/CheckpointSequence.res.mjs +99 -0
- package/src/db/EntityFilter.res +487 -275
- package/src/db/EntityFilter.res.mjs +557 -309
- package/src/db/EntityHistory.res +6 -5
- package/src/db/EntityHistory.res.mjs +5 -5
- package/src/db/Frontier.res +86 -0
- package/src/db/Frontier.res.mjs +126 -0
- package/src/db/InternalTable.res +145 -27
- package/src/db/InternalTable.res.mjs +115 -29
- package/src/db/RollbackFloors.res +34 -41
- package/src/db/RollbackFloors.res.mjs +27 -71
- package/src/db/Table.res +21 -6
- package/src/db/Table.res.mjs +13 -4
- package/src/sources/BlockStore.res +8 -0
- package/src/sources/EvmHyperSyncSource.res +5 -1
- package/src/sources/EvmHyperSyncSource.res.mjs +4 -3
- package/src/sources/FuelHyperSyncSource.res +1 -0
- package/src/sources/FuelHyperSyncSource.res.mjs +1 -1
- package/src/sources/HyperSync.res +6 -0
- package/src/sources/HyperSync.res.mjs +5 -2
- package/src/sources/HyperSync.resi +2 -0
- package/src/sources/HyperSyncClient.res +6 -0
- package/src/sources/HyperSyncSSE.res +1 -1
- package/src/sources/HyperSyncSSE.res.mjs +4 -10
- package/src/sources/RequestStat.res +8 -1
- package/src/sources/RpcSource.res +1 -0
- package/src/sources/RpcSource.res.mjs +1 -1
- package/src/sources/SimulateSource.res +1 -0
- package/src/sources/SimulateSource.res.mjs +1 -1
- package/src/sources/Source.res +11 -3
- package/src/sources/SourceManager.res +34 -13
- package/src/sources/SourceManager.res.mjs +26 -9
- package/src/sources/SourceManager.resi +2 -0
- package/src/sources/SvmHyperSyncClient.res +5 -0
- package/src/sources/SvmHyperSyncSource.res +3 -0
- package/src/sources/SvmHyperSyncSource.res.mjs +4 -2
- package/src/tui/Tui.res +56 -143
- package/src/tui/Tui.res.mjs +70 -166
- package/src/tui/components/TuiData.res +61 -16
- package/src/tui/components/TuiData.res.mjs +53 -15
- package/svm.schema.json +0 -7
- package/src/db/CheckpointBounds.res +0 -53
- package/src/db/CheckpointBounds.res.mjs +0 -44
package/src/Config.res
CHANGED
|
@@ -136,9 +136,17 @@ type t = {
|
|
|
136
136
|
reorgThresholdReadyTolerance: int,
|
|
137
137
|
lowercaseAddresses: bool,
|
|
138
138
|
isDev: bool,
|
|
139
|
+
// An `envio start --chain` process: drives a subset of the schema's chains
|
|
140
|
+
// while sibling processes drive the rest, so it only touches what its own
|
|
141
|
+
// chains own — their partitions' indexes, their `ready_at`, their resume.
|
|
142
|
+
isolated: bool,
|
|
139
143
|
userEntitiesByName: dict<Internal.entityConfig>,
|
|
140
144
|
userEntities: array<Internal.entityConfig>,
|
|
141
145
|
allEnums: array<Table.enumConfig<Table.enum>>,
|
|
146
|
+
// Whether checkpoint ids come from one counter or one per chain. Decided by
|
|
147
|
+
// the schema alone: a cross-chain entity has rows any chain's reorg can
|
|
148
|
+
// reach, so its checkpoints have to be comparable across chains.
|
|
149
|
+
checkpointSequence: CheckpointSequence.t,
|
|
142
150
|
}
|
|
143
151
|
|
|
144
152
|
type rpcSourceFor = | @as("sync") Sync | @as("fallback") Fallback | @as("realtime") Realtime
|
|
@@ -586,6 +594,7 @@ let publicConfigSchema = S.schema(s =>
|
|
|
586
594
|
"description": s.matches(S.option(S.string)),
|
|
587
595
|
"handlers": s.matches(S.option(S.string)),
|
|
588
596
|
"isDev": s.matches(S.option(S.bool)),
|
|
597
|
+
"isolatedChains": s.matches(S.option(S.array(ChainId.schema))),
|
|
589
598
|
"fullBatchSize": s.matches(S.option(S.int)),
|
|
590
599
|
"rollbackOnReorg": s.matches(S.option(S.bool)),
|
|
591
600
|
"saveFullHistory": s.matches(S.option(S.bool)),
|
|
@@ -609,6 +618,44 @@ let contractMappingOf = (~chainConfigs: array<chain>): ContractMapping.t => {
|
|
|
609
618
|
ContractMapping.make(~names)
|
|
610
619
|
}
|
|
611
620
|
|
|
621
|
+
let getChain = (config, ~chainId) =>
|
|
622
|
+
config.chainMap->ChainMap.has(chainId)
|
|
623
|
+
? chainId
|
|
624
|
+
: JsError.throwWithMessage(
|
|
625
|
+
"No chain with id " ++ chainId->ChainId.toString ++ " found in config.yaml",
|
|
626
|
+
)
|
|
627
|
+
|
|
628
|
+
// Narrows a config to the chains one `envio start --chain` process drives.
|
|
629
|
+
// `contractMapping` is deliberately left whole: its ids are what the migration
|
|
630
|
+
// that created the schema stored, and one rebuilt from a subset would hand the
|
|
631
|
+
// same contract a different id.
|
|
632
|
+
let isolate = (config: t, ~chainIds: array<ChainId.t>) => {
|
|
633
|
+
// Chains indexed in separate processes each advance their own checkpoint
|
|
634
|
+
// counter, which only holds while no entity has rows another chain can reach.
|
|
635
|
+
switch config.userEntities->Array.filter(entityConfig => entityConfig.crossChain) {
|
|
636
|
+
| [] => ()
|
|
637
|
+
| shared =>
|
|
638
|
+
JsError.throwWithMessage(
|
|
639
|
+
`Only a schema whose entities are all per-chain can be split across processes. Shared across chains: ${shared
|
|
640
|
+
->Array.map(entityConfig => entityConfig.name)
|
|
641
|
+
->Array.joinUnsafe(", ")}.`,
|
|
642
|
+
)
|
|
643
|
+
}
|
|
644
|
+
chainIds->Array.forEach(chainId => config->getChain(~chainId)->ignore)
|
|
645
|
+
|
|
646
|
+
// Filtered out of the config's own chain order rather than built from the
|
|
647
|
+
// argument order, so a repeated `--chain` collapses and `defaultChain` doesn't
|
|
648
|
+
// depend on how the flags were typed.
|
|
649
|
+
let chains =
|
|
650
|
+
config.chainMap->ChainMap.values->Array.filter(chain => chainIds->Array.includes(chain.id))
|
|
651
|
+
{
|
|
652
|
+
...config,
|
|
653
|
+
chainMap: chains->Array.map(chain => (chain.id, chain))->ChainMap.fromArrayUnsafe,
|
|
654
|
+
defaultChain: chains->Array.get(0),
|
|
655
|
+
isolated: true,
|
|
656
|
+
}
|
|
657
|
+
}
|
|
658
|
+
|
|
612
659
|
let fromPublic = (publicConfigJson: JSON.t) => {
|
|
613
660
|
let maxAddrInPartition = Env.maxAddrInPartition
|
|
614
661
|
// Parse public config
|
|
@@ -709,12 +756,7 @@ let fromPublic = (publicConfigJson: JSON.t) => {
|
|
|
709
756
|
}
|
|
710
757
|
let widened =
|
|
711
758
|
contractConfig->(
|
|
712
|
-
Utils.magic: _ => {
|
|
713
|
-
"svmAbi": option<{
|
|
714
|
-
"definedTypes": JSON.t,
|
|
715
|
-
"source": string,
|
|
716
|
-
}>,
|
|
717
|
-
}
|
|
759
|
+
Utils.magic: _ => {"svmAbi": option<{"definedTypes": JSON.t, "source": string}>}
|
|
718
760
|
)
|
|
719
761
|
contractDataByName->Dict.set(
|
|
720
762
|
capitalizedName,
|
|
@@ -1063,7 +1105,7 @@ let fromPublic = (publicConfigJson: JSON.t) => {
|
|
|
1063
1105
|
| None => []
|
|
1064
1106
|
}
|
|
1065
1107
|
|
|
1066
|
-
{
|
|
1108
|
+
let config = {
|
|
1067
1109
|
name: publicConfig["name"],
|
|
1068
1110
|
description: publicConfig["description"],
|
|
1069
1111
|
handlers: publicConfig["handlers"]->Option.getOr("src/handlers"),
|
|
@@ -1084,19 +1126,18 @@ let fromPublic = (publicConfigJson: JSON.t) => {
|
|
|
1084
1126
|
reorgThresholdReadyTolerance: 100,
|
|
1085
1127
|
lowercaseAddresses,
|
|
1086
1128
|
isDev: publicConfig["isDev"]->Option.getOr(false),
|
|
1129
|
+
isolated: false,
|
|
1087
1130
|
userEntitiesByName,
|
|
1088
1131
|
userEntities,
|
|
1089
1132
|
allEnums,
|
|
1133
|
+
checkpointSequence: CheckpointSequence.fromEntities(userEntities),
|
|
1090
1134
|
}
|
|
1091
|
-
}
|
|
1092
1135
|
|
|
1093
|
-
|
|
1094
|
-
|
|
1095
|
-
|
|
1096
|
-
|
|
1097
|
-
|
|
1098
|
-
config.chainMap->ChainMap.keys->Array.length > 1 &&
|
|
1099
|
-
config.userEntities->Array.every(entityConfig => !entityConfig.crossChain)
|
|
1136
|
+
switch publicConfig["isolatedChains"] {
|
|
1137
|
+
| None => config
|
|
1138
|
+
| Some(chainIds) => config->isolate(~chainIds)
|
|
1139
|
+
}
|
|
1140
|
+
}
|
|
1100
1141
|
|
|
1101
1142
|
// Canonicalize a user-provided address to the configured casing so it matches
|
|
1102
1143
|
// addresses parsed from config.yaml during routing. HyperSync/RPC data arrives
|
|
@@ -1169,33 +1210,12 @@ let getEventConfig = (config: t, ~contractName, ~eventName, ~chainId: option<Cha
|
|
|
1169
1210
|
})
|
|
1170
1211
|
}
|
|
1171
1212
|
|
|
1172
|
-
// A chain that can't be rolled back (maxReorgDepth = 0) has no history to keep,
|
|
1173
|
-
// unless a cross-chain entity lets another chain's rollback reach its rows.
|
|
1174
|
-
let shouldSaveHistory = (config, ~isInReorgThreshold, ~chainId: option<ChainId.t>=?) =>
|
|
1175
|
-
config.shouldSaveFullHistory ||
|
|
1176
|
-
(config.shouldRollbackOnReorg &&
|
|
1177
|
-
isInReorgThreshold &&
|
|
1178
|
-
switch chainId {
|
|
1179
|
-
| Some(chainId) if config->isIsolatedMultichain =>
|
|
1180
|
-
(config.chainMap->ChainMap.get(chainId)).maxReorgDepth > 0
|
|
1181
|
-
| _ => true
|
|
1182
|
-
})
|
|
1183
|
-
|
|
1184
|
-
let shouldPruneHistory = (config, ~isInReorgThreshold) =>
|
|
1185
|
-
!config.shouldSaveFullHistory && (config.shouldRollbackOnReorg && isInReorgThreshold)
|
|
1186
|
-
|
|
1187
|
-
let getChain = (config, ~chainId) =>
|
|
1188
|
-
config.chainMap->ChainMap.has(chainId)
|
|
1189
|
-
? chainId
|
|
1190
|
-
: JsError.throwWithMessage(
|
|
1191
|
-
"No chain with id " ++ chainId->ChainId.toString ++ " found in config.yaml",
|
|
1192
|
-
)
|
|
1193
|
-
|
|
1194
1213
|
// A CLI command payload already contains the resolved JSON; priming lets
|
|
1195
1214
|
// downstream callers skip the NAPI `getConfigJson` round-trip. Calling
|
|
1196
1215
|
// `prime` again invalidates the memo.
|
|
1197
1216
|
%%private(let primedJson: ref<option<JSON.t>> = ref(None))
|
|
1198
1217
|
%%private(let cached: ref<option<t>> = ref(None))
|
|
1218
|
+
|
|
1199
1219
|
let prime = (json: JSON.t): unit => {
|
|
1200
1220
|
primedJson := Some(json)
|
|
1201
1221
|
cached := None
|
|
@@ -1236,6 +1256,7 @@ let stripSensitiveData = (json: JSON.t): JSON.t => {
|
|
|
1236
1256
|
switch cloned {
|
|
1237
1257
|
| Object(obj) => {
|
|
1238
1258
|
obj->Utils.Dict.deleteInPlace("isDev")
|
|
1259
|
+
obj->Utils.Dict.deleteInPlace("isolatedChains")
|
|
1239
1260
|
stripChains(obj->Dict.get("evm"))
|
|
1240
1261
|
stripChains(obj->Dict.get("fuel"))
|
|
1241
1262
|
stripChains(obj->Dict.get("svm"))
|
package/src/Config.res.mjs
CHANGED
|
@@ -21,6 +21,7 @@ import * as Primitive_object from "@rescript/runtime/lib/es6/Primitive_object.js
|
|
|
21
21
|
import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js";
|
|
22
22
|
import * as Primitive_string from "@rescript/runtime/lib/es6/Primitive_string.js";
|
|
23
23
|
import * as S$RescriptSchema from "rescript-schema/src/S.res.mjs";
|
|
24
|
+
import * as CheckpointSequence from "./db/CheckpointSequence.res.mjs";
|
|
24
25
|
import * as EventConfigBuilder from "./EventConfigBuilder.res.mjs";
|
|
25
26
|
import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
|
|
26
27
|
|
|
@@ -436,6 +437,7 @@ let publicConfigSchema = S$RescriptSchema.schema(s => ({
|
|
|
436
437
|
description: s.m(S$RescriptSchema.option(S$RescriptSchema.string)),
|
|
437
438
|
handlers: s.m(S$RescriptSchema.option(S$RescriptSchema.string)),
|
|
438
439
|
isDev: s.m(S$RescriptSchema.option(S$RescriptSchema.bool)),
|
|
440
|
+
isolatedChains: s.m(S$RescriptSchema.option(S$RescriptSchema.array(ChainId.schema))),
|
|
439
441
|
fullBatchSize: s.m(S$RescriptSchema.option(S$RescriptSchema.int)),
|
|
440
442
|
rollbackOnReorg: s.m(S$RescriptSchema.option(S$RescriptSchema.bool)),
|
|
441
443
|
saveFullHistory: s.m(S$RescriptSchema.option(S$RescriptSchema.bool)),
|
|
@@ -460,6 +462,33 @@ function contractMappingOf(chainConfigs) {
|
|
|
460
462
|
return ContractMapping.make(names);
|
|
461
463
|
}
|
|
462
464
|
|
|
465
|
+
function getChain(config, chainId) {
|
|
466
|
+
if (ChainMap.has(config.chainMap, chainId)) {
|
|
467
|
+
return chainId;
|
|
468
|
+
} else {
|
|
469
|
+
return Stdlib_JsError.throwWithMessage("No chain with id " + ChainId.toString(chainId) + " found in config.yaml");
|
|
470
|
+
}
|
|
471
|
+
}
|
|
472
|
+
|
|
473
|
+
function isolate(config, chainIds) {
|
|
474
|
+
let shared = config.userEntities.filter(entityConfig => entityConfig.crossChain);
|
|
475
|
+
if (shared.length !== 0) {
|
|
476
|
+
Stdlib_JsError.throwWithMessage(`Only a schema whose entities are all per-chain can be split across processes. Shared across chains: ` + shared.map(entityConfig => entityConfig.name).join(", ") + `.`);
|
|
477
|
+
}
|
|
478
|
+
chainIds.forEach(chainId => {
|
|
479
|
+
getChain(config, chainId);
|
|
480
|
+
});
|
|
481
|
+
let chains = ChainMap.values(config.chainMap).filter(chain => chainIds.includes(chain.id));
|
|
482
|
+
let newrecord = {...config};
|
|
483
|
+
newrecord.isolated = true;
|
|
484
|
+
newrecord.defaultChain = chains[0];
|
|
485
|
+
newrecord.chainMap = ChainMap.fromArrayUnsafe(chains.map(chain => [
|
|
486
|
+
chain.id,
|
|
487
|
+
chain
|
|
488
|
+
]));
|
|
489
|
+
return newrecord;
|
|
490
|
+
}
|
|
491
|
+
|
|
463
492
|
function fromPublic(publicConfigJson) {
|
|
464
493
|
let publicConfig;
|
|
465
494
|
try {
|
|
@@ -755,38 +784,50 @@ function fromPublic(publicConfigJson) {
|
|
|
755
784
|
name: Utils.$$String.capitalize(param[0]),
|
|
756
785
|
handler: param[1].handler
|
|
757
786
|
})) : [];
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
787
|
+
let config_name = publicConfig.name;
|
|
788
|
+
let config_description = publicConfig.description;
|
|
789
|
+
let config_handlers = Stdlib_Option.getOr(publicConfig.handlers, "src/handlers");
|
|
790
|
+
let config_shouldRollbackOnReorg = Stdlib_Option.getOr(publicConfig.rollbackOnReorg, true);
|
|
791
|
+
let config_shouldSaveFullHistory = Stdlib_Option.getOr(publicConfig.saveFullHistory, false);
|
|
792
|
+
let config_chainIdMode = Stdlib_Option.getOr(publicConfig.chainIdMode, "int32");
|
|
793
|
+
let config_contractMapping = contractMappingOf(chains);
|
|
794
|
+
let config_defaultChain = chains[0];
|
|
795
|
+
let config_enableRawEvents = Stdlib_Option.getOr(publicConfig.rawEvents, false);
|
|
796
|
+
let config_batchSize = Stdlib_Option.getOr(publicConfig.fullBatchSize, 5000);
|
|
797
|
+
let config_isDev = Stdlib_Option.getOr(publicConfig.isDev, false);
|
|
798
|
+
let config_checkpointSequence = CheckpointSequence.fromEntities(userEntities);
|
|
799
|
+
let config = {
|
|
800
|
+
name: config_name,
|
|
801
|
+
description: config_description,
|
|
802
|
+
handlers: config_handlers,
|
|
762
803
|
contractHandlers: contractHandlers,
|
|
763
|
-
shouldRollbackOnReorg:
|
|
764
|
-
shouldSaveFullHistory:
|
|
804
|
+
shouldRollbackOnReorg: config_shouldRollbackOnReorg,
|
|
805
|
+
shouldSaveFullHistory: config_shouldSaveFullHistory,
|
|
765
806
|
defaultCrossChain: defaultCrossChain,
|
|
766
807
|
storage: globalStorage,
|
|
767
|
-
chainIdMode:
|
|
808
|
+
chainIdMode: config_chainIdMode,
|
|
768
809
|
chainMap: chainMap,
|
|
769
|
-
contractMapping:
|
|
770
|
-
defaultChain:
|
|
810
|
+
contractMapping: config_contractMapping,
|
|
811
|
+
defaultChain: config_defaultChain,
|
|
771
812
|
ecosystem: ecosystem,
|
|
772
|
-
enableRawEvents:
|
|
813
|
+
enableRawEvents: config_enableRawEvents,
|
|
773
814
|
maxAddrInPartition: Env.maxAddrInPartition,
|
|
774
815
|
clientFilterAddressThreshold: Env.clientFilterAddressThreshold,
|
|
775
|
-
batchSize:
|
|
816
|
+
batchSize: config_batchSize,
|
|
776
817
|
reorgThresholdReadyTolerance: 100,
|
|
777
818
|
lowercaseAddresses: lowercaseAddresses,
|
|
778
|
-
isDev:
|
|
819
|
+
isDev: config_isDev,
|
|
820
|
+
isolated: false,
|
|
779
821
|
userEntitiesByName: userEntitiesByName,
|
|
780
822
|
userEntities: userEntities,
|
|
781
|
-
allEnums: allEnums
|
|
823
|
+
allEnums: allEnums,
|
|
824
|
+
checkpointSequence: config_checkpointSequence
|
|
782
825
|
};
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
if (ChainMap.keys(config.chainMap).length > 1) {
|
|
787
|
-
return config.userEntities.every(entityConfig => !entityConfig.crossChain);
|
|
826
|
+
let chainIds = publicConfig.isolatedChains;
|
|
827
|
+
if (chainIds !== undefined) {
|
|
828
|
+
return isolate(config, chainIds);
|
|
788
829
|
} else {
|
|
789
|
-
return
|
|
830
|
+
return config;
|
|
790
831
|
}
|
|
791
832
|
}
|
|
792
833
|
|
|
@@ -853,36 +894,6 @@ function getEventConfig(config, contractName, eventName, chainId) {
|
|
|
853
894
|
});
|
|
854
895
|
}
|
|
855
896
|
|
|
856
|
-
function shouldSaveHistory(config, isInReorgThreshold, chainId) {
|
|
857
|
-
if (config.shouldSaveFullHistory) {
|
|
858
|
-
return true;
|
|
859
|
-
} else if (config.shouldRollbackOnReorg && isInReorgThreshold) {
|
|
860
|
-
if (chainId !== undefined && isIsolatedMultichain(config)) {
|
|
861
|
-
return ChainMap.get(config.chainMap, Primitive_option.valFromOption(chainId)).maxReorgDepth > 0;
|
|
862
|
-
} else {
|
|
863
|
-
return true;
|
|
864
|
-
}
|
|
865
|
-
} else {
|
|
866
|
-
return false;
|
|
867
|
-
}
|
|
868
|
-
}
|
|
869
|
-
|
|
870
|
-
function shouldPruneHistory(config, isInReorgThreshold) {
|
|
871
|
-
if (!config.shouldSaveFullHistory && config.shouldRollbackOnReorg) {
|
|
872
|
-
return isInReorgThreshold;
|
|
873
|
-
} else {
|
|
874
|
-
return false;
|
|
875
|
-
}
|
|
876
|
-
}
|
|
877
|
-
|
|
878
|
-
function getChain(config, chainId) {
|
|
879
|
-
if (ChainMap.has(config.chainMap, chainId)) {
|
|
880
|
-
return chainId;
|
|
881
|
-
} else {
|
|
882
|
-
return Stdlib_JsError.throwWithMessage("No chain with id " + ChainId.toString(chainId) + " found in config.yaml");
|
|
883
|
-
}
|
|
884
|
-
}
|
|
885
|
-
|
|
886
897
|
let primedJson = {
|
|
887
898
|
contents: undefined
|
|
888
899
|
};
|
|
@@ -931,6 +942,7 @@ function stripSensitiveData(json) {
|
|
|
931
942
|
};
|
|
932
943
|
if (typeof cloned === "object" && cloned !== null && !Array.isArray(cloned)) {
|
|
933
944
|
Utils.Dict.deleteInPlace(cloned, "isDev");
|
|
945
|
+
Utils.Dict.deleteInPlace(cloned, "isolatedChains");
|
|
934
946
|
stripChains(cloned["evm"]);
|
|
935
947
|
stripChains(cloned["fuel"]);
|
|
936
948
|
stripChains(cloned["svm"]);
|
|
@@ -1159,14 +1171,12 @@ export {
|
|
|
1159
1171
|
publicConfigStorageSchema,
|
|
1160
1172
|
publicConfigSchema,
|
|
1161
1173
|
contractMappingOf,
|
|
1174
|
+
getChain,
|
|
1175
|
+
isolate,
|
|
1162
1176
|
fromPublic,
|
|
1163
|
-
isIsolatedMultichain,
|
|
1164
1177
|
normalizeUserAddress,
|
|
1165
1178
|
normalizeSimulateAddress,
|
|
1166
1179
|
getEventConfig,
|
|
1167
|
-
shouldSaveHistory,
|
|
1168
|
-
shouldPruneHistory,
|
|
1169
|
-
getChain,
|
|
1170
1180
|
prime,
|
|
1171
1181
|
getPublicConfigJson,
|
|
1172
1182
|
stripSensitiveData,
|
package/src/CrossChainState.res
CHANGED
|
@@ -8,13 +8,13 @@ type t = {
|
|
|
8
8
|
// Chain ids in a stable order, so the cross-chain loops iterate the chains
|
|
9
9
|
// without allocating a values array on every tick.
|
|
10
10
|
chainIds: array<ChainId.t>,
|
|
11
|
-
// True once every chain has caught up to head/endBlock
|
|
11
|
+
// True once every chain this process drives has caught up to head/endBlock and
|
|
12
|
+
// the indexes they owe are committed. Monotonic during a run.
|
|
12
13
|
mutable isRealtime: bool,
|
|
13
|
-
// True once
|
|
14
|
-
// but before the deferred schema indexes and `ready_at` are committed. The
|
|
15
|
-
//
|
|
14
|
+
// True once those chains have caught up and there's nothing left to process,
|
|
15
|
+
// but before the deferred schema indexes and `ready_at` are committed. The gap
|
|
16
|
+
// between this and `isRealtime` is the FinalizingIndexes phase.
|
|
16
17
|
mutable isCaughtUp: bool,
|
|
17
|
-
mutable isInReorgThreshold: bool,
|
|
18
18
|
// Indexer-wide fetch buffer pool (item count), shared across all chains.
|
|
19
19
|
targetBufferSize: int,
|
|
20
20
|
}
|
|
@@ -26,18 +26,12 @@ let calculateTargetBufferSize = () =>
|
|
|
26
26
|
| None => 100_000
|
|
27
27
|
}
|
|
28
28
|
|
|
29
|
-
let make = (
|
|
30
|
-
~chainStates,
|
|
31
|
-
~isInReorgThreshold,
|
|
32
|
-
~isRealtime,
|
|
33
|
-
~targetBufferSize=calculateTargetBufferSize(),
|
|
34
|
-
): t => {
|
|
29
|
+
let make = (~chainStates, ~isRealtime, ~targetBufferSize=calculateTargetBufferSize()): t => {
|
|
35
30
|
{
|
|
36
31
|
chainStates,
|
|
37
32
|
chainIds: chainStates->Dict.valuesToArray->Array.map(cs => (cs->ChainState.chainConfig).id),
|
|
38
33
|
isRealtime,
|
|
39
34
|
isCaughtUp: isRealtime,
|
|
40
|
-
isInReorgThreshold,
|
|
41
35
|
targetBufferSize,
|
|
42
36
|
}
|
|
43
37
|
}
|
|
@@ -51,10 +45,21 @@ let getChainState = (crossChainState: t, chainId) =>
|
|
|
51
45
|
|
|
52
46
|
let chainStates = (crossChainState: t) => crossChainState.chainStates
|
|
53
47
|
let isRealtime = (crossChainState: t) => crossChainState.isRealtime
|
|
48
|
+
let chainIds = (crossChainState: t) => crossChainState.chainIds
|
|
54
49
|
let isCaughtUp = (crossChainState: t) => crossChainState.isCaughtUp
|
|
55
|
-
|
|
50
|
+
// Chains enter the threshold together. It is the run-wide reading; what a
|
|
51
|
+
// write keeps is decided per chain.
|
|
52
|
+
let isInReorgThreshold = (crossChainState: t) => {
|
|
53
|
+
let chainStates = crossChainState.chainStates->Dict.valuesToArray
|
|
54
|
+
chainStates->Utils.Array.notEmpty && chainStates->Array.every(ChainState.isInReorgThreshold)
|
|
55
|
+
}
|
|
56
56
|
let targetBufferSize = (crossChainState: t) => crossChainState.targetBufferSize
|
|
57
57
|
|
|
58
|
+
// Whether each chain's writes still need history, keyed by chain id — what the
|
|
59
|
+
// history policy is built from.
|
|
60
|
+
let shouldSaveHistory = (crossChainState: t) =>
|
|
61
|
+
crossChainState.chainStates->Utils.Dict.mapValues(ChainState.shouldSaveHistory)
|
|
62
|
+
|
|
58
63
|
// Ready-to-process items across every chain — the live draw against
|
|
59
64
|
// targetBufferSize, which is a budget of processable events (items stuck behind
|
|
60
65
|
// a gap don't count toward the goal of keeping ~targetBufferSize ready).
|
|
@@ -74,14 +79,20 @@ let nextItemIsNone = (crossChainState: t): bool =>
|
|
|
74
79
|
|
|
75
80
|
// Each chain's safe checkpoint: the last one a reorg on that chain can no longer
|
|
76
81
|
// reach, or None while it has nothing safe yet. A chain that can't be rolled
|
|
77
|
-
// back (maxReorgDepth = 0) tracks none, and everything
|
|
78
|
-
|
|
82
|
+
// back (maxReorgDepth = 0) tracks none, and everything committed is safe from
|
|
83
|
+
// it: under one shared sequence that is the run's highest id, not the chain's
|
|
84
|
+
// own — an idle chain's would hold every other chain's prune back.
|
|
85
|
+
let getSafeCheckpointIdByChain = (
|
|
86
|
+
crossChainState: t,
|
|
87
|
+
~sequence: CheckpointSequence.t,
|
|
88
|
+
~committedFrontier,
|
|
89
|
+
) =>
|
|
79
90
|
crossChainState.chainIds->Array.map(chainId => {
|
|
80
91
|
let cs = crossChainState->getChainState(chainId)
|
|
81
92
|
(
|
|
82
93
|
chainId,
|
|
83
94
|
switch cs->ChainState.safeCheckpointTracking {
|
|
84
|
-
| None => Some(
|
|
95
|
+
| None => Some(sequence->CheckpointSequence.position(committedFrontier, ~chainId))
|
|
85
96
|
| Some(tracking) =>
|
|
86
97
|
tracking->SafeCheckpointTracking.getSafeCheckpointId(
|
|
87
98
|
~sourceBlockNumber=cs->ChainState.knownHeight,
|
|
@@ -94,20 +105,16 @@ let getSafeCheckpointIdByChain = (crossChainState: t, ~committedCheckpointId) =>
|
|
|
94
105
|
|
|
95
106
|
let createBatch = (
|
|
96
107
|
crossChainState: t,
|
|
97
|
-
~
|
|
108
|
+
~config: Config.t,
|
|
109
|
+
~frontier,
|
|
98
110
|
~batchSizeTarget: int,
|
|
99
|
-
~isRollback: bool,
|
|
100
111
|
): Batch.t => {
|
|
101
112
|
Batch.make(
|
|
102
|
-
~
|
|
103
|
-
~
|
|
104
|
-
|
|
105
|
-
// This is needed to currectly overwrite old state
|
|
106
|
-
// in an append-only ClickHouse insert.
|
|
107
|
-
isRollback ? 1n : 0n,
|
|
108
|
-
),
|
|
113
|
+
~sequence=config.checkpointSequence,
|
|
114
|
+
~history=config->HistoryPolicy.decide(~shouldSaveHistory=crossChainState->shouldSaveHistory),
|
|
115
|
+
~frontier,
|
|
109
116
|
~chainsBeforeBatch=crossChainState.chainStates->Utils.Dict.mapValues(
|
|
110
|
-
ChainState.toChainBeforeBatch,
|
|
117
|
+
ChainState.toChainBeforeBatch(~isRealtime=crossChainState.isRealtime, ...),
|
|
111
118
|
),
|
|
112
119
|
~batchSizeTarget,
|
|
113
120
|
)
|
|
@@ -123,8 +130,6 @@ let enterReorgThreshold = (crossChainState: t) => {
|
|
|
123
130
|
->getChainState(crossChainState.chainIds->Array.getUnsafe(i))
|
|
124
131
|
->ChainState.enterReorgThreshold
|
|
125
132
|
}
|
|
126
|
-
|
|
127
|
-
crossChainState.isInReorgThreshold = true
|
|
128
133
|
}
|
|
129
134
|
|
|
130
135
|
// Commit each progressed chain's batch progress, then record whether the whole
|
|
@@ -146,17 +151,6 @@ let applyBatchProgress = (crossChainState: t, ~batch: Batch.t, ~blockTimestampNa
|
|
|
146
151
|
|
|
147
152
|
crossChainState.isCaughtUp =
|
|
148
153
|
crossChainState.isCaughtUp || (crossChainState->nextItemIsNone && everyChainCaughtUp.contents)
|
|
149
|
-
|
|
150
|
-
// A run resumed with every chain already stamped ready needs no finalize —
|
|
151
|
-
// the indexes were committed together with those stamps.
|
|
152
|
-
let allChainsReady = ref(true)
|
|
153
|
-
for i in 0 to chainIds->Array.length - 1 {
|
|
154
|
-
if !(crossChainState->getChainState(chainIds->Array.getUnsafe(i))->ChainState.isReady) {
|
|
155
|
-
allChainsReady := false
|
|
156
|
-
}
|
|
157
|
-
}
|
|
158
|
-
|
|
159
|
-
crossChainState.isRealtime = crossChainState.isRealtime || allChainsReady.contents
|
|
160
154
|
}
|
|
161
155
|
|
|
162
156
|
// Every chain has buffered up to its head (or endblock) with nothing
|
|
@@ -191,29 +185,25 @@ let markCaughtUpIfSettled = (crossChainState: t) =>
|
|
|
191
185
|
// may have moved on — at which point no chain looks at head any more and the
|
|
192
186
|
// indexes it still owes would wait out another whole backfill. Deciding here,
|
|
193
187
|
// before any source request, keeps that debt tied to the progress that was
|
|
194
|
-
// actually committed. Skipped
|
|
195
|
-
//
|
|
188
|
+
// actually committed. Skipped when the run resumed realtime, which is exactly
|
|
189
|
+
// the case where the indexes are already committed and nothing is owed.
|
|
196
190
|
let markCaughtUpOnResume = (crossChainState: t) => {
|
|
197
191
|
let everyChainCaughtUp = ref(crossChainState.chainIds->Array.length > 0)
|
|
198
|
-
let everyChainReady = ref(true)
|
|
199
192
|
for i in 0 to crossChainState.chainIds->Array.length - 1 {
|
|
200
193
|
let cs = crossChainState->getChainState(crossChainState.chainIds->Array.getUnsafe(i))
|
|
201
194
|
if !(cs->ChainState.isDurablyCaughtUp) {
|
|
202
195
|
everyChainCaughtUp := false
|
|
203
196
|
}
|
|
204
|
-
if !(cs->ChainState.isReady) {
|
|
205
|
-
everyChainReady := false
|
|
206
|
-
}
|
|
207
197
|
}
|
|
208
198
|
|
|
209
|
-
if everyChainCaughtUp.contents && !
|
|
199
|
+
if everyChainCaughtUp.contents && !crossChainState.isRealtime && crossChainState->nextItemIsNone {
|
|
210
200
|
crossChainState.isCaughtUp = true
|
|
211
201
|
}
|
|
212
202
|
}
|
|
213
203
|
|
|
214
|
-
// Concludes the FinalizingIndexes phase: stamps every chain
|
|
215
|
-
// already committed alongside
|
|
216
|
-
// indexer to realtime.
|
|
204
|
+
// Concludes the FinalizingIndexes phase: stamps every chain this process drives
|
|
205
|
+
// with the `ready_at` already committed alongside their deferred schema indexes,
|
|
206
|
+
// and switches the indexer to realtime.
|
|
217
207
|
let markReady = (crossChainState: t, ~readyAt) => {
|
|
218
208
|
for i in 0 to crossChainState.chainIds->Array.length - 1 {
|
|
219
209
|
crossChainState
|
|
@@ -267,6 +257,10 @@ let idleOrWaitAction = (cs: ChainState.t) =>
|
|
|
267
257
|
? FetchState.NothingToQuery
|
|
268
258
|
: FetchState.WaitingForNewBlock
|
|
269
259
|
|
|
260
|
+
// How far past the alignment anchor's frontier progress a follower may fetch,
|
|
261
|
+
// as a fraction of its own alignable range.
|
|
262
|
+
let alignmentMargin = 0.2
|
|
263
|
+
|
|
270
264
|
// Dispatch a fetch tick across the whole indexer from one shared pool of
|
|
271
265
|
// ~targetBufferSize ready events, as a waterfall: visit chains furthest-behind
|
|
272
266
|
// first, hand each the budget remaining at that point (plus its own
|
|
@@ -350,11 +344,14 @@ let checkAndFetch = async (
|
|
|
350
344
|
(isCold ? Pervasives.min(remaining.contents, coldChainBudget) : remaining.contents) +.
|
|
351
345
|
cs->ChainState.pendingBudget
|
|
352
346
|
let maxTargetBlock = switch alignment {
|
|
353
|
-
//
|
|
347
|
+
// 20% margin past the anchor's line: chains whose progress tracks the
|
|
354
348
|
// anchor closely would otherwise flap in and out of the clamp on every
|
|
355
|
-
// small frontier move, stalling their pipeline every other tick.
|
|
349
|
+
// small frontier move, stalling their pipeline every other tick. The
|
|
350
|
+
// margin is also the headroom a follower keeps buffered while the anchor
|
|
351
|
+
// is mid-fetch, so it has to outlast a slow anchor response, not just
|
|
352
|
+
// absorb frontier jitter.
|
|
356
353
|
| Some((anchorChainId, progress)) if anchorChainId !== chainId =>
|
|
357
|
-
Some(cs->ChainState.blockAtProgress(~progress=progress +.
|
|
354
|
+
Some(cs->ChainState.blockAtProgress(~progress=progress +. alignmentMargin))
|
|
358
355
|
| _ => None
|
|
359
356
|
}
|
|
360
357
|
switch cs->ChainState.getNextQuery(~chainTargetItems, ~maxTargetBlock?) {
|
|
@@ -404,7 +401,7 @@ let checkAndFetch = async (
|
|
|
404
401
|
switch actionByChain->ChainId.Dict.dangerouslyGetNonOption(chainId) {
|
|
405
402
|
| Some(NothingToQuery)
|
|
406
403
|
| None => ()
|
|
407
|
-
| Some(action) => promises->Array.push(dispatchChain(~chainId
|
|
404
|
+
| Some(action) => promises->Array.push(dispatchChain(~chainId, ~action))
|
|
408
405
|
}
|
|
409
406
|
}
|
|
410
407
|
let _ = await promises->Promise.all
|