envio 3.11.0 → 3.12.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (109) hide show
  1. package/index.d.ts +3 -0
  2. package/package.json +6 -6
  3. package/src/Batch.res +111 -78
  4. package/src/Batch.res.mjs +53 -46
  5. package/src/BatchProcessing.res +1 -11
  6. package/src/BatchProcessing.res.mjs +1 -4
  7. package/src/ChainFetching.res +7 -3
  8. package/src/ChainFetching.res.mjs +1 -1
  9. package/src/ChainState.res +82 -20
  10. package/src/ChainState.res.mjs +51 -13
  11. package/src/ChainState.resi +6 -1
  12. package/src/Config.res +58 -37
  13. package/src/Config.res.mjs +63 -53
  14. package/src/CrossChainState.res +50 -53
  15. package/src/CrossChainState.res.mjs +25 -23
  16. package/src/CrossChainState.resi +6 -13
  17. package/src/EffectState.res +13 -4
  18. package/src/EffectState.resi +7 -1
  19. package/src/EventProcessing.res +6 -6
  20. package/src/EventProcessing.res.mjs +6 -6
  21. package/src/FetchState.res +65 -46
  22. package/src/FetchState.res.mjs +68 -36
  23. package/src/FinalizeBackfill.res +16 -12
  24. package/src/FinalizeBackfill.res.mjs +3 -3
  25. package/src/HistoryPolicy.res +46 -0
  26. package/src/HistoryPolicy.res.mjs +50 -0
  27. package/src/InMemoryStore.res +30 -32
  28. package/src/InMemoryStore.res.mjs +33 -28
  29. package/src/InMemoryTable.res +155 -67
  30. package/src/InMemoryTable.res.mjs +151 -60
  31. package/src/IndexerState.res +66 -45
  32. package/src/IndexerState.res.mjs +57 -29
  33. package/src/IndexerState.resi +8 -13
  34. package/src/Internal.res +4 -2
  35. package/src/LoadLayer.res +68 -38
  36. package/src/LoadLayer.res.mjs +50 -64
  37. package/src/LoadLayer.resi +1 -1
  38. package/src/Main.res +25 -24
  39. package/src/Main.res.mjs +24 -25
  40. package/src/Metrics.res +49 -1
  41. package/src/Metrics.res.mjs +26 -3
  42. package/src/Persistence.res +55 -16
  43. package/src/Persistence.res.mjs +18 -3
  44. package/src/PgStorage.res +286 -180
  45. package/src/PgStorage.res.mjs +209 -122
  46. package/src/PruneStaleHistory.res +9 -15
  47. package/src/PruneStaleHistory.res.mjs +11 -20
  48. package/src/Rollback.res +9 -14
  49. package/src/Rollback.res.mjs +5 -3
  50. package/src/Sink.res +7 -5
  51. package/src/Sink.res.mjs +4 -4
  52. package/src/Staging.res +298 -0
  53. package/src/Staging.res.mjs +257 -0
  54. package/src/TestIndexer.res +14 -30
  55. package/src/TestIndexer.res.mjs +11 -9
  56. package/src/UserContext.res +20 -36
  57. package/src/UserContext.res.mjs +6 -9
  58. package/src/Utils.res +10 -4
  59. package/src/Utils.res.mjs +16 -16
  60. package/src/Writing.res +34 -20
  61. package/src/Writing.res.mjs +15 -20
  62. package/src/bindings/ClickHouse.res +41 -34
  63. package/src/bindings/ClickHouse.res.mjs +36 -30
  64. package/src/bindings/ClickHouseSink.res +45 -186
  65. package/src/bindings/ClickHouseSink.res.mjs +21 -174
  66. package/src/db/CheckpointSequence.res +109 -0
  67. package/src/db/CheckpointSequence.res.mjs +99 -0
  68. package/src/db/EntityFilter.res +487 -275
  69. package/src/db/EntityFilter.res.mjs +557 -309
  70. package/src/db/EntityHistory.res +6 -5
  71. package/src/db/EntityHistory.res.mjs +5 -5
  72. package/src/db/Frontier.res +86 -0
  73. package/src/db/Frontier.res.mjs +126 -0
  74. package/src/db/InternalTable.res +145 -27
  75. package/src/db/InternalTable.res.mjs +115 -29
  76. package/src/db/RollbackFloors.res +34 -41
  77. package/src/db/RollbackFloors.res.mjs +27 -71
  78. package/src/db/Table.res +21 -6
  79. package/src/db/Table.res.mjs +13 -4
  80. package/src/sources/BlockStore.res +8 -0
  81. package/src/sources/EvmHyperSyncSource.res +5 -1
  82. package/src/sources/EvmHyperSyncSource.res.mjs +4 -3
  83. package/src/sources/FuelHyperSyncSource.res +1 -0
  84. package/src/sources/FuelHyperSyncSource.res.mjs +1 -1
  85. package/src/sources/HyperSync.res +6 -0
  86. package/src/sources/HyperSync.res.mjs +5 -2
  87. package/src/sources/HyperSync.resi +2 -0
  88. package/src/sources/HyperSyncClient.res +6 -0
  89. package/src/sources/HyperSyncSSE.res +1 -1
  90. package/src/sources/HyperSyncSSE.res.mjs +4 -10
  91. package/src/sources/RequestStat.res +8 -1
  92. package/src/sources/RpcSource.res +1 -0
  93. package/src/sources/RpcSource.res.mjs +1 -1
  94. package/src/sources/SimulateSource.res +1 -0
  95. package/src/sources/SimulateSource.res.mjs +1 -1
  96. package/src/sources/Source.res +11 -3
  97. package/src/sources/SourceManager.res +34 -13
  98. package/src/sources/SourceManager.res.mjs +26 -9
  99. package/src/sources/SourceManager.resi +2 -0
  100. package/src/sources/SvmHyperSyncClient.res +5 -0
  101. package/src/sources/SvmHyperSyncSource.res +3 -0
  102. package/src/sources/SvmHyperSyncSource.res.mjs +4 -2
  103. package/src/tui/Tui.res +56 -143
  104. package/src/tui/Tui.res.mjs +70 -166
  105. package/src/tui/components/TuiData.res +61 -16
  106. package/src/tui/components/TuiData.res.mjs +53 -15
  107. package/svm.schema.json +0 -7
  108. package/src/db/CheckpointBounds.res +0 -53
  109. package/src/db/CheckpointBounds.res.mjs +0 -44
package/src/Config.res CHANGED
@@ -136,9 +136,17 @@ type t = {
136
136
  reorgThresholdReadyTolerance: int,
137
137
  lowercaseAddresses: bool,
138
138
  isDev: bool,
139
+ // An `envio start --chain` process: drives a subset of the schema's chains
140
+ // while sibling processes drive the rest, so it only touches what its own
141
+ // chains own — their partitions' indexes, their `ready_at`, their resume.
142
+ isolated: bool,
139
143
  userEntitiesByName: dict<Internal.entityConfig>,
140
144
  userEntities: array<Internal.entityConfig>,
141
145
  allEnums: array<Table.enumConfig<Table.enum>>,
146
+ // Whether checkpoint ids come from one counter or one per chain. Decided by
147
+ // the schema alone: a cross-chain entity has rows any chain's reorg can
148
+ // reach, so its checkpoints have to be comparable across chains.
149
+ checkpointSequence: CheckpointSequence.t,
142
150
  }
143
151
 
144
152
  type rpcSourceFor = | @as("sync") Sync | @as("fallback") Fallback | @as("realtime") Realtime
@@ -586,6 +594,7 @@ let publicConfigSchema = S.schema(s =>
586
594
  "description": s.matches(S.option(S.string)),
587
595
  "handlers": s.matches(S.option(S.string)),
588
596
  "isDev": s.matches(S.option(S.bool)),
597
+ "isolatedChains": s.matches(S.option(S.array(ChainId.schema))),
589
598
  "fullBatchSize": s.matches(S.option(S.int)),
590
599
  "rollbackOnReorg": s.matches(S.option(S.bool)),
591
600
  "saveFullHistory": s.matches(S.option(S.bool)),
@@ -609,6 +618,44 @@ let contractMappingOf = (~chainConfigs: array<chain>): ContractMapping.t => {
609
618
  ContractMapping.make(~names)
610
619
  }
611
620
 
621
+ let getChain = (config, ~chainId) =>
622
+ config.chainMap->ChainMap.has(chainId)
623
+ ? chainId
624
+ : JsError.throwWithMessage(
625
+ "No chain with id " ++ chainId->ChainId.toString ++ " found in config.yaml",
626
+ )
627
+
628
+ // Narrows a config to the chains one `envio start --chain` process drives.
629
+ // `contractMapping` is deliberately left whole: its ids are what the migration
630
+ // that created the schema stored, and one rebuilt from a subset would hand the
631
+ // same contract a different id.
632
+ let isolate = (config: t, ~chainIds: array<ChainId.t>) => {
633
+ // Chains indexed in separate processes each advance their own checkpoint
634
+ // counter, which only holds while no entity has rows another chain can reach.
635
+ switch config.userEntities->Array.filter(entityConfig => entityConfig.crossChain) {
636
+ | [] => ()
637
+ | shared =>
638
+ JsError.throwWithMessage(
639
+ `Only a schema whose entities are all per-chain can be split across processes. Shared across chains: ${shared
640
+ ->Array.map(entityConfig => entityConfig.name)
641
+ ->Array.joinUnsafe(", ")}.`,
642
+ )
643
+ }
644
+ chainIds->Array.forEach(chainId => config->getChain(~chainId)->ignore)
645
+
646
+ // Filtered out of the config's own chain order rather than built from the
647
+ // argument order, so a repeated `--chain` collapses and `defaultChain` doesn't
648
+ // depend on how the flags were typed.
649
+ let chains =
650
+ config.chainMap->ChainMap.values->Array.filter(chain => chainIds->Array.includes(chain.id))
651
+ {
652
+ ...config,
653
+ chainMap: chains->Array.map(chain => (chain.id, chain))->ChainMap.fromArrayUnsafe,
654
+ defaultChain: chains->Array.get(0),
655
+ isolated: true,
656
+ }
657
+ }
658
+
612
659
  let fromPublic = (publicConfigJson: JSON.t) => {
613
660
  let maxAddrInPartition = Env.maxAddrInPartition
614
661
  // Parse public config
@@ -709,12 +756,7 @@ let fromPublic = (publicConfigJson: JSON.t) => {
709
756
  }
710
757
  let widened =
711
758
  contractConfig->(
712
- Utils.magic: _ => {
713
- "svmAbi": option<{
714
- "definedTypes": JSON.t,
715
- "source": string,
716
- }>,
717
- }
759
+ Utils.magic: _ => {"svmAbi": option<{"definedTypes": JSON.t, "source": string}>}
718
760
  )
719
761
  contractDataByName->Dict.set(
720
762
  capitalizedName,
@@ -1063,7 +1105,7 @@ let fromPublic = (publicConfigJson: JSON.t) => {
1063
1105
  | None => []
1064
1106
  }
1065
1107
 
1066
- {
1108
+ let config = {
1067
1109
  name: publicConfig["name"],
1068
1110
  description: publicConfig["description"],
1069
1111
  handlers: publicConfig["handlers"]->Option.getOr("src/handlers"),
@@ -1084,19 +1126,18 @@ let fromPublic = (publicConfigJson: JSON.t) => {
1084
1126
  reorgThresholdReadyTolerance: 100,
1085
1127
  lowercaseAddresses,
1086
1128
  isDev: publicConfig["isDev"]->Option.getOr(false),
1129
+ isolated: false,
1087
1130
  userEntitiesByName,
1088
1131
  userEntities,
1089
1132
  allEnums,
1133
+ checkpointSequence: CheckpointSequence.fromEntities(userEntities),
1090
1134
  }
1091
- }
1092
1135
 
1093
- // With no cross-chain entity, a reorg on one chain can never have changed a row
1094
- // another chain owns, so its rollback stays isolated to that chain instead of
1095
- // dragging every sibling back with it. A single chain has no sibling to spare,
1096
- // and narrowing its rollback would only buy it a predicate that always holds.
1097
- let isIsolatedMultichain = (config: t) =>
1098
- config.chainMap->ChainMap.keys->Array.length > 1 &&
1099
- config.userEntities->Array.every(entityConfig => !entityConfig.crossChain)
1136
+ switch publicConfig["isolatedChains"] {
1137
+ | None => config
1138
+ | Some(chainIds) => config->isolate(~chainIds)
1139
+ }
1140
+ }
1100
1141
 
1101
1142
  // Canonicalize a user-provided address to the configured casing so it matches
1102
1143
  // addresses parsed from config.yaml during routing. HyperSync/RPC data arrives
@@ -1169,33 +1210,12 @@ let getEventConfig = (config: t, ~contractName, ~eventName, ~chainId: option<Cha
1169
1210
  })
1170
1211
  }
1171
1212
 
1172
- // A chain that can't be rolled back (maxReorgDepth = 0) has no history to keep,
1173
- // unless a cross-chain entity lets another chain's rollback reach its rows.
1174
- let shouldSaveHistory = (config, ~isInReorgThreshold, ~chainId: option<ChainId.t>=?) =>
1175
- config.shouldSaveFullHistory ||
1176
- (config.shouldRollbackOnReorg &&
1177
- isInReorgThreshold &&
1178
- switch chainId {
1179
- | Some(chainId) if config->isIsolatedMultichain =>
1180
- (config.chainMap->ChainMap.get(chainId)).maxReorgDepth > 0
1181
- | _ => true
1182
- })
1183
-
1184
- let shouldPruneHistory = (config, ~isInReorgThreshold) =>
1185
- !config.shouldSaveFullHistory && (config.shouldRollbackOnReorg && isInReorgThreshold)
1186
-
1187
- let getChain = (config, ~chainId) =>
1188
- config.chainMap->ChainMap.has(chainId)
1189
- ? chainId
1190
- : JsError.throwWithMessage(
1191
- "No chain with id " ++ chainId->ChainId.toString ++ " found in config.yaml",
1192
- )
1193
-
1194
1213
  // A CLI command payload already contains the resolved JSON; priming lets
1195
1214
  // downstream callers skip the NAPI `getConfigJson` round-trip. Calling
1196
1215
  // `prime` again invalidates the memo.
1197
1216
  %%private(let primedJson: ref<option<JSON.t>> = ref(None))
1198
1217
  %%private(let cached: ref<option<t>> = ref(None))
1218
+
1199
1219
  let prime = (json: JSON.t): unit => {
1200
1220
  primedJson := Some(json)
1201
1221
  cached := None
@@ -1236,6 +1256,7 @@ let stripSensitiveData = (json: JSON.t): JSON.t => {
1236
1256
  switch cloned {
1237
1257
  | Object(obj) => {
1238
1258
  obj->Utils.Dict.deleteInPlace("isDev")
1259
+ obj->Utils.Dict.deleteInPlace("isolatedChains")
1239
1260
  stripChains(obj->Dict.get("evm"))
1240
1261
  stripChains(obj->Dict.get("fuel"))
1241
1262
  stripChains(obj->Dict.get("svm"))
@@ -21,6 +21,7 @@ import * as Primitive_object from "@rescript/runtime/lib/es6/Primitive_object.js
21
21
  import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js";
22
22
  import * as Primitive_string from "@rescript/runtime/lib/es6/Primitive_string.js";
23
23
  import * as S$RescriptSchema from "rescript-schema/src/S.res.mjs";
24
+ import * as CheckpointSequence from "./db/CheckpointSequence.res.mjs";
24
25
  import * as EventConfigBuilder from "./EventConfigBuilder.res.mjs";
25
26
  import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
26
27
 
@@ -436,6 +437,7 @@ let publicConfigSchema = S$RescriptSchema.schema(s => ({
436
437
  description: s.m(S$RescriptSchema.option(S$RescriptSchema.string)),
437
438
  handlers: s.m(S$RescriptSchema.option(S$RescriptSchema.string)),
438
439
  isDev: s.m(S$RescriptSchema.option(S$RescriptSchema.bool)),
440
+ isolatedChains: s.m(S$RescriptSchema.option(S$RescriptSchema.array(ChainId.schema))),
439
441
  fullBatchSize: s.m(S$RescriptSchema.option(S$RescriptSchema.int)),
440
442
  rollbackOnReorg: s.m(S$RescriptSchema.option(S$RescriptSchema.bool)),
441
443
  saveFullHistory: s.m(S$RescriptSchema.option(S$RescriptSchema.bool)),
@@ -460,6 +462,33 @@ function contractMappingOf(chainConfigs) {
460
462
  return ContractMapping.make(names);
461
463
  }
462
464
 
465
+ function getChain(config, chainId) {
466
+ if (ChainMap.has(config.chainMap, chainId)) {
467
+ return chainId;
468
+ } else {
469
+ return Stdlib_JsError.throwWithMessage("No chain with id " + ChainId.toString(chainId) + " found in config.yaml");
470
+ }
471
+ }
472
+
473
+ function isolate(config, chainIds) {
474
+ let shared = config.userEntities.filter(entityConfig => entityConfig.crossChain);
475
+ if (shared.length !== 0) {
476
+ Stdlib_JsError.throwWithMessage(`Only a schema whose entities are all per-chain can be split across processes. Shared across chains: ` + shared.map(entityConfig => entityConfig.name).join(", ") + `.`);
477
+ }
478
+ chainIds.forEach(chainId => {
479
+ getChain(config, chainId);
480
+ });
481
+ let chains = ChainMap.values(config.chainMap).filter(chain => chainIds.includes(chain.id));
482
+ let newrecord = {...config};
483
+ newrecord.isolated = true;
484
+ newrecord.defaultChain = chains[0];
485
+ newrecord.chainMap = ChainMap.fromArrayUnsafe(chains.map(chain => [
486
+ chain.id,
487
+ chain
488
+ ]));
489
+ return newrecord;
490
+ }
491
+
463
492
  function fromPublic(publicConfigJson) {
464
493
  let publicConfig;
465
494
  try {
@@ -755,38 +784,50 @@ function fromPublic(publicConfigJson) {
755
784
  name: Utils.$$String.capitalize(param[0]),
756
785
  handler: param[1].handler
757
786
  })) : [];
758
- return {
759
- name: publicConfig.name,
760
- description: publicConfig.description,
761
- handlers: Stdlib_Option.getOr(publicConfig.handlers, "src/handlers"),
787
+ let config_name = publicConfig.name;
788
+ let config_description = publicConfig.description;
789
+ let config_handlers = Stdlib_Option.getOr(publicConfig.handlers, "src/handlers");
790
+ let config_shouldRollbackOnReorg = Stdlib_Option.getOr(publicConfig.rollbackOnReorg, true);
791
+ let config_shouldSaveFullHistory = Stdlib_Option.getOr(publicConfig.saveFullHistory, false);
792
+ let config_chainIdMode = Stdlib_Option.getOr(publicConfig.chainIdMode, "int32");
793
+ let config_contractMapping = contractMappingOf(chains);
794
+ let config_defaultChain = chains[0];
795
+ let config_enableRawEvents = Stdlib_Option.getOr(publicConfig.rawEvents, false);
796
+ let config_batchSize = Stdlib_Option.getOr(publicConfig.fullBatchSize, 5000);
797
+ let config_isDev = Stdlib_Option.getOr(publicConfig.isDev, false);
798
+ let config_checkpointSequence = CheckpointSequence.fromEntities(userEntities);
799
+ let config = {
800
+ name: config_name,
801
+ description: config_description,
802
+ handlers: config_handlers,
762
803
  contractHandlers: contractHandlers,
763
- shouldRollbackOnReorg: Stdlib_Option.getOr(publicConfig.rollbackOnReorg, true),
764
- shouldSaveFullHistory: Stdlib_Option.getOr(publicConfig.saveFullHistory, false),
804
+ shouldRollbackOnReorg: config_shouldRollbackOnReorg,
805
+ shouldSaveFullHistory: config_shouldSaveFullHistory,
765
806
  defaultCrossChain: defaultCrossChain,
766
807
  storage: globalStorage,
767
- chainIdMode: Stdlib_Option.getOr(publicConfig.chainIdMode, "int32"),
808
+ chainIdMode: config_chainIdMode,
768
809
  chainMap: chainMap,
769
- contractMapping: contractMappingOf(chains),
770
- defaultChain: chains[0],
810
+ contractMapping: config_contractMapping,
811
+ defaultChain: config_defaultChain,
771
812
  ecosystem: ecosystem,
772
- enableRawEvents: Stdlib_Option.getOr(publicConfig.rawEvents, false),
813
+ enableRawEvents: config_enableRawEvents,
773
814
  maxAddrInPartition: Env.maxAddrInPartition,
774
815
  clientFilterAddressThreshold: Env.clientFilterAddressThreshold,
775
- batchSize: Stdlib_Option.getOr(publicConfig.fullBatchSize, 5000),
816
+ batchSize: config_batchSize,
776
817
  reorgThresholdReadyTolerance: 100,
777
818
  lowercaseAddresses: lowercaseAddresses,
778
- isDev: Stdlib_Option.getOr(publicConfig.isDev, false),
819
+ isDev: config_isDev,
820
+ isolated: false,
779
821
  userEntitiesByName: userEntitiesByName,
780
822
  userEntities: userEntities,
781
- allEnums: allEnums
823
+ allEnums: allEnums,
824
+ checkpointSequence: config_checkpointSequence
782
825
  };
783
- }
784
-
785
- function isIsolatedMultichain(config) {
786
- if (ChainMap.keys(config.chainMap).length > 1) {
787
- return config.userEntities.every(entityConfig => !entityConfig.crossChain);
826
+ let chainIds = publicConfig.isolatedChains;
827
+ if (chainIds !== undefined) {
828
+ return isolate(config, chainIds);
788
829
  } else {
789
- return false;
830
+ return config;
790
831
  }
791
832
  }
792
833
 
@@ -853,36 +894,6 @@ function getEventConfig(config, contractName, eventName, chainId) {
853
894
  });
854
895
  }
855
896
 
856
- function shouldSaveHistory(config, isInReorgThreshold, chainId) {
857
- if (config.shouldSaveFullHistory) {
858
- return true;
859
- } else if (config.shouldRollbackOnReorg && isInReorgThreshold) {
860
- if (chainId !== undefined && isIsolatedMultichain(config)) {
861
- return ChainMap.get(config.chainMap, Primitive_option.valFromOption(chainId)).maxReorgDepth > 0;
862
- } else {
863
- return true;
864
- }
865
- } else {
866
- return false;
867
- }
868
- }
869
-
870
- function shouldPruneHistory(config, isInReorgThreshold) {
871
- if (!config.shouldSaveFullHistory && config.shouldRollbackOnReorg) {
872
- return isInReorgThreshold;
873
- } else {
874
- return false;
875
- }
876
- }
877
-
878
- function getChain(config, chainId) {
879
- if (ChainMap.has(config.chainMap, chainId)) {
880
- return chainId;
881
- } else {
882
- return Stdlib_JsError.throwWithMessage("No chain with id " + ChainId.toString(chainId) + " found in config.yaml");
883
- }
884
- }
885
-
886
897
  let primedJson = {
887
898
  contents: undefined
888
899
  };
@@ -931,6 +942,7 @@ function stripSensitiveData(json) {
931
942
  };
932
943
  if (typeof cloned === "object" && cloned !== null && !Array.isArray(cloned)) {
933
944
  Utils.Dict.deleteInPlace(cloned, "isDev");
945
+ Utils.Dict.deleteInPlace(cloned, "isolatedChains");
934
946
  stripChains(cloned["evm"]);
935
947
  stripChains(cloned["fuel"]);
936
948
  stripChains(cloned["svm"]);
@@ -1159,14 +1171,12 @@ export {
1159
1171
  publicConfigStorageSchema,
1160
1172
  publicConfigSchema,
1161
1173
  contractMappingOf,
1174
+ getChain,
1175
+ isolate,
1162
1176
  fromPublic,
1163
- isIsolatedMultichain,
1164
1177
  normalizeUserAddress,
1165
1178
  normalizeSimulateAddress,
1166
1179
  getEventConfig,
1167
- shouldSaveHistory,
1168
- shouldPruneHistory,
1169
- getChain,
1170
1180
  prime,
1171
1181
  getPublicConfigJson,
1172
1182
  stripSensitiveData,
@@ -8,13 +8,13 @@ type t = {
8
8
  // Chain ids in a stable order, so the cross-chain loops iterate the chains
9
9
  // without allocating a values array on every tick.
10
10
  chainIds: array<ChainId.t>,
11
- // True once every chain has caught up to head/endBlock. Monotonic during a run.
11
+ // True once every chain this process drives has caught up to head/endBlock and
12
+ // the indexes they owe are committed. Monotonic during a run.
12
13
  mutable isRealtime: bool,
13
- // True once every chain has caught up and there's nothing left to process,
14
- // but before the deferred schema indexes and `ready_at` are committed. The
15
- // gap between this and `isRealtime` is the FinalizingIndexes phase.
14
+ // True once those chains have caught up and there's nothing left to process,
15
+ // but before the deferred schema indexes and `ready_at` are committed. The gap
16
+ // between this and `isRealtime` is the FinalizingIndexes phase.
16
17
  mutable isCaughtUp: bool,
17
- mutable isInReorgThreshold: bool,
18
18
  // Indexer-wide fetch buffer pool (item count), shared across all chains.
19
19
  targetBufferSize: int,
20
20
  }
@@ -26,18 +26,12 @@ let calculateTargetBufferSize = () =>
26
26
  | None => 100_000
27
27
  }
28
28
 
29
- let make = (
30
- ~chainStates,
31
- ~isInReorgThreshold,
32
- ~isRealtime,
33
- ~targetBufferSize=calculateTargetBufferSize(),
34
- ): t => {
29
+ let make = (~chainStates, ~isRealtime, ~targetBufferSize=calculateTargetBufferSize()): t => {
35
30
  {
36
31
  chainStates,
37
32
  chainIds: chainStates->Dict.valuesToArray->Array.map(cs => (cs->ChainState.chainConfig).id),
38
33
  isRealtime,
39
34
  isCaughtUp: isRealtime,
40
- isInReorgThreshold,
41
35
  targetBufferSize,
42
36
  }
43
37
  }
@@ -51,10 +45,21 @@ let getChainState = (crossChainState: t, chainId) =>
51
45
 
52
46
  let chainStates = (crossChainState: t) => crossChainState.chainStates
53
47
  let isRealtime = (crossChainState: t) => crossChainState.isRealtime
48
+ let chainIds = (crossChainState: t) => crossChainState.chainIds
54
49
  let isCaughtUp = (crossChainState: t) => crossChainState.isCaughtUp
55
- let isInReorgThreshold = (crossChainState: t) => crossChainState.isInReorgThreshold
50
+ // Chains enter the threshold together. It is the run-wide reading; what a
51
+ // write keeps is decided per chain.
52
+ let isInReorgThreshold = (crossChainState: t) => {
53
+ let chainStates = crossChainState.chainStates->Dict.valuesToArray
54
+ chainStates->Utils.Array.notEmpty && chainStates->Array.every(ChainState.isInReorgThreshold)
55
+ }
56
56
  let targetBufferSize = (crossChainState: t) => crossChainState.targetBufferSize
57
57
 
58
+ // Whether each chain's writes still need history, keyed by chain id — what the
59
+ // history policy is built from.
60
+ let shouldSaveHistory = (crossChainState: t) =>
61
+ crossChainState.chainStates->Utils.Dict.mapValues(ChainState.shouldSaveHistory)
62
+
58
63
  // Ready-to-process items across every chain — the live draw against
59
64
  // targetBufferSize, which is a budget of processable events (items stuck behind
60
65
  // a gap don't count toward the goal of keeping ~targetBufferSize ready).
@@ -74,14 +79,20 @@ let nextItemIsNone = (crossChainState: t): bool =>
74
79
 
75
80
  // Each chain's safe checkpoint: the last one a reorg on that chain can no longer
76
81
  // reach, or None while it has nothing safe yet. A chain that can't be rolled
77
- // back (maxReorgDepth = 0) tracks none, and everything it has committed is safe.
78
- let getSafeCheckpointIdByChain = (crossChainState: t, ~committedCheckpointId) =>
82
+ // back (maxReorgDepth = 0) tracks none, and everything committed is safe from
83
+ // it: under one shared sequence that is the run's highest id, not the chain's
84
+ // own — an idle chain's would hold every other chain's prune back.
85
+ let getSafeCheckpointIdByChain = (
86
+ crossChainState: t,
87
+ ~sequence: CheckpointSequence.t,
88
+ ~committedFrontier,
89
+ ) =>
79
90
  crossChainState.chainIds->Array.map(chainId => {
80
91
  let cs = crossChainState->getChainState(chainId)
81
92
  (
82
93
  chainId,
83
94
  switch cs->ChainState.safeCheckpointTracking {
84
- | None => Some(committedCheckpointId)
95
+ | None => Some(sequence->CheckpointSequence.position(committedFrontier, ~chainId))
85
96
  | Some(tracking) =>
86
97
  tracking->SafeCheckpointTracking.getSafeCheckpointId(
87
98
  ~sourceBlockNumber=cs->ChainState.knownHeight,
@@ -94,20 +105,16 @@ let getSafeCheckpointIdByChain = (crossChainState: t, ~committedCheckpointId) =>
94
105
 
95
106
  let createBatch = (
96
107
  crossChainState: t,
97
- ~processedCheckpointId,
108
+ ~config: Config.t,
109
+ ~frontier,
98
110
  ~batchSizeTarget: int,
99
- ~isRollback: bool,
100
111
  ): Batch.t => {
101
112
  Batch.make(
102
- ~isInReorgThreshold=crossChainState.isInReorgThreshold,
103
- ~checkpointIdBeforeBatch=processedCheckpointId->BigInt.add(
104
- // Since for rollback we have a diff checkpoint id.
105
- // This is needed to currectly overwrite old state
106
- // in an append-only ClickHouse insert.
107
- isRollback ? 1n : 0n,
108
- ),
113
+ ~sequence=config.checkpointSequence,
114
+ ~history=config->HistoryPolicy.decide(~shouldSaveHistory=crossChainState->shouldSaveHistory),
115
+ ~frontier,
109
116
  ~chainsBeforeBatch=crossChainState.chainStates->Utils.Dict.mapValues(
110
- ChainState.toChainBeforeBatch,
117
+ ChainState.toChainBeforeBatch(~isRealtime=crossChainState.isRealtime, ...),
111
118
  ),
112
119
  ~batchSizeTarget,
113
120
  )
@@ -123,8 +130,6 @@ let enterReorgThreshold = (crossChainState: t) => {
123
130
  ->getChainState(crossChainState.chainIds->Array.getUnsafe(i))
124
131
  ->ChainState.enterReorgThreshold
125
132
  }
126
-
127
- crossChainState.isInReorgThreshold = true
128
133
  }
129
134
 
130
135
  // Commit each progressed chain's batch progress, then record whether the whole
@@ -146,17 +151,6 @@ let applyBatchProgress = (crossChainState: t, ~batch: Batch.t, ~blockTimestampNa
146
151
 
147
152
  crossChainState.isCaughtUp =
148
153
  crossChainState.isCaughtUp || (crossChainState->nextItemIsNone && everyChainCaughtUp.contents)
149
-
150
- // A run resumed with every chain already stamped ready needs no finalize —
151
- // the indexes were committed together with those stamps.
152
- let allChainsReady = ref(true)
153
- for i in 0 to chainIds->Array.length - 1 {
154
- if !(crossChainState->getChainState(chainIds->Array.getUnsafe(i))->ChainState.isReady) {
155
- allChainsReady := false
156
- }
157
- }
158
-
159
- crossChainState.isRealtime = crossChainState.isRealtime || allChainsReady.contents
160
154
  }
161
155
 
162
156
  // Every chain has buffered up to its head (or endblock) with nothing
@@ -191,29 +185,25 @@ let markCaughtUpIfSettled = (crossChainState: t) =>
191
185
  // may have moved on — at which point no chain looks at head any more and the
192
186
  // indexes it still owes would wait out another whole backfill. Deciding here,
193
187
  // before any source request, keeps that debt tied to the progress that was
194
- // actually committed. Skipped once every chain carries `ready_at`: those indexes
195
- // were committed together with the stamps.
188
+ // actually committed. Skipped when the run resumed realtime, which is exactly
189
+ // the case where the indexes are already committed and nothing is owed.
196
190
  let markCaughtUpOnResume = (crossChainState: t) => {
197
191
  let everyChainCaughtUp = ref(crossChainState.chainIds->Array.length > 0)
198
- let everyChainReady = ref(true)
199
192
  for i in 0 to crossChainState.chainIds->Array.length - 1 {
200
193
  let cs = crossChainState->getChainState(crossChainState.chainIds->Array.getUnsafe(i))
201
194
  if !(cs->ChainState.isDurablyCaughtUp) {
202
195
  everyChainCaughtUp := false
203
196
  }
204
- if !(cs->ChainState.isReady) {
205
- everyChainReady := false
206
- }
207
197
  }
208
198
 
209
- if everyChainCaughtUp.contents && !everyChainReady.contents && crossChainState->nextItemIsNone {
199
+ if everyChainCaughtUp.contents && !crossChainState.isRealtime && crossChainState->nextItemIsNone {
210
200
  crossChainState.isCaughtUp = true
211
201
  }
212
202
  }
213
203
 
214
- // Concludes the FinalizingIndexes phase: stamps every chain with the `ready_at`
215
- // already committed alongside the deferred schema indexes and switches the
216
- // indexer to realtime.
204
+ // Concludes the FinalizingIndexes phase: stamps every chain this process drives
205
+ // with the `ready_at` already committed alongside their deferred schema indexes,
206
+ // and switches the indexer to realtime.
217
207
  let markReady = (crossChainState: t, ~readyAt) => {
218
208
  for i in 0 to crossChainState.chainIds->Array.length - 1 {
219
209
  crossChainState
@@ -267,6 +257,10 @@ let idleOrWaitAction = (cs: ChainState.t) =>
267
257
  ? FetchState.NothingToQuery
268
258
  : FetchState.WaitingForNewBlock
269
259
 
260
+ // How far past the alignment anchor's frontier progress a follower may fetch,
261
+ // as a fraction of its own alignable range.
262
+ let alignmentMargin = 0.2
263
+
270
264
  // Dispatch a fetch tick across the whole indexer from one shared pool of
271
265
  // ~targetBufferSize ready events, as a waterfall: visit chains furthest-behind
272
266
  // first, hand each the budget remaining at that point (plus its own
@@ -350,11 +344,14 @@ let checkAndFetch = async (
350
344
  (isCold ? Pervasives.min(remaining.contents, coldChainBudget) : remaining.contents) +.
351
345
  cs->ChainState.pendingBudget
352
346
  let maxTargetBlock = switch alignment {
353
- // 10% margin past the anchor's line: chains whose progress tracks the
347
+ // 20% margin past the anchor's line: chains whose progress tracks the
354
348
  // anchor closely would otherwise flap in and out of the clamp on every
355
- // small frontier move, stalling their pipeline every other tick.
349
+ // small frontier move, stalling their pipeline every other tick. The
350
+ // margin is also the headroom a follower keeps buffered while the anchor
351
+ // is mid-fetch, so it has to outlast a slow anchor response, not just
352
+ // absorb frontier jitter.
356
353
  | Some((anchorChainId, progress)) if anchorChainId !== chainId =>
357
- Some(cs->ChainState.blockAtProgress(~progress=progress +. 0.1))
354
+ Some(cs->ChainState.blockAtProgress(~progress=progress +. alignmentMargin))
358
355
  | _ => None
359
356
  }
360
357
  switch cs->ChainState.getNextQuery(~chainTargetItems, ~maxTargetBlock?) {
@@ -404,7 +401,7 @@ let checkAndFetch = async (
404
401
  switch actionByChain->ChainId.Dict.dangerouslyGetNonOption(chainId) {
405
402
  | Some(NothingToQuery)
406
403
  | None => ()
407
- | Some(action) => promises->Array.push(dispatchChain(~chainId=chainId, ~action))
404
+ | Some(action) => promises->Array.push(dispatchChain(~chainId, ~action))
408
405
  }
409
406
  }
410
407
  let _ = await promises->Promise.all