envio 3.5.0-alpha.3 → 3.5.0-rc.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (127) hide show
  1. package/package.json +6 -6
  2. package/src/Batch.res +8 -8
  3. package/src/Batch.res.mjs +4 -3
  4. package/src/BatchProcessing.res +17 -7
  5. package/src/BatchProcessing.res.mjs +8 -4
  6. package/src/ChainFetching.res +20 -19
  7. package/src/ChainFetching.res.mjs +14 -14
  8. package/src/ChainId.res +58 -0
  9. package/src/ChainId.res.mjs +49 -0
  10. package/src/ChainId.resi +27 -0
  11. package/src/ChainMap.res +12 -26
  12. package/src/ChainMap.res.mjs +5 -41
  13. package/src/ChainMap.resi +5 -20
  14. package/src/ChainMetadata.res +1 -1
  15. package/src/ChainMetadata.res.mjs +3 -2
  16. package/src/ChainState.res +35 -20
  17. package/src/ChainState.res.mjs +34 -11
  18. package/src/ChainState.resi +2 -0
  19. package/src/Config.res +41 -24
  20. package/src/Config.res.mjs +17 -14
  21. package/src/ContractRegisterContext.res +1 -1
  22. package/src/ContractRegisterContext.res.mjs +1 -1
  23. package/src/Core.res +1 -0
  24. package/src/CrossChainState.res +61 -16
  25. package/src/CrossChainState.res.mjs +37 -13
  26. package/src/CrossChainState.resi +3 -2
  27. package/src/Env.res +0 -3
  28. package/src/Env.res.mjs +0 -3
  29. package/src/EventConfigBuilder.res +1 -1
  30. package/src/EventProcessing.res +4 -4
  31. package/src/EventProcessing.res.mjs +5 -5
  32. package/src/FetchState.res +261 -89
  33. package/src/FetchState.res.mjs +187 -63
  34. package/src/FinalizeBackfill.res +33 -6
  35. package/src/FinalizeBackfill.res.mjs +19 -1
  36. package/src/HandlerRegister.res +8 -8
  37. package/src/HandlerRegister.res.mjs +5 -4
  38. package/src/HandlerRegister.resi +1 -1
  39. package/src/InMemoryStore.res +4 -0
  40. package/src/InMemoryStore.res.mjs +1 -1
  41. package/src/IndexerLoop.res +9 -2
  42. package/src/IndexerLoop.res.mjs +8 -1
  43. package/src/IndexerState.res +41 -24
  44. package/src/IndexerState.res.mjs +36 -17
  45. package/src/IndexerState.resi +9 -7
  46. package/src/Internal.res +19 -17
  47. package/src/Internal.res.mjs +18 -9
  48. package/src/LoadLayer.res +1 -1
  49. package/src/LoadLayer.res.mjs +2 -1
  50. package/src/LogSelection.res +2 -2
  51. package/src/Main.res +7 -11
  52. package/src/Main.res.mjs +4 -6
  53. package/src/Metrics.res +6 -6
  54. package/src/Metrics.res.mjs +4 -3
  55. package/src/Persistence.res +15 -7
  56. package/src/PgStorage.res +284 -258
  57. package/src/PgStorage.res.mjs +181 -208
  58. package/src/RawEvent.res +1 -2
  59. package/src/RawEvent.res.mjs +1 -1
  60. package/src/Rollback.res +18 -20
  61. package/src/Rollback.res.mjs +9 -9
  62. package/src/RollbackCommit.res +2 -2
  63. package/src/RollbackCommit.res.mjs +3 -3
  64. package/src/SafeCheckpointTracking.res +2 -2
  65. package/src/SimulateDeadInputTracker.res +17 -21
  66. package/src/SimulateDeadInputTracker.res.mjs +9 -8
  67. package/src/SimulateItems.res +8 -10
  68. package/src/SimulateItems.res.mjs +9 -10
  69. package/src/Sink.res +2 -2
  70. package/src/Sink.res.mjs +3 -2
  71. package/src/TestIndexer.res +11 -14
  72. package/src/TestIndexer.res.mjs +14 -10
  73. package/src/UserContext.res +2 -2
  74. package/src/UserContext.res.mjs +2 -1
  75. package/src/Utils.res +0 -4
  76. package/src/Utils.res.mjs +1 -6
  77. package/src/bindings/ClickHouse.res +20 -2
  78. package/src/bindings/ClickHouse.res.mjs +25 -14
  79. package/src/db/EntityHistory.res +1 -1
  80. package/src/db/IndexCatalog.res +248 -0
  81. package/src/db/IndexCatalog.res.mjs +222 -0
  82. package/src/db/IndexDefinition.res +131 -0
  83. package/src/db/IndexDefinition.res.mjs +138 -0
  84. package/src/db/IndexManager.res +145 -0
  85. package/src/db/IndexManager.res.mjs +130 -0
  86. package/src/db/InternalTable.res +50 -26
  87. package/src/db/InternalTable.res.mjs +32 -24
  88. package/src/db/Table.res +11 -1
  89. package/src/db/Table.res.mjs +9 -4
  90. package/src/sources/AddressSet.res +5 -4
  91. package/src/sources/AddressStore.res +25 -14
  92. package/src/sources/AddressStore.res.mjs +10 -12
  93. package/src/sources/Evm.res +2 -2
  94. package/src/sources/Evm.res.mjs +1 -1
  95. package/src/sources/EvmChain.res +3 -3
  96. package/src/sources/EvmChain.res.mjs +3 -3
  97. package/src/sources/EvmHyperSyncSource.res +7 -7
  98. package/src/sources/EvmHyperSyncSource.res.mjs +6 -6
  99. package/src/sources/Fuel.res +2 -2
  100. package/src/sources/Fuel.res.mjs +1 -1
  101. package/src/sources/FuelHyperSyncClient.res +3 -0
  102. package/src/sources/FuelHyperSyncClient.res.mjs +1 -0
  103. package/src/sources/FuelHyperSyncSource.res +5 -7
  104. package/src/sources/FuelHyperSyncSource.res.mjs +5 -5
  105. package/src/sources/HyperSync.resi +2 -2
  106. package/src/sources/HyperSyncClient.res +5 -0
  107. package/src/sources/HyperSyncClient.res.mjs +1 -0
  108. package/src/sources/RpcSource.res +13 -14
  109. package/src/sources/RpcSource.res.mjs +13 -12
  110. package/src/sources/SimulateSource.res +42 -15
  111. package/src/sources/SimulateSource.res.mjs +27 -9
  112. package/src/sources/Source.res +1 -5
  113. package/src/sources/SourceManager.res +12 -12
  114. package/src/sources/SourceManager.res.mjs +11 -10
  115. package/src/sources/SourceManager.resi +2 -2
  116. package/src/sources/Svm.res +4 -5
  117. package/src/sources/Svm.res.mjs +5 -4
  118. package/src/sources/SvmHyperSyncClient.res +3 -0
  119. package/src/sources/SvmHyperSyncClient.res.mjs +1 -0
  120. package/src/sources/SvmHyperSyncSource.res +4 -4
  121. package/src/sources/SvmHyperSyncSource.res.mjs +3 -3
  122. package/src/tui/Tui.res +1 -1
  123. package/src/tui/Tui.res.mjs +2 -1
  124. package/src/tui/components/CustomHooks.res +5 -4
  125. package/src/tui/components/CustomHooks.res.mjs +4 -2
  126. package/src/db/IndexRegistry.res +0 -219
  127. package/src/db/IndexRegistry.res.mjs +0 -195
@@ -2,6 +2,7 @@
2
2
 
3
3
  import * as Env from "./Env.res.mjs";
4
4
  import * as Utils from "./Utils.res.mjs";
5
+ import * as ChainId from "./ChainId.res.mjs";
5
6
  import * as Js_math from "@rescript/runtime/lib/es6/Js_math.js";
6
7
  import * as Logging from "./Logging.res.mjs";
7
8
  import * as AddressSet from "./sources/AddressSet.res.mjs";
@@ -14,6 +15,38 @@ import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
14
15
  import * as Primitive_float from "@rescript/runtime/lib/es6/Primitive_float.js";
15
16
  import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js";
16
17
 
18
+ function deriveSelectionStartBlock(onEventRegistrations) {
19
+ let earliest;
20
+ let idx = 0;
21
+ let unrestricted = Utils.$$Array.isEmpty(onEventRegistrations);
22
+ while (!unrestricted && idx < onEventRegistrations.length) {
23
+ let startBlock = onEventRegistrations[idx].startBlock;
24
+ if (startBlock !== undefined) {
25
+ let current = earliest;
26
+ if (!(current !== undefined && current <= startBlock)) {
27
+ earliest = startBlock;
28
+ }
29
+ } else {
30
+ unrestricted = true;
31
+ }
32
+ idx = idx + 1 | 0;
33
+ };
34
+ if (unrestricted) {
35
+ return;
36
+ } else {
37
+ return earliest;
38
+ }
39
+ }
40
+
41
+ function makeSelection(onEventRegistrations, dependsOnAddresses, clientFilteredContracts) {
42
+ return {
43
+ onEventRegistrations: onEventRegistrations,
44
+ dependsOnAddresses: dependsOnAddresses,
45
+ clientFilteredContracts: clientFilteredContracts,
46
+ startBlock: deriveSelectionStartBlock(onEventRegistrations)
47
+ };
48
+ }
49
+
17
50
  function withAddresses(p, addresses) {
18
51
  return {
19
52
  id: p.id,
@@ -30,6 +63,26 @@ function withAddresses(p, addresses) {
30
63
  };
31
64
  }
32
65
 
66
+ function narrowSelectionToRange(selection, toBlock) {
67
+ if (toBlock === undefined) {
68
+ return selection;
69
+ }
70
+ let inRange = selection.onEventRegistrations.filter(reg => {
71
+ let startBlock = reg.startBlock;
72
+ if (startBlock !== undefined) {
73
+ return startBlock <= toBlock;
74
+ } else {
75
+ return true;
76
+ }
77
+ });
78
+ let length = inRange.length;
79
+ if (length !== 0 && length !== selection.onEventRegistrations.length) {
80
+ return makeSelection(inRange, selection.dependsOnAddresses, selection.clientFilteredContracts);
81
+ } else {
82
+ return selection;
83
+ }
84
+ }
85
+
33
86
  function densityItemsTarget(density, fromBlock, toBlock, chainTargetBlock) {
34
87
  return Primitive_int.max(1, Math.ceil(((Stdlib_Option.getOr(toBlock, chainTargetBlock) - fromBlock | 0) + 1 | 0) * density) | 0);
35
88
  }
@@ -46,6 +99,10 @@ function getTrustedDensity(p) {
46
99
  return p.eventDensity;
47
100
  }
48
101
 
102
+ function isFetching(p) {
103
+ return p.mutPendingQueries.some(pq => pq.fetchedBlock === undefined);
104
+ }
105
+
49
106
  function getMinQueryRange(partitions) {
50
107
  let min = 0;
51
108
  for (let i = 0, i_finish = partitions.length; i < i_finish; ++i) {
@@ -282,7 +339,7 @@ function make(partitions, maxAddrInPartition, nextPartitionIndex, dynamicContrac
282
339
  let p = partitions[idx];
283
340
  let mergeBlock = p.mergeBlock;
284
341
  if (mergeBlock !== undefined) {
285
- if (p.latestFetchedBlock.blockNumber < mergeBlock) {
342
+ if (p.latestFetchedBlock.blockNumber < mergeBlock || isFetching(p)) {
286
343
  newPartitions.push(p);
287
344
  }
288
345
  } else if (p.dynamicContract !== undefined && p.selection.dependsOnAddresses) {
@@ -379,6 +436,21 @@ function make(partitions, maxAddrInPartition, nextPartitionIndex, dynamicContrac
379
436
  latestSourceRangeCapacityUpdateBlock: p.latestSourceRangeCapacityUpdateBlock
380
437
  });
381
438
  return;
439
+ } else if (isFetching(p)) {
440
+ anchored.push({
441
+ id: p.id,
442
+ latestFetchedBlock: p.latestFetchedBlock,
443
+ selection: p.selection,
444
+ addresses: p.addresses,
445
+ mergeBlock: p.latestFetchedBlock.blockNumber,
446
+ dynamicContract: p.dynamicContract,
447
+ mutPendingQueries: p.mutPendingQueries,
448
+ sourceRangeCapacity: p.sourceRangeCapacity,
449
+ prevSourceRangeCapacity: p.prevSourceRangeCapacity,
450
+ eventDensity: p.eventDensity,
451
+ latestSourceRangeCapacityUpdateBlock: p.latestSourceRangeCapacityUpdateBlock
452
+ });
453
+ return;
382
454
  } else {
383
455
  return;
384
456
  }
@@ -505,7 +577,9 @@ function handleQueryResponseForPartition(optimizedPartitions, p, query, knownHei
505
577
  }
506
578
  let updatedLatestFetchedBlock = latestFetchedBlock$1;
507
579
  let mergeBlock = p.mergeBlock;
508
- let partitionReachedMergeBlock = mergeBlock !== undefined ? updatedLatestFetchedBlock.blockNumber >= mergeBlock : false;
580
+ let partitionReachedMergeBlock = (
581
+ mergeBlock !== undefined ? updatedLatestFetchedBlock.blockNumber >= mergeBlock : false
582
+ ) && !isFetching(p);
509
583
  if (partitionReachedMergeBlock) {
510
584
  Utils.Dict.deleteInPlace(mutEntities, p.id);
511
585
  return make(Object.values(mutEntities), optimizedPartitions.maxAddrInPartition, optimizedPartitions.nextPartitionIndex, optimizedPartitions.dynamicContracts, optimizedPartitions.clientFilteredContracts);
@@ -572,12 +646,9 @@ function handleQueryResponseForPartition(optimizedPartitions, p, query, knownHei
572
646
  }
573
647
 
574
648
  function handleQueryResponse(optimizedPartitions, query, knownHeight, itemsCount, latestFetchedBlock) {
575
- let p = optimizedPartitions.entities[query.partitionId];
576
- if (p !== undefined) {
577
- return handleQueryResponseForPartition(optimizedPartitions, p, query, knownHeight, itemsCount, latestFetchedBlock);
578
- } else {
579
- return optimizedPartitions;
580
- }
649
+ let partitionId = query.partitionId;
650
+ let p = optimizedPartitions.entities[partitionId];
651
+ return handleQueryResponseForPartition(optimizedPartitions, p !== undefined ? p : Stdlib_JsError.throwWithMessage(`Unexpected case: Couldn't find partition ` + partitionId), query, knownHeight, itemsCount, latestFetchedBlock);
581
652
  }
582
653
 
583
654
  function getLatestFullyFetchedBlock(optimizedPartitions) {
@@ -836,16 +907,42 @@ function collapseClientFilteredContracts(partitions, clientFilteredContracts, no
836
907
  let absorbedPartitions = [];
837
908
  let strippedFrontiers = [];
838
909
  let anchoredContracts$1 = anchoredContracts(partitions);
910
+ let retire = p => ({
911
+ id: p.id,
912
+ latestFetchedBlock: p.latestFetchedBlock,
913
+ selection: p.selection,
914
+ addresses: p.addresses,
915
+ mergeBlock: p.latestFetchedBlock.blockNumber,
916
+ dynamicContract: p.dynamicContract,
917
+ mutPendingQueries: p.mutPendingQueries,
918
+ sourceRangeCapacity: p.sourceRangeCapacity,
919
+ prevSourceRangeCapacity: p.prevSourceRangeCapacity,
920
+ eventDensity: p.eventDensity,
921
+ latestSourceRangeCapacityUpdateBlock: p.latestSourceRangeCapacityUpdateBlock
922
+ });
923
+ let absorb = p => {
924
+ absorbedPartitions.push(p);
925
+ if (isFetching(p)) {
926
+ kept.push(retire(p));
927
+ return;
928
+ }
929
+ };
839
930
  partitions.forEach(p => {
840
931
  if (!p.selection.dependsOnAddresses) {
841
932
  if (p.mergeBlock !== undefined) {
842
933
  backfills.push(p);
934
+ if (isFetching(p)) {
935
+ kept.push(retire(p));
936
+ return;
937
+ } else {
938
+ return;
939
+ }
843
940
  } else if (Stdlib_Option.isNone(standingRef.contents)) {
844
941
  standingRef.contents = p;
942
+ return;
845
943
  } else {
846
- absorbedPartitions.push(p);
944
+ return absorb(p);
847
945
  }
848
- return;
849
946
  }
850
947
  let contractNames = p.addresses.contractNames();
851
948
  let serverSideNames = contractNames.filter(c => !clientFilteredContracts.has(c));
@@ -856,11 +953,13 @@ function collapseClientFilteredContracts(partitions, clientFilteredContracts, no
856
953
  let contractName = p.dynamicContract;
857
954
  if (contractName !== undefined ? anchoredContracts$1.has(contractName) : false) {
858
955
  kept.push(p);
956
+ return;
859
957
  } else if (Utils.$$Array.isEmpty(serverSideNames)) {
860
- absorbedPartitions.push(p);
958
+ return absorb(p);
861
959
  } else {
862
960
  strippedFrontiers.push(p.latestFetchedBlock);
863
961
  kept.push(withAddresses(p, p.addresses.filterByContracts(serverSideNames)));
962
+ return;
864
963
  }
865
964
  });
866
965
  let standing = standingRef.contents;
@@ -900,19 +999,16 @@ function collapseClientFilteredContracts(partitions, clientFilteredContracts, no
900
999
  }
901
1000
  });
902
1001
  addRegs(normalSelection.onEventRegistrations.filter(reg => clientFilteredContracts.has(reg.eventConfig.contractName)));
903
- let newSelection_onEventRegistrations = Object.values(regByIndex);
904
- let newSelection_clientFilteredContracts = Array.from(clientFilteredContracts);
905
- let newSelection = {
906
- onEventRegistrations: newSelection_onEventRegistrations,
907
- dependsOnAddresses: false,
908
- clientFilteredContracts: newSelection_clientFilteredContracts
909
- };
1002
+ let newSelection = makeSelection(Object.values(regByIndex), false, Array.from(clientFilteredContracts));
910
1003
  let standing$2 = standingRef.contents;
911
1004
  if (standing$2 !== undefined) {
912
1005
  let standingOut;
913
1006
  if (selectionChanged) {
914
1007
  let id = nextPartitionIndexRef.contents.toString();
915
1008
  nextPartitionIndexRef.contents = nextPartitionIndexRef.contents + 1 | 0;
1009
+ if (isFetching(standing$2)) {
1010
+ kept.push(retire(standing$2));
1011
+ }
916
1012
  standingOut = {
917
1013
  id: id,
918
1014
  latestFetchedBlock: standing$2.latestFetchedBlock,
@@ -974,6 +1070,43 @@ function collapseClientFilteredContracts(partitions, clientFilteredContracts, no
974
1070
  return kept;
975
1071
  }
976
1072
 
1073
+ function warnAddressRegistration(chainId, contractAddress, params, message) {
1074
+ Logging.childWarn(Logging.createChild({
1075
+ chainId: chainId,
1076
+ contractAddress: contractAddress,
1077
+ details: params
1078
+ }), message);
1079
+ }
1080
+
1081
+ function warnRejectedRegistration(verdict, chainId, contractAddress, contractName) {
1082
+ if (typeof verdict !== "object") {
1083
+ return warnAddressRegistration(chainId, contractAddress, {
1084
+ contractName: contractName
1085
+ }, `Skipping contract registration: Not a valid address for this chain's ecosystem.`);
1086
+ }
1087
+ switch (verdict.TAG) {
1088
+ case "Added" :
1089
+ case "NoEvents" :
1090
+ return;
1091
+ case "Duplicate" :
1092
+ let existingEffectiveStartBlock = verdict.existingEffectiveStartBlock;
1093
+ let effectiveStartBlock = verdict.effectiveStartBlock;
1094
+ if (existingEffectiveStartBlock > effectiveStartBlock) {
1095
+ return warnAddressRegistration(chainId, contractAddress, {
1096
+ existingBlockNumber: existingEffectiveStartBlock,
1097
+ newBlockNumber: effectiveStartBlock
1098
+ }, `Skipping contract registration: Contract address is already registered at a later block number. Currently registration of the same contract address is not supported by Envio. Reach out to us if it's a problem for you.`);
1099
+ } else {
1100
+ return;
1101
+ }
1102
+ case "Conflict" :
1103
+ return warnAddressRegistration(chainId, contractAddress, {
1104
+ existingContractType: verdict.existingContractName,
1105
+ newContractType: contractName
1106
+ }, `Skipping contract registration: Contract address is already registered for one contract and cannot be registered for another contract.`);
1107
+ }
1108
+ }
1109
+
977
1110
  function createPartitions(registeringSetsByContract, addressStore, dynamicContracts, clientFilteredContracts, normalSelection, maxAddrInPartition, nextPartitionIndex, existingPartitions, progressBlockNumber, knownHeight) {
978
1111
  let nextPartitionIndexRef = {
979
1112
  contents: nextPartitionIndex
@@ -1117,25 +1250,17 @@ function registerDynamicContracts(fetchState, addressStore, claimCeilingOpt, ite
1117
1250
  }
1118
1251
  let idCursor = addressStore.nextId();
1119
1252
  let verdicts = AddressStore.registerBatch(addressStore, registrations);
1120
- let warn = (contractAddress, params, message) => Logging.childWarn(Logging.createChild({
1121
- chainId: fetchState.chainId,
1122
- contractAddress: contractAddress,
1123
- details: params
1124
- }), message);
1125
1253
  let registeringContractNames = [];
1126
1254
  let hasNoEventsUpdatesRef = false;
1127
1255
  for (let idx = 0, idx_finish = verdicts.length; idx < idx_finish; ++idx) {
1128
1256
  let registration = registrations[idx];
1129
- let contractAddress = registration.address;
1130
- let match = verdicts[idx];
1257
+ let verdict = verdicts[idx];
1131
1258
  let keep;
1132
- if (typeof match !== "object") {
1133
- warn(contractAddress, {
1134
- contractName: registration.contractName
1135
- }, `Skipping contract registration: Not a valid address for this chain's ecosystem.`);
1136
- keep = false;
1259
+ let exit = 0;
1260
+ if (typeof verdict !== "object") {
1261
+ exit = 1;
1137
1262
  } else {
1138
- switch (match.TAG) {
1263
+ switch (verdict.TAG) {
1139
1264
  case "Added" :
1140
1265
  if (!registeringContractNames.includes(registration.contractName)) {
1141
1266
  registeringContractNames.push(registration.contractName);
@@ -1144,31 +1269,19 @@ function registerDynamicContracts(fetchState, addressStore, claimCeilingOpt, ite
1144
1269
  break;
1145
1270
  case "NoEvents" :
1146
1271
  hasNoEventsUpdatesRef = true;
1147
- warn(contractAddress, {
1272
+ warnAddressRegistration(fetchState.chainId, registration.address, {
1148
1273
  contractName: registration.contractName
1149
1274
  }, `Persisting contract registration without fetching: Contract doesn't have any events to fetch. It'll be picked up on restart if you add events for the contract.`);
1150
1275
  keep = true;
1151
1276
  break;
1152
- case "Duplicate" :
1153
- let existingEffectiveStartBlock = match.existingEffectiveStartBlock;
1154
- let effectiveStartBlock = match.effectiveStartBlock;
1155
- if (existingEffectiveStartBlock > effectiveStartBlock) {
1156
- warn(contractAddress, {
1157
- existingBlockNumber: existingEffectiveStartBlock,
1158
- newBlockNumber: effectiveStartBlock
1159
- }, `Skipping contract registration: Contract address is already registered at a later block number. Currently registration of the same contract address is not supported by Envio. Reach out to us if it's a problem for you.`);
1160
- }
1161
- keep = false;
1162
- break;
1163
- case "Conflict" :
1164
- warn(contractAddress, {
1165
- existingContractType: match.existingContractName,
1166
- newContractType: registration.contractName
1167
- }, `Skipping contract registration: Contract address is already registered for one contract and cannot be registered for another contract.`);
1168
- keep = false;
1169
- break;
1277
+ default:
1278
+ exit = 1;
1170
1279
  }
1171
1280
  }
1281
+ if (exit === 1) {
1282
+ warnRejectedRegistration(verdict, fetchState.chainId, registration.address, registration.contractName);
1283
+ keep = false;
1284
+ }
1172
1285
  if (!keep) {
1173
1286
  sourceIndexes[idx] = -1 - sourceIndexes[idx] | 0;
1174
1287
  }
@@ -1191,8 +1304,8 @@ function registerDynamicContracts(fetchState, addressStore, claimCeilingOpt, ite
1191
1304
  dynamicContractsRef = Utils.$$Set.immutableAdd(dynamicContractsRef, contractName);
1192
1305
  for (let idx$3 = 0, idx_finish$2 = mutExistingPartitions.length; idx$3 < idx_finish$2; ++idx$3) {
1193
1306
  let p = mutExistingPartitions[idx$3];
1194
- let match$1 = p.addresses.countFor(contractName);
1195
- if (match$1 !== 0 && p.selection.dependsOnAddresses && p.mergeBlock === undefined) {
1307
+ let match = p.addresses.countFor(contractName);
1308
+ if (match !== 0 && p.selection.dependsOnAddresses && p.mergeBlock === undefined) {
1196
1309
  let allPartitionContractNames = p.addresses.contractNames();
1197
1310
  if (allPartitionContractNames.length !== 1) {
1198
1311
  let isFetching = p.mutPendingQueries.length !== 0;
@@ -1389,11 +1502,19 @@ function walkPartitionPending(p, partitionId, inFlightCount, candidates, headBlo
1389
1502
  }
1390
1503
  pqIdx = pqIdx + 1 | 0;
1391
1504
  };
1505
+ let startBlock = p.selection.startBlock;
1506
+ let cursor$1;
1507
+ if (startBlock !== undefined && startBlock > cursor) {
1508
+ let reachable = Utils.$$Math.minOptInt(headBlockNumber, queryEndBlock);
1509
+ cursor$1 = reachable !== undefined && startBlock <= reachable ? startBlock : cursor;
1510
+ } else {
1511
+ cursor$1 = cursor;
1512
+ }
1392
1513
  if (canContinue) {
1393
1514
  return {
1394
1515
  partitionId: partitionId,
1395
1516
  p: p,
1396
- cursor: cursor,
1517
+ cursor: cursor$1,
1397
1518
  chunksUsedThisCall: chunksUsedThisCall,
1398
1519
  inFlightCount: inFlightCount,
1399
1520
  queryEndBlock: queryEndBlock,
@@ -1653,10 +1774,7 @@ function make$1(startBlock, endBlock, onEventRegistrations, addressStore, addres
1653
1774
  partitions.push({
1654
1775
  id: partitions.length.toString(),
1655
1776
  latestFetchedBlock: latestFetchedBlock,
1656
- selection: {
1657
- onEventRegistrations: notDependingOnAddresses,
1658
- dependsOnAddresses: false
1659
- },
1777
+ selection: makeSelection(notDependingOnAddresses, false, undefined),
1660
1778
  addresses: addressStore.emptySet(),
1661
1779
  mergeBlock: undefined,
1662
1780
  dynamicContract: undefined,
@@ -1667,15 +1785,15 @@ function make$1(startBlock, endBlock, onEventRegistrations, addressStore, addres
1667
1785
  latestSourceRangeCapacityUpdateBlock: 0
1668
1786
  });
1669
1787
  }
1670
- let normalSelection = {
1671
- onEventRegistrations: normalRegistrations,
1672
- dependsOnAddresses: true
1673
- };
1788
+ let normalSelection = makeSelection(normalRegistrations, true, undefined);
1674
1789
  AddressStore.registerBatch(addressStore, addresses.map(contract => ({
1675
1790
  address: contract.address,
1676
1791
  contractName: contract.contractName,
1677
1792
  registrationBlock: contract.registrationBlock
1678
- })));
1793
+ }))).forEach((verdict, idx) => {
1794
+ let contract = addresses[idx];
1795
+ warnRejectedRegistration(verdict, chainId, contract.address, contract.contractName);
1796
+ });
1679
1797
  let dynamicContracts = new Set();
1680
1798
  let clientFilteredContracts = new Set();
1681
1799
  let registeringSetsByContract = {};
@@ -1703,7 +1821,7 @@ function make$1(startBlock, endBlock, onEventRegistrations, addressStore, addres
1703
1821
  }
1704
1822
  let optimizedPartitions = createPartitions(registeringSetsByContract, addressStore, dynamicContracts, clientFilteredContracts, normalSelection, maxAddrInPartition, partitions.length, partitions, progressBlockNumber, knownHeight);
1705
1823
  if (optimizedPartitions.idsInAscOrder.length === 0 && Utils.$$Array.isEmpty(onBlockRegistrations)) {
1706
- Stdlib_JsError.throwWithMessage(`Invalid configuration: Nothing to fetch on chain ` + chainId.toString() + `. ` + (`addresses=` + addresses.length.toString() + `, `) + (`onEventRegistrations=` + onEventRegistrations.length.toString() + `, `) + (`normalRegistrations=` + normalRegistrations.length.toString() + `. `) + `Make sure that you provided at least one contract address to index, or have events with Wildcard mode enabled, or have onBlock handlers.`);
1824
+ Stdlib_JsError.throwWithMessage(`Invalid configuration: Nothing to fetch on chain ` + ChainId.toString(chainId) + `. ` + (`addresses=` + addresses.length.toString() + `, `) + (`onEventRegistrations=` + onEventRegistrations.length.toString() + `, `) + (`normalRegistrations=` + normalRegistrations.length.toString() + `. `) + `Make sure that you provided at least one contract address to index, or have events with Wildcard mode enabled, or have onBlock handlers.`);
1707
1825
  }
1708
1826
  let buffer = [];
1709
1827
  let latestOnBlockBlockNumber;
@@ -2020,10 +2138,14 @@ let maxChainConcurrency = Env.maxChainConcurrency;
2020
2138
  let chunkRangeGrowthFactor = 1.8;
2021
2139
 
2022
2140
  export {
2141
+ deriveSelectionStartBlock,
2142
+ makeSelection,
2023
2143
  withAddresses,
2144
+ narrowSelectionToRange,
2024
2145
  densityItemsTarget,
2025
2146
  getMinHistoryRange,
2026
2147
  getTrustedDensity,
2148
+ isFetching,
2027
2149
  getMinQueryRange,
2028
2150
  OptimizedPartitions,
2029
2151
  bufferBlockNumber,
@@ -2038,6 +2160,8 @@ export {
2038
2160
  addClientFilteredContract,
2039
2161
  claimedFetchedBlock,
2040
2162
  collapseClientFilteredContracts,
2163
+ warnAddressRegistration,
2164
+ warnRejectedRegistration,
2041
2165
  createPartitions,
2042
2166
  registerDynamicContracts,
2043
2167
  handleQueryResult,
@@ -1,11 +1,12 @@
1
- // The FinalizingIndexes phase. Runs once, from the processing loop, when every
1
+ // The FinalizingIndexes phase. Reached from the processing loop when every
2
2
  // chain has caught up: processing is already paused (the loop awaits this),
3
- // pending writes are flushed, then storage creates every missing schema-defined
4
- // index and stamps `ready_at` in one transaction. A failure rolls back both, so
5
- // the next batch simply retries the indexer never reports ready with an index
6
- // the schema promised still missing.
3
+ // pending writes are flushed, then storage builds every missing schema-defined
4
+ // index and, once they all verify, commits `ready_at`. A failure part-way
5
+ // leaves the indexes built so far in place and reaches the processing loop's
6
+ // error boundary; the retry only owes what's left. Either way the indexer never
7
+ // reports ready with an index the schema promised still missing.
7
8
 
8
- let run = async (state: IndexerState.t) => {
9
+ let runOnce = async (state: IndexerState.t) => {
9
10
  Logging.info(
10
11
  "All chains are caught up. Finalizing the indexer before switching to realtime: flushing pending writes, then creating the indexes the schema promises.",
11
12
  )
@@ -28,7 +29,33 @@ let run = async (state: IndexerState.t) => {
28
29
  ~readyAt,
29
30
  )
30
31
 
32
+ // Only after the commit: in-memory readiness must never run ahead of the
33
+ // `ready_at` a restart would read back.
31
34
  state->IndexerState.markReady(~readyAt)
32
35
  Logging.info("The indexer is ready. Switching to realtime indexing.")
33
36
  }
34
37
  }
38
+
39
+ // Several paths reach the phase — a processed batch, a tick that progressed
40
+ // nothing — and a height update can bring another one round while the first is
41
+ // still building. They all join the in-flight run rather than starting a second
42
+ // pass over the same indexes.
43
+ let run = (state: IndexerState.t) =>
44
+ switch state->IndexerState.finalizeFiber {
45
+ | Some(fiber) => fiber
46
+ | None =>
47
+ let fiber = runOnce(state)->Promise.finally(() => state->IndexerState.endFinalizeFiber)
48
+ state->IndexerState.beginFinalizeFiber(fiber)
49
+ fiber
50
+ }
51
+
52
+ // An indexer resumed already ready never reaches `run`, so nothing above would
53
+ // notice an index the schema promises being dropped or invalidated while it was
54
+ // down — `ensureQueryIndexes` only covers what a getWhere actually asks for.
55
+ // Best-effort and not awaited by the loop: indexing is already live and correct
56
+ // without the index, just slower.
57
+ let repairSchemaIndexes = (state: IndexerState.t) => {
58
+ let persistence = state->IndexerState.persistence
59
+ let storage = persistence->Persistence.getInitializedStorageOrThrow
60
+ storage.ensureSchemaIndexes(~entities=persistence.allEntities)
61
+ }
@@ -6,7 +6,7 @@ import * as ChainState from "./ChainState.res.mjs";
6
6
  import * as Persistence from "./Persistence.res.mjs";
7
7
  import * as IndexerState from "./IndexerState.res.mjs";
8
8
 
9
- async function run(state) {
9
+ async function runOnce(state) {
10
10
  Logging.info("All chains are caught up. Finalizing the indexer before switching to realtime: flushing pending writes, then creating the indexes the schema promises.");
11
11
  await Writing.flush(state);
12
12
  if (IndexerState.hasFailedWrite(state)) {
@@ -20,7 +20,25 @@ async function run(state) {
20
20
  return Logging.info("The indexer is ready. Switching to realtime indexing.");
21
21
  }
22
22
 
23
+ function run(state) {
24
+ let fiber = IndexerState.finalizeFiber(state);
25
+ if (fiber !== undefined) {
26
+ return fiber;
27
+ }
28
+ let fiber$1 = runOnce(state).finally(() => IndexerState.endFinalizeFiber(state));
29
+ IndexerState.beginFinalizeFiber(state, fiber$1);
30
+ return fiber$1;
31
+ }
32
+
33
+ function repairSchemaIndexes(state) {
34
+ let persistence = IndexerState.persistence(state);
35
+ let storage = Persistence.getInitializedStorageOrThrow(persistence);
36
+ return storage.ensureSchemaIndexes(persistence.allEntities);
37
+ }
38
+
23
39
  export {
40
+ runOnce,
24
41
  run,
42
+ repairSchemaIndexes,
25
43
  }
26
44
  /* Logging Not a pure module */
@@ -82,8 +82,8 @@ let startRegistration = (~config: Config.t) => {
82
82
  }
83
83
  }
84
84
 
85
- let getChainRegistrations = (r: activeRegistration, ~chainId: int): chainRegistrations => {
86
- let key = chainId->Int.toString
85
+ let getChainRegistrations = (r: activeRegistration, ~chainId: ChainId.t): chainRegistrations => {
86
+ let key = chainId->ChainId.toString
87
87
  switch r.registrationsByChainId->Utils.Dict.dangerouslyGetNonOption(key) {
88
88
  | Some(existing) => existing
89
89
  | None =>
@@ -98,7 +98,7 @@ let getChainRegistrations = (r: activeRegistration, ~chainId: int): chainRegistr
98
98
 
99
99
  let buildOnEventRegistrationWith = (
100
100
  ~config: Config.t,
101
- ~chainId: int,
101
+ ~chainId: ChainId.t,
102
102
  ~eventConfig: Internal.eventConfig,
103
103
  ~isWildcard: bool,
104
104
  ~handler: option<Internal.handler>,
@@ -342,10 +342,10 @@ let setContractRegister = (~contractName, ~eventName, contractRegister, ~eventOp
342
342
  }
343
343
 
344
344
  // Raw onEvent registrations stored for a chain (empty if the chain has none).
345
- let storedOnEventRegistrations = (r: activeRegistration, ~chainId: int): array<
345
+ let storedOnEventRegistrations = (r: activeRegistration, ~chainId: ChainId.t): array<
346
346
  Internal.onEventRegistration,
347
347
  > =>
348
- switch r.registrationsByChainId->Utils.Dict.dangerouslyGetNonOption(chainId->Int.toString) {
348
+ switch r.registrationsByChainId->Utils.Dict.dangerouslyGetNonOption(chainId->ChainId.toString) {
349
349
  | Some(chainRegs) => chainRegs.onEventRegistrations
350
350
  | None => []
351
351
  }
@@ -373,7 +373,7 @@ let isWildcard = (~contractName, ~eventName) =>
373
373
  // item still produces an item to run.
374
374
  let getSimulateOnEventRegistrations = (
375
375
  ~config: Config.t,
376
- ~chainId: int,
376
+ ~chainId: ChainId.t,
377
377
  ~eventConfig: Internal.eventConfig,
378
378
  ): array<Internal.onEventRegistration> => {
379
379
  let stored = switch getActiveRegistration() {
@@ -412,7 +412,7 @@ let finishRegistration = (~config: Config.t): registrationsByChainId => {
412
412
  ->ChainMap.values
413
413
  ->Array.forEach(chainConfig => {
414
414
  let chainId = chainConfig.id
415
- let key = chainId->Int.toString
415
+ let key = chainId->ChainId.toString
416
416
 
417
417
  let builtRegs = mergeRegistrations(r->storedOnEventRegistrations(~chainId), ~config)
418
418
  let registeredKeys = Utils.Set.make()
@@ -638,7 +638,7 @@ let registerOnBlock = (
638
638
  ->ChainMap.values
639
639
  ->Array.forEach(chainConfig => {
640
640
  let chainId = chainConfig.id
641
- let chainObj = chainsDict->Dict.getUnsafe(chainId->Int.toString)
641
+ let chainObj = chainsDict->Dict.getUnsafe(chainId->ChainId.toString)
642
642
 
643
643
  // Predicate returns `true` → match with no filter; `false` → skip;
644
644
  // any plain object → structured filter. `undefined`/`null` returns
@@ -1,6 +1,7 @@
1
1
  // Generated by ReScript, PLEASE EDIT WITH CARE
2
2
 
3
3
  import * as Utils from "./Utils.res.mjs";
4
+ import * as ChainId from "./ChainId.res.mjs";
4
5
  import * as Logging from "./Logging.res.mjs";
5
6
  import * as ChainMap from "./ChainMap.res.mjs";
6
7
  import * as EnvioGlobal from "./EnvioGlobal.res.mjs";
@@ -54,7 +55,7 @@ function startRegistration(config) {
54
55
  }
55
56
 
56
57
  function getChainRegistrations(r, chainId) {
57
- let key = chainId.toString();
58
+ let key = ChainId.toString(chainId);
58
59
  let existing = r.registrationsByChainId[key];
59
60
  if (existing !== undefined) {
60
61
  return existing;
@@ -220,7 +221,7 @@ function setContractRegister(contractName, eventName, contractRegister, eventOpt
220
221
  }
221
222
 
222
223
  function storedOnEventRegistrations(r, chainId) {
223
- let chainRegs = r.registrationsByChainId[chainId.toString()];
224
+ let chainRegs = r.registrationsByChainId[ChainId.toString(chainId)];
224
225
  if (chainRegs !== undefined) {
225
226
  return chainRegs.onEventRegistrations;
226
227
  } else {
@@ -270,7 +271,7 @@ function finishRegistration(config) {
270
271
  let registrationsByChainId = {};
271
272
  ChainMap.values(config.chainMap).forEach(chainConfig => {
272
273
  let chainId = chainConfig.id;
273
- let key = chainId.toString();
274
+ let key = ChainId.toString(chainId);
274
275
  let builtRegs = mergeRegistrations(storedOnEventRegistrations(r, chainId), config);
275
276
  let registeredKeys = new Set();
276
277
  builtRegs.forEach(reg => {
@@ -385,7 +386,7 @@ function registerOnBlock(name, where, handler, getChainsObject) {
385
386
  };
386
387
  ChainMap.values(config.chainMap).forEach(chainConfig => {
387
388
  let chainId = chainConfig.id;
388
- let chainObj = chainsDict[chainId.toString()];
389
+ let chainObj = chainsDict[ChainId.toString(chainId)];
389
390
  let result = where$1 !== undefined ? where$1({
390
391
  chain: chainObj
391
392
  }) : true;
@@ -26,7 +26,7 @@ let isWildcard: (~contractName: string, ~eventName: string) => bool
26
26
  let isDroppedByWhere: (~config: Config.t, Internal.onEventRegistration) => bool
27
27
  let getSimulateOnEventRegistrations: (
28
28
  ~config: Config.t,
29
- ~chainId: int,
29
+ ~chainId: ChainId.t,
30
30
  ~eventConfig: Internal.eventConfig,
31
31
  ) => array<Internal.onEventRegistration>
32
32
 
@@ -108,7 +108,11 @@ let prepareRollbackDiff = async (
108
108
  let deletedEntities = Dict.make()
109
109
  let setEntities = Dict.make()
110
110
 
111
+ // Rollback data comes from Postgres entity history, which is kept only for
112
+ // Postgres-backed entities. ClickHouse-only entities have no history to
113
+ // restore from, so a reorg leaves them un-reverted in the sink.
111
114
  let _ = await persistence.allEntities
115
+ ->Array.filter(entityConfig => entityConfig.storage.postgres)
112
116
  ->Array.map(async entityConfig => {
113
117
  let entityTable = state->getInMemTable(~entityConfig)
114
118
 
@@ -86,7 +86,7 @@ async function prepareRollbackDiff(state, rollbackTargetCheckpointId, rollbackDi
86
86
  let committedCheckpointId = IndexerState.committedCheckpointId(state);
87
87
  let deletedEntities = {};
88
88
  let setEntities = {};
89
- await Promise.all(persistence.allEntities.map(async entityConfig => {
89
+ await Promise.all(persistence.allEntities.filter(entityConfig => entityConfig.storage.postgres).map(async entityConfig => {
90
90
  let entityTable = getInMemTable(state, entityConfig);
91
91
  let match = await persistence.storage.getRollbackData(entityConfig, rollbackTargetCheckpointId);
92
92
  match[0].forEach(entityId => {