envio 3.10.0 → 3.11.0-svm-alpha.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (105) hide show
  1. package/evm.schema.json +21 -4
  2. package/fuel.schema.json +21 -4
  3. package/index.d.ts +10 -6
  4. package/package.json +6 -6
  5. package/src/Batch.res +100 -78
  6. package/src/Batch.res.mjs +52 -46
  7. package/src/BatchProcessing.res +1 -11
  8. package/src/BatchProcessing.res.mjs +1 -4
  9. package/src/ChainState.res +46 -87
  10. package/src/ChainState.res.mjs +32 -66
  11. package/src/ChainState.resi +3 -0
  12. package/src/Config.res +131 -55
  13. package/src/Config.res.mjs +132 -69
  14. package/src/CrossChainState.res +39 -49
  15. package/src/CrossChainState.res.mjs +24 -22
  16. package/src/CrossChainState.resi +6 -13
  17. package/src/EffectState.res +13 -4
  18. package/src/EffectState.resi +7 -1
  19. package/src/EventConfigBuilder.res +14 -10
  20. package/src/EventConfigBuilder.res.mjs +6 -4
  21. package/src/EventProcessing.res +6 -6
  22. package/src/EventProcessing.res.mjs +6 -6
  23. package/src/FetchState.res +14 -0
  24. package/src/FetchState.res.mjs +14 -0
  25. package/src/FinalizeBackfill.res +16 -12
  26. package/src/FinalizeBackfill.res.mjs +3 -3
  27. package/src/HandlerRegister.res +12 -11
  28. package/src/HandlerRegister.res.mjs +6 -5
  29. package/src/HistoryPolicy.res +46 -0
  30. package/src/HistoryPolicy.res.mjs +50 -0
  31. package/src/InMemoryStore.res +30 -32
  32. package/src/InMemoryStore.res.mjs +33 -28
  33. package/src/IndexerState.res +66 -45
  34. package/src/IndexerState.res.mjs +57 -29
  35. package/src/IndexerState.resi +8 -13
  36. package/src/Internal.res +24 -6
  37. package/src/Internal.res.mjs +9 -0
  38. package/src/LoadLayer.res +11 -4
  39. package/src/LoadLayer.res.mjs +8 -5
  40. package/src/Main.res +33 -25
  41. package/src/Main.res.mjs +25 -26
  42. package/src/Metrics.res +41 -1
  43. package/src/Metrics.res.mjs +25 -3
  44. package/src/Persistence.res +65 -17
  45. package/src/Persistence.res.mjs +23 -4
  46. package/src/PgStorage.res +192 -119
  47. package/src/PgStorage.res.mjs +130 -73
  48. package/src/PruneStaleHistory.res +9 -15
  49. package/src/PruneStaleHistory.res.mjs +11 -20
  50. package/src/Rollback.res +8 -14
  51. package/src/Rollback.res.mjs +4 -3
  52. package/src/SimulateItems.res +13 -6
  53. package/src/SimulateItems.res.mjs +7 -6
  54. package/src/Sink.res +7 -5
  55. package/src/Sink.res.mjs +4 -4
  56. package/src/TestIndexer.res +10 -10
  57. package/src/TestIndexer.res.mjs +11 -10
  58. package/src/UserContext.res +7 -4
  59. package/src/UserContext.res.mjs +5 -2
  60. package/src/Utils.res +9 -0
  61. package/src/Utils.res.mjs +9 -0
  62. package/src/Writing.res +34 -20
  63. package/src/Writing.res.mjs +15 -20
  64. package/src/bindings/ClickHouse.res +24 -8
  65. package/src/bindings/ClickHouse.res.mjs +20 -10
  66. package/src/bindings/ClickHouseSink.res +16 -5
  67. package/src/bindings/ClickHouseSink.res.mjs +3 -1
  68. package/src/db/CheckpointSequence.res +109 -0
  69. package/src/db/CheckpointSequence.res.mjs +99 -0
  70. package/src/db/EntityHistory.res +6 -5
  71. package/src/db/EntityHistory.res.mjs +5 -5
  72. package/src/db/Frontier.res +86 -0
  73. package/src/db/Frontier.res.mjs +126 -0
  74. package/src/db/InternalTable.res +112 -21
  75. package/src/db/InternalTable.res.mjs +95 -28
  76. package/src/db/RollbackFloors.res +34 -41
  77. package/src/db/RollbackFloors.res.mjs +27 -71
  78. package/src/sources/ChainSources.res +72 -0
  79. package/src/sources/ChainSources.res.mjs +55 -0
  80. package/src/sources/EvmHyperSyncSource.res +13 -23
  81. package/src/sources/EvmHyperSyncSource.res.mjs +11 -42
  82. package/src/sources/FuelHyperSyncSource.res +10 -18
  83. package/src/sources/FuelHyperSyncSource.res.mjs +8 -38
  84. package/src/sources/HyperSync.res +33 -0
  85. package/src/sources/HyperSync.res.mjs +32 -0
  86. package/src/sources/HyperSync.resi +17 -0
  87. package/src/sources/HyperSyncClient.res +3 -0
  88. package/src/sources/RequestStat.res +8 -1
  89. package/src/sources/Source.res +4 -3
  90. package/src/sources/SourceManager.res +30 -10
  91. package/src/sources/SourceManager.res.mjs +24 -7
  92. package/src/sources/SourceManager.resi +2 -0
  93. package/src/sources/StartBlockResolver.res +173 -0
  94. package/src/sources/StartBlockResolver.res.mjs +134 -0
  95. package/src/sources/Svm.res +0 -51
  96. package/src/sources/Svm.res.mjs +0 -46
  97. package/src/sources/SvmHyperSyncSource.res +20 -9
  98. package/src/sources/SvmHyperSyncSource.res.mjs +34 -12
  99. package/src/tui/Tui.res +56 -143
  100. package/src/tui/Tui.res.mjs +70 -166
  101. package/src/tui/components/TuiData.res +61 -16
  102. package/src/tui/components/TuiData.res.mjs +53 -15
  103. package/svm.schema.json +80 -73
  104. package/src/db/CheckpointBounds.res +0 -53
  105. package/src/db/CheckpointBounds.res.mjs +0 -44
package/src/Batch.res.mjs CHANGED
@@ -1,34 +1,39 @@
1
1
  // Generated by ReScript, PLEASE EDIT WITH CARE
2
2
 
3
3
  import * as ChainId from "./ChainId.res.mjs";
4
+ import * as Frontier from "./db/Frontier.res.mjs";
4
5
  import * as FetchState from "./FetchState.res.mjs";
6
+ import * as CheckpointSequence from "./db/CheckpointSequence.res.mjs";
5
7
 
6
- function getChainAfterBatchIfProgressed(chainBeforeBatch, progressBlockNumberAfterBatch, fetchStateAfterBatch, batchSize) {
8
+ function getChainAfterBatchIfProgressed(chainBeforeBatch, progressBlockNumberAfterBatch, fetchStateAfterBatch, counts) {
7
9
  if (chainBeforeBatch.progressBlockNumber < progressBlockNumberAfterBatch) {
8
10
  return {
9
- batchSize: batchSize,
11
+ batchSize: counts.size,
10
12
  progressBlockNumber: progressBlockNumberAfterBatch,
11
13
  sourceBlockNumber: chainBeforeBatch.sourceBlockNumber,
12
- totalEventsProcessed: chainBeforeBatch.totalEventsProcessed + batchSize,
14
+ totalEventsProcessed: chainBeforeBatch.totalEventsProcessed + counts.eventsProcessed,
13
15
  fetchState: fetchStateAfterBatch,
14
16
  isProgressAtHeadWhenBatchCreated: progressBlockNumberAfterBatch >= chainBeforeBatch.sourceBlockNumber - chainBeforeBatch.chainConfig.blockLag
15
17
  };
16
18
  }
17
19
  }
18
20
 
19
- function getProgressedChainsById(chainsBeforeBatch, batchSizePerChain, progressBlockNumberPerChain) {
21
+ function getProgressedChainsById(chainsBeforeBatch, countsPerChain, progressBlockNumberPerChain) {
20
22
  let progressedChainsById = {};
21
23
  Object.values(chainsBeforeBatch).forEach(chainBeforeBatch => {
22
24
  let fetchState = chainBeforeBatch.fetchState;
23
25
  let progressBlockNumber = progressBlockNumberPerChain[ChainId.toString(fetchState.chainId)];
24
26
  let progressBlockNumberAfterBatch = progressBlockNumber !== undefined ? progressBlockNumber : chainBeforeBatch.progressBlockNumber;
25
- let batchSize = batchSizePerChain[ChainId.toString(fetchState.chainId)];
27
+ let counts = countsPerChain[ChainId.toString(fetchState.chainId)];
26
28
  let progressedChain;
27
- if (batchSize !== undefined) {
28
- let leftItems = fetchState.buffer.slice(batchSize);
29
- progressedChain = getChainAfterBatchIfProgressed(chainBeforeBatch, progressBlockNumberAfterBatch, FetchState.updateInternal(fetchState, undefined, leftItems, undefined, undefined, undefined), batchSize);
29
+ if (counts !== undefined) {
30
+ let leftItems = fetchState.buffer.slice(counts.size);
31
+ progressedChain = getChainAfterBatchIfProgressed(chainBeforeBatch, progressBlockNumberAfterBatch, FetchState.updateInternal(fetchState, undefined, leftItems, undefined, undefined, undefined), counts);
30
32
  } else {
31
- progressedChain = getChainAfterBatchIfProgressed(chainBeforeBatch, progressBlockNumberAfterBatch, chainBeforeBatch.fetchState, 0);
33
+ progressedChain = getChainAfterBatchIfProgressed(chainBeforeBatch, progressBlockNumberAfterBatch, chainBeforeBatch.fetchState, {
34
+ size: 0,
35
+ eventsProcessed: 0
36
+ });
32
37
  }
33
38
  if (progressedChain !== undefined) {
34
39
  progressedChainsById[chainBeforeBatch.fetchState.chainId] = progressedChain;
@@ -52,145 +57,146 @@ function seekFirstAbove(blockNumbers, blockNumber) {
52
57
  return low;
53
58
  }
54
59
 
55
- function addReorgCheckpoints(prevCheckpointId, scannedHashes, shouldRollbackOnReorg, fromBlockExclusive, toBlockExclusive, chainId, mutCheckpointIds, mutCheckpointChainIds, mutCheckpointBlockNumbers, mutCheckpointBlockHashes, mutCheckpointEventsProcessed) {
60
+ function addReorgCheckpoints(cursor, scannedHashes, shouldRollbackOnReorg, fromBlockExclusive, toBlockExclusive, chainId, mutCheckpointIds, mutCheckpointChainIds, mutCheckpointBlockNumbers, mutCheckpointBlockHashes, mutCheckpointItemsCount, mutCheckpointEventsProcessed) {
56
61
  if (!shouldRollbackOnReorg) {
57
- return prevCheckpointId;
62
+ return;
58
63
  }
59
- let prevCheckpointId$1 = prevCheckpointId;
60
64
  let blockNumbers = scannedHashes.blockNumbers;
61
65
  let length = blockNumbers.length;
62
66
  let idx = seekFirstAbove(blockNumbers, fromBlockExclusive);
63
67
  while (idx < length && blockNumbers[idx] < toBlockExclusive) {
64
68
  let blockNumber = blockNumbers[idx];
65
69
  let hash = scannedHashes.hashByBlockNumber[blockNumber];
66
- let checkpointId = prevCheckpointId$1 + 1n;
67
- prevCheckpointId$1 = checkpointId;
70
+ let checkpointId = CheckpointSequence.next(cursor, chainId);
68
71
  mutCheckpointIds.push(checkpointId);
69
72
  mutCheckpointChainIds.push(chainId);
70
73
  mutCheckpointBlockNumbers.push(blockNumber);
71
74
  mutCheckpointBlockHashes.push(hash);
75
+ mutCheckpointItemsCount.push(0);
72
76
  mutCheckpointEventsProcessed.push(0);
73
77
  idx = idx + 1;
74
78
  };
75
- return prevCheckpointId$1;
76
79
  }
77
80
 
78
- function prepareBatch(checkpointIdBeforeBatch, chainsBeforeBatch, batchSizeTarget, isInReorgThreshold) {
81
+ function make(sequence, frontier, chainsBeforeBatch, batchSizeTarget, history) {
79
82
  let preparedFetchStates = FetchState.sortForBatch(Object.values(chainsBeforeBatch).map(chainBeforeBatch => chainBeforeBatch.fetchState), batchSizeTarget);
80
83
  let chainIdx = 0;
81
84
  let preparedNumber = preparedFetchStates.length;
82
85
  let totalBatchSize = 0;
83
- let prevCheckpointId = checkpointIdBeforeBatch;
84
- let mutBatchSizePerChain = {};
86
+ let cursor = CheckpointSequence.cursor(sequence, frontier);
87
+ let mutCountsPerChain = {};
85
88
  let mutProgressBlockNumberPerChain = {};
86
89
  let items = [];
87
90
  let checkpointIds = [];
88
91
  let checkpointChainIds = [];
89
92
  let checkpointBlockNumbers = [];
90
93
  let checkpointBlockHashes = [];
94
+ let checkpointItemsCount = [];
91
95
  let checkpointEventsProcessed = [];
92
96
  while (totalBatchSize < batchSizeTarget && chainIdx < preparedNumber) {
93
97
  let fetchState = preparedFetchStates[chainIdx];
94
98
  let chainBatchSize = FetchState.getReadyItemsCount(fetchState, batchSizeTarget - totalBatchSize, 0);
95
99
  let chainBeforeBatch = chainsBeforeBatch[fetchState.chainId];
96
100
  let prevBlockNumber = chainBeforeBatch.progressBlockNumber;
101
+ let chainEventsProcessed = 0;
97
102
  if (chainBatchSize > 0) {
98
103
  for (let idx = 0, idx_finish = chainBatchSize - 1; idx <= idx_finish; ++idx) {
99
104
  let item = fetchState.buffer[idx];
100
105
  let blockNumber = item.blockNumber;
106
+ let isNewEvent = idx === 0 || !FetchState.isSameLog(fetchState.buffer[idx - 1], item);
107
+ if (isNewEvent) {
108
+ chainEventsProcessed = chainEventsProcessed + 1;
109
+ }
101
110
  if (blockNumber !== prevBlockNumber) {
102
111
  let chainId = fetchState.chainId;
103
112
  let fromBlockExclusive = prevBlockNumber;
104
113
  let scannedHashes = chainBeforeBatch.scannedHashes;
105
- let prevCheckpointId$1 = prevCheckpointId;
106
- let tmp;
107
114
  if (chainBeforeBatch.shouldRollbackOnReorg) {
108
- let prevCheckpointId$2 = prevCheckpointId$1;
109
115
  let blockNumbers = scannedHashes.blockNumbers;
110
116
  let length = blockNumbers.length;
111
117
  let idx$1 = seekFirstAbove(blockNumbers, fromBlockExclusive);
112
118
  while (idx$1 < length && blockNumbers[idx$1] < blockNumber) {
113
119
  let blockNumber$1 = blockNumbers[idx$1];
114
120
  let hash = scannedHashes.hashByBlockNumber[blockNumber$1];
115
- let checkpointId = prevCheckpointId$2 + 1n;
116
- prevCheckpointId$2 = checkpointId;
121
+ let checkpointId = CheckpointSequence.next(cursor, chainId);
117
122
  checkpointIds.push(checkpointId);
118
123
  checkpointChainIds.push(chainId);
119
124
  checkpointBlockNumbers.push(blockNumber$1);
120
125
  checkpointBlockHashes.push(hash);
126
+ checkpointItemsCount.push(0);
121
127
  checkpointEventsProcessed.push(0);
122
128
  idx$1 = idx$1 + 1;
123
129
  };
124
- tmp = prevCheckpointId$2;
125
- } else {
126
- tmp = prevCheckpointId$1;
127
130
  }
128
- prevCheckpointId = tmp;
129
- let checkpointId$1 = prevCheckpointId + 1n;
131
+ let checkpointId$1 = CheckpointSequence.next(cursor, fetchState.chainId);
130
132
  checkpointIds.push(checkpointId$1);
131
133
  checkpointChainIds.push(fetchState.chainId);
132
134
  checkpointBlockNumbers.push(blockNumber);
133
135
  let hash$1 = chainBeforeBatch.scannedHashes.hashByBlockNumber[blockNumber];
134
136
  checkpointBlockHashes.push(hash$1 !== undefined ? hash$1 : null);
137
+ checkpointItemsCount.push(1);
135
138
  checkpointEventsProcessed.push(1);
136
139
  prevBlockNumber = blockNumber;
137
- prevCheckpointId = checkpointId$1;
138
140
  } else {
139
- let lastIndex = checkpointEventsProcessed.length - 1;
140
- checkpointEventsProcessed[lastIndex] = checkpointEventsProcessed[lastIndex] + 1;
141
+ let lastIndex = checkpointItemsCount.length - 1;
142
+ checkpointItemsCount[lastIndex] = checkpointItemsCount[lastIndex] + 1;
143
+ if (isNewEvent) {
144
+ checkpointEventsProcessed[lastIndex] = checkpointEventsProcessed[lastIndex] + 1;
145
+ }
141
146
  }
142
147
  items.push(item);
143
148
  }
144
149
  totalBatchSize = totalBatchSize + chainBatchSize;
145
- mutBatchSizePerChain[fetchState.chainId] = chainBatchSize;
150
+ mutCountsPerChain[fetchState.chainId] = {
151
+ size: chainBatchSize,
152
+ eventsProcessed: chainEventsProcessed
153
+ };
146
154
  }
147
155
  let progressBlockNumberAfterBatch = FetchState.getProgressBlockNumberAt(fetchState, chainBatchSize);
148
156
  let chainId$1 = fetchState.chainId;
149
157
  let toBlockExclusive = progressBlockNumberAfterBatch + 1;
150
158
  let fromBlockExclusive$1 = prevBlockNumber;
151
159
  let scannedHashes$1 = chainBeforeBatch.scannedHashes;
152
- let prevCheckpointId$3 = prevCheckpointId;
153
- let tmp$1;
154
160
  if (chainBeforeBatch.shouldRollbackOnReorg) {
155
- let prevCheckpointId$4 = prevCheckpointId$3;
156
161
  let blockNumbers$1 = scannedHashes$1.blockNumbers;
157
162
  let length$1 = blockNumbers$1.length;
158
163
  let idx$2 = seekFirstAbove(blockNumbers$1, fromBlockExclusive$1);
159
164
  while (idx$2 < length$1 && blockNumbers$1[idx$2] < toBlockExclusive) {
160
165
  let blockNumber$2 = blockNumbers$1[idx$2];
161
166
  let hash$2 = scannedHashes$1.hashByBlockNumber[blockNumber$2];
162
- let checkpointId$2 = prevCheckpointId$4 + 1n;
163
- prevCheckpointId$4 = checkpointId$2;
167
+ let checkpointId$2 = CheckpointSequence.next(cursor, chainId$1);
164
168
  checkpointIds.push(checkpointId$2);
165
169
  checkpointChainIds.push(chainId$1);
166
170
  checkpointBlockNumbers.push(blockNumber$2);
167
171
  checkpointBlockHashes.push(hash$2);
172
+ checkpointItemsCount.push(0);
168
173
  checkpointEventsProcessed.push(0);
169
174
  idx$2 = idx$2 + 1;
170
175
  };
171
- tmp$1 = prevCheckpointId$4;
172
- } else {
173
- tmp$1 = prevCheckpointId$3;
174
176
  }
175
- prevCheckpointId = tmp$1;
176
177
  mutProgressBlockNumberPerChain[fetchState.chainId] = progressBlockNumberAfterBatch;
177
178
  chainIdx = chainIdx + 1;
178
179
  };
179
180
  return {
180
181
  totalBatchSize: totalBatchSize,
181
182
  items: items,
182
- progressedChainsById: getProgressedChainsById(chainsBeforeBatch, mutBatchSizePerChain, mutProgressBlockNumberPerChain),
183
- isInReorgThreshold: isInReorgThreshold,
183
+ progressedChainsById: getProgressedChainsById(chainsBeforeBatch, mutCountsPerChain, mutProgressBlockNumberPerChain),
184
+ history: history,
184
185
  checkpointIds: checkpointIds,
185
186
  checkpointChainIds: checkpointChainIds,
186
187
  checkpointBlockNumbers: checkpointBlockNumbers,
187
188
  checkpointBlockHashes: checkpointBlockHashes,
189
+ checkpointItemsCount: checkpointItemsCount,
188
190
  checkpointEventsProcessed: checkpointEventsProcessed,
189
191
  registeredAddresses: []
190
192
  };
191
193
  }
192
194
 
193
- let make = prepareBatch;
195
+ function checkpointFrontier(batch) {
196
+ let frontier = Frontier.empty();
197
+ batch.checkpointChainIds.forEach((chainId, index) => Frontier.set(frontier, chainId, batch.checkpointIds[index]));
198
+ return frontier;
199
+ }
194
200
 
195
201
  function findFirstEventBlockNumber(batch, chainId) {
196
202
  let idx = 0;
@@ -198,7 +204,7 @@ function findFirstEventBlockNumber(batch, chainId) {
198
204
  let checkpointsLength = batch.checkpointIds.length;
199
205
  while (idx < checkpointsLength && result === undefined) {
200
206
  let checkpointChainId = batch.checkpointChainIds[idx];
201
- if (checkpointChainId === chainId && batch.checkpointEventsProcessed[idx] > 0) {
207
+ if (checkpointChainId === chainId && batch.checkpointItemsCount[idx] > 0) {
202
208
  result = batch.checkpointBlockNumbers[idx];
203
209
  } else {
204
210
  idx = idx + 1;
@@ -225,8 +231,8 @@ export {
225
231
  getProgressedChainsById,
226
232
  seekFirstAbove,
227
233
  addReorgCheckpoints,
228
- prepareBatch,
229
234
  make,
235
+ checkpointFrontier,
230
236
  findFirstEventBlockNumber,
231
237
  findLastEventItem,
232
238
  }
@@ -69,17 +69,7 @@ and processNextBatch = async (state: IndexerState.t, ~scheduleFetch): unit => {
69
69
  let isInReorgThresholdBeforeUpdate = state->IndexerState.isInReorgThreshold
70
70
  let isRealtimeBeforeUpdate = state->IndexerState.isRealtime
71
71
 
72
- let isRollbackBatch = switch state->IndexerState.rollbackState {
73
- | RollbackReady(_) => true
74
- | _ => false
75
- }
76
-
77
- let batch =
78
- state->IndexerState.createBatch(
79
- ~processedCheckpointId=state->IndexerState.processedCheckpointId,
80
- ~batchSizeTarget=(state->IndexerState.config).batchSize,
81
- ~isRollback=isRollbackBatch,
82
- )
72
+ let batch = state->IndexerState.createBatch(~batchSizeTarget=(state->IndexerState.config).batchSize)
83
73
 
84
74
  let progressedChainsById = batch.progressedChainsById
85
75
 
@@ -23,10 +23,7 @@ function yieldTick() {
23
23
  async function processNextBatch(state, scheduleFetch) {
24
24
  let isInReorgThresholdBeforeUpdate = IndexerState.isInReorgThreshold(state);
25
25
  let isRealtimeBeforeUpdate = IndexerState.isRealtime(state);
26
- let match = IndexerState.rollbackState(state);
27
- let isRollbackBatch;
28
- isRollbackBatch = typeof match !== "object" ? false : match.TAG === "RollbackReady";
29
- let batch = IndexerState.createBatch(state, IndexerState.processedCheckpointId(state), IndexerState.config(state).batchSize, isRollbackBatch);
26
+ let batch = IndexerState.createBatch(state, IndexerState.config(state).batchSize);
30
27
  let progressedChainsById = batch.progressedChainsById;
31
28
  let isBelowReorgThreshold = !isInReorgThresholdBeforeUpdate && IndexerState.config(state).shouldRollbackOnReorg;
32
29
  let shouldEnterReorgThreshold = isBelowReorgThreshold && Object.values(IndexerState.chainStates(state)).every(cs => ChainState.isReadyToEnterReorgThresholdAfterBatch(cs, batch));
@@ -41,6 +41,8 @@ type t = {
41
41
  // detected reorg either rolls back or is only logged.
42
42
  shouldRollbackOnReorg: bool,
43
43
  maxReorgDepth: int,
44
+ // One-way: past it, everything the chain writes can still be rolled back.
45
+ mutable isInReorgThreshold: bool,
44
46
  // Holds this chain's transactions (kept in Rust) keyed by (blockNumber,
45
47
  // transactionIndex). Fetch responses merge their page in; entries are pruned
46
48
  // as the chain progresses and dropped above the target on rollback.
@@ -126,6 +128,7 @@ let make = (
126
128
  ~safeCheckpointTracking=None,
127
129
  ~shouldRollbackOnReorg,
128
130
  ~maxReorgDepth,
131
+ ~isInReorgThreshold=false,
129
132
  ~numEventsProcessed=0.,
130
133
  ~timestampCaughtUpToHeadOrEndblock=None,
131
134
  ~isProgressAtHead=false,
@@ -155,6 +158,7 @@ let make = (
155
158
  safeCheckpointTracking,
156
159
  shouldRollbackOnReorg,
157
160
  maxReorgDepth,
161
+ isInReorgThreshold,
158
162
  transactionStore,
159
163
  blockStore,
160
164
  reorgThresholdReadyTolerance,
@@ -203,7 +207,9 @@ let makeInternal = (
203
207
 
204
208
  chainConfig.contracts->Array.forEach(contract => {
205
209
  switch contract.startBlock {
206
- | Some(startBlock) if startBlock < chainConfig.startBlock =>
210
+ // Against the resolved `~startBlock`, which came from storage:
211
+ // `chainConfig.startBlock` still says whatever config.yaml said.
212
+ | Some(contractStartBlock) if contractStartBlock < startBlock =>
207
213
  JsError.throwWithMessage(
208
214
  `The start block for contract "${contract.name}" is less than the chain start block. This is not supported yet.`,
209
215
  )
@@ -262,78 +268,12 @@ let makeInternal = (
262
268
  })
263
269
 
264
270
  // Create sources lazily here - this is where API token validation happens
265
- let chainId = chainConfig.id
266
- let sources = switch chainConfig.sourceConfig {
267
- | Config.EvmSourceConfig({hypersync, rpcs}) =>
268
- let evmRpcs: array<EvmChain.rpc> = rpcs->Array.map((rpc): EvmChain.rpc => {
269
- let syncConfig = rpc.syncConfig
270
- let ws = rpc.ws
271
- let headers = rpc.headers
272
- {
273
- url: rpc.url,
274
- sourceFor: rpc.sourceFor,
275
- ?syncConfig,
276
- ?ws,
277
- ?headers,
278
- }
279
- })
280
- EvmChain.makeSources(
281
- ~chainId,
282
- ~onEventRegistrations=onEventRegistrations->(
283
- Utils.magic: array<Internal.onEventRegistration> => array<Internal.evmOnEventRegistration>
284
- ),
285
- ~hyperSync=hypersync,
286
- ~rpcs=evmRpcs,
287
- ~lowercaseAddresses,
288
- ~addressStore,
289
- )
290
- | Config.FuelSourceConfig({hypersync}) => [
291
- FuelHyperSyncSource.make({
292
- chainId,
293
- endpointUrl: hypersync,
294
- apiToken: Env.envioApiToken,
295
- onEventRegistrations,
296
- addressStore,
297
- }),
298
- ]
299
- | Config.SvmSourceConfig({hypersync, rpc}) =>
300
- switch (hypersync, rpc) {
301
- | (None, None) =>
302
- JsError.throwWithMessage(`Chain ${chainId->ChainId.toString} has no SVM data source`)
303
- | (None, Some(rpc)) => [Svm.makeRPCSource(~chainId, ~rpc)]
304
- | (Some(hypersyncUrl), _) =>
305
- // HyperSync drives instruction sync. A configured RPC is ignored for now
306
- // (RPC fallback isn't wired up yet).
307
- let apiToken = Env.envioApiToken
308
- [
309
- SvmHyperSyncSource.make({
310
- chainId,
311
- endpointUrl: hypersyncUrl,
312
- apiToken,
313
- onEventRegistrations: onEventRegistrations->(
314
- Utils.magic: array<Internal.onEventRegistration> => array<
315
- Internal.svmOnEventRegistration,
316
- >
317
- ),
318
- clientTimeoutMillis: Env.hyperSyncClientTimeoutMillis,
319
- addressStore,
320
- }),
321
- ]
322
- }
323
- | Config.SimulateSourceConfig({items, endBlock, ?transactionStore, ?blockStore}) => [
324
- SimulateSource.make(
325
- ~items,
326
- ~endBlock,
327
- ~chainId,
328
- ~addressStore,
329
- ~ecosystem=config.ecosystem.name,
330
- ~transactionStore,
331
- ~blockStore,
332
- ),
333
- ]
334
- // For tests: use ready-to-use sources directly
335
- | Config.CustomSources(sources) => sources
336
- }
271
+ let sources = ChainSources.make(
272
+ ~chainConfig,
273
+ ~onEventRegistrations,
274
+ ~addressStore,
275
+ ~lowercaseAddresses,
276
+ )
337
277
 
338
278
  let blockStore = BlockStore.make(
339
279
  ~ecosystem=config.ecosystem.name,
@@ -372,6 +312,7 @@ let makeInternal = (
372
312
  ~sourceManager=SourceManager.make(~sources, ~isRealtime, ~reducedPollingInterval?),
373
313
  ~shouldRollbackOnReorg=config.shouldRollbackOnReorg,
374
314
  ~maxReorgDepth,
315
+ ~isInReorgThreshold,
375
316
  ~safeCheckpointTracking=SafeCheckpointTracking.make(
376
317
  ~maxReorgDepth,
377
318
  ~shouldRollbackOnReorg=config.shouldRollbackOnReorg,
@@ -704,15 +645,21 @@ let hasProcessedToEndblock = (cs: t) => {
704
645
  // head has run away from it while the indexer was down.
705
646
  let isDurablyCaughtUp = (cs: t) => {
706
647
  let {committedProgressBlockNumber, fetchState} = cs
707
- switch fetchState.endBlock {
708
- | Some(endBlock) => committedProgressBlockNumber >= endBlock
709
- | None =>
710
- // The configured lag, not the fetch state's: pre-threshold that one also
711
- // carries maxReorgDepth, which would read a chain a whole reorg depth behind
712
- // the head as caught up.
648
+ let atEndBlock =
649
+ fetchState.endBlock->Option.mapOr(false, endBlock => committedProgressBlockNumber >= endBlock)
650
+ // Either one, like `isFetchingAtHead`: an `end_block` above the head is never
651
+ // reached, and testing only for it would leave such a chain reading as behind
652
+ // however long it sits at the head.
653
+ //
654
+ // The configured lag, not the fetch state's: pre-threshold that one also
655
+ // carries maxReorgDepth, which would read a chain a whole reorg depth behind
656
+ // the head as caught up. Clamped at zero: the -1 a run that has processed
657
+ // nothing carries would otherwise clear a chain younger than its own lag.
658
+ let atHead =
713
659
  fetchState.knownHeight > 0 &&
714
- committedProgressBlockNumber >= fetchState.knownHeight - cs.chainConfig.blockLag
715
- }
660
+ committedProgressBlockNumber >=
661
+ Pervasives.max(0, fetchState.knownHeight - cs.chainConfig.blockLag)
662
+ atEndBlock || atHead
716
663
  }
717
664
 
718
665
  let getHighestBlockBelowThreshold = (cs: t): int => {
@@ -993,8 +940,18 @@ let setEndBlockToFirstEvent = (cs: t, ~blockNumber) =>
993
940
  }
994
941
 
995
942
  // Shrink the fetch buffer by the configured blockLag on entering the reorg threshold.
996
- let enterReorgThreshold = (cs: t) =>
943
+ let enterReorgThreshold = (cs: t) => {
944
+ cs.isInReorgThreshold = true
997
945
  cs.fetchState = cs.fetchState->FetchState.updateInternal(~blockLag=cs.chainConfig.blockLag)
946
+ }
947
+
948
+ let isInReorgThreshold = (cs: t) => cs.isInReorgThreshold
949
+
950
+ // Whether the chain's writes need history: only what a rollback could still
951
+ // reach. A chain with no reorg depth is never rolled back, however far its
952
+ // progress has run.
953
+ let shouldSaveHistory = (cs: t) =>
954
+ cs.shouldRollbackOnReorg && cs.maxReorgDepth > 0 && cs.isInReorgThreshold
998
955
 
999
956
  // Snapshot the chain's metadata fields for staging into the chains table.
1000
957
  let toChainMetadata = (cs: t): InternalTable.Chains.metaFields => {
@@ -1043,6 +1000,8 @@ let toMetrics = (cs: t): Metrics.chainMetrics => {
1043
1000
  reorgCount: cs.reorgCount,
1044
1001
  reorgDetectedBlock: cs.reorgDetectedBlock,
1045
1002
  rollbackTargetBlock: cs.rollbackTargetBlock,
1003
+ rateLimitTimeMs: cs.sourceManager->SourceManager.getRateLimitTimeMs,
1004
+ rateLimitResetInMs: cs.sourceManager->SourceManager.getRateLimitResetInMs,
1046
1005
  }
1047
1006
 
1048
1007
  // Snapshot the inputs a batch build needs from this chain, including an
@@ -1199,11 +1158,11 @@ let applyBatchProgress = (cs: t, ~batch: Batch.t, ~blockTimestampName: string) =
1199
1158
  }
1200
1159
 
1201
1160
  // Mark the chain caught up to head/endblock. Called by CrossChainState only once
1202
- // every chain in the indexer is caught up and the deferred schema indexes are
1203
- // committed, so no chain flips to ready while another is still backfilling or
1204
- // while an index the schema promises is still missing. `readyAt` is the
1205
- // timestamp already committed to `envio_chains.ready_at` in that same
1206
- // transaction. Sticky: a chain stays ready once set.
1161
+ // every chain this process drives is caught up and the indexes the schema
1162
+ // promises them are committed, so no chain flips to ready while an index it
1163
+ // promised is still missing. `readyAt` is the timestamp already committed to
1164
+ // `envio_chains.ready_at` in that same transaction. Sticky: a chain stays ready
1165
+ // once set.
1207
1166
  let markReady = (cs: t, ~readyAt) =>
1208
1167
  if !(cs->isReady) {
1209
1168
  cs.timestampCaughtUpToHeadOrEndblock = Some(readyAt)