envio 3.10.0 → 3.11.0-svm-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/evm.schema.json +21 -4
- package/fuel.schema.json +21 -4
- package/index.d.ts +10 -6
- package/package.json +6 -6
- package/src/Batch.res +100 -78
- package/src/Batch.res.mjs +52 -46
- package/src/BatchProcessing.res +1 -11
- package/src/BatchProcessing.res.mjs +1 -4
- package/src/ChainState.res +46 -87
- package/src/ChainState.res.mjs +32 -66
- package/src/ChainState.resi +3 -0
- package/src/Config.res +131 -55
- package/src/Config.res.mjs +132 -69
- package/src/CrossChainState.res +39 -49
- package/src/CrossChainState.res.mjs +24 -22
- package/src/CrossChainState.resi +6 -13
- package/src/EffectState.res +13 -4
- package/src/EffectState.resi +7 -1
- package/src/EventConfigBuilder.res +14 -10
- package/src/EventConfigBuilder.res.mjs +6 -4
- package/src/EventProcessing.res +6 -6
- package/src/EventProcessing.res.mjs +6 -6
- package/src/FetchState.res +14 -0
- package/src/FetchState.res.mjs +14 -0
- package/src/FinalizeBackfill.res +16 -12
- package/src/FinalizeBackfill.res.mjs +3 -3
- package/src/HandlerRegister.res +12 -11
- package/src/HandlerRegister.res.mjs +6 -5
- package/src/HistoryPolicy.res +46 -0
- package/src/HistoryPolicy.res.mjs +50 -0
- package/src/InMemoryStore.res +30 -32
- package/src/InMemoryStore.res.mjs +33 -28
- package/src/IndexerState.res +66 -45
- package/src/IndexerState.res.mjs +57 -29
- package/src/IndexerState.resi +8 -13
- package/src/Internal.res +24 -6
- package/src/Internal.res.mjs +9 -0
- package/src/LoadLayer.res +11 -4
- package/src/LoadLayer.res.mjs +8 -5
- package/src/Main.res +33 -25
- package/src/Main.res.mjs +25 -26
- package/src/Metrics.res +41 -1
- package/src/Metrics.res.mjs +25 -3
- package/src/Persistence.res +65 -17
- package/src/Persistence.res.mjs +23 -4
- package/src/PgStorage.res +192 -119
- package/src/PgStorage.res.mjs +130 -73
- package/src/PruneStaleHistory.res +9 -15
- package/src/PruneStaleHistory.res.mjs +11 -20
- package/src/Rollback.res +8 -14
- package/src/Rollback.res.mjs +4 -3
- package/src/SimulateItems.res +13 -6
- package/src/SimulateItems.res.mjs +7 -6
- package/src/Sink.res +7 -5
- package/src/Sink.res.mjs +4 -4
- package/src/TestIndexer.res +10 -10
- package/src/TestIndexer.res.mjs +11 -10
- package/src/UserContext.res +7 -4
- package/src/UserContext.res.mjs +5 -2
- package/src/Utils.res +9 -0
- package/src/Utils.res.mjs +9 -0
- package/src/Writing.res +34 -20
- package/src/Writing.res.mjs +15 -20
- package/src/bindings/ClickHouse.res +24 -8
- package/src/bindings/ClickHouse.res.mjs +20 -10
- package/src/bindings/ClickHouseSink.res +16 -5
- package/src/bindings/ClickHouseSink.res.mjs +3 -1
- package/src/db/CheckpointSequence.res +109 -0
- package/src/db/CheckpointSequence.res.mjs +99 -0
- package/src/db/EntityHistory.res +6 -5
- package/src/db/EntityHistory.res.mjs +5 -5
- package/src/db/Frontier.res +86 -0
- package/src/db/Frontier.res.mjs +126 -0
- package/src/db/InternalTable.res +112 -21
- package/src/db/InternalTable.res.mjs +95 -28
- package/src/db/RollbackFloors.res +34 -41
- package/src/db/RollbackFloors.res.mjs +27 -71
- package/src/sources/ChainSources.res +72 -0
- package/src/sources/ChainSources.res.mjs +55 -0
- package/src/sources/EvmHyperSyncSource.res +13 -23
- package/src/sources/EvmHyperSyncSource.res.mjs +11 -42
- package/src/sources/FuelHyperSyncSource.res +10 -18
- package/src/sources/FuelHyperSyncSource.res.mjs +8 -38
- package/src/sources/HyperSync.res +33 -0
- package/src/sources/HyperSync.res.mjs +32 -0
- package/src/sources/HyperSync.resi +17 -0
- package/src/sources/HyperSyncClient.res +3 -0
- package/src/sources/RequestStat.res +8 -1
- package/src/sources/Source.res +4 -3
- package/src/sources/SourceManager.res +30 -10
- package/src/sources/SourceManager.res.mjs +24 -7
- package/src/sources/SourceManager.resi +2 -0
- package/src/sources/StartBlockResolver.res +173 -0
- package/src/sources/StartBlockResolver.res.mjs +134 -0
- package/src/sources/Svm.res +0 -51
- package/src/sources/Svm.res.mjs +0 -46
- package/src/sources/SvmHyperSyncSource.res +20 -9
- package/src/sources/SvmHyperSyncSource.res.mjs +34 -12
- package/src/tui/Tui.res +56 -143
- package/src/tui/Tui.res.mjs +70 -166
- package/src/tui/components/TuiData.res +61 -16
- package/src/tui/components/TuiData.res.mjs +53 -15
- package/svm.schema.json +80 -73
- package/src/db/CheckpointBounds.res +0 -53
- package/src/db/CheckpointBounds.res.mjs +0 -44
package/src/Batch.res.mjs
CHANGED
|
@@ -1,34 +1,39 @@
|
|
|
1
1
|
// Generated by ReScript, PLEASE EDIT WITH CARE
|
|
2
2
|
|
|
3
3
|
import * as ChainId from "./ChainId.res.mjs";
|
|
4
|
+
import * as Frontier from "./db/Frontier.res.mjs";
|
|
4
5
|
import * as FetchState from "./FetchState.res.mjs";
|
|
6
|
+
import * as CheckpointSequence from "./db/CheckpointSequence.res.mjs";
|
|
5
7
|
|
|
6
|
-
function getChainAfterBatchIfProgressed(chainBeforeBatch, progressBlockNumberAfterBatch, fetchStateAfterBatch,
|
|
8
|
+
function getChainAfterBatchIfProgressed(chainBeforeBatch, progressBlockNumberAfterBatch, fetchStateAfterBatch, counts) {
|
|
7
9
|
if (chainBeforeBatch.progressBlockNumber < progressBlockNumberAfterBatch) {
|
|
8
10
|
return {
|
|
9
|
-
batchSize:
|
|
11
|
+
batchSize: counts.size,
|
|
10
12
|
progressBlockNumber: progressBlockNumberAfterBatch,
|
|
11
13
|
sourceBlockNumber: chainBeforeBatch.sourceBlockNumber,
|
|
12
|
-
totalEventsProcessed: chainBeforeBatch.totalEventsProcessed +
|
|
14
|
+
totalEventsProcessed: chainBeforeBatch.totalEventsProcessed + counts.eventsProcessed,
|
|
13
15
|
fetchState: fetchStateAfterBatch,
|
|
14
16
|
isProgressAtHeadWhenBatchCreated: progressBlockNumberAfterBatch >= chainBeforeBatch.sourceBlockNumber - chainBeforeBatch.chainConfig.blockLag
|
|
15
17
|
};
|
|
16
18
|
}
|
|
17
19
|
}
|
|
18
20
|
|
|
19
|
-
function getProgressedChainsById(chainsBeforeBatch,
|
|
21
|
+
function getProgressedChainsById(chainsBeforeBatch, countsPerChain, progressBlockNumberPerChain) {
|
|
20
22
|
let progressedChainsById = {};
|
|
21
23
|
Object.values(chainsBeforeBatch).forEach(chainBeforeBatch => {
|
|
22
24
|
let fetchState = chainBeforeBatch.fetchState;
|
|
23
25
|
let progressBlockNumber = progressBlockNumberPerChain[ChainId.toString(fetchState.chainId)];
|
|
24
26
|
let progressBlockNumberAfterBatch = progressBlockNumber !== undefined ? progressBlockNumber : chainBeforeBatch.progressBlockNumber;
|
|
25
|
-
let
|
|
27
|
+
let counts = countsPerChain[ChainId.toString(fetchState.chainId)];
|
|
26
28
|
let progressedChain;
|
|
27
|
-
if (
|
|
28
|
-
let leftItems = fetchState.buffer.slice(
|
|
29
|
-
progressedChain = getChainAfterBatchIfProgressed(chainBeforeBatch, progressBlockNumberAfterBatch, FetchState.updateInternal(fetchState, undefined, leftItems, undefined, undefined, undefined),
|
|
29
|
+
if (counts !== undefined) {
|
|
30
|
+
let leftItems = fetchState.buffer.slice(counts.size);
|
|
31
|
+
progressedChain = getChainAfterBatchIfProgressed(chainBeforeBatch, progressBlockNumberAfterBatch, FetchState.updateInternal(fetchState, undefined, leftItems, undefined, undefined, undefined), counts);
|
|
30
32
|
} else {
|
|
31
|
-
progressedChain = getChainAfterBatchIfProgressed(chainBeforeBatch, progressBlockNumberAfterBatch, chainBeforeBatch.fetchState,
|
|
33
|
+
progressedChain = getChainAfterBatchIfProgressed(chainBeforeBatch, progressBlockNumberAfterBatch, chainBeforeBatch.fetchState, {
|
|
34
|
+
size: 0,
|
|
35
|
+
eventsProcessed: 0
|
|
36
|
+
});
|
|
32
37
|
}
|
|
33
38
|
if (progressedChain !== undefined) {
|
|
34
39
|
progressedChainsById[chainBeforeBatch.fetchState.chainId] = progressedChain;
|
|
@@ -52,145 +57,146 @@ function seekFirstAbove(blockNumbers, blockNumber) {
|
|
|
52
57
|
return low;
|
|
53
58
|
}
|
|
54
59
|
|
|
55
|
-
function addReorgCheckpoints(
|
|
60
|
+
function addReorgCheckpoints(cursor, scannedHashes, shouldRollbackOnReorg, fromBlockExclusive, toBlockExclusive, chainId, mutCheckpointIds, mutCheckpointChainIds, mutCheckpointBlockNumbers, mutCheckpointBlockHashes, mutCheckpointItemsCount, mutCheckpointEventsProcessed) {
|
|
56
61
|
if (!shouldRollbackOnReorg) {
|
|
57
|
-
return
|
|
62
|
+
return;
|
|
58
63
|
}
|
|
59
|
-
let prevCheckpointId$1 = prevCheckpointId;
|
|
60
64
|
let blockNumbers = scannedHashes.blockNumbers;
|
|
61
65
|
let length = blockNumbers.length;
|
|
62
66
|
let idx = seekFirstAbove(blockNumbers, fromBlockExclusive);
|
|
63
67
|
while (idx < length && blockNumbers[idx] < toBlockExclusive) {
|
|
64
68
|
let blockNumber = blockNumbers[idx];
|
|
65
69
|
let hash = scannedHashes.hashByBlockNumber[blockNumber];
|
|
66
|
-
let checkpointId =
|
|
67
|
-
prevCheckpointId$1 = checkpointId;
|
|
70
|
+
let checkpointId = CheckpointSequence.next(cursor, chainId);
|
|
68
71
|
mutCheckpointIds.push(checkpointId);
|
|
69
72
|
mutCheckpointChainIds.push(chainId);
|
|
70
73
|
mutCheckpointBlockNumbers.push(blockNumber);
|
|
71
74
|
mutCheckpointBlockHashes.push(hash);
|
|
75
|
+
mutCheckpointItemsCount.push(0);
|
|
72
76
|
mutCheckpointEventsProcessed.push(0);
|
|
73
77
|
idx = idx + 1;
|
|
74
78
|
};
|
|
75
|
-
return prevCheckpointId$1;
|
|
76
79
|
}
|
|
77
80
|
|
|
78
|
-
function
|
|
81
|
+
function make(sequence, frontier, chainsBeforeBatch, batchSizeTarget, history) {
|
|
79
82
|
let preparedFetchStates = FetchState.sortForBatch(Object.values(chainsBeforeBatch).map(chainBeforeBatch => chainBeforeBatch.fetchState), batchSizeTarget);
|
|
80
83
|
let chainIdx = 0;
|
|
81
84
|
let preparedNumber = preparedFetchStates.length;
|
|
82
85
|
let totalBatchSize = 0;
|
|
83
|
-
let
|
|
84
|
-
let
|
|
86
|
+
let cursor = CheckpointSequence.cursor(sequence, frontier);
|
|
87
|
+
let mutCountsPerChain = {};
|
|
85
88
|
let mutProgressBlockNumberPerChain = {};
|
|
86
89
|
let items = [];
|
|
87
90
|
let checkpointIds = [];
|
|
88
91
|
let checkpointChainIds = [];
|
|
89
92
|
let checkpointBlockNumbers = [];
|
|
90
93
|
let checkpointBlockHashes = [];
|
|
94
|
+
let checkpointItemsCount = [];
|
|
91
95
|
let checkpointEventsProcessed = [];
|
|
92
96
|
while (totalBatchSize < batchSizeTarget && chainIdx < preparedNumber) {
|
|
93
97
|
let fetchState = preparedFetchStates[chainIdx];
|
|
94
98
|
let chainBatchSize = FetchState.getReadyItemsCount(fetchState, batchSizeTarget - totalBatchSize, 0);
|
|
95
99
|
let chainBeforeBatch = chainsBeforeBatch[fetchState.chainId];
|
|
96
100
|
let prevBlockNumber = chainBeforeBatch.progressBlockNumber;
|
|
101
|
+
let chainEventsProcessed = 0;
|
|
97
102
|
if (chainBatchSize > 0) {
|
|
98
103
|
for (let idx = 0, idx_finish = chainBatchSize - 1; idx <= idx_finish; ++idx) {
|
|
99
104
|
let item = fetchState.buffer[idx];
|
|
100
105
|
let blockNumber = item.blockNumber;
|
|
106
|
+
let isNewEvent = idx === 0 || !FetchState.isSameLog(fetchState.buffer[idx - 1], item);
|
|
107
|
+
if (isNewEvent) {
|
|
108
|
+
chainEventsProcessed = chainEventsProcessed + 1;
|
|
109
|
+
}
|
|
101
110
|
if (blockNumber !== prevBlockNumber) {
|
|
102
111
|
let chainId = fetchState.chainId;
|
|
103
112
|
let fromBlockExclusive = prevBlockNumber;
|
|
104
113
|
let scannedHashes = chainBeforeBatch.scannedHashes;
|
|
105
|
-
let prevCheckpointId$1 = prevCheckpointId;
|
|
106
|
-
let tmp;
|
|
107
114
|
if (chainBeforeBatch.shouldRollbackOnReorg) {
|
|
108
|
-
let prevCheckpointId$2 = prevCheckpointId$1;
|
|
109
115
|
let blockNumbers = scannedHashes.blockNumbers;
|
|
110
116
|
let length = blockNumbers.length;
|
|
111
117
|
let idx$1 = seekFirstAbove(blockNumbers, fromBlockExclusive);
|
|
112
118
|
while (idx$1 < length && blockNumbers[idx$1] < blockNumber) {
|
|
113
119
|
let blockNumber$1 = blockNumbers[idx$1];
|
|
114
120
|
let hash = scannedHashes.hashByBlockNumber[blockNumber$1];
|
|
115
|
-
let checkpointId =
|
|
116
|
-
prevCheckpointId$2 = checkpointId;
|
|
121
|
+
let checkpointId = CheckpointSequence.next(cursor, chainId);
|
|
117
122
|
checkpointIds.push(checkpointId);
|
|
118
123
|
checkpointChainIds.push(chainId);
|
|
119
124
|
checkpointBlockNumbers.push(blockNumber$1);
|
|
120
125
|
checkpointBlockHashes.push(hash);
|
|
126
|
+
checkpointItemsCount.push(0);
|
|
121
127
|
checkpointEventsProcessed.push(0);
|
|
122
128
|
idx$1 = idx$1 + 1;
|
|
123
129
|
};
|
|
124
|
-
tmp = prevCheckpointId$2;
|
|
125
|
-
} else {
|
|
126
|
-
tmp = prevCheckpointId$1;
|
|
127
130
|
}
|
|
128
|
-
|
|
129
|
-
let checkpointId$1 = prevCheckpointId + 1n;
|
|
131
|
+
let checkpointId$1 = CheckpointSequence.next(cursor, fetchState.chainId);
|
|
130
132
|
checkpointIds.push(checkpointId$1);
|
|
131
133
|
checkpointChainIds.push(fetchState.chainId);
|
|
132
134
|
checkpointBlockNumbers.push(blockNumber);
|
|
133
135
|
let hash$1 = chainBeforeBatch.scannedHashes.hashByBlockNumber[blockNumber];
|
|
134
136
|
checkpointBlockHashes.push(hash$1 !== undefined ? hash$1 : null);
|
|
137
|
+
checkpointItemsCount.push(1);
|
|
135
138
|
checkpointEventsProcessed.push(1);
|
|
136
139
|
prevBlockNumber = blockNumber;
|
|
137
|
-
prevCheckpointId = checkpointId$1;
|
|
138
140
|
} else {
|
|
139
|
-
let lastIndex =
|
|
140
|
-
|
|
141
|
+
let lastIndex = checkpointItemsCount.length - 1;
|
|
142
|
+
checkpointItemsCount[lastIndex] = checkpointItemsCount[lastIndex] + 1;
|
|
143
|
+
if (isNewEvent) {
|
|
144
|
+
checkpointEventsProcessed[lastIndex] = checkpointEventsProcessed[lastIndex] + 1;
|
|
145
|
+
}
|
|
141
146
|
}
|
|
142
147
|
items.push(item);
|
|
143
148
|
}
|
|
144
149
|
totalBatchSize = totalBatchSize + chainBatchSize;
|
|
145
|
-
|
|
150
|
+
mutCountsPerChain[fetchState.chainId] = {
|
|
151
|
+
size: chainBatchSize,
|
|
152
|
+
eventsProcessed: chainEventsProcessed
|
|
153
|
+
};
|
|
146
154
|
}
|
|
147
155
|
let progressBlockNumberAfterBatch = FetchState.getProgressBlockNumberAt(fetchState, chainBatchSize);
|
|
148
156
|
let chainId$1 = fetchState.chainId;
|
|
149
157
|
let toBlockExclusive = progressBlockNumberAfterBatch + 1;
|
|
150
158
|
let fromBlockExclusive$1 = prevBlockNumber;
|
|
151
159
|
let scannedHashes$1 = chainBeforeBatch.scannedHashes;
|
|
152
|
-
let prevCheckpointId$3 = prevCheckpointId;
|
|
153
|
-
let tmp$1;
|
|
154
160
|
if (chainBeforeBatch.shouldRollbackOnReorg) {
|
|
155
|
-
let prevCheckpointId$4 = prevCheckpointId$3;
|
|
156
161
|
let blockNumbers$1 = scannedHashes$1.blockNumbers;
|
|
157
162
|
let length$1 = blockNumbers$1.length;
|
|
158
163
|
let idx$2 = seekFirstAbove(blockNumbers$1, fromBlockExclusive$1);
|
|
159
164
|
while (idx$2 < length$1 && blockNumbers$1[idx$2] < toBlockExclusive) {
|
|
160
165
|
let blockNumber$2 = blockNumbers$1[idx$2];
|
|
161
166
|
let hash$2 = scannedHashes$1.hashByBlockNumber[blockNumber$2];
|
|
162
|
-
let checkpointId$2 =
|
|
163
|
-
prevCheckpointId$4 = checkpointId$2;
|
|
167
|
+
let checkpointId$2 = CheckpointSequence.next(cursor, chainId$1);
|
|
164
168
|
checkpointIds.push(checkpointId$2);
|
|
165
169
|
checkpointChainIds.push(chainId$1);
|
|
166
170
|
checkpointBlockNumbers.push(blockNumber$2);
|
|
167
171
|
checkpointBlockHashes.push(hash$2);
|
|
172
|
+
checkpointItemsCount.push(0);
|
|
168
173
|
checkpointEventsProcessed.push(0);
|
|
169
174
|
idx$2 = idx$2 + 1;
|
|
170
175
|
};
|
|
171
|
-
tmp$1 = prevCheckpointId$4;
|
|
172
|
-
} else {
|
|
173
|
-
tmp$1 = prevCheckpointId$3;
|
|
174
176
|
}
|
|
175
|
-
prevCheckpointId = tmp$1;
|
|
176
177
|
mutProgressBlockNumberPerChain[fetchState.chainId] = progressBlockNumberAfterBatch;
|
|
177
178
|
chainIdx = chainIdx + 1;
|
|
178
179
|
};
|
|
179
180
|
return {
|
|
180
181
|
totalBatchSize: totalBatchSize,
|
|
181
182
|
items: items,
|
|
182
|
-
progressedChainsById: getProgressedChainsById(chainsBeforeBatch,
|
|
183
|
-
|
|
183
|
+
progressedChainsById: getProgressedChainsById(chainsBeforeBatch, mutCountsPerChain, mutProgressBlockNumberPerChain),
|
|
184
|
+
history: history,
|
|
184
185
|
checkpointIds: checkpointIds,
|
|
185
186
|
checkpointChainIds: checkpointChainIds,
|
|
186
187
|
checkpointBlockNumbers: checkpointBlockNumbers,
|
|
187
188
|
checkpointBlockHashes: checkpointBlockHashes,
|
|
189
|
+
checkpointItemsCount: checkpointItemsCount,
|
|
188
190
|
checkpointEventsProcessed: checkpointEventsProcessed,
|
|
189
191
|
registeredAddresses: []
|
|
190
192
|
};
|
|
191
193
|
}
|
|
192
194
|
|
|
193
|
-
|
|
195
|
+
function checkpointFrontier(batch) {
|
|
196
|
+
let frontier = Frontier.empty();
|
|
197
|
+
batch.checkpointChainIds.forEach((chainId, index) => Frontier.set(frontier, chainId, batch.checkpointIds[index]));
|
|
198
|
+
return frontier;
|
|
199
|
+
}
|
|
194
200
|
|
|
195
201
|
function findFirstEventBlockNumber(batch, chainId) {
|
|
196
202
|
let idx = 0;
|
|
@@ -198,7 +204,7 @@ function findFirstEventBlockNumber(batch, chainId) {
|
|
|
198
204
|
let checkpointsLength = batch.checkpointIds.length;
|
|
199
205
|
while (idx < checkpointsLength && result === undefined) {
|
|
200
206
|
let checkpointChainId = batch.checkpointChainIds[idx];
|
|
201
|
-
if (checkpointChainId === chainId && batch.
|
|
207
|
+
if (checkpointChainId === chainId && batch.checkpointItemsCount[idx] > 0) {
|
|
202
208
|
result = batch.checkpointBlockNumbers[idx];
|
|
203
209
|
} else {
|
|
204
210
|
idx = idx + 1;
|
|
@@ -225,8 +231,8 @@ export {
|
|
|
225
231
|
getProgressedChainsById,
|
|
226
232
|
seekFirstAbove,
|
|
227
233
|
addReorgCheckpoints,
|
|
228
|
-
prepareBatch,
|
|
229
234
|
make,
|
|
235
|
+
checkpointFrontier,
|
|
230
236
|
findFirstEventBlockNumber,
|
|
231
237
|
findLastEventItem,
|
|
232
238
|
}
|
package/src/BatchProcessing.res
CHANGED
|
@@ -69,17 +69,7 @@ and processNextBatch = async (state: IndexerState.t, ~scheduleFetch): unit => {
|
|
|
69
69
|
let isInReorgThresholdBeforeUpdate = state->IndexerState.isInReorgThreshold
|
|
70
70
|
let isRealtimeBeforeUpdate = state->IndexerState.isRealtime
|
|
71
71
|
|
|
72
|
-
let
|
|
73
|
-
| RollbackReady(_) => true
|
|
74
|
-
| _ => false
|
|
75
|
-
}
|
|
76
|
-
|
|
77
|
-
let batch =
|
|
78
|
-
state->IndexerState.createBatch(
|
|
79
|
-
~processedCheckpointId=state->IndexerState.processedCheckpointId,
|
|
80
|
-
~batchSizeTarget=(state->IndexerState.config).batchSize,
|
|
81
|
-
~isRollback=isRollbackBatch,
|
|
82
|
-
)
|
|
72
|
+
let batch = state->IndexerState.createBatch(~batchSizeTarget=(state->IndexerState.config).batchSize)
|
|
83
73
|
|
|
84
74
|
let progressedChainsById = batch.progressedChainsById
|
|
85
75
|
|
|
@@ -23,10 +23,7 @@ function yieldTick() {
|
|
|
23
23
|
async function processNextBatch(state, scheduleFetch) {
|
|
24
24
|
let isInReorgThresholdBeforeUpdate = IndexerState.isInReorgThreshold(state);
|
|
25
25
|
let isRealtimeBeforeUpdate = IndexerState.isRealtime(state);
|
|
26
|
-
let
|
|
27
|
-
let isRollbackBatch;
|
|
28
|
-
isRollbackBatch = typeof match !== "object" ? false : match.TAG === "RollbackReady";
|
|
29
|
-
let batch = IndexerState.createBatch(state, IndexerState.processedCheckpointId(state), IndexerState.config(state).batchSize, isRollbackBatch);
|
|
26
|
+
let batch = IndexerState.createBatch(state, IndexerState.config(state).batchSize);
|
|
30
27
|
let progressedChainsById = batch.progressedChainsById;
|
|
31
28
|
let isBelowReorgThreshold = !isInReorgThresholdBeforeUpdate && IndexerState.config(state).shouldRollbackOnReorg;
|
|
32
29
|
let shouldEnterReorgThreshold = isBelowReorgThreshold && Object.values(IndexerState.chainStates(state)).every(cs => ChainState.isReadyToEnterReorgThresholdAfterBatch(cs, batch));
|
package/src/ChainState.res
CHANGED
|
@@ -41,6 +41,8 @@ type t = {
|
|
|
41
41
|
// detected reorg either rolls back or is only logged.
|
|
42
42
|
shouldRollbackOnReorg: bool,
|
|
43
43
|
maxReorgDepth: int,
|
|
44
|
+
// One-way: past it, everything the chain writes can still be rolled back.
|
|
45
|
+
mutable isInReorgThreshold: bool,
|
|
44
46
|
// Holds this chain's transactions (kept in Rust) keyed by (blockNumber,
|
|
45
47
|
// transactionIndex). Fetch responses merge their page in; entries are pruned
|
|
46
48
|
// as the chain progresses and dropped above the target on rollback.
|
|
@@ -126,6 +128,7 @@ let make = (
|
|
|
126
128
|
~safeCheckpointTracking=None,
|
|
127
129
|
~shouldRollbackOnReorg,
|
|
128
130
|
~maxReorgDepth,
|
|
131
|
+
~isInReorgThreshold=false,
|
|
129
132
|
~numEventsProcessed=0.,
|
|
130
133
|
~timestampCaughtUpToHeadOrEndblock=None,
|
|
131
134
|
~isProgressAtHead=false,
|
|
@@ -155,6 +158,7 @@ let make = (
|
|
|
155
158
|
safeCheckpointTracking,
|
|
156
159
|
shouldRollbackOnReorg,
|
|
157
160
|
maxReorgDepth,
|
|
161
|
+
isInReorgThreshold,
|
|
158
162
|
transactionStore,
|
|
159
163
|
blockStore,
|
|
160
164
|
reorgThresholdReadyTolerance,
|
|
@@ -203,7 +207,9 @@ let makeInternal = (
|
|
|
203
207
|
|
|
204
208
|
chainConfig.contracts->Array.forEach(contract => {
|
|
205
209
|
switch contract.startBlock {
|
|
206
|
-
|
|
210
|
+
// Against the resolved `~startBlock`, which came from storage:
|
|
211
|
+
// `chainConfig.startBlock` still says whatever config.yaml said.
|
|
212
|
+
| Some(contractStartBlock) if contractStartBlock < startBlock =>
|
|
207
213
|
JsError.throwWithMessage(
|
|
208
214
|
`The start block for contract "${contract.name}" is less than the chain start block. This is not supported yet.`,
|
|
209
215
|
)
|
|
@@ -262,78 +268,12 @@ let makeInternal = (
|
|
|
262
268
|
})
|
|
263
269
|
|
|
264
270
|
// Create sources lazily here - this is where API token validation happens
|
|
265
|
-
let
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
let headers = rpc.headers
|
|
272
|
-
{
|
|
273
|
-
url: rpc.url,
|
|
274
|
-
sourceFor: rpc.sourceFor,
|
|
275
|
-
?syncConfig,
|
|
276
|
-
?ws,
|
|
277
|
-
?headers,
|
|
278
|
-
}
|
|
279
|
-
})
|
|
280
|
-
EvmChain.makeSources(
|
|
281
|
-
~chainId,
|
|
282
|
-
~onEventRegistrations=onEventRegistrations->(
|
|
283
|
-
Utils.magic: array<Internal.onEventRegistration> => array<Internal.evmOnEventRegistration>
|
|
284
|
-
),
|
|
285
|
-
~hyperSync=hypersync,
|
|
286
|
-
~rpcs=evmRpcs,
|
|
287
|
-
~lowercaseAddresses,
|
|
288
|
-
~addressStore,
|
|
289
|
-
)
|
|
290
|
-
| Config.FuelSourceConfig({hypersync}) => [
|
|
291
|
-
FuelHyperSyncSource.make({
|
|
292
|
-
chainId,
|
|
293
|
-
endpointUrl: hypersync,
|
|
294
|
-
apiToken: Env.envioApiToken,
|
|
295
|
-
onEventRegistrations,
|
|
296
|
-
addressStore,
|
|
297
|
-
}),
|
|
298
|
-
]
|
|
299
|
-
| Config.SvmSourceConfig({hypersync, rpc}) =>
|
|
300
|
-
switch (hypersync, rpc) {
|
|
301
|
-
| (None, None) =>
|
|
302
|
-
JsError.throwWithMessage(`Chain ${chainId->ChainId.toString} has no SVM data source`)
|
|
303
|
-
| (None, Some(rpc)) => [Svm.makeRPCSource(~chainId, ~rpc)]
|
|
304
|
-
| (Some(hypersyncUrl), _) =>
|
|
305
|
-
// HyperSync drives instruction sync. A configured RPC is ignored for now
|
|
306
|
-
// (RPC fallback isn't wired up yet).
|
|
307
|
-
let apiToken = Env.envioApiToken
|
|
308
|
-
[
|
|
309
|
-
SvmHyperSyncSource.make({
|
|
310
|
-
chainId,
|
|
311
|
-
endpointUrl: hypersyncUrl,
|
|
312
|
-
apiToken,
|
|
313
|
-
onEventRegistrations: onEventRegistrations->(
|
|
314
|
-
Utils.magic: array<Internal.onEventRegistration> => array<
|
|
315
|
-
Internal.svmOnEventRegistration,
|
|
316
|
-
>
|
|
317
|
-
),
|
|
318
|
-
clientTimeoutMillis: Env.hyperSyncClientTimeoutMillis,
|
|
319
|
-
addressStore,
|
|
320
|
-
}),
|
|
321
|
-
]
|
|
322
|
-
}
|
|
323
|
-
| Config.SimulateSourceConfig({items, endBlock, ?transactionStore, ?blockStore}) => [
|
|
324
|
-
SimulateSource.make(
|
|
325
|
-
~items,
|
|
326
|
-
~endBlock,
|
|
327
|
-
~chainId,
|
|
328
|
-
~addressStore,
|
|
329
|
-
~ecosystem=config.ecosystem.name,
|
|
330
|
-
~transactionStore,
|
|
331
|
-
~blockStore,
|
|
332
|
-
),
|
|
333
|
-
]
|
|
334
|
-
// For tests: use ready-to-use sources directly
|
|
335
|
-
| Config.CustomSources(sources) => sources
|
|
336
|
-
}
|
|
271
|
+
let sources = ChainSources.make(
|
|
272
|
+
~chainConfig,
|
|
273
|
+
~onEventRegistrations,
|
|
274
|
+
~addressStore,
|
|
275
|
+
~lowercaseAddresses,
|
|
276
|
+
)
|
|
337
277
|
|
|
338
278
|
let blockStore = BlockStore.make(
|
|
339
279
|
~ecosystem=config.ecosystem.name,
|
|
@@ -372,6 +312,7 @@ let makeInternal = (
|
|
|
372
312
|
~sourceManager=SourceManager.make(~sources, ~isRealtime, ~reducedPollingInterval?),
|
|
373
313
|
~shouldRollbackOnReorg=config.shouldRollbackOnReorg,
|
|
374
314
|
~maxReorgDepth,
|
|
315
|
+
~isInReorgThreshold,
|
|
375
316
|
~safeCheckpointTracking=SafeCheckpointTracking.make(
|
|
376
317
|
~maxReorgDepth,
|
|
377
318
|
~shouldRollbackOnReorg=config.shouldRollbackOnReorg,
|
|
@@ -704,15 +645,21 @@ let hasProcessedToEndblock = (cs: t) => {
|
|
|
704
645
|
// head has run away from it while the indexer was down.
|
|
705
646
|
let isDurablyCaughtUp = (cs: t) => {
|
|
706
647
|
let {committedProgressBlockNumber, fetchState} = cs
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
648
|
+
let atEndBlock =
|
|
649
|
+
fetchState.endBlock->Option.mapOr(false, endBlock => committedProgressBlockNumber >= endBlock)
|
|
650
|
+
// Either one, like `isFetchingAtHead`: an `end_block` above the head is never
|
|
651
|
+
// reached, and testing only for it would leave such a chain reading as behind
|
|
652
|
+
// however long it sits at the head.
|
|
653
|
+
//
|
|
654
|
+
// The configured lag, not the fetch state's: pre-threshold that one also
|
|
655
|
+
// carries maxReorgDepth, which would read a chain a whole reorg depth behind
|
|
656
|
+
// the head as caught up. Clamped at zero: the -1 a run that has processed
|
|
657
|
+
// nothing carries would otherwise clear a chain younger than its own lag.
|
|
658
|
+
let atHead =
|
|
713
659
|
fetchState.knownHeight > 0 &&
|
|
714
|
-
committedProgressBlockNumber >=
|
|
715
|
-
|
|
660
|
+
committedProgressBlockNumber >=
|
|
661
|
+
Pervasives.max(0, fetchState.knownHeight - cs.chainConfig.blockLag)
|
|
662
|
+
atEndBlock || atHead
|
|
716
663
|
}
|
|
717
664
|
|
|
718
665
|
let getHighestBlockBelowThreshold = (cs: t): int => {
|
|
@@ -993,8 +940,18 @@ let setEndBlockToFirstEvent = (cs: t, ~blockNumber) =>
|
|
|
993
940
|
}
|
|
994
941
|
|
|
995
942
|
// Shrink the fetch buffer by the configured blockLag on entering the reorg threshold.
|
|
996
|
-
let enterReorgThreshold = (cs: t) =>
|
|
943
|
+
let enterReorgThreshold = (cs: t) => {
|
|
944
|
+
cs.isInReorgThreshold = true
|
|
997
945
|
cs.fetchState = cs.fetchState->FetchState.updateInternal(~blockLag=cs.chainConfig.blockLag)
|
|
946
|
+
}
|
|
947
|
+
|
|
948
|
+
let isInReorgThreshold = (cs: t) => cs.isInReorgThreshold
|
|
949
|
+
|
|
950
|
+
// Whether the chain's writes need history: only what a rollback could still
|
|
951
|
+
// reach. A chain with no reorg depth is never rolled back, however far its
|
|
952
|
+
// progress has run.
|
|
953
|
+
let shouldSaveHistory = (cs: t) =>
|
|
954
|
+
cs.shouldRollbackOnReorg && cs.maxReorgDepth > 0 && cs.isInReorgThreshold
|
|
998
955
|
|
|
999
956
|
// Snapshot the chain's metadata fields for staging into the chains table.
|
|
1000
957
|
let toChainMetadata = (cs: t): InternalTable.Chains.metaFields => {
|
|
@@ -1043,6 +1000,8 @@ let toMetrics = (cs: t): Metrics.chainMetrics => {
|
|
|
1043
1000
|
reorgCount: cs.reorgCount,
|
|
1044
1001
|
reorgDetectedBlock: cs.reorgDetectedBlock,
|
|
1045
1002
|
rollbackTargetBlock: cs.rollbackTargetBlock,
|
|
1003
|
+
rateLimitTimeMs: cs.sourceManager->SourceManager.getRateLimitTimeMs,
|
|
1004
|
+
rateLimitResetInMs: cs.sourceManager->SourceManager.getRateLimitResetInMs,
|
|
1046
1005
|
}
|
|
1047
1006
|
|
|
1048
1007
|
// Snapshot the inputs a batch build needs from this chain, including an
|
|
@@ -1199,11 +1158,11 @@ let applyBatchProgress = (cs: t, ~batch: Batch.t, ~blockTimestampName: string) =
|
|
|
1199
1158
|
}
|
|
1200
1159
|
|
|
1201
1160
|
// Mark the chain caught up to head/endblock. Called by CrossChainState only once
|
|
1202
|
-
// every chain
|
|
1203
|
-
// committed, so no chain flips to ready while
|
|
1204
|
-
//
|
|
1205
|
-
//
|
|
1206
|
-
//
|
|
1161
|
+
// every chain this process drives is caught up and the indexes the schema
|
|
1162
|
+
// promises them are committed, so no chain flips to ready while an index it
|
|
1163
|
+
// promised is still missing. `readyAt` is the timestamp already committed to
|
|
1164
|
+
// `envio_chains.ready_at` in that same transaction. Sticky: a chain stays ready
|
|
1165
|
+
// once set.
|
|
1207
1166
|
let markReady = (cs: t, ~readyAt) =>
|
|
1208
1167
|
if !(cs->isReady) {
|
|
1209
1168
|
cs.timestampCaughtUpToHeadOrEndblock = Some(readyAt)
|