envio 3.12.0 → 3.13.0-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +6 -6
- package/src/BatchProcessing.res +9 -9
- package/src/BatchProcessing.res.mjs +5 -5
- package/src/Bin.res +24 -17
- package/src/Bin.res.mjs +5 -0
- package/src/ChainFetching.res +31 -13
- package/src/ChainFetching.res.mjs +9 -4
- package/src/ChainState.res +55 -3
- package/src/ChainState.res.mjs +46 -3
- package/src/ChainState.resi +3 -1
- package/src/Config.res +35 -0
- package/src/Config.res.mjs +39 -0
- package/src/Core.res +4 -0
- package/src/Core.res.mjs +4 -0
- package/src/CrossChainState.res +61 -4
- package/src/CrossChainState.res.mjs +39 -5
- package/src/CrossChainState.resi +10 -1
- package/src/Env.res +4 -0
- package/src/FetchState.res +51 -46
- package/src/FetchState.res.mjs +54 -36
- package/src/InMemoryTable.res +155 -67
- package/src/InMemoryTable.res.mjs +151 -60
- package/src/IndexerLoop.res +2 -0
- package/src/IndexerLoop.res.mjs +1 -0
- package/src/IndexerState.res +41 -1
- package/src/IndexerState.res.mjs +43 -4
- package/src/IndexerState.resi +10 -0
- package/src/LoadLayer.res +57 -34
- package/src/LoadLayer.res.mjs +45 -62
- package/src/LoadLayer.resi +1 -1
- package/src/Logging.res +38 -6
- package/src/Logging.res.mjs +32 -5
- package/src/Main.res +131 -268
- package/src/Main.res.mjs +28 -152
- package/src/Metrics.res +263 -102
- package/src/Metrics.res.mjs +227 -48
- package/src/Persistence.res +27 -2
- package/src/Persistence.res.mjs +9 -2
- package/src/PgStorage.res +109 -71
- package/src/PgStorage.res.mjs +88 -58
- package/src/Server.res +181 -0
- package/src/Server.res.mjs +143 -0
- package/src/Supervisor.res +415 -0
- package/src/Supervisor.res.mjs +325 -0
- package/src/TestIndexer.res +9 -25
- package/src/TestIndexer.res.mjs +5 -4
- package/src/UserContext.res +13 -32
- package/src/UserContext.res.mjs +1 -7
- package/src/Utils.res +1 -4
- package/src/Utils.res.mjs +7 -16
- package/src/Worker.res +95 -0
- package/src/Worker.res.mjs +80 -0
- package/src/bindings/NodeJs.res +41 -0
- package/src/db/EntityFilter.res +487 -275
- package/src/db/EntityFilter.res.mjs +557 -309
- package/src/db/InternalTable.res +8 -1
- package/src/db/InternalTable.res.mjs +5 -1
- package/src/db/Table.res +21 -6
- package/src/db/Table.res.mjs +13 -4
- package/src/sources/BlockStore.res +7 -2
- package/src/sources/EvmHyperSyncSource.res +2 -0
- package/src/sources/EvmHyperSyncSource.res.mjs +2 -2
- package/src/sources/FuelHyperSyncSource.res +1 -0
- package/src/sources/FuelHyperSyncSource.res.mjs +1 -1
- package/src/sources/HyperSync.res +4 -0
- package/src/sources/HyperSync.res.mjs +4 -2
- package/src/sources/HyperSync.resi +1 -0
- package/src/sources/HyperSyncClient.res +3 -0
- package/src/sources/HyperSyncSSE.res +1 -1
- package/src/sources/HyperSyncSSE.res.mjs +4 -10
- package/src/sources/RpcSource.res +1 -0
- package/src/sources/RpcSource.res.mjs +1 -1
- package/src/sources/SimulateSource.res +1 -0
- package/src/sources/SimulateSource.res.mjs +1 -1
- package/src/sources/Source.res +7 -0
- package/src/sources/SourceManager.res +4 -3
- package/src/sources/SourceManager.res.mjs +2 -2
- package/src/sources/SvmHyperSyncClient.res +5 -0
- package/src/sources/SvmHyperSyncSource.res +3 -0
- package/src/sources/SvmHyperSyncSource.res.mjs +4 -2
- package/src/tui/Tui.res +24 -0
- package/src/tui/Tui.res.mjs +18 -0
- package/src/tui/components/SyncETA.res +12 -6
- package/src/tui/components/SyncETA.res.mjs +12 -8
package/src/Core.res.mjs
CHANGED
|
@@ -88,6 +88,10 @@ let loadDevAddon = (function(req, envioDir) {
|
|
|
88
88
|
fs.copyFileSync(srcPath, nodePath);
|
|
89
89
|
}
|
|
90
90
|
|
|
91
|
+
// Forked workers inherit this, so only the first process in a run pays for
|
|
92
|
+
// the cargo build (and they don't contend over the cargo lock).
|
|
93
|
+
process.env.ENVIO_DEV_ADDON = nodePath;
|
|
94
|
+
|
|
91
95
|
return req(nodePath);
|
|
92
96
|
});
|
|
93
97
|
|
package/src/CrossChainState.res
CHANGED
|
@@ -17,6 +17,11 @@ type t = {
|
|
|
17
17
|
mutable isCaughtUp: bool,
|
|
18
18
|
// Indexer-wide fetch buffer pool (item count), shared across all chains.
|
|
19
19
|
targetBufferSize: int,
|
|
20
|
+
// Set on a process driving part of a split run: the chains it drives may be
|
|
21
|
+
// at the head while chains in another process are still backfilling, and an
|
|
22
|
+
// indexer switches to realtime as a whole or not at all. Cleared by the
|
|
23
|
+
// supervisor once every chain in the run has arrived.
|
|
24
|
+
mutable holdRealtime: bool,
|
|
20
25
|
}
|
|
21
26
|
|
|
22
27
|
// The whole-indexer fetch buffer pool, independent of chain count.
|
|
@@ -26,16 +31,50 @@ let calculateTargetBufferSize = () =>
|
|
|
26
31
|
| None => 100_000
|
|
27
32
|
}
|
|
28
33
|
|
|
29
|
-
let make = (
|
|
34
|
+
let make = (
|
|
35
|
+
~chainStates,
|
|
36
|
+
~isRealtime,
|
|
37
|
+
~targetBufferSize=calculateTargetBufferSize(),
|
|
38
|
+
~holdRealtime=false,
|
|
39
|
+
): t => {
|
|
30
40
|
{
|
|
31
41
|
chainStates,
|
|
32
42
|
chainIds: chainStates->Dict.valuesToArray->Array.map(cs => (cs->ChainState.chainConfig).id),
|
|
33
43
|
isRealtime,
|
|
34
44
|
isCaughtUp: isRealtime,
|
|
35
45
|
targetBufferSize,
|
|
46
|
+
holdRealtime,
|
|
36
47
|
}
|
|
37
48
|
}
|
|
38
49
|
|
|
50
|
+
// The supervisor's go-ahead: every chain in the run has reached the head, so
|
|
51
|
+
// this process may make the transitions it has been holding back.
|
|
52
|
+
let releaseRealtime = (crossChainState: t) => crossChainState.holdRealtime = false
|
|
53
|
+
|
|
54
|
+
let isHoldingRealtime = (crossChainState: t) => crossChainState.holdRealtime
|
|
55
|
+
|
|
56
|
+
// Whether this process has got as far as it can without the run's leave. What a
|
|
57
|
+
// supervisor reads to decide that a split run may go realtime as one.
|
|
58
|
+
//
|
|
59
|
+
// Three ways to have arrived, because a chain can be as far along as it can get
|
|
60
|
+
// in three different states. Its chains have caught up; or it resumed already
|
|
61
|
+
// realtime; or every chain is waiting to enter the reorg threshold, which is as
|
|
62
|
+
// far as one can fetch while the pre-threshold lag holds it at the safe block —
|
|
63
|
+
// entering the threshold is what lifts that lag, so a run held until its chains
|
|
64
|
+
// reached the head would be holding back the transition that gets them there.
|
|
65
|
+
//
|
|
66
|
+
// The process's own conclusion rather than a reading a supervisor reassembles:
|
|
67
|
+
// a chain committed at what was the head and resumed once the head had moved on
|
|
68
|
+
// has arrived, and no live reading of it can say so — which is the same reason
|
|
69
|
+
// `markCaughtUpOnResume` decides before any source request.
|
|
70
|
+
let hasArrivedAtHead = (crossChainState: t) =>
|
|
71
|
+
crossChainState.isCaughtUp ||
|
|
72
|
+
crossChainState.isRealtime || {
|
|
73
|
+
let chainStates = crossChainState.chainStates->Dict.valuesToArray
|
|
74
|
+
chainStates->Utils.Array.notEmpty &&
|
|
75
|
+
chainStates->Array.every(ChainState.isReadyToEnterReorgThreshold)
|
|
76
|
+
}
|
|
77
|
+
|
|
39
78
|
// Resolve a chain's state by id. The id always comes from `chainIds`, which is
|
|
40
79
|
// derived from `chainStates`, so the entry is guaranteed present.
|
|
41
80
|
let getChainState = (crossChainState: t, chainId) =>
|
|
@@ -114,7 +153,7 @@ let createBatch = (
|
|
|
114
153
|
~history=config->HistoryPolicy.decide(~shouldSaveHistory=crossChainState->shouldSaveHistory),
|
|
115
154
|
~frontier,
|
|
116
155
|
~chainsBeforeBatch=crossChainState.chainStates->Utils.Dict.mapValues(
|
|
117
|
-
ChainState.toChainBeforeBatch,
|
|
156
|
+
ChainState.toChainBeforeBatch(~isRealtime=crossChainState.isRealtime, ...),
|
|
118
157
|
),
|
|
119
158
|
~batchSizeTarget,
|
|
120
159
|
)
|
|
@@ -122,6 +161,16 @@ let createBatch = (
|
|
|
122
161
|
|
|
123
162
|
// Enter the reorg threshold: shrink each chain's buffer by its configured
|
|
124
163
|
// blockLag and flip the flag.
|
|
164
|
+
// Whether every chain this process drives has buffered close enough to the head
|
|
165
|
+
// to enter the threshold together — and, in a split run, whether the rest of the
|
|
166
|
+
// run has too. Chains enter it as one indexer, so one chain still backfilling
|
|
167
|
+
// holds the others back whatever process it runs in.
|
|
168
|
+
let isReadyToEnterReorgThreshold = (crossChainState: t, ~batch) =>
|
|
169
|
+
!crossChainState.holdRealtime &&
|
|
170
|
+
crossChainState.chainStates
|
|
171
|
+
->Dict.valuesToArray
|
|
172
|
+
->Array.every(cs => cs->ChainState.isReadyToEnterReorgThresholdAfterBatch(~batch))
|
|
173
|
+
|
|
125
174
|
let enterReorgThreshold = (crossChainState: t) => {
|
|
126
175
|
Logging.info("Reorg threshold reached")
|
|
127
176
|
|
|
@@ -150,7 +199,10 @@ let applyBatchProgress = (crossChainState: t, ~batch: Batch.t, ~blockTimestampNa
|
|
|
150
199
|
}
|
|
151
200
|
|
|
152
201
|
crossChainState.isCaughtUp =
|
|
153
|
-
crossChainState.isCaughtUp ||
|
|
202
|
+
crossChainState.isCaughtUp ||
|
|
203
|
+
(!crossChainState.holdRealtime &&
|
|
204
|
+
crossChainState->nextItemIsNone &&
|
|
205
|
+
everyChainCaughtUp.contents)
|
|
154
206
|
}
|
|
155
207
|
|
|
156
208
|
// Every chain has buffered up to its head (or endblock) with nothing
|
|
@@ -174,8 +226,13 @@ let isSettledAtHead = (crossChainState: t) => {
|
|
|
174
226
|
}
|
|
175
227
|
|
|
176
228
|
// Enter the FinalizingIndexes phase without a batch, for the resume above.
|
|
229
|
+
// Not while the run holds this process back: the hold keeps the pre-threshold
|
|
230
|
+
// lag in place, and a chain that has fetched to a lagged head it was never
|
|
231
|
+
// going to get past reads as settled without having indexed anything. What a
|
|
232
|
+
// held process may conclude about where it stands, it concludes from what was
|
|
233
|
+
// persisted — see `markCaughtUpOnResume`, which the hold leaves alone.
|
|
177
234
|
let markCaughtUpIfSettled = (crossChainState: t) =>
|
|
178
|
-
if crossChainState->isSettledAtHead {
|
|
235
|
+
if !crossChainState.holdRealtime && crossChainState->isSettledAtHead {
|
|
179
236
|
crossChainState.isCaughtUp = true
|
|
180
237
|
}
|
|
181
238
|
|
|
@@ -20,19 +20,41 @@ function calculateTargetBufferSize() {
|
|
|
20
20
|
}
|
|
21
21
|
}
|
|
22
22
|
|
|
23
|
-
function make(chainStates, isRealtime, targetBufferSizeOpt) {
|
|
23
|
+
function make(chainStates, isRealtime, targetBufferSizeOpt, holdRealtimeOpt) {
|
|
24
24
|
let targetBufferSize = targetBufferSizeOpt !== undefined ? targetBufferSizeOpt : (
|
|
25
25
|
Env.targetBufferSize !== undefined ? Env.targetBufferSize : 100000
|
|
26
26
|
);
|
|
27
|
+
let holdRealtime = holdRealtimeOpt !== undefined ? holdRealtimeOpt : false;
|
|
27
28
|
return {
|
|
28
29
|
chainStates: chainStates,
|
|
29
30
|
chainIds: Object.values(chainStates).map(cs => ChainState.chainConfig(cs).id),
|
|
30
31
|
isRealtime: isRealtime,
|
|
31
32
|
isCaughtUp: isRealtime,
|
|
32
|
-
targetBufferSize: targetBufferSize
|
|
33
|
+
targetBufferSize: targetBufferSize,
|
|
34
|
+
holdRealtime: holdRealtime
|
|
33
35
|
};
|
|
34
36
|
}
|
|
35
37
|
|
|
38
|
+
function releaseRealtime(crossChainState) {
|
|
39
|
+
crossChainState.holdRealtime = false;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
function isHoldingRealtime(crossChainState) {
|
|
43
|
+
return crossChainState.holdRealtime;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
function hasArrivedAtHead(crossChainState) {
|
|
47
|
+
if (crossChainState.isCaughtUp || crossChainState.isRealtime) {
|
|
48
|
+
return true;
|
|
49
|
+
}
|
|
50
|
+
let chainStates = Object.values(crossChainState.chainStates);
|
|
51
|
+
if (Utils.$$Array.notEmpty(chainStates)) {
|
|
52
|
+
return chainStates.every(ChainState.isReadyToEnterReorgThreshold);
|
|
53
|
+
} else {
|
|
54
|
+
return false;
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
|
|
36
58
|
function chainStates(crossChainState) {
|
|
37
59
|
return crossChainState.chainStates;
|
|
38
60
|
}
|
|
@@ -91,7 +113,15 @@ function getSafeCheckpointIdByChain(crossChainState, sequence, committedFrontier
|
|
|
91
113
|
}
|
|
92
114
|
|
|
93
115
|
function createBatch(crossChainState, config, frontier, batchSizeTarget) {
|
|
94
|
-
return Batch.make(config.checkpointSequence, frontier, Utils.Dict.mapValues(crossChainState.chainStates, ChainState.toChainBeforeBatch), batchSizeTarget, HistoryPolicy.decide(config, shouldSaveHistory(crossChainState)));
|
|
116
|
+
return Batch.make(config.checkpointSequence, frontier, Utils.Dict.mapValues(crossChainState.chainStates, none => ChainState.toChainBeforeBatch(none, crossChainState.isRealtime)), batchSizeTarget, HistoryPolicy.decide(config, shouldSaveHistory(crossChainState)));
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
function isReadyToEnterReorgThreshold(crossChainState, batch) {
|
|
120
|
+
if (crossChainState.holdRealtime) {
|
|
121
|
+
return false;
|
|
122
|
+
} else {
|
|
123
|
+
return Object.values(crossChainState.chainStates).every(cs => ChainState.isReadyToEnterReorgThresholdAfterBatch(cs, batch));
|
|
124
|
+
}
|
|
95
125
|
}
|
|
96
126
|
|
|
97
127
|
function enterReorgThreshold(crossChainState) {
|
|
@@ -111,7 +141,7 @@ function applyBatchProgress(crossChainState, batch, blockTimestampName) {
|
|
|
111
141
|
everyChainCaughtUp = false;
|
|
112
142
|
}
|
|
113
143
|
}
|
|
114
|
-
crossChainState.isCaughtUp = crossChainState.isCaughtUp || nextItemIsNone(crossChainState) && everyChainCaughtUp;
|
|
144
|
+
crossChainState.isCaughtUp = crossChainState.isCaughtUp || !crossChainState.holdRealtime && nextItemIsNone(crossChainState) && everyChainCaughtUp;
|
|
115
145
|
}
|
|
116
146
|
|
|
117
147
|
function isSettledAtHead(crossChainState) {
|
|
@@ -125,7 +155,7 @@ function isSettledAtHead(crossChainState) {
|
|
|
125
155
|
}
|
|
126
156
|
|
|
127
157
|
function markCaughtUpIfSettled(crossChainState) {
|
|
128
|
-
if (isSettledAtHead(crossChainState)) {
|
|
158
|
+
if (!crossChainState.holdRealtime && isSettledAtHead(crossChainState)) {
|
|
129
159
|
crossChainState.isCaughtUp = true;
|
|
130
160
|
return;
|
|
131
161
|
}
|
|
@@ -256,6 +286,10 @@ export {
|
|
|
256
286
|
shouldSaveHistory,
|
|
257
287
|
nextItemIsNone,
|
|
258
288
|
getSafeCheckpointIdByChain,
|
|
289
|
+
releaseRealtime,
|
|
290
|
+
isHoldingRealtime,
|
|
291
|
+
hasArrivedAtHead,
|
|
292
|
+
isReadyToEnterReorgThreshold,
|
|
259
293
|
createBatch,
|
|
260
294
|
enterReorgThreshold,
|
|
261
295
|
applyBatchProgress,
|
package/src/CrossChainState.resi
CHANGED
|
@@ -5,7 +5,12 @@ type t
|
|
|
5
5
|
|
|
6
6
|
let calculateTargetBufferSize: unit => int
|
|
7
7
|
|
|
8
|
-
let make: (
|
|
8
|
+
let make: (
|
|
9
|
+
~chainStates: dict<ChainState.t>,
|
|
10
|
+
~isRealtime: bool,
|
|
11
|
+
~targetBufferSize: int=?,
|
|
12
|
+
~holdRealtime: bool=?,
|
|
13
|
+
) => t
|
|
9
14
|
|
|
10
15
|
// Accessors.
|
|
11
16
|
let chainStates: t => dict<ChainState.t>
|
|
@@ -25,6 +30,10 @@ let getSafeCheckpointIdByChain: (
|
|
|
25
30
|
) => array<(ChainId.t, option<Internal.checkpointId>)>
|
|
26
31
|
|
|
27
32
|
// Cross-chain transitions.
|
|
33
|
+
let releaseRealtime: t => unit
|
|
34
|
+
let isHoldingRealtime: t => bool
|
|
35
|
+
let hasArrivedAtHead: t => bool
|
|
36
|
+
let isReadyToEnterReorgThreshold: (t, ~batch: Batch.t) => bool
|
|
28
37
|
let createBatch: (t, ~config: Config.t, ~frontier: Frontier.t, ~batchSizeTarget: int) => Batch.t
|
|
29
38
|
let enterReorgThreshold: t => unit
|
|
30
39
|
let applyBatchProgress: (t, ~batch: Batch.t, ~blockTimestampName: string) => unit
|
package/src/Env.res
CHANGED
|
@@ -127,6 +127,10 @@ module Db = {
|
|
|
127
127
|
//the SSL modes should be provided as string otherwise as 'require' | 'allow' | 'prefer' | 'verify-full'
|
|
128
128
|
~devFallback=Bool(false),
|
|
129
129
|
)
|
|
130
|
+
// The budget for the whole run, not for one process: a run that splits across
|
|
131
|
+
// workers divides it among them, and each caps its own pool to its share.
|
|
132
|
+
// The default buys a single worker, so a run splits only once the operator
|
|
133
|
+
// raises the budget it may spend.
|
|
130
134
|
let maxConnections = envSafe->EnvSafe.get("ENVIO_PG_MAX_CONNECTIONS", S.int, ~fallback=2)
|
|
131
135
|
}
|
|
132
136
|
|
package/src/FetchState.res
CHANGED
|
@@ -50,6 +50,22 @@ let makeSelection = (~onEventRegistrations, ~dependsOnAddresses, ~clientFiltered
|
|
|
50
50
|
startBlock: ?deriveSelectionStartBlock(onEventRegistrations),
|
|
51
51
|
}
|
|
52
52
|
|
|
53
|
+
// A partition's frontier never sits below the block before its selection can
|
|
54
|
+
// first match: the blocks below hold nothing for it, so they count as fetched
|
|
55
|
+
// the same way the blocks before an address's start block do. The chain's
|
|
56
|
+
// buffer frontier is its lowest partition frontier, so a partition left at the
|
|
57
|
+
// chain start would hold every other partition's events back as unprocessable
|
|
58
|
+
// and keep the chain's query target — sized from that frontier — short of
|
|
59
|
+
// anything worth asking for. A start block the chain has not reached yet leaves
|
|
60
|
+
// the partition waiting there, like an address partition does, rather than
|
|
61
|
+
// scanning empty ranges up to it; bufferBlockNumber caps the chain's frontier at
|
|
62
|
+
// the head meanwhile.
|
|
63
|
+
let floorAtSelectionStart = (latestFetchedBlock, ~selection) =>
|
|
64
|
+
switch selection.startBlock {
|
|
65
|
+
| Some(startBlock) => Pervasives.max(latestFetchedBlock, startBlock - 1)
|
|
66
|
+
| None => latestFetchedBlock
|
|
67
|
+
}
|
|
68
|
+
|
|
53
69
|
type pendingQuery = {
|
|
54
70
|
fromBlock: int,
|
|
55
71
|
toBlock: option<int>,
|
|
@@ -398,6 +414,10 @@ module OptimizedPartitions = {
|
|
|
398
414
|
~dynamicContracts: Utils.Set.t<string>,
|
|
399
415
|
~clientFilteredContracts: Utils.Set.t<string>,
|
|
400
416
|
) => {
|
|
417
|
+
let partitions = partitions->Array.map(p => {
|
|
418
|
+
let floored = p.latestFetchedBlock->floorAtSelectionStart(~selection=p.selection)
|
|
419
|
+
floored === p.latestFetchedBlock ? p : {...p, latestFetchedBlock: floored}
|
|
420
|
+
})
|
|
401
421
|
let newPartitions = []
|
|
402
422
|
let mergingPartitions = Dict.make()
|
|
403
423
|
let nextPartitionIndexRef = ref(nextPartitionIndex)
|
|
@@ -847,28 +867,17 @@ type t = {
|
|
|
847
867
|
clientFilterAddressThreshold: option<int>,
|
|
848
868
|
}
|
|
849
869
|
|
|
870
|
+
// The latest block whose items are all in the buffer: the lowest partition
|
|
871
|
+
// frontier, held back by the onBlock pointer. Without onBlock registrations the
|
|
872
|
+
// pointer tracks the known height, so this also caps the frontier at the head: a
|
|
873
|
+
// partition can sit past it, on a response that reported a block the chain
|
|
874
|
+
// hadn't heard of yet or on a start block the chain hasn't reached.
|
|
850
875
|
@inline
|
|
851
876
|
let bufferBlockNumber = ({latestOnBlockBlockNumber, optimizedPartitions}: t) => {
|
|
852
877
|
switch optimizedPartitions->OptimizedPartitions.getLatestFullyFetchedBlock {
|
|
853
878
|
| None => latestOnBlockBlockNumber
|
|
854
879
|
| Some(latestFullyFetchedBlock) =>
|
|
855
|
-
latestOnBlockBlockNumber
|
|
856
|
-
? latestOnBlockBlockNumber
|
|
857
|
-
: latestFullyFetchedBlock
|
|
858
|
-
}
|
|
859
|
-
}
|
|
860
|
-
|
|
861
|
-
/**
|
|
862
|
-
* Returns the latest block which is ready to be consumed
|
|
863
|
-
*/
|
|
864
|
-
@inline
|
|
865
|
-
let bufferBlock = ({optimizedPartitions, latestOnBlockBlockNumber}: t) => {
|
|
866
|
-
switch optimizedPartitions->OptimizedPartitions.getLatestFullyFetchedBlock {
|
|
867
|
-
| None => latestOnBlockBlockNumber
|
|
868
|
-
| Some(latestFullyFetchedBlock) =>
|
|
869
|
-
latestOnBlockBlockNumber < latestFullyFetchedBlock
|
|
870
|
-
? latestOnBlockBlockNumber
|
|
871
|
-
: latestFullyFetchedBlock
|
|
880
|
+
Pervasives.min(latestOnBlockBlockNumber, latestFullyFetchedBlock)
|
|
872
881
|
}
|
|
873
882
|
}
|
|
874
883
|
|
|
@@ -1138,14 +1147,20 @@ let updateInternal = (
|
|
|
1138
1147
|
// Use maxOnBlockBufferSize to get the last target item in the buffer
|
|
1139
1148
|
// (sorted, so this is the highest-block item within the buffer cap).
|
|
1140
1149
|
// All this needed to prevent OOM when adding too many block items to the queue
|
|
1141
|
-
|
|
1142
|
-
|
|
1143
|
-
|
|
1144
|
-
|
|
1145
|
-
|
|
1146
|
-
| Some(
|
|
1147
|
-
|
|
1148
|
-
|
|
1150
|
+
// Never past the head: a partition can sit there, on a start block the
|
|
1151
|
+
// chain hasn't reached, and a response is applied before the height it
|
|
1152
|
+
// reported, so its events can run past the height still known here.
|
|
1153
|
+
let maxBlockNumber = Pervasives.min(
|
|
1154
|
+
switch base->Array.get(fetchState.maxOnBlockBufferSize - 1) {
|
|
1155
|
+
| Some(item) => item->Internal.getItemBlockNumber
|
|
1156
|
+
| None =>
|
|
1157
|
+
switch optimizedPartitions->OptimizedPartitions.getLatestFullyFetchedBlock {
|
|
1158
|
+
| None => knownHeight
|
|
1159
|
+
| Some(latestFullyFetchedBlock) => latestFullyFetchedBlock
|
|
1160
|
+
}
|
|
1161
|
+
},
|
|
1162
|
+
knownHeight,
|
|
1163
|
+
)
|
|
1149
1164
|
appendOnBlockItems(
|
|
1150
1165
|
~mutItems=blockItems,
|
|
1151
1166
|
~onBlockRegistrations,
|
|
@@ -2165,27 +2180,11 @@ let walkPartitionPending = (
|
|
|
2165
2180
|
pqIdx := pqIdx.contents + 1
|
|
2166
2181
|
}
|
|
2167
2182
|
|
|
2168
|
-
// Nothing in this partition's selection can match below its earliest start
|
|
2169
|
-
// block, so forward work skips straight to it instead of scanning up to it and
|
|
2170
|
-
// discarding whole pages. Only the cursor moves — `latestFetchedBlock` still
|
|
2171
|
-
// advances solely on a response, so no block is ever reported fetched that
|
|
2172
|
-
// wasn't. Bounded by the head and the query end block: past either, the
|
|
2173
|
-
// partition would offer no candidate at all, so it queries as before rather
|
|
2174
|
-
// than going quiet until the chain reaches its start block.
|
|
2175
|
-
let cursor = switch p.selection.startBlock {
|
|
2176
|
-
| Some(startBlock) if startBlock > cursor.contents =>
|
|
2177
|
-
switch Utils.Math.minOptInt(Some(headBlockNumber), queryEndBlock) {
|
|
2178
|
-
| Some(reachable) if startBlock <= reachable => startBlock
|
|
2179
|
-
| _ => cursor.contents
|
|
2180
|
-
}
|
|
2181
|
-
| _ => cursor.contents
|
|
2182
|
-
}
|
|
2183
|
-
|
|
2184
2183
|
canContinue.contents
|
|
2185
2184
|
? Some({
|
|
2186
2185
|
partitionId,
|
|
2187
2186
|
p,
|
|
2188
|
-
cursor,
|
|
2187
|
+
cursor: cursor.contents,
|
|
2189
2188
|
chunksUsedThisCall: chunksUsedThisCall.contents,
|
|
2190
2189
|
inFlightCount,
|
|
2191
2190
|
queryEndBlock,
|
|
@@ -2780,10 +2779,17 @@ let make = (
|
|
|
2780
2779
|
// fetching, so without seeding the buffer here getNextQuery would return
|
|
2781
2780
|
// NothingToQuery and the indexer would get stuck.
|
|
2782
2781
|
let buffer = []
|
|
2783
|
-
let latestOnBlockBlockNumber =
|
|
2782
|
+
let latestOnBlockBlockNumber = switch onBlockRegistrations {
|
|
2783
|
+
// As updateInternal keeps it: with nothing to generate per block the pointer
|
|
2784
|
+
// is the head cap on the buffer frontier, and left at the progress block it
|
|
2785
|
+
// would hold the frontier there until the first height update — below a
|
|
2786
|
+
// partition that starts later, whose queries the chain sizes off that
|
|
2787
|
+
// frontier and so never reaches.
|
|
2788
|
+
| [] if knownHeight > 0 => knownHeight
|
|
2789
|
+
| onBlockRegistrations if knownHeight > 0 =>
|
|
2784
2790
|
let maxBlockNumber = switch optimizedPartitions->OptimizedPartitions.getLatestFullyFetchedBlock {
|
|
2785
2791
|
| None => knownHeight
|
|
2786
|
-
| Some(latestFullyFetchedBlock) => latestFullyFetchedBlock
|
|
2792
|
+
| Some(latestFullyFetchedBlock) => Pervasives.min(latestFullyFetchedBlock, knownHeight)
|
|
2787
2793
|
}
|
|
2788
2794
|
appendOnBlockItems(
|
|
2789
2795
|
~mutItems=buffer,
|
|
@@ -2793,8 +2799,7 @@ let make = (
|
|
|
2793
2799
|
~maxBlockNumber,
|
|
2794
2800
|
~maxOnBlockBufferSize,
|
|
2795
2801
|
)
|
|
2796
|
-
|
|
2797
|
-
progressBlockNumber
|
|
2802
|
+
| _ => progressBlockNumber
|
|
2798
2803
|
}
|
|
2799
2804
|
|
|
2800
2805
|
let fetchState = {
|
package/src/FetchState.res.mjs
CHANGED
|
@@ -47,6 +47,15 @@ function makeSelection(onEventRegistrations, dependsOnAddresses, clientFilteredC
|
|
|
47
47
|
};
|
|
48
48
|
}
|
|
49
49
|
|
|
50
|
+
function floorAtSelectionStart(latestFetchedBlock, selection) {
|
|
51
|
+
let startBlock = selection.startBlock;
|
|
52
|
+
if (startBlock !== undefined) {
|
|
53
|
+
return Primitive_int.max(latestFetchedBlock, startBlock - 1 | 0);
|
|
54
|
+
} else {
|
|
55
|
+
return latestFetchedBlock;
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
|
|
50
59
|
function withAddresses(p, addresses) {
|
|
51
60
|
return {
|
|
52
61
|
id: p.id,
|
|
@@ -305,6 +314,26 @@ function getAnchorSafeBlock(p) {
|
|
|
305
314
|
}
|
|
306
315
|
|
|
307
316
|
function make(partitions, maxAddrInPartition, nextPartitionIndex, dynamicContracts, clientFilteredContracts) {
|
|
317
|
+
let partitions$1 = partitions.map(p => {
|
|
318
|
+
let floored = floorAtSelectionStart(p.latestFetchedBlock, p.selection);
|
|
319
|
+
if (floored === p.latestFetchedBlock) {
|
|
320
|
+
return p;
|
|
321
|
+
} else {
|
|
322
|
+
return {
|
|
323
|
+
id: p.id,
|
|
324
|
+
latestFetchedBlock: floored,
|
|
325
|
+
selection: p.selection,
|
|
326
|
+
addresses: p.addresses,
|
|
327
|
+
mergeBlock: p.mergeBlock,
|
|
328
|
+
dynamicContract: p.dynamicContract,
|
|
329
|
+
mutPendingQueries: p.mutPendingQueries,
|
|
330
|
+
sourceRangeCapacity: p.sourceRangeCapacity,
|
|
331
|
+
prevSourceRangeCapacity: p.prevSourceRangeCapacity,
|
|
332
|
+
eventDensity: p.eventDensity,
|
|
333
|
+
latestSourceRangeCapacityUpdateBlock: p.latestSourceRangeCapacityUpdateBlock
|
|
334
|
+
};
|
|
335
|
+
}
|
|
336
|
+
});
|
|
308
337
|
let newPartitions = [];
|
|
309
338
|
let mergingPartitions = {};
|
|
310
339
|
let nextPartitionIndexRef = {
|
|
@@ -313,7 +342,7 @@ function make(partitions, maxAddrInPartition, nextPartitionIndex, dynamicContrac
|
|
|
313
342
|
let anchorSafeBlocks = {};
|
|
314
343
|
let anchoredContractsSet = new Set();
|
|
315
344
|
if (clientFilteredContracts.size > 0) {
|
|
316
|
-
partitions.forEach(p => {
|
|
345
|
+
partitions$1.forEach(p => {
|
|
317
346
|
let match = p.mergeBlock;
|
|
318
347
|
let match$1 = p.selection.clientFilteredContracts;
|
|
319
348
|
if (match !== undefined) {
|
|
@@ -332,8 +361,8 @@ function make(partitions, maxAddrInPartition, nextPartitionIndex, dynamicContrac
|
|
|
332
361
|
});
|
|
333
362
|
});
|
|
334
363
|
}
|
|
335
|
-
for (let idx = 0, idx_finish = partitions.length; idx < idx_finish; ++idx) {
|
|
336
|
-
let p = partitions[idx];
|
|
364
|
+
for (let idx = 0, idx_finish = partitions$1.length; idx < idx_finish; ++idx) {
|
|
365
|
+
let p = partitions$1[idx];
|
|
337
366
|
let mergeBlock = p.mergeBlock;
|
|
338
367
|
if (mergeBlock !== undefined) {
|
|
339
368
|
if (p.latestFetchedBlock < mergeBlock || isFetching(p)) {
|
|
@@ -676,20 +705,8 @@ function bufferBlockNumber(param) {
|
|
|
676
705
|
let optimizedPartitions = param.optimizedPartitions;
|
|
677
706
|
let id = optimizedPartitions.idsInAscOrder[0];
|
|
678
707
|
let latestFullyFetchedBlock = id !== undefined ? optimizedPartitions.entities[id].latestFetchedBlock : undefined;
|
|
679
|
-
if (latestFullyFetchedBlock !== undefined
|
|
680
|
-
return latestFullyFetchedBlock;
|
|
681
|
-
} else {
|
|
682
|
-
return latestOnBlockBlockNumber;
|
|
683
|
-
}
|
|
684
|
-
}
|
|
685
|
-
|
|
686
|
-
function bufferBlock(param) {
|
|
687
|
-
let latestOnBlockBlockNumber = param.latestOnBlockBlockNumber;
|
|
688
|
-
let optimizedPartitions = param.optimizedPartitions;
|
|
689
|
-
let id = optimizedPartitions.idsInAscOrder[0];
|
|
690
|
-
let latestFullyFetchedBlock = id !== undefined ? optimizedPartitions.entities[id].latestFetchedBlock : undefined;
|
|
691
|
-
if (latestFullyFetchedBlock !== undefined && latestOnBlockBlockNumber >= latestFullyFetchedBlock) {
|
|
692
|
-
return latestFullyFetchedBlock;
|
|
708
|
+
if (latestFullyFetchedBlock !== undefined) {
|
|
709
|
+
return Primitive_int.min(latestOnBlockBlockNumber, latestFullyFetchedBlock);
|
|
693
710
|
} else {
|
|
694
711
|
return latestOnBlockBlockNumber;
|
|
695
712
|
}
|
|
@@ -897,14 +914,15 @@ function updateInternal(fetchState, optimizedPartitionsOpt, mutItems, mutItemsSo
|
|
|
897
914
|
let latestOnBlockBlockNumber;
|
|
898
915
|
if (onBlockRegistrations.length !== 0) {
|
|
899
916
|
let item = base[fetchState.maxOnBlockBufferSize - 1 | 0];
|
|
900
|
-
let
|
|
917
|
+
let tmp;
|
|
901
918
|
if (item !== undefined) {
|
|
902
|
-
|
|
919
|
+
tmp = item.blockNumber;
|
|
903
920
|
} else {
|
|
904
921
|
let id = optimizedPartitions.idsInAscOrder[0];
|
|
905
922
|
let latestFullyFetchedBlock = id !== undefined ? optimizedPartitions.entities[id].latestFetchedBlock : undefined;
|
|
906
|
-
|
|
923
|
+
tmp = latestFullyFetchedBlock !== undefined ? latestFullyFetchedBlock : knownHeight;
|
|
907
924
|
}
|
|
925
|
+
let maxBlockNumber = Primitive_int.min(tmp, knownHeight);
|
|
908
926
|
latestOnBlockBlockNumber = appendOnBlockItems(blockItems, onBlockRegistrations, fetchState.startBlock, fetchState.latestOnBlockBlockNumber, maxBlockNumber, fetchState.maxOnBlockBufferSize);
|
|
909
927
|
} else {
|
|
910
928
|
latestOnBlockBlockNumber = knownHeight;
|
|
@@ -1486,19 +1504,11 @@ function walkPartitionPending(p, partitionId, inFlightCount, candidates, headBlo
|
|
|
1486
1504
|
}
|
|
1487
1505
|
pqIdx = pqIdx + 1 | 0;
|
|
1488
1506
|
};
|
|
1489
|
-
let startBlock = p.selection.startBlock;
|
|
1490
|
-
let cursor$1;
|
|
1491
|
-
if (startBlock !== undefined && startBlock > cursor) {
|
|
1492
|
-
let reachable = Utils.$$Math.minOptInt(headBlockNumber, queryEndBlock);
|
|
1493
|
-
cursor$1 = reachable !== undefined && startBlock <= reachable ? startBlock : cursor;
|
|
1494
|
-
} else {
|
|
1495
|
-
cursor$1 = cursor;
|
|
1496
|
-
}
|
|
1497
1507
|
if (canContinue) {
|
|
1498
1508
|
return {
|
|
1499
1509
|
partitionId: partitionId,
|
|
1500
1510
|
p: p,
|
|
1501
|
-
cursor: cursor
|
|
1511
|
+
cursor: cursor,
|
|
1502
1512
|
chunksUsedThisCall: chunksUsedThisCall,
|
|
1503
1513
|
inFlightCount: inFlightCount,
|
|
1504
1514
|
queryEndBlock: queryEndBlock,
|
|
@@ -1800,13 +1810,21 @@ function make$1(startBlock, endBlock, onEventRegistrations, addressStore, addres
|
|
|
1800
1810
|
}
|
|
1801
1811
|
let buffer = [];
|
|
1802
1812
|
let latestOnBlockBlockNumber;
|
|
1803
|
-
|
|
1804
|
-
|
|
1805
|
-
|
|
1806
|
-
let maxBlockNumber = latestFullyFetchedBlock !== undefined ? latestFullyFetchedBlock : knownHeight;
|
|
1807
|
-
latestOnBlockBlockNumber = appendOnBlockItems(buffer, onBlockRegistrations, startBlock, progressBlockNumber, maxBlockNumber, maxOnBlockBufferSize);
|
|
1813
|
+
let exit = 0;
|
|
1814
|
+
if (onBlockRegistrations.length !== 0 || knownHeight <= 0) {
|
|
1815
|
+
exit = 1;
|
|
1808
1816
|
} else {
|
|
1809
|
-
latestOnBlockBlockNumber =
|
|
1817
|
+
latestOnBlockBlockNumber = knownHeight;
|
|
1818
|
+
}
|
|
1819
|
+
if (exit === 1) {
|
|
1820
|
+
if (knownHeight > 0) {
|
|
1821
|
+
let id = optimizedPartitions.idsInAscOrder[0];
|
|
1822
|
+
let latestFullyFetchedBlock = id !== undefined ? optimizedPartitions.entities[id].latestFetchedBlock : undefined;
|
|
1823
|
+
let maxBlockNumber = latestFullyFetchedBlock !== undefined ? Primitive_int.min(latestFullyFetchedBlock, knownHeight) : knownHeight;
|
|
1824
|
+
latestOnBlockBlockNumber = appendOnBlockItems(buffer, onBlockRegistrations, startBlock, progressBlockNumber, maxBlockNumber, maxOnBlockBufferSize);
|
|
1825
|
+
} else {
|
|
1826
|
+
latestOnBlockBlockNumber = progressBlockNumber;
|
|
1827
|
+
}
|
|
1810
1828
|
}
|
|
1811
1829
|
return {
|
|
1812
1830
|
optimizedPartitions: optimizedPartitions,
|
|
@@ -2111,6 +2129,7 @@ let chunkRangeGrowthFactor = 1.8;
|
|
|
2111
2129
|
export {
|
|
2112
2130
|
deriveSelectionStartBlock,
|
|
2113
2131
|
makeSelection,
|
|
2132
|
+
floorAtSelectionStart,
|
|
2114
2133
|
withAddresses,
|
|
2115
2134
|
narrowSelectionToRange,
|
|
2116
2135
|
densityItemsTarget,
|
|
@@ -2120,7 +2139,6 @@ export {
|
|
|
2120
2139
|
getMinQueryRange,
|
|
2121
2140
|
OptimizedPartitions,
|
|
2122
2141
|
bufferBlockNumber,
|
|
2123
|
-
bufferBlock,
|
|
2124
2142
|
bufferReadyCount,
|
|
2125
2143
|
getRegistrationIndex,
|
|
2126
2144
|
comparePath,
|