envio 3.12.0 → 3.13.0-alpha.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (84) hide show
  1. package/package.json +6 -6
  2. package/src/BatchProcessing.res +9 -9
  3. package/src/BatchProcessing.res.mjs +5 -5
  4. package/src/Bin.res +24 -17
  5. package/src/Bin.res.mjs +5 -0
  6. package/src/ChainFetching.res +31 -13
  7. package/src/ChainFetching.res.mjs +9 -4
  8. package/src/ChainState.res +55 -3
  9. package/src/ChainState.res.mjs +46 -3
  10. package/src/ChainState.resi +3 -1
  11. package/src/Config.res +35 -0
  12. package/src/Config.res.mjs +39 -0
  13. package/src/Core.res +4 -0
  14. package/src/Core.res.mjs +4 -0
  15. package/src/CrossChainState.res +61 -4
  16. package/src/CrossChainState.res.mjs +39 -5
  17. package/src/CrossChainState.resi +10 -1
  18. package/src/Env.res +4 -0
  19. package/src/FetchState.res +51 -46
  20. package/src/FetchState.res.mjs +54 -36
  21. package/src/InMemoryTable.res +155 -67
  22. package/src/InMemoryTable.res.mjs +151 -60
  23. package/src/IndexerLoop.res +2 -0
  24. package/src/IndexerLoop.res.mjs +1 -0
  25. package/src/IndexerState.res +41 -1
  26. package/src/IndexerState.res.mjs +43 -4
  27. package/src/IndexerState.resi +10 -0
  28. package/src/LoadLayer.res +57 -34
  29. package/src/LoadLayer.res.mjs +45 -62
  30. package/src/LoadLayer.resi +1 -1
  31. package/src/Logging.res +38 -6
  32. package/src/Logging.res.mjs +32 -5
  33. package/src/Main.res +131 -268
  34. package/src/Main.res.mjs +28 -152
  35. package/src/Metrics.res +263 -102
  36. package/src/Metrics.res.mjs +227 -48
  37. package/src/Persistence.res +27 -2
  38. package/src/Persistence.res.mjs +9 -2
  39. package/src/PgStorage.res +109 -71
  40. package/src/PgStorage.res.mjs +88 -58
  41. package/src/Server.res +181 -0
  42. package/src/Server.res.mjs +143 -0
  43. package/src/Supervisor.res +415 -0
  44. package/src/Supervisor.res.mjs +325 -0
  45. package/src/TestIndexer.res +9 -25
  46. package/src/TestIndexer.res.mjs +5 -4
  47. package/src/UserContext.res +13 -32
  48. package/src/UserContext.res.mjs +1 -7
  49. package/src/Utils.res +1 -4
  50. package/src/Utils.res.mjs +7 -16
  51. package/src/Worker.res +95 -0
  52. package/src/Worker.res.mjs +80 -0
  53. package/src/bindings/NodeJs.res +41 -0
  54. package/src/db/EntityFilter.res +487 -275
  55. package/src/db/EntityFilter.res.mjs +557 -309
  56. package/src/db/InternalTable.res +8 -1
  57. package/src/db/InternalTable.res.mjs +5 -1
  58. package/src/db/Table.res +21 -6
  59. package/src/db/Table.res.mjs +13 -4
  60. package/src/sources/BlockStore.res +7 -2
  61. package/src/sources/EvmHyperSyncSource.res +2 -0
  62. package/src/sources/EvmHyperSyncSource.res.mjs +2 -2
  63. package/src/sources/FuelHyperSyncSource.res +1 -0
  64. package/src/sources/FuelHyperSyncSource.res.mjs +1 -1
  65. package/src/sources/HyperSync.res +4 -0
  66. package/src/sources/HyperSync.res.mjs +4 -2
  67. package/src/sources/HyperSync.resi +1 -0
  68. package/src/sources/HyperSyncClient.res +3 -0
  69. package/src/sources/HyperSyncSSE.res +1 -1
  70. package/src/sources/HyperSyncSSE.res.mjs +4 -10
  71. package/src/sources/RpcSource.res +1 -0
  72. package/src/sources/RpcSource.res.mjs +1 -1
  73. package/src/sources/SimulateSource.res +1 -0
  74. package/src/sources/SimulateSource.res.mjs +1 -1
  75. package/src/sources/Source.res +7 -0
  76. package/src/sources/SourceManager.res +4 -3
  77. package/src/sources/SourceManager.res.mjs +2 -2
  78. package/src/sources/SvmHyperSyncClient.res +5 -0
  79. package/src/sources/SvmHyperSyncSource.res +3 -0
  80. package/src/sources/SvmHyperSyncSource.res.mjs +4 -2
  81. package/src/tui/Tui.res +24 -0
  82. package/src/tui/Tui.res.mjs +18 -0
  83. package/src/tui/components/SyncETA.res +12 -6
  84. package/src/tui/components/SyncETA.res.mjs +12 -8
package/src/Core.res.mjs CHANGED
@@ -88,6 +88,10 @@ let loadDevAddon = (function(req, envioDir) {
88
88
  fs.copyFileSync(srcPath, nodePath);
89
89
  }
90
90
 
91
+ // Forked workers inherit this, so only the first process in a run pays for
92
+ // the cargo build (and they don't contend over the cargo lock).
93
+ process.env.ENVIO_DEV_ADDON = nodePath;
94
+
91
95
  return req(nodePath);
92
96
  });
93
97
 
@@ -17,6 +17,11 @@ type t = {
17
17
  mutable isCaughtUp: bool,
18
18
  // Indexer-wide fetch buffer pool (item count), shared across all chains.
19
19
  targetBufferSize: int,
20
+ // Set on a process driving part of a split run: the chains it drives may be
21
+ // at the head while chains in another process are still backfilling, and an
22
+ // indexer switches to realtime as a whole or not at all. Cleared by the
23
+ // supervisor once every chain in the run has arrived.
24
+ mutable holdRealtime: bool,
20
25
  }
21
26
 
22
27
  // The whole-indexer fetch buffer pool, independent of chain count.
@@ -26,16 +31,50 @@ let calculateTargetBufferSize = () =>
26
31
  | None => 100_000
27
32
  }
28
33
 
29
- let make = (~chainStates, ~isRealtime, ~targetBufferSize=calculateTargetBufferSize()): t => {
34
+ let make = (
35
+ ~chainStates,
36
+ ~isRealtime,
37
+ ~targetBufferSize=calculateTargetBufferSize(),
38
+ ~holdRealtime=false,
39
+ ): t => {
30
40
  {
31
41
  chainStates,
32
42
  chainIds: chainStates->Dict.valuesToArray->Array.map(cs => (cs->ChainState.chainConfig).id),
33
43
  isRealtime,
34
44
  isCaughtUp: isRealtime,
35
45
  targetBufferSize,
46
+ holdRealtime,
36
47
  }
37
48
  }
38
49
 
50
+ // The supervisor's go-ahead: every chain in the run has reached the head, so
51
+ // this process may make the transitions it has been holding back.
52
+ let releaseRealtime = (crossChainState: t) => crossChainState.holdRealtime = false
53
+
54
+ let isHoldingRealtime = (crossChainState: t) => crossChainState.holdRealtime
55
+
56
+ // Whether this process has got as far as it can without the run's leave. What a
57
+ // supervisor reads to decide that a split run may go realtime as one.
58
+ //
59
+ // Three ways to have arrived, because a chain can be as far along as it can get
60
+ // in three different states. Its chains have caught up; or it resumed already
61
+ // realtime; or every chain is waiting to enter the reorg threshold, which is as
62
+ // far as one can fetch while the pre-threshold lag holds it at the safe block —
63
+ // entering the threshold is what lifts that lag, so a run held until its chains
64
+ // reached the head would be holding back the transition that gets them there.
65
+ //
66
+ // The process's own conclusion rather than a reading a supervisor reassembles:
67
+ // a chain committed at what was the head and resumed once the head had moved on
68
+ // has arrived, and no live reading of it can say so — which is the same reason
69
+ // `markCaughtUpOnResume` decides before any source request.
70
+ let hasArrivedAtHead = (crossChainState: t) =>
71
+ crossChainState.isCaughtUp ||
72
+ crossChainState.isRealtime || {
73
+ let chainStates = crossChainState.chainStates->Dict.valuesToArray
74
+ chainStates->Utils.Array.notEmpty &&
75
+ chainStates->Array.every(ChainState.isReadyToEnterReorgThreshold)
76
+ }
77
+
39
78
  // Resolve a chain's state by id. The id always comes from `chainIds`, which is
40
79
  // derived from `chainStates`, so the entry is guaranteed present.
41
80
  let getChainState = (crossChainState: t, chainId) =>
@@ -114,7 +153,7 @@ let createBatch = (
114
153
  ~history=config->HistoryPolicy.decide(~shouldSaveHistory=crossChainState->shouldSaveHistory),
115
154
  ~frontier,
116
155
  ~chainsBeforeBatch=crossChainState.chainStates->Utils.Dict.mapValues(
117
- ChainState.toChainBeforeBatch,
156
+ ChainState.toChainBeforeBatch(~isRealtime=crossChainState.isRealtime, ...),
118
157
  ),
119
158
  ~batchSizeTarget,
120
159
  )
@@ -122,6 +161,16 @@ let createBatch = (
122
161
 
123
162
  // Enter the reorg threshold: shrink each chain's buffer by its configured
124
163
  // blockLag and flip the flag.
164
+ // Whether every chain this process drives has buffered close enough to the head
165
+ // to enter the threshold together — and, in a split run, whether the rest of the
166
+ // run has too. Chains enter it as one indexer, so one chain still backfilling
167
+ // holds the others back whatever process it runs in.
168
+ let isReadyToEnterReorgThreshold = (crossChainState: t, ~batch) =>
169
+ !crossChainState.holdRealtime &&
170
+ crossChainState.chainStates
171
+ ->Dict.valuesToArray
172
+ ->Array.every(cs => cs->ChainState.isReadyToEnterReorgThresholdAfterBatch(~batch))
173
+
125
174
  let enterReorgThreshold = (crossChainState: t) => {
126
175
  Logging.info("Reorg threshold reached")
127
176
 
@@ -150,7 +199,10 @@ let applyBatchProgress = (crossChainState: t, ~batch: Batch.t, ~blockTimestampNa
150
199
  }
151
200
 
152
201
  crossChainState.isCaughtUp =
153
- crossChainState.isCaughtUp || (crossChainState->nextItemIsNone && everyChainCaughtUp.contents)
202
+ crossChainState.isCaughtUp ||
203
+ (!crossChainState.holdRealtime &&
204
+ crossChainState->nextItemIsNone &&
205
+ everyChainCaughtUp.contents)
154
206
  }
155
207
 
156
208
  // Every chain has buffered up to its head (or endblock) with nothing
@@ -174,8 +226,13 @@ let isSettledAtHead = (crossChainState: t) => {
174
226
  }
175
227
 
176
228
  // Enter the FinalizingIndexes phase without a batch, for the resume above.
229
+ // Not while the run holds this process back: the hold keeps the pre-threshold
230
+ // lag in place, and a chain that has fetched to a lagged head it was never
231
+ // going to get past reads as settled without having indexed anything. What a
232
+ // held process may conclude about where it stands, it concludes from what was
233
+ // persisted — see `markCaughtUpOnResume`, which the hold leaves alone.
177
234
  let markCaughtUpIfSettled = (crossChainState: t) =>
178
- if crossChainState->isSettledAtHead {
235
+ if !crossChainState.holdRealtime && crossChainState->isSettledAtHead {
179
236
  crossChainState.isCaughtUp = true
180
237
  }
181
238
 
@@ -20,19 +20,41 @@ function calculateTargetBufferSize() {
20
20
  }
21
21
  }
22
22
 
23
- function make(chainStates, isRealtime, targetBufferSizeOpt) {
23
+ function make(chainStates, isRealtime, targetBufferSizeOpt, holdRealtimeOpt) {
24
24
  let targetBufferSize = targetBufferSizeOpt !== undefined ? targetBufferSizeOpt : (
25
25
  Env.targetBufferSize !== undefined ? Env.targetBufferSize : 100000
26
26
  );
27
+ let holdRealtime = holdRealtimeOpt !== undefined ? holdRealtimeOpt : false;
27
28
  return {
28
29
  chainStates: chainStates,
29
30
  chainIds: Object.values(chainStates).map(cs => ChainState.chainConfig(cs).id),
30
31
  isRealtime: isRealtime,
31
32
  isCaughtUp: isRealtime,
32
- targetBufferSize: targetBufferSize
33
+ targetBufferSize: targetBufferSize,
34
+ holdRealtime: holdRealtime
33
35
  };
34
36
  }
35
37
 
38
+ function releaseRealtime(crossChainState) {
39
+ crossChainState.holdRealtime = false;
40
+ }
41
+
42
+ function isHoldingRealtime(crossChainState) {
43
+ return crossChainState.holdRealtime;
44
+ }
45
+
46
+ function hasArrivedAtHead(crossChainState) {
47
+ if (crossChainState.isCaughtUp || crossChainState.isRealtime) {
48
+ return true;
49
+ }
50
+ let chainStates = Object.values(crossChainState.chainStates);
51
+ if (Utils.$$Array.notEmpty(chainStates)) {
52
+ return chainStates.every(ChainState.isReadyToEnterReorgThreshold);
53
+ } else {
54
+ return false;
55
+ }
56
+ }
57
+
36
58
  function chainStates(crossChainState) {
37
59
  return crossChainState.chainStates;
38
60
  }
@@ -91,7 +113,15 @@ function getSafeCheckpointIdByChain(crossChainState, sequence, committedFrontier
91
113
  }
92
114
 
93
115
  function createBatch(crossChainState, config, frontier, batchSizeTarget) {
94
- return Batch.make(config.checkpointSequence, frontier, Utils.Dict.mapValues(crossChainState.chainStates, ChainState.toChainBeforeBatch), batchSizeTarget, HistoryPolicy.decide(config, shouldSaveHistory(crossChainState)));
116
+ return Batch.make(config.checkpointSequence, frontier, Utils.Dict.mapValues(crossChainState.chainStates, none => ChainState.toChainBeforeBatch(none, crossChainState.isRealtime)), batchSizeTarget, HistoryPolicy.decide(config, shouldSaveHistory(crossChainState)));
117
+ }
118
+
119
+ function isReadyToEnterReorgThreshold(crossChainState, batch) {
120
+ if (crossChainState.holdRealtime) {
121
+ return false;
122
+ } else {
123
+ return Object.values(crossChainState.chainStates).every(cs => ChainState.isReadyToEnterReorgThresholdAfterBatch(cs, batch));
124
+ }
95
125
  }
96
126
 
97
127
  function enterReorgThreshold(crossChainState) {
@@ -111,7 +141,7 @@ function applyBatchProgress(crossChainState, batch, blockTimestampName) {
111
141
  everyChainCaughtUp = false;
112
142
  }
113
143
  }
114
- crossChainState.isCaughtUp = crossChainState.isCaughtUp || nextItemIsNone(crossChainState) && everyChainCaughtUp;
144
+ crossChainState.isCaughtUp = crossChainState.isCaughtUp || !crossChainState.holdRealtime && nextItemIsNone(crossChainState) && everyChainCaughtUp;
115
145
  }
116
146
 
117
147
  function isSettledAtHead(crossChainState) {
@@ -125,7 +155,7 @@ function isSettledAtHead(crossChainState) {
125
155
  }
126
156
 
127
157
  function markCaughtUpIfSettled(crossChainState) {
128
- if (isSettledAtHead(crossChainState)) {
158
+ if (!crossChainState.holdRealtime && isSettledAtHead(crossChainState)) {
129
159
  crossChainState.isCaughtUp = true;
130
160
  return;
131
161
  }
@@ -256,6 +286,10 @@ export {
256
286
  shouldSaveHistory,
257
287
  nextItemIsNone,
258
288
  getSafeCheckpointIdByChain,
289
+ releaseRealtime,
290
+ isHoldingRealtime,
291
+ hasArrivedAtHead,
292
+ isReadyToEnterReorgThreshold,
259
293
  createBatch,
260
294
  enterReorgThreshold,
261
295
  applyBatchProgress,
@@ -5,7 +5,12 @@ type t
5
5
 
6
6
  let calculateTargetBufferSize: unit => int
7
7
 
8
- let make: (~chainStates: dict<ChainState.t>, ~isRealtime: bool, ~targetBufferSize: int=?) => t
8
+ let make: (
9
+ ~chainStates: dict<ChainState.t>,
10
+ ~isRealtime: bool,
11
+ ~targetBufferSize: int=?,
12
+ ~holdRealtime: bool=?,
13
+ ) => t
9
14
 
10
15
  // Accessors.
11
16
  let chainStates: t => dict<ChainState.t>
@@ -25,6 +30,10 @@ let getSafeCheckpointIdByChain: (
25
30
  ) => array<(ChainId.t, option<Internal.checkpointId>)>
26
31
 
27
32
  // Cross-chain transitions.
33
+ let releaseRealtime: t => unit
34
+ let isHoldingRealtime: t => bool
35
+ let hasArrivedAtHead: t => bool
36
+ let isReadyToEnterReorgThreshold: (t, ~batch: Batch.t) => bool
28
37
  let createBatch: (t, ~config: Config.t, ~frontier: Frontier.t, ~batchSizeTarget: int) => Batch.t
29
38
  let enterReorgThreshold: t => unit
30
39
  let applyBatchProgress: (t, ~batch: Batch.t, ~blockTimestampName: string) => unit
package/src/Env.res CHANGED
@@ -127,6 +127,10 @@ module Db = {
127
127
  //the SSL modes should be provided as string otherwise as 'require' | 'allow' | 'prefer' | 'verify-full'
128
128
  ~devFallback=Bool(false),
129
129
  )
130
+ // The budget for the whole run, not for one process: a run that splits across
131
+ // workers divides it among them, and each caps its own pool to its share.
132
+ // The default buys a single worker, so a run splits only once the operator
133
+ // raises the budget it may spend.
130
134
  let maxConnections = envSafe->EnvSafe.get("ENVIO_PG_MAX_CONNECTIONS", S.int, ~fallback=2)
131
135
  }
132
136
 
@@ -50,6 +50,22 @@ let makeSelection = (~onEventRegistrations, ~dependsOnAddresses, ~clientFiltered
50
50
  startBlock: ?deriveSelectionStartBlock(onEventRegistrations),
51
51
  }
52
52
 
53
+ // A partition's frontier never sits below the block before its selection can
54
+ // first match: the blocks below hold nothing for it, so they count as fetched
55
+ // the same way the blocks before an address's start block do. The chain's
56
+ // buffer frontier is its lowest partition frontier, so a partition left at the
57
+ // chain start would hold every other partition's events back as unprocessable
58
+ // and keep the chain's query target — sized from that frontier — short of
59
+ // anything worth asking for. A start block the chain has not reached yet leaves
60
+ // the partition waiting there, like an address partition does, rather than
61
+ // scanning empty ranges up to it; bufferBlockNumber caps the chain's frontier at
62
+ // the head meanwhile.
63
+ let floorAtSelectionStart = (latestFetchedBlock, ~selection) =>
64
+ switch selection.startBlock {
65
+ | Some(startBlock) => Pervasives.max(latestFetchedBlock, startBlock - 1)
66
+ | None => latestFetchedBlock
67
+ }
68
+
53
69
  type pendingQuery = {
54
70
  fromBlock: int,
55
71
  toBlock: option<int>,
@@ -398,6 +414,10 @@ module OptimizedPartitions = {
398
414
  ~dynamicContracts: Utils.Set.t<string>,
399
415
  ~clientFilteredContracts: Utils.Set.t<string>,
400
416
  ) => {
417
+ let partitions = partitions->Array.map(p => {
418
+ let floored = p.latestFetchedBlock->floorAtSelectionStart(~selection=p.selection)
419
+ floored === p.latestFetchedBlock ? p : {...p, latestFetchedBlock: floored}
420
+ })
401
421
  let newPartitions = []
402
422
  let mergingPartitions = Dict.make()
403
423
  let nextPartitionIndexRef = ref(nextPartitionIndex)
@@ -847,28 +867,17 @@ type t = {
847
867
  clientFilterAddressThreshold: option<int>,
848
868
  }
849
869
 
870
+ // The latest block whose items are all in the buffer: the lowest partition
871
+ // frontier, held back by the onBlock pointer. Without onBlock registrations the
872
+ // pointer tracks the known height, so this also caps the frontier at the head: a
873
+ // partition can sit past it, on a response that reported a block the chain
874
+ // hadn't heard of yet or on a start block the chain hasn't reached.
850
875
  @inline
851
876
  let bufferBlockNumber = ({latestOnBlockBlockNumber, optimizedPartitions}: t) => {
852
877
  switch optimizedPartitions->OptimizedPartitions.getLatestFullyFetchedBlock {
853
878
  | None => latestOnBlockBlockNumber
854
879
  | Some(latestFullyFetchedBlock) =>
855
- latestOnBlockBlockNumber < latestFullyFetchedBlock
856
- ? latestOnBlockBlockNumber
857
- : latestFullyFetchedBlock
858
- }
859
- }
860
-
861
- /**
862
- * Returns the latest block which is ready to be consumed
863
- */
864
- @inline
865
- let bufferBlock = ({optimizedPartitions, latestOnBlockBlockNumber}: t) => {
866
- switch optimizedPartitions->OptimizedPartitions.getLatestFullyFetchedBlock {
867
- | None => latestOnBlockBlockNumber
868
- | Some(latestFullyFetchedBlock) =>
869
- latestOnBlockBlockNumber < latestFullyFetchedBlock
870
- ? latestOnBlockBlockNumber
871
- : latestFullyFetchedBlock
880
+ Pervasives.min(latestOnBlockBlockNumber, latestFullyFetchedBlock)
872
881
  }
873
882
  }
874
883
 
@@ -1138,14 +1147,20 @@ let updateInternal = (
1138
1147
  // Use maxOnBlockBufferSize to get the last target item in the buffer
1139
1148
  // (sorted, so this is the highest-block item within the buffer cap).
1140
1149
  // All this needed to prevent OOM when adding too many block items to the queue
1141
- let maxBlockNumber = switch base->Array.get(fetchState.maxOnBlockBufferSize - 1) {
1142
- | Some(item) => item->Internal.getItemBlockNumber
1143
- | None =>
1144
- switch optimizedPartitions->OptimizedPartitions.getLatestFullyFetchedBlock {
1145
- | None => knownHeight
1146
- | Some(latestFullyFetchedBlock) => latestFullyFetchedBlock
1147
- }
1148
- }
1150
+ // Never past the head: a partition can sit there, on a start block the
1151
+ // chain hasn't reached, and a response is applied before the height it
1152
+ // reported, so its events can run past the height still known here.
1153
+ let maxBlockNumber = Pervasives.min(
1154
+ switch base->Array.get(fetchState.maxOnBlockBufferSize - 1) {
1155
+ | Some(item) => item->Internal.getItemBlockNumber
1156
+ | None =>
1157
+ switch optimizedPartitions->OptimizedPartitions.getLatestFullyFetchedBlock {
1158
+ | None => knownHeight
1159
+ | Some(latestFullyFetchedBlock) => latestFullyFetchedBlock
1160
+ }
1161
+ },
1162
+ knownHeight,
1163
+ )
1149
1164
  appendOnBlockItems(
1150
1165
  ~mutItems=blockItems,
1151
1166
  ~onBlockRegistrations,
@@ -2165,27 +2180,11 @@ let walkPartitionPending = (
2165
2180
  pqIdx := pqIdx.contents + 1
2166
2181
  }
2167
2182
 
2168
- // Nothing in this partition's selection can match below its earliest start
2169
- // block, so forward work skips straight to it instead of scanning up to it and
2170
- // discarding whole pages. Only the cursor moves — `latestFetchedBlock` still
2171
- // advances solely on a response, so no block is ever reported fetched that
2172
- // wasn't. Bounded by the head and the query end block: past either, the
2173
- // partition would offer no candidate at all, so it queries as before rather
2174
- // than going quiet until the chain reaches its start block.
2175
- let cursor = switch p.selection.startBlock {
2176
- | Some(startBlock) if startBlock > cursor.contents =>
2177
- switch Utils.Math.minOptInt(Some(headBlockNumber), queryEndBlock) {
2178
- | Some(reachable) if startBlock <= reachable => startBlock
2179
- | _ => cursor.contents
2180
- }
2181
- | _ => cursor.contents
2182
- }
2183
-
2184
2183
  canContinue.contents
2185
2184
  ? Some({
2186
2185
  partitionId,
2187
2186
  p,
2188
- cursor,
2187
+ cursor: cursor.contents,
2189
2188
  chunksUsedThisCall: chunksUsedThisCall.contents,
2190
2189
  inFlightCount,
2191
2190
  queryEndBlock,
@@ -2780,10 +2779,17 @@ let make = (
2780
2779
  // fetching, so without seeding the buffer here getNextQuery would return
2781
2780
  // NothingToQuery and the indexer would get stuck.
2782
2781
  let buffer = []
2783
- let latestOnBlockBlockNumber = if knownHeight > 0 && onBlockRegistrations->Utils.Array.notEmpty {
2782
+ let latestOnBlockBlockNumber = switch onBlockRegistrations {
2783
+ // As updateInternal keeps it: with nothing to generate per block the pointer
2784
+ // is the head cap on the buffer frontier, and left at the progress block it
2785
+ // would hold the frontier there until the first height update — below a
2786
+ // partition that starts later, whose queries the chain sizes off that
2787
+ // frontier and so never reaches.
2788
+ | [] if knownHeight > 0 => knownHeight
2789
+ | onBlockRegistrations if knownHeight > 0 =>
2784
2790
  let maxBlockNumber = switch optimizedPartitions->OptimizedPartitions.getLatestFullyFetchedBlock {
2785
2791
  | None => knownHeight
2786
- | Some(latestFullyFetchedBlock) => latestFullyFetchedBlock
2792
+ | Some(latestFullyFetchedBlock) => Pervasives.min(latestFullyFetchedBlock, knownHeight)
2787
2793
  }
2788
2794
  appendOnBlockItems(
2789
2795
  ~mutItems=buffer,
@@ -2793,8 +2799,7 @@ let make = (
2793
2799
  ~maxBlockNumber,
2794
2800
  ~maxOnBlockBufferSize,
2795
2801
  )
2796
- } else {
2797
- progressBlockNumber
2802
+ | _ => progressBlockNumber
2798
2803
  }
2799
2804
 
2800
2805
  let fetchState = {
@@ -47,6 +47,15 @@ function makeSelection(onEventRegistrations, dependsOnAddresses, clientFilteredC
47
47
  };
48
48
  }
49
49
 
50
+ function floorAtSelectionStart(latestFetchedBlock, selection) {
51
+ let startBlock = selection.startBlock;
52
+ if (startBlock !== undefined) {
53
+ return Primitive_int.max(latestFetchedBlock, startBlock - 1 | 0);
54
+ } else {
55
+ return latestFetchedBlock;
56
+ }
57
+ }
58
+
50
59
  function withAddresses(p, addresses) {
51
60
  return {
52
61
  id: p.id,
@@ -305,6 +314,26 @@ function getAnchorSafeBlock(p) {
305
314
  }
306
315
 
307
316
  function make(partitions, maxAddrInPartition, nextPartitionIndex, dynamicContracts, clientFilteredContracts) {
317
+ let partitions$1 = partitions.map(p => {
318
+ let floored = floorAtSelectionStart(p.latestFetchedBlock, p.selection);
319
+ if (floored === p.latestFetchedBlock) {
320
+ return p;
321
+ } else {
322
+ return {
323
+ id: p.id,
324
+ latestFetchedBlock: floored,
325
+ selection: p.selection,
326
+ addresses: p.addresses,
327
+ mergeBlock: p.mergeBlock,
328
+ dynamicContract: p.dynamicContract,
329
+ mutPendingQueries: p.mutPendingQueries,
330
+ sourceRangeCapacity: p.sourceRangeCapacity,
331
+ prevSourceRangeCapacity: p.prevSourceRangeCapacity,
332
+ eventDensity: p.eventDensity,
333
+ latestSourceRangeCapacityUpdateBlock: p.latestSourceRangeCapacityUpdateBlock
334
+ };
335
+ }
336
+ });
308
337
  let newPartitions = [];
309
338
  let mergingPartitions = {};
310
339
  let nextPartitionIndexRef = {
@@ -313,7 +342,7 @@ function make(partitions, maxAddrInPartition, nextPartitionIndex, dynamicContrac
313
342
  let anchorSafeBlocks = {};
314
343
  let anchoredContractsSet = new Set();
315
344
  if (clientFilteredContracts.size > 0) {
316
- partitions.forEach(p => {
345
+ partitions$1.forEach(p => {
317
346
  let match = p.mergeBlock;
318
347
  let match$1 = p.selection.clientFilteredContracts;
319
348
  if (match !== undefined) {
@@ -332,8 +361,8 @@ function make(partitions, maxAddrInPartition, nextPartitionIndex, dynamicContrac
332
361
  });
333
362
  });
334
363
  }
335
- for (let idx = 0, idx_finish = partitions.length; idx < idx_finish; ++idx) {
336
- let p = partitions[idx];
364
+ for (let idx = 0, idx_finish = partitions$1.length; idx < idx_finish; ++idx) {
365
+ let p = partitions$1[idx];
337
366
  let mergeBlock = p.mergeBlock;
338
367
  if (mergeBlock !== undefined) {
339
368
  if (p.latestFetchedBlock < mergeBlock || isFetching(p)) {
@@ -676,20 +705,8 @@ function bufferBlockNumber(param) {
676
705
  let optimizedPartitions = param.optimizedPartitions;
677
706
  let id = optimizedPartitions.idsInAscOrder[0];
678
707
  let latestFullyFetchedBlock = id !== undefined ? optimizedPartitions.entities[id].latestFetchedBlock : undefined;
679
- if (latestFullyFetchedBlock !== undefined && latestOnBlockBlockNumber >= latestFullyFetchedBlock) {
680
- return latestFullyFetchedBlock;
681
- } else {
682
- return latestOnBlockBlockNumber;
683
- }
684
- }
685
-
686
- function bufferBlock(param) {
687
- let latestOnBlockBlockNumber = param.latestOnBlockBlockNumber;
688
- let optimizedPartitions = param.optimizedPartitions;
689
- let id = optimizedPartitions.idsInAscOrder[0];
690
- let latestFullyFetchedBlock = id !== undefined ? optimizedPartitions.entities[id].latestFetchedBlock : undefined;
691
- if (latestFullyFetchedBlock !== undefined && latestOnBlockBlockNumber >= latestFullyFetchedBlock) {
692
- return latestFullyFetchedBlock;
708
+ if (latestFullyFetchedBlock !== undefined) {
709
+ return Primitive_int.min(latestOnBlockBlockNumber, latestFullyFetchedBlock);
693
710
  } else {
694
711
  return latestOnBlockBlockNumber;
695
712
  }
@@ -897,14 +914,15 @@ function updateInternal(fetchState, optimizedPartitionsOpt, mutItems, mutItemsSo
897
914
  let latestOnBlockBlockNumber;
898
915
  if (onBlockRegistrations.length !== 0) {
899
916
  let item = base[fetchState.maxOnBlockBufferSize - 1 | 0];
900
- let maxBlockNumber;
917
+ let tmp;
901
918
  if (item !== undefined) {
902
- maxBlockNumber = item.blockNumber;
919
+ tmp = item.blockNumber;
903
920
  } else {
904
921
  let id = optimizedPartitions.idsInAscOrder[0];
905
922
  let latestFullyFetchedBlock = id !== undefined ? optimizedPartitions.entities[id].latestFetchedBlock : undefined;
906
- maxBlockNumber = latestFullyFetchedBlock !== undefined ? latestFullyFetchedBlock : knownHeight;
923
+ tmp = latestFullyFetchedBlock !== undefined ? latestFullyFetchedBlock : knownHeight;
907
924
  }
925
+ let maxBlockNumber = Primitive_int.min(tmp, knownHeight);
908
926
  latestOnBlockBlockNumber = appendOnBlockItems(blockItems, onBlockRegistrations, fetchState.startBlock, fetchState.latestOnBlockBlockNumber, maxBlockNumber, fetchState.maxOnBlockBufferSize);
909
927
  } else {
910
928
  latestOnBlockBlockNumber = knownHeight;
@@ -1486,19 +1504,11 @@ function walkPartitionPending(p, partitionId, inFlightCount, candidates, headBlo
1486
1504
  }
1487
1505
  pqIdx = pqIdx + 1 | 0;
1488
1506
  };
1489
- let startBlock = p.selection.startBlock;
1490
- let cursor$1;
1491
- if (startBlock !== undefined && startBlock > cursor) {
1492
- let reachable = Utils.$$Math.minOptInt(headBlockNumber, queryEndBlock);
1493
- cursor$1 = reachable !== undefined && startBlock <= reachable ? startBlock : cursor;
1494
- } else {
1495
- cursor$1 = cursor;
1496
- }
1497
1507
  if (canContinue) {
1498
1508
  return {
1499
1509
  partitionId: partitionId,
1500
1510
  p: p,
1501
- cursor: cursor$1,
1511
+ cursor: cursor,
1502
1512
  chunksUsedThisCall: chunksUsedThisCall,
1503
1513
  inFlightCount: inFlightCount,
1504
1514
  queryEndBlock: queryEndBlock,
@@ -1800,13 +1810,21 @@ function make$1(startBlock, endBlock, onEventRegistrations, addressStore, addres
1800
1810
  }
1801
1811
  let buffer = [];
1802
1812
  let latestOnBlockBlockNumber;
1803
- if (knownHeight > 0 && Utils.$$Array.notEmpty(onBlockRegistrations)) {
1804
- let id = optimizedPartitions.idsInAscOrder[0];
1805
- let latestFullyFetchedBlock = id !== undefined ? optimizedPartitions.entities[id].latestFetchedBlock : undefined;
1806
- let maxBlockNumber = latestFullyFetchedBlock !== undefined ? latestFullyFetchedBlock : knownHeight;
1807
- latestOnBlockBlockNumber = appendOnBlockItems(buffer, onBlockRegistrations, startBlock, progressBlockNumber, maxBlockNumber, maxOnBlockBufferSize);
1813
+ let exit = 0;
1814
+ if (onBlockRegistrations.length !== 0 || knownHeight <= 0) {
1815
+ exit = 1;
1808
1816
  } else {
1809
- latestOnBlockBlockNumber = progressBlockNumber;
1817
+ latestOnBlockBlockNumber = knownHeight;
1818
+ }
1819
+ if (exit === 1) {
1820
+ if (knownHeight > 0) {
1821
+ let id = optimizedPartitions.idsInAscOrder[0];
1822
+ let latestFullyFetchedBlock = id !== undefined ? optimizedPartitions.entities[id].latestFetchedBlock : undefined;
1823
+ let maxBlockNumber = latestFullyFetchedBlock !== undefined ? Primitive_int.min(latestFullyFetchedBlock, knownHeight) : knownHeight;
1824
+ latestOnBlockBlockNumber = appendOnBlockItems(buffer, onBlockRegistrations, startBlock, progressBlockNumber, maxBlockNumber, maxOnBlockBufferSize);
1825
+ } else {
1826
+ latestOnBlockBlockNumber = progressBlockNumber;
1827
+ }
1810
1828
  }
1811
1829
  return {
1812
1830
  optimizedPartitions: optimizedPartitions,
@@ -2111,6 +2129,7 @@ let chunkRangeGrowthFactor = 1.8;
2111
2129
  export {
2112
2130
  deriveSelectionStartBlock,
2113
2131
  makeSelection,
2132
+ floorAtSelectionStart,
2114
2133
  withAddresses,
2115
2134
  narrowSelectionToRange,
2116
2135
  densityItemsTarget,
@@ -2120,7 +2139,6 @@ export {
2120
2139
  getMinQueryRange,
2121
2140
  OptimizedPartitions,
2122
2141
  bufferBlockNumber,
2123
- bufferBlock,
2124
2142
  bufferReadyCount,
2125
2143
  getRegistrationIndex,
2126
2144
  comparePath,