envio 3.3.0-alpha.8 → 3.3.0-alpha.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +6 -6
- package/src/ChainState.res +104 -37
- package/src/ChainState.res.mjs +80 -19
- package/src/ChainState.resi +4 -1
- package/src/CrossChainState.res +56 -29
- package/src/CrossChainState.res.mjs +11 -2
- package/src/EnvioGlobal.res +53 -0
- package/src/EnvioGlobal.res.mjs +31 -0
- package/src/EventConfigBuilder.res +16 -17
- package/src/EventConfigBuilder.res.mjs +7 -6
- package/src/FetchState.res +151 -130
- package/src/FetchState.res.mjs +105 -88
- package/src/HandlerLoader.res +1 -1
- package/src/HandlerLoader.res.mjs +1 -1
- package/src/HandlerRegister.res +476 -305
- package/src/HandlerRegister.res.mjs +276 -209
- package/src/HandlerRegister.resi +11 -6
- package/src/Internal.res +23 -4
- package/src/LogSelection.res +102 -165
- package/src/LogSelection.res.mjs +101 -116
- package/src/Main.res +29 -143
- package/src/Main.res.mjs +25 -88
- package/src/RollbackCommit.res +4 -1
- package/src/RollbackCommit.res.mjs +3 -2
- package/src/SimulateItems.res +1 -1
- package/src/sources/Evm.res +4 -1
- package/src/sources/Evm.res.mjs +1 -1
- package/src/sources/Fuel.res +3 -1
- package/src/sources/Fuel.res.mjs +1 -1
- package/src/sources/HyperSyncSource.res +22 -33
- package/src/sources/HyperSyncSource.res.mjs +17 -26
- package/src/sources/RpcSource.res +28 -39
- package/src/sources/RpcSource.res.mjs +18 -31
- package/src/sources/SourceManager.res +1 -7
- package/src/sources/SourceManager.res.mjs +1 -2
|
@@ -243,9 +243,13 @@ let getTopicEncoder = (abiType: string): (unknown => EvmTypes.Hex.t) => {
|
|
|
243
243
|
} else {
|
|
244
244
|
switch abiType {
|
|
245
245
|
| "address" =>
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
246
|
+
// Lowercase before encoding so mixed-case (checksummed) user input
|
|
247
|
+
// still matches the lowercase hex topics returned by sources.
|
|
248
|
+
|
|
249
|
+
(
|
|
250
|
+
(value: string) =>
|
|
251
|
+
value->String.toLowerCase->Address.unsafeFromString->TopicFilter.fromAddress
|
|
252
|
+
)->(Utils.magic: (string => EvmTypes.Hex.t) => unknown => EvmTypes.Hex.t)
|
|
249
253
|
|
|
250
254
|
| "bool" =>
|
|
251
255
|
TopicFilter.fromBool->(Utils.magic: (bool => EvmTypes.Hex.t) => unknown => EvmTypes.Hex.t)
|
|
@@ -413,31 +417,26 @@ let buildEvmEventConfig = (
|
|
|
413
417
|
}
|
|
414
418
|
|
|
415
419
|
// Enrich an EVM definition into a per-(event,chain) registration: resolve the
|
|
416
|
-
// registered `where`
|
|
417
|
-
//
|
|
420
|
+
// registered `where` for this chain into `resolvedWhere` + address filters,
|
|
421
|
+
// and override `startBlock` with `where.block._gte`.
|
|
418
422
|
let buildEvmOnEventRegistration = (
|
|
419
423
|
~eventConfig: Internal.evmEventConfig,
|
|
420
424
|
~isWildcard: bool,
|
|
421
425
|
~handler: option<Internal.handler>,
|
|
422
426
|
~contractRegister: option<Internal.contractRegister>,
|
|
423
|
-
~
|
|
424
|
-
~
|
|
427
|
+
~where: option<JSON.t>,
|
|
428
|
+
~chainId: int,
|
|
425
429
|
~onEventBlockFilterSchema: S.t<option<unknown>>,
|
|
426
430
|
~startBlock: option<int>=?,
|
|
427
431
|
): Internal.evmOnEventRegistration => {
|
|
428
432
|
let indexedParams = eventConfig.paramsMetadata->Array.filter(p => p.indexed)
|
|
429
433
|
|
|
430
|
-
let {
|
|
431
|
-
|
|
432
|
-
filterByAddresses,
|
|
433
|
-
addressFilterParamGroups,
|
|
434
|
-
startBlock: whereStartBlock,
|
|
435
|
-
} = LogSelection.parseEventFiltersOrThrow(
|
|
436
|
-
~eventFilters,
|
|
434
|
+
let {resolvedWhere, filterByAddresses, addressFilterParamGroups} = LogSelection.parseWhereOrThrow(
|
|
435
|
+
~where,
|
|
437
436
|
~sighash=eventConfig.sighash,
|
|
438
437
|
~params=indexedParams->Array.map(p => p.name),
|
|
439
438
|
~contractName=eventConfig.contractName,
|
|
440
|
-
~
|
|
439
|
+
~chainId,
|
|
441
440
|
~onEventBlockFilterSchema,
|
|
442
441
|
~topic1=?indexedParams->Array.get(0)->Option.map(buildTopicGetter),
|
|
443
442
|
~topic2=?indexedParams->Array.get(1)->Option.map(buildTopicGetter),
|
|
@@ -447,7 +446,7 @@ let buildEvmOnEventRegistration = (
|
|
|
447
446
|
// `where.block.number._gte` overrides the contract-level startBlock when
|
|
448
447
|
// present (an explicit per-event opt-in that wins over `config.yaml`);
|
|
449
448
|
// otherwise the contract/chain value passes through.
|
|
450
|
-
let resolvedStartBlock = switch
|
|
449
|
+
let resolvedStartBlock = switch resolvedWhere.startBlock {
|
|
451
450
|
| Some(_) as sb => sb
|
|
452
451
|
| None => startBlock
|
|
453
452
|
}
|
|
@@ -457,7 +456,7 @@ let buildEvmOnEventRegistration = (
|
|
|
457
456
|
isWildcard,
|
|
458
457
|
handler,
|
|
459
458
|
contractRegister,
|
|
460
|
-
|
|
459
|
+
resolvedWhere,
|
|
461
460
|
filterByAddresses,
|
|
462
461
|
clientAddressFilter: ?buildAddressFilter(addressFilterParamGroups, ~isWildcard),
|
|
463
462
|
dependsOnAddresses: Internal.dependsOnAddresses(~isWildcard, ~filterByAddresses),
|
|
@@ -213,7 +213,7 @@ function getTopicEncoder(abiType) {
|
|
|
213
213
|
}
|
|
214
214
|
switch (abiType) {
|
|
215
215
|
case "address" :
|
|
216
|
-
return TopicFilter.fromAddress;
|
|
216
|
+
return value => TopicFilter.fromAddress(value.toLowerCase());
|
|
217
217
|
case "bool" :
|
|
218
218
|
return TopicFilter.fromBool;
|
|
219
219
|
case "bytes" :
|
|
@@ -309,12 +309,13 @@ function buildEvmEventConfig(contractName, eventName, sighash, params, blockFiel
|
|
|
309
309
|
};
|
|
310
310
|
}
|
|
311
311
|
|
|
312
|
-
function buildEvmOnEventRegistration(eventConfig, isWildcard, handler, contractRegister,
|
|
312
|
+
function buildEvmOnEventRegistration(eventConfig, isWildcard, handler, contractRegister, where, chainId, onEventBlockFilterSchema, startBlock) {
|
|
313
313
|
let indexedParams = eventConfig.paramsMetadata.filter(p => p.indexed);
|
|
314
|
-
let match = LogSelection.
|
|
315
|
-
let whereStartBlock = match.startBlock;
|
|
314
|
+
let match = LogSelection.parseWhereOrThrow(where, eventConfig.sighash, indexedParams.map(p => p.name), eventConfig.contractName, chainId, onEventBlockFilterSchema, Stdlib_Option.map(indexedParams[0], buildTopicGetter), Stdlib_Option.map(indexedParams[1], buildTopicGetter), Stdlib_Option.map(indexedParams[2], buildTopicGetter));
|
|
316
315
|
let filterByAddresses = match.filterByAddresses;
|
|
317
|
-
let
|
|
316
|
+
let resolvedWhere = match.resolvedWhere;
|
|
317
|
+
let sb = resolvedWhere.startBlock;
|
|
318
|
+
let resolvedStartBlock = sb !== undefined ? sb : startBlock;
|
|
318
319
|
return {
|
|
319
320
|
eventConfig: eventConfig,
|
|
320
321
|
handler: handler,
|
|
@@ -324,7 +325,7 @@ function buildEvmOnEventRegistration(eventConfig, isWildcard, handler, contractR
|
|
|
324
325
|
dependsOnAddresses: Internal.dependsOnAddresses(isWildcard, filterByAddresses),
|
|
325
326
|
clientAddressFilter: buildAddressFilter(match.addressFilterParamGroups, isWildcard, undefined),
|
|
326
327
|
startBlock: resolvedStartBlock,
|
|
327
|
-
|
|
328
|
+
resolvedWhere: resolvedWhere
|
|
328
329
|
};
|
|
329
330
|
}
|
|
330
331
|
|
package/src/FetchState.res
CHANGED
|
@@ -19,7 +19,7 @@ type pendingQuery = {
|
|
|
19
19
|
// Items this in-flight query is targeting (server maxNumLogs-style cap), carried
|
|
20
20
|
// from the query so the shared buffer budget can account for what's already
|
|
21
21
|
// being fetched.
|
|
22
|
-
itemsTarget:
|
|
22
|
+
itemsTarget: int,
|
|
23
23
|
// Stores latestFetchedBlock when query completes. Only needed to persist
|
|
24
24
|
// timestamp while earlier queries are still pending before updating
|
|
25
25
|
// the partition's latestFetchedBlock.
|
|
@@ -68,7 +68,7 @@ type query = {
|
|
|
68
68
|
// with known density this is density × the query's block range; for a
|
|
69
69
|
// partition with no signal yet it's whatever budget share the query was
|
|
70
70
|
// sized against.
|
|
71
|
-
itemsTarget:
|
|
71
|
+
itemsTarget: int,
|
|
72
72
|
selection: selection,
|
|
73
73
|
addressesByContractName: dict<array<Address.t>>,
|
|
74
74
|
}
|
|
@@ -91,27 +91,19 @@ let deriveContractNameByAddress: dict<array<Address.t>> => dict<
|
|
|
91
91
|
result
|
|
92
92
|
})
|
|
93
93
|
|
|
94
|
-
//
|
|
95
|
-
//
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
//
|
|
100
|
-
//
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
// owning chain wants to reach (see getNextQuery). A partition that responded
|
|
108
|
-
// with no items has density 0, so its queries cost 0 — correct, they don't
|
|
109
|
-
// fill the buffer. Only called for partitions with a known density
|
|
110
|
-
// (prevQueryRange > 0); partitions with no signal yet are sized instead
|
|
111
|
-
// against their even split of the chain's range budget.
|
|
112
|
-
let densityItemsTarget = (p: partition, ~fromBlock, ~toBlock, ~chainTargetBlock) => {
|
|
113
|
-
let density = p.prevRangeSize->Int.toFloat /. p.prevQueryRange->Int.toFloat
|
|
114
|
-
(toBlock->Option.getOr(chainTargetBlock) - fromBlock + 1)->Int.toFloat *. density
|
|
94
|
+
// itemsTarget for a query over [fromBlock, toBlock] at the given event density
|
|
95
|
+
// (items/block). toBlock None is the open-ended tail, capped at
|
|
96
|
+
// chainTargetBlock — the soft per-tick horizon the owning chain wants to reach
|
|
97
|
+
// (see getNextQuery).
|
|
98
|
+
let densityItemsTarget = (~density, ~fromBlock, ~toBlock, ~chainTargetBlock) => {
|
|
99
|
+
// Floor at 1: the reservation must equal the server-side cap SourceManager
|
|
100
|
+
// sends, and a 0 cap would ask the backend for nothing.
|
|
101
|
+
Pervasives.max(
|
|
102
|
+
1,
|
|
103
|
+
((toBlock->Option.getOr(chainTargetBlock) - fromBlock + 1)->Int.toFloat *. density)
|
|
104
|
+
->Math.ceil
|
|
105
|
+
->Float.toInt,
|
|
106
|
+
)
|
|
115
107
|
}
|
|
116
108
|
|
|
117
109
|
// Calculate the chunk range from history using min-of-last-3-ranges heuristic
|
|
@@ -122,6 +114,15 @@ let getMinHistoryRange = (p: partition) => {
|
|
|
122
114
|
}
|
|
123
115
|
}
|
|
124
116
|
|
|
117
|
+
// Density (items/block) from the last response, trusted only after two
|
|
118
|
+
// responses — a single sample is too noisy to size the next query by.
|
|
119
|
+
let getTrustedDensity = (p: partition) => {
|
|
120
|
+
switch (p.prevQueryRange, p.prevPrevQueryRange) {
|
|
121
|
+
| (0, _) | (_, 0) => None
|
|
122
|
+
| (prevQueryRange, _) => Some(p.prevRangeSize->Int.toFloat /. prevQueryRange->Int.toFloat)
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
|
|
125
126
|
let getMinQueryRange = (partitions: array<partition>) => {
|
|
126
127
|
let min = ref(0)
|
|
127
128
|
for i in 0 to partitions->Array.length - 1 {
|
|
@@ -201,6 +202,12 @@ module OptimizedPartitions = {
|
|
|
201
202
|
let newId = nextPartitionIndexRef.contents->Int.toString
|
|
202
203
|
nextPartitionIndexRef := nextPartitionIndexRef.contents + 1
|
|
203
204
|
let minRange = getMinQueryRange([p1, p2])
|
|
205
|
+
// The merged partition indexes both parents' addresses, so its expected
|
|
206
|
+
// event rate is the sum of their densities. Parents without a trusted
|
|
207
|
+
// density contribute 0; if none has one, prevRangeSize stays 0 and the
|
|
208
|
+
// partition probes for a fresh signal instead of chunking.
|
|
209
|
+
let inheritedDensity =
|
|
210
|
+
p1->getTrustedDensity->Option.getOr(0.) +. p2->getTrustedDensity->Option.getOr(0.)
|
|
204
211
|
{
|
|
205
212
|
id: newId,
|
|
206
213
|
dynamicContract: Some(contractName),
|
|
@@ -211,7 +218,7 @@ module OptimizedPartitions = {
|
|
|
211
218
|
mutPendingQueries: [],
|
|
212
219
|
prevQueryRange: minRange,
|
|
213
220
|
prevPrevQueryRange: minRange,
|
|
214
|
-
prevRangeSize:
|
|
221
|
+
prevRangeSize: (inheritedDensity *. minRange->Int.toFloat)->Math.ceil->Float.toInt,
|
|
215
222
|
latestBlockRangeUpdateBlock: 0,
|
|
216
223
|
}
|
|
217
224
|
}
|
|
@@ -1352,9 +1359,13 @@ let startFetchingQueries = ({optimizedPartitions}: t, ~queries: array<query>) =>
|
|
|
1352
1359
|
let maxPendingChunksPerPartition = 10
|
|
1353
1360
|
|
|
1354
1361
|
// Fills a gap range (a hole left between completed/pending chunks, e.g. from an
|
|
1355
|
-
// out-of-order partial response) unconditionally
|
|
1356
|
-
//
|
|
1357
|
-
//
|
|
1362
|
+
// out-of-order partial response) unconditionally — gaps are already-committed
|
|
1363
|
+
// range, not subject to this tick's water-fill budget. Priced by the
|
|
1364
|
+
// partition's trusted density when it has one; otherwise by the "available
|
|
1365
|
+
// density" — the partition's equal-divide budget spread over its remaining
|
|
1366
|
+
// range this tick — so a small gap reserves proportionally little instead of a
|
|
1367
|
+
// noisy one-sample estimate. Chunks only on a trusted POSITIVE density, same
|
|
1368
|
+
// rule as the water-fill. Returns the created queries' total itemsTarget.
|
|
1358
1369
|
let pushGapFillQueries = (
|
|
1359
1370
|
queries: array<query>,
|
|
1360
1371
|
~partitionId: string,
|
|
@@ -1365,6 +1376,7 @@ let pushGapFillQueries = (
|
|
|
1365
1376
|
~maybeChunkRange: option<int>,
|
|
1366
1377
|
~maxChunks: int,
|
|
1367
1378
|
~partition: partition,
|
|
1379
|
+
~partitionBudget: float,
|
|
1368
1380
|
~selection: selection,
|
|
1369
1381
|
~addressesByContractName: dict<array<Address.t>>,
|
|
1370
1382
|
) => {
|
|
@@ -1373,10 +1385,14 @@ let pushGapFillQueries = (
|
|
|
1373
1385
|
switch rangeEndBlock {
|
|
1374
1386
|
| Some(endBlock) if rangeFromBlock > endBlock => ()
|
|
1375
1387
|
| _ =>
|
|
1376
|
-
|
|
1377
|
-
|
|
1388
|
+
let trustedDensity = partition->getTrustedDensity
|
|
1389
|
+
let maxBlock = switch rangeEndBlock {
|
|
1390
|
+
| Some(eb) => eb
|
|
1391
|
+
| None => chainTargetBlock
|
|
1392
|
+
}
|
|
1393
|
+
let pushSingleQuery = (~density, ~isChunk) => {
|
|
1378
1394
|
let itemsTarget = densityItemsTarget(
|
|
1379
|
-
|
|
1395
|
+
~density,
|
|
1380
1396
|
~fromBlock=rangeFromBlock,
|
|
1381
1397
|
~toBlock=rangeEndBlock,
|
|
1382
1398
|
~chainTargetBlock,
|
|
@@ -1386,16 +1402,14 @@ let pushGapFillQueries = (
|
|
|
1386
1402
|
fromBlock: rangeFromBlock,
|
|
1387
1403
|
toBlock: rangeEndBlock,
|
|
1388
1404
|
selection,
|
|
1389
|
-
isChunk
|
|
1405
|
+
isChunk,
|
|
1390
1406
|
itemsTarget,
|
|
1391
1407
|
addressesByContractName,
|
|
1392
1408
|
})
|
|
1393
|
-
cost := cost.contents +. itemsTarget
|
|
1394
|
-
|
|
1395
|
-
|
|
1396
|
-
|
|
1397
|
-
| None => chainTargetBlock
|
|
1398
|
-
}
|
|
1409
|
+
cost := cost.contents +. itemsTarget->Int.toFloat
|
|
1410
|
+
}
|
|
1411
|
+
switch (trustedDensity, maybeChunkRange) {
|
|
1412
|
+
| (Some(density), Some(chunkRange)) if density > 0. =>
|
|
1399
1413
|
let chunkSize = Js.Math.ceil_int(chunkRange->Int.toFloat *. 1.8)
|
|
1400
1414
|
if rangeFromBlock + chunkSize * 2 - 1 <= maxBlock {
|
|
1401
1415
|
let chunkFromBlock = ref(rangeFromBlock)
|
|
@@ -1405,7 +1419,7 @@ let pushGapFillQueries = (
|
|
|
1405
1419
|
) {
|
|
1406
1420
|
let chunkToBlock = chunkFromBlock.contents + chunkSize - 1
|
|
1407
1421
|
let itemsTarget = densityItemsTarget(
|
|
1408
|
-
|
|
1422
|
+
~density,
|
|
1409
1423
|
~fromBlock=chunkFromBlock.contents,
|
|
1410
1424
|
~toBlock=Some(chunkToBlock),
|
|
1411
1425
|
~chainTargetBlock,
|
|
@@ -1419,35 +1433,49 @@ let pushGapFillQueries = (
|
|
|
1419
1433
|
itemsTarget,
|
|
1420
1434
|
addressesByContractName,
|
|
1421
1435
|
})
|
|
1422
|
-
cost := cost.contents +. itemsTarget
|
|
1436
|
+
cost := cost.contents +. itemsTarget->Int.toFloat
|
|
1423
1437
|
chunkFromBlock := chunkToBlock + 1
|
|
1424
1438
|
chunkIdx := chunkIdx.contents + 1
|
|
1425
1439
|
}
|
|
1426
1440
|
} else {
|
|
1427
1441
|
// Not enough room for 2 chunks, fall back to a single query
|
|
1428
|
-
|
|
1429
|
-
partition,
|
|
1430
|
-
~fromBlock=rangeFromBlock,
|
|
1431
|
-
~toBlock=rangeEndBlock,
|
|
1432
|
-
~chainTargetBlock,
|
|
1433
|
-
)
|
|
1434
|
-
queries->Array.push({
|
|
1435
|
-
partitionId,
|
|
1436
|
-
fromBlock: rangeFromBlock,
|
|
1437
|
-
toBlock: rangeEndBlock,
|
|
1438
|
-
selection,
|
|
1439
|
-
isChunk: rangeEndBlock !== None,
|
|
1440
|
-
itemsTarget,
|
|
1441
|
-
addressesByContractName,
|
|
1442
|
-
})
|
|
1443
|
-
cost := cost.contents +. itemsTarget
|
|
1442
|
+
pushSingleQuery(~density, ~isChunk=rangeEndBlock !== None)
|
|
1444
1443
|
}
|
|
1444
|
+
| (Some(density), _) => pushSingleQuery(~density, ~isChunk=false)
|
|
1445
|
+
| (None, _) =>
|
|
1446
|
+
let remainingRange = Pervasives.max(1, chainTargetBlock - rangeFromBlock + 1)
|
|
1447
|
+
pushSingleQuery(~density=partitionBudget /. remainingRange->Int.toFloat, ~isChunk=false)
|
|
1445
1448
|
}
|
|
1446
1449
|
}
|
|
1447
1450
|
}
|
|
1448
1451
|
cost.contents
|
|
1449
1452
|
}
|
|
1450
1453
|
|
|
1454
|
+
// The level every partition ends at when `budget` is poured across partitions
|
|
1455
|
+
// already holding `footprints`: the unique L with Σ max(0, L - fᵢ) = budget.
|
|
1456
|
+
// Partitions above L get nothing (their head start is their share); the rest
|
|
1457
|
+
// are topped up exactly to L, so the pour equals the budget no matter how
|
|
1458
|
+
// uneven the footprints are.
|
|
1459
|
+
let waterLevel = (~budget: float, ~footprints: array<float>) => {
|
|
1460
|
+
let sorted = footprints->Array.toSorted(Float.compare)
|
|
1461
|
+
let n = sorted->Array.length
|
|
1462
|
+
let prefix = ref(0.)
|
|
1463
|
+
let level = ref(None)
|
|
1464
|
+
let idx = ref(0)
|
|
1465
|
+
while level.contents == None && idx.contents < n {
|
|
1466
|
+
let i = idx.contents
|
|
1467
|
+
prefix := prefix.contents +. sorted->Array.getUnsafe(i)
|
|
1468
|
+
// The level if only the i+1 lowest footprints receive water — correct once
|
|
1469
|
+
// it doesn't reach the next footprint up.
|
|
1470
|
+
let candidate = (budget +. prefix.contents) /. (i + 1)->Int.toFloat
|
|
1471
|
+
if i == n - 1 || candidate <= sorted->Array.getUnsafe(i + 1) {
|
|
1472
|
+
level := Some(candidate)
|
|
1473
|
+
}
|
|
1474
|
+
idx := idx.contents + 1
|
|
1475
|
+
}
|
|
1476
|
+
level.contents->Option.getOr(0.)
|
|
1477
|
+
}
|
|
1478
|
+
|
|
1451
1479
|
// Per-partition mutable state threaded through the water-fill rounds below.
|
|
1452
1480
|
type waterFillState = {
|
|
1453
1481
|
partitionId: string,
|
|
@@ -1473,21 +1501,24 @@ type waterFillState = {
|
|
|
1473
1501
|
// query with no other ceiling: the true hard bounds stay
|
|
1474
1502
|
// endBlock/mergeBlock/the lagged head.
|
|
1475
1503
|
//
|
|
1476
|
-
// In-range partitions
|
|
1477
|
-
//
|
|
1478
|
-
//
|
|
1479
|
-
//
|
|
1480
|
-
//
|
|
1481
|
-
//
|
|
1504
|
+
// In-range partitions share chainTargetItems (this chain's target total
|
|
1505
|
+
// footprint — in-flight plus new) by water-fill: each round pours exactly the
|
|
1506
|
+
// remaining fresh budget at the level computed by waterLevel, so a partition
|
|
1507
|
+
// already holding more than the level (from earlier ticks' in-flight queries)
|
|
1508
|
+
// gets nothing and its implicit share flows to the others, while totals stay
|
|
1509
|
+
// even and the pour never exceeds the budget. Rounds repeat only because
|
|
1510
|
+
// emits are quantized: a partition may consume less than its allotment (range
|
|
1511
|
+
// or chunk-cap runs out) or slightly more (min one chunk), and the leftover
|
|
1512
|
+
// re-pours over whoever still has range left.
|
|
1482
1513
|
//
|
|
1483
|
-
// A partition with a trusted density (two or more responses — see
|
|
1514
|
+
// A partition with a trusted positive density (two or more responses — see
|
|
1484
1515
|
// getMinHistoryRange) always emits at least one full-size chunk/query once
|
|
1485
|
-
//
|
|
1486
|
-
//
|
|
1516
|
+
// given an allotment, sized by its real density — may overshoot the
|
|
1517
|
+
// allotment by at most one chunk (the server also enforces itemsTarget via a
|
|
1487
1518
|
// maxNumLogs-style cap, so an overshoot truncates the response rather than
|
|
1488
|
-
// the buffer).
|
|
1489
|
-
//
|
|
1490
|
-
//
|
|
1519
|
+
// the buffer). Any other partition (no signal, one noisy sample, or a
|
|
1520
|
+
// density-0 estimate) emits one open-ended probe sized exactly at its
|
|
1521
|
+
// allotment instead.
|
|
1491
1522
|
let getNextQuery = (
|
|
1492
1523
|
{optimizedPartitions, blockLag, latestOnBlockBlockNumber, knownHeight, endBlock}: t,
|
|
1493
1524
|
~chainTargetBlock: int,
|
|
@@ -1557,7 +1588,8 @@ let getNextQuery = (
|
|
|
1557
1588
|
let p = optimizedPartitions.entities->Dict.getUnsafe(partitionId)
|
|
1558
1589
|
let cost = ref(0.)
|
|
1559
1590
|
for pqIdx in 0 to p.mutPendingQueries->Array.length - 1 {
|
|
1560
|
-
cost :=
|
|
1591
|
+
cost :=
|
|
1592
|
+
cost.contents +. (p.mutPendingQueries->Array.getUnsafe(pqIdx)).itemsTarget->Int.toFloat
|
|
1561
1593
|
}
|
|
1562
1594
|
existingReservedByPartition->Dict.set(partitionId, cost.contents)
|
|
1563
1595
|
chainReserved := chainReserved.contents +. cost.contents
|
|
@@ -1597,6 +1629,7 @@ let getNextQuery = (
|
|
|
1597
1629
|
~maybeChunkRange,
|
|
1598
1630
|
~maxChunks=maxPendingChunksPerPartition - pendingCount - chunksUsedThisCall.contents,
|
|
1599
1631
|
~partition=p,
|
|
1632
|
+
~partitionBudget=chainTargetItems /. partitionsCount->Int.toFloat,
|
|
1600
1633
|
~selection=p.selection,
|
|
1601
1634
|
~addressesByContractName=p.addressesByContractName,
|
|
1602
1635
|
)
|
|
@@ -1646,50 +1679,57 @@ let getNextQuery = (
|
|
|
1646
1679
|
fillStates->Array.forEach(fs => {
|
|
1647
1680
|
let cost = ref(existingReservedByPartition->Dict.getUnsafe(fs.partitionId))
|
|
1648
1681
|
for i in 0 to fs.bucket->Array.length - 1 {
|
|
1649
|
-
cost := cost.contents +. (fs.bucket->Array.getUnsafe(i)).itemsTarget
|
|
1682
|
+
cost := cost.contents +. (fs.bucket->Array.getUnsafe(i)).itemsTarget->Int.toFloat
|
|
1650
1683
|
}
|
|
1651
1684
|
reservedByPartition->Dict.set(fs.partitionId, cost.contents)
|
|
1652
1685
|
})
|
|
1653
1686
|
|
|
1654
|
-
|
|
1655
|
-
|
|
1687
|
+
let isInRange = (fs: waterFillState) =>
|
|
1688
|
+
fs.cursor <= chainTargetBlock &&
|
|
1689
|
+
switch fs.queryEndBlock {
|
|
1690
|
+
| Some(eb) => fs.cursor <= eb
|
|
1691
|
+
| None => true
|
|
1692
|
+
} &&
|
|
1693
|
+
fs.pendingCount + fs.chunksUsedThisCall < maxPendingChunksPerPartition
|
|
1694
|
+
|
|
1695
|
+
let inRangeStates = fillStates->Array.filter(isInRange)
|
|
1696
|
+
|
|
1697
|
+
// Emits this round's queries for one partition, given its water-fill
|
|
1698
|
+
// allotment. Mutates fs.cursor/chunksUsedThisCall and returns the
|
|
1656
1699
|
// itemsTarget consumed.
|
|
1657
1700
|
//
|
|
1658
|
-
//
|
|
1659
|
-
//
|
|
1660
|
-
//
|
|
1661
|
-
//
|
|
1662
|
-
// would otherwise mis-size the very next one), so it's treated the same
|
|
1663
|
-
// as a partition with no signal at all: sized exactly to this round's
|
|
1664
|
-
// share instead of an uncapped, one-sample density estimate.
|
|
1701
|
+
// Chunks require a POSITIVE trusted density: density 0 prices every chunk
|
|
1702
|
+
// at ~nothing, letting a partition flood its full chunk pipeline with
|
|
1703
|
+
// hard-bounded queries that crawl 1.8× per two responses — an open-ended
|
|
1704
|
+
// probe instead gets the server's full scan range in one response.
|
|
1665
1705
|
let emitQueries = (fs: waterFillState, ~budget: float) => {
|
|
1666
1706
|
let p = fs.p
|
|
1667
1707
|
let maxBlock = switch fs.queryEndBlock {
|
|
1668
1708
|
| Some(eb) => eb
|
|
1669
1709
|
| None => chainTargetBlock
|
|
1670
1710
|
}
|
|
1671
|
-
switch fs.maybeChunkRange {
|
|
1672
|
-
| Some(minHistoryRange) =>
|
|
1711
|
+
switch (fs.maybeChunkRange, p->getTrustedDensity) {
|
|
1712
|
+
| (Some(minHistoryRange), Some(density)) if density > 0. =>
|
|
1673
1713
|
// Chunking active: strict chunks with a hard endBlock, uncapped real
|
|
1674
1714
|
// density — at least one full chunk this round even if budget falls
|
|
1675
1715
|
// short.
|
|
1676
|
-
let density = p.prevRangeSize->Int.toFloat /. p.prevQueryRange->Int.toFloat
|
|
1677
1716
|
let chunkSize = Js.Math.ceil_int(minHistoryRange->Int.toFloat *. 1.8)
|
|
1678
1717
|
let chunkCost = density *. chunkSize->Int.toFloat
|
|
1679
1718
|
let maxChunksRemaining =
|
|
1680
1719
|
maxPendingChunksPerPartition - fs.pendingCount - fs.chunksUsedThisCall
|
|
1681
|
-
let affordable =
|
|
1682
|
-
Math.floor(budget /. chunkCost)->Float.toInt
|
|
1683
|
-
} else {
|
|
1684
|
-
maxChunksRemaining
|
|
1685
|
-
}
|
|
1720
|
+
let affordable = Math.floor(budget /. chunkCost)->Float.toInt
|
|
1686
1721
|
let numChunks = Pervasives.max(1, Pervasives.min(affordable, maxChunksRemaining))
|
|
1687
1722
|
let consumed = ref(0.)
|
|
1688
1723
|
let created = ref(0)
|
|
1689
1724
|
let chunkFromBlock = ref(fs.cursor)
|
|
1690
1725
|
while created.contents < numChunks && chunkFromBlock.contents <= maxBlock {
|
|
1691
1726
|
let chunkToBlock = Pervasives.min(chunkFromBlock.contents + chunkSize - 1, maxBlock)
|
|
1692
|
-
let itemsTarget =
|
|
1727
|
+
let itemsTarget = Pervasives.max(
|
|
1728
|
+
1,
|
|
1729
|
+
(density *. (chunkToBlock - chunkFromBlock.contents + 1)->Int.toFloat)
|
|
1730
|
+
->Math.ceil
|
|
1731
|
+
->Float.toInt,
|
|
1732
|
+
)
|
|
1693
1733
|
fs.bucket->Array.push({
|
|
1694
1734
|
partitionId: fs.partitionId,
|
|
1695
1735
|
fromBlock: chunkFromBlock.contents,
|
|
@@ -1699,19 +1739,15 @@ let getNextQuery = (
|
|
|
1699
1739
|
itemsTarget,
|
|
1700
1740
|
addressesByContractName: p.addressesByContractName,
|
|
1701
1741
|
})
|
|
1702
|
-
consumed := consumed.contents +. itemsTarget
|
|
1742
|
+
consumed := consumed.contents +. itemsTarget->Int.toFloat
|
|
1703
1743
|
chunkFromBlock := chunkToBlock + 1
|
|
1704
1744
|
created := created.contents + 1
|
|
1705
1745
|
}
|
|
1706
1746
|
fs.cursor = chunkFromBlock.contents
|
|
1707
1747
|
fs.chunksUsedThisCall = fs.chunksUsedThisCall + created.contents
|
|
1708
1748
|
consumed.contents
|
|
1709
|
-
|
|
|
1710
|
-
|
|
1711
|
-
// to this round's share, capped so a single probe can't swallow the
|
|
1712
|
-
// whole (possibly cross-chain-wide) budget and starve other partitions
|
|
1713
|
-
// or chains before they get their own first probe.
|
|
1714
|
-
let itemsTarget = Pervasives.min(Math.round(budget), maxItemsTarget)
|
|
1749
|
+
| _ =>
|
|
1750
|
+
let itemsTarget = Pervasives.max(1, Math.round(budget)->Float.toInt)
|
|
1715
1751
|
fs.bucket->Array.push({
|
|
1716
1752
|
partitionId: fs.partitionId,
|
|
1717
1753
|
fromBlock: fs.cursor,
|
|
@@ -1723,56 +1759,41 @@ let getNextQuery = (
|
|
|
1723
1759
|
})
|
|
1724
1760
|
fs.cursor = maxBlock + 1
|
|
1725
1761
|
fs.chunksUsedThisCall = fs.chunksUsedThisCall + 1
|
|
1726
|
-
itemsTarget
|
|
1762
|
+
itemsTarget->Int.toFloat
|
|
1727
1763
|
}
|
|
1728
1764
|
}
|
|
1729
1765
|
|
|
1730
|
-
let isInRange = (fs: waterFillState) =>
|
|
1731
|
-
fs.cursor <= chainTargetBlock &&
|
|
1732
|
-
switch fs.queryEndBlock {
|
|
1733
|
-
| Some(eb) => fs.cursor <= eb
|
|
1734
|
-
| None => true
|
|
1735
|
-
} &&
|
|
1736
|
-
fs.pendingCount + fs.chunksUsedThisCall < maxPendingChunksPerPartition
|
|
1737
|
-
|
|
1738
1766
|
// Water-fill. Range membership is fixed by chainTargetBlock/queryEndBlock,
|
|
1739
|
-
// so the in-range partitions are filtered once up front; each round
|
|
1740
|
-
//
|
|
1741
|
-
// those still in range for the next
|
|
1742
|
-
//
|
|
1743
|
-
//
|
|
1744
|
-
//
|
|
1745
|
-
//
|
|
1746
|
-
// rest are topped up to the line — so leftover from an under-using or
|
|
1747
|
-
// filled partition redistributes to whoever can still use it, and totals
|
|
1748
|
-
// stay even regardless of processing order (the line is fixed before the
|
|
1749
|
-
// round's emits).
|
|
1767
|
+
// so the in-range partitions are filtered once up front; each round pours
|
|
1768
|
+
// the remaining fresh budget at the waterLevel of the still-not-filled
|
|
1769
|
+
// partitions' footprints and keeps only those still in range for the next
|
|
1770
|
+
// round. Allotments sum to exactly the poured budget, so the outcome is
|
|
1771
|
+
// order-independent and never exceeds it — the only overshoot left is the
|
|
1772
|
+
// min-one-chunk quantization in emitQueries, bounded by one chunk per
|
|
1773
|
+
// partition per round.
|
|
1750
1774
|
//
|
|
1751
1775
|
// No explicit round cap: a partition survives a round only by advancing
|
|
1752
1776
|
// chunksUsedThisCall (capped at maxPendingChunksPerPartition) or by
|
|
1753
|
-
// consuming fresh budget
|
|
1754
|
-
// loop also stops once the whole
|
|
1755
|
-
|
|
1756
|
-
let
|
|
1757
|
-
|
|
1758
|
-
|
|
1759
|
-
|
|
1760
|
-
|
|
1761
|
-
|
|
1762
|
-
|
|
1763
|
-
|
|
1764
|
-
footprintSum := footprintSum.contents +. reservedByPartition->Dict.getUnsafe(fs.partitionId)
|
|
1777
|
+
// consuming fresh budget (every emit consumes at least 1), so the
|
|
1778
|
+
// not-filled set drains on its own and the loop also stops once the whole
|
|
1779
|
+
// budget is poured.
|
|
1780
|
+
let notFilledPartitions = ref(inRangeStates)
|
|
1781
|
+
let remainingBudget = ref(rangeItemsTarget)
|
|
1782
|
+
while notFilledPartitions.contents->Array.length > 0 && remainingBudget.contents > 0. {
|
|
1783
|
+
let level = waterLevel(
|
|
1784
|
+
~budget=remainingBudget.contents,
|
|
1785
|
+
~footprints=notFilledPartitions.contents->Array.map(fs =>
|
|
1786
|
+
reservedByPartition->Dict.getUnsafe(fs.partitionId)
|
|
1787
|
+
),
|
|
1765
1788
|
)
|
|
1766
|
-
let line =
|
|
1767
|
-
(rangeItemsTarget -. reservedFromRange.contents +. footprintSum.contents) /. n->Int.toFloat
|
|
1768
1789
|
let next = []
|
|
1769
1790
|
notFilledPartitions.contents->Array.forEach(fs => {
|
|
1770
1791
|
let reserved = reservedByPartition->Dict.getUnsafe(fs.partitionId)
|
|
1771
|
-
let budget =
|
|
1792
|
+
let budget = level -. reserved
|
|
1772
1793
|
if budget > 0. {
|
|
1773
1794
|
let consumed = emitQueries(fs, ~budget)
|
|
1774
1795
|
reservedByPartition->Dict.set(fs.partitionId, reserved +. consumed)
|
|
1775
|
-
|
|
1796
|
+
remainingBudget := remainingBudget.contents -. consumed
|
|
1776
1797
|
if fs->isInRange {
|
|
1777
1798
|
next->Array.push(fs)->ignore
|
|
1778
1799
|
}
|