envio 3.3.0-alpha.8 → 3.3.0-alpha.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -243,9 +243,13 @@ let getTopicEncoder = (abiType: string): (unknown => EvmTypes.Hex.t) => {
243
243
  } else {
244
244
  switch abiType {
245
245
  | "address" =>
246
- TopicFilter.fromAddress->(
247
- Utils.magic: (Address.t => EvmTypes.Hex.t) => unknown => EvmTypes.Hex.t
248
- )
246
+ // Lowercase before encoding so mixed-case (checksummed) user input
247
+ // still matches the lowercase hex topics returned by sources.
248
+
249
+ (
250
+ (value: string) =>
251
+ value->String.toLowerCase->Address.unsafeFromString->TopicFilter.fromAddress
252
+ )->(Utils.magic: (string => EvmTypes.Hex.t) => unknown => EvmTypes.Hex.t)
249
253
 
250
254
  | "bool" =>
251
255
  TopicFilter.fromBool->(Utils.magic: (bool => EvmTypes.Hex.t) => unknown => EvmTypes.Hex.t)
@@ -413,31 +417,26 @@ let buildEvmEventConfig = (
413
417
  }
414
418
 
415
419
  // Enrich an EVM definition into a per-(event,chain) registration: resolve the
416
- // registered `where` (with `probeChainId`) into the lazy `getEventFiltersOrThrow`
417
- // closure + address filters, and override `startBlock` with `where.block._gte`.
420
+ // registered `where` for this chain into `resolvedWhere` + address filters,
421
+ // and override `startBlock` with `where.block._gte`.
418
422
  let buildEvmOnEventRegistration = (
419
423
  ~eventConfig: Internal.evmEventConfig,
420
424
  ~isWildcard: bool,
421
425
  ~handler: option<Internal.handler>,
422
426
  ~contractRegister: option<Internal.contractRegister>,
423
- ~eventFilters: option<JSON.t>,
424
- ~probeChainId: int,
427
+ ~where: option<JSON.t>,
428
+ ~chainId: int,
425
429
  ~onEventBlockFilterSchema: S.t<option<unknown>>,
426
430
  ~startBlock: option<int>=?,
427
431
  ): Internal.evmOnEventRegistration => {
428
432
  let indexedParams = eventConfig.paramsMetadata->Array.filter(p => p.indexed)
429
433
 
430
- let {
431
- getEventFiltersOrThrow,
432
- filterByAddresses,
433
- addressFilterParamGroups,
434
- startBlock: whereStartBlock,
435
- } = LogSelection.parseEventFiltersOrThrow(
436
- ~eventFilters,
434
+ let {resolvedWhere, filterByAddresses, addressFilterParamGroups} = LogSelection.parseWhereOrThrow(
435
+ ~where,
437
436
  ~sighash=eventConfig.sighash,
438
437
  ~params=indexedParams->Array.map(p => p.name),
439
438
  ~contractName=eventConfig.contractName,
440
- ~probeChainId,
439
+ ~chainId,
441
440
  ~onEventBlockFilterSchema,
442
441
  ~topic1=?indexedParams->Array.get(0)->Option.map(buildTopicGetter),
443
442
  ~topic2=?indexedParams->Array.get(1)->Option.map(buildTopicGetter),
@@ -447,7 +446,7 @@ let buildEvmOnEventRegistration = (
447
446
  // `where.block.number._gte` overrides the contract-level startBlock when
448
447
  // present (an explicit per-event opt-in that wins over `config.yaml`);
449
448
  // otherwise the contract/chain value passes through.
450
- let resolvedStartBlock = switch whereStartBlock {
449
+ let resolvedStartBlock = switch resolvedWhere.startBlock {
451
450
  | Some(_) as sb => sb
452
451
  | None => startBlock
453
452
  }
@@ -457,7 +456,7 @@ let buildEvmOnEventRegistration = (
457
456
  isWildcard,
458
457
  handler,
459
458
  contractRegister,
460
- getEventFiltersOrThrow,
459
+ resolvedWhere,
461
460
  filterByAddresses,
462
461
  clientAddressFilter: ?buildAddressFilter(addressFilterParamGroups, ~isWildcard),
463
462
  dependsOnAddresses: Internal.dependsOnAddresses(~isWildcard, ~filterByAddresses),
@@ -213,7 +213,7 @@ function getTopicEncoder(abiType) {
213
213
  }
214
214
  switch (abiType) {
215
215
  case "address" :
216
- return TopicFilter.fromAddress;
216
+ return value => TopicFilter.fromAddress(value.toLowerCase());
217
217
  case "bool" :
218
218
  return TopicFilter.fromBool;
219
219
  case "bytes" :
@@ -309,12 +309,13 @@ function buildEvmEventConfig(contractName, eventName, sighash, params, blockFiel
309
309
  };
310
310
  }
311
311
 
312
- function buildEvmOnEventRegistration(eventConfig, isWildcard, handler, contractRegister, eventFilters, probeChainId, onEventBlockFilterSchema, startBlock) {
312
+ function buildEvmOnEventRegistration(eventConfig, isWildcard, handler, contractRegister, where, chainId, onEventBlockFilterSchema, startBlock) {
313
313
  let indexedParams = eventConfig.paramsMetadata.filter(p => p.indexed);
314
- let match = LogSelection.parseEventFiltersOrThrow(eventFilters, eventConfig.sighash, indexedParams.map(p => p.name), eventConfig.contractName, probeChainId, onEventBlockFilterSchema, Stdlib_Option.map(indexedParams[0], buildTopicGetter), Stdlib_Option.map(indexedParams[1], buildTopicGetter), Stdlib_Option.map(indexedParams[2], buildTopicGetter));
315
- let whereStartBlock = match.startBlock;
314
+ let match = LogSelection.parseWhereOrThrow(where, eventConfig.sighash, indexedParams.map(p => p.name), eventConfig.contractName, chainId, onEventBlockFilterSchema, Stdlib_Option.map(indexedParams[0], buildTopicGetter), Stdlib_Option.map(indexedParams[1], buildTopicGetter), Stdlib_Option.map(indexedParams[2], buildTopicGetter));
316
315
  let filterByAddresses = match.filterByAddresses;
317
- let resolvedStartBlock = whereStartBlock !== undefined ? whereStartBlock : startBlock;
316
+ let resolvedWhere = match.resolvedWhere;
317
+ let sb = resolvedWhere.startBlock;
318
+ let resolvedStartBlock = sb !== undefined ? sb : startBlock;
318
319
  return {
319
320
  eventConfig: eventConfig,
320
321
  handler: handler,
@@ -324,7 +325,7 @@ function buildEvmOnEventRegistration(eventConfig, isWildcard, handler, contractR
324
325
  dependsOnAddresses: Internal.dependsOnAddresses(isWildcard, filterByAddresses),
325
326
  clientAddressFilter: buildAddressFilter(match.addressFilterParamGroups, isWildcard, undefined),
326
327
  startBlock: resolvedStartBlock,
327
- getEventFiltersOrThrow: match.getEventFiltersOrThrow
328
+ resolvedWhere: resolvedWhere
328
329
  };
329
330
  }
330
331
 
@@ -19,7 +19,7 @@ type pendingQuery = {
19
19
  // Items this in-flight query is targeting (server maxNumLogs-style cap), carried
20
20
  // from the query so the shared buffer budget can account for what's already
21
21
  // being fetched.
22
- itemsTarget: float,
22
+ itemsTarget: int,
23
23
  // Stores latestFetchedBlock when query completes. Only needed to persist
24
24
  // timestamp while earlier queries are still pending before updating
25
25
  // the partition's latestFetchedBlock.
@@ -68,7 +68,7 @@ type query = {
68
68
  // with known density this is density × the query's block range; for a
69
69
  // partition with no signal yet it's whatever budget share the query was
70
70
  // sized against.
71
- itemsTarget: float,
71
+ itemsTarget: int,
72
72
  selection: selection,
73
73
  addressesByContractName: dict<array<Address.t>>,
74
74
  }
@@ -91,27 +91,19 @@ let deriveContractNameByAddress: dict<array<Address.t>> => dict<
91
91
  result
92
92
  })
93
93
 
94
- // Floor for a query's itemsTarget. Also used by SourceManager as the floor for
95
- // its own maxNumLogs-style safety net.
96
- let minItemsTarget = 2_000.
97
-
98
- // Ceiling for an unknown-density probe query's itemsTarget. A partition with no
99
- // trusted density yet gets one open-ended probe; without a cap it would claim its
100
- // full even share of the chain's (possibly whole-pool) budget and starve sibling
101
- // chains needing their own first probe in the same cross-chain tick.
102
- let maxItemsTarget = 10_000.
103
-
104
- // itemsTarget for a query over [fromBlock, toBlock], from the partition's event
105
- // density (items/block derived from its last response). toBlock None is the
106
- // open-ended tail, capped at chainTargetBlock — the soft per-tick horizon the
107
- // owning chain wants to reach (see getNextQuery). A partition that responded
108
- // with no items has density 0, so its queries cost 0 — correct, they don't
109
- // fill the buffer. Only called for partitions with a known density
110
- // (prevQueryRange > 0); partitions with no signal yet are sized instead
111
- // against their even split of the chain's range budget.
112
- let densityItemsTarget = (p: partition, ~fromBlock, ~toBlock, ~chainTargetBlock) => {
113
- let density = p.prevRangeSize->Int.toFloat /. p.prevQueryRange->Int.toFloat
114
- (toBlock->Option.getOr(chainTargetBlock) - fromBlock + 1)->Int.toFloat *. density
94
+ // itemsTarget for a query over [fromBlock, toBlock] at the given event density
95
+ // (items/block). toBlock None is the open-ended tail, capped at
96
+ // chainTargetBlock the soft per-tick horizon the owning chain wants to reach
97
+ // (see getNextQuery).
98
+ let densityItemsTarget = (~density, ~fromBlock, ~toBlock, ~chainTargetBlock) => {
99
+ // Floor at 1: the reservation must equal the server-side cap SourceManager
100
+ // sends, and a 0 cap would ask the backend for nothing.
101
+ Pervasives.max(
102
+ 1,
103
+ ((toBlock->Option.getOr(chainTargetBlock) - fromBlock + 1)->Int.toFloat *. density)
104
+ ->Math.ceil
105
+ ->Float.toInt,
106
+ )
115
107
  }
116
108
 
117
109
  // Calculate the chunk range from history using min-of-last-3-ranges heuristic
@@ -122,6 +114,15 @@ let getMinHistoryRange = (p: partition) => {
122
114
  }
123
115
  }
124
116
 
117
+ // Density (items/block) from the last response, trusted only after two
118
+ // responses — a single sample is too noisy to size the next query by.
119
+ let getTrustedDensity = (p: partition) => {
120
+ switch (p.prevQueryRange, p.prevPrevQueryRange) {
121
+ | (0, _) | (_, 0) => None
122
+ | (prevQueryRange, _) => Some(p.prevRangeSize->Int.toFloat /. prevQueryRange->Int.toFloat)
123
+ }
124
+ }
125
+
125
126
  let getMinQueryRange = (partitions: array<partition>) => {
126
127
  let min = ref(0)
127
128
  for i in 0 to partitions->Array.length - 1 {
@@ -201,6 +202,12 @@ module OptimizedPartitions = {
201
202
  let newId = nextPartitionIndexRef.contents->Int.toString
202
203
  nextPartitionIndexRef := nextPartitionIndexRef.contents + 1
203
204
  let minRange = getMinQueryRange([p1, p2])
205
+ // The merged partition indexes both parents' addresses, so its expected
206
+ // event rate is the sum of their densities. Parents without a trusted
207
+ // density contribute 0; if none has one, prevRangeSize stays 0 and the
208
+ // partition probes for a fresh signal instead of chunking.
209
+ let inheritedDensity =
210
+ p1->getTrustedDensity->Option.getOr(0.) +. p2->getTrustedDensity->Option.getOr(0.)
204
211
  {
205
212
  id: newId,
206
213
  dynamicContract: Some(contractName),
@@ -211,7 +218,7 @@ module OptimizedPartitions = {
211
218
  mutPendingQueries: [],
212
219
  prevQueryRange: minRange,
213
220
  prevPrevQueryRange: minRange,
214
- prevRangeSize: 0,
221
+ prevRangeSize: (inheritedDensity *. minRange->Int.toFloat)->Math.ceil->Float.toInt,
215
222
  latestBlockRangeUpdateBlock: 0,
216
223
  }
217
224
  }
@@ -1352,9 +1359,13 @@ let startFetchingQueries = ({optimizedPartitions}: t, ~queries: array<query>) =>
1352
1359
  let maxPendingChunksPerPartition = 10
1353
1360
 
1354
1361
  // Fills a gap range (a hole left between completed/pending chunks, e.g. from an
1355
- // out-of-order partial response) unconditionally against the partition's real
1356
- // density — gaps are already-committed range, not subject to this tick's
1357
- // water-fill budget. Returns the created queries' total itemsTarget.
1362
+ // out-of-order partial response) unconditionally gaps are already-committed
1363
+ // range, not subject to this tick's water-fill budget. Priced by the
1364
+ // partition's trusted density when it has one; otherwise by the "available
1365
+ // density" — the partition's equal-divide budget spread over its remaining
1366
+ // range this tick — so a small gap reserves proportionally little instead of a
1367
+ // noisy one-sample estimate. Chunks only on a trusted POSITIVE density, same
1368
+ // rule as the water-fill. Returns the created queries' total itemsTarget.
1358
1369
  let pushGapFillQueries = (
1359
1370
  queries: array<query>,
1360
1371
  ~partitionId: string,
@@ -1365,6 +1376,7 @@ let pushGapFillQueries = (
1365
1376
  ~maybeChunkRange: option<int>,
1366
1377
  ~maxChunks: int,
1367
1378
  ~partition: partition,
1379
+ ~partitionBudget: float,
1368
1380
  ~selection: selection,
1369
1381
  ~addressesByContractName: dict<array<Address.t>>,
1370
1382
  ) => {
@@ -1373,10 +1385,14 @@ let pushGapFillQueries = (
1373
1385
  switch rangeEndBlock {
1374
1386
  | Some(endBlock) if rangeFromBlock > endBlock => ()
1375
1387
  | _ =>
1376
- switch maybeChunkRange {
1377
- | None =>
1388
+ let trustedDensity = partition->getTrustedDensity
1389
+ let maxBlock = switch rangeEndBlock {
1390
+ | Some(eb) => eb
1391
+ | None => chainTargetBlock
1392
+ }
1393
+ let pushSingleQuery = (~density, ~isChunk) => {
1378
1394
  let itemsTarget = densityItemsTarget(
1379
- partition,
1395
+ ~density,
1380
1396
  ~fromBlock=rangeFromBlock,
1381
1397
  ~toBlock=rangeEndBlock,
1382
1398
  ~chainTargetBlock,
@@ -1386,16 +1402,14 @@ let pushGapFillQueries = (
1386
1402
  fromBlock: rangeFromBlock,
1387
1403
  toBlock: rangeEndBlock,
1388
1404
  selection,
1389
- isChunk: false,
1405
+ isChunk,
1390
1406
  itemsTarget,
1391
1407
  addressesByContractName,
1392
1408
  })
1393
- cost := cost.contents +. itemsTarget
1394
- | Some(chunkRange) =>
1395
- let maxBlock = switch rangeEndBlock {
1396
- | Some(eb) => eb
1397
- | None => chainTargetBlock
1398
- }
1409
+ cost := cost.contents +. itemsTarget->Int.toFloat
1410
+ }
1411
+ switch (trustedDensity, maybeChunkRange) {
1412
+ | (Some(density), Some(chunkRange)) if density > 0. =>
1399
1413
  let chunkSize = Js.Math.ceil_int(chunkRange->Int.toFloat *. 1.8)
1400
1414
  if rangeFromBlock + chunkSize * 2 - 1 <= maxBlock {
1401
1415
  let chunkFromBlock = ref(rangeFromBlock)
@@ -1405,7 +1419,7 @@ let pushGapFillQueries = (
1405
1419
  ) {
1406
1420
  let chunkToBlock = chunkFromBlock.contents + chunkSize - 1
1407
1421
  let itemsTarget = densityItemsTarget(
1408
- partition,
1422
+ ~density,
1409
1423
  ~fromBlock=chunkFromBlock.contents,
1410
1424
  ~toBlock=Some(chunkToBlock),
1411
1425
  ~chainTargetBlock,
@@ -1419,35 +1433,49 @@ let pushGapFillQueries = (
1419
1433
  itemsTarget,
1420
1434
  addressesByContractName,
1421
1435
  })
1422
- cost := cost.contents +. itemsTarget
1436
+ cost := cost.contents +. itemsTarget->Int.toFloat
1423
1437
  chunkFromBlock := chunkToBlock + 1
1424
1438
  chunkIdx := chunkIdx.contents + 1
1425
1439
  }
1426
1440
  } else {
1427
1441
  // Not enough room for 2 chunks, fall back to a single query
1428
- let itemsTarget = densityItemsTarget(
1429
- partition,
1430
- ~fromBlock=rangeFromBlock,
1431
- ~toBlock=rangeEndBlock,
1432
- ~chainTargetBlock,
1433
- )
1434
- queries->Array.push({
1435
- partitionId,
1436
- fromBlock: rangeFromBlock,
1437
- toBlock: rangeEndBlock,
1438
- selection,
1439
- isChunk: rangeEndBlock !== None,
1440
- itemsTarget,
1441
- addressesByContractName,
1442
- })
1443
- cost := cost.contents +. itemsTarget
1442
+ pushSingleQuery(~density, ~isChunk=rangeEndBlock !== None)
1444
1443
  }
1444
+ | (Some(density), _) => pushSingleQuery(~density, ~isChunk=false)
1445
+ | (None, _) =>
1446
+ let remainingRange = Pervasives.max(1, chainTargetBlock - rangeFromBlock + 1)
1447
+ pushSingleQuery(~density=partitionBudget /. remainingRange->Int.toFloat, ~isChunk=false)
1445
1448
  }
1446
1449
  }
1447
1450
  }
1448
1451
  cost.contents
1449
1452
  }
1450
1453
 
1454
+ // The level every partition ends at when `budget` is poured across partitions
1455
+ // already holding `footprints`: the unique L with Σ max(0, L - fᵢ) = budget.
1456
+ // Partitions above L get nothing (their head start is their share); the rest
1457
+ // are topped up exactly to L, so the pour equals the budget no matter how
1458
+ // uneven the footprints are.
1459
+ let waterLevel = (~budget: float, ~footprints: array<float>) => {
1460
+ let sorted = footprints->Array.toSorted(Float.compare)
1461
+ let n = sorted->Array.length
1462
+ let prefix = ref(0.)
1463
+ let level = ref(None)
1464
+ let idx = ref(0)
1465
+ while level.contents == None && idx.contents < n {
1466
+ let i = idx.contents
1467
+ prefix := prefix.contents +. sorted->Array.getUnsafe(i)
1468
+ // The level if only the i+1 lowest footprints receive water — correct once
1469
+ // it doesn't reach the next footprint up.
1470
+ let candidate = (budget +. prefix.contents) /. (i + 1)->Int.toFloat
1471
+ if i == n - 1 || candidate <= sorted->Array.getUnsafe(i + 1) {
1472
+ level := Some(candidate)
1473
+ }
1474
+ idx := idx.contents + 1
1475
+ }
1476
+ level.contents->Option.getOr(0.)
1477
+ }
1478
+
1451
1479
  // Per-partition mutable state threaded through the water-fill rounds below.
1452
1480
  type waterFillState = {
1453
1481
  partitionId: string,
@@ -1473,21 +1501,24 @@ type waterFillState = {
1473
1501
  // query with no other ceiling: the true hard bounds stay
1474
1502
  // endBlock/mergeBlock/the lagged head.
1475
1503
  //
1476
- // In-range partitions evenly split chainTargetItems (this chain's target
1477
- // total footprint — in-flight plus new). A partition already holding more
1478
- // than its even share (from earlier ticks' in-flight queries) is skipped this
1479
- // round so its share flows to the others; the split is recomputed every round
1480
- // against the shrinking set of partitions still with range left, so leftover
1481
- // keeps redistributing until the budget or the range runs out.
1504
+ // In-range partitions share chainTargetItems (this chain's target total
1505
+ // footprint — in-flight plus new) by water-fill: each round pours exactly the
1506
+ // remaining fresh budget at the level computed by waterLevel, so a partition
1507
+ // already holding more than the level (from earlier ticks' in-flight queries)
1508
+ // gets nothing and its implicit share flows to the others, while totals stay
1509
+ // even and the pour never exceeds the budget. Rounds repeat only because
1510
+ // emits are quantized: a partition may consume less than its allotment (range
1511
+ // or chunk-cap runs out) or slightly more (min one chunk), and the leftover
1512
+ // re-pours over whoever still has range left.
1482
1513
  //
1483
- // A partition with a trusted density (two or more responses — see
1514
+ // A partition with a trusted positive density (two or more responses — see
1484
1515
  // getMinHistoryRange) always emits at least one full-size chunk/query once
1485
- // included in a round, sized by its real density — uncapped, may overshoot
1486
- // that round's share (the server also enforces itemsTarget via a
1516
+ // given an allotment, sized by its real density — may overshoot the
1517
+ // allotment by at most one chunk (the server also enforces itemsTarget via a
1487
1518
  // maxNumLogs-style cap, so an overshoot truncates the response rather than
1488
- // the buffer). A partition with no trusted density yet (zero or one
1489
- // response a single sample is too noisy to size by) emits one query sized
1490
- // exactly to its round's share instead.
1519
+ // the buffer). Any other partition (no signal, one noisy sample, or a
1520
+ // density-0 estimate) emits one open-ended probe sized exactly at its
1521
+ // allotment instead.
1491
1522
  let getNextQuery = (
1492
1523
  {optimizedPartitions, blockLag, latestOnBlockBlockNumber, knownHeight, endBlock}: t,
1493
1524
  ~chainTargetBlock: int,
@@ -1557,7 +1588,8 @@ let getNextQuery = (
1557
1588
  let p = optimizedPartitions.entities->Dict.getUnsafe(partitionId)
1558
1589
  let cost = ref(0.)
1559
1590
  for pqIdx in 0 to p.mutPendingQueries->Array.length - 1 {
1560
- cost := cost.contents +. (p.mutPendingQueries->Array.getUnsafe(pqIdx)).itemsTarget
1591
+ cost :=
1592
+ cost.contents +. (p.mutPendingQueries->Array.getUnsafe(pqIdx)).itemsTarget->Int.toFloat
1561
1593
  }
1562
1594
  existingReservedByPartition->Dict.set(partitionId, cost.contents)
1563
1595
  chainReserved := chainReserved.contents +. cost.contents
@@ -1597,6 +1629,7 @@ let getNextQuery = (
1597
1629
  ~maybeChunkRange,
1598
1630
  ~maxChunks=maxPendingChunksPerPartition - pendingCount - chunksUsedThisCall.contents,
1599
1631
  ~partition=p,
1632
+ ~partitionBudget=chainTargetItems /. partitionsCount->Int.toFloat,
1600
1633
  ~selection=p.selection,
1601
1634
  ~addressesByContractName=p.addressesByContractName,
1602
1635
  )
@@ -1646,50 +1679,57 @@ let getNextQuery = (
1646
1679
  fillStates->Array.forEach(fs => {
1647
1680
  let cost = ref(existingReservedByPartition->Dict.getUnsafe(fs.partitionId))
1648
1681
  for i in 0 to fs.bucket->Array.length - 1 {
1649
- cost := cost.contents +. (fs.bucket->Array.getUnsafe(i)).itemsTarget
1682
+ cost := cost.contents +. (fs.bucket->Array.getUnsafe(i)).itemsTarget->Int.toFloat
1650
1683
  }
1651
1684
  reservedByPartition->Dict.set(fs.partitionId, cost.contents)
1652
1685
  })
1653
1686
 
1654
- // Emits this round's queries for one partition, given its share of the
1655
- // range budget. Mutates fs.cursor/chunksUsedThisCall and returns the
1687
+ let isInRange = (fs: waterFillState) =>
1688
+ fs.cursor <= chainTargetBlock &&
1689
+ switch fs.queryEndBlock {
1690
+ | Some(eb) => fs.cursor <= eb
1691
+ | None => true
1692
+ } &&
1693
+ fs.pendingCount + fs.chunksUsedThisCall < maxPendingChunksPerPartition
1694
+
1695
+ let inRangeStates = fillStates->Array.filter(isInRange)
1696
+
1697
+ // Emits this round's queries for one partition, given its water-fill
1698
+ // allotment. Mutates fs.cursor/chunksUsedThisCall and returns the
1656
1699
  // itemsTarget consumed.
1657
1700
  //
1658
- // Density is trusted only once the chunking heuristic is active
1659
- // (maybeChunkRange requires prevQueryRange AND prevPrevQueryRange, i.e.
1660
- // two responses see getMinHistoryRange) a single response is noisy
1661
- // (a first query that happens to land on an unusually dense/sparse block
1662
- // would otherwise mis-size the very next one), so it's treated the same
1663
- // as a partition with no signal at all: sized exactly to this round's
1664
- // share instead of an uncapped, one-sample density estimate.
1701
+ // Chunks require a POSITIVE trusted density: density 0 prices every chunk
1702
+ // at ~nothing, letting a partition flood its full chunk pipeline with
1703
+ // hard-bounded queries that crawl 1.8× per two responses an open-ended
1704
+ // probe instead gets the server's full scan range in one response.
1665
1705
  let emitQueries = (fs: waterFillState, ~budget: float) => {
1666
1706
  let p = fs.p
1667
1707
  let maxBlock = switch fs.queryEndBlock {
1668
1708
  | Some(eb) => eb
1669
1709
  | None => chainTargetBlock
1670
1710
  }
1671
- switch fs.maybeChunkRange {
1672
- | Some(minHistoryRange) =>
1711
+ switch (fs.maybeChunkRange, p->getTrustedDensity) {
1712
+ | (Some(minHistoryRange), Some(density)) if density > 0. =>
1673
1713
  // Chunking active: strict chunks with a hard endBlock, uncapped real
1674
1714
  // density — at least one full chunk this round even if budget falls
1675
1715
  // short.
1676
- let density = p.prevRangeSize->Int.toFloat /. p.prevQueryRange->Int.toFloat
1677
1716
  let chunkSize = Js.Math.ceil_int(minHistoryRange->Int.toFloat *. 1.8)
1678
1717
  let chunkCost = density *. chunkSize->Int.toFloat
1679
1718
  let maxChunksRemaining =
1680
1719
  maxPendingChunksPerPartition - fs.pendingCount - fs.chunksUsedThisCall
1681
- let affordable = if chunkCost > 0. {
1682
- Math.floor(budget /. chunkCost)->Float.toInt
1683
- } else {
1684
- maxChunksRemaining
1685
- }
1720
+ let affordable = Math.floor(budget /. chunkCost)->Float.toInt
1686
1721
  let numChunks = Pervasives.max(1, Pervasives.min(affordable, maxChunksRemaining))
1687
1722
  let consumed = ref(0.)
1688
1723
  let created = ref(0)
1689
1724
  let chunkFromBlock = ref(fs.cursor)
1690
1725
  while created.contents < numChunks && chunkFromBlock.contents <= maxBlock {
1691
1726
  let chunkToBlock = Pervasives.min(chunkFromBlock.contents + chunkSize - 1, maxBlock)
1692
- let itemsTarget = density *. (chunkToBlock - chunkFromBlock.contents + 1)->Int.toFloat
1727
+ let itemsTarget = Pervasives.max(
1728
+ 1,
1729
+ (density *. (chunkToBlock - chunkFromBlock.contents + 1)->Int.toFloat)
1730
+ ->Math.ceil
1731
+ ->Float.toInt,
1732
+ )
1693
1733
  fs.bucket->Array.push({
1694
1734
  partitionId: fs.partitionId,
1695
1735
  fromBlock: chunkFromBlock.contents,
@@ -1699,19 +1739,15 @@ let getNextQuery = (
1699
1739
  itemsTarget,
1700
1740
  addressesByContractName: p.addressesByContractName,
1701
1741
  })
1702
- consumed := consumed.contents +. itemsTarget
1742
+ consumed := consumed.contents +. itemsTarget->Int.toFloat
1703
1743
  chunkFromBlock := chunkToBlock + 1
1704
1744
  created := created.contents + 1
1705
1745
  }
1706
1746
  fs.cursor = chunkFromBlock.contents
1707
1747
  fs.chunksUsedThisCall = fs.chunksUsedThisCall + created.contents
1708
1748
  consumed.contents
1709
- | None =>
1710
- // No trusted density yet (zero or one response): one open query sized
1711
- // to this round's share, capped so a single probe can't swallow the
1712
- // whole (possibly cross-chain-wide) budget and starve other partitions
1713
- // or chains before they get their own first probe.
1714
- let itemsTarget = Pervasives.min(Math.round(budget), maxItemsTarget)
1749
+ | _ =>
1750
+ let itemsTarget = Pervasives.max(1, Math.round(budget)->Float.toInt)
1715
1751
  fs.bucket->Array.push({
1716
1752
  partitionId: fs.partitionId,
1717
1753
  fromBlock: fs.cursor,
@@ -1723,56 +1759,41 @@ let getNextQuery = (
1723
1759
  })
1724
1760
  fs.cursor = maxBlock + 1
1725
1761
  fs.chunksUsedThisCall = fs.chunksUsedThisCall + 1
1726
- itemsTarget
1762
+ itemsTarget->Int.toFloat
1727
1763
  }
1728
1764
  }
1729
1765
 
1730
- let isInRange = (fs: waterFillState) =>
1731
- fs.cursor <= chainTargetBlock &&
1732
- switch fs.queryEndBlock {
1733
- | Some(eb) => fs.cursor <= eb
1734
- | None => true
1735
- } &&
1736
- fs.pendingCount + fs.chunksUsedThisCall < maxPendingChunksPerPartition
1737
-
1738
1766
  // Water-fill. Range membership is fixed by chainTargetBlock/queryEndBlock,
1739
- // so the in-range partitions are filtered once up front; each round levels
1740
- // every still-not-filled partition up toward a shared line and keeps only
1741
- // those still in range for the next round.
1742
- //
1743
- // The line is (remaining fresh budget + those partitions' current
1744
- // footprint) / count: a partition already holding more than the line gets
1745
- // nothing and drops out (its head start is its whole share), while the
1746
- // rest are topped up to the line — so leftover from an under-using or
1747
- // filled partition redistributes to whoever can still use it, and totals
1748
- // stay even regardless of processing order (the line is fixed before the
1749
- // round's emits).
1767
+ // so the in-range partitions are filtered once up front; each round pours
1768
+ // the remaining fresh budget at the waterLevel of the still-not-filled
1769
+ // partitions' footprints and keeps only those still in range for the next
1770
+ // round. Allotments sum to exactly the poured budget, so the outcome is
1771
+ // order-independent and never exceeds it the only overshoot left is the
1772
+ // min-one-chunk quantization in emitQueries, bounded by one chunk per
1773
+ // partition per round.
1750
1774
  //
1751
1775
  // No explicit round cap: a partition survives a round only by advancing
1752
1776
  // chunksUsedThisCall (capped at maxPendingChunksPerPartition) or by
1753
- // consuming fresh budget, so the not-filled set drains on its own and the
1754
- // loop also stops once the whole budget is reserved.
1755
- let notFilledPartitions = ref(fillStates->Array.filter(isInRange))
1756
- let reservedFromRange = ref(0.)
1757
- while (
1758
- notFilledPartitions.contents->Array.length > 0 &&
1759
- reservedFromRange.contents < rangeItemsTarget
1760
- ) {
1761
- let n = notFilledPartitions.contents->Array.length
1762
- let footprintSum = ref(0.)
1763
- notFilledPartitions.contents->Array.forEach(fs =>
1764
- footprintSum := footprintSum.contents +. reservedByPartition->Dict.getUnsafe(fs.partitionId)
1777
+ // consuming fresh budget (every emit consumes at least 1), so the
1778
+ // not-filled set drains on its own and the loop also stops once the whole
1779
+ // budget is poured.
1780
+ let notFilledPartitions = ref(inRangeStates)
1781
+ let remainingBudget = ref(rangeItemsTarget)
1782
+ while notFilledPartitions.contents->Array.length > 0 && remainingBudget.contents > 0. {
1783
+ let level = waterLevel(
1784
+ ~budget=remainingBudget.contents,
1785
+ ~footprints=notFilledPartitions.contents->Array.map(fs =>
1786
+ reservedByPartition->Dict.getUnsafe(fs.partitionId)
1787
+ ),
1765
1788
  )
1766
- let line =
1767
- (rangeItemsTarget -. reservedFromRange.contents +. footprintSum.contents) /. n->Int.toFloat
1768
1789
  let next = []
1769
1790
  notFilledPartitions.contents->Array.forEach(fs => {
1770
1791
  let reserved = reservedByPartition->Dict.getUnsafe(fs.partitionId)
1771
- let budget = line -. reserved
1792
+ let budget = level -. reserved
1772
1793
  if budget > 0. {
1773
1794
  let consumed = emitQueries(fs, ~budget)
1774
1795
  reservedByPartition->Dict.set(fs.partitionId, reserved +. consumed)
1775
- reservedFromRange := reservedFromRange.contents +. consumed
1796
+ remainingBudget := remainingBudget.contents -. consumed
1776
1797
  if fs->isInRange {
1777
1798
  next->Array.push(fs)->ignore
1778
1799
  }