envio 3.6.1 → 3.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.d.ts +197 -9
- package/package.json +6 -6
- package/src/Batch.res +64 -25
- package/src/Batch.res.mjs +69 -46
- package/src/ChainFetching.res +13 -17
- package/src/ChainFetching.res.mjs +9 -14
- package/src/ChainState.res +160 -119
- package/src/ChainState.res.mjs +75 -46
- package/src/ChainState.resi +15 -12
- package/src/Config.res +9 -2
- package/src/Config.res.mjs +4 -4
- package/src/Core.res +5 -0
- package/src/Ecosystem.res +0 -1
- package/src/Envio.res +16 -3
- package/src/EventConfigBuilder.res +155 -40
- package/src/EventConfigBuilder.res.mjs +76 -24
- package/src/EventProcessing.res +4 -8
- package/src/EventProcessing.res.mjs +4 -4
- package/src/EventUtils.res +3 -47
- package/src/FetchState.res +124 -104
- package/src/FetchState.res.mjs +116 -100
- package/src/HandlerRegister.res +108 -2
- package/src/HandlerRegister.res.mjs +59 -10
- package/src/HandlerRegister.resi +4 -0
- package/src/Internal.res +84 -24
- package/src/Internal.res.mjs +33 -2
- package/src/LazyLoader.res +12 -11
- package/src/LazyLoader.res.mjs +13 -7
- package/src/Main.res +22 -9
- package/src/Main.res.mjs +18 -5
- package/src/MemoryStorage.res +724 -0
- package/src/MemoryStorage.res.mjs +601 -0
- package/src/PgStorage.res +18 -7
- package/src/PgStorage.res.mjs +14 -11
- package/src/ReorgDetection.res +0 -222
- package/src/ReorgDetection.res.mjs +0 -163
- package/src/Rollback.res +14 -15
- package/src/Rollback.res.mjs +4 -5
- package/src/SimulateItems.res +2 -2
- package/src/Utils.res +3 -0
- package/src/bindings/ClickHouse.res +16 -11
- package/src/bindings/ClickHouse.res.mjs +10 -10
- package/src/bindings/Vitest.res +1 -1
- package/src/sources/BlockStore.res +115 -5
- package/src/sources/BlockStore.res.mjs +25 -0
- package/src/sources/Evm.res +0 -1
- package/src/sources/Evm.res.mjs +0 -1
- package/src/sources/EvmHyperSyncSource.res +19 -84
- package/src/sources/EvmHyperSyncSource.res.mjs +24 -60
- package/src/sources/Fuel.res +24 -4
- package/src/sources/Fuel.res.mjs +23 -3
- package/src/sources/FuelHyperSync.res +8 -3
- package/src/sources/FuelHyperSync.res.mjs +5 -4
- package/src/sources/FuelHyperSync.resi +3 -1
- package/src/sources/FuelHyperSyncClient.res +7 -10
- package/src/sources/FuelHyperSyncSource.res +18 -45
- package/src/sources/FuelHyperSyncSource.res.mjs +18 -35
- package/src/sources/HyperSync.res +49 -229
- package/src/sources/HyperSync.res.mjs +74 -191
- package/src/sources/HyperSync.resi +11 -35
- package/src/sources/HyperSyncClient.res +9 -79
- package/src/sources/HyperSyncClient.res.mjs +2 -6
- package/src/sources/RequestStat.res +5 -0
- package/src/sources/RequestStat.res.mjs +2 -0
- package/src/sources/RpcSource.res +149 -43
- package/src/sources/RpcSource.res.mjs +104 -41
- package/src/sources/SimulateSource.res +8 -5
- package/src/sources/SimulateSource.res.mjs +6 -6
- package/src/sources/Source.res +89 -16
- package/src/sources/Source.res.mjs +50 -1
- package/src/sources/SourceManager.res +255 -50
- package/src/sources/SourceManager.res.mjs +149 -55
- package/src/sources/SourceManager.resi +2 -6
- package/src/sources/Svm.res +0 -7
- package/src/sources/Svm.res.mjs +0 -8
- package/src/sources/SvmHyperSyncClient.res +21 -16
- package/src/sources/SvmHyperSyncClient.res.mjs +3 -4
- package/src/sources/SvmHyperSyncSource.res +25 -117
- package/src/sources/SvmHyperSyncSource.res.mjs +8 -121
- package/svm.schema.json +3 -2
package/src/FetchState.res
CHANGED
|
@@ -1,12 +1,5 @@
|
|
|
1
1
|
type indexingAddress = Internal.indexingContract
|
|
2
2
|
|
|
3
|
-
type blockNumberAndTimestamp = {
|
|
4
|
-
blockNumber: int,
|
|
5
|
-
blockTimestamp: int,
|
|
6
|
-
}
|
|
7
|
-
|
|
8
|
-
type blockNumberAndLogIndex = {blockNumber: int, logIndex: int}
|
|
9
|
-
|
|
10
3
|
type selection = {
|
|
11
4
|
onEventRegistrations: array<Internal.onEventRegistration>,
|
|
12
5
|
// Whether the partition's queries are built from its own address list
|
|
@@ -70,7 +63,7 @@ type pendingQuery = {
|
|
|
70
63
|
// Stores latestFetchedBlock when query completes. Only needed to persist
|
|
71
64
|
// timestamp while earlier queries are still pending before updating
|
|
72
65
|
// the partition's latestFetchedBlock.
|
|
73
|
-
mutable fetchedBlock: option<
|
|
66
|
+
mutable fetchedBlock: option<int>,
|
|
74
67
|
}
|
|
75
68
|
|
|
76
69
|
/**
|
|
@@ -83,7 +76,7 @@ type partition = {
|
|
|
83
76
|
id: string,
|
|
84
77
|
// The block number of the latest fetched query
|
|
85
78
|
// which added all its events to the queue
|
|
86
|
-
latestFetchedBlock:
|
|
79
|
+
latestFetchedBlock: int,
|
|
87
80
|
selection: selection,
|
|
88
81
|
// The partition's slice of the chain's address index. Ordered by
|
|
89
82
|
// (effectiveStartBlock, address) inside Rust, so a partition layout is a pure
|
|
@@ -109,7 +102,7 @@ type partition = {
|
|
|
109
102
|
// cap is useful density evidence while saying nothing about source capacity.
|
|
110
103
|
// None distinguishes a new partition from a real zero-density observation.
|
|
111
104
|
eventDensity: option<float>,
|
|
112
|
-
// Tracks the latestFetchedBlock
|
|
105
|
+
// Tracks the latestFetchedBlock of the most recent response
|
|
113
106
|
// that updated sourceRangeCapacity. Prevents degradation of the chunking
|
|
114
107
|
// heuristic when parallel query responses arrive out of order.
|
|
115
108
|
latestSourceRangeCapacityUpdateBlock: int,
|
|
@@ -270,8 +263,8 @@ module OptimizedPartitions = {
|
|
|
270
263
|
) => {
|
|
271
264
|
let combinedAddresses = p1.addresses->AddressSet.merge(p2.addresses)
|
|
272
265
|
|
|
273
|
-
let p1Below = p1.latestFetchedBlock
|
|
274
|
-
let p2Below = p2.latestFetchedBlock
|
|
266
|
+
let p1Below = p1.latestFetchedBlock < potentialMergeBlock
|
|
267
|
+
let p2Below = p2.latestFetchedBlock < potentialMergeBlock
|
|
275
268
|
|
|
276
269
|
// Build the continuing partition (at potentialMergeBlock with combined addresses),
|
|
277
270
|
// collecting completed partitions (with mergeBlock) along the way
|
|
@@ -303,7 +296,7 @@ module OptimizedPartitions = {
|
|
|
303
296
|
id: newId,
|
|
304
297
|
dynamicContract: Some(contractName),
|
|
305
298
|
selection: p1.selection,
|
|
306
|
-
latestFetchedBlock:
|
|
299
|
+
latestFetchedBlock: potentialMergeBlock,
|
|
307
300
|
mergeBlock: None,
|
|
308
301
|
addresses: p1.addresses, // replaced below
|
|
309
302
|
mutPendingQueries: [],
|
|
@@ -348,8 +341,7 @@ module OptimizedPartitions = {
|
|
|
348
341
|
// quering the same block range multiple times
|
|
349
342
|
let tooFarBlockRange = 20_000
|
|
350
343
|
|
|
351
|
-
let ascSortFn = (a, b) =>
|
|
352
|
-
Int.compare(a.latestFetchedBlock.blockNumber, b.latestFetchedBlock.blockNumber)
|
|
344
|
+
let ascSortFn = (a, b) => Int.compare(a.latestFetchedBlock, b.latestFetchedBlock)
|
|
353
345
|
|
|
354
346
|
// Contracts a standing address-free partition already fetches client-side.
|
|
355
347
|
// Addresses registered for them after that partition passed get a normal
|
|
@@ -374,11 +366,11 @@ module OptimizedPartitions = {
|
|
|
374
366
|
// bounded query claims its whole toBlock. An in-flight unbounded query has no
|
|
375
367
|
// ceiling at all, so nothing is safe to stop a catch-up partition at yet.
|
|
376
368
|
let getAnchorSafeBlock = (p: partition) => {
|
|
377
|
-
let safeRef = ref(Some(p.latestFetchedBlock
|
|
369
|
+
let safeRef = ref(Some(p.latestFetchedBlock))
|
|
378
370
|
p.mutPendingQueries->Array.forEach(pq =>
|
|
379
371
|
switch (safeRef.contents, pq) {
|
|
380
372
|
| (None, _) => ()
|
|
381
|
-
| (Some(safe), {fetchedBlock: Some(
|
|
373
|
+
| (Some(safe), {fetchedBlock: Some(blockNumber)}) =>
|
|
382
374
|
if blockNumber > safe {
|
|
383
375
|
safeRef := Some(blockNumber)
|
|
384
376
|
}
|
|
@@ -446,7 +438,7 @@ module OptimizedPartitions = {
|
|
|
446
438
|
// response to ever remove it on. A retired partition still awaiting a
|
|
447
439
|
// response stays until it lands.
|
|
448
440
|
| {mergeBlock: Some(mergeBlock)} =>
|
|
449
|
-
if p.latestFetchedBlock
|
|
441
|
+
if p.latestFetchedBlock < mergeBlock || p->isFetching {
|
|
450
442
|
newPartitions->Array.push(p)->ignore
|
|
451
443
|
}
|
|
452
444
|
// Since it's not a dynamic contract partition,
|
|
@@ -461,7 +453,7 @@ module OptimizedPartitions = {
|
|
|
461
453
|
let potentialMergeBlock = switch p.mutPendingQueries->Utils.Array.last {
|
|
462
454
|
| Some({isChunk: true, toBlock: Some(toBlock)}) => Some(toBlock)
|
|
463
455
|
| Some(_) => None // unbounded query -- can't merge
|
|
464
|
-
| None => Some(p.latestFetchedBlock
|
|
456
|
+
| None => Some(p.latestFetchedBlock)
|
|
465
457
|
}
|
|
466
458
|
switch potentialMergeBlock {
|
|
467
459
|
| None => newPartitions->Array.push(p)->ignore
|
|
@@ -567,13 +559,13 @@ module OptimizedPartitions = {
|
|
|
567
559
|
switch anchorSafeBlocks->Utils.Dict.dangerouslyGetNonOption(contractName) {
|
|
568
560
|
| None => anchored->Array.push(p)->ignore
|
|
569
561
|
| Some(anchorSafeBlock) =>
|
|
570
|
-
if p.latestFetchedBlock
|
|
562
|
+
if p.latestFetchedBlock < anchorSafeBlock {
|
|
571
563
|
anchored->Array.push({...p, mergeBlock: Some(anchorSafeBlock)})->ignore
|
|
572
564
|
} else if p->isFetching {
|
|
573
565
|
// Caught up, but a response is still owed: keep it until that
|
|
574
566
|
// lands rather than dropping the range it owns.
|
|
575
567
|
anchored
|
|
576
|
-
->Array.push({...p, mergeBlock: Some(p.latestFetchedBlock
|
|
568
|
+
->Array.push({...p, mergeBlock: Some(p.latestFetchedBlock)})
|
|
577
569
|
->ignore
|
|
578
570
|
}
|
|
579
571
|
}
|
|
@@ -612,7 +604,7 @@ module OptimizedPartitions = {
|
|
|
612
604
|
@inline
|
|
613
605
|
let consumeFetchedQueries = (
|
|
614
606
|
mutPendingQueries: array<pendingQuery>,
|
|
615
|
-
~initialLatestFetchedBlock:
|
|
607
|
+
~initialLatestFetchedBlock: int,
|
|
616
608
|
) => {
|
|
617
609
|
let latestFetchedBlock = ref(initialLatestFetchedBlock)
|
|
618
610
|
|
|
@@ -621,7 +613,7 @@ module OptimizedPartitions = {
|
|
|
621
613
|
while canContinue.contents {
|
|
622
614
|
switch mutPendingQueries->Array.get(consumedCount.contents) {
|
|
623
615
|
| Some({fetchedBlock: Some(fetchedBlock), fromBlock})
|
|
624
|
-
if fromBlock <= latestFetchedBlock.contents
|
|
616
|
+
if fromBlock <= latestFetchedBlock.contents + 1 =>
|
|
625
617
|
latestFetchedBlock := fetchedBlock
|
|
626
618
|
consumedCount := consumedCount.contents + 1
|
|
627
619
|
| _ => canContinue := false
|
|
@@ -662,7 +654,7 @@ module OptimizedPartitions = {
|
|
|
662
654
|
~query,
|
|
663
655
|
~knownHeight,
|
|
664
656
|
~itemsCount,
|
|
665
|
-
~latestFetchedBlock:
|
|
657
|
+
~latestFetchedBlock: int,
|
|
666
658
|
) =>
|
|
667
659
|
optimizedPartitions->handleQueryResponseForPartition(
|
|
668
660
|
~p=optimizedPartitions->getOrThrow(~partitionId=query.partitionId),
|
|
@@ -678,7 +670,7 @@ module OptimizedPartitions = {
|
|
|
678
670
|
~query,
|
|
679
671
|
~knownHeight,
|
|
680
672
|
~itemsCount,
|
|
681
|
-
~latestFetchedBlock:
|
|
673
|
+
~latestFetchedBlock: int,
|
|
682
674
|
) => {
|
|
683
675
|
let mutEntities = optimizedPartitions.entities->Utils.Dict.shallowCopy
|
|
684
676
|
|
|
@@ -686,7 +678,7 @@ module OptimizedPartitions = {
|
|
|
686
678
|
let pendingQuery = getPendingQueryOrThrow(p, ~fromBlock=query.fromBlock)
|
|
687
679
|
pendingQuery.fetchedBlock = Some(latestFetchedBlock)
|
|
688
680
|
|
|
689
|
-
let blockRange = latestFetchedBlock
|
|
681
|
+
let blockRange = latestFetchedBlock - query.fromBlock + 1
|
|
690
682
|
// Update density for every response, independently from whether this range
|
|
691
683
|
// is valid evidence of source capacity. A cap hit is still useful density
|
|
692
684
|
// evidence because it reports items returned across the scanned range.
|
|
@@ -704,13 +696,13 @@ module OptimizedPartitions = {
|
|
|
704
696
|
// responses arrive out of order (e.g. earlier query with smaller range
|
|
705
697
|
// arriving after a later query with bigger range).
|
|
706
698
|
let shouldUpdateSourceRangeCapacity =
|
|
707
|
-
latestFetchedBlock
|
|
699
|
+
latestFetchedBlock > p.latestSourceRangeCapacityUpdateBlock &&
|
|
708
700
|
switch query.toBlock {
|
|
709
701
|
| None =>
|
|
710
702
|
// Don't update source capacity when very close to the head.
|
|
711
|
-
latestFetchedBlock
|
|
703
|
+
latestFetchedBlock < knownHeight - 10
|
|
712
704
|
| Some(queryToBlock) =>
|
|
713
|
-
if latestFetchedBlock
|
|
705
|
+
if latestFetchedBlock < queryToBlock {
|
|
714
706
|
// Partial response is direct capacity evidence — unless it was
|
|
715
707
|
// truncated by our own itemsTarget cap: that reflects the
|
|
716
708
|
// reservation we asked for, not what the server could return. A
|
|
@@ -748,7 +740,7 @@ module OptimizedPartitions = {
|
|
|
748
740
|
// the last of them has landed.
|
|
749
741
|
let partitionReachedMergeBlock =
|
|
750
742
|
switch p.mergeBlock {
|
|
751
|
-
| Some(mergeBlock) => updatedLatestFetchedBlock
|
|
743
|
+
| Some(mergeBlock) => updatedLatestFetchedBlock >= mergeBlock
|
|
752
744
|
| None => false
|
|
753
745
|
} &&
|
|
754
746
|
!(p->isFetching)
|
|
@@ -772,7 +764,7 @@ module OptimizedPartitions = {
|
|
|
772
764
|
prevSourceRangeCapacity: updatedPrevSourceRangeCapacity,
|
|
773
765
|
eventDensity: updatedEventDensity,
|
|
774
766
|
latestSourceRangeCapacityUpdateBlock: shouldUpdateSourceRangeCapacity
|
|
775
|
-
? latestFetchedBlock
|
|
767
|
+
? latestFetchedBlock
|
|
776
768
|
: p.latestSourceRangeCapacityUpdateBlock,
|
|
777
769
|
}
|
|
778
770
|
|
|
@@ -789,10 +781,8 @@ module OptimizedPartitions = {
|
|
|
789
781
|
let idx = ids->Array.indexOf(p.id)
|
|
790
782
|
let isAfter = jdx =>
|
|
791
783
|
jdx < count &&
|
|
792
|
-
(
|
|
793
|
-
|
|
794
|
-
).latestFetchedBlock.blockNumber <
|
|
795
|
-
updatedMainPartition.latestFetchedBlock.blockNumber
|
|
784
|
+
(mutEntities->Dict.getUnsafe(ids->Array.getUnsafe(jdx))).latestFetchedBlock <
|
|
785
|
+
updatedMainPartition.latestFetchedBlock
|
|
796
786
|
if isAfter(idx + 1) {
|
|
797
787
|
let reordered = ids->Array.copy
|
|
798
788
|
let jdx = ref(idx)
|
|
@@ -862,9 +852,9 @@ let bufferBlockNumber = ({latestOnBlockBlockNumber, optimizedPartitions}: t) =>
|
|
|
862
852
|
switch optimizedPartitions->OptimizedPartitions.getLatestFullyFetchedBlock {
|
|
863
853
|
| None => latestOnBlockBlockNumber
|
|
864
854
|
| Some(latestFullyFetchedBlock) =>
|
|
865
|
-
latestOnBlockBlockNumber < latestFullyFetchedBlock
|
|
855
|
+
latestOnBlockBlockNumber < latestFullyFetchedBlock
|
|
866
856
|
? latestOnBlockBlockNumber
|
|
867
|
-
: latestFullyFetchedBlock
|
|
857
|
+
: latestFullyFetchedBlock
|
|
868
858
|
}
|
|
869
859
|
}
|
|
870
860
|
|
|
@@ -874,16 +864,10 @@ let bufferBlockNumber = ({latestOnBlockBlockNumber, optimizedPartitions}: t) =>
|
|
|
874
864
|
@inline
|
|
875
865
|
let bufferBlock = ({optimizedPartitions, latestOnBlockBlockNumber}: t) => {
|
|
876
866
|
switch optimizedPartitions->OptimizedPartitions.getLatestFullyFetchedBlock {
|
|
877
|
-
| None =>
|
|
878
|
-
blockNumber: latestOnBlockBlockNumber,
|
|
879
|
-
blockTimestamp: 0,
|
|
880
|
-
}
|
|
867
|
+
| None => latestOnBlockBlockNumber
|
|
881
868
|
| Some(latestFullyFetchedBlock) =>
|
|
882
|
-
latestOnBlockBlockNumber < latestFullyFetchedBlock
|
|
883
|
-
?
|
|
884
|
-
blockNumber: latestOnBlockBlockNumber,
|
|
885
|
-
blockTimestamp: 0,
|
|
886
|
-
}
|
|
869
|
+
latestOnBlockBlockNumber < latestFullyFetchedBlock
|
|
870
|
+
? latestOnBlockBlockNumber
|
|
887
871
|
: latestFullyFetchedBlock
|
|
888
872
|
}
|
|
889
873
|
}
|
|
@@ -916,31 +900,87 @@ let getRegistrationIndex = (item: Internal.item): int =>
|
|
|
916
900
|
| Block({onBlockRegistration}) => onBlockRegistration.index
|
|
917
901
|
}
|
|
918
902
|
|
|
919
|
-
//
|
|
920
|
-
//
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
903
|
+
// Lexicographic order on two call paths, parent before child: `[1]` precedes
|
|
904
|
+
// `[1, 0]`, which is the order the runtime executed them in.
|
|
905
|
+
let comparePath = (a: array<int>, b: array<int>): int => {
|
|
906
|
+
let la = a->Array.length
|
|
907
|
+
let lb = b->Array.length
|
|
908
|
+
let shared = la < lb ? la : lb
|
|
909
|
+
let i = ref(0)
|
|
910
|
+
let result = ref(0)
|
|
911
|
+
while result.contents === 0 && i.contents < shared {
|
|
912
|
+
let x = a->Array.getUnsafe(i.contents)
|
|
913
|
+
let y = b->Array.getUnsafe(i.contents)
|
|
914
|
+
if x !== y {
|
|
915
|
+
result := (x < y ? -1 : 1)
|
|
916
|
+
}
|
|
917
|
+
i := i.contents + 1
|
|
918
|
+
}
|
|
919
|
+
if result.contents !== 0 {
|
|
920
|
+
result.contents
|
|
921
|
+
} else if la === lb {
|
|
922
|
+
0
|
|
923
|
+
} else if la < lb {
|
|
924
|
+
-1
|
|
925
|
+
} else {
|
|
926
|
+
1
|
|
927
|
+
}
|
|
928
|
+
}
|
|
929
|
+
|
|
930
|
+
// Cold tail of `compareBufferItem`, out of line so the block/kind/log-index
|
|
931
|
+
// comparison that every merge step runs stays small enough for V8 to inline.
|
|
932
|
+
let compareTiebreak = (a: Internal.item, b: Internal.item): int => {
|
|
933
|
+
// Two items an ecosystem's scalar key can't separate: instructions of one
|
|
934
|
+
// Solana transaction, ordered by their position in its CPI tree.
|
|
935
|
+
let byPath = switch (a->Internal.getItemOrderPath, b->Internal.getItemOrderPath) {
|
|
936
|
+
| (Value(pa), Value(pb)) => comparePath(pa, pb)
|
|
937
|
+
| _ => 0
|
|
938
|
+
}
|
|
939
|
+
if byPath !== 0 {
|
|
940
|
+
byPath
|
|
941
|
+
} else {
|
|
942
|
+
let ia = a->getRegistrationIndex
|
|
943
|
+
let ib = b->getRegistrationIndex
|
|
944
|
+
ia < ib ? -1 : ia > ib ? 1 : 0
|
|
945
|
+
}
|
|
946
|
+
}
|
|
947
|
+
|
|
948
|
+
// Total order on buffer items: block, then item kind, then the ecosystem's
|
|
949
|
+
// within-block order, then registration index. Returns a plain int (-1/0/1)
|
|
950
|
+
// with explicit field comparisons so it can be called directly from the
|
|
951
|
+
// merge/insertion loops below — no Array.sort callback, no allocated key. `0`
|
|
952
|
+
// means a true duplicate: the same log routed to the same registration (two
|
|
953
|
+
// registrations for one log differ by index and are kept).
|
|
954
|
+
//
|
|
955
|
+
// Kind outranks the log index so that every event of a block precedes that
|
|
956
|
+
// block's handlers by construction. A sentinel log index for block items
|
|
957
|
+
// would put the same guarantee at the mercy of how large an ecosystem's key
|
|
958
|
+
// grows — which is how SVM, keyed by transaction index, came to run slot
|
|
959
|
+
// handlers ahead of most of a slot's instructions.
|
|
924
960
|
let compareBufferItem = (a: Internal.item, b: Internal.item): int => {
|
|
925
961
|
let ba = a->Internal.getItemBlockNumber
|
|
926
962
|
let bb = b->Internal.getItemBlockNumber
|
|
927
963
|
if ba != bb {
|
|
928
964
|
ba < bb ? -1 : 1
|
|
929
965
|
} else {
|
|
930
|
-
let
|
|
931
|
-
let
|
|
932
|
-
if
|
|
933
|
-
|
|
966
|
+
let ka = a->Internal.getItemKind
|
|
967
|
+
let kb = b->Internal.getItemKind
|
|
968
|
+
if ka !== kb {
|
|
969
|
+
ka < kb ? -1 : 1
|
|
934
970
|
} else {
|
|
935
|
-
let
|
|
936
|
-
let
|
|
937
|
-
|
|
971
|
+
let la = a->Internal.getItemLogIndex
|
|
972
|
+
let lb = b->Internal.getItemLogIndex
|
|
973
|
+
if la != lb {
|
|
974
|
+
la < lb ? -1 : 1
|
|
975
|
+
} else {
|
|
976
|
+
compareTiebreak(a, b)
|
|
977
|
+
}
|
|
938
978
|
}
|
|
939
979
|
}
|
|
940
980
|
}
|
|
941
981
|
|
|
942
982
|
// Merge a maybe-unsorted `newItems` run into the already-sorted, already-deduped
|
|
943
|
-
// `buffer`, dropping items equal on
|
|
983
|
+
// `buffer`, dropping items equal on every component of `compareBufferItem`.
|
|
944
984
|
// Single linear pass over both runs after ordering `newItems` in place; every
|
|
945
985
|
// comparison is a direct `compareBufferItem` call (V8 inlines it) rather than a
|
|
946
986
|
// callback through `Array.sort`.
|
|
@@ -996,9 +1036,6 @@ let mergeIntoBuffer = (buffer: array<Internal.item>, newItems: array<Internal.it
|
|
|
996
1036
|
merged
|
|
997
1037
|
}
|
|
998
1038
|
|
|
999
|
-
// Some big number which should be bigger than any log index
|
|
1000
|
-
let blockItemLogIndex = 16777216
|
|
1001
|
-
|
|
1002
1039
|
// Appends Block items produced by the onBlock handlers for every block in
|
|
1003
1040
|
// (fromBlock, maxBlockNumber] into mutItems and returns the new
|
|
1004
1041
|
// latestOnBlockBlockNumber pointer. maxOnBlockBufferSize bounds how many items
|
|
@@ -1045,7 +1082,6 @@ let appendOnBlockItems = (
|
|
|
1045
1082
|
Block({
|
|
1046
1083
|
onBlockRegistration,
|
|
1047
1084
|
blockNumber,
|
|
1048
|
-
logIndex: blockItemLogIndex + onBlockRegistration.index,
|
|
1049
1085
|
}),
|
|
1050
1086
|
)
|
|
1051
1087
|
newItemsCounter := newItemsCounter.contents + 1
|
|
@@ -1078,7 +1114,7 @@ let updateInternal = (
|
|
|
1078
1114
|
| None => fetchState.buffer
|
|
1079
1115
|
}
|
|
1080
1116
|
|
|
1081
|
-
// onBlock items are generated as their own ascending
|
|
1117
|
+
// onBlock items are generated as their own ascending run and
|
|
1082
1118
|
// folded into `base` by the single merge below.
|
|
1083
1119
|
let blockItems = []
|
|
1084
1120
|
let latestOnBlockBlockNumber = switch fetchState.onBlockRegistrations {
|
|
@@ -1093,7 +1129,7 @@ let updateInternal = (
|
|
|
1093
1129
|
| None =>
|
|
1094
1130
|
switch optimizedPartitions->OptimizedPartitions.getLatestFullyFetchedBlock {
|
|
1095
1131
|
| None => knownHeight
|
|
1096
|
-
| Some(latestFullyFetchedBlock) => latestFullyFetchedBlock
|
|
1132
|
+
| Some(latestFullyFetchedBlock) => latestFullyFetchedBlock
|
|
1097
1133
|
}
|
|
1098
1134
|
}
|
|
1099
1135
|
appendOnBlockItems(
|
|
@@ -1164,11 +1200,11 @@ let addClientFilteredContract = (
|
|
|
1164
1200
|
// refetch it. An in-flight open-ended query has no toBlock of its own; it can't
|
|
1165
1201
|
// return past the chain's known height, so that bounds it.
|
|
1166
1202
|
let claimedFetchedBlock = (p: partition, ~knownHeight) =>
|
|
1167
|
-
p.mutPendingQueries->Array.reduce(p.latestFetchedBlock
|
|
1203
|
+
p.mutPendingQueries->Array.reduce(p.latestFetchedBlock, (max, q) =>
|
|
1168
1204
|
Pervasives.max(
|
|
1169
1205
|
max,
|
|
1170
1206
|
switch q.fetchedBlock {
|
|
1171
|
-
| Some(
|
|
1207
|
+
| Some(blockNumber) => blockNumber
|
|
1172
1208
|
| None => q.toBlock->Option.getOr(knownHeight)
|
|
1173
1209
|
},
|
|
1174
1210
|
)
|
|
@@ -1213,7 +1249,7 @@ let collapseClientFilteredContracts = (
|
|
|
1213
1249
|
// Frontiers of the addresses that were never given a server-side partition
|
|
1214
1250
|
// because their contract is already client-filtered. They stand in for the
|
|
1215
1251
|
// partitions that would otherwise have been created only to be absorbed here.
|
|
1216
|
-
~clientFilteredFrontiers: array<
|
|
1252
|
+
~clientFilteredFrontiers: array<int>=[],
|
|
1217
1253
|
~knownHeight: int,
|
|
1218
1254
|
) => {
|
|
1219
1255
|
if clientFilteredContracts->Utils.Set.size === 0 {
|
|
@@ -1232,7 +1268,7 @@ let collapseClientFilteredContracts = (
|
|
|
1232
1268
|
// then handleQueryResponse drops it.
|
|
1233
1269
|
let retire = (p: partition) => {
|
|
1234
1270
|
...p,
|
|
1235
|
-
mergeBlock: Some(p.latestFetchedBlock
|
|
1271
|
+
mergeBlock: Some(p.latestFetchedBlock),
|
|
1236
1272
|
}
|
|
1237
1273
|
let absorb = p => {
|
|
1238
1274
|
absorbedPartitions->Array.push(p)->ignore
|
|
@@ -1301,10 +1337,10 @@ let collapseClientFilteredContracts = (
|
|
|
1301
1337
|
// partition (and any in-progress backfill) untouched.
|
|
1302
1338
|
partitions
|
|
1303
1339
|
} else {
|
|
1304
|
-
let minFrontierRef: ref<option<
|
|
1305
|
-
let considerFrontier = (b:
|
|
1340
|
+
let minFrontierRef: ref<option<int>> = ref(None)
|
|
1341
|
+
let considerFrontier = (b: int) =>
|
|
1306
1342
|
switch minFrontierRef.contents {
|
|
1307
|
-
| Some(m) if m
|
|
1343
|
+
| Some(m) if m <= b => ()
|
|
1308
1344
|
| _ => minFrontierRef := Some(b)
|
|
1309
1345
|
}
|
|
1310
1346
|
absorbedPartitions->Array.forEach(p => considerFrontier(p.latestFetchedBlock))
|
|
@@ -1386,7 +1422,7 @@ let collapseClientFilteredContracts = (
|
|
|
1386
1422
|
// all the partition will ever claim.
|
|
1387
1423
|
let catchUpToBlock = standingOut->claimedFetchedBlock(~knownHeight)
|
|
1388
1424
|
switch minFrontierRef.contents {
|
|
1389
|
-
| Some(minFrontier) if minFrontier
|
|
1425
|
+
| Some(minFrontier) if minFrontier < catchUpToBlock =>
|
|
1390
1426
|
let id = nextPartitionIndexRef.contents->Int.toString
|
|
1391
1427
|
nextPartitionIndexRef := nextPartitionIndexRef.contents + 1
|
|
1392
1428
|
kept
|
|
@@ -1535,10 +1571,7 @@ OptimizedPartitions.t => {
|
|
|
1535
1571
|
// the first group's since groups are ascending.
|
|
1536
1572
|
switch groups->Array.get(0) {
|
|
1537
1573
|
| Some({startBlock}) =>
|
|
1538
|
-
clientFilteredFrontiers->Array.push(
|
|
1539
|
-
blockNumber: Pervasives.max(startBlock - 1, progressBlockNumber),
|
|
1540
|
-
blockTimestamp: 0,
|
|
1541
|
-
})
|
|
1574
|
+
clientFilteredFrontiers->Array.push(Pervasives.max(startBlock - 1, progressBlockNumber))
|
|
1542
1575
|
| None => ()
|
|
1543
1576
|
}
|
|
1544
1577
|
} else {
|
|
@@ -1562,10 +1595,7 @@ OptimizedPartitions.t => {
|
|
|
1562
1595
|
}
|
|
1563
1596
|
}
|
|
1564
1597
|
|
|
1565
|
-
let latestFetchedBlock =
|
|
1566
|
-
blockNumber: Pervasives.max(startBlock - 1, progressBlockNumber),
|
|
1567
|
-
blockTimestamp: 0,
|
|
1568
|
-
}
|
|
1598
|
+
let latestFetchedBlock = Pervasives.max(startBlock - 1, progressBlockNumber)
|
|
1569
1599
|
let remainingRef = ref(countRef.contents)
|
|
1570
1600
|
let chunkOffsetRef = ref(offsetRef.contents)
|
|
1571
1601
|
while remainingRef.contents > 0 {
|
|
@@ -1609,8 +1639,8 @@ OptimizedPartitions.t => {
|
|
|
1609
1639
|
while nextIdx.contents < nonDynamicPartitions->Array.length {
|
|
1610
1640
|
let nextP = nonDynamicPartitions->Array.getUnsafe(nextIdx.contents)
|
|
1611
1641
|
let currentP = currentPRef.contents
|
|
1612
|
-
let currentPBlock = currentP.latestFetchedBlock
|
|
1613
|
-
let nextPBlock = nextP.latestFetchedBlock
|
|
1642
|
+
let currentPBlock = currentP.latestFetchedBlock
|
|
1643
|
+
let nextPBlock = nextP.latestFetchedBlock
|
|
1614
1644
|
|
|
1615
1645
|
let totalCount = currentP.addresses->AddressSet.size + nextP.addresses->AddressSet.size
|
|
1616
1646
|
|
|
@@ -1869,12 +1899,7 @@ Throws if the partition with given query cannot be found (unexpected)
|
|
|
1869
1899
|
|
|
1870
1900
|
newItems are ordered earliest to latest (as they are returned from the worker)
|
|
1871
1901
|
*/
|
|
1872
|
-
let handleQueryResult = (
|
|
1873
|
-
fetchState: t,
|
|
1874
|
-
~query: query,
|
|
1875
|
-
~latestFetchedBlock: blockNumberAndTimestamp,
|
|
1876
|
-
~newItems,
|
|
1877
|
-
): t => {
|
|
1902
|
+
let handleQueryResult = (fetchState: t, ~query: query, ~latestFetchedBlock: int, ~newItems): t => {
|
|
1878
1903
|
fetchState->updateInternal(
|
|
1879
1904
|
~optimizedPartitions=fetchState.optimizedPartitions->OptimizedPartitions.handleQueryResponse(
|
|
1880
1905
|
~query,
|
|
@@ -2104,7 +2129,7 @@ let walkPartitionPending = (
|
|
|
2104
2129
|
let maybeChunkRange = getMinHistoryRange(p)
|
|
2105
2130
|
let pendingCount = p.mutPendingQueries->Array.length
|
|
2106
2131
|
|
|
2107
|
-
let cursor = ref(p.latestFetchedBlock
|
|
2132
|
+
let cursor = ref(p.latestFetchedBlock + 1)
|
|
2108
2133
|
let canContinue = ref(true)
|
|
2109
2134
|
let chunksUsedThisCall = ref(0)
|
|
2110
2135
|
let pqIdx = ref(0)
|
|
@@ -2130,7 +2155,7 @@ let walkPartitionPending = (
|
|
|
2130
2155
|
chunksUsedThisCall := chunksUsedThisCall.contents + (candidates->Array.length - beforeLen)
|
|
2131
2156
|
}
|
|
2132
2157
|
switch pq {
|
|
2133
|
-
| {isChunk: true, toBlock: Some(toBlock), fetchedBlock: Some(
|
|
2158
|
+
| {isChunk: true, toBlock: Some(toBlock), fetchedBlock: Some(blockNumber)}
|
|
2134
2159
|
if blockNumber < toBlock =>
|
|
2135
2160
|
cursor := blockNumber + 1
|
|
2136
2161
|
| {isChunk: true, toBlock: Some(toBlock)} => cursor := toBlock + 1
|
|
@@ -2432,9 +2457,7 @@ let getNextQuery = (
|
|
|
2432
2457
|
}
|
|
2433
2458
|
}
|
|
2434
2459
|
inFlightCounts->Array.setUnsafe(idx, inFlightCount.contents)
|
|
2435
|
-
if
|
|
2436
|
-
p.mutPendingQueries->Array.length > 0 || p.latestFetchedBlock.blockNumber < headBlockNumber
|
|
2437
|
-
) {
|
|
2460
|
+
if p.mutPendingQueries->Array.length > 0 || p.latestFetchedBlock < headBlockNumber {
|
|
2438
2461
|
// Even if there are some partitions waiting for the new block
|
|
2439
2462
|
// We still want to wait for all partitions reaching the head
|
|
2440
2463
|
// because they might update knownHeight in their response
|
|
@@ -2616,10 +2639,7 @@ let make = (
|
|
|
2616
2639
|
~clientFilterAddressThreshold=None,
|
|
2617
2640
|
~isResumed=false,
|
|
2618
2641
|
): t => {
|
|
2619
|
-
let latestFetchedBlock =
|
|
2620
|
-
blockTimestamp: 0,
|
|
2621
|
-
blockNumber: progressBlockNumber,
|
|
2622
|
-
}
|
|
2642
|
+
let latestFetchedBlock = progressBlockNumber
|
|
2623
2643
|
|
|
2624
2644
|
let notDependingOnAddresses = []
|
|
2625
2645
|
let normalRegistrations = []
|
|
@@ -2764,7 +2784,7 @@ let make = (
|
|
|
2764
2784
|
let latestOnBlockBlockNumber = if knownHeight > 0 && onBlockRegistrations->Utils.Array.notEmpty {
|
|
2765
2785
|
let maxBlockNumber = switch optimizedPartitions->OptimizedPartitions.getLatestFullyFetchedBlock {
|
|
2766
2786
|
| None => knownHeight
|
|
2767
|
-
| Some(latestFullyFetchedBlock) => latestFullyFetchedBlock
|
|
2787
|
+
| Some(latestFullyFetchedBlock) => latestFullyFetchedBlock
|
|
2768
2788
|
}
|
|
2769
2789
|
appendOnBlockItems(
|
|
2770
2790
|
~mutItems=buffer,
|
|
@@ -2809,11 +2829,11 @@ let rollbackPendingQueries = (mutPendingQueries: array<pendingQuery>, ~targetBlo
|
|
|
2809
2829
|
let pq = mutPendingQueries->Array.getUnsafe(qIdx)
|
|
2810
2830
|
if pq.fromBlock <= targetBlockNumber {
|
|
2811
2831
|
switch pq.fetchedBlock {
|
|
2812
|
-
| Some(
|
|
2832
|
+
| Some(blockNumber) if blockNumber > targetBlockNumber =>
|
|
2813
2833
|
adjusted
|
|
2814
2834
|
->Array.push({
|
|
2815
2835
|
...pq,
|
|
2816
|
-
fetchedBlock: Some(
|
|
2836
|
+
fetchedBlock: Some(targetBlockNumber),
|
|
2817
2837
|
})
|
|
2818
2838
|
->ignore
|
|
2819
2839
|
| Some(_) => adjusted->Array.push(pq)->ignore
|
|
@@ -2868,8 +2888,8 @@ let rollback = (fetchState: t, ~addressStore: AddressStore.t, ~targetBlockNumber
|
|
|
2868
2888
|
->Array.push({
|
|
2869
2889
|
...p,
|
|
2870
2890
|
id,
|
|
2871
|
-
latestFetchedBlock: p.latestFetchedBlock
|
|
2872
|
-
?
|
|
2891
|
+
latestFetchedBlock: p.latestFetchedBlock > targetBlockNumber
|
|
2892
|
+
? targetBlockNumber
|
|
2873
2893
|
: p.latestFetchedBlock,
|
|
2874
2894
|
// Everything above the target is refetched by whichever partition this
|
|
2875
2895
|
// one was catching up to, so there is nothing left to catch up on past
|
|
@@ -2884,7 +2904,7 @@ let rollback = (fetchState: t, ~addressStore: AddressStore.t, ~targetBlockNumber
|
|
|
2884
2904
|
->ignore
|
|
2885
2905
|
|
|
2886
2906
|
// Non-wildcard with lfb > target: delete, collect addresses for recreation
|
|
2887
|
-
| _ if p.latestFetchedBlock
|
|
2907
|
+
| _ if p.latestFetchedBlock > targetBlockNumber =>
|
|
2888
2908
|
collectForRecreation(p.addresses->AddressSet.filterByRegistrationBlock(targetBlockNumber))
|
|
2889
2909
|
|
|
2890
2910
|
// Non-wildcard with lfb <= target: keep, adjust pending queries and mergeBlock
|