@peerbit/shared-log 10.0.6 → 10.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/benchmark/get-samples.js +1 -1
- package/dist/benchmark/get-samples.js.map +1 -1
- package/dist/benchmark/utils.js +1 -1
- package/dist/benchmark/utils.js.map +1 -1
- package/dist/src/index.d.ts +15 -10
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +138 -65
- package/dist/src/index.js.map +1 -1
- package/dist/src/ranges.d.ts +95 -11
- package/dist/src/ranges.d.ts.map +1 -1
- package/dist/src/ranges.js +437 -83
- package/dist/src/ranges.js.map +1 -1
- package/dist/src/replication-domain-hash.d.ts +2 -2
- package/dist/src/replication-domain-hash.d.ts.map +1 -1
- package/dist/src/replication-domain-hash.js +2 -17
- package/dist/src/replication-domain-hash.js.map +1 -1
- package/dist/src/replication-domain-time.d.ts +7 -2
- package/dist/src/replication-domain-time.d.ts.map +1 -1
- package/dist/src/replication-domain-time.js +7 -12
- package/dist/src/replication-domain-time.js.map +1 -1
- package/dist/src/replication-domain.d.ts +3 -20
- package/dist/src/replication-domain.d.ts.map +1 -1
- package/dist/src/replication-domain.js +0 -33
- package/dist/src/replication-domain.js.map +1 -1
- package/package.json +4 -4
- package/src/index.ts +205 -107
- package/src/ranges.ts +669 -127
- package/src/replication-domain-hash.ts +16 -29
- package/src/replication-domain-time.ts +46 -40
- package/src/replication-domain.ts +7 -59
package/src/index.ts
CHANGED
|
@@ -63,6 +63,7 @@ import {
|
|
|
63
63
|
} from "./exchange-heads.js";
|
|
64
64
|
import {
|
|
65
65
|
MAX_U32,
|
|
66
|
+
MAX_U64,
|
|
66
67
|
type NumberFromType,
|
|
67
68
|
type Numbers,
|
|
68
69
|
bytesToNumber,
|
|
@@ -75,6 +76,8 @@ import {
|
|
|
75
76
|
type EntryReplicated,
|
|
76
77
|
EntryReplicatedU32,
|
|
77
78
|
EntryReplicatedU64,
|
|
79
|
+
type ReplicationChange,
|
|
80
|
+
type ReplicationChanges,
|
|
78
81
|
ReplicationIntent,
|
|
79
82
|
type ReplicationRangeIndexable,
|
|
80
83
|
ReplicationRangeIndexableU32,
|
|
@@ -82,9 +85,11 @@ import {
|
|
|
82
85
|
ReplicationRangeMessage,
|
|
83
86
|
SyncStatus,
|
|
84
87
|
appromixateCoverage,
|
|
88
|
+
countCoveringRangesSameOwner,
|
|
89
|
+
debounceAggregationChanges,
|
|
90
|
+
getAllMergeCandiates,
|
|
85
91
|
getCoverSet,
|
|
86
92
|
getSamples,
|
|
87
|
-
iHaveCoveringRange,
|
|
88
93
|
isMatured,
|
|
89
94
|
isReplicationRangeMessage,
|
|
90
95
|
mergeRanges,
|
|
@@ -102,11 +107,8 @@ import {
|
|
|
102
107
|
} from "./replication-domain-time.js";
|
|
103
108
|
import {
|
|
104
109
|
type ExtractDomainArgs,
|
|
105
|
-
type ReplicationChange,
|
|
106
|
-
type ReplicationChanges,
|
|
107
110
|
type ReplicationDomain,
|
|
108
|
-
|
|
109
|
-
mergeReplicationChanges,
|
|
111
|
+
type ReplicationDomainConstructor,
|
|
110
112
|
} from "./replication-domain.js";
|
|
111
113
|
import {
|
|
112
114
|
AbsoluteReplicas,
|
|
@@ -143,6 +145,7 @@ export {
|
|
|
143
145
|
EntryReplicatedU32,
|
|
144
146
|
EntryReplicatedU64,
|
|
145
147
|
};
|
|
148
|
+
export { MAX_U32, MAX_U64, type NumberFromType };
|
|
146
149
|
export const logger = loggerFn({ module: "shared-log" });
|
|
147
150
|
|
|
148
151
|
const getLatestEntry = (
|
|
@@ -183,7 +186,6 @@ export type FixedReplicationOptions = {
|
|
|
183
186
|
factor: number | bigint | "all" | "right";
|
|
184
187
|
strict?: boolean; // if true, only this range will be replicated
|
|
185
188
|
offset?: number | bigint;
|
|
186
|
-
syncStatus?: SyncStatus;
|
|
187
189
|
};
|
|
188
190
|
|
|
189
191
|
export type ReplicationOptions<R extends "u32" | "u64" = any> =
|
|
@@ -249,7 +251,7 @@ interface IndexableDomain<R extends "u32" | "u64"> {
|
|
|
249
251
|
properties: {
|
|
250
252
|
id?: Uint8Array;
|
|
251
253
|
offset: NumberFromType<R>;
|
|
252
|
-
|
|
254
|
+
width: NumberFromType<R>;
|
|
253
255
|
mode?: ReplicationIntent;
|
|
254
256
|
timestamp?: bigint;
|
|
255
257
|
} & ({ publicKeyHash: string } | { publicKey: PublicSignKey }),
|
|
@@ -299,7 +301,7 @@ export type SharedLogOptions<
|
|
|
299
301
|
waitForPruneDelay?: number;
|
|
300
302
|
distributionDebounceTime?: number;
|
|
301
303
|
compatibility?: number;
|
|
302
|
-
domain?: D
|
|
304
|
+
domain?: ReplicationDomainConstructor<D>;
|
|
303
305
|
};
|
|
304
306
|
|
|
305
307
|
export const DEFAULT_MIN_REPLICAS = 2;
|
|
@@ -374,6 +376,8 @@ export class SharedLog<
|
|
|
374
376
|
private _replicationRangeIndex!: Index<ReplicationRangeIndexable<R>>;
|
|
375
377
|
private _entryCoordinatesIndex!: Index<EntryReplicated<R>>;
|
|
376
378
|
private coordinateToHash!: Cache<string>;
|
|
379
|
+
private recentlyRebalanced!: Cache<string>;
|
|
380
|
+
|
|
377
381
|
private uniqueReplicators!: Set<string>;
|
|
378
382
|
|
|
379
383
|
/* private _totalParticipation!: number; */
|
|
@@ -459,7 +463,7 @@ export class SharedLog<
|
|
|
459
463
|
private _requestIPruneResponseReplicatorSet!: Map<string, Set<string>>; // tracks entry hash to peer hash
|
|
460
464
|
|
|
461
465
|
private replicationChangeDebounceFn!: ReturnType<
|
|
462
|
-
typeof debounceAggregationChanges
|
|
466
|
+
typeof debounceAggregationChanges<ReplicationRangeIndexable<R>>
|
|
463
467
|
>;
|
|
464
468
|
|
|
465
469
|
// regular distribution checks
|
|
@@ -557,13 +561,11 @@ export class SharedLog<
|
|
|
557
561
|
{
|
|
558
562
|
reset,
|
|
559
563
|
checkDuplicates,
|
|
560
|
-
syncStatus,
|
|
561
564
|
announce,
|
|
562
565
|
mergeSegments,
|
|
563
566
|
}: {
|
|
564
567
|
reset?: boolean;
|
|
565
568
|
checkDuplicates?: boolean;
|
|
566
|
-
syncStatus?: SyncStatus;
|
|
567
569
|
mergeSegments?: boolean;
|
|
568
570
|
announce?: (
|
|
569
571
|
msg: AddedReplicationSegmentMessage | AllReplicatingSegmentsMessage,
|
|
@@ -574,7 +576,8 @@ export class SharedLog<
|
|
|
574
576
|
if (isUnreplicationOptions(options)) {
|
|
575
577
|
await this.unreplicate();
|
|
576
578
|
} else {
|
|
577
|
-
let
|
|
579
|
+
let rangesToReplicate: ReplicationRangeIndexable<R>[] = [];
|
|
580
|
+
let rangesToUnreplicate: ReplicationRangeIndexable<R>[] = [];
|
|
578
581
|
|
|
579
582
|
if (options == null) {
|
|
580
583
|
options = {};
|
|
@@ -595,11 +598,11 @@ export class SharedLog<
|
|
|
595
598
|
// not allowed
|
|
596
599
|
return;
|
|
597
600
|
}
|
|
598
|
-
|
|
601
|
+
rangesToReplicate = [maybeRange];
|
|
599
602
|
|
|
600
603
|
offsetWasProvided = true;
|
|
601
604
|
} else if (isReplicationRangeMessage(options)) {
|
|
602
|
-
|
|
605
|
+
rangesToReplicate = [
|
|
603
606
|
options.toReplicationRangeIndexable(this.node.identity.publicKey),
|
|
604
607
|
];
|
|
605
608
|
|
|
@@ -650,60 +653,62 @@ export class SharedLog<
|
|
|
650
653
|
let factorDenormalized = !normalized
|
|
651
654
|
? factor
|
|
652
655
|
: this.indexableDomain.numbers.denormalize(factor as number);
|
|
653
|
-
|
|
656
|
+
rangesToReplicate.push(
|
|
654
657
|
new this.indexableDomain.constructorRange({
|
|
655
658
|
id: rangeArg.id,
|
|
656
659
|
// @ts-ignore
|
|
657
660
|
offset: offset,
|
|
658
661
|
// @ts-ignore
|
|
659
|
-
|
|
662
|
+
width: (factor === "all"
|
|
660
663
|
? fullWidth
|
|
661
664
|
: factor === "right"
|
|
662
665
|
? // @ts-ignore
|
|
663
666
|
fullWidth - offset
|
|
664
667
|
: factorDenormalized) as NumberFromType<R>,
|
|
665
|
-
/* typeof factor === "number"
|
|
666
|
-
? factor
|
|
667
|
-
: factor === "all"
|
|
668
|
-
? width
|
|
669
|
-
// @ts-ignore
|
|
670
|
-
: width - offset, */
|
|
671
668
|
publicKeyHash: this.node.identity.publicKey.hashcode(),
|
|
672
669
|
mode: rangeArg.strict
|
|
673
670
|
? ReplicationIntent.Strict
|
|
674
671
|
: ReplicationIntent.NonStrict, // automatic means that this range might be reused later for dynamic replication behaviour
|
|
675
672
|
timestamp: timestamp ?? BigInt(+new Date()),
|
|
676
673
|
}),
|
|
677
|
-
/* new ReplicationRangeIndexable({
|
|
678
|
-
id: rangeArg.id,
|
|
679
|
-
normalized,
|
|
680
|
-
offset: offset,
|
|
681
|
-
length:
|
|
682
|
-
typeof factor === "number"
|
|
683
|
-
? factor
|
|
684
|
-
: factor === "all"
|
|
685
|
-
? width
|
|
686
|
-
// @ts-ignore
|
|
687
|
-
: width - offset,
|
|
688
|
-
publicKeyHash: this.node.identity.publicKey.hashcode(),
|
|
689
|
-
mode: rangeArg.strict
|
|
690
|
-
? ReplicationIntent.Strict
|
|
691
|
-
: ReplicationIntent.NonStrict, // automatic means that this range might be reused later for dynamic replication behaviour
|
|
692
|
-
timestamp: BigInt(+new Date()),
|
|
693
|
-
}), */
|
|
694
674
|
);
|
|
695
675
|
}
|
|
696
676
|
|
|
697
|
-
if (mergeSegments
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
677
|
+
if (mergeSegments) {
|
|
678
|
+
let range =
|
|
679
|
+
rangesToReplicate.length > 1
|
|
680
|
+
? mergeRanges(rangesToReplicate, this.indexableDomain.numbers)
|
|
681
|
+
: rangesToReplicate[0];
|
|
682
|
+
|
|
683
|
+
// also merge segments that are already in the index
|
|
684
|
+
if (this.domain.canMerge) {
|
|
685
|
+
const mergable = await getAllMergeCandiates(
|
|
686
|
+
this.replicationIndex,
|
|
687
|
+
range,
|
|
688
|
+
this.indexableDomain.numbers,
|
|
689
|
+
);
|
|
690
|
+
const mergeableFiltered: ReplicationRangeIndexable<R>[] = [range];
|
|
691
|
+
|
|
692
|
+
for (const mergeCandidate of mergable) {
|
|
693
|
+
if (this.domain.canMerge(mergeCandidate, range)) {
|
|
694
|
+
mergeableFiltered.push(mergeCandidate);
|
|
695
|
+
if (mergeCandidate.idString !== range.idString) {
|
|
696
|
+
rangesToUnreplicate.push(mergeCandidate);
|
|
697
|
+
}
|
|
698
|
+
}
|
|
699
|
+
}
|
|
700
|
+
if (mergeableFiltered.length > 1) {
|
|
701
|
+
range = mergeRanges(
|
|
702
|
+
mergeableFiltered,
|
|
703
|
+
this.indexableDomain.numbers,
|
|
704
|
+
);
|
|
705
|
+
}
|
|
706
|
+
}
|
|
707
|
+
rangesToReplicate = [range];
|
|
703
708
|
}
|
|
704
709
|
}
|
|
705
710
|
|
|
706
|
-
for (const range of
|
|
711
|
+
for (const range of rangesToReplicate) {
|
|
707
712
|
this.oldestOpenTime = Math.min(
|
|
708
713
|
Number(range.timestamp),
|
|
709
714
|
this.oldestOpenTime,
|
|
@@ -717,14 +722,29 @@ export class SharedLog<
|
|
|
717
722
|
// but ({ replicate: 0.5, offset: 0.5 }) means that we want to add a range
|
|
718
723
|
// TODO make behaviour more clear
|
|
719
724
|
}
|
|
720
|
-
await this.startAnnounceReplicating(
|
|
725
|
+
await this.startAnnounceReplicating(rangesToReplicate, {
|
|
721
726
|
reset: resetRanges ?? false,
|
|
722
727
|
checkDuplicates,
|
|
723
728
|
announce,
|
|
724
|
-
syncStatus,
|
|
725
729
|
});
|
|
726
730
|
|
|
727
|
-
|
|
731
|
+
if (rangesToUnreplicate.length > 0) {
|
|
732
|
+
await this.removeReplicationRanges(
|
|
733
|
+
rangesToUnreplicate,
|
|
734
|
+
this.node.identity.publicKey,
|
|
735
|
+
);
|
|
736
|
+
|
|
737
|
+
await this.rpc.send(
|
|
738
|
+
new StoppedReplicating({
|
|
739
|
+
segmentIds: rangesToUnreplicate.map((x) => x.id),
|
|
740
|
+
}),
|
|
741
|
+
{
|
|
742
|
+
priority: 1,
|
|
743
|
+
},
|
|
744
|
+
);
|
|
745
|
+
}
|
|
746
|
+
|
|
747
|
+
return rangesToReplicate;
|
|
728
748
|
}
|
|
729
749
|
}
|
|
730
750
|
|
|
@@ -773,7 +793,6 @@ export class SharedLog<
|
|
|
773
793
|
| ReplicationRangeMessage<any>[]
|
|
774
794
|
| ReplicationOptions<R>
|
|
775
795
|
| undefined = undefined;
|
|
776
|
-
let syncStatus = SyncStatus.Unsynced;
|
|
777
796
|
|
|
778
797
|
if (rangeOrEntry instanceof ReplicationRangeMessage) {
|
|
779
798
|
range = rangeOrEntry;
|
|
@@ -783,7 +802,6 @@ export class SharedLog<
|
|
|
783
802
|
offset: await this.domain.fromEntry(rangeOrEntry),
|
|
784
803
|
normalized: false,
|
|
785
804
|
};
|
|
786
|
-
syncStatus = SyncStatus.Synced; /// we already have the entries
|
|
787
805
|
} else if (Array.isArray(rangeOrEntry)) {
|
|
788
806
|
let ranges: (ReplicationRangeMessage<any> | FixedReplicationOptions)[] =
|
|
789
807
|
[];
|
|
@@ -793,9 +811,8 @@ export class SharedLog<
|
|
|
793
811
|
factor: 1,
|
|
794
812
|
offset: await this.domain.fromEntry(entry),
|
|
795
813
|
normalized: false,
|
|
814
|
+
strict: true,
|
|
796
815
|
});
|
|
797
|
-
|
|
798
|
-
syncStatus = SyncStatus.Synced; /// we already have the entries
|
|
799
816
|
} else {
|
|
800
817
|
ranges.push(entry);
|
|
801
818
|
}
|
|
@@ -805,18 +822,34 @@ export class SharedLog<
|
|
|
805
822
|
range = rangeOrEntry ?? true;
|
|
806
823
|
}
|
|
807
824
|
|
|
808
|
-
return this._replicate(range,
|
|
825
|
+
return this._replicate(range, options);
|
|
809
826
|
}
|
|
810
827
|
|
|
811
|
-
async unreplicate(rangeOrEntry?: Entry<T> |
|
|
812
|
-
let
|
|
828
|
+
async unreplicate(rangeOrEntry?: Entry<T> | { id: Uint8Array }[]) {
|
|
829
|
+
let segmentIds: Uint8Array<ArrayBufferLike>[];
|
|
813
830
|
if (rangeOrEntry instanceof Entry) {
|
|
814
|
-
range = {
|
|
831
|
+
let range: FixedReplicationOptions = {
|
|
815
832
|
factor: 1,
|
|
816
833
|
offset: await this.domain.fromEntry(rangeOrEntry),
|
|
817
834
|
};
|
|
818
|
-
|
|
819
|
-
|
|
835
|
+
const indexed = this.replicationIndex.iterate({
|
|
836
|
+
query: {
|
|
837
|
+
width: 1,
|
|
838
|
+
start1: range.offset /* ,
|
|
839
|
+
hash: this.node.identity.publicKey.hashcode(), */,
|
|
840
|
+
},
|
|
841
|
+
});
|
|
842
|
+
segmentIds = (await indexed.all()).map((x) => x.id.key as Uint8Array);
|
|
843
|
+
if (segmentIds.length === 0) {
|
|
844
|
+
logger.warn("No segment found to unreplicate");
|
|
845
|
+
return;
|
|
846
|
+
}
|
|
847
|
+
} else if (Array.isArray(rangeOrEntry)) {
|
|
848
|
+
segmentIds = rangeOrEntry.map((x) => x.id);
|
|
849
|
+
if (segmentIds.length === 0) {
|
|
850
|
+
logger.warn("No segment found to unreplicate");
|
|
851
|
+
return;
|
|
852
|
+
}
|
|
820
853
|
} else {
|
|
821
854
|
this._isReplicating = false;
|
|
822
855
|
this._isAdaptiveReplicating = false;
|
|
@@ -830,15 +863,14 @@ export class SharedLog<
|
|
|
830
863
|
throw new Error("Unsupported when adaptive replicating");
|
|
831
864
|
}
|
|
832
865
|
|
|
833
|
-
const
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
await this.removeReplicationRange(segmentIds, this.node.identity.publicKey);
|
|
866
|
+
const rangesToRemove = await this.resolveReplicationRangesFromIdsAndKey(
|
|
867
|
+
segmentIds,
|
|
868
|
+
this.node.identity.publicKey,
|
|
869
|
+
);
|
|
870
|
+
await this.removeReplicationRanges(
|
|
871
|
+
rangesToRemove,
|
|
872
|
+
this.node.identity.publicKey,
|
|
873
|
+
);
|
|
842
874
|
await this.rpc.send(new StoppedReplicating({ segmentIds }), {
|
|
843
875
|
priority: 1,
|
|
844
876
|
});
|
|
@@ -881,10 +913,12 @@ export class SharedLog<
|
|
|
881
913
|
}
|
|
882
914
|
}
|
|
883
915
|
|
|
916
|
+
const timestamp = BigInt(+new Date());
|
|
884
917
|
for (const x of deleted) {
|
|
885
918
|
this.replicationChangeDebounceFn.add({
|
|
886
919
|
range: x.value,
|
|
887
920
|
type: "removed",
|
|
921
|
+
timestamp,
|
|
888
922
|
});
|
|
889
923
|
}
|
|
890
924
|
|
|
@@ -917,7 +951,10 @@ export class SharedLog<
|
|
|
917
951
|
: +new Date();
|
|
918
952
|
}
|
|
919
953
|
|
|
920
|
-
private async
|
|
954
|
+
private async resolveReplicationRangesFromIdsAndKey(
|
|
955
|
+
ids: Uint8Array[],
|
|
956
|
+
from: PublicSignKey,
|
|
957
|
+
) {
|
|
921
958
|
let idMatcher = new Or(
|
|
922
959
|
ids.map((x) => new ByteMatchQuery({ key: "id", value: x })),
|
|
923
960
|
);
|
|
@@ -929,10 +966,17 @@ export class SharedLog<
|
|
|
929
966
|
});
|
|
930
967
|
|
|
931
968
|
let query = new And([idMatcher, identityMatcher]);
|
|
932
|
-
|
|
969
|
+
return (await this.replicationIndex.iterate({ query }).all()).map(
|
|
970
|
+
(x) => x.value,
|
|
971
|
+
);
|
|
972
|
+
}
|
|
973
|
+
private async removeReplicationRanges(
|
|
974
|
+
ranges: ReplicationRangeIndexable<R>[],
|
|
975
|
+
from: PublicSignKey,
|
|
976
|
+
) {
|
|
933
977
|
const pendingMaturity = this.pendingMaturity.get(from.hashcode());
|
|
934
978
|
if (pendingMaturity) {
|
|
935
|
-
for (const id of
|
|
979
|
+
for (const id of ranges) {
|
|
936
980
|
const info = pendingMaturity.get(id.toString());
|
|
937
981
|
if (info) {
|
|
938
982
|
clearTimeout(info.timeout);
|
|
@@ -944,7 +988,11 @@ export class SharedLog<
|
|
|
944
988
|
}
|
|
945
989
|
}
|
|
946
990
|
|
|
947
|
-
await this.replicationIndex.del({
|
|
991
|
+
await this.replicationIndex.del({
|
|
992
|
+
query: new Or(
|
|
993
|
+
ranges.map((x) => new ByteMatchQuery({ key: "id", value: x.id })),
|
|
994
|
+
),
|
|
995
|
+
});
|
|
948
996
|
|
|
949
997
|
const otherSegmentsIterator = this.replicationIndex.iterate(
|
|
950
998
|
{ query: { hash: from.hashcode() } },
|
|
@@ -974,15 +1022,17 @@ export class SharedLog<
|
|
|
974
1022
|
{
|
|
975
1023
|
reset,
|
|
976
1024
|
checkDuplicates,
|
|
977
|
-
|
|
1025
|
+
timestamp: ts,
|
|
1026
|
+
}: { reset?: boolean; checkDuplicates?: boolean; timestamp?: number } = {},
|
|
978
1027
|
) {
|
|
979
1028
|
if (this._isTrustedReplicator && !(await this._isTrustedReplicator(from))) {
|
|
980
1029
|
return undefined;
|
|
981
1030
|
}
|
|
982
1031
|
let isNewReplicator = false;
|
|
1032
|
+
let timestamp = BigInt(ts ?? +new Date());
|
|
983
1033
|
|
|
984
|
-
let diffs: ReplicationChanges
|
|
985
|
-
let deleted: ReplicationRangeIndexable<
|
|
1034
|
+
let diffs: ReplicationChanges<ReplicationRangeIndexable<R>>;
|
|
1035
|
+
let deleted: ReplicationRangeIndexable<R>[] | undefined = undefined;
|
|
986
1036
|
if (reset) {
|
|
987
1037
|
deleted = (
|
|
988
1038
|
await this.replicationIndex
|
|
@@ -998,10 +1048,10 @@ export class SharedLog<
|
|
|
998
1048
|
|
|
999
1049
|
diffs = [
|
|
1000
1050
|
...deleted.map((x) => {
|
|
1001
|
-
return { range: x, type: "removed" as const };
|
|
1051
|
+
return { range: x, type: "removed" as const, timestamp };
|
|
1002
1052
|
}),
|
|
1003
1053
|
...ranges.map((x) => {
|
|
1004
|
-
return { range: x, type: "added" as const };
|
|
1054
|
+
return { range: x, type: "added" as const, timestamp };
|
|
1005
1055
|
}),
|
|
1006
1056
|
];
|
|
1007
1057
|
|
|
@@ -1031,7 +1081,9 @@ export class SharedLog<
|
|
|
1031
1081
|
|
|
1032
1082
|
// TODO also deduplicate/de-overlap among the ranges that ought to be inserted?
|
|
1033
1083
|
for (const range of ranges) {
|
|
1034
|
-
if (
|
|
1084
|
+
if (
|
|
1085
|
+
!(await countCoveringRangesSameOwner(this.replicationIndex, range))
|
|
1086
|
+
) {
|
|
1035
1087
|
deduplicated.push(range);
|
|
1036
1088
|
}
|
|
1037
1089
|
}
|
|
@@ -1042,19 +1094,31 @@ export class SharedLog<
|
|
|
1042
1094
|
existingMap.set(result.value.idString, result.value);
|
|
1043
1095
|
}
|
|
1044
1096
|
|
|
1045
|
-
let changes: ReplicationChanges = ranges
|
|
1097
|
+
let changes: ReplicationChanges<ReplicationRangeIndexable<R>> = ranges
|
|
1046
1098
|
.map((x) => {
|
|
1047
1099
|
const prev = existingMap.get(x.idString);
|
|
1048
1100
|
if (prev) {
|
|
1049
1101
|
if (prev.equalRange(x)) {
|
|
1050
|
-
return
|
|
1102
|
+
return [];
|
|
1051
1103
|
}
|
|
1052
|
-
return
|
|
1104
|
+
return [
|
|
1105
|
+
{
|
|
1106
|
+
range: prev,
|
|
1107
|
+
timestamp: x.timestamp - 1n,
|
|
1108
|
+
prev,
|
|
1109
|
+
type: "replaced" as const,
|
|
1110
|
+
},
|
|
1111
|
+
{
|
|
1112
|
+
range: x,
|
|
1113
|
+
timestamp: x.timestamp,
|
|
1114
|
+
type: "added" as const,
|
|
1115
|
+
},
|
|
1116
|
+
];
|
|
1053
1117
|
} else {
|
|
1054
|
-
return { range: x, type: "added" };
|
|
1118
|
+
return { range: x, timestamp: x.timestamp, type: "added" as const };
|
|
1055
1119
|
}
|
|
1056
1120
|
})
|
|
1057
|
-
.
|
|
1121
|
+
.flat() as ReplicationChanges<ReplicationRangeIndexable<R>>;
|
|
1058
1122
|
diffs = changes;
|
|
1059
1123
|
}
|
|
1060
1124
|
|
|
@@ -1065,7 +1129,7 @@ export class SharedLog<
|
|
|
1065
1129
|
let isAllMature = true;
|
|
1066
1130
|
|
|
1067
1131
|
for (const diff of diffs) {
|
|
1068
|
-
if (diff.type === "added"
|
|
1132
|
+
if (diff.type === "added") {
|
|
1069
1133
|
/* if (this.closed) {
|
|
1070
1134
|
return;
|
|
1071
1135
|
} */
|
|
@@ -1104,7 +1168,7 @@ export class SharedLog<
|
|
|
1104
1168
|
}),
|
|
1105
1169
|
);
|
|
1106
1170
|
|
|
1107
|
-
this.replicationChangeDebounceFn.add(diff); // we need to call this here because the outcom of findLeaders will be different when some ranges become mature, i.e. some of data we own might be prunable!
|
|
1171
|
+
this.replicationChangeDebounceFn.add({ ...diff, matured: true }); // we need to call this here because the outcom of findLeaders will be different when some ranges become mature, i.e. some of data we own might be prunable!
|
|
1108
1172
|
pendingRanges.delete(diff.range.idString);
|
|
1109
1173
|
if (pendingRanges.size === 0) {
|
|
1110
1174
|
this.pendingMaturity.delete(diff.range.hash);
|
|
@@ -1123,7 +1187,7 @@ export class SharedLog<
|
|
|
1123
1187
|
});
|
|
1124
1188
|
}
|
|
1125
1189
|
}
|
|
1126
|
-
} else {
|
|
1190
|
+
} else if (diff.type === "removed") {
|
|
1127
1191
|
const pendingFromPeer = this.pendingMaturity.get(diff.range.hash);
|
|
1128
1192
|
if (pendingFromPeer) {
|
|
1129
1193
|
const prev = pendingFromPeer.get(diff.range.idString);
|
|
@@ -1136,6 +1200,7 @@ export class SharedLog<
|
|
|
1136
1200
|
}
|
|
1137
1201
|
}
|
|
1138
1202
|
}
|
|
1203
|
+
// else replaced, do nothing
|
|
1139
1204
|
}
|
|
1140
1205
|
|
|
1141
1206
|
if (reset) {
|
|
@@ -1394,10 +1459,10 @@ export class SharedLog<
|
|
|
1394
1459
|
|
|
1395
1460
|
// TODO types
|
|
1396
1461
|
this.domain = options?.domain
|
|
1397
|
-
? (options.domain as
|
|
1462
|
+
? (options.domain(this) as D)
|
|
1398
1463
|
: (createReplicationDomainHash(
|
|
1399
1464
|
options?.compatibility && options?.compatibility < 10 ? "u32" : "u64",
|
|
1400
|
-
) as D);
|
|
1465
|
+
)(this) as D);
|
|
1401
1466
|
this.indexableDomain = createIndexableDomainFromResolution(
|
|
1402
1467
|
this.domain.resolution,
|
|
1403
1468
|
);
|
|
@@ -1406,6 +1471,7 @@ export class SharedLog<
|
|
|
1406
1471
|
this._pendingIHave = new Map();
|
|
1407
1472
|
this.latestReplicationInfoMessage = new Map();
|
|
1408
1473
|
this.coordinateToHash = new Cache<string>({ max: 1e6, ttl: 1e4 });
|
|
1474
|
+
this.recentlyRebalanced = new Cache<string>({ max: 1e4, ttl: 1e5 });
|
|
1409
1475
|
|
|
1410
1476
|
this.uniqueReplicators = new Set();
|
|
1411
1477
|
|
|
@@ -1476,7 +1542,9 @@ export class SharedLog<
|
|
|
1476
1542
|
this._requestIPruneSent = new Map();
|
|
1477
1543
|
this._requestIPruneResponseReplicatorSet = new Map();
|
|
1478
1544
|
|
|
1479
|
-
this.replicationChangeDebounceFn = debounceAggregationChanges
|
|
1545
|
+
this.replicationChangeDebounceFn = debounceAggregationChanges<
|
|
1546
|
+
ReplicationRangeIndexable<R>
|
|
1547
|
+
>(
|
|
1480
1548
|
(change) =>
|
|
1481
1549
|
this.onReplicationChange(change).then(() =>
|
|
1482
1550
|
this.rebalanceParticipationDebounced?.(),
|
|
@@ -1800,17 +1868,18 @@ export class SharedLog<
|
|
|
1800
1868
|
) {
|
|
1801
1869
|
let roleAge = options?.roleAge ?? (await this.getDefaultMinRoleAge());
|
|
1802
1870
|
let eager = options?.eager ?? false;
|
|
1803
|
-
const range = await this.domain.fromArgs(args
|
|
1804
|
-
|
|
1871
|
+
const range = await this.domain.fromArgs(args);
|
|
1872
|
+
|
|
1873
|
+
const width =
|
|
1874
|
+
range.length ??
|
|
1875
|
+
(await minimumWidthToCover<R>(
|
|
1876
|
+
this.replicas.min.getValue(this),
|
|
1877
|
+
this.indexableDomain.numbers,
|
|
1878
|
+
));
|
|
1805
1879
|
const set = await getCoverSet<R>({
|
|
1806
1880
|
peers: this.replicationIndex,
|
|
1807
1881
|
start: range.offset,
|
|
1808
|
-
widthToCoverScaled:
|
|
1809
|
-
range.length ??
|
|
1810
|
-
(await minimumWidthToCover<R>(
|
|
1811
|
-
this.replicas.min.getValue(this),
|
|
1812
|
-
this.indexableDomain.numbers,
|
|
1813
|
-
)),
|
|
1882
|
+
widthToCoverScaled: width,
|
|
1814
1883
|
roleAge,
|
|
1815
1884
|
eager,
|
|
1816
1885
|
numbers: this.indexableDomain.numbers,
|
|
@@ -1837,6 +1906,7 @@ export class SharedLog<
|
|
|
1837
1906
|
|
|
1838
1907
|
this.distributeQueue?.clear();
|
|
1839
1908
|
this.coordinateToHash.clear();
|
|
1909
|
+
this.recentlyRebalanced.clear();
|
|
1840
1910
|
this.uniqueReplicators.clear();
|
|
1841
1911
|
|
|
1842
1912
|
this._closeController.abort();
|
|
@@ -2378,7 +2448,11 @@ export class SharedLog<
|
|
|
2378
2448
|
x.toReplicationRangeIndexable(context.from!),
|
|
2379
2449
|
),
|
|
2380
2450
|
context.from!,
|
|
2381
|
-
{
|
|
2451
|
+
{
|
|
2452
|
+
reset,
|
|
2453
|
+
checkDuplicates: true,
|
|
2454
|
+
timestamp: Number(context.timestamp),
|
|
2455
|
+
},
|
|
2382
2456
|
);
|
|
2383
2457
|
|
|
2384
2458
|
/* await this._modifyReplicators(msg.role, context.from!); */
|
|
@@ -2403,7 +2477,19 @@ export class SharedLog<
|
|
|
2403
2477
|
return;
|
|
2404
2478
|
}
|
|
2405
2479
|
|
|
2406
|
-
await this.
|
|
2480
|
+
const rangesToRemove = await this.resolveReplicationRangesFromIdsAndKey(
|
|
2481
|
+
msg.segmentIds,
|
|
2482
|
+
context.from,
|
|
2483
|
+
);
|
|
2484
|
+
await this.removeReplicationRanges(rangesToRemove, context.from);
|
|
2485
|
+
const timestamp = BigInt(+new Date());
|
|
2486
|
+
for (const range of rangesToRemove) {
|
|
2487
|
+
this.replicationChangeDebounceFn.add({
|
|
2488
|
+
range,
|
|
2489
|
+
type: "removed",
|
|
2490
|
+
timestamp,
|
|
2491
|
+
});
|
|
2492
|
+
}
|
|
2407
2493
|
} else {
|
|
2408
2494
|
throw new Error("Unexpected message");
|
|
2409
2495
|
}
|
|
@@ -2563,6 +2649,7 @@ export class SharedLog<
|
|
|
2563
2649
|
| boolean
|
|
2564
2650
|
| {
|
|
2565
2651
|
mergeSegments?: boolean;
|
|
2652
|
+
assumeSynced?: boolean;
|
|
2566
2653
|
};
|
|
2567
2654
|
},
|
|
2568
2655
|
): Promise<void> {
|
|
@@ -2605,11 +2692,20 @@ export class SharedLog<
|
|
|
2605
2692
|
|
|
2606
2693
|
const persistCoordinate = async (entry: Entry<T>) => {
|
|
2607
2694
|
const minReplicas = decodeReplicas(entry).getValue(this);
|
|
2608
|
-
await this.findLeaders(
|
|
2695
|
+
const leaders = await this.findLeaders(
|
|
2609
2696
|
await this.createCoordinates(entry, minReplicas),
|
|
2610
2697
|
entry,
|
|
2611
2698
|
{ persist: {} },
|
|
2612
2699
|
);
|
|
2700
|
+
|
|
2701
|
+
if (
|
|
2702
|
+
options?.replicate &&
|
|
2703
|
+
typeof options.replicate !== "boolean" &&
|
|
2704
|
+
options.replicate.assumeSynced
|
|
2705
|
+
) {
|
|
2706
|
+
// make sure we dont start to initate syncing process outwards for this entry
|
|
2707
|
+
this.addPeersToGidPeerHistory(entry.meta.gid, leaders.keys());
|
|
2708
|
+
}
|
|
2613
2709
|
};
|
|
2614
2710
|
let entriesToPersist: Entry<T>[] = [];
|
|
2615
2711
|
let joinOptions = {
|
|
@@ -3468,9 +3564,10 @@ export class SharedLog<
|
|
|
3468
3564
|
this._gidPeersHistory.clear();
|
|
3469
3565
|
}
|
|
3470
3566
|
|
|
3567
|
+
const timestamp = BigInt(+new Date());
|
|
3471
3568
|
this.onReplicationChange(
|
|
3472
3569
|
(await this.getAllReplicationSegments()).map((x) => {
|
|
3473
|
-
return { range: x, type: "added" };
|
|
3570
|
+
return { range: x, type: "added", timestamp };
|
|
3474
3571
|
}),
|
|
3475
3572
|
);
|
|
3476
3573
|
}
|
|
@@ -3480,7 +3577,9 @@ export class SharedLog<
|
|
|
3480
3577
|
}
|
|
3481
3578
|
|
|
3482
3579
|
async onReplicationChange(
|
|
3483
|
-
changeOrChanges:
|
|
3580
|
+
changeOrChanges:
|
|
3581
|
+
| ReplicationChanges<ReplicationRangeIndexable<R>>
|
|
3582
|
+
| ReplicationChanges<ReplicationRangeIndexable<R>>[],
|
|
3484
3583
|
) {
|
|
3485
3584
|
/**
|
|
3486
3585
|
* TODO use information of new joined/leaving peer to create a subset of heads
|
|
@@ -3493,7 +3592,6 @@ export class SharedLog<
|
|
|
3493
3592
|
|
|
3494
3593
|
await this.log.trim();
|
|
3495
3594
|
|
|
3496
|
-
const change = mergeReplicationChanges(changeOrChanges);
|
|
3497
3595
|
const changed = false;
|
|
3498
3596
|
|
|
3499
3597
|
try {
|
|
@@ -3503,13 +3601,13 @@ export class SharedLog<
|
|
|
3503
3601
|
> = new Map();
|
|
3504
3602
|
|
|
3505
3603
|
for await (const entryReplicated of toRebalance<R>(
|
|
3506
|
-
|
|
3604
|
+
changeOrChanges,
|
|
3507
3605
|
this.entryCoordinatesIndex,
|
|
3606
|
+
this.recentlyRebalanced,
|
|
3508
3607
|
)) {
|
|
3509
3608
|
if (this.closed) {
|
|
3510
3609
|
break;
|
|
3511
3610
|
}
|
|
3512
|
-
|
|
3513
3611
|
let oldPeersSet = this._gidPeersHistory.get(entryReplicated.gid);
|
|
3514
3612
|
let isLeader = false;
|
|
3515
3613
|
|
|
@@ -3660,7 +3758,7 @@ export class SharedLog<
|
|
|
3660
3758
|
// TODO can not reuse old range, since it will (potentially) affect the index because of sideeffects
|
|
3661
3759
|
dynamicRange = new this.indexableDomain.constructorRange({
|
|
3662
3760
|
offset: dynamicRange.start1,
|
|
3663
|
-
|
|
3761
|
+
width: this.indexableDomain.numbers.denormalize(newFactor),
|
|
3664
3762
|
publicKeyHash: dynamicRange.hash,
|
|
3665
3763
|
id: dynamicRange.id,
|
|
3666
3764
|
mode: dynamicRange.mode,
|
|
@@ -3729,7 +3827,7 @@ export class SharedLog<
|
|
|
3729
3827
|
if (!range) {
|
|
3730
3828
|
range = new this.indexableDomain.constructorRange({
|
|
3731
3829
|
offset: this.getDynamicRangeOffset(),
|
|
3732
|
-
|
|
3830
|
+
width: this.indexableDomain.numbers.zero,
|
|
3733
3831
|
publicKeyHash: this.node.identity.publicKey.hashcode(),
|
|
3734
3832
|
mode: ReplicationIntent.NonStrict,
|
|
3735
3833
|
timestamp: BigInt(+new Date()),
|