@peerbit/shared-log 10.0.6 → 10.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/index.ts CHANGED
@@ -63,6 +63,7 @@ import {
63
63
  } from "./exchange-heads.js";
64
64
  import {
65
65
  MAX_U32,
66
+ MAX_U64,
66
67
  type NumberFromType,
67
68
  type Numbers,
68
69
  bytesToNumber,
@@ -75,6 +76,8 @@ import {
75
76
  type EntryReplicated,
76
77
  EntryReplicatedU32,
77
78
  EntryReplicatedU64,
79
+ type ReplicationChange,
80
+ type ReplicationChanges,
78
81
  ReplicationIntent,
79
82
  type ReplicationRangeIndexable,
80
83
  ReplicationRangeIndexableU32,
@@ -82,9 +85,11 @@ import {
82
85
  ReplicationRangeMessage,
83
86
  SyncStatus,
84
87
  appromixateCoverage,
88
+ countCoveringRangesSameOwner,
89
+ debounceAggregationChanges,
90
+ getAllMergeCandiates,
85
91
  getCoverSet,
86
92
  getSamples,
87
- iHaveCoveringRange,
88
93
  isMatured,
89
94
  isReplicationRangeMessage,
90
95
  mergeRanges,
@@ -102,11 +107,8 @@ import {
102
107
  } from "./replication-domain-time.js";
103
108
  import {
104
109
  type ExtractDomainArgs,
105
- type ReplicationChange,
106
- type ReplicationChanges,
107
110
  type ReplicationDomain,
108
- debounceAggregationChanges,
109
- mergeReplicationChanges,
111
+ type ReplicationDomainConstructor,
110
112
  } from "./replication-domain.js";
111
113
  import {
112
114
  AbsoluteReplicas,
@@ -143,6 +145,7 @@ export {
143
145
  EntryReplicatedU32,
144
146
  EntryReplicatedU64,
145
147
  };
148
+ export { MAX_U32, MAX_U64, type NumberFromType };
146
149
  export const logger = loggerFn({ module: "shared-log" });
147
150
 
148
151
  const getLatestEntry = (
@@ -183,7 +186,6 @@ export type FixedReplicationOptions = {
183
186
  factor: number | bigint | "all" | "right";
184
187
  strict?: boolean; // if true, only this range will be replicated
185
188
  offset?: number | bigint;
186
- syncStatus?: SyncStatus;
187
189
  };
188
190
 
189
191
  export type ReplicationOptions<R extends "u32" | "u64" = any> =
@@ -249,7 +251,7 @@ interface IndexableDomain<R extends "u32" | "u64"> {
249
251
  properties: {
250
252
  id?: Uint8Array;
251
253
  offset: NumberFromType<R>;
252
- length: NumberFromType<R>;
254
+ width: NumberFromType<R>;
253
255
  mode?: ReplicationIntent;
254
256
  timestamp?: bigint;
255
257
  } & ({ publicKeyHash: string } | { publicKey: PublicSignKey }),
@@ -299,7 +301,7 @@ export type SharedLogOptions<
299
301
  waitForPruneDelay?: number;
300
302
  distributionDebounceTime?: number;
301
303
  compatibility?: number;
302
- domain?: D;
304
+ domain?: ReplicationDomainConstructor<D>;
303
305
  };
304
306
 
305
307
  export const DEFAULT_MIN_REPLICAS = 2;
@@ -374,6 +376,8 @@ export class SharedLog<
374
376
  private _replicationRangeIndex!: Index<ReplicationRangeIndexable<R>>;
375
377
  private _entryCoordinatesIndex!: Index<EntryReplicated<R>>;
376
378
  private coordinateToHash!: Cache<string>;
379
+ private recentlyRebalanced!: Cache<string>;
380
+
377
381
  private uniqueReplicators!: Set<string>;
378
382
 
379
383
  /* private _totalParticipation!: number; */
@@ -459,7 +463,7 @@ export class SharedLog<
459
463
  private _requestIPruneResponseReplicatorSet!: Map<string, Set<string>>; // tracks entry hash to peer hash
460
464
 
461
465
  private replicationChangeDebounceFn!: ReturnType<
462
- typeof debounceAggregationChanges
466
+ typeof debounceAggregationChanges<ReplicationRangeIndexable<R>>
463
467
  >;
464
468
 
465
469
  // regular distribution checks
@@ -557,13 +561,11 @@ export class SharedLog<
557
561
  {
558
562
  reset,
559
563
  checkDuplicates,
560
- syncStatus,
561
564
  announce,
562
565
  mergeSegments,
563
566
  }: {
564
567
  reset?: boolean;
565
568
  checkDuplicates?: boolean;
566
- syncStatus?: SyncStatus;
567
569
  mergeSegments?: boolean;
568
570
  announce?: (
569
571
  msg: AddedReplicationSegmentMessage | AllReplicatingSegmentsMessage,
@@ -574,7 +576,8 @@ export class SharedLog<
574
576
  if (isUnreplicationOptions(options)) {
575
577
  await this.unreplicate();
576
578
  } else {
577
- let ranges: ReplicationRangeIndexable<any>[] = [];
579
+ let rangesToReplicate: ReplicationRangeIndexable<R>[] = [];
580
+ let rangesToUnreplicate: ReplicationRangeIndexable<R>[] = [];
578
581
 
579
582
  if (options == null) {
580
583
  options = {};
@@ -595,11 +598,11 @@ export class SharedLog<
595
598
  // not allowed
596
599
  return;
597
600
  }
598
- ranges = [maybeRange];
601
+ rangesToReplicate = [maybeRange];
599
602
 
600
603
  offsetWasProvided = true;
601
604
  } else if (isReplicationRangeMessage(options)) {
602
- ranges = [
605
+ rangesToReplicate = [
603
606
  options.toReplicationRangeIndexable(this.node.identity.publicKey),
604
607
  ];
605
608
 
@@ -650,60 +653,62 @@ export class SharedLog<
650
653
  let factorDenormalized = !normalized
651
654
  ? factor
652
655
  : this.indexableDomain.numbers.denormalize(factor as number);
653
- ranges.push(
656
+ rangesToReplicate.push(
654
657
  new this.indexableDomain.constructorRange({
655
658
  id: rangeArg.id,
656
659
  // @ts-ignore
657
660
  offset: offset,
658
661
  // @ts-ignore
659
- length: (factor === "all"
662
+ width: (factor === "all"
660
663
  ? fullWidth
661
664
  : factor === "right"
662
665
  ? // @ts-ignore
663
666
  fullWidth - offset
664
667
  : factorDenormalized) as NumberFromType<R>,
665
- /* typeof factor === "number"
666
- ? factor
667
- : factor === "all"
668
- ? width
669
- // @ts-ignore
670
- : width - offset, */
671
668
  publicKeyHash: this.node.identity.publicKey.hashcode(),
672
669
  mode: rangeArg.strict
673
670
  ? ReplicationIntent.Strict
674
671
  : ReplicationIntent.NonStrict, // automatic means that this range might be reused later for dynamic replication behaviour
675
672
  timestamp: timestamp ?? BigInt(+new Date()),
676
673
  }),
677
- /* new ReplicationRangeIndexable({
678
- id: rangeArg.id,
679
- normalized,
680
- offset: offset,
681
- length:
682
- typeof factor === "number"
683
- ? factor
684
- : factor === "all"
685
- ? width
686
- // @ts-ignore
687
- : width - offset,
688
- publicKeyHash: this.node.identity.publicKey.hashcode(),
689
- mode: rangeArg.strict
690
- ? ReplicationIntent.Strict
691
- : ReplicationIntent.NonStrict, // automatic means that this range might be reused later for dynamic replication behaviour
692
- timestamp: BigInt(+new Date()),
693
- }), */
694
674
  );
695
675
  }
696
676
 
697
- if (mergeSegments && ranges.length > 1) {
698
- const mergedSegment = mergeRanges(
699
- ranges,
700
- this.indexableDomain.numbers,
701
- );
702
- ranges = [mergedSegment];
677
+ if (mergeSegments) {
678
+ let range =
679
+ rangesToReplicate.length > 1
680
+ ? mergeRanges(rangesToReplicate, this.indexableDomain.numbers)
681
+ : rangesToReplicate[0];
682
+
683
+ // also merge segments that are already in the index
684
+ if (this.domain.canMerge) {
685
+ const mergable = await getAllMergeCandiates(
686
+ this.replicationIndex,
687
+ range,
688
+ this.indexableDomain.numbers,
689
+ );
690
+ const mergeableFiltered: ReplicationRangeIndexable<R>[] = [range];
691
+
692
+ for (const mergeCandidate of mergable) {
693
+ if (this.domain.canMerge(mergeCandidate, range)) {
694
+ mergeableFiltered.push(mergeCandidate);
695
+ if (mergeCandidate.idString !== range.idString) {
696
+ rangesToUnreplicate.push(mergeCandidate);
697
+ }
698
+ }
699
+ }
700
+ if (mergeableFiltered.length > 1) {
701
+ range = mergeRanges(
702
+ mergeableFiltered,
703
+ this.indexableDomain.numbers,
704
+ );
705
+ }
706
+ }
707
+ rangesToReplicate = [range];
703
708
  }
704
709
  }
705
710
 
706
- for (const range of ranges) {
711
+ for (const range of rangesToReplicate) {
707
712
  this.oldestOpenTime = Math.min(
708
713
  Number(range.timestamp),
709
714
  this.oldestOpenTime,
@@ -717,14 +722,29 @@ export class SharedLog<
717
722
  // but ({ replicate: 0.5, offset: 0.5 }) means that we want to add a range
718
723
  // TODO make behaviour more clear
719
724
  }
720
- await this.startAnnounceReplicating(ranges, {
725
+ await this.startAnnounceReplicating(rangesToReplicate, {
721
726
  reset: resetRanges ?? false,
722
727
  checkDuplicates,
723
728
  announce,
724
- syncStatus,
725
729
  });
726
730
 
727
- return ranges;
731
+ if (rangesToUnreplicate.length > 0) {
732
+ await this.removeReplicationRanges(
733
+ rangesToUnreplicate,
734
+ this.node.identity.publicKey,
735
+ );
736
+
737
+ await this.rpc.send(
738
+ new StoppedReplicating({
739
+ segmentIds: rangesToUnreplicate.map((x) => x.id),
740
+ }),
741
+ {
742
+ priority: 1,
743
+ },
744
+ );
745
+ }
746
+
747
+ return rangesToReplicate;
728
748
  }
729
749
  }
730
750
 
@@ -773,7 +793,6 @@ export class SharedLog<
773
793
  | ReplicationRangeMessage<any>[]
774
794
  | ReplicationOptions<R>
775
795
  | undefined = undefined;
776
- let syncStatus = SyncStatus.Unsynced;
777
796
 
778
797
  if (rangeOrEntry instanceof ReplicationRangeMessage) {
779
798
  range = rangeOrEntry;
@@ -783,7 +802,6 @@ export class SharedLog<
783
802
  offset: await this.domain.fromEntry(rangeOrEntry),
784
803
  normalized: false,
785
804
  };
786
- syncStatus = SyncStatus.Synced; /// we already have the entries
787
805
  } else if (Array.isArray(rangeOrEntry)) {
788
806
  let ranges: (ReplicationRangeMessage<any> | FixedReplicationOptions)[] =
789
807
  [];
@@ -793,9 +811,8 @@ export class SharedLog<
793
811
  factor: 1,
794
812
  offset: await this.domain.fromEntry(entry),
795
813
  normalized: false,
814
+ strict: true,
796
815
  });
797
-
798
- syncStatus = SyncStatus.Synced; /// we already have the entries
799
816
  } else {
800
817
  ranges.push(entry);
801
818
  }
@@ -805,18 +822,34 @@ export class SharedLog<
805
822
  range = rangeOrEntry ?? true;
806
823
  }
807
824
 
808
- return this._replicate(range, { ...options, syncStatus });
825
+ return this._replicate(range, options);
809
826
  }
810
827
 
811
- async unreplicate(rangeOrEntry?: Entry<T> | ReplicationRangeMessage<R>) {
812
- let range: FixedReplicationOptions;
828
+ async unreplicate(rangeOrEntry?: Entry<T> | { id: Uint8Array }[]) {
829
+ let segmentIds: Uint8Array<ArrayBufferLike>[];
813
830
  if (rangeOrEntry instanceof Entry) {
814
- range = {
831
+ let range: FixedReplicationOptions = {
815
832
  factor: 1,
816
833
  offset: await this.domain.fromEntry(rangeOrEntry),
817
834
  };
818
- } else if (rangeOrEntry instanceof ReplicationRangeMessage) {
819
- range = rangeOrEntry;
835
+ const indexed = this.replicationIndex.iterate({
836
+ query: {
837
+ width: 1,
838
+ start1: range.offset /* ,
839
+ hash: this.node.identity.publicKey.hashcode(), */,
840
+ },
841
+ });
842
+ segmentIds = (await indexed.all()).map((x) => x.id.key as Uint8Array);
843
+ if (segmentIds.length === 0) {
844
+ logger.warn("No segment found to unreplicate");
845
+ return;
846
+ }
847
+ } else if (Array.isArray(rangeOrEntry)) {
848
+ segmentIds = rangeOrEntry.map((x) => x.id);
849
+ if (segmentIds.length === 0) {
850
+ logger.warn("No segment found to unreplicate");
851
+ return;
852
+ }
820
853
  } else {
821
854
  this._isReplicating = false;
822
855
  this._isAdaptiveReplicating = false;
@@ -830,15 +863,14 @@ export class SharedLog<
830
863
  throw new Error("Unsupported when adaptive replicating");
831
864
  }
832
865
 
833
- const indexed = this.replicationIndex.iterate({
834
- query: {
835
- width: 1,
836
- start1: range.offset,
837
- },
838
- });
839
-
840
- const segmentIds = (await indexed.all()).map((x) => x.id.key as Uint8Array);
841
- await this.removeReplicationRange(segmentIds, this.node.identity.publicKey);
866
+ const rangesToRemove = await this.resolveReplicationRangesFromIdsAndKey(
867
+ segmentIds,
868
+ this.node.identity.publicKey,
869
+ );
870
+ await this.removeReplicationRanges(
871
+ rangesToRemove,
872
+ this.node.identity.publicKey,
873
+ );
842
874
  await this.rpc.send(new StoppedReplicating({ segmentIds }), {
843
875
  priority: 1,
844
876
  });
@@ -881,10 +913,12 @@ export class SharedLog<
881
913
  }
882
914
  }
883
915
 
916
+ const timestamp = BigInt(+new Date());
884
917
  for (const x of deleted) {
885
918
  this.replicationChangeDebounceFn.add({
886
919
  range: x.value,
887
920
  type: "removed",
921
+ timestamp,
888
922
  });
889
923
  }
890
924
 
@@ -917,7 +951,10 @@ export class SharedLog<
917
951
  : +new Date();
918
952
  }
919
953
 
920
- private async removeReplicationRange(ids: Uint8Array[], from: PublicSignKey) {
954
+ private async resolveReplicationRangesFromIdsAndKey(
955
+ ids: Uint8Array[],
956
+ from: PublicSignKey,
957
+ ) {
921
958
  let idMatcher = new Or(
922
959
  ids.map((x) => new ByteMatchQuery({ key: "id", value: x })),
923
960
  );
@@ -929,10 +966,17 @@ export class SharedLog<
929
966
  });
930
967
 
931
968
  let query = new And([idMatcher, identityMatcher]);
932
-
969
+ return (await this.replicationIndex.iterate({ query }).all()).map(
970
+ (x) => x.value,
971
+ );
972
+ }
973
+ private async removeReplicationRanges(
974
+ ranges: ReplicationRangeIndexable<R>[],
975
+ from: PublicSignKey,
976
+ ) {
933
977
  const pendingMaturity = this.pendingMaturity.get(from.hashcode());
934
978
  if (pendingMaturity) {
935
- for (const id of ids) {
979
+ for (const id of ranges) {
936
980
  const info = pendingMaturity.get(id.toString());
937
981
  if (info) {
938
982
  clearTimeout(info.timeout);
@@ -944,7 +988,11 @@ export class SharedLog<
944
988
  }
945
989
  }
946
990
 
947
- await this.replicationIndex.del({ query });
991
+ await this.replicationIndex.del({
992
+ query: new Or(
993
+ ranges.map((x) => new ByteMatchQuery({ key: "id", value: x.id })),
994
+ ),
995
+ });
948
996
 
949
997
  const otherSegmentsIterator = this.replicationIndex.iterate(
950
998
  { query: { hash: from.hashcode() } },
@@ -974,15 +1022,17 @@ export class SharedLog<
974
1022
  {
975
1023
  reset,
976
1024
  checkDuplicates,
977
- }: { reset?: boolean; checkDuplicates?: boolean } = {},
1025
+ timestamp: ts,
1026
+ }: { reset?: boolean; checkDuplicates?: boolean; timestamp?: number } = {},
978
1027
  ) {
979
1028
  if (this._isTrustedReplicator && !(await this._isTrustedReplicator(from))) {
980
1029
  return undefined;
981
1030
  }
982
1031
  let isNewReplicator = false;
1032
+ let timestamp = BigInt(ts ?? +new Date());
983
1033
 
984
- let diffs: ReplicationChanges;
985
- let deleted: ReplicationRangeIndexable<any>[] | undefined = undefined;
1034
+ let diffs: ReplicationChanges<ReplicationRangeIndexable<R>>;
1035
+ let deleted: ReplicationRangeIndexable<R>[] | undefined = undefined;
986
1036
  if (reset) {
987
1037
  deleted = (
988
1038
  await this.replicationIndex
@@ -998,10 +1048,10 @@ export class SharedLog<
998
1048
 
999
1049
  diffs = [
1000
1050
  ...deleted.map((x) => {
1001
- return { range: x, type: "removed" as const };
1051
+ return { range: x, type: "removed" as const, timestamp };
1002
1052
  }),
1003
1053
  ...ranges.map((x) => {
1004
- return { range: x, type: "added" as const };
1054
+ return { range: x, type: "added" as const, timestamp };
1005
1055
  }),
1006
1056
  ];
1007
1057
 
@@ -1031,7 +1081,9 @@ export class SharedLog<
1031
1081
 
1032
1082
  // TODO also deduplicate/de-overlap among the ranges that ought to be inserted?
1033
1083
  for (const range of ranges) {
1034
- if (!(await iHaveCoveringRange(this.replicationIndex, range))) {
1084
+ if (
1085
+ !(await countCoveringRangesSameOwner(this.replicationIndex, range))
1086
+ ) {
1035
1087
  deduplicated.push(range);
1036
1088
  }
1037
1089
  }
@@ -1042,19 +1094,31 @@ export class SharedLog<
1042
1094
  existingMap.set(result.value.idString, result.value);
1043
1095
  }
1044
1096
 
1045
- let changes: ReplicationChanges = ranges
1097
+ let changes: ReplicationChanges<ReplicationRangeIndexable<R>> = ranges
1046
1098
  .map((x) => {
1047
1099
  const prev = existingMap.get(x.idString);
1048
1100
  if (prev) {
1049
1101
  if (prev.equalRange(x)) {
1050
- return undefined;
1102
+ return [];
1051
1103
  }
1052
- return { range: x, prev, type: "updated" };
1104
+ return [
1105
+ {
1106
+ range: prev,
1107
+ timestamp: x.timestamp - 1n,
1108
+ prev,
1109
+ type: "replaced" as const,
1110
+ },
1111
+ {
1112
+ range: x,
1113
+ timestamp: x.timestamp,
1114
+ type: "added" as const,
1115
+ },
1116
+ ];
1053
1117
  } else {
1054
- return { range: x, type: "added" };
1118
+ return { range: x, timestamp: x.timestamp, type: "added" as const };
1055
1119
  }
1056
1120
  })
1057
- .filter((x) => x != null) as ReplicationChanges;
1121
+ .flat() as ReplicationChanges<ReplicationRangeIndexable<R>>;
1058
1122
  diffs = changes;
1059
1123
  }
1060
1124
 
@@ -1065,7 +1129,7 @@ export class SharedLog<
1065
1129
  let isAllMature = true;
1066
1130
 
1067
1131
  for (const diff of diffs) {
1068
- if (diff.type === "added" || diff.type === "updated") {
1132
+ if (diff.type === "added") {
1069
1133
  /* if (this.closed) {
1070
1134
  return;
1071
1135
  } */
@@ -1104,7 +1168,7 @@ export class SharedLog<
1104
1168
  }),
1105
1169
  );
1106
1170
 
1107
- this.replicationChangeDebounceFn.add(diff); // we need to call this here because the outcom of findLeaders will be different when some ranges become mature, i.e. some of data we own might be prunable!
1171
+ this.replicationChangeDebounceFn.add({ ...diff, matured: true }); // we need to call this here because the outcom of findLeaders will be different when some ranges become mature, i.e. some of data we own might be prunable!
1108
1172
  pendingRanges.delete(diff.range.idString);
1109
1173
  if (pendingRanges.size === 0) {
1110
1174
  this.pendingMaturity.delete(diff.range.hash);
@@ -1123,7 +1187,7 @@ export class SharedLog<
1123
1187
  });
1124
1188
  }
1125
1189
  }
1126
- } else {
1190
+ } else if (diff.type === "removed") {
1127
1191
  const pendingFromPeer = this.pendingMaturity.get(diff.range.hash);
1128
1192
  if (pendingFromPeer) {
1129
1193
  const prev = pendingFromPeer.get(diff.range.idString);
@@ -1136,6 +1200,7 @@ export class SharedLog<
1136
1200
  }
1137
1201
  }
1138
1202
  }
1203
+ // else replaced, do nothing
1139
1204
  }
1140
1205
 
1141
1206
  if (reset) {
@@ -1394,10 +1459,10 @@ export class SharedLog<
1394
1459
 
1395
1460
  // TODO types
1396
1461
  this.domain = options?.domain
1397
- ? (options.domain as any as D)
1462
+ ? (options.domain(this) as D)
1398
1463
  : (createReplicationDomainHash(
1399
1464
  options?.compatibility && options?.compatibility < 10 ? "u32" : "u64",
1400
- ) as D);
1465
+ )(this) as D);
1401
1466
  this.indexableDomain = createIndexableDomainFromResolution(
1402
1467
  this.domain.resolution,
1403
1468
  );
@@ -1406,6 +1471,7 @@ export class SharedLog<
1406
1471
  this._pendingIHave = new Map();
1407
1472
  this.latestReplicationInfoMessage = new Map();
1408
1473
  this.coordinateToHash = new Cache<string>({ max: 1e6, ttl: 1e4 });
1474
+ this.recentlyRebalanced = new Cache<string>({ max: 1e4, ttl: 1e5 });
1409
1475
 
1410
1476
  this.uniqueReplicators = new Set();
1411
1477
 
@@ -1476,7 +1542,9 @@ export class SharedLog<
1476
1542
  this._requestIPruneSent = new Map();
1477
1543
  this._requestIPruneResponseReplicatorSet = new Map();
1478
1544
 
1479
- this.replicationChangeDebounceFn = debounceAggregationChanges(
1545
+ this.replicationChangeDebounceFn = debounceAggregationChanges<
1546
+ ReplicationRangeIndexable<R>
1547
+ >(
1480
1548
  (change) =>
1481
1549
  this.onReplicationChange(change).then(() =>
1482
1550
  this.rebalanceParticipationDebounced?.(),
@@ -1800,17 +1868,18 @@ export class SharedLog<
1800
1868
  ) {
1801
1869
  let roleAge = options?.roleAge ?? (await this.getDefaultMinRoleAge());
1802
1870
  let eager = options?.eager ?? false;
1803
- const range = await this.domain.fromArgs(args, this);
1804
-
1871
+ const range = await this.domain.fromArgs(args);
1872
+
1873
+ const width =
1874
+ range.length ??
1875
+ (await minimumWidthToCover<R>(
1876
+ this.replicas.min.getValue(this),
1877
+ this.indexableDomain.numbers,
1878
+ ));
1805
1879
  const set = await getCoverSet<R>({
1806
1880
  peers: this.replicationIndex,
1807
1881
  start: range.offset,
1808
- widthToCoverScaled:
1809
- range.length ??
1810
- (await minimumWidthToCover<R>(
1811
- this.replicas.min.getValue(this),
1812
- this.indexableDomain.numbers,
1813
- )),
1882
+ widthToCoverScaled: width,
1814
1883
  roleAge,
1815
1884
  eager,
1816
1885
  numbers: this.indexableDomain.numbers,
@@ -1837,6 +1906,7 @@ export class SharedLog<
1837
1906
 
1838
1907
  this.distributeQueue?.clear();
1839
1908
  this.coordinateToHash.clear();
1909
+ this.recentlyRebalanced.clear();
1840
1910
  this.uniqueReplicators.clear();
1841
1911
 
1842
1912
  this._closeController.abort();
@@ -2378,7 +2448,11 @@ export class SharedLog<
2378
2448
  x.toReplicationRangeIndexable(context.from!),
2379
2449
  ),
2380
2450
  context.from!,
2381
- { reset, checkDuplicates: true },
2451
+ {
2452
+ reset,
2453
+ checkDuplicates: true,
2454
+ timestamp: Number(context.timestamp),
2455
+ },
2382
2456
  );
2383
2457
 
2384
2458
  /* await this._modifyReplicators(msg.role, context.from!); */
@@ -2403,7 +2477,19 @@ export class SharedLog<
2403
2477
  return;
2404
2478
  }
2405
2479
 
2406
- await this.removeReplicationRange(msg.segmentIds, context.from);
2480
+ const rangesToRemove = await this.resolveReplicationRangesFromIdsAndKey(
2481
+ msg.segmentIds,
2482
+ context.from,
2483
+ );
2484
+ await this.removeReplicationRanges(rangesToRemove, context.from);
2485
+ const timestamp = BigInt(+new Date());
2486
+ for (const range of rangesToRemove) {
2487
+ this.replicationChangeDebounceFn.add({
2488
+ range,
2489
+ type: "removed",
2490
+ timestamp,
2491
+ });
2492
+ }
2407
2493
  } else {
2408
2494
  throw new Error("Unexpected message");
2409
2495
  }
@@ -2563,6 +2649,7 @@ export class SharedLog<
2563
2649
  | boolean
2564
2650
  | {
2565
2651
  mergeSegments?: boolean;
2652
+ assumeSynced?: boolean;
2566
2653
  };
2567
2654
  },
2568
2655
  ): Promise<void> {
@@ -2605,11 +2692,20 @@ export class SharedLog<
2605
2692
 
2606
2693
  const persistCoordinate = async (entry: Entry<T>) => {
2607
2694
  const minReplicas = decodeReplicas(entry).getValue(this);
2608
- await this.findLeaders(
2695
+ const leaders = await this.findLeaders(
2609
2696
  await this.createCoordinates(entry, minReplicas),
2610
2697
  entry,
2611
2698
  { persist: {} },
2612
2699
  );
2700
+
2701
+ if (
2702
+ options?.replicate &&
2703
+ typeof options.replicate !== "boolean" &&
2704
+ options.replicate.assumeSynced
2705
+ ) {
2706
+ // make sure we dont start to initate syncing process outwards for this entry
2707
+ this.addPeersToGidPeerHistory(entry.meta.gid, leaders.keys());
2708
+ }
2613
2709
  };
2614
2710
  let entriesToPersist: Entry<T>[] = [];
2615
2711
  let joinOptions = {
@@ -3468,9 +3564,10 @@ export class SharedLog<
3468
3564
  this._gidPeersHistory.clear();
3469
3565
  }
3470
3566
 
3567
+ const timestamp = BigInt(+new Date());
3471
3568
  this.onReplicationChange(
3472
3569
  (await this.getAllReplicationSegments()).map((x) => {
3473
- return { range: x, type: "added" };
3570
+ return { range: x, type: "added", timestamp };
3474
3571
  }),
3475
3572
  );
3476
3573
  }
@@ -3480,7 +3577,9 @@ export class SharedLog<
3480
3577
  }
3481
3578
 
3482
3579
  async onReplicationChange(
3483
- changeOrChanges: ReplicationChanges | ReplicationChanges[],
3580
+ changeOrChanges:
3581
+ | ReplicationChanges<ReplicationRangeIndexable<R>>
3582
+ | ReplicationChanges<ReplicationRangeIndexable<R>>[],
3484
3583
  ) {
3485
3584
  /**
3486
3585
  * TODO use information of new joined/leaving peer to create a subset of heads
@@ -3493,7 +3592,6 @@ export class SharedLog<
3493
3592
 
3494
3593
  await this.log.trim();
3495
3594
 
3496
- const change = mergeReplicationChanges(changeOrChanges);
3497
3595
  const changed = false;
3498
3596
 
3499
3597
  try {
@@ -3503,13 +3601,13 @@ export class SharedLog<
3503
3601
  > = new Map();
3504
3602
 
3505
3603
  for await (const entryReplicated of toRebalance<R>(
3506
- change,
3604
+ changeOrChanges,
3507
3605
  this.entryCoordinatesIndex,
3606
+ this.recentlyRebalanced,
3508
3607
  )) {
3509
3608
  if (this.closed) {
3510
3609
  break;
3511
3610
  }
3512
-
3513
3611
  let oldPeersSet = this._gidPeersHistory.get(entryReplicated.gid);
3514
3612
  let isLeader = false;
3515
3613
 
@@ -3660,7 +3758,7 @@ export class SharedLog<
3660
3758
  // TODO can not reuse old range, since it will (potentially) affect the index because of sideeffects
3661
3759
  dynamicRange = new this.indexableDomain.constructorRange({
3662
3760
  offset: dynamicRange.start1,
3663
- length: this.indexableDomain.numbers.denormalize(newFactor),
3761
+ width: this.indexableDomain.numbers.denormalize(newFactor),
3664
3762
  publicKeyHash: dynamicRange.hash,
3665
3763
  id: dynamicRange.id,
3666
3764
  mode: dynamicRange.mode,
@@ -3729,7 +3827,7 @@ export class SharedLog<
3729
3827
  if (!range) {
3730
3828
  range = new this.indexableDomain.constructorRange({
3731
3829
  offset: this.getDynamicRangeOffset(),
3732
- length: this.indexableDomain.numbers.zero,
3830
+ width: this.indexableDomain.numbers.zero,
3733
3831
  publicKeyHash: this.node.identity.publicKey.hashcode(),
3734
3832
  mode: ReplicationIntent.NonStrict,
3735
3833
  timestamp: BigInt(+new Date()),