@peerbit/shared-log 10.0.6 → 10.1.0

package/dist/src/index.js

@@ -26,13 +26,13 @@ import { BlocksMessage } from "./blocks.js";
 import { CPUUsageIntervalLag } from "./cpu.js";
 import { debounceAccumulator, debounceFixedInterval, debouncedAccumulatorMap, } from "./debounce.js";
 import { EntryWithRefs, ExchangeHeadsMessage, RequestIPrune, ResponseIPrune, createExchangeHeadsMessages, } from "./exchange-heads.js";
-import { MAX_U32, bytesToNumber, createNumbers, denormalizer, } from "./integers.js";
+import { MAX_U32, MAX_U64, bytesToNumber, createNumbers, denormalizer, } from "./integers.js";
 import { TransportMessage } from "./message.js";
 import { PIDReplicationController } from "./pid.js";
-import { EntryReplicatedU32, EntryReplicatedU64, ReplicationIntent, ReplicationRangeIndexableU32, ReplicationRangeIndexableU64, ReplicationRangeMessage, SyncStatus, appromixateCoverage, getCoverSet, getSamples, iHaveCoveringRange, isMatured, isReplicationRangeMessage, mergeRanges, minimumWidthToCover, shouldAssigneToRangeBoundary as shouldAssignToRangeBoundary, toRebalance, } from "./ranges.js";
+import { EntryReplicatedU32, EntryReplicatedU64, ReplicationIntent, ReplicationRangeIndexableU32, ReplicationRangeIndexableU64, ReplicationRangeMessage, SyncStatus, appromixateCoverage, countCoveringRangesSameOwner, debounceAggregationChanges, getAllMergeCandiates, getCoverSet, getSamples, isMatured, isReplicationRangeMessage, mergeRanges, minimumWidthToCover, shouldAssigneToRangeBoundary as shouldAssignToRangeBoundary, toRebalance, } from "./ranges.js";
 import { createReplicationDomainHash, } from "./replication-domain-hash.js";
 import { createReplicationDomainTime, } from "./replication-domain-time.js";
-import { debounceAggregationChanges, mergeReplicationChanges, } from "./replication-domain.js";
+import {} from "./replication-domain.js";
 import { AbsoluteReplicas, AddedReplicationSegmentMessage, AllReplicatingSegmentsMessage, MinReplicas, ReplicationError, RequestReplicationInfoMessage, ResponseRoleMessage, StoppedReplicating, decodeReplicas, encodeReplicas, maxReplicas, } from "./replication.js";
 import { Observer, Replicator } from "./role.js";
 import { RatelessIBLTSynchronizer } from "./sync/rateless-iblt.js";
@@ -42,6 +42,7 @@ export { createReplicationDomainHash, createReplicationDomainTime, };
 export { CPUUsageIntervalLag };
 export * from "./replication.js";
 export { EntryReplicatedU32, EntryReplicatedU64, };
+export { MAX_U32, MAX_U64 };
 export const logger = loggerFn({ module: "shared-log" });
 const getLatestEntry = (entries) => {
     let latest = undefined;
@@ -135,6 +136,7 @@ let SharedLog = class SharedLog extends Program {
     _replicationRangeIndex;
     _entryCoordinatesIndex;
     coordinateToHash;
+    recentlyRebalanced;
     uniqueReplicators;
     /* private _totalParticipation!: number; */
     // gid -> coordinate -> publicKeyHash list (of owners)
@@ -230,13 +232,14 @@ let SharedLog = class SharedLog extends Program {
         ) */
         () => intervalTime);
     }
-    async _replicate(options, { reset, checkDuplicates, syncStatus, announce, mergeSegments, } = {}) {
+    async _replicate(options, { reset, checkDuplicates, announce, mergeSegments, } = {}) {
         let offsetWasProvided = false;
         if (isUnreplicationOptions(options)) {
             await this.unreplicate();
         }
         else {
-            let ranges = [];
+            let rangesToReplicate = [];
+            let rangesToUnreplicate = [];
             if (options == null) {
                 options = {};
             }
@@ -254,11 +257,11 @@ let SharedLog = class SharedLog extends Program {
                     // not allowed
                     return;
                 }
-                ranges = [maybeRange];
+                rangesToReplicate = [maybeRange];
                 offsetWasProvided = true;
             }
             else if (isReplicationRangeMessage(options)) {
-                ranges = [
+                rangesToReplicate = [
                     options.toReplicationRangeIndexable(this.node.identity.publicKey),
                 ];
                 offsetWasProvided = true;
@@ -302,23 +305,17 @@ let SharedLog = class SharedLog extends Program {
                     let factorDenormalized = !normalized
                         ? factor
                         : this.indexableDomain.numbers.denormalize(factor);
-                    ranges.push(new this.indexableDomain.constructorRange({
+                    rangesToReplicate.push(new this.indexableDomain.constructorRange({
                         id: rangeArg.id,
                         // @ts-ignore
                         offset: offset,
                         // @ts-ignore
-                        length: (factor === "all"
+                        width: (factor === "all"
                             ? fullWidth
                             : factor === "right"
                                 ? // @ts-ignore
                                     fullWidth - offset
                                 : factorDenormalized),
-                        /* typeof factor === "number"
-                            ? factor
-                            : factor === "all"
-                                ? width
-                                // @ts-ignore
-                                : width - offset, */
                         publicKeyHash: this.node.identity.publicKey.hashcode(),
                         mode: rangeArg.strict
                             ? ReplicationIntent.Strict
@@ -326,12 +323,30 @@ let SharedLog = class SharedLog extends Program {
                         timestamp: timestamp ?? BigInt(+new Date()),
                     }));
                 }
-                if (mergeSegments && ranges.length > 1) {
-                    const mergedSegment = mergeRanges(ranges, this.indexableDomain.numbers);
-                    ranges = [mergedSegment];
+                if (mergeSegments) {
+                    let range = rangesToReplicate.length > 1
+                        ? mergeRanges(rangesToReplicate, this.indexableDomain.numbers)
+                        : rangesToReplicate[0];
+                    // also merge segments that are already in the index
+                    if (this.domain.canMerge) {
+                        const mergable = await getAllMergeCandiates(this.replicationIndex, range, this.indexableDomain.numbers);
+                        const mergeableFiltered = [range];
+                        for (const mergeCandidate of mergable) {
+                            if (this.domain.canMerge(mergeCandidate, range)) {
+                                mergeableFiltered.push(mergeCandidate);
+                                if (mergeCandidate.idString !== range.idString) {
+                                    rangesToUnreplicate.push(mergeCandidate);
+                                }
+                            }
+                        }
+                        if (mergeableFiltered.length > 1) {
+                            range = mergeRanges(mergeableFiltered, this.indexableDomain.numbers);
+                        }
+                    }
+                    rangesToReplicate = [range];
                 }
             }
-            for (const range of ranges) {
+            for (const range of rangesToReplicate) {
                 this.oldestOpenTime = Math.min(Number(range.timestamp), this.oldestOpenTime);
             }
             let resetRanges = reset;
@@ -341,13 +356,20 @@ let SharedLog = class SharedLog extends Program {
                 // but ({ replicate: 0.5, offset: 0.5 }) means that we want to add a range
                 // TODO make behaviour more clear
             }
-            await this.startAnnounceReplicating(ranges, {
+            await this.startAnnounceReplicating(rangesToReplicate, {
                 reset: resetRanges ?? false,
                 checkDuplicates,
                 announce,
-                syncStatus,
             });
-            return ranges;
+            if (rangesToUnreplicate.length > 0) {
+                await this.removeReplicationRanges(rangesToUnreplicate, this.node.identity.publicKey);
+                await this.rpc.send(new StoppedReplicating({
+                    segmentIds: rangesToUnreplicate.map((x) => x.id),
+                }), {
+                    priority: 1,
+                });
+            }
+            return rangesToReplicate;
         }
     }
     setupDebouncedRebalancing(options) {
@@ -373,7 +395,6 @@ let SharedLog = class SharedLog extends Program {
     }
     async replicate(rangeOrEntry, options) {
         let range = undefined;
-        let syncStatus = SyncStatus.Unsynced;
         if (rangeOrEntry instanceof ReplicationRangeMessage) {
             range = rangeOrEntry;
         }
@@ -383,7 +404,6 @@ let SharedLog = class SharedLog extends Program {
                 offset: await this.domain.fromEntry(rangeOrEntry),
                 normalized: false,
             };
-            syncStatus = SyncStatus.Synced; /// we already have the entries
         }
         else if (Array.isArray(rangeOrEntry)) {
             let ranges = [];
@@ -393,8 +413,8 @@ let SharedLog = class SharedLog extends Program {
                         factor: 1,
                         offset: await this.domain.fromEntry(entry),
                         normalized: false,
+                        strict: true,
                     });
-                    syncStatus = SyncStatus.Synced; /// we already have the entries
                 }
                 else {
                     ranges.push(entry);
@@ -405,18 +425,34 @@ let SharedLog = class SharedLog extends Program {
         else {
             range = rangeOrEntry ?? true;
         }
-        return this._replicate(range, { ...options, syncStatus });
+        return this._replicate(range, options);
     }
     async unreplicate(rangeOrEntry) {
-        let range;
+        let segmentIds;
         if (rangeOrEntry instanceof Entry) {
-            range = {
+            let range = {
                 factor: 1,
                 offset: await this.domain.fromEntry(rangeOrEntry),
             };
+            const indexed = this.replicationIndex.iterate({
+                query: {
+                    width: 1,
+                    start1: range.offset /* ,
+                    hash: this.node.identity.publicKey.hashcode(), */,
+                },
+            });
+            segmentIds = (await indexed.all()).map((x) => x.id.key);
+            if (segmentIds.length === 0) {
+                logger.warn("No segment found to unreplicate");
+                return;
+            }
         }
-        else if (rangeOrEntry instanceof ReplicationRangeMessage) {
-            range = rangeOrEntry;
+        else if (Array.isArray(rangeOrEntry)) {
+            segmentIds = rangeOrEntry.map((x) => x.id);
+            if (segmentIds.length === 0) {
+                logger.warn("No segment found to unreplicate");
+                return;
+            }
         }
         else {
             this._isReplicating = false;
@@ -429,14 +465,8 @@ let SharedLog = class SharedLog extends Program {
             // TODO support this by never deleting the range with the segment id that is generated by the dynamic replication method
             throw new Error("Unsupported when adaptive replicating");
         }
-        const indexed = this.replicationIndex.iterate({
-            query: {
-                width: 1,
-                start1: range.offset,
-            },
-        });
-        const segmentIds = (await indexed.all()).map((x) => x.id.key);
-        await this.removeReplicationRange(segmentIds, this.node.identity.publicKey);
+        const rangesToRemove = await this.resolveReplicationRangesFromIdsAndKey(segmentIds, this.node.identity.publicKey);
+        await this.removeReplicationRanges(rangesToRemove, this.node.identity.publicKey);
         await this.rpc.send(new StoppedReplicating({ segmentIds }), {
             priority: 1,
         });
@@ -468,10 +498,12 @@ let SharedLog = class SharedLog extends Program {
                 throw new Error("Key was not a PublicSignKey");
             }
         }
+        const timestamp = BigInt(+new Date());
         for (const x of deleted) {
             this.replicationChangeDebounceFn.add({
                 range: x.value,
                 type: "removed",
+                timestamp,
             });
         }
         const pendingMaturity = this.pendingMaturity.get(keyHash);
@@ -496,7 +528,7 @@ let SharedLog = class SharedLog extends Program {
                 ? Number(oldestTimestampFromDB)
                 : +new Date();
     }
-    async removeReplicationRange(ids, from) {
+    async resolveReplicationRangesFromIdsAndKey(ids, from) {
         let idMatcher = new Or(ids.map((x) => new ByteMatchQuery({ key: "id", value: x })));
         // make sure we are not removing something that is owned by the replicator
         let identityMatcher = new StringMatch({
@@ -504,9 +536,12 @@ let SharedLog = class SharedLog extends Program {
             value: from.hashcode(),
         });
         let query = new And([idMatcher, identityMatcher]);
+        return (await this.replicationIndex.iterate({ query }).all()).map((x) => x.value);
+    }
+    async removeReplicationRanges(ranges, from) {
         const pendingMaturity = this.pendingMaturity.get(from.hashcode());
         if (pendingMaturity) {
-            for (const id of ids) {
+            for (const id of ranges) {
                 const info = pendingMaturity.get(id.toString());
                 if (info) {
                     clearTimeout(info.timeout);
@@ -517,7 +552,9 @@ let SharedLog = class SharedLog extends Program {
                 this.pendingMaturity.delete(from.hashcode());
             }
         }
-        await this.replicationIndex.del({ query });
+        await this.replicationIndex.del({
+            query: new Or(ranges.map((x) => new ByteMatchQuery({ key: "id", value: x.id }))),
+        });
         const otherSegmentsIterator = this.replicationIndex.iterate({ query: { hash: from.hashcode() } }, { shape: { id: true } });
         if ((await otherSegmentsIterator.next(1)).length === 0) {
             this.uniqueReplicators.delete(from.hashcode());
@@ -531,11 +568,12 @@ let SharedLog = class SharedLog extends Program {
             this.rebalanceParticipationDebounced?.();
         }
     }
-    async addReplicationRange(ranges, from, { reset, checkDuplicates, } = {}) {
+    async addReplicationRange(ranges, from, { reset, checkDuplicates, timestamp: ts, } = {}) {
         if (this._isTrustedReplicator && !(await this._isTrustedReplicator(from))) {
             return undefined;
         }
         let isNewReplicator = false;
+        let timestamp = BigInt(ts ?? +new Date());
         let diffs;
         let deleted = undefined;
         if (reset) {
@@ -548,10 +586,10 @@ let SharedLog = class SharedLog extends Program {
             await this.replicationIndex.del({ query: { hash: from.hashcode() } });
             diffs = [
                 ...deleted.map((x) => {
-                    return { range: x, type: "removed" };
+                    return { range: x, type: "removed", timestamp };
                 }),
                 ...ranges.map((x) => {
-                    return { range: x, type: "added" };
+                    return { range: x, type: "added", timestamp };
                 }),
             ];
             isNewReplicator = prevCount === 0 && ranges.length > 0;
@@ -575,7 +613,7 @@ let SharedLog = class SharedLog extends Program {
                 let deduplicated = [];
                 // TODO also deduplicate/de-overlap among the ranges that ought to be inserted?
                 for (const range of ranges) {
-                    if (!(await iHaveCoveringRange(this.replicationIndex, range))) {
+                    if (!(await countCoveringRangesSameOwner(this.replicationIndex, range))) {
                         deduplicated.push(range);
                     }
                 }
@@ -590,15 +628,27 @@ let SharedLog = class SharedLog extends Program {
                 const prev = existingMap.get(x.idString);
                 if (prev) {
                     if (prev.equalRange(x)) {
-                        return undefined;
+                        return [];
                     }
-                    return { range: x, prev, type: "updated" };
+                    return [
+                        {
+                            range: prev,
+                            timestamp: x.timestamp - 1n,
+                            prev,
+                            type: "replaced",
+                        },
+                        {
+                            range: x,
+                            timestamp: x.timestamp,
+                            type: "added",
+                        },
+                    ];
                 }
                 else {
-                    return { range: x, type: "added" };
+                    return { range: x, timestamp: x.timestamp, type: "added" };
                 }
             })
-                .filter((x) => x != null);
+                .flat();
             diffs = changes;
         }
         this.uniqueReplicators.add(from.hashcode());
@@ -606,7 +656,7 @@ let SharedLog = class SharedLog extends Program {
         let minRoleAge = await this.getDefaultMinRoleAge();
         let isAllMature = true;
         for (const diff of diffs) {
-            if (diff.type === "added" || diff.type === "updated") {
+            if (diff.type === "added") {
                 /* if (this.closed) {
                     return;
                 } */
@@ -628,7 +678,7 @@ let SharedLog = class SharedLog extends Program {
                         this.events.dispatchEvent(new CustomEvent("replicator:mature", {
                             detail: { publicKey: from },
                         }));
-                        this.replicationChangeDebounceFn.add(diff); // we need to call this here because the outcom of findLeaders will be different when some ranges become mature, i.e. some of data we own might be prunable!
+                        this.replicationChangeDebounceFn.add({ ...diff, matured: true }); // we need to call this here because the outcom of findLeaders will be different when some ranges become mature, i.e. some of data we own might be prunable!
                         pendingRanges.delete(diff.range.idString);
                         if (pendingRanges.size === 0) {
                             this.pendingMaturity.delete(diff.range.hash);
@@ -648,7 +698,7 @@ let SharedLog = class SharedLog extends Program {
                     }
                 }
             }
-            else {
+            else if (diff.type === "removed") {
                 const pendingFromPeer = this.pendingMaturity.get(diff.range.hash);
                 if (pendingFromPeer) {
                     const prev = pendingFromPeer.get(diff.range.idString);
@@ -661,6 +711,7 @@ let SharedLog = class SharedLog extends Program {
                     }
                 }
             }
+            // else replaced, do nothing
         }
         if (reset) {
             await this.updateOldestTimestampFromIndex();
@@ -850,14 +901,15 @@ let SharedLog = class SharedLog extends Program {
         this._logProperties = options;
         // TODO types
         this.domain = options?.domain
-            ? options.domain
-            : createReplicationDomainHash(options?.compatibility && options?.compatibility < 10 ? "u32" : "u64");
+            ? options.domain(this)
+            : createReplicationDomainHash(options?.compatibility && options?.compatibility < 10 ? "u32" : "u64")(this);
         this.indexableDomain = createIndexableDomainFromResolution(this.domain.resolution);
         this._respondToIHaveTimeout = options?.respondToIHaveTimeout ?? 2e4;
         this._pendingDeletes = new Map();
         this._pendingIHave = new Map();
         this.latestReplicationInfoMessage = new Map();
         this.coordinateToHash = new Cache({ max: 1e6, ttl: 1e4 });
+        this.recentlyRebalanced = new Cache({ max: 1e4, ttl: 1e5 });
         this.uniqueReplicators = new Set();
         this.openTime = +new Date();
         this.oldestOpenTime = this.openTime;
@@ -1151,12 +1203,13 @@ let SharedLog = class SharedLog extends Program {
     async getCover(args, options) {
         let roleAge = options?.roleAge ?? (await this.getDefaultMinRoleAge());
         let eager = options?.eager ?? false;
-        const range = await this.domain.fromArgs(args, this);
+        const range = await this.domain.fromArgs(args);
+        const width = range.length ??
+            (await minimumWidthToCover(this.replicas.min.getValue(this), this.indexableDomain.numbers));
         const set = await getCoverSet({
             peers: this.replicationIndex,
             start: range.offset,
-            widthToCoverScaled: range.length ??
-                (await minimumWidthToCover(this.replicas.min.getValue(this), this.indexableDomain.numbers)),
+            widthToCoverScaled: width,
             roleAge,
             eager,
             numbers: this.indexableDomain.numbers,
@@ -1177,6 +1230,7 @@ let SharedLog = class SharedLog extends Program {
         this.pendingMaturity.clear();
         this.distributeQueue?.clear();
         this.coordinateToHash.clear();
+        this.recentlyRebalanced.clear();
         this.uniqueReplicators.clear();
         this._closeController.abort();
         clearInterval(this.interval);
@@ -1586,7 +1640,11 @@ let SharedLog = class SharedLog extends Program {
                     if (this.closed) {
                         return;
                     }
-                    await this.addReplicationRange(replicationInfoMessage.segments.map((x) => x.toReplicationRangeIndexable(context.from)), context.from, { reset, checkDuplicates: true });
+                    await this.addReplicationRange(replicationInfoMessage.segments.map((x) => x.toReplicationRangeIndexable(context.from)), context.from, {
+                        reset,
+                        checkDuplicates: true,
+                        timestamp: Number(context.timestamp),
+                    });
                     /* await this._modifyReplicators(msg.role, context.from!); */
                 })
                     .catch((e) => {
@@ -1607,7 +1665,16 @@ let SharedLog = class SharedLog extends Program {
                 if (context.from.equals(this.node.identity.publicKey)) {
                     return;
                 }
-                await this.removeReplicationRange(msg.segmentIds, context.from);
+                const rangesToRemove = await this.resolveReplicationRangesFromIdsAndKey(msg.segmentIds, context.from);
+                await this.removeReplicationRanges(rangesToRemove, context.from);
+                const timestamp = BigInt(+new Date());
+                for (const range of rangesToRemove) {
+                    this.replicationChangeDebounceFn.add({
+                        range,
+                        type: "removed",
+                        timestamp,
+                    });
+                }
             }
             else {
                 throw new Error("Unexpected message");
@@ -1767,7 +1834,13 @@ let SharedLog = class SharedLog extends Program {
             : undefined;
         const persistCoordinate = async (entry) => {
             const minReplicas = decodeReplicas(entry).getValue(this);
-            await this.findLeaders(await this.createCoordinates(entry, minReplicas), entry, { persist: {} });
+            const leaders = await this.findLeaders(await this.createCoordinates(entry, minReplicas), entry, { persist: {} });
+            if (options?.replicate &&
+                typeof options.replicate !== "boolean" &&
+                options.replicate.assumeSynced) {
+                // make sure we dont start to initate syncing process outwards for this entry
+                this.addPeersToGidPeerHistory(entry.meta.gid, leaders.keys());
+            }
         };
         let entriesToPersist = [];
         let joinOptions = {
@@ -2376,8 +2449,9 @@ let SharedLog = class SharedLog extends Program {
         if (options?.clearCache) {
             this._gidPeersHistory.clear();
         }
+        const timestamp = BigInt(+new Date());
         this.onReplicationChange((await this.getAllReplicationSegments()).map((x) => {
-            return { range: x, type: "added" };
+            return { range: x, type: "added", timestamp };
         }));
     }
     async waitForPruned() {
@@ -2392,11 +2466,10 @@ let SharedLog = class SharedLog extends Program {
             return;
         }
         await this.log.trim();
-        const change = mergeReplicationChanges(changeOrChanges);
         const changed = false;
         try {
             const uncheckedDeliver = new Map();
-            for await (const entryReplicated of toRebalance(change, this.entryCoordinatesIndex)) {
+            for await (const entryReplicated of toRebalance(changeOrChanges, this.entryCoordinatesIndex, this.recentlyRebalanced)) {
                 if (this.closed) {
                     break;
                 }
@@ -2505,7 +2578,7 @@ let SharedLog = class SharedLog extends Program {
                     // TODO can not reuse old range, since it will (potentially) affect the index because of sideeffects
                     dynamicRange = new this.indexableDomain.constructorRange({
                         offset: dynamicRange.start1,
-                        length: this.indexableDomain.numbers.denormalize(newFactor),
+                        width: this.indexableDomain.numbers.denormalize(newFactor),
                         publicKeyHash: dynamicRange.hash,
                         id: dynamicRange.id,
                         mode: dynamicRange.mode,
@@ -2560,7 +2633,7 @@ let SharedLog = class SharedLog extends Program {
         if (!range) {
             range = new this.indexableDomain.constructorRange({
                 offset: this.getDynamicRangeOffset(),
-                length: this.indexableDomain.numbers.zero,
+                width: this.indexableDomain.numbers.zero,
                 publicKeyHash: this.node.identity.publicKey.hashcode(),
                 mode: ReplicationIntent.NonStrict,
                 timestamp: BigInt(+new Date()),