@peerbit/shared-log 12.2.0-62829ef → 12.2.0-6aaa5dd

package/dist/src/index.js

@@ -169,10 +169,17 @@ const createIndexableDomainFromResolution = (resolution) => {
 export const DEFAULT_MIN_REPLICAS = 2;
 export const WAIT_FOR_REPLICATOR_TIMEOUT = 9000;
 export const WAIT_FOR_ROLE_MATURITY = 5000;
-export const WAIT_FOR_PRUNE_DELAY = 5000;
+// TODO(prune): Investigate if/when a non-zero prune delay is required for correctness
+// (e.g. responsibility/replication-info message reordering in multi-peer scenarios).
+// Prefer making pruning robust without timing-based heuristics.
+export const WAIT_FOR_PRUNE_DELAY = 0;
 const PRUNE_DEBOUNCE_INTERVAL = 500;
 // DONT SET THIS ANY LOWER, because it will make the pid controller unstable as the system responses are not fast enough to updates from the pid controller
 const RECALCULATE_PARTICIPATION_DEBOUNCE_INTERVAL = 1000;
+const RECALCULATE_PARTICIPATION_MIN_RELATIVE_CHANGE = 0.01;
+const RECALCULATE_PARTICIPATION_MIN_RELATIVE_CHANGE_WITH_CPU_LIMIT = 0.005;
+const RECALCULATE_PARTICIPATION_MIN_RELATIVE_CHANGE_WITH_MEMORY_LIMIT = 0.001;
+const RECALCULATE_PARTICIPATION_RELATIVE_DENOMINATOR_FLOOR = 1e-3;
 const DEFAULT_DISTRIBUTION_DEBOUNCE_TIME = 500;
 const getIdForDynamicRange = (publicKey) => {
     return sha256Sync(concat([publicKey.bytes, new TextEncoder().encode("dynamic")]));
@@ -217,6 +224,7 @@ let SharedLog = (() => {
         coordinateToHash;
         recentlyRebalanced;
         uniqueReplicators;
+        _replicatorsReconciled;
         /* private _totalParticipation!: number; */
         // gid -> coordinate -> publicKeyHash list (of owners)
         _gidPeersHistory;
@@ -296,14 +304,6 @@ let SharedLog = (() => {
         }
         setupRebalanceDebounceFunction(interval = RECALCULATE_PARTICIPATION_DEBOUNCE_INTERVAL) {
             this.rebalanceParticipationDebounced = undefined;
-            // make the rebalancing to respect warmup time
-            let intervalTime = interval * 2;
-            let timeout = setTimeout(() => {
-                intervalTime = interval;
-            }, this.timeUntilRoleMaturity);
-            this._closeController.signal.addEventListener("abort", () => {
-                clearTimeout(timeout);
-            });
             this.rebalanceParticipationDebounced = debounceFixedInterval(() => this.rebalanceParticipation(), 
             /* Math.max(
                 REBALANCE_DEBOUNCE_INTERVAL,
@@ -312,7 +312,7 @@ let SharedLog = (() => {
                     REBALANCE_DEBOUNCE_INTERVAL
                 )
             ) */
-            () => intervalTime);
+            interval);
         }
         async _replicate(options, { reset, checkDuplicates, announce, mergeSegments, rebalance, } = {}) {
             let offsetWasProvided = false;
@@ -690,15 +690,29 @@ let SharedLog = (() => {
                 })
                     .all()).map((x) => x.value);
                 let prevCount = deleted.length;
-                await this.replicationIndex.del({ query: { hash: from.hashcode() } });
-                diffs = [
-                    ...deleted.map((x) => {
-                        return { range: x, type: "removed", timestamp };
-                    }),
-                    ...ranges.map((x) => {
-                        return { range: x, type: "added", timestamp };
-                    }),
-                ];
+                const existingById = new Map(deleted.map((x) => [x.idString, x]));
+                const hasSameRanges = deleted.length === ranges.length &&
+                    ranges.every((range) => {
+                        const existing = existingById.get(range.idString);
+                        return existing != null && existing.equalRange(range);
+                    });
+                // Avoid churn on repeated full-state announcements that don't change any
+                // replication ranges. This prevents unnecessary `replication:change`
+                // events and rebalancing cascades.
+                if (hasSameRanges) {
+                    diffs = [];
+                }
+                else {
+                    await this.replicationIndex.del({ query: { hash: from.hashcode() } });
+                    diffs = [
+                        ...deleted.map((x) => {
+                            return { range: x, type: "removed", timestamp };
+                        }),
+                        ...ranges.map((x) => {
+                            return { range: x, type: "added", timestamp };
+                        }),
+                    ];
+                }
                 isNewReplicator = prevCount === 0 && ranges.length > 0;
             }
             else {
@@ -716,16 +730,18 @@ let SharedLog = (() => {
                         existing.push(result.value);
                     }
                 }
+                let prevCountForOwner = undefined;
                 if (existing.length === 0) {
-                    let prevCount = await this.replicationIndex.count({
+                    prevCountForOwner = await this.replicationIndex.count({
                         query: new StringMatch({ key: "hash", value: from.hashcode() }),
                     });
-                    isNewReplicator = prevCount === 0;
+                    isNewReplicator = prevCountForOwner === 0;
                 }
                 else {
                     isNewReplicator = false;
                 }
-                if (checkDuplicates) {
+                if (checkDuplicates &&
+                    (existing.length > 0 || (prevCountForOwner ?? 0) > 0)) {
                     let deduplicated = [];
                     // TODO also deduplicate/de-overlap among the ranges that ought to be inserted?
                     for (const range of ranges) {
@@ -1252,6 +1268,7 @@ let SharedLog = (() => {
             this.coordinateToHash = new Cache({ max: 1e6, ttl: 1e4 });
             this.recentlyRebalanced = new Cache({ max: 1e4, ttl: 1e5 });
             this.uniqueReplicators = new Set();
+            this._replicatorsReconciled = false;
             this.openTime = +new Date();
             this.oldestOpenTime = this.openTime;
             this.distributionDebounceTime =
@@ -1259,8 +1276,8 @@ let SharedLog = (() => {
             this.timeUntilRoleMaturity =
                 options?.timeUntilRoleMaturity ?? WAIT_FOR_ROLE_MATURITY;
             this.waitForReplicatorTimeout =
-                options?.waitForReplicatorTimeout || WAIT_FOR_REPLICATOR_TIMEOUT;
-            this.waitForPruneDelay = options?.waitForPruneDelay || WAIT_FOR_PRUNE_DELAY;
+                options?.waitForReplicatorTimeout ?? WAIT_FOR_REPLICATOR_TIMEOUT;
+            this.waitForPruneDelay = options?.waitForPruneDelay ?? WAIT_FOR_PRUNE_DELAY;
             if (this.waitForReplicatorTimeout < this.timeUntilRoleMaturity) {
                 this.waitForReplicatorTimeout = this.timeUntilRoleMaturity; // does not makes sense to expect a replicator to mature faster than it is reachable
             }
@@ -1473,7 +1490,16 @@ let SharedLog = (() => {
         async afterOpen() {
             await super.afterOpen();
             // We do this here, because these calls requires this.closed == false
-            this.pruneOfflineReplicators();
+            void this.pruneOfflineReplicators()
+                .then(() => {
+                this._replicatorsReconciled = true;
+            })
+                .catch((error) => {
+                if (isNotStartedError(error)) {
+                    return;
+                }
+                logger.error(error);
+            });
             await this.rebalanceParticipation();
             // Take into account existing subscription
             (await this.node.services.pubsub.getSubscribers(this.topic))?.forEach((v, k) => {
@@ -1631,13 +1657,29 @@ let SharedLog = (() => {
                     set.add(key);
                 }
                 if (options?.reachableOnly) {
-                    let reachableSet = [];
+                    // Prefer the live pubsub subscriber set when filtering reachability.
+                    // `uniqueReplicators` is primarily driven by replication messages and can lag during
+                    // joins/restarts; using subscribers prevents excluding peers that are reachable but
+                    // whose replication ranges were loaded from disk or haven't been processed yet.
+                    const subscribers = (await this.node.services.pubsub.getSubscribers(this.topic)) ??
+                        undefined;
+                    const subscriberHashcodes = subscribers
+                        ? new Set(subscribers.map((key) => key.hashcode()))
+                        : undefined;
+                    const reachable = [];
+                    const selfHash = this.node.identity.publicKey.hashcode();
                     for (const peer of set) {
-                        if (this.uniqueReplicators.has(peer)) {
-                            reachableSet.push(peer);
+                        if (peer === selfHash) {
+                            reachable.push(peer);
+                            continue;
+                        }
+                        if (subscriberHashcodes
+                            ? subscriberHashcodes.has(peer)
+                            : this.uniqueReplicators.has(peer)) {
+                            reachable.push(peer);
                         }
                     }
-                    return reachableSet;
+                    return reachable;
                 }
                 return [...set];
             }
@@ -1977,15 +2019,15 @@ let SharedLog = (() => {
                     await this.remoteBlocks.onMessage(msg.message, context.from.hashcode());
                 }
                 else if (msg instanceof RequestReplicationInfoMessage) {
-                    // TODO this message type is never used, should we remove it?
                     if (context.from.equals(this.node.identity.publicKey)) {
                         return;
                     }
-                    await this.rpc.send(new AllReplicatingSegmentsMessage({
-                        segments: (await this.getMyReplicationSegments()).map((x) => x.toReplicationRange()),
-                    }), {
-                        mode: new SilentDelivery({ to: [context.from], redundancy: 1 }),
-                    });
+                    const segments = (await this.getMyReplicationSegments()).map((x) => x.toReplicationRange());
+                    this.rpc
+                        .send(new AllReplicatingSegmentsMessage({ segments }), {
+                        mode: new SeekDelivery({ to: [context.from], redundancy: 1 }),
+                    })
+                        .catch((e) => logger.error(e.toString()));
                     // for backwards compatibility (v8) remove this when we are sure that all nodes are v9+
                     if (this.v8Behaviour) {
                         const role = this.getRole();
@@ -2009,38 +2051,32 @@ let SharedLog = (() => {
                     if (context.from.equals(this.node.identity.publicKey)) {
                         return;
                     }
-                    let replicationInfoMessage = msg;
-                    // we have this statement because peers might have changed/announced their role,
-                    // but we don't know them as "subscribers" yet. i.e. they are not online
-                    this.waitFor(context.from, {
-                        signal: this._closeController.signal,
-                        timeout: this.waitForReplicatorTimeout,
-                    })
-                        .then(async () => {
-                        // do use an operation log here, because we want to make sure that we don't miss any updates
-                        // and do them in the right order
-                        const prev = this.latestReplicationInfoMessage.get(context.from.hashcode());
-                        if (prev && prev > context.message.header.timestamp) {
+                    const replicationInfoMessage = msg;
+                    // Process replication updates even if the sender isn't yet considered "ready" by
+                    // `Program.waitFor()`. Dropping these messages can lead to missing replicator info
+                    // (and downstream `waitForReplicator()` timeouts) under timing-sensitive joins.
+                    const from = context.from;
+                    const messageTimestamp = context.message.header.timestamp;
+                    (async () => {
+                        const prev = this.latestReplicationInfoMessage.get(from.hashcode());
+                        if (prev && prev > messageTimestamp) {
                             return;
                         }
-                        this.latestReplicationInfoMessage.set(context.from.hashcode(), context.message.header.timestamp);
-                        let reset = msg instanceof AllReplicatingSegmentsMessage;
+                        this.latestReplicationInfoMessage.set(from.hashcode(), messageTimestamp);
                         if (this.closed) {
                             return;
                         }
-                        await this.addReplicationRange(replicationInfoMessage.segments.map((x) => x.toReplicationRangeIndexable(context.from)), context.from, {
+                        const reset = msg instanceof AllReplicatingSegmentsMessage;
+                        await this.addReplicationRange(replicationInfoMessage.segments.map((x) => x.toReplicationRangeIndexable(from)), from, {
                             reset,
                             checkDuplicates: true,
-                            timestamp: Number(context.message.header.timestamp),
+                            timestamp: Number(messageTimestamp),
                         });
-                        /* await this._modifyReplicators(msg.role, context.from!); */
-                    })
-                        .catch((e) => {
+                    })().catch((e) => {
                         if (isNotStartedError(e)) {
                             return;
                         }
-                        logger.error("Failed to find peer who updated replication settings: " +
-                            e?.message);
+                        logger.error(`Failed to apply replication settings from '${from.hashcode()}': ${e?.message ?? e}`);
                     });
                 }
                 else if (msg instanceof StoppedReplicating) {
@@ -2181,15 +2217,17 @@ let SharedLog = (() => {
         }
         async join(entries, options) {
             let entriesToReplicate = [];
-            if (options?.replicate) {
+            if (options?.replicate && this.log.length > 0) {
                 // TODO this block should perhaps be called from a callback on the this.log.join method on all the ignored element because already joined, like "onAlreadyJoined"
                 // check which entrise we already have but not are replicating, and replicate them
                 // we can not just do the 'join' call because it will ignore the already joined entries
                 for (const element of entries) {
                     if (typeof element === "string") {
-                        const entry = await this.log.get(element);
-                        if (entry) {
-                            entriesToReplicate.push(entry);
+                        if (await this.log.has(element)) {
+                            const entry = await this.log.get(element);
+                            if (entry) {
+                                entriesToReplicate.push(entry);
+                            }
                         }
                     }
                     else if (element instanceof Entry) {
@@ -2198,9 +2236,11 @@ let SharedLog = (() => {
                         }
                     }
                     else {
-                        const entry = await this.log.get(element.hash);
-                        if (entry) {
-                            entriesToReplicate.push(entry);
+                        if (await this.log.has(element.hash)) {
+                            const entry = await this.log.get(element.hash);
+                            if (entry) {
+                                entriesToReplicate.push(entry);
+                            }
                         }
                     }
                 }
@@ -2293,6 +2333,7 @@ let SharedLog = (() => {
                 : (options?.roleAge ?? (await this.getDefaultMinRoleAge()));
             let settled = false;
             let timer;
+            let requestTimer;
             const clear = () => {
                 this.events.removeEventListener("replicator:mature", check);
                 this.events.removeEventListener("replication:change", check);
@@ -2301,6 +2342,10 @@ let SharedLog = (() => {
                     clearTimeout(timer);
                     timer = undefined;
                 }
+                if (requestTimer != null) {
+                    clearTimeout(requestTimer);
+                    requestTimer = undefined;
+                }
             };
             const resolve = () => {
                 if (settled) {
@@ -2325,6 +2370,31 @@ let SharedLog = (() => {
             timer = setTimeout(() => {
                 reject(new TimeoutError(`Timeout waiting for replicator ${key.hashcode()}`));
             }, timeoutMs);
+            let requestAttempts = 0;
+            const requestIntervalMs = 1000;
+            const maxRequestAttempts = Math.max(3, Math.ceil(timeoutMs / requestIntervalMs));
+            const requestReplicationInfo = () => {
+                if (settled || this.closed) {
+                    return;
+                }
+                if (requestAttempts >= maxRequestAttempts) {
+                    return;
+                }
+                requestAttempts++;
+                this.rpc
+                    .send(new RequestReplicationInfoMessage(), {
+                    mode: new SeekDelivery({ redundancy: 1, to: [key] }),
+                })
+                    .catch((e) => {
+                    // Best-effort: missing peers / unopened RPC should not fail the wait logic.
+                    if (isNotStartedError(e)) {
+                        return;
+                    }
+                });
+                if (requestAttempts < maxRequestAttempts) {
+                    requestTimer = setTimeout(requestReplicationInfo, requestIntervalMs);
+                }
+            };
             const check = async () => {
                 const iterator = this.replicationIndex?.iterate({ query: new StringMatch({ key: "hash", value: key.hashcode() }) }, { reference: true });
                 try {
@@ -2347,6 +2417,7 @@ let SharedLog = (() => {
                     await iterator?.close();
                 }
             };
+            requestReplicationInfo();
             check();
             this.events.addEventListener("replicator:mature", check);
             this.events.addEventListener("replication:change", check);
@@ -2504,15 +2575,49 @@ let SharedLog = (() => {
             if (this._isReplicating === false) {
                 return 0;
             }
-            const now = +new Date();
-            const subscribers = this.rpc.closed
-                ? 1
-                : ((await this.node.services.pubsub.getSubscribers(this.rpc.topic))
-                    ?.length ?? 1);
-            const diffToOldest = subscribers > 1 ? now - this.oldestOpenTime - 1 : Number.MAX_SAFE_INTEGER;
-            const result = Math.min(this.timeUntilRoleMaturity, Math.max(diffToOldest, this.timeUntilRoleMaturity), Math.max(Math.round((this.timeUntilRoleMaturity * Math.log(subscribers + 1)) / 3), this.timeUntilRoleMaturity)); // / 3 so that if 2 replicators and timeUntilRoleMaturity = 1e4 the result will be 1
-            return result;
-            /* return Math.min(1e3, this.timeUntilRoleMaturity); */
+            // Explicitly disable maturity gating (used by many tests).
+            if (this.timeUntilRoleMaturity <= 0) {
+                return 0;
+            }
+            // If we're alone (or pubsub isn't ready), a fixed maturity time is sufficient.
+            // When there are multiple replicators we want a stable threshold that doesn't
+            // depend on "now" (otherwise it can drift and turn into a flake).
+            let subscribers = 1;
+            if (!this.rpc.closed) {
+                try {
+                    subscribers =
+                        (await this.node.services.pubsub.getSubscribers(this.rpc.topic))
+                            ?.length ?? 1;
+                }
+                catch {
+                    // Best-effort only; fall back to 1.
+                }
+            }
+            if (subscribers <= 1) {
+                return this.timeUntilRoleMaturity;
+            }
+            // Use replication range timestamps to compute a stable "age gap" between the
+            // newest and oldest known roles. This keeps the oldest role mature while
+            // preventing newer roles from being treated as mature purely because time
+            // passes between test steps / network events.
+            let newestOpenTime = this.openTime;
+            try {
+                const newestIterator = await this.replicationIndex.iterate({
+                    sort: [new Sort({ key: "timestamp", direction: "desc" })],
+                }, { shape: { timestamp: true }, reference: true });
+                const newestTimestampFromDB = (await newestIterator.next(1))[0]?.value
+                    .timestamp;
+                await newestIterator.close();
+                if (newestTimestampFromDB != null) {
+                    newestOpenTime = Number(newestTimestampFromDB);
+                }
+            }
+            catch {
+                // Best-effort only; fall back to local open time.
+            }
+            const ageGapToOldest = newestOpenTime - this.oldestOpenTime;
+            const roleAge = Math.max(this.timeUntilRoleMaturity, ageGapToOldest);
+            return roleAge < 0 ? 0 : roleAge;
         }
         async findLeaders(cursors, entry, options) {
             // we consume a list of coordinates in this method since if we are leader of one coordinate we want to persist all of them
@@ -2548,8 +2653,32 @@ let SharedLog = (() => {
         }
         async _findLeaders(cursors, options) {
             const roleAge = options?.roleAge ?? (await this.getDefaultMinRoleAge()); // TODO -500 as is added so that i f someone else is just as new as us, then we treat them as mature as us. without -500 we might be slower syncing if two nodes starts almost at the same time
+            const selfHash = this.node.identity.publicKey.hashcode();
+            // Use `uniqueReplicators` (replicator cache) once we've reconciled it against the
+            // persisted replication index. Until then, fall back to live pubsub subscribers
+            // and avoid relying on `uniqueReplicators` being complete.
+            let peerFilter = undefined;
+            if (this._replicatorsReconciled && this.uniqueReplicators.size > 0) {
+                peerFilter = this.uniqueReplicators.has(selfHash)
+                    ? this.uniqueReplicators
+                    : new Set([...this.uniqueReplicators, selfHash]);
+            }
+            else {
+                try {
+                    const subscribers = (await this.node.services.pubsub.getSubscribers(this.topic)) ??
+                        undefined;
+                    if (subscribers && subscribers.length > 0) {
+                        peerFilter = new Set(subscribers.map((key) => key.hashcode()));
+                        peerFilter.add(selfHash);
+                    }
+                }
+                catch {
+                    // Best-effort only; if pubsub isn't ready, do a full scan.
+                }
+            }
             return getSamples(cursors, this.replicationIndex, roleAge, this.indexableDomain.numbers, {
-                uniqueReplicators: this.uniqueReplicators,
+                peerFilter,
+                uniqueReplicators: peerFilter,
             });
         }
         async findLeadersFromEntry(entry, replicas, options) {
@@ -2578,7 +2707,7 @@ let SharedLog = (() => {
                 for (const [k, v] of this._requestIPruneResponseReplicatorSet) {
                     v.delete(publicKey.hashcode());
                     if (v.size === 0) {
-                        this._requestIPruneSent.delete(k);
+                        this._requestIPruneResponseReplicatorSet.delete(k);
                     }
                 }
                 this.syncronizer.onPeerDisconnected(publicKey);
@@ -2608,6 +2737,14 @@ let SharedLog = (() => {
                             .catch((e) => logger.error(e.toString()));
                     }
                 }
+                // Request the remote peer's replication info. This makes joins resilient to
+                // timing-sensitive delivery/order issues where we may miss their initial
+                // replication announcement.
+                this.rpc
+                    .send(new RequestReplicationInfoMessage(), {
+                    mode: new SeekDelivery({ redundancy: 1, to: [publicKey] }),
+                })
+                    .catch((e) => logger.error(e.toString()));
             }
             else {
                 await this.removeReplicator(publicKey);
@@ -2728,9 +2865,19 @@ let SharedLog = (() => {
                     deferredPromise.reject(e);
                 };
                 let cursor = undefined;
-                const timeout = setTimeout(async () => {
-                    reject(new Error("Timeout for checked pruning: Closed: " + this.closed));
-                }, options?.timeout ?? 1e4);
+                // Checked prune requests can legitimately take longer than a fixed 10s:
+                // - The remote may not have the entry yet and will wait up to `_respondToIHaveTimeout`
+                // - Leadership/replicator information may take up to `waitForReplicatorTimeout` to settle
+                // If we time out too early we can end up with permanently prunable heads that never
+                // get retried (a common CI flake in "prune before join" tests).
+                const checkedPruneTimeoutMs = options?.timeout ??
+                    Math.max(10_000, Number(this._respondToIHaveTimeout ?? 0) +
+                        this.waitForReplicatorTimeout +
+                        PRUNE_DEBOUNCE_INTERVAL * 2);
+                const timeout = setTimeout(() => {
+                    reject(new Error(`Timeout for checked pruning after ${checkedPruneTimeoutMs}ms (closed=${this.closed})`));
+                }, checkedPruneTimeoutMs);
+                timeout.unref?.();
                 this._pendingDeletes.set(entry.hash, {
                     promise: deferredPromise,
                     clear,
@@ -2838,8 +2985,8 @@ let SharedLog = (() => {
                 return { range: x, type: "added", timestamp };
             }));
         }
-        async waitForPruned() {
-            await waitFor(() => this._pendingDeletes.size === 0);
+        async waitForPruned(options) {
+            await waitFor(() => this._pendingDeletes.size === 0, options);
         }
         async onReplicationChange(changeOrChanges) {
             /**
@@ -2956,9 +3103,19 @@ let SharedLog = (() => {
                         peerCount: peersSize,
                         cpuUsage: this.cpuUsage?.value(),
                     });
-                    const relativeDifference = Math.abs(dynamicRange.widthNormalized - newFactor) /
-                        dynamicRange.widthNormalized;
-                    if (relativeDifference > 0.0001) {
+                    const absoluteDifference = Math.abs(dynamicRange.widthNormalized - newFactor);
+                    const relativeDifference = absoluteDifference /
+                        Math.max(dynamicRange.widthNormalized, RECALCULATE_PARTICIPATION_RELATIVE_DENOMINATOR_FLOOR);
+                    let minRelativeChange = RECALCULATE_PARTICIPATION_MIN_RELATIVE_CHANGE;
+                    if (this.replicationController.maxMemoryLimit != null) {
+                        minRelativeChange =
+                            RECALCULATE_PARTICIPATION_MIN_RELATIVE_CHANGE_WITH_MEMORY_LIMIT;
+                    }
+                    else if (this.replicationController.maxCPUUsage != null) {
+                        minRelativeChange =
+                            RECALCULATE_PARTICIPATION_MIN_RELATIVE_CHANGE_WITH_CPU_LIMIT;
+                    }
+                    if (relativeDifference > minRelativeChange) {
                         // TODO can not reuse old range, since it will (potentially) affect the index because of sideeffects
                         dynamicRange = new this.indexableDomain.constructorRange({
                             offset: dynamicRange.start1,