@peerbit/shared-log 12.2.0-62829ef → 12.2.0-6aaa5dd

package/src/index.ts

@@ -376,11 +376,18 @@ export type SharedLogOptions<
 export const DEFAULT_MIN_REPLICAS = 2;
 export const WAIT_FOR_REPLICATOR_TIMEOUT = 9000;
 export const WAIT_FOR_ROLE_MATURITY = 5000;
-export const WAIT_FOR_PRUNE_DELAY = 5000;
+// TODO(prune): Investigate if/when a non-zero prune delay is required for correctness
+// (e.g. responsibility/replication-info message reordering in multi-peer scenarios).
+// Prefer making pruning robust without timing-based heuristics.
+export const WAIT_FOR_PRUNE_DELAY = 0;
 const PRUNE_DEBOUNCE_INTERVAL = 500;
 
 // DONT SET THIS ANY LOWER, because it will make the pid controller unstable as the system responses are not fast enough to updates from the pid controller
 const RECALCULATE_PARTICIPATION_DEBOUNCE_INTERVAL = 1000;
+const RECALCULATE_PARTICIPATION_MIN_RELATIVE_CHANGE = 0.01;
+const RECALCULATE_PARTICIPATION_MIN_RELATIVE_CHANGE_WITH_CPU_LIMIT = 0.005;
+const RECALCULATE_PARTICIPATION_MIN_RELATIVE_CHANGE_WITH_MEMORY_LIMIT = 0.001;
+const RECALCULATE_PARTICIPATION_RELATIVE_DENOMINATOR_FLOOR = 1e-3;
 
 const DEFAULT_DISTRIBUTION_DEBOUNCE_TIME = 500;
 
@@ -456,6 +463,7 @@ export class SharedLog<
 	private recentlyRebalanced!: Cache<string>;
 
 	uniqueReplicators!: Set<string>;
+	private _replicatorsReconciled!: boolean;
 
 	/* private _totalParticipation!: number; */
 
@@ -617,15 +625,6 @@ export class SharedLog<
 	) {
 		this.rebalanceParticipationDebounced = undefined;
 
-		// make the rebalancing to respect warmup time
-		let intervalTime = interval * 2;
-		let timeout = setTimeout(() => {
-			intervalTime = interval;
-		}, this.timeUntilRoleMaturity);
-		this._closeController.signal.addEventListener("abort", () => {
-			clearTimeout(timeout);
-		});
-
 		this.rebalanceParticipationDebounced = debounceFixedInterval(
 			() => this.rebalanceParticipation(),
 			/* Math.max(
@@ -635,7 +634,7 @@ export class SharedLog<
 					REBALANCE_DEBOUNCE_INTERVAL
 				)
 			) */
-			() => intervalTime, // TODO make this dynamic on the number of replicators
+			interval, // TODO make this dynamic on the number of replicators
 		);
 	}
 
@@ -1166,16 +1165,31 @@ export class SharedLog<
 
 			let prevCount = deleted.length;
 
-			await this.replicationIndex.del({ query: { hash: from.hashcode() } });
+			const existingById = new Map(deleted.map((x) => [x.idString, x]));
+			const hasSameRanges =
+				deleted.length === ranges.length &&
+				ranges.every((range) => {
+					const existing = existingById.get(range.idString);
+					return existing != null && existing.equalRange(range);
+				});
 
-			diffs = [
-				...deleted.map((x) => {
-					return { range: x, type: "removed" as const, timestamp };
-				}),
-				...ranges.map((x) => {
-					return { range: x, type: "added" as const, timestamp };
-				}),
-			];
+			// Avoid churn on repeated full-state announcements that don't change any
+			// replication ranges. This prevents unnecessary `replication:change`
+			// events and rebalancing cascades.
+			if (hasSameRanges) {
+				diffs = [];
+			} else {
+				await this.replicationIndex.del({ query: { hash: from.hashcode() } });
+
+				diffs = [
+					...deleted.map((x) => {
+						return { range: x, type: "removed" as const, timestamp };
+					}),
+					...ranges.map((x) => {
+						return { range: x, type: "added" as const, timestamp };
+					}),
+				];
+			}
 
 			isNewReplicator = prevCount === 0 && ranges.length > 0;
 		} else {
@@ -1198,16 +1212,20 @@ export class SharedLog<
 				}
 			}
 
+			let prevCountForOwner: number | undefined = undefined;
 			if (existing.length === 0) {
-				let prevCount = await this.replicationIndex.count({
+				prevCountForOwner = await this.replicationIndex.count({
 					query: new StringMatch({ key: "hash", value: from.hashcode() }),
 				});
-				isNewReplicator = prevCount === 0;
+				isNewReplicator = prevCountForOwner === 0;
 			} else {
 				isNewReplicator = false;
 			}
 
-			if (checkDuplicates) {
+			if (
+				checkDuplicates &&
+				(existing.length > 0 || (prevCountForOwner ?? 0) > 0)
+			) {
 				let deduplicated: ReplicationRangeIndexable<any>[] = [];
 
 				// TODO also deduplicate/de-overlap among the ranges that ought to be inserted?
@@ -1874,6 +1892,7 @@ export class SharedLog<
 		this.recentlyRebalanced = new Cache<string>({ max: 1e4, ttl: 1e5 });
 
 		this.uniqueReplicators = new Set();
+		this._replicatorsReconciled = false;
 
 		this.openTime = +new Date();
 		this.oldestOpenTime = this.openTime;
@@ -1883,8 +1902,8 @@ export class SharedLog<
 		this.timeUntilRoleMaturity =
 			options?.timeUntilRoleMaturity ?? WAIT_FOR_ROLE_MATURITY;
 		this.waitForReplicatorTimeout =
-			options?.waitForReplicatorTimeout || WAIT_FOR_REPLICATOR_TIMEOUT;
-		this.waitForPruneDelay = options?.waitForPruneDelay || WAIT_FOR_PRUNE_DELAY;
+			options?.waitForReplicatorTimeout ?? WAIT_FOR_REPLICATOR_TIMEOUT;
+		this.waitForPruneDelay = options?.waitForPruneDelay ?? WAIT_FOR_PRUNE_DELAY;
 
 		if (this.waitForReplicatorTimeout < this.timeUntilRoleMaturity) {
 			this.waitForReplicatorTimeout = this.timeUntilRoleMaturity; // does not makes sense to expect a replicator to mature faster than it is reachable
@@ -2176,7 +2195,16 @@ export class SharedLog<
 		await super.afterOpen();
 
 		// We do this here, because these calls requires this.closed == false
-		this.pruneOfflineReplicators();
+		void this.pruneOfflineReplicators()
+			.then(() => {
+				this._replicatorsReconciled = true;
+			})
+			.catch((error) => {
+				if (isNotStartedError(error as Error)) {
+					return;
+				}
+				logger.error(error);
+			});
 
 		await this.rebalanceParticipation();
 
@@ -2392,15 +2420,35 @@ export class SharedLog<
 				set.add(key);
 			}
 
-			if (options?.reachableOnly) {
-				let reachableSet: string[] = [];
-				for (const peer of set) {
-					if (this.uniqueReplicators.has(peer)) {
-						reachableSet.push(peer);
+				if (options?.reachableOnly) {
+					// Prefer the live pubsub subscriber set when filtering reachability.
+					// `uniqueReplicators` is primarily driven by replication messages and can lag during
+					// joins/restarts; using subscribers prevents excluding peers that are reachable but
+					// whose replication ranges were loaded from disk or haven't been processed yet.
+					const subscribers =
+						(await this.node.services.pubsub.getSubscribers(this.topic)) ??
+						undefined;
+					const subscriberHashcodes = subscribers
+						? new Set(subscribers.map((key) => key.hashcode()))
+						: undefined;
+
+					const reachable: string[] = [];
+					const selfHash = this.node.identity.publicKey.hashcode();
+					for (const peer of set) {
+						if (peer === selfHash) {
+							reachable.push(peer);
+							continue;
+						}
+						if (
+							subscriberHashcodes
+								? subscriberHashcodes.has(peer)
+								: this.uniqueReplicators.has(peer)
+						) {
+							reachable.push(peer);
+						}
 					}
+					return reachable;
 				}
-				return reachableSet;
-			}
 
 			return [...set];
 		} catch (error) {
@@ -2844,22 +2892,20 @@ export class SharedLog<
 					context.from!.hashcode(),
 				);
 			} else if (msg instanceof RequestReplicationInfoMessage) {
-				// TODO this message type is never used, should we remove it?
-
 				if (context.from.equals(this.node.identity.publicKey)) {
 					return;
 				}
-				await this.rpc.send(
-					new AllReplicatingSegmentsMessage({
-						segments: (await this.getMyReplicationSegments()).map((x) =>
-							x.toReplicationRange(),
-						),
-					}),
-					{
-						mode: new SilentDelivery({ to: [context.from], redundancy: 1 }),
-					},
+
+				const segments = (await this.getMyReplicationSegments()).map((x) =>
+					x.toReplicationRange(),
 				);
 
+				this.rpc
+					.send(new AllReplicatingSegmentsMessage({ segments }), {
+						mode: new SeekDelivery({ to: [context.from], redundancy: 1 }),
+					})
+					.catch((e) => logger.error(e.toString()));
+
 				// for backwards compatibility (v8) remove this when we are sure that all nodes are v9+
 				if (this.v8Behaviour) {
 					const role = this.getRole();
@@ -2881,73 +2927,60 @@ export class SharedLog<
 					}
 				}
 			} else if (
-				msg instanceof AllReplicatingSegmentsMessage ||
-				msg instanceof AddedReplicationSegmentMessage
-			) {
-				if (context.from.equals(this.node.identity.publicKey)) {
-					return;
-				}
-
-				let replicationInfoMessage = msg as
-					| AllReplicatingSegmentsMessage
-					| AddedReplicationSegmentMessage;
-
-				// we have this statement because peers might have changed/announced their role,
-				// but we don't know them as "subscribers" yet. i.e. they are not online
-
-				this.waitFor(context.from, {
-					signal: this._closeController.signal,
-					timeout: this.waitForReplicatorTimeout,
-				})
-					.then(async () => {
-						// do use an operation log here, because we want to make sure that we don't miss any updates
-						// and do them in the right order
-						const prev = this.latestReplicationInfoMessage.get(
-							context.from!.hashcode(),
-						);
+					msg instanceof AllReplicatingSegmentsMessage ||
+					msg instanceof AddedReplicationSegmentMessage
+				) {
+					if (context.from.equals(this.node.identity.publicKey)) {
+						return;
+					}
 
-						if (prev && prev > context.message.header.timestamp) {
+					const replicationInfoMessage = msg as
+						| AllReplicatingSegmentsMessage
+						| AddedReplicationSegmentMessage;
+
+					// Process replication updates even if the sender isn't yet considered "ready" by
+					// `Program.waitFor()`. Dropping these messages can lead to missing replicator info
+					// (and downstream `waitForReplicator()` timeouts) under timing-sensitive joins.
+					const from = context.from!;
+					const messageTimestamp = context.message.header.timestamp;
+					(async () => {
+						const prev = this.latestReplicationInfoMessage.get(from.hashcode());
+						if (prev && prev > messageTimestamp) {
 							return;
 						}
 
-						this.latestReplicationInfoMessage.set(
-							context.from!.hashcode(),
-							context.message.header.timestamp,
-						);
-
-						let reset = msg instanceof AllReplicatingSegmentsMessage;
+						this.latestReplicationInfoMessage.set(from.hashcode(), messageTimestamp);
 
 						if (this.closed) {
 							return;
 						}
 
+						const reset = msg instanceof AllReplicatingSegmentsMessage;
 						await this.addReplicationRange(
 							replicationInfoMessage.segments.map((x) =>
-								x.toReplicationRangeIndexable(context.from!),
+								x.toReplicationRangeIndexable(from),
 							),
-							context.from!,
+							from,
 							{
 								reset,
 								checkDuplicates: true,
-								timestamp: Number(context.message.header.timestamp),
+								timestamp: Number(messageTimestamp),
 							},
 						);
-
-						/* await this._modifyReplicators(msg.role, context.from!); */
-					})
-					.catch((e) => {
+					})().catch((e) => {
 						if (isNotStartedError(e)) {
 							return;
 						}
 						logger.error(
-							"Failed to find peer who updated replication settings: " +
-								e?.message,
+							`Failed to apply replication settings from '${from.hashcode()}': ${
+								e?.message ?? e
+							}`,
 						);
 					});
-			} else if (msg instanceof StoppedReplicating) {
-				if (context.from.equals(this.node.identity.publicKey)) {
-					return;
-				}
+				} else if (msg instanceof StoppedReplicating) {
+					if (context.from.equals(this.node.identity.publicKey)) {
+						return;
+					}
 
 				const rangesToRemove = await this.resolveReplicationRangesFromIdsAndKey(
 					msg.segmentIds,
@@ -3140,25 +3173,29 @@ export class SharedLog<
 		},
 	): Promise<void> {
 		let entriesToReplicate: Entry<T>[] = [];
-		if (options?.replicate) {
+		if (options?.replicate && this.log.length > 0) {
 			// TODO this block should perhaps be called from a callback on the this.log.join method on all the ignored element because already joined, like "onAlreadyJoined"
 
 			// check which entrise we already have but not are replicating, and replicate them
 			// we can not just do the 'join' call because it will ignore the already joined entries
 			for (const element of entries) {
 				if (typeof element === "string") {
-					const entry = await this.log.get(element);
-					if (entry) {
-						entriesToReplicate.push(entry);
+					if (await this.log.has(element)) {
+						const entry = await this.log.get(element);
+						if (entry) {
+							entriesToReplicate.push(entry);
+						}
 					}
 				} else if (element instanceof Entry) {
 					if (await this.log.has(element.hash)) {
 						entriesToReplicate.push(element);
 					}
 				} else {
-					const entry = await this.log.get(element.hash);
-					if (entry) {
-						entriesToReplicate.push(entry);
+					if (await this.log.has(element.hash)) {
+						const entry = await this.log.get(element.hash);
+						if (entry) {
+							entriesToReplicate.push(entry);
+						}
 					}
 				}
 			}
@@ -3277,6 +3314,7 @@ export class SharedLog<
 
 		let settled = false;
 		let timer: ReturnType<typeof setTimeout> | undefined;
+		let requestTimer: ReturnType<typeof setTimeout> | undefined;
 
 		const clear = () => {
 			this.events.removeEventListener("replicator:mature", check);
@@ -3286,6 +3324,10 @@ export class SharedLog<
 				clearTimeout(timer);
 				timer = undefined;
 			}
+			if (requestTimer != null) {
+				clearTimeout(requestTimer);
+				requestTimer = undefined;
+			}
 		};
 
 		const resolve = () => {
@@ -3317,6 +3359,40 @@ export class SharedLog<
 			);
 		}, timeoutMs);
 
+		let requestAttempts = 0;
+		const requestIntervalMs = 1000;
+		const maxRequestAttempts = Math.max(
+			3,
+			Math.ceil(timeoutMs / requestIntervalMs),
+		);
+
+		const requestReplicationInfo = () => {
+			if (settled || this.closed) {
+				return;
+			}
+
+			if (requestAttempts >= maxRequestAttempts) {
+				return;
+			}
+
+			requestAttempts++;
+
+			this.rpc
+				.send(new RequestReplicationInfoMessage(), {
+					mode: new SeekDelivery({ redundancy: 1, to: [key] }),
+				})
+				.catch((e) => {
+					// Best-effort: missing peers / unopened RPC should not fail the wait logic.
+					if (isNotStartedError(e as Error)) {
+						return;
+					}
+				});
+
+			if (requestAttempts < maxRequestAttempts) {
+				requestTimer = setTimeout(requestReplicationInfo, requestIntervalMs);
+			}
+		};
+
 		const check = async () => {
 			const iterator = this.replicationIndex?.iterate(
 				{ query: new StringMatch({ key: "hash", value: key.hashcode() }) },
@@ -3341,6 +3417,7 @@ export class SharedLog<
 			}
 		};
 
+		requestReplicationInfo();
 		check();
 		this.events.addEventListener("replicator:mature", check);
 		this.events.addEventListener("replication:change", check);
@@ -3590,27 +3667,54 @@ export class SharedLog<
 			return 0;
 		}
 
-		const now = +new Date();
-		const subscribers = this.rpc.closed
-			? 1
-			: ((await this.node.services.pubsub.getSubscribers(this.rpc.topic))
-					?.length ?? 1);
-		const diffToOldest =
-			subscribers > 1 ? now - this.oldestOpenTime - 1 : Number.MAX_SAFE_INTEGER;
-
-		const result = Math.min(
-			this.timeUntilRoleMaturity,
-			Math.max(diffToOldest, this.timeUntilRoleMaturity),
-			Math.max(
-				Math.round(
-					(this.timeUntilRoleMaturity * Math.log(subscribers + 1)) / 3,
-				),
-				this.timeUntilRoleMaturity,
-			),
-		); // / 3 so that if 2 replicators and timeUntilRoleMaturity = 1e4 the result will be 1
+		// Explicitly disable maturity gating (used by many tests).
+		if (this.timeUntilRoleMaturity <= 0) {
+			return 0;
+		}
 
-		return result;
-		/* return Math.min(1e3, this.timeUntilRoleMaturity); */
+		// If we're alone (or pubsub isn't ready), a fixed maturity time is sufficient.
+		// When there are multiple replicators we want a stable threshold that doesn't
+		// depend on "now" (otherwise it can drift and turn into a flake).
+		let subscribers = 1;
+		if (!this.rpc.closed) {
+			try {
+				subscribers =
+					(await this.node.services.pubsub.getSubscribers(this.rpc.topic))
+						?.length ?? 1;
+			} catch {
+				// Best-effort only; fall back to 1.
+			}
+		}
+
+		if (subscribers <= 1) {
+			return this.timeUntilRoleMaturity;
+		}
+
+		// Use replication range timestamps to compute a stable "age gap" between the
+		// newest and oldest known roles. This keeps the oldest role mature while
+		// preventing newer roles from being treated as mature purely because time
+		// passes between test steps / network events.
+		let newestOpenTime = this.openTime;
+		try {
+			const newestIterator = await this.replicationIndex.iterate(
+				{
+					sort: [new Sort({ key: "timestamp", direction: "desc" })],
+				},
+				{ shape: { timestamp: true }, reference: true },
+			);
+			const newestTimestampFromDB = (await newestIterator.next(1))[0]?.value
+				.timestamp;
+			await newestIterator.close();
+			if (newestTimestampFromDB != null) {
+				newestOpenTime = Number(newestTimestampFromDB);
+			}
+		} catch {
+			// Best-effort only; fall back to local open time.
+		}
+
+		const ageGapToOldest = newestOpenTime - this.oldestOpenTime;
+		const roleAge = Math.max(this.timeUntilRoleMaturity, ageGapToOldest);
+		return roleAge < 0 ? 0 : roleAge;
 	}
 
 	async findLeaders(
@@ -3689,13 +3793,37 @@ export class SharedLog<
 		},
 	): Promise<Map<string, { intersecting: boolean }>> {
 		const roleAge = options?.roleAge ?? (await this.getDefaultMinRoleAge()); // TODO -500 as is added so that i f someone else is just as new as us, then we treat them as mature as us. without -500 we might be slower syncing if two nodes starts almost at the same time
+		const selfHash = this.node.identity.publicKey.hashcode();
+
+		// Use `uniqueReplicators` (replicator cache) once we've reconciled it against the
+		// persisted replication index. Until then, fall back to live pubsub subscribers
+		// and avoid relying on `uniqueReplicators` being complete.
+		let peerFilter: Set<string> | undefined = undefined;
+		if (this._replicatorsReconciled && this.uniqueReplicators.size > 0) {
+			peerFilter = this.uniqueReplicators.has(selfHash)
+				? this.uniqueReplicators
+				: new Set([...this.uniqueReplicators, selfHash]);
+		} else {
+			try {
+				const subscribers =
+					(await this.node.services.pubsub.getSubscribers(this.topic)) ??
+					undefined;
+				if (subscribers && subscribers.length > 0) {
+					peerFilter = new Set(subscribers.map((key) => key.hashcode()));
+					peerFilter.add(selfHash);
+				}
+			} catch {
+				// Best-effort only; if pubsub isn't ready, do a full scan.
+			}
+		}
 		return getSamples<R>(
 			cursors,
 			this.replicationIndex,
 			roleAge,
 			this.indexableDomain.numbers,
 			{
-				uniqueReplicators: this.uniqueReplicators,
+				peerFilter,
+				uniqueReplicators: peerFilter,
 			},
 		);
 	}
@@ -3750,7 +3878,7 @@ export class SharedLog<
 			for (const [k, v] of this._requestIPruneResponseReplicatorSet) {
 				v.delete(publicKey.hashcode());
 				if (v.size === 0) {
-					this._requestIPruneSent.delete(k);
+					this._requestIPruneResponseReplicatorSet.delete(k);
 				}
 			}
 
@@ -3789,6 +3917,15 @@ export class SharedLog<
 						.catch((e) => logger.error(e.toString()));
 				}
 			}
+
+			// Request the remote peer's replication info. This makes joins resilient to
+			// timing-sensitive delivery/order issues where we may miss their initial
+			// replication announcement.
+			this.rpc
+				.send(new RequestReplicationInfoMessage(), {
+					mode: new SeekDelivery({ redundancy: 1, to: [publicKey] }),
+				})
+				.catch((e) => logger.error(e.toString()));
 		} else {
 			await this.removeReplicator(publicKey);
 		}
@@ -3948,11 +4085,28 @@ export class SharedLog<
 
 			let cursor: NumberFromType<R>[] | undefined = undefined;
 
-			const timeout = setTimeout(async () => {
+			// Checked prune requests can legitimately take longer than a fixed 10s:
+			// - The remote may not have the entry yet and will wait up to `_respondToIHaveTimeout`
+			// - Leadership/replicator information may take up to `waitForReplicatorTimeout` to settle
+			// If we time out too early we can end up with permanently prunable heads that never
+			// get retried (a common CI flake in "prune before join" tests).
+			const checkedPruneTimeoutMs =
+				options?.timeout ??
+				Math.max(
+					10_000,
+					Number(this._respondToIHaveTimeout ?? 0) +
+						this.waitForReplicatorTimeout +
+						PRUNE_DEBOUNCE_INTERVAL * 2,
+				);
+
+			const timeout = setTimeout(() => {
 				reject(
-					new Error("Timeout for checked pruning: Closed: " + this.closed),
+					new Error(
+						`Timeout for checked pruning after ${checkedPruneTimeoutMs}ms (closed=${this.closed})`,
+					),
 				);
-			}, options?.timeout ?? 1e4);
+			}, checkedPruneTimeoutMs);
+			timeout.unref?.();
 
 			this._pendingDeletes.set(entry.hash, {
 				promise: deferredPromise,
@@ -4100,8 +4254,13 @@ export class SharedLog<
 		);
 	}
 
-	async waitForPruned() {
-		await waitFor(() => this._pendingDeletes.size === 0);
+	async waitForPruned(options?: {
+		timeout?: number;
+		signal?: AbortSignal;
+		delayInterval?: number;
+		timeoutMessage?: string;
+	}) {
+		await waitFor(() => this._pendingDeletes.size === 0, options);
 	}
 
 	async onReplicationChange(
@@ -4280,11 +4439,24 @@ export class SharedLog<
 					cpuUsage: this.cpuUsage?.value(),
 				});
 
+				const absoluteDifference = Math.abs(dynamicRange.widthNormalized - newFactor);
 				const relativeDifference =
-					Math.abs(dynamicRange.widthNormalized - newFactor) /
-					dynamicRange.widthNormalized;
+					absoluteDifference /
+					Math.max(
+						dynamicRange.widthNormalized,
+						RECALCULATE_PARTICIPATION_RELATIVE_DENOMINATOR_FLOOR,
+					);
+
+				let minRelativeChange = RECALCULATE_PARTICIPATION_MIN_RELATIVE_CHANGE;
+				if (this.replicationController.maxMemoryLimit != null) {
+					minRelativeChange =
+						RECALCULATE_PARTICIPATION_MIN_RELATIVE_CHANGE_WITH_MEMORY_LIMIT;
+				} else if (this.replicationController.maxCPUUsage != null) {
+					minRelativeChange =
+						RECALCULATE_PARTICIPATION_MIN_RELATIVE_CHANGE_WITH_CPU_LIMIT;
+				}
 
-				if (relativeDifference > 0.0001) {
+				if (relativeDifference > minRelativeChange) {
 					// TODO can not reuse old range, since it will (potentially) affect the index because of sideeffects
 					dynamicRange = new this.indexableDomain.constructorRange({
 						offset: dynamicRange.start1,