@peerbit/shared-log 7.0.10 → 7.0.11

package/src/index.ts

@@ -51,7 +51,6 @@ import yallist from "yallist";
 import {
 	AcknowledgeDelivery,
 	DeliveryMode,
-	SeekDelivery,
 	SilentDelivery,
 	NotStartedError
 } from "@peerbit/stream-interface";
@@ -62,7 +61,7 @@ import { PIDReplicationController } from "./pid.js";
 export * from "./replication.js";
 import PQueue from "p-queue";
 import { CPUUsage, CPUUsageIntervalLag } from "./cpu.js";
-import { collectNodesAroundPoint, getCover } from "./ranges.js";
+import { getCoverSet, getSamples, isMatured } from "./ranges.js";
 export { type CPUUsage, CPUUsageIntervalLag };
 export { Observer, Replicator, Role };
 
@@ -204,6 +203,7 @@ export class SharedLog<T = Uint8Array> extends Program<
 	private remoteBlocks: RemoteBlocks;
 
 	private openTime: number;
+	private oldestOpenTime: number;
 
 	private sync?: (entry: Entry<T>) => boolean;
 
@@ -219,6 +219,7 @@ export class SharedLog<T = Uint8Array> extends Program<
 	private syncMoreInterval?: ReturnType<typeof setTimeout>;
 	private syncInFlightQueue: Map<string, PublicSignKey[]>;
 	private syncInFlightQueueInverted: Map<string, Set<string>>;
+	private syncInFlight: Map<string, Map<string, { timestamp: number }>>;
 
 	replicas: ReplicationLimits;
 
@@ -301,7 +302,7 @@ export class SharedLog<T = Uint8Array> extends Program<
 				} else {
 					this._roleOptions = new Replicator({
 						factor: options.factor,
-						offset: hashToUniformNumber(this.node.identity.publicKey.bytes)
+						offset: this.getReplicationOffset()
 					});
 				}
 			} else {
@@ -326,12 +327,12 @@ export class SharedLog<T = Uint8Array> extends Program<
 			if (this._roleOptions?.limits) {
 				this._role = new Replicator({
 					factor: this._role instanceof Replicator ? this._role.factor : 0,
-					offset: hashToUniformNumber(this.node.identity.publicKey.bytes)
+					offset: this.getReplicationOffset()
 				});
 			} else {
 				this._role = new Replicator({
 					factor: this._role instanceof Replicator ? this._role.factor : 1,
-					offset: hashToUniformNumber(this.node.identity.publicKey.bytes)
+					offset: this.getReplicationOffset()
 				});
 			}
 		}
@@ -354,12 +355,7 @@ export class SharedLog<T = Uint8Array> extends Program<
 		);
 
 		await this.rpc.subscribe();
-
-		await this.rpc.send(new ResponseRoleMessage({ role: this._role }), {
-			mode: new SeekDelivery({
-				redundancy: 1
-			})
-		});
+		await this.rpc.send(new ResponseRoleMessage({ role: this._role }));
 
 		if (onRoleChange && changed !== "none") {
 			this.onRoleChange(this._role, this.node.identity.publicKey);
@@ -418,6 +414,15 @@ export class SharedLog<T = Uint8Array> extends Program<
 					}
 					leaders = newAndOldLeaders;
 				}
+				let set = this._gidPeersHistory.get(result.entry.meta.gid);
+				if (!set) {
+					set = new Set(leaders);
+					this._gidPeersHistory.set(result.entry.meta.gid, set);
+				} else {
+					for (const receiver of leaders) {
+						set.add(receiver);
+					}
+				}
 				mode = isLeader
 					? new SilentDelivery({ redundancy: 1, to: leaders })
 					: new AcknowledgeDelivery({ redundancy: 1, to: leaders });
@@ -453,7 +458,9 @@ export class SharedLog<T = Uint8Array> extends Program<
 		this.latestRoleMessages = new Map();
 		this.syncInFlightQueue = new Map();
 		this.syncInFlightQueueInverted = new Map();
+		this.syncInFlight = new Map();
 		this.openTime = +new Date();
+		this.oldestOpenTime = this.openTime;
 		this.timeUntilRoleMaturity =
 			options?.timeUntilRoleMaturity || WAIT_FOR_ROLE_MATURITY;
 		this.waitForReplicatorTimeout =
@@ -590,21 +597,25 @@ export class SharedLog<T = Uint8Array> extends Program<
 					this.syncInFlightQueue.delete(key);
 				}
 			}
-			this.rpc
-				.send(
-					new ResponseMaybeSync({
-						hashes: requestHashes
-					}),
-					{
-						mode: new SilentDelivery({ to: from, redundancy: 1 })
-					}
-				)
-				.finally(() => {
-					if (this.closed) {
-						return;
+
+			const nowMin10s = +new Date() - 1e4;
+			for (const [key, map] of this.syncInFlight) {
+				// cleanup "old" missing syncs
+				for (const [hash, { timestamp }] of map) {
+					if (timestamp < nowMin10s) {
+						map.delete(hash);
 					}
-					this.syncMoreInterval = setTimeout(requestSync, 1e4);
-				});
+				}
+				if (map.size === 0) {
+					this.syncInFlight.delete(key);
+				}
+			}
+			this.requestSync(requestHashes, from).finally(() => {
+				if (this.closed) {
+					return;
+				}
+				this.syncMoreInterval = setTimeout(requestSync, 1e4);
+			});
 		};
 		requestSync();
 	}
@@ -668,6 +679,7 @@ export class SharedLog<T = Uint8Array> extends Program<
 		this._pendingIHave.clear();
 		this.syncInFlightQueue.clear();
 		this.syncInFlightQueueInverted.clear();
+		this.syncInFlight.clear();
 		this.latestRoleMessages.clear();
 		this._gidPeersHistory.clear();
 
@@ -824,6 +836,17 @@ export class SharedLog<T = Uint8Array> extends Program<
 						this.rebalanceParticipationDebounced?.();
 					}
 
+					/// we clear sync in flight here because we want to join before that, so that entries are totally accounted for
+					for (const head of heads) {
+						const set = this.syncInFlight.get(context.from.hashcode());
+						if (set) {
+							set.delete(head.entry.hash);
+							if (set?.size === 0) {
+								this.syncInFlight.delete(context.from.hashcode());
+							}
+						}
+					}
+
 					if (maybeDelete) {
 						for (const entries of maybeDelete as EntryWithRefs<any>[][]) {
 							const headsWithGid = this.log.headsIndex.gids.get(
@@ -930,18 +953,11 @@ export class SharedLog<T = Uint8Array> extends Program<
 						inverted.add(hash);
 					} else if (!this.log.has(hash)) {
 						this.syncInFlightQueue.set(hash, []);
-						requestHashes.push(hash);
+						requestHashes.push(hash); // request immediately (first time we have seen this hash)
 					}
 				}
 
-				await this.rpc.send(
-					new ResponseMaybeSync({
-						hashes: requestHashes
-					}),
-					{
-						mode: new SilentDelivery({ to: [context.from], redundancy: 1 })
-					}
-				);
+				await this.requestSync(requestHashes, [context.from.hashcode()]);
 			} else if (msg instanceof ResponseMaybeSync) {
 				// TODO better choice of step size
 				const entries = (
@@ -1042,10 +1058,12 @@ export class SharedLog<T = Uint8Array> extends Program<
 	async waitForReplicator(...keys: PublicSignKey[]) {
 		const check = () => {
 			for (const k of keys) {
+				const rect = this.getReplicatorsSorted()
+					?.toArray()
+					?.find((x) => x.publicKey.equals(k));
 				if (
-					!this.getReplicatorsSorted()
-						?.toArray()
-						?.find((x) => x.publicKey.equals(k))
+					!rect ||
+					!isMatured(rect.role, +new Date(), this.getDefaultMinRoleAge())
 				) {
 					return false;
 				}
@@ -1069,6 +1087,10 @@ export class SharedLog<T = Uint8Array> extends Program<
 		return !!isLeader;
 	}
 
+	private getReplicationOffset() {
+		return hashToUniformNumber(this.node.identity.publicKey.bytes);
+	}
+
 	private async waitForIsLeader(
 		slot: { toString(): string },
 		numberOfLeaders: number,
@@ -1137,6 +1159,17 @@ export class SharedLog<T = Uint8Array> extends Program<
 		return this.findLeadersFromUniformNumber(cursor, numberOfLeaders, options);
 	}
 
+	getDefaultMinRoleAge(): number {
+		const now = +new Date();
+		const replLength = this.getReplicatorsSorted()!.length;
+		const diffToOldest =
+			replLength > 1 ? now - this.oldestOpenTime : Number.MAX_SAFE_INTEGER;
+		return Math.min(
+			this.timeUntilRoleMaturity,
+			diffToOldest,
+			(this.timeUntilRoleMaturity * Math.log(replLength)) / 3
+		); // / 3 so that if 2 replicators and timeUntilRoleMaturity = 1e4 the result will be 1
+	}
 	private findLeadersFromUniformNumber(
 		cursor: number,
 		numberOfLeaders: number,
@@ -1144,33 +1177,26 @@ export class SharedLog<T = Uint8Array> extends Program<
 			roleAge?: number;
 		}
 	) {
-		const leaders: Set<string> = new Set();
-		const width = 1;
-		const peers = this.getReplicatorsSorted();
-		if (!peers || peers?.length === 0) {
-			return [];
-		}
-		numberOfLeaders = Math.min(numberOfLeaders, peers.length);
+		const roleAge = options?.roleAge ?? this.getDefaultMinRoleAge(); // TODO -500 as is added so that i f someone else is just as new as us, then we treat them as mature as us. without -500 we might be slower syncing if two nodes starts almost at the same time
 
-		const t = +new Date();
-		const roleAge =
-			options?.roleAge ??
-			Math.min(this.timeUntilRoleMaturity, +new Date() - this.openTime - 500); // TODO -500 as is added so that i f someone else is just as new as us, then we treat them as mature as us. without -500 we might be slower syncing if two nodes starts almost at the same time
-
-		for (let i = 0; i < numberOfLeaders; i++) {
-			const point = ((cursor + i / numberOfLeaders) % 1) * width;
-			const currentNode = peers.head;
-			collectNodesAroundPoint(t, roleAge, peers, currentNode, leaders, point);
-		}
+		const sampes = getSamples(
+			cursor,
+			this.getReplicatorsSorted()!,
+			numberOfLeaders,
+			roleAge,
+			this.role instanceof Replicator && this.role.factor === 0
+				? this.node.identity.publicKey.hashcode()
+				: undefined
+		);
 
-		return [...leaders];
+		return sampes;
 	}
 
 	/**
 	 *
 	 * @returns groups where at least one in any group will have the entry you are looking for
 	 */
-	getReplicatorUnion(roleAge: number = this.timeUntilRoleMaturity) {
+	getReplicatorUnion(roleAge: number = this.getDefaultMinRoleAge()) {
 		if (this.closed === true) {
 			throw new Error("Closed");
 		}
@@ -1192,12 +1218,18 @@ export class SharedLog<T = Uint8Array> extends Program<
 		// the entry we are looking for
 		const coveringWidth = width / minReplicas;
 
-		return getCover(
+		const set = getCoverSet(
 			coveringWidth,
 			peers,
 			roleAge,
 			this.role instanceof Replicator ? this.node.identity.publicKey : undefined
 		);
+
+		// add all in flight
+		for (const [key, _] of this.syncInFlight) {
+			set.add(key);
+		}
+		return [...set];
 	}
 
 	async replicator(
@@ -1255,7 +1287,7 @@ export class SharedLog<T = Uint8Array> extends Program<
 			await this.rebalanceParticipationDebounced?.(); /* await this.rebalanceParticipation(false); */
 			if (update.changed === "added") {
 				await this.rpc.send(new ResponseRoleMessage({ role: this._role }), {
-					mode: new SeekDelivery({
+					mode: new SilentDelivery({
 						to: [publicKey.hashcode()],
 						redundancy: 1
 					})
@@ -1303,6 +1335,10 @@ export class SharedLog<T = Uint8Array> extends Program<
 				// TODO should we remove replicators if they are already added?
 				return { changed: "none" };
 			}
+			this.oldestOpenTime = Math.min(
+				this.oldestOpenTime,
+				Number(role.timestamp)
+			);
 
 			// insert or if already there do nothing
 			const rect: ReplicatorRect = {
@@ -1380,13 +1416,29 @@ export class SharedLog<T = Uint8Array> extends Program<
 			for (const [_a, b] of this._gidPeersHistory) {
 				b.delete(publicKey.hashcode());
 			}
+			this.syncInFlight.delete(publicKey.hashcode());
+			const waitingHashes = this.syncInFlightQueueInverted.get(
+				publicKey.hashcode()
+			);
+			if (waitingHashes) {
+				for (const hash of waitingHashes) {
+					let arr = this.syncInFlightQueue.get(hash);
+					if (arr) {
+						arr = arr.filter((x) => !x.equals(publicKey));
+					}
+					if (this.syncInFlightQueue.size === 0) {
+						this.syncInFlightQueue.delete(hash);
+					}
+				}
+			}
+			this.syncInFlightQueueInverted.delete(publicKey.hashcode());
 		}
 
 		if (subscribed) {
 			if (this.role instanceof Replicator) {
 				this.rpc
 					.send(new ResponseRoleMessage({ role: this._role }), {
-						mode: new SeekDelivery({ redundancy: 1, to: [publicKey] })
+						mode: new SilentDelivery({ redundancy: 1, to: [publicKey] })
 					})
 					.catch((e) => logger.error(e.toString()));
 			}
@@ -1637,6 +1689,29 @@ export class SharedLog<T = Uint8Array> extends Program<
 		return changed;
 	}
 
+	private async requestSync(hashes: string[], to: Set<string> | string[]) {
+		const now = +new Date();
+		for (const node of to) {
+			let map = this.syncInFlight.get(node);
+			if (!map) {
+				map = new Map();
+				this.syncInFlight.set(node, map);
+			}
+			for (const hash of hashes) {
+				map.set(hash, { timestamp: now });
+			}
+		}
+
+		await this.rpc.send(
+			new ResponseMaybeSync({
+				hashes: hashes
+			}),
+			{
+				mode: new SilentDelivery({ to, redundancy: 1 })
+			}
+		);
+	}
+
 	async _onUnsubscription(evt: CustomEvent<UnsubcriptionEvent>) {
 		logger.debug(
 			`Peer disconnected '${evt.detail.from.hashcode()}' from '${JSON.stringify(