@peerbit/shared-log 12.1.3 → 12.2.0-3333888

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@peerbit/shared-log",
-  "version": "12.1.3",
+  "version": "12.2.0-3333888",
   "description": "Shared log",
   "sideEffects": false,
   "type": "module",
@@ -54,35 +54,35 @@
   "dependencies": {
     "@dao-xyz/borsh": "^6.0.0",
     "@libp2p/crypto": "^5.1.10",
+    "@peerbit/log": "5.0.6-3333888",
+    "@peerbit/logger": "2.0.0-3333888",
+    "@peerbit/program": "5.6.0-3333888",
+    "@peerbit/riblt": "1.2.0-3333888",
+    "@peerbit/rpc": "5.4.15-3333888",
+    "@peerbit/any-store": "2.2.4-3333888",
+    "@peerbit/blocks": "3.1.6-3333888",
+    "@peerbit/blocks-interface": "1.5.1-3333888",
+    "@peerbit/cache": "2.2.0-3333888",
+    "@peerbit/crypto": "2.4.1-3333888",
+    "@peerbit/indexer-interface": "2.1.1-3333888",
+    "@peerbit/indexer-sqlite3": "2.1.0-3333888",
+    "@peerbit/pubsub": "4.1.3-3333888",
+    "@peerbit/pubsub-interface": "4.1.1-3333888",
+    "@peerbit/stream-interface": "5.3.1-3333888",
+    "@peerbit/time": "2.3.0-3333888",
     "json-stringify-deterministic": "^1.0.7",
     "p-each-series": "^3.0.0",
     "p-defer": "^4.0.0",
     "p-queue": "^8.0.1",
     "pidusage": "^4.0.1",
     "pino": "^9.4.0",
-    "uint8arrays": "^5.1.0",
-    "@peerbit/log": "5.0.5",
-    "@peerbit/logger": "2.0.0",
-    "@peerbit/program": "5.5.2",
-    "@peerbit/rpc": "5.4.14",
-    "@peerbit/any-store": "2.2.4",
-    "@peerbit/riblt": "1.1.0",
-    "@peerbit/blocks": "3.1.6",
-    "@peerbit/cache": "2.2.0",
-    "@peerbit/blocks-interface": "1.5.1",
-    "@peerbit/crypto": "2.4.1",
-    "@peerbit/indexer-interface": "2.1.1",
-    "@peerbit/indexer-sqlite3": "2.0.2",
-    "@peerbit/pubsub": "4.1.3",
-    "@peerbit/pubsub-interface": "4.1.1",
-    "@peerbit/stream-interface": "5.3.1",
-    "@peerbit/time": "2.3.0"
+    "uint8arrays": "^5.1.0"
   },
   "devDependencies": {
+    "@peerbit/test-utils": "2.3.15-3333888",
     "@types/libsodium-wrappers": "^0.7.14",
     "@types/pidusage": "^2.0.5",
-    "uuid": "^10.0.0",
-    "@peerbit/test-utils": "2.3.14"
+    "uuid": "^10.0.0"
   },
   "scripts": {
     "clean": "aegir clean",

package/src/index.ts

@@ -135,7 +135,11 @@ import {
 	maxReplicas,
 } from "./replication.js";
 import { Observer, Replicator } from "./role.js";
-import type { SynchronizerConstructor, Syncronizer } from "./sync/index.js";
+import type {
+	SyncOptions,
+	SynchronizerConstructor,
+	Syncronizer,
+} from "./sync/index.js";
 import { RatelessIBLTSynchronizer } from "./sync/rateless-iblt.js";
 import { SimpleSyncronizer } from "./sync/simple.js";
 import { groupByGid } from "./utils.js";
@@ -149,6 +153,12 @@ export {
 };
 export { type CPUUsage, CPUUsageIntervalLag };
 export * from "./replication.js";
+export type {
+	LogLike,
+	LogResultsIterator,
+	SharedLogLike,
+	SharedLogReplicationIndexLike,
+} from "./like.js";
 export {
 	type ReplicationRangeIndexable,
 	ReplicationRangeIndexableU32,
@@ -352,6 +362,7 @@ export type SharedLogOptions<
 	keep?: (
 		entry: ShallowOrFullEntry<T> | EntryReplicated<R>,
 	) => Promise<boolean> | boolean;
+	sync?: SyncOptions<R>;
 	syncronizer?: SynchronizerConstructor<R>;
 	timeUntilRoleMaturity?: number;
 	waitForReplicatorTimeout?: number;
@@ -365,11 +376,18 @@ export type SharedLogOptions<
 export const DEFAULT_MIN_REPLICAS = 2;
 export const WAIT_FOR_REPLICATOR_TIMEOUT = 9000;
 export const WAIT_FOR_ROLE_MATURITY = 5000;
-export const WAIT_FOR_PRUNE_DELAY = 5000;
+// TODO(prune): Investigate if/when a non-zero prune delay is required for correctness
+// (e.g. responsibility/replication-info message reordering in multi-peer scenarios).
+// Prefer making pruning robust without timing-based heuristics.
+export const WAIT_FOR_PRUNE_DELAY = 0;
 const PRUNE_DEBOUNCE_INTERVAL = 500;
 
 // DONT SET THIS ANY LOWER, because it will make the pid controller unstable as the system responses are not fast enough to updates from the pid controller
 const RECALCULATE_PARTICIPATION_DEBOUNCE_INTERVAL = 1000;
+const RECALCULATE_PARTICIPATION_MIN_RELATIVE_CHANGE = 0.01;
+const RECALCULATE_PARTICIPATION_MIN_RELATIVE_CHANGE_WITH_CPU_LIMIT = 0.005;
+const RECALCULATE_PARTICIPATION_MIN_RELATIVE_CHANGE_WITH_MEMORY_LIMIT = 0.001;
+const RECALCULATE_PARTICIPATION_RELATIVE_DENOMINATOR_FLOOR = 1e-3;
 
 const DEFAULT_DISTRIBUTION_DEBOUNCE_TIME = 500;
 
@@ -606,15 +624,6 @@ export class SharedLog<
 	) {
 		this.rebalanceParticipationDebounced = undefined;
 
-		// make the rebalancing to respect warmup time
-		let intervalTime = interval * 2;
-		let timeout = setTimeout(() => {
-			intervalTime = interval;
-		}, this.timeUntilRoleMaturity);
-		this._closeController.signal.addEventListener("abort", () => {
-			clearTimeout(timeout);
-		});
-
 		this.rebalanceParticipationDebounced = debounceFixedInterval(
 			() => this.rebalanceParticipation(),
 			/* Math.max(
@@ -624,7 +633,7 @@ export class SharedLog<
 					REBALANCE_DEBOUNCE_INTERVAL
 				)
 			) */
-			() => intervalTime, // TODO make this dynamic on the number of replicators
+			interval, // TODO make this dynamic on the number of replicators
 		);
 	}
 
@@ -1187,16 +1196,20 @@ export class SharedLog<
 				}
 			}
 
+			let prevCountForOwner: number | undefined = undefined;
 			if (existing.length === 0) {
-				let prevCount = await this.replicationIndex.count({
+				prevCountForOwner = await this.replicationIndex.count({
 					query: new StringMatch({ key: "hash", value: from.hashcode() }),
 				});
-				isNewReplicator = prevCount === 0;
+				isNewReplicator = prevCountForOwner === 0;
 			} else {
 				isNewReplicator = false;
 			}
 
-			if (checkDuplicates) {
+			if (
+				checkDuplicates &&
+				(existing.length > 0 || (prevCountForOwner ?? 0) > 0)
+			) {
 				let deduplicated: ReplicationRangeIndexable<any>[] = [];
 
 				// TODO also deduplicate/de-overlap among the ranges that ought to be inserted?
@@ -1872,8 +1885,8 @@ export class SharedLog<
 		this.timeUntilRoleMaturity =
 			options?.timeUntilRoleMaturity ?? WAIT_FOR_ROLE_MATURITY;
 		this.waitForReplicatorTimeout =
-			options?.waitForReplicatorTimeout || WAIT_FOR_REPLICATOR_TIMEOUT;
-		this.waitForPruneDelay = options?.waitForPruneDelay || WAIT_FOR_PRUNE_DELAY;
+			options?.waitForReplicatorTimeout ?? WAIT_FOR_REPLICATOR_TIMEOUT;
+		this.waitForPruneDelay = options?.waitForPruneDelay ?? WAIT_FOR_PRUNE_DELAY;
 
 		if (this.waitForReplicatorTimeout < this.timeUntilRoleMaturity) {
 			this.waitForReplicatorTimeout = this.timeUntilRoleMaturity; // does not makes sense to expect a replicator to mature faster than it is reachable
@@ -2037,6 +2050,7 @@ export class SharedLog<
 				rangeIndex: this._replicationRangeIndex,
 				rpc: this.rpc,
 				coordinateToHash: this.coordinateToHash,
+				sync: options?.sync,
 			});
 		} else {
 			if (
@@ -2048,6 +2062,7 @@ export class SharedLog<
 					rpc: this.rpc,
 					entryIndex: this.entryCoordinatesIndex,
 					coordinateToHash: this.coordinateToHash,
+					sync: options?.sync,
 				});
 			} else {
 				if (this.domain.resolution === "u32") {
@@ -2063,6 +2078,7 @@ export class SharedLog<
 					rangeIndex: this._replicationRangeIndex,
 					rpc: this.rpc,
 					coordinateToHash: this.coordinateToHash,
+					sync: options?.sync,
 				}) as Syncronizer<R>;
 			}
 		}
@@ -2378,15 +2394,35 @@ export class SharedLog<
 				set.add(key);
 			}
 
-			if (options?.reachableOnly) {
-				let reachableSet: string[] = [];
-				for (const peer of set) {
-					if (this.uniqueReplicators.has(peer)) {
-						reachableSet.push(peer);
+				if (options?.reachableOnly) {
+					// Prefer the live pubsub subscriber set when filtering reachability.
+					// `uniqueReplicators` is primarily driven by replication messages and can lag during
+					// joins/restarts; using subscribers prevents excluding peers that are reachable but
+					// whose replication ranges were loaded from disk or haven't been processed yet.
+					const subscribers =
+						(await this.node.services.pubsub.getSubscribers(this.topic)) ??
+						undefined;
+					const subscriberHashcodes = subscribers
+						? new Set(subscribers.map((key) => key.hashcode()))
+						: undefined;
+
+					const reachable: string[] = [];
+					const selfHash = this.node.identity.publicKey.hashcode();
+					for (const peer of set) {
+						if (peer === selfHash) {
+							reachable.push(peer);
+							continue;
+						}
+						if (
+							subscriberHashcodes
+								? subscriberHashcodes.has(peer)
+								: this.uniqueReplicators.has(peer)
+						) {
+							reachable.push(peer);
+						}
 					}
+					return reachable;
 				}
-				return reachableSet;
-			}
 
 			return [...set];
 		} catch (error) {
@@ -3126,25 +3162,29 @@ export class SharedLog<
 		},
 	): Promise<void> {
 		let entriesToReplicate: Entry<T>[] = [];
-		if (options?.replicate) {
+		if (options?.replicate && this.log.length > 0) {
 			// TODO this block should perhaps be called from a callback on the this.log.join method on all the ignored element because already joined, like "onAlreadyJoined"
 
 			// check which entrise we already have but not are replicating, and replicate them
 			// we can not just do the 'join' call because it will ignore the already joined entries
 			for (const element of entries) {
 				if (typeof element === "string") {
-					const entry = await this.log.get(element);
-					if (entry) {
-						entriesToReplicate.push(entry);
+					if (await this.log.has(element)) {
+						const entry = await this.log.get(element);
+						if (entry) {
+							entriesToReplicate.push(entry);
+						}
 					}
 				} else if (element instanceof Entry) {
 					if (await this.log.has(element.hash)) {
 						entriesToReplicate.push(element);
 					}
 				} else {
-					const entry = await this.log.get(element.hash);
-					if (entry) {
-						entriesToReplicate.push(entry);
+					if (await this.log.has(element.hash)) {
+						const entry = await this.log.get(element.hash);
+						if (entry) {
+							entriesToReplicate.push(entry);
+						}
 					}
 				}
 			}
@@ -3736,7 +3776,7 @@ export class SharedLog<
 			for (const [k, v] of this._requestIPruneResponseReplicatorSet) {
 				v.delete(publicKey.hashcode());
 				if (v.size === 0) {
-					this._requestIPruneSent.delete(k);
+					this._requestIPruneResponseReplicatorSet.delete(k);
 				}
 			}
 
@@ -4086,8 +4126,13 @@ export class SharedLog<
 		);
 	}
 
-	async waitForPruned() {
-		await waitFor(() => this._pendingDeletes.size === 0);
+	async waitForPruned(options?: {
+		timeout?: number;
+		signal?: AbortSignal;
+		delayInterval?: number;
+		timeoutMessage?: string;
+	}) {
+		await waitFor(() => this._pendingDeletes.size === 0, options);
 	}
 
 	async onReplicationChange(
@@ -4266,11 +4311,24 @@ export class SharedLog<
 					cpuUsage: this.cpuUsage?.value(),
 				});
 
+				const absoluteDifference = Math.abs(dynamicRange.widthNormalized - newFactor);
 				const relativeDifference =
-					Math.abs(dynamicRange.widthNormalized - newFactor) /
-					dynamicRange.widthNormalized;
+					absoluteDifference /
+					Math.max(
+						dynamicRange.widthNormalized,
+						RECALCULATE_PARTICIPATION_RELATIVE_DENOMINATOR_FLOOR,
+					);
+
+				let minRelativeChange = RECALCULATE_PARTICIPATION_MIN_RELATIVE_CHANGE;
+				if (this.replicationController.maxMemoryLimit != null) {
+					minRelativeChange =
+						RECALCULATE_PARTICIPATION_MIN_RELATIVE_CHANGE_WITH_MEMORY_LIMIT;
+				} else if (this.replicationController.maxCPUUsage != null) {
+					minRelativeChange =
+						RECALCULATE_PARTICIPATION_MIN_RELATIVE_CHANGE_WITH_CPU_LIMIT;
+				}
 
-				if (relativeDifference > 0.0001) {
+				if (relativeDifference > minRelativeChange) {
 					// TODO can not reuse old range, since it will (potentially) affect the index because of sideeffects
 					dynamicRange = new this.indexableDomain.constructorRange({
 						offset: dynamicRange.start1,

package/src/like.ts

@@ -0,0 +1,84 @@
+import type { PublicSignKey } from "@peerbit/crypto";
+import type {
+	CountOptions,
+	IndexIterator,
+	IterateOptions,
+} from "@peerbit/indexer-interface";
+import type { Entry } from "@peerbit/log";
+import type { ShallowEntry } from "@peerbit/log";
+import type { ReplicationOptions, ReplicationRangeIndexable } from "./index.js";
+
+export type LogBlocksLike = {
+	has: (hash: string) => Promise<boolean> | boolean;
+};
+
+export type LogResultsIterator<T> = {
+	close: () => void | Promise<void>;
+	next: (amount: number) => T[] | Promise<T[]>;
+	done: () => boolean | undefined;
+	all: () => T[] | Promise<T[]>;
+};
+
+export type LogLike<T = any> = {
+	idString?: string;
+	length: number;
+	get: (
+		hash: string,
+		options?: any,
+	) => Promise<Entry<T> | undefined> | Entry<T> | undefined;
+	has: (hash: string) => Promise<boolean> | boolean;
+	getHeads: (resolve?: boolean) => LogResultsIterator<Entry<T> | ShallowEntry>;
+	toArray: () => Promise<Entry<T>[]>;
+	blocks?: LogBlocksLike;
+};
+
+export type SharedLogReplicationIndexLike<R extends "u32" | "u64" = any> = {
+	iterate: (
+		request?: IterateOptions,
+	) => IndexIterator<ReplicationRangeIndexable<R>, undefined>;
+	count: (options?: CountOptions) => Promise<number> | number;
+	getSize?: () => Promise<number> | number;
+};
+
+export type SharedLogLike<T = any, R extends "u32" | "u64" = any> = {
+	closed?: boolean;
+	events: EventTarget;
+	log: LogLike<T>;
+	replicationIndex: SharedLogReplicationIndexLike<R>;
+	node?: { identity: { publicKey: PublicSignKey } };
+	getReplicators: () => Promise<Set<string>>;
+	waitForReplicator: (
+		publicKey: PublicSignKey,
+		options?: {
+			eager?: boolean;
+			roleAge?: number;
+			timeout?: number;
+			signal?: AbortSignal;
+		},
+	) => Promise<void>;
+	waitForReplicators: (options?: {
+		timeout?: number;
+		roleAge?: number;
+		coverageThreshold?: number;
+		waitForNewPeers?: boolean;
+		signal?: AbortSignal;
+	}) => Promise<void>;
+	replicate: (
+		rangeOrEntry?: ReplicationOptions<R> | any,
+		options?: {
+			reset?: boolean;
+			checkDuplicates?: boolean;
+			rebalance?: boolean;
+			mergeSegments?: boolean;
+		},
+	) => Promise<void | ReplicationRangeIndexable<R>[]>;
+	unreplicate: (rangeOrEntry?: { id: Uint8Array }[]) => Promise<void>;
+	calculateCoverage: (options?: {
+		start?: number | bigint;
+		end?: number | bigint;
+		roleAge?: number;
+	}) => Promise<number>;
+	getMyReplicationSegments: () => Promise<ReplicationRangeIndexable<R>[]>;
+	getAllReplicationSegments: () => Promise<ReplicationRangeIndexable<R>[]>;
+	close: () => Promise<void | boolean>;
+};

package/src/sync/index.ts

@@ -8,6 +8,24 @@ import type { Numbers } from "../integers.js";
 import type { TransportMessage } from "../message.js";
 import type { EntryReplicated, ReplicationRangeIndexable } from "../ranges.js";
 
+export type SyncPriorityFn<R extends "u32" | "u64"> = (
+	entry: EntryReplicated<R>,
+) => number;
+
+export type SyncOptions<R extends "u32" | "u64"> = {
+	/**
+	 * Higher numbers are synced first.
+	 * The callback should be fast and side-effect free.
+	 */
+	priority?: SyncPriorityFn<R>;
+
+	/**
+	 * When using rateless IBLT sync, optionally pre-sync up to this many
+	 * high-priority entries using the simple synchronizer.
+	 */
+	maxSimpleEntries?: number;
+};
+
 export type SynchronizerComponents<R extends "u32" | "u64"> = {
 	rpc: RPC<TransportMessage, TransportMessage>;
 	rangeIndex: Index<ReplicationRangeIndexable<R>, any>;
@@ -15,6 +33,7 @@ export type SynchronizerComponents<R extends "u32" | "u64"> = {
 	log: Log<any>;
 	coordinateToHash: Cache<string>;
 	numbers: Numbers<R>;
+	sync?: SyncOptions<R>;
 };
 export type SynchronizerConstructor<R extends "u32" | "u64"> = new (
 	properties: SynchronizerComponents<R>,