@peerbit/shared-log 10.2.0 → 10.3.0-a4ac71a

package/src/index.ts

@@ -37,10 +37,12 @@ import {
 import { RPC, type RequestContext } from "@peerbit/rpc";
 import {
 	AcknowledgeDelivery,
+	AnyWhere,
 	DeliveryMode,
 	NotStartedError,
 	SeekDelivery,
 	SilentDelivery,
+	type WithMode,
 } from "@peerbit/stream-interface";
 import { AbortError, waitFor } from "@peerbit/time";
 import pDefer, { type DeferredPromise } from "p-defer";
@@ -83,7 +85,6 @@ import {
 	ReplicationRangeIndexableU32,
 	ReplicationRangeIndexableU64,
 	ReplicationRangeMessage,
-	SyncStatus,
 	appromixateCoverage,
 	countCoveringRangesSameOwner,
 	debounceAggregationChanges,
@@ -195,6 +196,7 @@ export type ReplicationOptions<R extends "u32" | "u64" = any> =
 	| number
 	| boolean
 	| "resume";
+export { BlocksMessage };
 
 const isAdaptiveReplicatorOption = (
 	options: ReplicationOptions<any>,
@@ -307,6 +309,7 @@ export type SharedLogOptions<
 	distributionDebounceTime?: number;
 	compatibility?: number;
 	domain?: ReplicationDomainConstructor<D>;
+	earlyBlocks?: boolean | { cacheSize?: number };
 };
 
 export const DEFAULT_MIN_REPLICAS = 2;
@@ -568,15 +571,17 @@ export class SharedLog<
 			checkDuplicates,
 			announce,
 			mergeSegments,
+			rebalance,
 		}: {
 			reset?: boolean;
 			checkDuplicates?: boolean;
 			mergeSegments?: boolean;
+			rebalance?: boolean;
 			announce?: (
 				msg: AddedReplicationSegmentMessage | AllReplicatingSegmentsMessage,
 			) => void;
 		} = {},
-	) {
+	): Promise<ReplicationRangeIndexable<R>[]> {
 		let offsetWasProvided = false;
 		if (isUnreplicationOptions(options)) {
 			await this.unreplicate();
@@ -603,7 +608,7 @@ export class SharedLog<
 				const maybeRange = await this.getDynamicRange();
 				if (!maybeRange) {
 					// not allowed
-					return;
+					return [];
 				}
 				rangesToReplicate = [maybeRange];
 
@@ -630,7 +635,7 @@ export class SharedLog<
 
 				if (rangeArgs.length === 0) {
 					// nothing to do
-					return;
+					return [];
 				}
 
 				for (const rangeArg of rangeArgs) {
@@ -740,6 +745,7 @@ export class SharedLog<
 				reset: resetRanges ?? false,
 				checkDuplicates,
 				announce,
+				rebalance,
 			});
 
 			if (rangesToUnreplicate.length > 0) {
@@ -755,6 +761,8 @@ export class SharedLog<
 
 			return rangesToReplicate;
 		}
+
+		return [];
 	}
 
 	setupDebouncedRebalancing(options?: DynamicReplicationOptions<R>) {
@@ -792,6 +800,7 @@ export class SharedLog<
 		options?: {
 			reset?: boolean;
 			checkDuplicates?: boolean;
+			rebalance?: boolean;
 			mergeSegments?: boolean;
 			announce?: (
 				msg: AllReplicatingSegmentsMessage | AddedReplicationSegmentMessage,
@@ -1032,13 +1041,20 @@ export class SharedLog<
 			reset,
 			checkDuplicates,
 			timestamp: ts,
-		}: { reset?: boolean; checkDuplicates?: boolean; timestamp?: number } = {},
+			rebalance,
+		}: {
+			reset?: boolean;
+			rebalance?: boolean;
+			checkDuplicates?: boolean;
+			timestamp?: number;
+		} = {},
 	) {
 		if (this._isTrustedReplicator && !(await this._isTrustedReplicator(from))) {
 			return undefined;
 		}
 		let isNewReplicator = false;
 		let timestamp = BigInt(ts ?? +new Date());
+		rebalance = rebalance == null ? true : rebalance;
 
 		let diffs: ReplicationChanges<ReplicationRangeIndexable<R>>;
 		let deleted: ReplicationRangeIndexable<R>[] | undefined = undefined;
@@ -1066,16 +1082,25 @@ export class SharedLog<
 
 			isNewReplicator = prevCount === 0 && ranges.length > 0;
 		} else {
-			let existing = await this.replicationIndex
-				.iterate(
-					{
-						query: ranges.map(
-							(x) => new ByteMatchQuery({ key: "id", value: x.id }),
-						),
-					},
-					{ reference: true },
-				)
-				.all();
+			let batchSize = 100;
+			let existing: ReplicationRangeIndexable<R>[] = [];
+			for (let i = 0; i < ranges.length; i += batchSize) {
+				const results = await this.replicationIndex
+					.iterate(
+						{
+							query: (ranges.length <= batchSize
+								? ranges
+								: ranges.slice(i, i + batchSize)
+							).map((x) => new ByteMatchQuery({ key: "id", value: x.id })),
+						},
+						{ reference: true },
+					)
+					.all();
+				for (const result of results) {
+					existing.push(result.value);
+				}
+			}
+
 			if (existing.length === 0) {
 				let prevCount = await this.replicationIndex.count({
 					query: new StringMatch({ key: "hash", value: from.hashcode() }),
@@ -1100,7 +1125,7 @@ export class SharedLog<
 			}
 			let existingMap = new Map<string, ReplicationRangeIndexable<any>>();
 			for (const result of existing) {
-				existingMap.set(result.value.idString, result.value);
+				existingMap.set(result.idString, result);
 			}
 
 			let changes: ReplicationChanges<ReplicationRangeIndexable<R>> = ranges
@@ -1177,7 +1202,13 @@ export class SharedLog<
 								}),
 							);
 
-							this.replicationChangeDebounceFn.add({ ...diff, matured: true }); // we need to call this here because the outcom of findLeaders will be different when some ranges become mature, i.e. some of data we own might be prunable!
+							if (rebalance && diff.range.mode !== ReplicationIntent.Strict) {
+								// TODO this statement (might) cause issues with triggering pruning if the segment is strict and maturity timings will affect the outcome of rebalancing
+								this.replicationChangeDebounceFn.add({
+									...diff,
+									matured: true,
+								}); // we need to call this here because the outcom of findLeaders will be different when some ranges become mature, i.e. some of data we own might be prunable!
+							}
 							pendingRanges.delete(diff.range.idString);
 							if (pendingRanges.size === 0) {
 								this.pendingMaturity.delete(diff.range.hash);
@@ -1238,7 +1269,7 @@ export class SharedLog<
 			}
 		}
 
-		if (diffs.length > 0) {
+		if (rebalance && diffs.length > 0) {
 			for (const diff of diffs) {
 				this.replicationChangeDebounceFn.add(diff);
 			}
@@ -1254,9 +1285,9 @@ export class SharedLog<
 	async startAnnounceReplicating(
 		range: ReplicationRangeIndexable<R>[],
 		options: {
-			syncStatus?: SyncStatus;
 			reset?: boolean;
 			checkDuplicates?: boolean;
+			rebalance?: boolean;
 			announce?: (
 				msg: AllReplicatingSegmentsMessage | AddedReplicationSegmentMessage,
 			) => void;
@@ -1311,7 +1342,7 @@ export class SharedLog<
 		}
 	}
 
-	private addPeersToGidPeerHistory(
+	addPeersToGidPeerHistory(
 		gid: string,
 		publicKeys: Iterable<string>,
 		reset?: boolean,
@@ -1513,12 +1544,13 @@ export class SharedLog<
 		this.remoteBlocks = new RemoteBlocks({
 			local: localBlocks,
 			publish: (message, options) =>
-				this.rpc.send(new BlocksMessage(message), {
-					mode: options?.to
-						? new SilentDelivery({ to: options.to, redundancy: 1 })
-						: undefined,
-				}),
+				this.rpc.send(
+					new BlocksMessage(message),
+					(options as WithMode).mode instanceof AnyWhere ? undefined : options,
+				),
 			waitFor: this.rpc.waitFor.bind(this.rpc),
+			publicKey: this.node.identity.publicKey,
+			earlyBlocks: options?.earlyBlocks ?? true,
 		});
 
 		await this.remoteBlocks.start();
@@ -1691,7 +1723,7 @@ export class SharedLog<
 		await this.rpc.open({
 			queryType: TransportMessage,
 			responseType: TransportMessage,
-			responseHandler: (query, context) => this._onMessage(query, context),
+			responseHandler: (query, context) => this.onMessage(query, context),
 			topic: this.topic,
 		});
 
@@ -1917,7 +1949,6 @@ export class SharedLog<
 		this.coordinateToHash.clear();
 		this.recentlyRebalanced.clear();
 		this.uniqueReplicators.clear();
-
 		this._closeController.abort();
 
 		clearInterval(this.interval);
@@ -1984,7 +2015,7 @@ export class SharedLog<
 	}
 
 	// Callback for receiving a message from the network
-	async _onMessage(
+	async onMessage(
 		msg: TransportMessage,
 		context: RequestContext,
 	): Promise<void> {
@@ -2332,48 +2363,11 @@ export class SharedLog<
 				}
 			} else if (await this.syncronizer.onMessage(msg, context)) {
 				return; // the syncronizer has handled the message
-			} /* else if (msg instanceof RequestMaybeSync) {
-				const requestHashes: string[] = [];
-
-				for (const hash of msg.hashes) {
-					const inFlight = this.syncInFlightQueue.get(hash);
-					if (inFlight) {
-						if (
-							!inFlight.find((x) => x.hashcode() === context.from!.hashcode())
-						) {
-							inFlight.push(context.from);
-							let inverted = this.syncInFlightQueueInverted.get(
-								context.from.hashcode(),
-							);
-							if (!inverted) {
-								inverted = new Set();
-								this.syncInFlightQueueInverted.set(
-									context.from.hashcode(),
-									inverted,
-								);
-							}
-							inverted.add(hash);
-						}
-					} else if (!(await this.log.has(hash))) {
-						this.syncInFlightQueue.set(hash, []);
-						requestHashes.push(hash); // request immediately (first time we have seen this hash)
-					}
-				}
-				requestHashes.length > 0 &&
-					(await this.requestSync(requestHashes, [context.from.hashcode()]));
-			} else if (msg instanceof ResponseMaybeSync) {
-				// TODO perhaps send less messages to more receivers for performance reasons?
-				// TODO wait for previous send to target before trying to send more?
-				for await (const message of createExchangeHeadsMessages(
-					this.log,
-					msg.hashes,
-				)) {
-					await this.rpc.send(message, {
-						mode: new SilentDelivery({ to: [context.from!], redundancy: 1 }),
-					});
-				}
-			}  */ else if (msg instanceof BlocksMessage) {
-				await this.remoteBlocks.onMessage(msg.message);
+			} else if (msg instanceof BlocksMessage) {
+				await this.remoteBlocks.onMessage(
+					msg.message,
+					context.from!.hashcode(),
+				);
 			} else if (msg instanceof RequestReplicationInfoMessage) {
 				// TODO this message type is never used, should we remove it?
 
@@ -2699,6 +2693,10 @@ export class SharedLog<
 				}
 			: undefined;
 
+		let assumeSynced =
+			options?.replicate &&
+			typeof options.replicate !== "boolean" &&
+			options.replicate.assumeSynced;
 		const persistCoordinate = async (entry: Entry<T>) => {
 			const minReplicas = decodeReplicas(entry).getValue(this);
 			const leaders = await this.findLeaders(
@@ -2707,11 +2705,7 @@ export class SharedLog<
 				{ persist: {} },
 			);
 
-			if (
-				options?.replicate &&
-				typeof options.replicate !== "boolean" &&
-				options.replicate.assumeSynced
-			) {
+			if (assumeSynced) {
 				// make sure we dont start to initate syncing process outwards for this entry
 				this.addPeersToGidPeerHistory(entry.meta.gid, leaders.keys());
 			}
@@ -2743,7 +2737,9 @@ export class SharedLog<
 
 		if (options?.replicate) {
 			let messageToSend: AddedReplicationSegmentMessage | undefined = undefined;
+
 			await this.replicate(entriesToReplicate, {
+				rebalance: assumeSynced ? false : true,
 				checkDuplicates: true,
 				mergeSegments:
 					typeof options.replicate !== "boolean" && options.replicate
@@ -2768,6 +2764,7 @@ export class SharedLog<
 				},
 			});
 
+			// it is importat that we call persistCoordinate after this.replicate(entries) as else there might be a prune job deleting the entry before replication duties has been assigned to self
 			for (const entry of entriesToPersist) {
 				await persistCoordinate(entry);
 			}
@@ -2779,39 +2776,6 @@ export class SharedLog<
 			}
 		}
 	}
-	/* 
-		private async updateLeaders(
-			cursor: NumberFromType<R>,
-			prev: EntryReplicated<R>,
-			options?: {
-				roleAge?: number;
-			},
-		): Promise<{
-			isLeader: boolean;
-			leaders: Map<string, { intersecting: boolean }>;
-		}> {
-			// we consume a list of coordinates in this method since if we are leader of one coordinate we want to persist all of them
-			const leaders = await this._findLeaders(cursor, options);
-			const isLeader = leaders.has(this.node.identity.publicKey.hashcode());
-			const isAtRangeBoundary = shouldAssignToRangeBoundary(leaders, 1);
-	
-			// dont do anthing if nothing has changed
-			if (prev.assignedToRangeBoundary !== isAtRangeBoundary) {
-				return { isLeader, leaders };
-			}
-	
-			await this.entryCoordinatesIndex.put(
-				new this.indexableDomain.constructorEntry({
-					assignedToRangeBoundary: isAtRangeBoundary,
-					coordinate: cursor,
-					meta: prev.meta,
-					hash: prev.hash,
-				}),
-			);
-	
-			return { isLeader, leaders };
-		}
-	 */
 
 	private async _waitForReplicators(
 		cursors: NumberFromType<R>[],
@@ -3443,7 +3407,6 @@ export class SharedLog<
 					const minReplicasValue = minReplicasObj.getValue(this);
 
 					// TODO is this check necessary
-
 					if (
 						!(await this._waitForReplicators(
 							cursor ??
@@ -3574,7 +3537,7 @@ export class SharedLog<
 		}
 
 		const timestamp = BigInt(+new Date());
-		this.onReplicationChange(
+		return this.onReplicationChange(
 			(await this.getAllReplicationSegments()).map((x) => {
 				return { range: x, type: "added", timestamp };
 			}),
@@ -3617,6 +3580,7 @@ export class SharedLog<
 				if (this.closed) {
 					break;
 				}
+
 				let oldPeersSet = this._gidPeersHistory.get(entryReplicated.gid);
 				let isLeader = false;
 

package/src/ranges.ts

@@ -43,11 +43,6 @@ export enum ReplicationIntent {
 	Strict = 1, // only replicate data in the segment to the specified replicator, not any other data
 }
 
-export enum SyncStatus {
-	Unsynced = 0,
-	Synced = 1,
-}
-
 const min = (a: number | bigint, b: number | bigint) => (a < b ? a : b);
 
 const getSegmentsFromOffsetAndRange = <T extends number | bigint>(
@@ -1897,7 +1892,6 @@ export const getSamples = async <R extends "u32" | "u64">(
 	const now = +new Date();
 	let matured = 0;
 
-	/* let missingForCursors: NumberFromType<R>[] = [] */
 	let uniqueVisited = new Set<string>();
 	for (let i = 0; i < cursor.length; i++) {
 		let point = cursor[i];
@@ -1962,6 +1956,7 @@ export const getSamples = async <R extends "u32" | "u64">(
 	/* if (leaders.size < cursor.length) {
 		throw new Error("Missing leaders got: " + leaders.size + " -- expected -- " + cursor.length + " role age " + roleAge + " missing " + missingForCursors.length + " replication index size: " + (await peers.count()));
 	} */
+
 	return leaders;
 };
 
@@ -2277,6 +2272,9 @@ export const debounceAggregationChanges = <
 ) => {
 	return debounceAccumulator(
 		(result) => {
+			if (result.size === 0) {
+				return;
+			}
 			return fn([...result.values()]);
 		},
 		() => {
@@ -2450,12 +2448,6 @@ export const toRebalance = <R extends "u32" | "u64">(
 
 			while (iterator.done() !== true) {
 				const entries = await iterator.all(); // TODO choose right batch sizes here for optimal memory usage / speed
-
-				/* const grouped = await groupByGidSync(entries.map((x) => x.value));
-				for (const [gid, entries] of grouped.entries()) {
-					yield { gid, entries };
-				} */
-
 				for (const entry of entries) {
 					yield entry.value;
 				}

package/src/sync/rateless-iblt.ts

@@ -3,6 +3,7 @@ import { Cache } from "@peerbit/cache";
 import { type PublicSignKey, randomBytes, toBase64 } from "@peerbit/crypto";
 import { type Index } from "@peerbit/indexer-interface";
 import type { Entry, Log } from "@peerbit/log";
+import { logger as loggerFn } from "@peerbit/logger";
 import { DecoderWrapper, EncoderWrapper } from "@peerbit/riblt";
 import type { RPC, RequestContext } from "@peerbit/rpc";
 import { SilentDelivery } from "@peerbit/stream-interface";
@@ -17,6 +18,8 @@ import {
 } from "../ranges.js";
 import { SimpleSyncronizer } from "./simple.js";
 
+export const logger = loggerFn({ module: "shared-log" });
+
 type NumberOrBigint = number | bigint;
 
 const coerceBigInt = (value: NumberOrBigint): bigint =>
@@ -470,7 +473,16 @@ export class RatelessIBLTSynchronizer<
 					for (const symbol of message.symbols) {
 						decoder.add_coded_symbol(symbol);
 					}
-					decoder.try_decode();
+					try {
+						decoder.try_decode();
+					} catch (error: any) {
+						if (error?.message === "Invalid degree") {
+							logger.error(error?.message);
+							return false;
+						} else {
+							throw error;
+						}
+					}
 					count += message.symbols.length;
 
 					if (decoder.decoded()) {