@peerbit/shared-log 3.1.10 → 4.0.1

package/src/index.ts

@@ -7,19 +7,16 @@ import {
 	LogEvents,
 	LogProperties
 } from "@peerbit/log";
-import { Program } from "@peerbit/program";
+import { Program, ProgramEvents } from "@peerbit/program";
 import {
-	BinaryReader,
 	BinaryWriter,
 	BorshError,
-	deserialize,
 	field,
 	serialize,
 	variant
 } from "@dao-xyz/borsh";
 import {
 	AccessError,
-	getPublicKeyFromPeerId,
 	PublicSignKey,
 	sha256,
 	sha256Base64Sync
@@ -36,24 +33,34 @@ import {
 	SubscriptionEvent,
 	UnsubcriptionEvent
 } from "@peerbit/pubsub-interface";
-import { startsWith } from "@peerbit/uint8arrays";
-import { TimeoutError } from "@peerbit/time";
-import { REPLICATOR_TYPE_VARIANT, Observer, Replicator, Role } from "./role.js";
+import { AbortError, delay, TimeoutError, waitFor } from "@peerbit/time";
+import { Observer, Replicator, Role } from "./role.js";
 import {
 	AbsoluteReplicas,
-	MinReplicas,
 	ReplicationError,
+	ReplicationLimits,
+	ReplicatorRect,
+	RequestRoleMessage,
+	ResponseRoleMessage,
 	decodeReplicas,
 	encodeReplicas,
+	hashToUniformNumber,
 	maxReplicas
 } from "./replication.js";
 import pDefer, { DeferredPromise } from "p-defer";
 import { Cache } from "@peerbit/cache";
+import { CustomEvent } from "@libp2p/interface";
+import yallist from "yallist";
+import { AcknowledgeDelivery, SilentDelivery } from "@peerbit/stream-interface";
+import { AnyBlockStore, RemoteBlocks } from "@peerbit/blocks";
+import { BlocksMessage } from "./blocks.js";
+import debounce from "p-debounce";
 
 export * from "./replication.js";
+
 export { Observer, Replicator, Role };
 
-export const logger = loggerFn({ module: "peer" });
+export const logger = loggerFn({ module: "shared-log" });
 
 const groupByGid = async <T extends Entry<any> | EntryWithRefs<any>>(
 	entries: T[]
@@ -73,30 +80,80 @@ const groupByGid = async <T extends Entry<any> | EntryWithRefs<any>>(
 	return groupByGid;
 };
 
-export type SyncFilter = (entries: Entry<any>) => Promise<boolean> | boolean;
-
-type ReplicationLimits = { min: MinReplicas; max?: MinReplicas };
 export type ReplicationLimitsOptions =
 	| Partial<ReplicationLimits>
 	| { min?: number; max?: number };
 
-export interface SharedLogOptions {
+type StringRoleOptions = "observer" | "replicator";
+
+export type ReplicationErrorFunction = (objectives: {
+	coverage: number;
+	balance: number;
+	memory: number;
+}) => number;
+
+type AdaptiveReplicatorOptions = {
+	type: "replicator";
+	limits?: { memory: number };
+	error?: ReplicationErrorFunction;
+};
+
+type FixedReplicatorOptions = {
+	type: "replicator";
+	factor: number;
+};
+
+type ObserverType = {
+	type: "observer";
+};
+
+export type RoleOptions =
+	| StringRoleOptions
+	| ObserverType
+	| FixedReplicatorOptions
+	| AdaptiveReplicatorOptions;
+
+const isAdaptiveReplicatorOption = (
+	options: FixedReplicatorOptions | AdaptiveReplicatorOptions
+): options is AdaptiveReplicatorOptions => {
+	if (
+		(options as AdaptiveReplicatorOptions).limits ||
+		(options as AdaptiveReplicatorOptions).error ||
+		(options as FixedReplicatorOptions).factor == null
+	) {
+		return true;
+	}
+	return false;
+};
+
+export type SharedLogOptions = {
+	role?: RoleOptions;
 	replicas?: ReplicationLimitsOptions;
-	sync?: SyncFilter;
-	role?: Role;
 	respondToIHaveTimeout?: number;
 	canReplicate?: (publicKey: PublicSignKey) => Promise<boolean> | boolean;
-}
+};
 
 export const DEFAULT_MIN_REPLICAS = 2;
+export const WAIT_FOR_REPLICATOR_TIMEOUT = 9000;
+export const WAIT_FOR_ROLE_MATURITY = 5000;
+const REBALANCE_DEBOUNCE_INTERAVAL = 50;
 
 export type Args<T> = LogProperties<T> & LogEvents<T> & SharedLogOptions;
+
 export type SharedAppendOptions<T> = AppendOptions<T> & {
 	replicas?: AbsoluteReplicas | number;
 };
 
+type UpdateRoleEvent = { publicKey: PublicSignKey; role: Role };
+export interface SharedLogEvents extends ProgramEvents {
+	role: CustomEvent<UpdateRoleEvent>;
+}
+
 @variant("shared_log")
-export class SharedLog<T = Uint8Array> extends Program<Args<T>> {
+export class SharedLog<T = Uint8Array> extends Program<
+	Args<T>,
+	SharedLogEvents
+> {
 	@field({ type: Log })
 	log: Log<T>;
 
@@ -104,22 +161,22 @@ export class SharedLog<T = Uint8Array> extends Program<Args<T>> {
 	rpc: RPC<TransportMessage, TransportMessage>;
 
 	// options
-	private _sync?: SyncFilter;
 	private _role: Observer | Replicator;
+	private _roleOptions: AdaptiveReplicatorOptions | Observer | Replicator;
 
-	private _sortedPeersCache: { hash: string; timestamp: number }[] | undefined;
-	private _lastSubscriptionMessageId: number;
+	private _sortedPeersCache: yallist<ReplicatorRect> | undefined;
 	private _gidPeersHistory: Map<string, Set<string>>;
 
 	private _onSubscriptionFn: (arg: any) => any;
 	private _onUnsubscriptionFn: (arg: any) => any;
 
 	private _canReplicate?: (
-		publicKey: PublicSignKey
+		publicKey: PublicSignKey,
+		role: Replicator
 	) => Promise<boolean> | boolean;
 
 	private _logProperties?: LogProperties<T> & LogEvents<T>;
-
+	private _closeController: AbortController;
 	private _loadedOnce = false;
 	private _gidParentCache: Cache<Entry<any>[]>;
 	private _respondToIHaveTimeout;
@@ -134,9 +191,19 @@ export class SharedLog<T = Uint8Array> extends Program<Args<T>> {
 
 	private _pendingIHave: Map<
 		string,
-		{ clear: () => void; callback: () => void }
+		{ clear: () => void; callback: (entry: Entry<T>) => void }
 	>;
 
+	private latestRoleMessages: Map<string, bigint>;
+
+	private remoteBlocks: RemoteBlocks;
+
+	private openTime: number;
+
+	private rebalanceParticipationDebounced:
+		| ReturnType<typeof debounce>
+		| undefined;
+
 	replicas: ReplicationLimits;
 
 	constructor(properties?: { id?: Uint8Array }) {
@@ -149,37 +216,89 @@ export class SharedLog<T = Uint8Array> extends Program<Args<T>> {
 		return this._role;
 	}
 
-	async updateRole(role: Observer | Replicator) {
-		const wasRepicators = this._role instanceof Replicator;
-		this._role = role;
-		await this.initializeWithRole();
-		await this.rpc.subscribe(serialize(this._role));
+	private setupRole(options?: RoleOptions) {
+		this.rebalanceParticipationDebounced = undefined;
 
-		if (wasRepicators) {
-			await this.replicationReorganization();
-		}
-	}
+		const setupDebouncedRebalancing = (options?: AdaptiveReplicatorOptions) => {
+			this.replicationController = new ReplicationController({
+				targetMemoryLimit: options?.limits?.memory,
+				errorFunction: options?.error
+			});
 
-	private async initializeWithRole() {
-		try {
-			await this.modifySortedSubscriptionCache(
-				this._role instanceof Replicator ? true : false,
-				getPublicKeyFromPeerId(this.node.peerId)
+			this.rebalanceParticipationDebounced = debounce(
+				() => this.rebalanceParticipation(),
+				REBALANCE_DEBOUNCE_INTERAVAL // TODO make dynamic
 			);
+		};
 
-			if (!this._loadedOnce) {
-				await this.log.load();
-				this._loadedOnce = true;
-			}
-		} catch (error) {
-			if (error instanceof AccessError) {
-				logger.error(
-					"Failed to load all entries due to access error, make sure you are opening the program with approate keychain configuration"
-				);
+		if (options instanceof Observer || options instanceof Replicator) {
+			throw new Error("Unsupported role option type");
+		} else if (options === "observer") {
+			this._roleOptions = new Observer();
+		} else if (options === "replicator") {
+			setupDebouncedRebalancing();
+			this._roleOptions = { type: options };
+		} else if (options) {
+			if (options.type === "replicator") {
+				if (isAdaptiveReplicatorOption(options)) {
+					setupDebouncedRebalancing(options);
+					this._roleOptions = options;
+				} else {
+					this._roleOptions = new Replicator({ factor: options.factor });
+				}
 			} else {
-				throw error;
+				this._roleOptions = new Observer();
 			}
+		} else {
+			// Default option
+			setupDebouncedRebalancing();
+			this._roleOptions = { type: "replicator" };
 		}
+
+		// setup the initial role
+		if (
+			this._roleOptions instanceof Replicator ||
+			this._roleOptions instanceof Observer
+		) {
+			this._role = this._roleOptions; // Fixed
+		} else {
+			this._role = new Replicator({
+				// initial role in a dynamic setup
+				factor: 1,
+				timestamp: BigInt(+new Date())
+			});
+		}
+
+		return this._role;
+	}
+
+	async updateRole(role: RoleOptions, onRoleChange = true) {
+		return this._updateRole(this.setupRole(role), onRoleChange);
+	}
+
+	private async _updateRole(
+		role: Observer | Replicator = this._role,
+		onRoleChange = true
+	) {
+		this._role = role;
+		const { changed } = await this._modifyReplicators(
+			this.role,
+			this.node.identity.publicKey
+		);
+
+		if (!this._loadedOnce) {
+			await this.log.load();
+			this._loadedOnce = true;
+		}
+		await this.rpc.subscribe();
+
+		await this.rpc.send(new ResponseRoleMessage(role));
+
+		if (onRoleChange && changed) {
+			this.onRoleChange(undefined, this._role, this.node.identity.publicKey);
+		}
+
+		return changed;
 	}
 
 	async append(
@@ -207,14 +326,27 @@ export class SharedLog<T = Uint8Array> extends Program<Args<T>> {
 		}
 
 		const result = await this.log.append(data, appendOptions);
+		const leaders = await this.findLeaders(
+			result.entry.meta.gid,
+			decodeReplicas(result.entry).getValue(this)
+		);
+		const isLeader = leaders.includes(this.node.identity.publicKey.hashcode());
 
 		await this.rpc.send(
 			await createExchangeHeadsMessage(
 				this.log,
 				[result.entry],
 				this._gidParentCache
-			)
+			),
+			{
+				mode: isLeader
+					? new SilentDelivery({ redundancy: 1, to: leaders })
+					: new AcknowledgeDelivery({ redundancy: 1, to: leaders })
+			}
 		);
+
+		this.rebalanceParticipationDebounced?.();
+
 		return result;
 	}
 
@@ -231,21 +363,23 @@ export class SharedLog<T = Uint8Array> extends Program<Args<T>> {
 					: options.replicas.max
 				: undefined
 		};
+
 		this._respondToIHaveTimeout = options?.respondToIHaveTimeout ?? 10 * 1000; // TODO make into arg
 		this._pendingDeletes = new Map();
 		this._pendingIHave = new Map();
+		this.latestRoleMessages = new Map();
+		this.openTime = +new Date();
 
 		this._gidParentCache = new Cache({ max: 1000 });
+		this._closeController = new AbortController();
 
 		this._canReplicate = options?.canReplicate;
-		this._sync = options?.sync;
 		this._logProperties = options;
-		this._role = options?.role || new Replicator();
 
-		this._lastSubscriptionMessageId = 0;
-		this._onSubscriptionFn = this._onSubscription.bind(this);
+		this.setupRole(options?.role);
 
-		this._sortedPeersCache = [];
+		this._onSubscriptionFn = this._onSubscription.bind(this);
+		this._sortedPeersCache = yallist.create();
 		this._gidPeersHistory = new Map();
 
 		await this.node.services.pubsub.addEventListener(
@@ -259,9 +393,26 @@ export class SharedLog<T = Uint8Array> extends Program<Args<T>> {
 			this._onUnsubscriptionFn
 		);
 
-		await this.log.open(this.node.services.blocks, this.node.identity, {
-			keychain: this.node.keychain,
+		const id = sha256Base64Sync(this.log.id);
+		const storage = await this.node.memory.sublevel(id);
+		const localBlocks = await new AnyBlockStore(
+			await storage.sublevel("blocks")
+		);
+		const cache = await storage.sublevel("cache");
+
+		this.remoteBlocks = new RemoteBlocks({
+			local: localBlocks,
+			publish: (message, options) =>
+				this.rpc.send(new BlocksMessage(message), {
+					to: options?.to
+				}),
+			waitFor: this.rpc.waitFor.bind(this.rpc)
+		});
 
+		await this.remoteBlocks.start();
+
+		await this.log.open(this.remoteBlocks, this.node.identity, {
+			keychain: this.node.services.keychain,
 			...this._logProperties,
 			onChange: (change) => {
 				if (this._pendingIHave.size > 0) {
@@ -269,7 +420,7 @@ export class SharedLog<T = Uint8Array> extends Program<Args<T>> {
 						const ih = this._pendingIHave.get(added.hash);
 						if (ih) {
 							ih.clear();
-							ih.callback();
+							ih.callback(added);
 						}
 					}
 				}
@@ -304,42 +455,37 @@ export class SharedLog<T = Uint8Array> extends Program<Args<T>> {
 				return this._logProperties?.canAppend?.(entry) ?? true;
 			},
 			trim: this._logProperties?.trim && {
-				...this._logProperties?.trim,
-				filter: {
-					canTrim: async (entry) =>
-						!(await this.isLeader(
-							entry.meta.gid,
-							decodeReplicas(entry).getValue(this)
-						)), // TODO types
-					cacheId: () => this._lastSubscriptionMessageId
-				}
+				...this._logProperties?.trim
 			},
-			cache:
-				this.node.memory &&
-				(await this.node.memory.sublevel(sha256Base64Sync(this.log.id)))
+			cache: cache
 		});
 
-		await this.initializeWithRole();
+		// Open for communcation
+		await this.rpc.open({
+			queryType: TransportMessage,
+			responseType: TransportMessage,
+			responseHandler: this._onMessage.bind(this),
+			topic: this.topic
+		});
+
+		await this._updateRole();
+		await this.rebalanceParticipation();
 
 		// Take into account existing subscription
 		(await this.node.services.pubsub.getSubscribers(this.topic))?.forEach(
 			(v, k) => {
-				this.handleSubscriptionChange(
-					v.publicKey,
-					[{ topic: this.topic, data: v.data }],
-					true
-				);
+				if (v.equals(this.node.identity.publicKey)) {
+					return;
+				}
+				this.handleSubscriptionChange(v, [this.topic], true);
 			}
 		);
+	}
 
-		// Open for communcation
-		await this.rpc.open({
-			queryType: TransportMessage,
-			responseType: TransportMessage,
-			responseHandler: this._onMessage.bind(this),
-			topic: this.topic,
-			subscriptionData: serialize(this.role)
-		});
+	async getMemoryUsage() {
+		return (
+			((await this.log.memory?.size()) || 0) + (await this.log.blocks.size())
+		);
 	}
 
 	get topic() {
@@ -347,6 +493,19 @@ export class SharedLog<T = Uint8Array> extends Program<Args<T>> {
 	}
 
 	private async _close() {
+		this._closeController.abort();
+
+		this.node.services.pubsub.removeEventListener(
+			"subscribe",
+			this._onSubscriptionFn
+		);
+
+		this._onUnsubscriptionFn = this._onUnsubscription.bind(this);
+		this.node.services.pubsub.removeEventListener(
+			"unsubscribe",
+			this._onUnsubscriptionFn
+		);
+
 		for (const [k, v] of this._pendingDeletes) {
 			v.clear();
 			v.promise.resolve(); // TODO or reject?
@@ -355,24 +514,15 @@ export class SharedLog<T = Uint8Array> extends Program<Args<T>> {
 			v.clear();
 		}
 
+		await this.remoteBlocks.stop();
 		this._gidParentCache.clear();
 		this._pendingDeletes = new Map();
 		this._pendingIHave = new Map();
+		this.latestRoleMessages.clear();
 
 		this._gidPeersHistory = new Map();
 		this._sortedPeersCache = undefined;
 		this._loadedOnce = false;
-
-		this.node.services.pubsub.removeEventListener(
-			"subscribe",
-			this._onSubscriptionFn
-		);
-
-		this._onUnsubscriptionFn = this._onUnsubscription.bind(this);
-		this.node.services.pubsub.removeEventListener(
-			"unsubscribe",
-			this._onUnsubscriptionFn
-		);
 	}
 	async close(from?: Program): Promise<boolean> {
 		const superClosed = await super.close(from);
@@ -389,8 +539,8 @@ export class SharedLog<T = Uint8Array> extends Program<Args<T>> {
 		if (!superDropped) {
 			return superDropped;
 		}
-		await this._close();
 		await this.log.drop();
+		await this._close();
 		return true;
 	}
 
@@ -411,7 +561,6 @@ export class SharedLog<T = Uint8Array> extends Program<Args<T>> {
 				 */
 
 				const { heads } = msg;
-				// replication topic === trustedNetwork address
 
 				logger.debug(
 					`${this.node.identity.publicKey.hashcode()}: Recieved heads: ${
@@ -431,26 +580,32 @@ export class SharedLog<T = Uint8Array> extends Program<Args<T>> {
 						}
 					}
 
-					if (!this._sync) {
-						const toMerge: EntryWithRefs<any>[] = [];
+					if (filteredHeads.length === 0) {
+						return;
+					}
 
-						let toDelete: Entry<any>[] | undefined = undefined;
-						let maybeDelete: EntryWithRefs<any>[][] | undefined = undefined;
+					const toMerge: EntryWithRefs<any>[] = [];
+					let toDelete: Entry<any>[] | undefined = undefined;
+					let maybeDelete: EntryWithRefs<any>[][] | undefined = undefined;
 
-						const groupedByGid = await groupByGid(filteredHeads);
+					const groupedByGid = await groupByGid(filteredHeads);
+					const promises: Promise<void>[] = [];
 
-						for (const [gid, entries] of groupedByGid) {
+					for (const [gid, entries] of groupedByGid) {
+						const fn = async () => {
 							const headsWithGid = this.log.headsIndex.gids.get(gid);
+
 							const maxReplicasFromHead =
 								headsWithGid && headsWithGid.size > 0
 									? maxReplicas(this, [...headsWithGid.values()])
 									: this.replicas.min.getValue(this);
 
-							const maxReplicasFromNewEntries = maxReplicas(this, [
-								...entries.map((x) => x.entry)
-							]);
+							const maxReplicasFromNewEntries = maxReplicas(
+								this,
+								entries.map((x) => x.entry)
+							);
 
-							const isLeader = await this.isLeader(
+							const isLeader = await this.waitForIsLeader(
 								gid,
 								Math.max(maxReplicasFromHead, maxReplicasFromNewEntries)
 							);
@@ -481,50 +636,49 @@ export class SharedLog<T = Uint8Array> extends Program<Args<T>> {
 									}. Because not leader`
 								);
 							}
-						}
+						};
+						promises.push(fn());
+					}
+					await Promise.all(promises);
 
-						if (toMerge.length > 0) {
-							await this.log.join(toMerge);
-							toDelete &&
-								Promise.all(this.pruneSafely(toDelete)).catch((e) => {
-									logger.error(e.toString());
-								});
-						}
+					if (this.closed) {
+						return;
+					}
 
-						if (maybeDelete) {
-							for (const entries of maybeDelete) {
-								const headsWithGid = this.log.headsIndex.gids.get(
-									entries[0].entry.meta.gid
+					if (toMerge.length > 0) {
+						await this.log.join(toMerge);
+						toDelete &&
+							this.prune(toDelete).catch((e) => {
+								logger.error(e.toString());
+							});
+						this.rebalanceParticipationDebounced?.();
+					}
+
+					if (maybeDelete) {
+						for (const entries of maybeDelete as EntryWithRefs<any>[][]) {
+							const headsWithGid = this.log.headsIndex.gids.get(
+								entries[0].entry.meta.gid
+							);
+							if (headsWithGid && headsWithGid.size > 0) {
+								const minReplicas = maxReplicas(this, headsWithGid.values());
+
+								const isLeader = await this.isLeader(
+									entries[0].entry.meta.gid,
+									minReplicas
 								);
-								if (headsWithGid && headsWithGid.size > 0) {
-									const minReplicas = maxReplicas(this, [
-										...headsWithGid.values()
-									]);
-
-									const isLeader = await this.isLeader(
-										entries[0].entry.meta.gid,
-										minReplicas
-									);
-									if (!isLeader) {
-										Promise.all(
-											this.pruneSafely(entries.map((x) => x.entry))
-										).catch((e) => {
-											logger.error(e.toString());
-										});
-									}
+
+								if (!isLeader) {
+									this.prune(entries.map((x) => x.entry)).catch((e) => {
+										logger.error(e.toString());
+									});
 								}
 							}
 						}
-					} else {
-						await this.log.join(
-							await Promise.all(
-								filteredHeads.map((x) => this._sync!(x.entry))
-							).then((filter) => filteredHeads.filter((v, ix) => filter[ix]))
-						);
 					}
 				}
 			} else if (msg instanceof RequestIHave) {
 				const hasAndIsLeader: string[] = [];
+
 				for (const hash of msg.hashes) {
 					const indexedEntry = this.log.entryIndex.getShallow(hash);
 					if (
@@ -542,12 +696,21 @@ export class SharedLog<T = Uint8Array> extends Program<Args<T>> {
 								clearTimeout(timeout);
 								prevPendingIHave?.clear();
 							},
-							callback: () => {
-								prevPendingIHave && prevPendingIHave.callback();
-								this.rpc.send(new ResponseIHave({ hashes: [hash] }), {
-									to: [context.from!]
-								});
-								this._pendingIHave.delete(hash);
+							callback: async (entry) => {
+								if (
+									await this.isLeader(
+										entry.meta.gid,
+										decodeReplicas(entry).getValue(this)
+									)
+								) {
+									this.rpc.send(new ResponseIHave({ hashes: [entry.hash] }), {
+										to: [context.from!]
+									});
+								}
+
+								prevPendingIHave && prevPendingIHave.callback(entry);
+
+								this._pendingIHave.delete(entry.hash);
 							}
 						};
 						const timeout = setTimeout(() => {
@@ -560,17 +723,71 @@ export class SharedLog<T = Uint8Array> extends Program<Args<T>> {
 						this._pendingIHave.set(hash, pendingIHave);
 					}
 				}
-				this.rpc.send(new ResponseIHave({ hashes: hasAndIsLeader }), {
+				await this.rpc.send(new ResponseIHave({ hashes: hasAndIsLeader }), {
 					to: [context.from!]
 				});
 			} else if (msg instanceof ResponseIHave) {
 				for (const hash of msg.hashes) {
 					this._pendingDeletes.get(hash)?.callback(context.from!.hashcode());
 				}
+			} else if (msg instanceof BlocksMessage) {
+				await this.remoteBlocks.onMessage(msg.message);
+			} else if (msg instanceof RequestRoleMessage) {
+				if (!context.from) {
+					throw new Error("Missing form in update role message");
+				}
+
+				if (context.from.equals(this.node.identity.publicKey)) {
+					return;
+				}
+
+				await this.rpc.send(new ResponseRoleMessage({ role: this.role }), {
+					to: [context.from!]
+				});
+			} else if (msg instanceof ResponseRoleMessage) {
+				if (!context.from) {
+					throw new Error("Missing form in update role message");
+				}
+
+				if (context.from.equals(this.node.identity.publicKey)) {
+					return;
+				}
+
+				this.waitFor(context.from, {
+					signal: this._closeController.signal,
+					timeout: WAIT_FOR_REPLICATOR_TIMEOUT
+				})
+					.then(async () => {
+						/* await delay(1000 * Math.random()) */
+
+						const prev = this.latestRoleMessages.get(context.from!.hashcode());
+						if (prev && prev > context.timestamp) {
+							return;
+						}
+						this.latestRoleMessages.set(
+							context.from!.hashcode(),
+							context.timestamp
+						);
+
+						await this.modifyReplicators(msg.role, context.from!);
+					})
+					.catch((e) => {
+						if (e instanceof AbortError) {
+							return;
+						}
+
+						logger.error(
+							"Failed to find peer who updated their role: " + e?.message
+						);
+					});
 			} else {
 				throw new Error("Unexpected message");
 			}
 		} catch (e: any) {
+			if (e instanceof AbortError) {
+				return;
+			}
+
 			if (e instanceof BorshError) {
 				logger.trace(
 					`${this.node.identity.publicKey.hashcode()}: Failed to handle message on topic: ${JSON.stringify(
@@ -579,6 +796,7 @@ export class SharedLog<T = Uint8Array> extends Program<Args<T>> {
 				);
 				return;
 			}
+
 			if (e instanceof AccessError) {
 				logger.trace(
 					`${this.node.identity.publicKey.hashcode()}: Failed to handle message for log: ${JSON.stringify(
@@ -591,41 +809,89 @@ export class SharedLog<T = Uint8Array> extends Program<Args<T>> {
 		}
 	}
 
-	getReplicatorsSorted(): { hash: string; timestamp: number }[] | undefined {
+	getReplicatorsSorted(): yallist<ReplicatorRect> | undefined {
 		return this._sortedPeersCache;
 	}
 
+	async waitForReplicator(...keys: PublicSignKey[]) {
+		const check = () => {
+			for (const k of keys) {
+				if (
+					!this.getReplicatorsSorted()
+						?.toArray()
+						?.find((x) => x.publicKey.equals(k))
+				) {
+					return false;
+				}
+			}
+			return true;
+		};
+		return waitFor(() => check(), { signal: this._closeController.signal });
+	}
+
 	async isLeader(
 		slot: { toString(): string },
-		numberOfLeaders: number
+		numberOfLeaders: number,
+		options?: {
+			candidates?: string[];
+			roleAge?: number;
+		}
 	): Promise<boolean> {
-		const isLeader = (await this.findLeaders(slot, numberOfLeaders)).find(
-			(l) => l === this.node.identity.publicKey.hashcode()
-		);
+		const isLeader = (
+			await this.findLeaders(slot, numberOfLeaders, options)
+		).find((l) => l === this.node.identity.publicKey.hashcode());
 		return !!isLeader;
 	}
 
+	private async waitForIsLeader(
+		slot: { toString(): string },
+		numberOfLeaders: number,
+		timeout = WAIT_FOR_REPLICATOR_TIMEOUT
+	): Promise<boolean> {
+		return new Promise((res, rej) => {
+			const removeListeners = () => {
+				this.events.removeEventListener("role", roleListener);
+				this._closeController.signal.addEventListener("abort", abortListener);
+			};
+			const abortListener = () => {
+				removeListeners();
+				clearTimeout(timer);
+				res(false);
+			};
+
+			const timer = setTimeout(() => {
+				removeListeners();
+				res(false);
+			}, timeout);
+
+			const check = () =>
+				this.isLeader(slot, numberOfLeaders).then((isLeader) => {
+					if (isLeader) {
+						removeListeners();
+						clearTimeout(timer);
+						res(isLeader);
+					}
+				});
+
+			const roleListener = () => {
+				check();
+			};
+			this.events.addEventListener("role", roleListener);
+			this._closeController.signal.addEventListener("abort", abortListener);
+
+			check();
+		});
+	}
+
 	async findLeaders(
 		subject: { toString(): string },
-		numberOfLeadersUnbounded: number
+		numberOfLeaders: number,
+		options?: {
+			roleAge?: number;
+		}
 	): Promise<string[]> {
-		const lower = this.replicas.min.getValue(this);
-		const higher = this.replicas.max?.getValue(this) ?? Number.MAX_SAFE_INTEGER;
-		let numberOfLeaders = Math.max(
-			Math.min(higher, numberOfLeadersUnbounded),
-			lower
-		);
-
 		// For a fixed set or members, the choosen leaders will always be the same (address invariant)
 		// This allows for that same content is always chosen to be distributed to same peers, to remove unecessary copies
-		const peers: { hash: string; timestamp: number }[] =
-			this.getReplicatorsSorted() || [];
-
-		if (peers.length === 0) {
-			return [];
-		}
-
-		numberOfLeaders = Math.min(numberOfLeaders, peers.length);
 
 		// Convert this thing we wan't to distribute to 8 bytes so we get can convert it into a u64
 		// modulus into an index
@@ -634,31 +900,220 @@ export class SharedLog<T = Uint8Array> extends Program<Args<T>> {
 		const seed = await sha256(utf8writer.finalize());
 
 		// convert hash of slot to a number
-		const seedNumber = new BinaryReader(
-			seed.subarray(seed.length - 8, seed.length)
-		).u64();
-		const startIndex = Number(seedNumber % BigInt(peers.length));
-
-		// we only step forward 1 step (ignoring that step backward 1 could be 'closer')
-		// This does not matter, we only have to make sure all nodes running the code comes to somewhat the
-		// same conclusion (are running the same leader selection algorithm)
-		const leaders = new Array(numberOfLeaders);
+		const cursor = hashToUniformNumber(seed); // bounded between 0 and 1
+		return this.findLeadersFromUniformNumber(cursor, numberOfLeaders, options);
+	}
+
+	private findLeadersFromUniformNumber(
+		cursor: number,
+		numberOfLeaders: number,
+		options?: {
+			roleAge?: number;
+		}
+	) {
+		const leaders: Set<string> = new Set();
+		const width = 1; // this.getParticipationSum(roleAge);
+		const peers = this.getReplicatorsSorted();
+		if (!peers || peers?.length === 0) {
+			return [];
+		}
+		numberOfLeaders = Math.min(numberOfLeaders, peers.length);
+
+		const t = +new Date();
+		const roleAge =
+			options?.roleAge ??
+			Math.min(WAIT_FOR_ROLE_MATURITY, +new Date() - this.openTime);
+
 		for (let i = 0; i < numberOfLeaders; i++) {
-			leaders[i] = peers[(i + startIndex) % peers.length].hash;
+			let matured = 0;
+			const maybeIncrementMatured = (role: Replicator) => {
+				if (t - Number(role.timestamp) > roleAge) {
+					matured++;
+				}
+			};
+
+			const x = ((cursor + i / numberOfLeaders) % 1) * width;
+			let currentNode = peers.head;
+			const diffs: { diff: number; rect: ReplicatorRect }[] = [];
+			while (currentNode) {
+				const start = currentNode.value.offset % width;
+				const absDelta = Math.abs(start - x);
+				const diff = Math.min(absDelta, width - absDelta);
+
+				if (diff < currentNode.value.role.factor / 2 + 0.00001) {
+					leaders.add(currentNode.value.publicKey.hashcode());
+					maybeIncrementMatured(currentNode.value.role);
+				} else {
+					diffs.push({
+						diff:
+							currentNode.value.role.factor > 0
+								? diff / currentNode.value.role.factor
+								: Number.MAX_SAFE_INTEGER,
+						rect: currentNode.value
+					});
+				}
+
+				currentNode = currentNode.next;
+			}
+
+			if (matured === 0) {
+				diffs.sort((x, y) => x.diff - y.diff);
+				for (const node of diffs) {
+					leaders.add(node.rect.publicKey.hashcode());
+					maybeIncrementMatured(node.rect.role);
+					if (matured > 0) {
+						break;
+					}
+				}
+			}
+		}
+
+		return [...leaders];
+	}
+
+	/**
+	 *
+	 * @returns groups where at least one in any group will have the entry you are looking for
+	 */
+	getReplicatorUnion(roleAge: number = WAIT_FOR_ROLE_MATURITY) {
+		// Total replication "width"
+		const width = 1; //this.getParticipationSum(roleAge);
+
+		// How much width you need to "query" to
+
+		const peers = this.getReplicatorsSorted()!; // TODO types
+		const minReplicas = Math.min(
+			peers.length,
+			this.replicas.min.getValue(this)
+		);
+		const coveringWidth = width / minReplicas;
+
+		let walker = peers.head;
+		if (this.role instanceof Replicator) {
+			// start at our node (local first)
+			while (walker) {
+				if (walker.value.publicKey.equals(this.node.identity.publicKey)) {
+					break;
+				}
+				walker = walker.next;
+			}
+		} else {
+			const seed = Math.round(peers.length * Math.random()); // start at a random point
+			for (let i = 0; i < seed - 1; i++) {
+				if (walker?.next == null) {
+					break;
+				}
+				walker = walker.next;
+			}
+		}
+
+		const set: string[] = [];
+		let distance = 0;
+		const startNode = walker;
+		if (!startNode) {
+			return [];
+		}
+
+		let nextPoint = startNode.value.offset;
+		const t = +new Date();
+		while (walker && distance < coveringWidth) {
+			const absDelta = Math.abs(walker!.value.offset - nextPoint);
+			const diff = Math.min(absDelta, width - absDelta);
+
+			if (diff < walker!.value.role.factor / 2 + 0.00001) {
+				set.push(walker!.value.publicKey.hashcode());
+				if (
+					t - Number(walker!.value.role.timestamp) >
+					roleAge /* ||
+					walker!.value.publicKey.equals(this.node.identity.publicKey)) */
+				) {
+					nextPoint = (nextPoint + walker!.value.role.factor) % 1;
+					distance += walker!.value.role.factor;
+				}
+			}
+
+			walker = walker.next || peers.head;
+
+			if (
+				walker?.value.publicKey &&
+				startNode?.value.publicKey.equals(walker?.value.publicKey)
+			) {
+				break; // TODO throw error for failing to fetch ffull width
+			}
+		}
+
+		return set;
+	}
+
+	async replicator(
+		entry: Entry<any>,
+		options?: {
+			candidates?: string[];
+			roleAge?: number;
 		}
-		return leaders;
+	) {
+		return this.isLeader(
+			entry.gid,
+			decodeReplicas(entry).getValue(this),
+			options
+		);
 	}
 
-	private async modifySortedSubscriptionCache(
-		subscribed: boolean,
+	private onRoleChange(
+		prev: Observer | Replicator | undefined,
+		role: Observer | Replicator,
 		publicKey: PublicSignKey
 	) {
+		if (this.closed) {
+			return;
+		}
+
+		this.distribute();
+
+		if (role instanceof Replicator) {
+			const timer = setTimeout(async () => {
+				this._closeController.signal.removeEventListener("abort", listener);
+				await this.rebalanceParticipationDebounced?.();
+				this.distribute();
+			}, WAIT_FOR_ROLE_MATURITY + 2000);
+
+			const listener = () => {
+				clearTimeout(timer);
+			};
+
+			this._closeController.signal.addEventListener("abort", listener);
+		}
+
+		this.events.dispatchEvent(
+			new CustomEvent<UpdateRoleEvent>("role", {
+				detail: { publicKey, role }
+			})
+		);
+	}
+
+	private async modifyReplicators(
+		role: Observer | Replicator,
+		publicKey: PublicSignKey
+	) {
+		const { prev, changed } = await this._modifyReplicators(role, publicKey);
+		if (changed) {
+			await this.rebalanceParticipationDebounced?.(); // await this.rebalanceParticipation(false);
+			this.onRoleChange(prev, role, publicKey);
+			return true;
+		}
+		return false;
+	}
+
+	private async _modifyReplicators(
+		role: Observer | Replicator,
+		publicKey: PublicSignKey
+	): Promise<{ prev?: Replicator; changed: boolean }> {
 		if (
-			subscribed &&
+			role instanceof Replicator &&
 			this._canReplicate &&
-			!(await this._canReplicate(publicKey))
+			!(await this._canReplicate(publicKey, role))
 		) {
-			return false;
+			return { changed: false };
 		}
 
 		const sortedPeer = this._sortedPeersCache;
@@ -666,81 +1121,125 @@ export class SharedLog<T = Uint8Array> extends Program<Args<T>> {
 			if (this.closed === false) {
 				throw new Error("Unexpected, sortedPeersCache is undefined");
 			}
-			return false;
+			return { changed: false };
 		}
-		const code = publicKey.hashcode();
-		if (subscribed) {
+
+		if (role instanceof Replicator && role.factor > 0) {
 			// TODO use Set + list for fast lookup
-			if (!sortedPeer.find((x) => x.hash === code)) {
-				sortedPeer.push({ hash: code, timestamp: +new Date() });
-				sortedPeer.sort((a, b) => a.hash.localeCompare(b.hash));
-				return true;
+			// check also that peer is online
+
+			const isOnline =
+				this.node.identity.publicKey.equals(publicKey) ||
+				(await this.waitFor(publicKey, { signal: this._closeController.signal })
+					.then(() => true)
+					.catch(() => false));
+
+			if (isOnline) {
+				// insert or if already there do nothing
+				const code = hashToUniformNumber(publicKey.bytes);
+				const rect: ReplicatorRect = {
+					publicKey,
+					offset: code,
+					role
+				};
+
+				let currentNode = sortedPeer.head;
+				if (!currentNode) {
+					sortedPeer.push(rect);
+					return { changed: true };
+				} else {
+					while (currentNode) {
+						if (currentNode.value.publicKey.equals(publicKey)) {
+							// update the value
+							// rect.timestamp = currentNode.value.timestamp;
+							const prev = currentNode.value;
+							currentNode.value = rect;
+							// TODO change detection and only do change stuff if diff?
+							return { prev: prev.role, changed: true };
+						}
+
+						if (code > currentNode.value.offset) {
+							const next = currentNode?.next;
+							if (next) {
+								currentNode = next;
+								continue;
+							} else {
+								break;
+							}
+						} else {
+							currentNode = currentNode.prev;
+							break;
+						}
+					}
+
+					const prev = currentNode;
+					if (!prev?.next?.value.publicKey.equals(publicKey)) {
+						_insertAfter(sortedPeer, prev || undefined, rect);
+					} else {
+						throw new Error("Unexpected");
+					}
+					return { changed: true };
+				}
 			} else {
-				return false;
+				return { changed: false };
 			}
 		} else {
-			const deleteIndex = sortedPeer.findIndex((x) => x.hash === code);
-			if (deleteIndex >= 0) {
-				sortedPeer.splice(deleteIndex, 1);
-				return true;
-			} else {
-				return false;
+			let currentNode = sortedPeer.head;
+			while (currentNode) {
+				if (currentNode.value.publicKey.equals(publicKey)) {
+					sortedPeer.removeNode(currentNode);
+					return { prev: currentNode.value.role, changed: true };
+				}
+				currentNode = currentNode.next;
 			}
+			return { changed: false };
 		}
 	}
 
 	async handleSubscriptionChange(
 		publicKey: PublicSignKey,
-		changes: { topic: string; data?: Uint8Array }[],
+		changes: string[],
 		subscribed: boolean
 	) {
-		// TODO why are we doing two loops?
-		const prev: boolean[] = [];
-		for (const subscription of changes) {
-			if (this.log.idString !== subscription.topic) {
-				continue;
-			}
-
-			if (
-				!subscription.data ||
-				!startsWith(subscription.data, REPLICATOR_TYPE_VARIANT)
-			) {
-				prev.push(await this.modifySortedSubscriptionCache(false, publicKey));
-				continue;
-			} else {
-				this._lastSubscriptionMessageId += 1;
-				prev.push(
-					await this.modifySortedSubscriptionCache(subscribed, publicKey)
-				);
-			}
-		}
-
-		// TODO don't do this i fnot is replicator?
-		for (const [i, subscription] of changes.entries()) {
-			if (this.log.idString !== subscription.topic) {
-				continue;
+		if (subscribed) {
+			if (this.role instanceof Replicator) {
+				for (const subscription of changes) {
+					if (this.log.idString !== subscription) {
+						continue;
+					}
+					this.rpc
+						.send(new ResponseRoleMessage(this.role), {
+							mode: new AcknowledgeDelivery({ redundancy: 1, to: [publicKey] })
+						})
+						.catch((e) => logger.error(e.toString()));
+				}
 			}
-			if (subscription.data) {
-				try {
-					const type = deserialize(subscription.data, Role);
 
-					// Reorganize if the new subscriber is a replicator, or observers AND was replicator
-					if (type instanceof Replicator || prev[i]) {
-						await this.replicationReorganization();
-					}
-				} catch (error: any) {
-					logger.warn(
-						"Recieved subscription with invalid data on topic: " +
-							subscription.topic +
-							". Error: " +
-							error?.message
-					);
+			//if(evt.detail.subscriptions.map((x) => x.topic).includes())
+		} else {
+			for (const topic of changes) {
+				if (this.log.idString !== topic) {
+					continue;
 				}
+
+				await this.modifyReplicators(new Observer(), publicKey);
 			}
 		}
 	}
 
-	pruneSafely(entries: Entry<any>[], options?: { timeout: number }) {
+	async prune(
+		entries: Entry<any>[],
+		options?: { timeout?: number; unchecked?: boolean }
+	): Promise<any> {
+		if (options?.unchecked) {
+			return Promise.all(
+				entries.map((x) =>
+					this.log.remove(x, {
+						recursively: true
+					})
+				)
+			);
+		}
 		// ask network if they have they entry,
 		// so I can delete it
 
@@ -754,14 +1253,17 @@ export class SharedLog<T = Uint8Array> extends Program<Args<T>> {
 		const filteredEntries: Entry<any>[] = [];
 		for (const entry of entries) {
 			const pendingPrev = this._pendingDeletes.get(entry.hash);
-
+			if (pendingPrev) {
+				promises.push(pendingPrev.promise.promise);
+				continue;
+			}
 			filteredEntries.push(entry);
 			const existCounter = new Set<string>();
 			const minReplicas = decodeReplicas(entry);
 			const deferredPromise: DeferredPromise<void> = pDefer();
 
 			const clear = () => {
-				pendingPrev?.clear();
+				//pendingPrev?.clear();
 				const pending = this._pendingDeletes.get(entry.hash);
 				if (pending?.promise == deferredPromise) {
 					this._pendingDeletes.delete(entry.hash);
@@ -780,7 +1282,7 @@ export class SharedLog<T = Uint8Array> extends Program<Args<T>> {
 
 			const timeout = setTimeout(
 				() => {
-					reject(new Error("Timeout"));
+					reject(new Error("Timeout for checked pruning"));
 				},
 				options?.timeout ?? 10 * 1000
 			);
@@ -792,10 +1294,25 @@ export class SharedLog<T = Uint8Array> extends Program<Args<T>> {
 				},
 				callback: async (publicKeyHash: string) => {
 					const minReplicasValue = minReplicas.getValue(this);
-					const l = await this.findLeaders(entry.gid, minReplicasValue);
-					if (l.find((x) => x === publicKeyHash)) {
+					const minMinReplicasValue = this.replicas.max
+						? Math.min(minReplicasValue, this.replicas.max.getValue(this))
+						: minReplicasValue;
+
+					const leaders = await this.findLeaders(
+						entry.gid,
+						minMinReplicasValue
+					);
+
+					if (
+						leaders.find((x) => x === this.node.identity.publicKey.hashcode())
+					) {
+						reject(new Error("Failed to delete, is leader"));
+						return;
+					}
+
+					if (leaders.find((x) => x === publicKeyHash)) {
 						existCounter.add(publicKeyHash);
-						if (minReplicas.getValue(this) <= existCounter.size) {
+						if (minMinReplicasValue <= existCounter.size) {
 							this.log
 								.remove(entry, {
 									recursively: true
@@ -812,28 +1329,52 @@ export class SharedLog<T = Uint8Array> extends Program<Args<T>> {
 			});
 			promises.push(deferredPromise.promise);
 		}
-		if (filteredEntries.length > 0) {
-			this.rpc.send(
-				new RequestIHave({ hashes: filteredEntries.map((x) => x.hash) })
-			);
+		if (filteredEntries.length == 0) {
+			return;
 		}
 
-		return promises;
+		this.rpc.send(
+			new RequestIHave({ hashes: filteredEntries.map((x) => x.hash) })
+		);
+
+		const onNewPeer = async (e: CustomEvent<UpdateRoleEvent>) => {
+			if (e.detail.role instanceof Replicator) {
+				await this.rpc.send(
+					new RequestIHave({ hashes: filteredEntries.map((x) => x.hash) }),
+					{
+						to: [e.detail.publicKey.hashcode()]
+					}
+				);
+			}
+		};
+		// check joining peers
+		this.events.addEventListener("role", onNewPeer);
+		return Promise.all(promises).finally(() =>
+			this.events.removeEventListener("role", onNewPeer)
+		);
 	}
 
-	/**
-	 * When a peers join the networkk and want to participate the leaders for particular log subgraphs might change, hence some might start replicating, might some stop
-	 * This method will go through my owned entries, and see whether I should share them with a new leader, and/or I should stop care about specific entries
-	 * @param channel
-	 */
-	async replicationReorganization() {
+	async distribute() {
+		/**
+		 * TODO use information of new joined/leaving peer to create a subset of heads
+		 * that we potentially need to share with other peers
+		 */
+
+		if (this.closed) {
+			return;
+		}
+
 		const changed = false;
+		await this.log.trim();
 		const heads = await this.log.getHeads();
 		const groupedByGid = await groupByGid(heads);
-		let storeChanged = false;
+		const toDeliver: Map<string, Entry<any>[]> = new Map();
+		const allEntriesToDelete: Entry<any>[] = [];
+
 		for (const [gid, entries] of groupedByGid) {
-			const toSend: Map<string, Entry<any>> = new Map();
-			const newPeers: string[] = [];
+			if (this.closed) {
+				break;
+			}
 
 			if (entries.length === 0) {
 				continue; // TODO maybe close store?
@@ -844,121 +1385,91 @@ export class SharedLog<T = Uint8Array> extends Program<Args<T>> {
 				gid,
 				maxReplicas(this, entries) // pick max replication policy of all entries, so all information is treated equally important as the most important
 			);
+			const currentPeersSet = new Set(currentPeers);
+			this._gidPeersHistory.set(gid, currentPeersSet);
 
 			for (const currentPeer of currentPeers) {
-				if (
-					!oldPeersSet?.has(currentPeer) &&
-					currentPeer !== this.node.identity.publicKey.hashcode()
-				) {
-					storeChanged = true;
+				if (currentPeer == this.node.identity.publicKey.hashcode()) {
+					continue;
+				}
+
+				if (!oldPeersSet?.has(currentPeer)) {
 					// second condition means that if the new peer is us, we should not do anything, since we are expecting to receive heads, not send
-					newPeers.push(currentPeer);
-
-					// send heads to the new peer
-					// console.log('new gid for peer', newPeers.length, this.id.toString(), newPeer, gid, entries.length, newPeers)
-					try {
-						logger.debug(
-							`${this.node.identity.publicKey.hashcode()}: Exchange heads ${
-								entries.length === 1 ? entries[0].hash : "#" + entries.length
-							}  on rebalance`
-						);
-						for (const entry of entries) {
-							toSend.set(entry.hash, entry);
-						}
-					} catch (error) {
-						if (error instanceof TimeoutError) {
-							logger.error(
-								"Missing channel when reorg to peer: " + currentPeer.toString()
-							);
-							continue;
-						}
-						throw error;
+					let arr = toDeliver.get(currentPeer);
+					if (!arr) {
+						arr = [];
+						toDeliver.set(currentPeer, arr);
+					}
+
+					for (const entry of entries) {
+						arr.push(entry);
 					}
 				}
 			}
 
-			// We don't need this clause anymore because we got the trim option!
 			if (
 				!currentPeers.find((x) => x === this.node.identity.publicKey.hashcode())
 			) {
-				let entriesToDelete = entries.filter((e) => !e.createdLocally);
-
-				if (this._sync) {
-					// dont delete entries which we wish to keep
-					entriesToDelete = await Promise.all(
-						entriesToDelete.map((x) => this._sync!(x))
-					).then((filter) => entriesToDelete.filter((v, ix) => !filter[ix]));
+				if (currentPeers.length > 0) {
+					// If we are observer, never prune locally created entries, since we dont really know who can store them
+					// if we are replicator, we will always persist entries that we need to so filtering on createdLocally will not make a difference
+					const entriesToDelete =
+						this._role instanceof Observer
+							? entries.filter((e) => !e.createdLocally)
+							: entries;
+					entriesToDelete.map((x) => this._gidPeersHistory.delete(x.meta.gid));
+					allEntriesToDelete.push(...entriesToDelete);
 				}
-
-				// delete entries since we are not suppose to replicate this anymore
-				// TODO add delay? freeze time? (to ensure resiliance for bad io)
-				if (entriesToDelete.length > 0) {
-					Promise.all(this.pruneSafely(entriesToDelete)).catch((e) => {
-						logger.error(e.toString());
-					});
+			} else {
+				for (const entry of entries) {
+					this._pendingDeletes
+						.get(entry.hash)
+						?.promise.reject(
+							new Error(
+								"Failed to delete, is leader: " +
+									this.role.constructor.name +
+									". " +
+									this.node.identity.publicKey.hashcode()
+							)
+						);
 				}
-
-				// TODO if length === 0 maybe close store?
 			}
-			this._gidPeersHistory.set(gid, new Set(currentPeers));
+		}
 
-			if (toSend.size === 0) {
-				continue;
-			}
+		for (const [target, entries] of toDeliver) {
 			const message = await createExchangeHeadsMessage(
 				this.log,
-				[...toSend.values()], // TODO send to peers directly
+				entries, // TODO send to peers directly
 				this._gidParentCache
 			);
-
 			// TODO perhaps send less messages to more receivers for performance reasons?
 			await this.rpc.send(message, {
-				to: newPeers,
-				strict: true
+				to: [target]
 			});
 		}
-		if (storeChanged) {
-			await this.log.trim(); // because for entries createdLocally,we can have trim options that still allow us to delete them
-		}
-		return storeChanged || changed;
-	}
 
-	/**
-	 *
-	 * @returns groups where at least one in any group will have the entry you are looking for
-	 */
-	getDiscoveryGroups() {
-		// TODO Optimize this so we don't have to recreate the array all the time!
-		const minReplicas = this.replicas.min.getValue(this);
-		const replicators = this.getReplicatorsSorted();
-		if (!replicators) {
-			return []; // No subscribers and we are not replicating
-		}
-		const numberOfGroups = Math.min(
-			Math.ceil(replicators!.length / minReplicas)
-		);
-		const groups = new Array<{ hash: string; timestamp: number }[]>(
-			numberOfGroups
-		);
-		for (let i = 0; i < groups.length; i++) {
-			groups[i] = [];
-		}
-		for (let i = 0; i < replicators!.length; i++) {
-			groups[i % numberOfGroups].push(replicators![i]);
+		if (allEntriesToDelete.length > 0) {
+			this.prune(allEntriesToDelete).catch((e) => {
+				logger.error(e.toString());
+			});
 		}
 
-		return groups;
-	}
-	async replicator(entry: Entry<any>) {
-		return this.isLeader(entry.gid, decodeReplicas(entry).getValue(this));
+		return changed;
 	}
 
 	async _onUnsubscription(evt: CustomEvent<UnsubcriptionEvent>) {
 		logger.debug(
 			`Peer disconnected '${evt.detail.from.hashcode()}' from '${JSON.stringify(
-				evt.detail.unsubscriptions.map((x) => x.topic)
+				evt.detail.unsubscriptions.map((x) => x)
 			)}'`
 		);
+		this.latestRoleMessages.delete(evt.detail.from.hashcode());
+
+		this.events.dispatchEvent(
+			new CustomEvent<UpdateRoleEvent>("role", {
+				detail: { publicKey: evt.detail.from, role: new Observer() }
+			})
+		);
 
 		return this.handleSubscriptionChange(
 			evt.detail.from,
@@ -970,13 +1481,257 @@ export class SharedLog<T = Uint8Array> extends Program<Args<T>> {
 	async _onSubscription(evt: CustomEvent<SubscriptionEvent>) {
 		logger.debug(
 			`New peer '${evt.detail.from.hashcode()}' connected to '${JSON.stringify(
-				evt.detail.subscriptions.map((x) => x.topic)
+				evt.detail.subscriptions.map((x) => x)
 			)}'`
 		);
+		this.remoteBlocks.onReachable(evt.detail.from);
+
 		return this.handleSubscriptionChange(
 			evt.detail.from,
 			evt.detail.subscriptions,
 			true
 		);
 	}
+	replicationController: ReplicationController;
+
+	history: { usedMemory: number; factor: number }[];
+	async addToHistory(usedMemory: number, factor: number) {
+		(this.history || (this.history = [])).push({ usedMemory, factor });
+
+		// Keep only the last N entries in the history array (you can adjust N based on your needs)
+		const maxHistoryLength = 10;
+		if (this.history.length > maxHistoryLength) {
+			this.history.shift();
+		}
+	}
+
+	async calculateTrend() {
+		// Calculate the average change in factor per unit change in memory usage
+		const factorChanges = this.history.map((entry, index) => {
+			if (index > 0) {
+				const memoryChange =
+					entry.usedMemory - this.history[index - 1].usedMemory;
+				if (memoryChange !== 0) {
+					const factorChange = entry.factor - this.history[index - 1].factor;
+					return factorChange / memoryChange;
+				}
+			}
+			return 0;
+		});
+
+		// Return the average factor change per unit memory change
+		return (
+			factorChanges.reduce((sum, change) => sum + change, 0) /
+			factorChanges.length
+		);
+	}
+
+	private sumFactors() {
+		let sum = 0;
+		for (const element of this.getReplicatorsSorted()?.toArray() || []) {
+			sum += element.role.factor;
+		}
+
+		return sum;
+	}
+
+	async rebalanceParticipation(onRoleChange = true) {
+		// update more participation rate to converge to the average expected rate or bounded by
+		// resources such as memory and or cpu
+
+		if (this.closed) {
+			return false;
+		}
+
+		// The role is fixed (no changes depending on memory usage or peer count etc)
+		if (this._roleOptions instanceof Role) {
+			return false;
+		}
+
+		// TODO second condition: what if the current role is Observer?
+		if (
+			this._roleOptions.type == "replicator" &&
+			this._role instanceof Replicator
+		) {
+			const peers = this.getReplicatorsSorted();
+			const usedMemory = await this.getMemoryUsage();
+
+			const newFactor =
+				await this.replicationController.adjustReplicationFactor(
+					usedMemory,
+					this._role.factor,
+					this.sumFactors(),
+					peers?.length || 1
+				);
+
+			const newRole = new Replicator({
+				factor: newFactor,
+				timestamp: this._role.timestamp
+			});
+
+			const relativeDifference =
+				Math.abs(this._role.factor - newRole.factor) / this._role.factor;
+
+			if (relativeDifference > 0.0001) {
+				const canReplicate =
+					!this._canReplicate ||
+					(await this._canReplicate(this.node.identity.publicKey, newRole));
+				if (!canReplicate) {
+					return false;
+				}
+
+				await this._updateRole(newRole, onRoleChange);
+				return true;
+			}
+			return false;
+		}
+		return false;
+	}
+}
+
+function _insertAfter(
+	self: yallist<any>,
+	node: yallist.Node<ReplicatorRect> | undefined,
+	value: ReplicatorRect
+) {
+	const inserted = !node
+		? new yallist.Node(
+				value,
+				null as any,
+				self.head as yallist.Node<ReplicatorRect> | undefined,
+				self
+			)
+		: new yallist.Node(
+				value,
+				node,
+				node.next as yallist.Node<ReplicatorRect> | undefined,
+				self
+			);
+
+	// is tail
+	if (inserted.next === null) {
+		self.tail = inserted;
+	}
+
+	// is head
+	if (inserted.prev === null) {
+		self.head = inserted;
+	}
+
+	self.length++;
+	return inserted;
+}
+
+class ReplicationController {
+	integral = 0;
+	prevError = 0;
+	prevMemoryUsage = 0;
+	lastTs = 0;
+	kp: number;
+	ki: number;
+	kd: number;
+	errorFunction: ReplicationErrorFunction;
+	targetMemoryLimit?: number;
+	constructor(
+		options: {
+			errorFunction?: ReplicationErrorFunction;
+			targetMemoryLimit?: number;
+			kp?: number;
+			ki?: number;
+			kd?: number;
+		} = {}
+	) {
+		const {
+			targetMemoryLimit,
+			kp = 0.1,
+			ki = 0 /* 0.01, */,
+			kd = 0.1,
+			errorFunction = ({ balance, coverage, memory }) =>
+				memory * 0.8 + balance * 0.1 + coverage * 0.1
+		} = options;
+		this.kp = kp;
+		this.ki = ki;
+		this.kd = kd;
+		this.targetMemoryLimit = targetMemoryLimit;
+		this.errorFunction = errorFunction;
+	}
+
+	async adjustReplicationFactor(
+		memoryUsage: number,
+		currentFactor: number,
+		totalFactor: number,
+		peerCount: number
+	) {
+		/* 	if (memoryUsage === this.prevMemoryUsage) {
+				return Math.max(Math.min(currentFactor, maxFraction), minFraction);
+			} */
+
+		this.prevMemoryUsage = memoryUsage;
+		if (memoryUsage <= 0) {
+			this.integral = 0;
+		}
+		const normalizedFactor = currentFactor / totalFactor;
+		/*const estimatedTotalSize = memoryUsage / normalizedFactor;
+
+		 const errorMemory =
+			currentFactor > 0 && memoryUsage > 0
+				? (Math.max(Math.min(maxFraction, this.targetMemoryLimit / estimatedTotalSize), minFraction) -
+					normalizedFactor) * totalFactor
+				: minFraction; */
+
+		const estimatedTotalSize = memoryUsage / currentFactor;
+		const errorCoverage = Math.min(1 - totalFactor, 1);
+
+		let errorMemory = 0;
+		const errorTarget = 1 / peerCount - currentFactor;
+
+		if (this.targetMemoryLimit != null) {
+			errorMemory =
+				currentFactor > 0 && memoryUsage > 0
+					? Math.max(
+							Math.min(1, this.targetMemoryLimit / estimatedTotalSize),
+							0
+						) - currentFactor
+					: 0.0001;
+
+			/* errorTarget = errorMemory + (1 / peerCount - currentFactor) / 10; */
+		}
+
+		// alpha * errorCoverage + (1 - alpha) * errorTarget;
+		const totalError = this.errorFunction({
+			balance: errorTarget,
+			coverage: errorCoverage,
+			memory: errorMemory
+		});
+
+		if (this.lastTs === 0) {
+			this.lastTs = +new Date();
+		}
+		const kpAdjusted = Math.min(
+			Math.max(this.kp, (+new Date() - this.lastTs) / 100),
+			0.8
+		);
+		const pTerm = kpAdjusted * totalError;
+
+		this.lastTs = +new Date();
+
+		// Integral term
+		this.integral += totalError;
+		const beta = 0.4;
+		this.integral = beta * totalError + (1 - beta) * this.integral;
+
+		const iTerm = this.ki * this.integral;
+
+		// Derivative term
+		const derivative = totalError - this.prevError;
+		const dTerm = this.kd * derivative;
+
+		// Calculate the new replication factor
+		const newFactor = currentFactor + pTerm + iTerm + dTerm;
+
+		// Update state for the next iteration
+		this.prevError = totalError;
+
+		return Math.max(Math.min(newFactor, 1), 0);
+	}
 }