@peerbit/shared-log 9.0.9-e1db01f → 9.0.10-ccaf4f4

package/src/index.ts

@@ -1,11 +1,10 @@
-import { BinaryWriter, BorshError, field, variant } from "@dao-xyz/borsh";
+import { BorshError, field, variant } from "@dao-xyz/borsh";
 import { CustomEvent } from "@libp2p/interface";
 import { AnyBlockStore, RemoteBlocks } from "@peerbit/blocks";
 import { Cache } from "@peerbit/cache";
 import {
 	AccessError,
 	PublicSignKey,
-	sha256,
 	sha256Base64Sync,
 	sha256Sync,
 } from "@peerbit/crypto";
@@ -15,13 +14,11 @@ import {
 	CountRequest,
 	DeleteRequest,
 	type Index,
-	IntegerCompare,
 	Or,
 	SearchRequest,
 	Sort,
 	StringMatch,
 	SumRequest,
-	toId,
 } from "@peerbit/indexer-interface";
 import {
 	type AppendOptions,
@@ -50,6 +47,7 @@ import { AbortError, delay, waitFor } from "@peerbit/time";
 import debounce from "p-debounce";
 import pDefer, { type DeferredPromise } from "p-defer";
 import PQueue from "p-queue";
+import { concat } from "uint8arrays";
 import { BlocksMessage } from "./blocks.js";
 import { type CPUUsage, CPUUsageIntervalLag } from "./cpu.js";
 import {
@@ -63,32 +61,73 @@ import {
 } from "./exchange-heads.js";
 import { TransportMessage } from "./message.js";
 import { PIDReplicationController } from "./pid.js";
-import { getCoverSet, getSamples, isMatured } from "./ranges.js";
+import {
+	getCoverSet,
+	getSamples,
+	hasCoveringRange,
+	isMatured,
+	minimumWidthToCover,
+} from "./ranges.js";
+import {
+	type ReplicationDomainHash,
+	createReplicationDomainHash,
+	hashToU32,
+} from "./replication-domain-hash.js";
+import {
+	type ReplicationDomainTime,
+	createReplicationDomainTime,
+} from "./replication-domain-time.js";
+import {
+	type ExtractDomainArgs,
+	type ReplicationDomain,
+	type u32,
+} from "./replication-domain.js";
 import {
 	AbsoluteReplicas,
+	AddedReplicationSegmentMessage,
+	AllReplicatingSegmentsMessage,
 	ReplicationError,
 	ReplicationIntent,
 	type ReplicationLimits,
 	ReplicationRange,
 	ReplicationRangeIndexable,
 	RequestReplicationInfoMessage,
-	ResponseReplicationInfoMessage,
 	ResponseRoleMessage,
-	StartedReplicating,
 	StoppedReplicating,
 	decodeReplicas,
 	encodeReplicas,
-	hashToUniformNumber,
 	maxReplicas,
 } from "./replication.js";
-import { Observer, Replicator, SEGMENT_COORDINATE_SCALE } from "./role.js";
-
-export * from "./replication.js";
-
+import { MAX_U32, Observer, Replicator, scaleToU32 } from "./role.js";
+
+export {
+	type ReplicationDomain,
+	type ReplicationDomainHash,
+	type ReplicationDomainTime,
+	createReplicationDomainHash,
+	createReplicationDomainTime,
+};
 export { type CPUUsage, CPUUsageIntervalLag };
+export * from "./replication.js";
 
 export const logger = loggerFn({ module: "shared-log" });
 
+const getLatestEntry = (
+	entries: (ShallowOrFullEntry<any> | EntryWithRefs<any>)[],
+) => {
+	let latest: ShallowOrFullEntry<any> | undefined = undefined;
+	for (const element of entries) {
+		let entry = element instanceof EntryWithRefs ? element.entry : element;
+		if (
+			!latest ||
+			entry.meta.clock.timestamp.compare(latest.meta.clock.timestamp) > 0
+		) {
+			latest = entry;
+		}
+	}
+	return latest;
+};
+
 const groupByGid = async <
 	T extends ShallowEntry | Entry<any> | EntryWithRefs<any>,
 >(
@@ -123,13 +162,16 @@ export type DynamicReplicationOptions = {
 };
 
 export type FixedReplicationOptions = {
-	factor: number;
+	normalized?: boolean;
+	factor: number | "all" | "right";
+	strict?: boolean; // if true, only this range will be replicated
 	offset?: number;
 };
 
 export type ReplicationOptions =
 	| DynamicReplicationOptions
 	| FixedReplicationOptions
+	| FixedReplicationOptions[]
 	| number
 	| boolean;
 
@@ -145,10 +187,19 @@ const isAdaptiveReplicatorOption = (
 	if ((options as FixedReplicationOptions).factor != null) {
 		return false;
 	}
+	if (Array.isArray(options)) {
+		return false;
+	}
 	return true;
 };
 
-export type SharedLogOptions<T> = {
+const isUnreplicationOptions = (options?: ReplicationOptions): boolean =>
+	options === false ||
+	options === 0 ||
+	((options as FixedReplicationOptions)?.offset === undefined &&
+		(options as FixedReplicationOptions)?.factor === 0);
+
+export type SharedLogOptions<T, D extends ReplicationDomain<any, T>> = {
 	replicate?: ReplicationOptions;
 	replicas?: ReplicationLimitsOptions;
 	respondToIHaveTimeout?: number;
@@ -158,6 +209,7 @@ export type SharedLogOptions<T> = {
 	waitForReplicatorTimeout?: number;
 	distributionDebounceTime?: number;
 	compatibility?: number;
+	domain?: D;
 };
 
 export const DEFAULT_MIN_REPLICAS = 2;
@@ -166,10 +218,14 @@ export const WAIT_FOR_ROLE_MATURITY = 5000;
 const REBALANCE_DEBOUNCE_INTERVAL = 100;
 const DEFAULT_DISTRIBUTION_DEBOUNCE_TIME = 500;
 
-export type Args<T> = LogProperties<T> & LogEvents<T> & SharedLogOptions<T>;
+export type Args<
+	T,
+	D extends ReplicationDomain<any, T> = ReplicationDomainHash,
+> = LogProperties<T> & LogEvents<T> & SharedLogOptions<T, D>;
 
 export type SharedAppendOptions<T> = AppendOptions<T> & {
 	replicas?: AbsoluteReplicas | number;
+	replicate?: boolean;
 	target?: "all" | "replicators";
 };
 
@@ -184,10 +240,10 @@ export interface SharedLogEvents extends ProgramEvents {
 }
 
 @variant("shared_log")
-export class SharedLog<T = Uint8Array> extends Program<
-	Args<T>,
-	SharedLogEvents
-> {
+export class SharedLog<
+	T = Uint8Array,
+	D extends ReplicationDomain<any, T> = ReplicationDomainHash,
+> extends Program<Args<T, D>, SharedLogEvents> {
 	@field({ type: Log })
 	log: Log<T>;
 
@@ -195,9 +251,11 @@ export class SharedLog<T = Uint8Array> extends Program<
 	rpc: RPC<TransportMessage, TransportMessage>;
 
 	// options
-	private _replicationSettings?: ReplicationOptions;
+	private _isReplicating: boolean;
+	private _isAdaptiveReplicating: boolean;
+
 	private _replicationRangeIndex!: Index<ReplicationRangeIndexable>;
-	private _totalParticipation!: number;
+	/* private _totalParticipation!: number; */
 	private _gidPeersHistory!: Map<string, Set<string>>;
 
 	private _onSubscriptionFn!: (arg: any) => any;
@@ -209,7 +267,7 @@ export class SharedLog<T = Uint8Array> extends Program<
 
 	private _logProperties?: LogProperties<T> &
 		LogEvents<T> &
-		SharedLogOptions<T>;
+		SharedLogOptions<T, D>;
 	private _closeController!: AbortController;
 	private _gidParentCache!: Cache<Entry<any>[]>;
 	private _respondToIHaveTimeout!: any;
@@ -228,7 +286,7 @@ export class SharedLog<T = Uint8Array> extends Program<
 		{ clear: () => void; callback: (entry: Entry<T>) => void }
 	>;
 
-	private latestRoleMessages!: Map<string, bigint>;
+	private latestReplicationInfoMessage!: Map<string, bigint>;
 
 	private remoteBlocks!: RemoteBlocks;
 
@@ -254,7 +312,7 @@ export class SharedLog<T = Uint8Array> extends Program<
 	private syncInFlightQueueInverted!: Map<string, Set<string>>;
 
 	// map of hash to public keys that we have asked for entries
-	private syncInFlight!: Map<string, Map<string, { timestamp: number }>>;
+	syncInFlight!: Map<string, Map<string, { timestamp: number }>>;
 
 	replicas!: ReplicationLimits;
 
@@ -266,6 +324,7 @@ export class SharedLog<T = Uint8Array> extends Program<
 
 	replicationController!: PIDReplicationController;
 	history!: { usedMemory: number; factor: number }[];
+	domain: D;
 
 	private pq: PQueue<any>;
 
@@ -275,13 +334,6 @@ export class SharedLog<T = Uint8Array> extends Program<
 		this.rpc = new RPC();
 	}
 
-	/**
-	 * Return the
-	 */
-	get replicationSettings(): ReplicationOptions | undefined {
-		return this._replicationSettings;
-	}
-
 	get compatibility(): number | undefined {
 		return this._logProperties?.compatibility;
 	}
@@ -291,16 +343,19 @@ export class SharedLog<T = Uint8Array> extends Program<
 	}
 
 	// @deprecated
-	private getRole() {
-		let isFixedReplicationSettings =
-			(this._replicationSettings as FixedReplicationOptions).factor !==
-			undefined;
-		if (isFixedReplicationSettings) {
-			const fixedSettings = this
-				._replicationSettings as FixedReplicationOptions;
+	private async getRole() {
+		const segments = await this.getMyReplicationSegments();
+		if (segments.length > 1) {
+			throw new Error(
+				"More than one replication segment found. Can only use one segment for compatbility with v8",
+			);
+		}
+
+		if (segments.length > 0) {
+			const segment = segments[0].toReplicationRange();
 			return new Replicator({
-				factor: fixedSettings.factor,
-				offset: fixedSettings.offset ?? 0,
+				factor: segment.factor / MAX_U32,
+				offset: segment.offset / MAX_U32,
 			});
 		}
 
@@ -309,15 +364,17 @@ export class SharedLog<T = Uint8Array> extends Program<
 	}
 
 	async isReplicating() {
-		if (!this._replicationSettings) {
+		if (!this._isReplicating) {
 			return false;
 		}
+		/* 
 		if (isAdaptiveReplicatorOption(this._replicationSettings)) {
 			return true;
 		}
-		if ((this.replicationSettings as FixedReplicationOptions).factor > 0) {
+
+		if ((this.replicationSettings as FixedReplicationOptions).factor !== 0) {
 			return true;
-		}
+		} */
 
 		return (await this.countReplicationSegments()) > 0;
 	}
@@ -330,7 +387,7 @@ export class SharedLog<T = Uint8Array> extends Program<
 		const sum = await this.replicationIndex.sum(
 			new SumRequest({ key: "width" }),
 		);
-		return Number(sum) / SEGMENT_COORDINATE_SCALE;
+		return Number(sum) / MAX_U32;
 	}
 
 	async countReplicationSegments() {
@@ -346,6 +403,7 @@ export class SharedLog<T = Uint8Array> extends Program<
 	}
 
 	private setupRebalanceDebounceFunction() {
+		this.rebalanceParticipationDebounced = undefined;
 		this.rebalanceParticipationDebounced = debounce(
 			() => this.rebalanceParticipation(),
 			/* Math.max(
@@ -358,135 +416,256 @@ export class SharedLog<T = Uint8Array> extends Program<
 			REBALANCE_DEBOUNCE_INTERVAL, // TODO make this dynamic on the number of replicators
 		);
 	}
-	private async setupReplicationSettings(options?: ReplicationOptions) {
-		this.rebalanceParticipationDebounced = undefined;
-		const setupDebouncedRebalancing = (options?: DynamicReplicationOptions) => {
-			this.cpuUsage?.stop?.();
-			this.replicationController = new PIDReplicationController(
-				this.node.identity.publicKey.hashcode(),
-				{
-					storage:
-						options?.limits?.storage != null
-							? { max: options?.limits?.storage }
-							: undefined,
-					cpu:
-						options?.limits?.cpu != null
-							? {
-									max:
-										typeof options?.limits?.cpu === "object"
-											? options.limits.cpu.max
-											: options?.limits?.cpu,
-								}
-							: undefined,
-				},
-			);
+	private async _replicate(
+		options?: ReplicationOptions,
+		{
+			reset,
+			checkDuplicates,
+			announce,
+		}: {
+			reset?: boolean;
+			checkDuplicates?: boolean;
+			announce?: (
+				msg: AddedReplicationSegmentMessage | AllReplicatingSegmentsMessage,
+			) => void;
+		} = {},
+	) {
+		let offsetWasProvided = false;
+		if (isUnreplicationOptions(options)) {
+			await this.unreplicate();
+		} else {
+			let ranges: ReplicationRangeIndexable[] = [];
+
+			if (options == null) {
+				options = {};
+			} else if (options === true) {
+				options = {};
+			}
 
-			this.cpuUsage =
-				options?.limits?.cpu && typeof options?.limits?.cpu === "object"
-					? options?.limits?.cpu?.monitor || new CPUUsageIntervalLag()
-					: new CPUUsageIntervalLag();
-			this.cpuUsage?.start?.();
+			this._isReplicating = true;
+			this._isAdaptiveReplicating = false;
 
-			this.setupRebalanceDebounceFunction();
-		};
+			if (isAdaptiveReplicatorOption(options!)) {
+				this._isAdaptiveReplicating = true;
+				this.setupDebouncedRebalancing(options);
 
-		if (options) {
-			if (isAdaptiveReplicatorOption(options)) {
-				this._replicationSettings = options;
-				setupDebouncedRebalancing(this._replicationSettings);
-			} else if (
-				options === true ||
-				(options && Object.keys(options).length === 0)
-			) {
-				this._replicationSettings = {};
-				setupDebouncedRebalancing(this._replicationSettings);
+				// initial role in a dynamic setup
+				const maybeRange = await this.getDynamicRange();
+				if (!maybeRange) {
+					// not allowed
+					return;
+				}
+				ranges = [maybeRange];
+
+				offsetWasProvided = true;
+			} else if (options instanceof ReplicationRange) {
+				ranges = [
+					options.toReplicationRangeIndexable(this.node.identity.publicKey),
+				];
+
+				offsetWasProvided = true;
 			} else {
+				let rangeArgs: FixedReplicationOptions[];
 				if (typeof options === "number") {
-					this._replicationSettings = {
-						factor: options,
-					} as FixedReplicationOptions;
+					rangeArgs = [
+						{
+							factor: options,
+						} as FixedReplicationOptions,
+					];
 				} else {
-					this._replicationSettings = { ...options } as FixedReplicationOptions;
+					rangeArgs = (
+						Array.isArray(options) ? options : [{ ...options }]
+					) as FixedReplicationOptions[];
 				}
-			}
-		} else {
-			return;
-		}
 
-		if (isAdaptiveReplicatorOption(this._replicationSettings!)) {
-			// initial role in a dynamic setup
-			await this.getDynamicRange();
-		} else {
-			// fixed
-			const range = new ReplicationRangeIndexable({
-				offset:
-					(this._replicationSettings as FixedReplicationOptions).offset ??
-					Math.random(),
-				length: (this._replicationSettings as FixedReplicationOptions).factor,
-				publicKeyHash: this.node.identity.publicKey.hashcode(),
-				replicationIntent: ReplicationIntent.Explicit, // automatic means that this range might be reused later for dynamic replication behaviour
-				timestamp: BigInt(+new Date()),
-				id: sha256Sync(this.node.identity.publicKey.bytes),
-			});
-			await this.startAnnounceReplicating(range);
-		}
-	}
+				if (rangeArgs.length === 0) {
+					// nothing to do
+					return;
+				}
 
-	async replicate(range?: ReplicationRange | ReplicationOptions) {
-		if (range === false || range === 0) {
-			this._replicationSettings = undefined;
-			await this.removeReplicator(this.node.identity.publicKey);
-		} else {
-			await this.rpc.subscribe();
+				for (const rangeArg of rangeArgs) {
+					const normalized = rangeArg.normalized ?? true;
+					offsetWasProvided = rangeArg.offset != null;
+					const offset =
+						rangeArg.offset ??
+						(normalized ? Math.random() : scaleToU32(Math.random()));
+					let factor = rangeArg.factor;
+					let width = normalized ? 1 : scaleToU32(1);
+					ranges.push(
+						new ReplicationRangeIndexable({
+							normalized,
+							offset: offset,
+							length:
+								typeof factor === "number"
+									? factor
+									: factor === "all"
+										? width
+										: width - offset,
+							publicKeyHash: this.node.identity.publicKey.hashcode(),
+							mode: rangeArg.strict
+								? ReplicationIntent.Strict
+								: ReplicationIntent.NonStrict, // automatic means that this range might be reused later for dynamic replication behaviour
+							timestamp: BigInt(+new Date()),
+						}),
+					);
+				}
+			}
 
-			if (range instanceof ReplicationRange) {
+			for (const range of ranges) {
 				this.oldestOpenTime = Math.min(
 					Number(range.timestamp),
 					this.oldestOpenTime,
 				);
+			}
 
-				await this.startAnnounceReplicating(
-					range.toReplicationRangeIndexable(this.node.identity.publicKey),
-				);
-			} else {
-				await this.setupReplicationSettings(range ?? true);
+			let resetRanges = reset;
+			if (!resetRanges && !offsetWasProvided) {
+				resetRanges = true;
+				// because if we do something like replicate ({ factor: 0.5 }) it means that we want to replicate 50%
+				// but ({ replicate: 0.5, offset: 0.5 }) means that we want to add a range
+				// TODO make behaviour more clear
 			}
+			await this.startAnnounceReplicating(ranges, {
+				reset: resetRanges ?? false,
+				checkDuplicates,
+				announce,
+			});
 		}
+	}
+
+	setupDebouncedRebalancing(options?: DynamicReplicationOptions) {
+		this.cpuUsage?.stop?.();
+		this.replicationController = new PIDReplicationController(
+			this.node.identity.publicKey.hashcode(),
+			{
+				storage:
+					options?.limits?.storage != null
+						? { max: options?.limits?.storage }
+						: undefined,
+				cpu:
+					options?.limits?.cpu != null
+						? {
+								max:
+									typeof options?.limits?.cpu === "object"
+										? options.limits.cpu.max
+										: options?.limits?.cpu,
+							}
+						: undefined,
+			},
+		);
+
+		this.cpuUsage =
+			options?.limits?.cpu && typeof options?.limits?.cpu === "object"
+				? options?.limits?.cpu?.monitor || new CPUUsageIntervalLag()
+				: new CPUUsageIntervalLag();
+		this.cpuUsage?.start?.();
+		this.setupRebalanceDebounceFunction();
+	}
+
+	async replicate(
+		rangeOrEntry?:
+			| ReplicationRange
+			| ReplicationOptions
+			| Entry<T>
+			| Entry<T>[],
+		options?: {
+			reset?: boolean;
+			checkDuplicates?: boolean;
+			announce?: (
+				msg: AllReplicatingSegmentsMessage | AddedReplicationSegmentMessage,
+			) => void;
+		},
+	) {
+		let range: ReplicationRange[] | ReplicationOptions | undefined = undefined;
+
+		if (rangeOrEntry instanceof ReplicationRange) {
+			range = rangeOrEntry;
+		} else if (rangeOrEntry instanceof Entry) {
+			range = {
+				factor: 1,
+				offset: await this.domain.fromEntry(rangeOrEntry),
+				normalized: false,
+			};
+		} else if (Array.isArray(rangeOrEntry)) {
+			let ranges: (ReplicationRange | FixedReplicationOptions)[] = [];
+			for (const entry of rangeOrEntry) {
+				if (entry instanceof Entry) {
+					ranges.push({
+						factor: 1,
+						offset: await this.domain.fromEntry(entry),
+						normalized: false,
+					});
+				} else {
+					ranges.push(entry);
+				}
+			}
+			range = ranges;
+		} else {
+			range = rangeOrEntry ?? true;
+		}
+
+		const newRanges = await this._replicate(range, options);
 
 		// assume new role
 		await this.distribute();
+
+		return newRanges;
 	}
 
-	private async removeReplicator(key: PublicSignKey) {
-		const fn = async () => {
-			let prev = await this.replicationIndex.query(
-				new SearchRequest({
-					query: { hash: key.hashcode() },
-					fetch: 0xffffffff,
-				}),
-				{ reference: true },
-			);
+	async unreplicate(rangeOrEntry?: Entry<T> | ReplicationRange) {
+		let range: FixedReplicationOptions;
+		if (rangeOrEntry instanceof Entry) {
+			range = {
+				factor: 1,
+				offset: await this.domain.fromEntry(rangeOrEntry),
+			};
+		} else if (rangeOrEntry instanceof ReplicationRange) {
+			range = rangeOrEntry;
+		} else {
+			this._isReplicating = false;
+			this._isAdaptiveReplicating = false;
+			await this.removeReplicator(this.node.identity.publicKey);
+			return;
+		}
 
-			if (prev.results.length === 0) {
-				return;
-			}
+		if (this._isAdaptiveReplicating) {
+			// we can not unreplicate individual ranges when dynamically replicating (yet)
+			// TODO support this by never deleting the range with the segment id that is generated by the dynamic replication method
+			throw new Error("Unsupported when adaptive replicating");
+		}
 
-			let sumWidth = prev.results.reduce(
-				(acc, x) => acc + x.value.widthNormalized,
-				0,
-			);
-			this._totalParticipation -= sumWidth;
+		const indexed = await this.replicationIndex.query(
+			new SearchRequest({
+				query: {
+					width: 1,
+					start1: range.offset,
+				},
+			}),
+		);
 
-			let idMatcher = new Or(
-				prev.results.map(
-					(x) => new ByteMatchQuery({ key: "id", value: x.value.id }),
-				),
-			);
+		const segmentIds = indexed.results.map((x) => x.id.key as Uint8Array);
+		await this.removeReplicationRange(segmentIds, this.node.identity.publicKey);
+		await this.rpc.send(new StoppedReplicating({ segmentIds }), {
+			priority: 1,
+		});
+	}
 
-			await this.replicationIndex.del(new DeleteRequest({ query: idMatcher }));
+	private async removeReplicator(key: PublicSignKey) {
+		const fn = async () => {
+			await this.replicationIndex.del(
+				new DeleteRequest({ query: { hash: key.hashcode() } }),
+			);
 
 			await this.updateOldestTimestampFromIndex();
 
+			if (this.node.identity.publicKey.equals(key)) {
+				// announce that we are no longer replicating
+				await this.rpc.send(
+					new AllReplicatingSegmentsMessage({ segments: [] }),
+					{ priority: 1 },
+				);
+			}
+
 			this.events.dispatchEvent(
 				new CustomEvent<ReplicationChange>("replication:change", {
 					detail: { publicKey: key },
@@ -531,11 +710,6 @@ export class SharedLog<T = Uint8Array> extends Program<
 
 			let query = new And([idMatcher, identityMatcher]);
 
-			const prevSum = await this.replicationIndex.sum(
-				new SumRequest({ query, key: "width" }),
-			);
-			const prevSumNormalized = Number(prevSum) / SEGMENT_COORDINATE_SCALE;
-			this._totalParticipation -= prevSumNormalized;
 			await this.replicationIndex.del(new DeleteRequest({ query }));
 
 			await this.updateOldestTimestampFromIndex();
@@ -555,8 +729,12 @@ export class SharedLog<T = Uint8Array> extends Program<
 	}
 
 	private async addReplicationRange(
-		range: ReplicationRangeIndexable,
+		ranges: ReplicationRangeIndexable[],
 		from: PublicSignKey,
+		{
+			reset,
+			checkDuplicates,
+		}: { reset?: boolean; checkDuplicates?: boolean } = {},
 	) {
 		const fn = async () => {
 			if (
@@ -566,7 +744,6 @@ export class SharedLog<T = Uint8Array> extends Program<
 				return false;
 			}
 
-			range.id = new Uint8Array(range.id);
 			let prevCount = await this.replicationIndex.count(
 				new CountRequest({
 					query: new StringMatch({ key: "hash", value: from.hashcode() }),
@@ -574,26 +751,35 @@ export class SharedLog<T = Uint8Array> extends Program<
 			);
 			const isNewReplicator = prevCount === 0;
 
-			let prev = await this.replicationIndex.get(toId(range.id));
-			if (prev) {
-				if (prev.value.equals(range)) {
-					return false;
+			if (reset) {
+				await this.replicationIndex.del(
+					new DeleteRequest({ query: { hash: from.hashcode() } }),
+				);
+			} else if (checkDuplicates) {
+				let deduplicated: any[] = [];
+
+				// TODO also deduplicate/de-overlap among the ranges that ought to be inserted?
+				for (const range of ranges) {
+					if (!(await hasCoveringRange(this.replicationIndex, range))) {
+						deduplicated.push(range);
+					}
 				}
-				this._totalParticipation -= prev.value.widthNormalized;
+				ranges = deduplicated;
 			}
 
-			await this.replicationIndex.put(range);
-			let inserted = await this.replicationIndex.get(toId(range.id));
-			if (!inserted?.value.equals(range)) {
-				throw new Error("Failed to insert range");
+			for (const range of ranges) {
+				await this.replicationIndex.put(range);
+				if (!reset) {
+					this.oldestOpenTime = Math.min(
+						Number(range.timestamp),
+						this.oldestOpenTime,
+					);
+				}
 			}
 
-			this._totalParticipation += range.widthNormalized;
-
-			this.oldestOpenTime = Math.min(
-				Number(range.timestamp),
-				this.oldestOpenTime,
-			);
+			if (reset) {
+				await this.updateOldestTimestampFromIndex();
+			}
 
 			this.events.dispatchEvent(
 				new CustomEvent<ReplicationChange>("replication:change", {
@@ -614,25 +800,50 @@ export class SharedLog<T = Uint8Array> extends Program<
 			}
 			return true;
 		};
+
+		// we sequialize this because we are going to queries to check wether to add or not
+		// if two processes do the same this both process might add a range while only one in practice should
 		return this.pq.add(fn);
 	}
 
-	async startAnnounceReplicating(range: ReplicationRangeIndexable) {
+	async startAnnounceReplicating(
+		range: ReplicationRangeIndexable[],
+		options: {
+			reset?: boolean;
+			checkDuplicates?: boolean;
+			announce?: (
+				msg: AllReplicatingSegmentsMessage | AddedReplicationSegmentMessage,
+			) => void;
+		} = {},
+	) {
 		const added = await this.addReplicationRange(
 			range,
 			this.node.identity.publicKey,
+			options,
 		);
 		if (!added) {
 			logger.warn("Not allowed to replicate by canReplicate");
 		}
 
+		let message: AllReplicatingSegmentsMessage | AddedReplicationSegmentMessage;
 		if (added) {
-			await this.rpc.send(
-				new StartedReplicating({ segments: [range.toReplicationRange()] }),
-				{
+			if (options.reset) {
+				message = new AllReplicatingSegmentsMessage({
+					segments: range.map((x) => x.toReplicationRange()),
+				});
+			} else {
+				message = new AddedReplicationSegmentMessage({
+					segments: range.map((x) => x.toReplicationRange()),
+				});
+			}
+
+			if (options.announce) {
+				return options.announce(message);
+			} else {
+				await this.rpc.send(message, {
 					priority: 1,
-				},
-			);
+				});
+			}
 		}
 	}
 
@@ -678,6 +889,10 @@ export class SharedLog<T = Uint8Array> extends Program<
 		const result = await this.log.append(data, appendOptions);
 		let mode: DeliveryMode | undefined = undefined;
 
+		if (options?.replicate) {
+			await this.replicate(result.entry, { checkDuplicates: true });
+		}
+
 		for (const message of await createExchangeHeadsMessages(
 			this.log,
 			[result.entry],
@@ -685,22 +900,29 @@ export class SharedLog<T = Uint8Array> extends Program<
 		)) {
 			if (options?.target === "replicators" || !options?.target) {
 				const minReplicas = decodeReplicas(result.entry).getValue(this);
+
 				let leaders: string[] | Set<string> = await this.findLeaders(
-					result.entry.meta.gid,
+					result.entry,
 					minReplicas,
 				);
+
 				const isLeader = leaders.includes(
 					this.node.identity.publicKey.hashcode(),
 				);
+
 				if (message.heads[0].gidRefrences.length > 0) {
 					const newAndOldLeaders = new Set(leaders);
 					for (const ref of message.heads[0].gidRefrences) {
-						for (const hash of await this.findLeaders(ref, minReplicas)) {
-							newAndOldLeaders.add(hash);
+						const entryFromGid = this.log.entryIndex.getHeads(ref, false);
+						for (const entry of await entryFromGid.all()) {
+							for (const hash of await this.findLeaders(entry, minReplicas)) {
+								newAndOldLeaders.add(hash);
+							}
 						}
 					}
 					leaders = newAndOldLeaders;
 				}
+
 				let set = this._gidPeersHistory.get(result.entry.meta.gid);
 				if (!set) {
 					set = new Set(leaders);
@@ -725,7 +947,7 @@ export class SharedLog<T = Uint8Array> extends Program<
 		return result;
 	}
 
-	async open(options?: Args<T>): Promise<void> {
+	async open(options?: Args<T, D>): Promise<void> {
 		this.replicas = {
 			min: options?.replicas?.min
 				? typeof options?.replicas?.min === "number"
@@ -738,11 +960,11 @@ export class SharedLog<T = Uint8Array> extends Program<
 					: options.replicas.max
 				: undefined,
 		};
-
+		this.domain = options?.domain ?? (createReplicationDomainHash() as D);
 		this._respondToIHaveTimeout = options?.respondToIHaveTimeout ?? 10 * 1000; // TODO make into arg
 		this._pendingDeletes = new Map();
 		this._pendingIHave = new Map();
-		this.latestRoleMessages = new Map();
+		this.latestReplicationInfoMessage = new Map();
 		this.syncInFlightQueue = new Map();
 		this.syncInFlightQueueInverted = new Map();
 		this.syncInFlight = new Map();
@@ -780,16 +1002,21 @@ export class SharedLog<T = Uint8Array> extends Program<
 
 		await this.remoteBlocks.start();
 
-		this._totalParticipation = 0;
+		/* this._totalParticipation = 0; */
 		const logScope = await this.node.indexer.scope(id);
 		const replicationIndex = await logScope.scope("replication");
 		this._replicationRangeIndex = await replicationIndex.init({
 			schema: ReplicationRangeIndexable,
 		});
+
 		const logIndex = await logScope.scope("log");
+
 		await this.node.indexer.start(); // TODO why do we need to start the indexer here?
 
-		this._totalParticipation = await this.calculateTotalParticipation();
+		const hasIndexedReplicationInfo =
+			(await this.replicationIndex.getSize()) > 0;
+
+		/* this._totalParticipation = await this.calculateTotalParticipation(); */
 
 		this._gidPeersHistory = new Map();
 
@@ -834,6 +1061,8 @@ export class SharedLog<T = Uint8Array> extends Program<
 			this._onUnsubscriptionFn,
 		);
 
+		await this.rpc.subscribe();
+
 		// await this.log.load();
 
 		// TODO (do better)
@@ -888,7 +1117,15 @@ export class SharedLog<T = Uint8Array> extends Program<
 			});
 		};
 
-		await this.replicate(options?.replicate);
+		// if we had a previous session with replication info, and new replication info dictates that we unreplicate
+		// we should do that. Otherwise if options is a unreplication we dont need to do anything because
+		// we are already unreplicated (as we are just opening)
+		if (
+			hasIndexedReplicationInfo ||
+			isUnreplicationOptions(options?.replicate) === false
+		) {
+			await this.replicate(options?.replicate, { checkDuplicates: true });
+		}
 		requestSync();
 	}
 
@@ -925,7 +1162,7 @@ export class SharedLog<T = Uint8Array> extends Program<
 
 	async onChange(change: Change<T>) {
 		for (const added of change.added) {
-			this.onEntryAdded(added);
+			this.onEntryAdded(added.entry);
 		}
 		for (const removed of change.removed) {
 			this.onEntryRemoved(removed.hash);
@@ -957,6 +1194,28 @@ export class SharedLog<T = Uint8Array> extends Program<
 		}
 	}
 
+	async getCover(args: ExtractDomainArgs<D>, roleAge?: number) {
+		roleAge = roleAge ?? (await this.getDefaultMinRoleAge());
+
+		const range = await this.domain.fromArgs(args, this);
+
+		const set = await getCoverSet(
+			this.replicationIndex,
+			roleAge,
+			range.offset,
+			range.length ??
+				(await minimumWidthToCover(this.replicas.min.getValue(this))),
+			MAX_U32,
+		);
+
+		// add all in flight
+		for (const [key, _] of this.syncInFlight) {
+			set.add(key);
+		}
+
+		return [...set];
+	}
+
 	private async _close() {
 		clearTimeout(this.syncMoreInterval);
 		clearInterval(this.distributeInterval);
@@ -989,12 +1248,12 @@ export class SharedLog<T = Uint8Array> extends Program<
 		this.syncInFlightQueue.clear();
 		this.syncInFlightQueueInverted.clear();
 		this.syncInFlight.clear();
-		this.latestRoleMessages.clear();
+		this.latestReplicationInfoMessage.clear();
 		this._gidPeersHistory.clear();
 
 		this._replicationRangeIndex = undefined as any;
 		this.cpuUsage?.stop?.();
-		this._totalParticipation = 0;
+		/* this._totalParticipation = 0; */
 		this.pq.clear();
 	}
 	async close(from?: Program): Promise<boolean> {
@@ -1078,6 +1337,7 @@ export class SharedLog<T = Uint8Array> extends Program<
 							const headsWithGid = await this.log.entryIndex
 								.getHeads(gid)
 								.all();
+							const latestEntry = getLatestEntry(entries)!;
 
 							const maxReplicasFromHead =
 								headsWithGid && headsWithGid.length > 0
@@ -1095,12 +1355,12 @@ export class SharedLog<T = Uint8Array> extends Program<
 
 							if (isReplicating) {
 								isLeader = await this.waitForIsLeader(
-									gid,
+									latestEntry,
 									Math.max(maxReplicasFromHead, maxReplicasFromNewEntries),
 								);
 							} else {
 								isLeader = await this.findLeaders(
-									gid,
+									latestEntry,
 									Math.max(maxReplicasFromHead, maxReplicasFromNewEntries),
 								);
 
@@ -1184,7 +1444,7 @@ export class SharedLog<T = Uint8Array> extends Program<
 								const minReplicas = maxReplicas(this, headsWithGid.values());
 
 								const isLeader = await this.isLeader(
-									entries[0].entry.meta.gid,
+									entries[0].entry,
 									minReplicas,
 								);
 
@@ -1206,7 +1466,7 @@ export class SharedLog<T = Uint8Array> extends Program<
 					if (
 						indexedEntry &&
 						(await this.isLeader(
-							indexedEntry.value.meta.gid,
+							indexedEntry.value,
 							decodeReplicas(indexedEntry.value).getValue(this),
 						))
 					) {
@@ -1224,7 +1484,7 @@ export class SharedLog<T = Uint8Array> extends Program<
 							callback: async (entry: any) => {
 								if (
 									await this.isLeader(
-										entry.meta.gid,
+										entry,
 										decodeReplicas(entry).getValue(this),
 									)
 								) {
@@ -1316,7 +1576,7 @@ export class SharedLog<T = Uint8Array> extends Program<
 					return;
 				}
 				await this.rpc.send(
-					new ResponseReplicationInfoMessage({
+					new AllReplicatingSegmentsMessage({
 						segments: (await this.getMyReplicationSegments()).map((x) =>
 							x.toReplicationRange(),
 						),
@@ -1330,9 +1590,8 @@ export class SharedLog<T = Uint8Array> extends Program<
 				if (this.v8Behaviour) {
 					const role = this.getRole();
 					if (role instanceof Replicator) {
-						const fixedSettings = this
-							._replicationSettings as FixedReplicationOptions;
-						if (fixedSettings.factor === 1) {
+						const fixedSettings = !this._isAdaptiveReplicating;
+						if (fixedSettings) {
 							await this.rpc.send(
 								new ResponseRoleMessage({
 									role,
@@ -1348,16 +1607,16 @@ export class SharedLog<T = Uint8Array> extends Program<
 					}
 				}
 			} else if (
-				msg instanceof ResponseReplicationInfoMessage ||
-				msg instanceof StartedReplicating
+				msg instanceof AllReplicatingSegmentsMessage ||
+				msg instanceof AddedReplicationSegmentMessage
 			) {
 				if (context.from.equals(this.node.identity.publicKey)) {
 					return;
 				}
 
 				let replicationInfoMessage = msg as
-					| ResponseReplicationInfoMessage
-					| StartedReplicating;
+					| AllReplicatingSegmentsMessage
+					| AddedReplicationSegmentMessage;
 
 				// we have this statement because peers might have changed/announced their role,
 				// but we don't know them as "subscribers" yet. i.e. they are not online
@@ -1367,30 +1626,30 @@ export class SharedLog<T = Uint8Array> extends Program<
 					timeout: this.waitForReplicatorTimeout,
 				})
 					.then(async () => {
-						// peer should not be online (for us)
-						const prev = this.latestRoleMessages.get(context.from!.hashcode());
+						// do use an operation log here, because we want to make sure that we don't miss any updates
+						// and do them in the right order
+						const prev = this.latestReplicationInfoMessage.get(
+							context.from!.hashcode(),
+						);
+
 						if (prev && prev > context.timestamp) {
 							return;
 						}
-						this.latestRoleMessages.set(
+						this.latestReplicationInfoMessage.set(
 							context.from!.hashcode(),
 							context.timestamp,
 						);
 
-						if (msg instanceof ResponseReplicationInfoMessage) {
-							await this.removeReplicator(context.from!);
-						}
-						let addedOnce = false;
-						for (const segment of replicationInfoMessage.segments) {
-							const added = await this.addReplicationRange(
-								segment.toReplicationRangeIndexable(context.from!),
-								context.from!,
-							);
-							if (typeof added === "boolean") {
-								addedOnce = addedOnce || added;
-							}
-						}
-						addedOnce && (await this.distribute());
+						let reset = msg instanceof AllReplicatingSegmentsMessage;
+						const added = await this.addReplicationRange(
+							replicationInfoMessage.segments.map((x) =>
+								x.toReplicationRangeIndexable(context.from!),
+							),
+							context.from!,
+							{ reset, checkDuplicates: true },
+						);
+
+						added && (await this.distribute());
 
 						/* await this._modifyReplicators(msg.role, context.from!); */
 					})
@@ -1456,7 +1715,7 @@ export class SharedLog<T = Uint8Array> extends Program<
 		return ranges.results.map((x) => x.value);
 	}
 
-	async getTotalParticipation() {
+	async getMyTotalParticipation() {
 		// sum all of my replicator rects
 		return (await this.getMyReplicationSegments()).reduce(
 			(acc, { widthNormalized }) => acc + widthNormalized,
@@ -1518,8 +1777,96 @@ export class SharedLog<T = Uint8Array> extends Program<
 		});
 	}
 
+	async join(
+		entries: (string | Entry<T> | ShallowEntry)[],
+		options?: {
+			verifySignatures?: boolean;
+			timeout?: number;
+			replicate?: boolean;
+		},
+	): Promise<void> {
+		let messageToSend: AddedReplicationSegmentMessage | undefined = undefined;
+
+		if (options?.replicate) {
+			// TODO this block should perhaps be called from a callback on the this.log.join method on all the ignored element because already joined, like "onAlreadyJoined"
+
+			// check which entrise we already have but not are replicating, and replicate them
+			let alreadyJoined: Entry<T>[] = [];
+			for (const element of entries) {
+				if (typeof element === "string") {
+					const entry = await this.log.get(element);
+					if (entry) {
+						alreadyJoined.push(entry);
+					}
+				} else if (element instanceof Entry) {
+					if (await this.log.has(element.hash)) {
+						alreadyJoined.push(element);
+					}
+				} else {
+					const entry = await this.log.get(element.hash);
+					if (entry) {
+						alreadyJoined.push(entry);
+					}
+				}
+			}
+
+			// assume is heads
+			await this.replicate(alreadyJoined, {
+				checkDuplicates: true,
+				announce: (msg) => {
+					if (msg instanceof AllReplicatingSegmentsMessage) {
+						throw new Error("Unexpected");
+					}
+					messageToSend = msg;
+				},
+			});
+		}
+
+		let joinOptions = options?.replicate
+			? {
+					...options,
+					onChange: async (change: Change<T>) => {
+						if (change.added) {
+							for (const entry of change.added) {
+								if (entry.head) {
+									await this.replicate(entry.entry, {
+										checkDuplicates: true,
+
+										// we override the announce step here to make sure we announce all new replication info
+										// in one large message instead
+										announce: (msg) => {
+											if (msg instanceof AllReplicatingSegmentsMessage) {
+												throw new Error("Unexpected");
+											}
+
+											if (messageToSend) {
+												// merge segments to make it into one messages
+												for (const segment of msg.segments) {
+													messageToSend.segments.push(segment);
+												}
+											} else {
+												messageToSend = msg;
+											}
+										},
+									});
+								}
+							}
+						}
+					},
+				}
+			: options;
+
+		await this.log.join(entries, joinOptions);
+
+		if (messageToSend) {
+			await this.rpc.send(messageToSend, {
+				priority: 1,
+			});
+		}
+	}
+
 	async isLeader(
-		slot: { toString(): string },
+		entry: ShallowOrFullEntry<any>,
 		numberOfLeaders: number,
 		options?: {
 			candidates?: string[];
@@ -1527,13 +1874,13 @@ export class SharedLog<T = Uint8Array> extends Program<
 		},
 	): Promise<boolean> {
 		const isLeader = (
-			await this.findLeaders(slot, numberOfLeaders, options)
+			await this.findLeaders(entry, numberOfLeaders, options)
 		).find((l) => l === this.node.identity.publicKey.hashcode());
 		return !!isLeader;
 	}
 
 	private async waitForIsLeader(
-		slot: { toString(): string },
+		entry: ShallowOrFullEntry<T>,
 		numberOfLeaders: number,
 		timeout = this.waitForReplicatorTimeout,
 	): Promise<string[] | false> {
@@ -1554,7 +1901,7 @@ export class SharedLog<T = Uint8Array> extends Program<
 			}, timeout);
 
 			const check = () =>
-				this.findLeaders(slot, numberOfLeaders).then((leaders) => {
+				this.findLeaders(entry, numberOfLeaders).then((leaders) => {
 					const isLeader = leaders.find(
 						(l) => l === this.node.identity.publicKey.hashcode(),
 					);
@@ -1577,7 +1924,7 @@ export class SharedLog<T = Uint8Array> extends Program<
 	}
 
 	async findLeaders(
-		subject: { toString(): string },
+		entry: ShallowOrFullEntry<any>,
 		numberOfLeaders: number,
 		options?: {
 			roleAge?: number;
@@ -1587,18 +1934,8 @@ export class SharedLog<T = Uint8Array> extends Program<
 			return [this.node.identity.publicKey.hashcode()]; // Assumption: if the store is closed, always assume we have responsibility over the data
 		}
 
-		// For a fixed set or members, the choosen leaders will always be the same (address invariant)
-		// This allows for that same content is always chosen to be distributed to same peers, to remove unecessary copies
-
-		// Convert this thing we wan't to distribute to 8 bytes so we get can convert it into a u64
-		// modulus into an index
-		const utf8writer = new BinaryWriter();
-		utf8writer.string(subject.toString());
-		const seed = await sha256(utf8writer.finalize());
-
-		// convert hash of slot to a number
-		const cursor = hashToUniformNumber(seed); // bounded between 0 and 1
-		return this.findLeadersFromUniformNumber(cursor, numberOfLeaders, options);
+		const cursor = await this.domain.fromEntry(entry);
+		return this.findLeadersFromU32(cursor, numberOfLeaders, options);
 	}
 
 	async getDefaultMinRoleAge(): Promise<number> {
@@ -1617,63 +1954,15 @@ export class SharedLog<T = Uint8Array> extends Program<
 		); // / 3 so that if 2 replicators and timeUntilRoleMaturity = 1e4 the result will be 1
 	}
 
-	private async findLeadersFromUniformNumber(
-		cursor: number,
+	private async findLeadersFromU32(
+		cursor: u32,
 		numberOfLeaders: number,
 		options?: {
 			roleAge?: number;
 		},
 	) {
 		const roleAge = options?.roleAge ?? (await this.getDefaultMinRoleAge()); // TODO -500 as is added so that i f someone else is just as new as us, then we treat them as mature as us. without -500 we might be slower syncing if two nodes starts almost at the same time
-
-		const samples = await getSamples(
-			cursor,
-			this.replicationIndex,
-			numberOfLeaders,
-			roleAge,
-		);
-
-		return samples;
-	}
-
-	/**
-	 *
-	 * @returns groups where at least one in any group will have the entry you are looking for
-	 */
-	async getReplicatorUnion(roleAge?: number) {
-		roleAge = roleAge ?? (await this.getDefaultMinRoleAge());
-		if (this.closed === true) {
-			throw new ClosedError();
-		}
-
-		// Total replication "width"
-		const width = 1;
-
-		// How much width you need to "query" to
-		const peers = this.replicationIndex; // TODO types
-		const minReplicas = Math.min(
-			await peers.getSize(),
-			this.replicas.min.getValue(this),
-		);
-
-		// If min replicas = 2
-		// then we need to make sure we cover 0.5 of the total 'width' of the replication space
-		// to make sure we reach sufficient amount of nodes such that at least one one has
-		// the entry we are looking for
-		const coveringWidth = width / minReplicas;
-
-		const set = await getCoverSet(
-			coveringWidth,
-			peers,
-			roleAge,
-			this.node.identity.publicKey,
-		);
-
-		// add all in flight
-		for (const [key, _] of this.syncInFlight) {
-			set.add(key);
-		}
-		return [...set];
+		return getSamples(cursor, this.replicationIndex, numberOfLeaders, roleAge);
 	}
 
 	async isReplicator(
@@ -1683,11 +1972,7 @@ export class SharedLog<T = Uint8Array> extends Program<
 			roleAge?: number;
 		},
 	) {
-		return this.isLeader(
-			entry.meta.gid,
-			decodeReplicas(entry).getValue(this),
-			options,
-		);
+		return this.isLeader(entry, decodeReplicas(entry).getValue(this), options);
 	}
 
 	async handleSubscriptionChange(
@@ -1728,7 +2013,7 @@ export class SharedLog<T = Uint8Array> extends Program<
 			if (replicationSegments.length > 0) {
 				this.rpc
 					.send(
-						new ResponseReplicationInfoMessage({
+						new AllReplicatingSegmentsMessage({
 							segments: replicationSegments.map((x) => x.toReplicationRange()),
 						}),
 						{
@@ -1740,7 +2025,7 @@ export class SharedLog<T = Uint8Array> extends Program<
 				if (this.v8Behaviour) {
 					// for backwards compatibility
 					this.rpc
-						.send(new ResponseRoleMessage({ role: this.getRole() }), {
+						.send(new ResponseRoleMessage({ role: await this.getRole() }), {
 							mode: new SilentDelivery({ redundancy: 1, to: [publicKey] }),
 						})
 						.catch((e) => logger.error(e.toString()));
@@ -1825,10 +2110,7 @@ export class SharedLog<T = Uint8Array> extends Program<
 						? Math.min(minReplicasValue, this.replicas.max.getValue(this))
 						: minReplicasValue;
 
-					const leaders = await this.findLeaders(
-						entry.meta.gid,
-						minMinReplicasValue,
-					);
+					const leaders = await this.findLeaders(entry, minMinReplicasValue);
 
 					if (
 						leaders.find((x) => x === this.node.identity.publicKey.hashcode())
@@ -1941,7 +2223,7 @@ export class SharedLog<T = Uint8Array> extends Program<
 
 			const oldPeersSet = this._gidPeersHistory.get(gid);
 			const currentPeers = await this.findLeaders(
-				gid,
+				getLatestEntry(entries)!,
 				maxReplicas(this, entries), // pick max replication policy of all entries, so all information is treated equally important as the most important
 			);
 
@@ -2041,8 +2323,9 @@ export class SharedLog<T = Uint8Array> extends Program<
 				evt.detail.unsubscriptions.map((x) => x),
 			)}'`,
 		);
-		this.latestRoleMessages.delete(evt.detail.from.hashcode());
+		this.latestReplicationInfoMessage.delete(evt.detail.from.hashcode());
 
+		// TODO only emit this if the peer is actually replicating anything
 		this.events.dispatchEvent(
 			new CustomEvent<ReplicatorLeaveEvent>("replicator:leave", {
 				detail: { publicKey: evt.detail.from },
@@ -2111,11 +2394,11 @@ export class SharedLog<T = Uint8Array> extends Program<
 		}
 
 		// The role is fixed (no changes depending on memory usage or peer count etc)
-		if (!this._replicationSettings) {
+		if (!this._isReplicating) {
 			return false;
 		}
 
-		if (isAdaptiveReplicatorOption(this._replicationSettings)) {
+		if (this._isAdaptiveReplicating) {
 			const peers = this.replicationIndex;
 			const usedMemory = await this.getMemoryUsage();
 			let dynamicRange = await this.getDynamicRange();
@@ -2125,10 +2408,12 @@ export class SharedLog<T = Uint8Array> extends Program<
 			}
 
 			const peersSize = (await peers.getSize()) || 1;
+			const totalParticipation = await this.calculateTotalParticipation();
+
 			const newFactor = this.replicationController.step({
 				memoryUsage: usedMemory,
 				currentFactor: dynamicRange.widthNormalized,
-				totalFactor: await this.calculateTotalParticipation(), // TODO use this._totalParticipation when flakiness is fixed
+				totalFactor: totalParticipation, // TODO use this._totalParticipation when flakiness is fixed
 				peerCount: peersSize,
 				cpuUsage: this.cpuUsage?.value(),
 			});
@@ -2140,11 +2425,11 @@ export class SharedLog<T = Uint8Array> extends Program<
 			if (relativeDifference > 0.0001) {
 				// TODO can not reuse old range, since it will (potentially) affect the index because of sideeffects
 				dynamicRange = new ReplicationRangeIndexable({
-					offset: hashToUniformNumber(this.node.identity.publicKey.bytes),
-					length: newFactor,
+					offset: hashToU32(this.node.identity.publicKey.bytes),
+					length: scaleToU32(newFactor),
 					publicKeyHash: dynamicRange.hash,
 					id: dynamicRange.id,
-					replicationIntent: dynamicRange.replicationIntent,
+					mode: dynamicRange.mode,
 					timestamp: dynamicRange.timestamp,
 				});
 
@@ -2155,7 +2440,10 @@ export class SharedLog<T = Uint8Array> extends Program<
 					return false;
 				}
 
-				await this.startAnnounceReplicating(dynamicRange);
+				await this.startAnnounceReplicating([dynamicRange], {
+					checkDuplicates: false,
+					reset: false,
+				});
 
 				/* await this._updateRole(newRole, onRoleChange); */
 				this.rebalanceParticipationDebounced?.();
@@ -2169,18 +2457,19 @@ export class SharedLog<T = Uint8Array> extends Program<
 		return false;
 	}
 	async getDynamicRange() {
+		let dynamicRangeId = sha256Sync(
+			concat([
+				this.node.identity.publicKey.bytes,
+				new TextEncoder().encode("dynamic"),
+			]),
+		);
 		let range = (
 			await this.replicationIndex.query(
 				new SearchRequest({
 					query: [
-						new StringMatch({
-							key: "hash",
-							value: this.node.identity.publicKey.hashcode(),
-						}),
-						new IntegerCompare({
-							key: "replicationIntent",
-							value: ReplicationIntent.Automatic,
-							compare: "eq",
+						new ByteMatchQuery({
+							key: "id",
+							value: dynamicRangeId,
 						}),
 					],
 					fetch: 1,
@@ -2188,18 +2477,19 @@ export class SharedLog<T = Uint8Array> extends Program<
 			)
 		)?.results[0]?.value;
 		if (!range) {
-			let seed = Math.random();
 			range = new ReplicationRangeIndexable({
-				offset: seed,
+				normalized: true,
+				offset: Math.random(),
 				length: 0,
 				publicKeyHash: this.node.identity.publicKey.hashcode(),
-				replicationIntent: ReplicationIntent.Automatic,
+				mode: ReplicationIntent.NonStrict,
 				timestamp: BigInt(+new Date()),
-				id: sha256Sync(this.node.identity.publicKey.bytes),
+				id: dynamicRangeId,
 			});
 			const added = await this.addReplicationRange(
-				range,
+				[range],
 				this.node.identity.publicKey,
+				{ reset: false, checkDuplicates: false },
 			);
 			if (!added) {
 				logger.warn("Not allowed to replicate by canReplicate");