@peerbit/shared-log 9.2.13 → 10.0.0

package/dist/src/index.js

@@ -9,37 +9,39 @@ var __metadata = (this && this.__metadata) || function (k, v) {
 };
 import { BorshError, field, variant } from "@dao-xyz/borsh";
 import { AnyBlockStore, RemoteBlocks } from "@peerbit/blocks";
+import { Cache } from "@peerbit/cache";
 import { AccessError, PublicSignKey, sha256Base64Sync, sha256Sync, } from "@peerbit/crypto";
-import { And, ByteMatchQuery, Or, Sort, StringMatch, } from "@peerbit/indexer-interface";
-import { Entry, Log, ShallowEntry, } from "@peerbit/log";
+import { And, ByteMatchQuery, NotStartedError as IndexNotStartedError, Or, Sort, StringMatch, toId, } from "@peerbit/indexer-interface";
+import { Entry, Log, Meta, ShallowEntry, } from "@peerbit/log";
 import { logger as loggerFn } from "@peerbit/logger";
 import { ClosedError, Program } from "@peerbit/program";
 import { SubscriptionEvent, UnsubcriptionEvent, } from "@peerbit/pubsub-interface";
 import { RPC } from "@peerbit/rpc";
-import { AcknowledgeDelivery, DeliveryMode, NotStartedError, SilentDelivery, } from "@peerbit/stream-interface";
-import { AbortError, 
-/* delay, */
-waitFor, } from "@peerbit/time";
+import { AcknowledgeDelivery, DeliveryMode, NotStartedError, SeekDelivery, SilentDelivery, } from "@peerbit/stream-interface";
+import { AbortError, waitFor } from "@peerbit/time";
 import pDefer, {} from "p-defer";
 import PQueue from "p-queue";
 import { concat } from "uint8arrays";
 import { BlocksMessage } from "./blocks.js";
 import { CPUUsageIntervalLag } from "./cpu.js";
-import { debounceAcculmulator, debounceFixedInterval, debouncedAccumulatorMap, } from "./debounce.js";
-import { EntryWithRefs, ExchangeHeadsMessage, RequestIPrune, RequestMaybeSync, ResponseIPrune, ResponseMaybeSync, createExchangeHeadsMessages, } from "./exchange-heads.js";
+import { debounceAccumulator, debounceFixedInterval, debouncedAccumulatorMap, } from "./debounce.js";
+import { EntryWithRefs, ExchangeHeadsMessage, RequestIPrune, ResponseIPrune, createExchangeHeadsMessages, } from "./exchange-heads.js";
+import { MAX_U32, bytesToNumber, createNumbers, denormalizer, } from "./integers.js";
 import { TransportMessage } from "./message.js";
 import { PIDReplicationController } from "./pid.js";
-import { EntryReplicated, ReplicationIntent, ReplicationRange, ReplicationRangeIndexable, getCoverSet, getEvenlySpacedU32, getSamples, hasCoveringRange, isMatured, minimumWidthToCover, shouldAssigneToRangeBoundary, toRebalance, } from "./ranges.js";
-import { createReplicationDomainHash, hashToU32, } from "./replication-domain-hash.js";
+import { EntryReplicatedU32, EntryReplicatedU64, ReplicationIntent, ReplicationRangeIndexableU32, ReplicationRangeIndexableU64, ReplicationRangeMessage, SyncStatus, appromixateCoverage, getCoverSet, getSamples, iHaveCoveringRange, isMatured, isReplicationRangeMessage, mergeRanges, minimumWidthToCover, shouldAssigneToRangeBoundary as shouldAssignToRangeBoundary, toRebalance, } from "./ranges.js";
+import { createReplicationDomainHash, } from "./replication-domain-hash.js";
 import { createReplicationDomainTime, } from "./replication-domain-time.js";
 import { debounceAggregationChanges, mergeReplicationChanges, } from "./replication-domain.js";
-import { AbsoluteReplicas, AddedReplicationSegmentMessage, AllReplicatingSegmentsMessage, ReplicationError, RequestReplicationInfoMessage, ResponseRoleMessage, StoppedReplicating, decodeReplicas, encodeReplicas, maxReplicas, } from "./replication.js";
-import { MAX_U32, Observer, Replicator, scaleToU32 } from "./role.js";
+import { AbsoluteReplicas, AddedReplicationSegmentMessage, AllReplicatingSegmentsMessage, MinReplicas, ReplicationError, RequestReplicationInfoMessage, ResponseRoleMessage, StoppedReplicating, decodeReplicas, encodeReplicas, maxReplicas, } from "./replication.js";
+import { Observer, Replicator } from "./role.js";
+import { RatelessIBLTSynchronizer } from "./sync/rateless-iblt.js";
+import { SimpleSyncronizer } from "./sync/simple.js";
 import { groupByGid } from "./utils.js";
 export { createReplicationDomainHash, createReplicationDomainTime, };
 export { CPUUsageIntervalLag };
 export * from "./replication.js";
-export { EntryReplicated, ReplicationRangeIndexable };
+export { EntryReplicatedU32, EntryReplicatedU64, };
 export const logger = loggerFn({ module: "shared-log" });
 const getLatestEntry = (entries) => {
     let latest = undefined;
@@ -85,9 +87,33 @@ const isReplicationOptionsDependentOnPreviousState = (options) => {
     }
     return false;
 };
+const createIndexableDomainFromResolution = (resolution) => {
+    const denormalizerFn = denormalizer(resolution);
+    const byteToNumberFn = bytesToNumber(resolution);
+    if (resolution === "u32") {
+        return {
+            constructorEntry: EntryReplicatedU32,
+            constructorRange: ReplicationRangeIndexableU32,
+            denormalize: denormalizerFn,
+            bytesToNumber: byteToNumberFn,
+            numbers: createNumbers(resolution),
+        };
+    }
+    else if (resolution === "u64") {
+        return {
+            constructorEntry: EntryReplicatedU64,
+            constructorRange: ReplicationRangeIndexableU64,
+            denormalize: denormalizerFn,
+            bytesToNumber: byteToNumberFn,
+            numbers: createNumbers(resolution),
+        };
+    }
+    throw new Error("Unsupported resolution");
+};
 export const DEFAULT_MIN_REPLICAS = 2;
 export const WAIT_FOR_REPLICATOR_TIMEOUT = 9000;
 export const WAIT_FOR_ROLE_MATURITY = 5000;
+export const WAIT_FOR_PRUNE_DELAY = 5000;
 const PRUNE_DEBOUNCE_INTERVAL = 500;
 // DONT SET THIS ANY LOWER, because it will make the pid controller unstable as the system responses are not fast enough to updates from the pid controller
 const RECALCULATE_PARTICIPATION_DEBOUNCE_INTERVAL = 1000;
@@ -108,7 +134,10 @@ let SharedLog = class SharedLog extends Program {
     _isAdaptiveReplicating;
     _replicationRangeIndex;
     _entryCoordinatesIndex;
+    coordinateToHash;
+    uniqueReplicators;
     /* private _totalParticipation!: number; */
+    // gid -> coordinate -> publicKeyHash list (of owners)
     _gidPeersHistory;
     _onSubscriptionFn;
     _onUnsubscriptionFn;
@@ -118,6 +147,7 @@ let SharedLog = class SharedLog extends Program {
     _respondToIHaveTimeout;
     _pendingDeletes;
     _pendingIHave;
+    // public key hash to range id to range
     pendingMaturity; // map of peerId to timeout
     latestReplicationInfoMessage;
     remoteBlocks;
@@ -129,24 +159,22 @@ let SharedLog = class SharedLog extends Program {
     // A fn for debouncing the calls for pruning
     pruneDebouncedFn;
     responseToPruneDebouncedFn;
+    _requestIPruneSent; // tracks entry hash to peer hash for requesting I prune messages
+    _requestIPruneResponseReplicatorSet; // tracks entry hash to peer hash
     replicationChangeDebounceFn;
     // regular distribution checks
     distributeQueue;
-    // Syncing and dedeplucation work
-    syncMoreInterval;
-    // map of hash to public keys that we can ask for entries
-    syncInFlightQueue;
-    syncInFlightQueueInverted;
-    // map of hash to public keys that we have asked for entries
-    syncInFlight;
+    syncronizer;
     replicas;
     cpuUsage;
     timeUntilRoleMaturity;
     waitForReplicatorTimeout;
+    waitForPruneDelay;
     distributionDebounceTime;
     replicationController;
     history;
     domain;
+    indexableDomain;
     interval;
     constructor(properties) {
         super();
@@ -168,6 +196,7 @@ let SharedLog = class SharedLog extends Program {
         if (segments.length > 0) {
             const segment = segments[0].toReplicationRange();
             return new Replicator({
+                // TODO types
                 factor: segment.factor / MAX_U32,
                 offset: segment.offset / MAX_U32,
             });
@@ -179,32 +208,8 @@ let SharedLog = class SharedLog extends Program {
         if (!this._isReplicating) {
             return false;
         }
-        /*
-        if (isAdaptiveReplicatorOption(this._replicationSettings)) {
-            return true;
-        }
-
-        if ((this.replicationSettings as FixedReplicationOptions).factor !== 0) {
-            return true;
-        } */
         return (await this.countReplicationSegments()) > 0;
     }
-    /* get totalParticipation(): number {
-        return this._totalParticipation;
-    } */
-    async calculateTotalParticipation() {
-        const sum = await this.replicationIndex.sum({ key: "width" });
-        return Number(sum) / MAX_U32;
-    }
-    async countReplicationSegments() {
-        const count = await this.replicationIndex.count({
-            query: new StringMatch({
-                key: "hash",
-                value: this.node.identity.publicKey.hashcode(),
-            }),
-        });
-        return count;
-    }
     setupRebalanceDebounceFunction(interval = RECALCULATE_PARTICIPATION_DEBOUNCE_INTERVAL) {
         this.rebalanceParticipationDebounced = undefined;
         // make the rebalancing to respect warmup time
@@ -225,7 +230,7 @@ let SharedLog = class SharedLog extends Program {
         ) */
         () => intervalTime);
     }
-    async _replicate(options, { reset, checkDuplicates, announce, } = {}) {
+    async _replicate(options, { reset, checkDuplicates, syncStatus, announce, mergeSegments, } = {}) {
         let offsetWasProvided = false;
         if (isUnreplicationOptions(options)) {
             await this.unreplicate();
@@ -252,7 +257,7 @@ let SharedLog = class SharedLog extends Program {
                 ranges = [maybeRange];
                 offsetWasProvided = true;
             }
-            else if (options instanceof ReplicationRange) {
+            else if (isReplicationRangeMessage(options)) {
                 ranges = [
                     options.toReplicationRangeIndexable(this.node.identity.publicKey),
                 ];
@@ -275,28 +280,56 @@ let SharedLog = class SharedLog extends Program {
                     return;
                 }
                 for (const rangeArg of rangeArgs) {
+                    let timestamp = undefined;
+                    if (rangeArg.id != null) {
+                        // fetch the previous timestamp if it exists
+                        const indexed = await this.replicationIndex.get(toId(rangeArg.id), {
+                            shape: { id: true, timestamp: true },
+                        });
+                        if (indexed) {
+                            timestamp = indexed.value.timestamp;
+                        }
+                    }
                     const normalized = rangeArg.normalized ?? true;
                     offsetWasProvided = rangeArg.offset != null;
-                    const offset = rangeArg.offset ??
-                        (normalized ? Math.random() : scaleToU32(Math.random()));
+                    const offset = rangeArg.offset != null
+                        ? normalized
+                            ? this.indexableDomain.numbers.denormalize(rangeArg.offset)
+                            : rangeArg.offset
+                        : this.indexableDomain.numbers.random();
                     let factor = rangeArg.factor;
-                    let width = normalized ? 1 : scaleToU32(1);
-                    ranges.push(new ReplicationRangeIndexable({
+                    let fullWidth = this.indexableDomain.numbers.maxValue;
+                    let factorDenormalized = !normalized
+                        ? factor
+                        : this.indexableDomain.numbers.denormalize(factor);
+                    ranges.push(new this.indexableDomain.constructorRange({
                         id: rangeArg.id,
-                        normalized,
+                        // @ts-ignore
                         offset: offset,
-                        length: typeof factor === "number"
+                        // @ts-ignore
+                        length: (factor === "all"
+                            ? fullWidth
+                            : factor === "right"
+                                ? // @ts-ignore
+                                    fullWidth - offset
+                                : factorDenormalized),
+                        /* typeof factor === "number"
                             ? factor
                             : factor === "all"
                                 ? width
-                                : width - offset,
+                                // @ts-ignore
+                                : width - offset, */
                         publicKeyHash: this.node.identity.publicKey.hashcode(),
                         mode: rangeArg.strict
                             ? ReplicationIntent.Strict
                             : ReplicationIntent.NonStrict, // automatic means that this range might be reused later for dynamic replication behaviour
-                        timestamp: BigInt(+new Date()),
+                        timestamp: timestamp ?? BigInt(+new Date()),
                     }));
                 }
+                if (mergeSegments && ranges.length > 1) {
+                    const mergedSegment = mergeRanges(ranges, this.indexableDomain.numbers);
+                    ranges = [mergedSegment];
+                }
             }
             for (const range of ranges) {
                 this.oldestOpenTime = Math.min(Number(range.timestamp), this.oldestOpenTime);
@@ -312,6 +345,7 @@ let SharedLog = class SharedLog extends Program {
                 reset: resetRanges ?? false,
                 checkDuplicates,
                 announce,
+                syncStatus,
             });
             return ranges;
         }
@@ -339,7 +373,8 @@ let SharedLog = class SharedLog extends Program {
     }
     async replicate(rangeOrEntry, options) {
         let range = undefined;
-        if (rangeOrEntry instanceof ReplicationRange) {
+        let syncStatus = SyncStatus.Unsynced;
+        if (rangeOrEntry instanceof ReplicationRangeMessage) {
             range = rangeOrEntry;
         }
         else if (rangeOrEntry instanceof Entry) {
@@ -348,6 +383,7 @@ let SharedLog = class SharedLog extends Program {
                 offset: await this.domain.fromEntry(rangeOrEntry),
                 normalized: false,
             };
+            syncStatus = SyncStatus.Synced; /// we already have the entries
         }
         else if (Array.isArray(rangeOrEntry)) {
             let ranges = [];
@@ -358,6 +394,7 @@ let SharedLog = class SharedLog extends Program {
                         offset: await this.domain.fromEntry(entry),
                         normalized: false,
                     });
+                    syncStatus = SyncStatus.Synced; /// we already have the entries
                 }
                 else {
                     ranges.push(entry);
@@ -368,7 +405,7 @@ let SharedLog = class SharedLog extends Program {
         else {
             range = rangeOrEntry ?? true;
         }
-        return this._replicate(range, options);
+        return this._replicate(range, { ...options, syncStatus });
     }
     async unreplicate(rangeOrEntry) {
         let range;
@@ -378,7 +415,7 @@ let SharedLog = class SharedLog extends Program {
                 offset: await this.domain.fromEntry(rangeOrEntry),
             };
         }
-        else if (rangeOrEntry instanceof ReplicationRange) {
+        else if (rangeOrEntry instanceof ReplicationRangeMessage) {
             range = rangeOrEntry;
         }
         else {
@@ -405,46 +442,48 @@ let SharedLog = class SharedLog extends Program {
         });
     }
     async removeReplicator(key, options) {
-        const fn = async () => {
-            const keyHash = typeof key === "string" ? key : key.hashcode();
-            const deleted = await this.replicationIndex
-                .iterate({
-                query: { hash: keyHash },
-            })
-                .all();
-            await this.replicationIndex.del({ query: { hash: keyHash } });
-            await this.updateOldestTimestampFromIndex();
-            const isMe = this.node.identity.publicKey.hashcode() === keyHash;
-            if (isMe) {
-                // announce that we are no longer replicating
-                await this.rpc.send(new AllReplicatingSegmentsMessage({ segments: [] }), { priority: 1 });
+        const keyHash = typeof key === "string" ? key : key.hashcode();
+        const deleted = await this.replicationIndex
+            .iterate({
+            query: { hash: keyHash },
+        })
+            .all();
+        this.uniqueReplicators.delete(keyHash);
+        await this.replicationIndex.del({ query: { hash: keyHash } });
+        await this.updateOldestTimestampFromIndex();
+        const isMe = this.node.identity.publicKey.hashcode() === keyHash;
+        if (isMe) {
+            // announce that we are no longer replicating
+            await this.rpc.send(new AllReplicatingSegmentsMessage({ segments: [] }), {
+                priority: 1,
+            });
+        }
+        if (options?.noEvent !== true) {
+            if (key instanceof PublicSignKey) {
+                this.events.dispatchEvent(new CustomEvent("replication:change", {
+                    detail: { publicKey: key },
+                }));
             }
-            if (options?.noEvent !== true) {
-                if (key instanceof PublicSignKey) {
-                    this.events.dispatchEvent(new CustomEvent("replication:change", {
-                        detail: { publicKey: key },
-                    }));
-                }
-                else {
-                    throw new Error("Key was not a PublicSignKey");
-                }
+            else {
+                throw new Error("Key was not a PublicSignKey");
             }
-            deleted.forEach((x) => {
-                return this.replicationChangeDebounceFn.add({
-                    range: x.value,
-                    type: "removed",
-                });
+        }
+        for (const x of deleted) {
+            this.replicationChangeDebounceFn.add({
+                range: x.value,
+                type: "removed",
             });
-            const pendingMaturity = this.pendingMaturity.get(keyHash);
-            if (pendingMaturity) {
-                clearTimeout(pendingMaturity.timeout);
-                this.pendingMaturity.delete(keyHash);
-            }
-            if (!isMe) {
-                this.rebalanceParticipationDebounced?.();
+        }
+        const pendingMaturity = this.pendingMaturity.get(keyHash);
+        if (pendingMaturity) {
+            for (const [_k, v] of pendingMaturity) {
+                clearTimeout(v.timeout);
             }
-        };
-        return fn();
+            this.pendingMaturity.delete(keyHash);
+        }
+        if (!isMe) {
+            this.rebalanceParticipationDebounced?.();
+        }
     }
     async updateOldestTimestampFromIndex() {
         const iterator = await this.replicationIndex.iterate({
@@ -458,190 +497,196 @@ let SharedLog = class SharedLog extends Program {
                 : +new Date();
     }
     async removeReplicationRange(ids, from) {
-        const fn = async () => {
-            let idMatcher = new Or(ids.map((x) => new ByteMatchQuery({ key: "id", value: x })));
-            // make sure we are not removing something that is owned by the replicator
-            let identityMatcher = new StringMatch({
-                key: "hash",
-                value: from.hashcode(),
-            });
-            let query = new And([idMatcher, identityMatcher]);
-            const pendingMaturity = this.pendingMaturity.get(from.hashcode());
-            if (pendingMaturity) {
-                for (const id of ids) {
-                    pendingMaturity.ranges.delete(id.toString());
-                }
-                if (pendingMaturity.ranges.size === 0) {
-                    clearTimeout(pendingMaturity.timeout);
-                    this.pendingMaturity.delete(from.hashcode());
+        let idMatcher = new Or(ids.map((x) => new ByteMatchQuery({ key: "id", value: x })));
+        // make sure we are not removing something that is owned by the replicator
+        let identityMatcher = new StringMatch({
+            key: "hash",
+            value: from.hashcode(),
+        });
+        let query = new And([idMatcher, identityMatcher]);
+        const pendingMaturity = this.pendingMaturity.get(from.hashcode());
+        if (pendingMaturity) {
+            for (const id of ids) {
+                const info = pendingMaturity.get(id.toString());
+                if (info) {
+                    clearTimeout(info.timeout);
+                    pendingMaturity.delete(id.toString());
                 }
             }
-            await this.replicationIndex.del({ query });
-            await this.updateOldestTimestampFromIndex();
-            this.events.dispatchEvent(new CustomEvent("replication:change", {
-                detail: { publicKey: from },
-            }));
-            if (!from.equals(this.node.identity.publicKey)) {
-                this.rebalanceParticipationDebounced?.();
+            if (pendingMaturity.size === 0) {
+                this.pendingMaturity.delete(from.hashcode());
             }
-        };
-        return fn();
+        }
+        await this.replicationIndex.del({ query });
+        const otherSegmentsIterator = this.replicationIndex.iterate({ query: { hash: from.hashcode() } }, { shape: { id: true } });
+        if ((await otherSegmentsIterator.next(1)).length === 0) {
+            this.uniqueReplicators.delete(from.hashcode());
+        }
+        await otherSegmentsIterator.close();
+        await this.updateOldestTimestampFromIndex();
+        this.events.dispatchEvent(new CustomEvent("replication:change", {
+            detail: { publicKey: from },
+        }));
+        if (!from.equals(this.node.identity.publicKey)) {
+            this.rebalanceParticipationDebounced?.();
+        }
     }
     async addReplicationRange(ranges, from, { reset, checkDuplicates, } = {}) {
-        const fn = async () => {
-            if (this._isTrustedReplicator &&
-                !(await this._isTrustedReplicator(from))) {
-                return undefined;
-            }
-            let isNewReplicator = false;
-            let diffs;
-            let deleted = undefined;
-            if (reset) {
-                deleted = (await this.replicationIndex
-                    .iterate({
-                    query: { hash: from.hashcode() },
-                })
-                    .all()).map((x) => x.value);
-                let prevCount = deleted.length;
-                await this.replicationIndex.del({ query: { hash: from.hashcode() } });
-                diffs = [
-                    ...deleted.map((x) => {
-                        return { range: x, type: "removed" };
-                    }),
-                    ...ranges.map((x) => {
-                        return { range: x, type: "added" };
-                    }),
-                ];
-                isNewReplicator = prevCount === 0 && ranges.length > 0;
+        if (this._isTrustedReplicator && !(await this._isTrustedReplicator(from))) {
+            return undefined;
+        }
+        let isNewReplicator = false;
+        let diffs;
+        let deleted = undefined;
+        if (reset) {
+            deleted = (await this.replicationIndex
+                .iterate({
+                query: { hash: from.hashcode() },
+            })
+                .all()).map((x) => x.value);
+            let prevCount = deleted.length;
+            await this.replicationIndex.del({ query: { hash: from.hashcode() } });
+            diffs = [
+                ...deleted.map((x) => {
+                    return { range: x, type: "removed" };
+                }),
+                ...ranges.map((x) => {
+                    return { range: x, type: "added" };
+                }),
+            ];
+            isNewReplicator = prevCount === 0 && ranges.length > 0;
+        }
+        else {
+            let existing = await this.replicationIndex
+                .iterate({
+                query: ranges.map((x) => new ByteMatchQuery({ key: "id", value: x.id })),
+            }, { reference: true })
+                .all();
+            if (existing.length === 0) {
+                let prevCount = await this.replicationIndex.count({
+                    query: new StringMatch({ key: "hash", value: from.hashcode() }),
+                });
+                isNewReplicator = prevCount === 0;
             }
             else {
-                let existing = await this.replicationIndex
-                    .iterate({
-                    query: ranges.map((x) => new ByteMatchQuery({ key: "id", value: x.id })),
-                }, { reference: true })
-                    .all();
-                if (existing.length === 0) {
-                    let prevCount = await this.replicationIndex.count({
-                        query: new StringMatch({ key: "hash", value: from.hashcode() }),
-                    });
-                    isNewReplicator = prevCount === 0;
-                }
-                else {
-                    isNewReplicator = false;
+                isNewReplicator = false;
+            }
+            if (checkDuplicates) {
+                let deduplicated = [];
+                // TODO also deduplicate/de-overlap among the ranges that ought to be inserted?
+                for (const range of ranges) {
+                    if (!(await iHaveCoveringRange(this.replicationIndex, range))) {
+                        deduplicated.push(range);
+                    }
                 }
-                if (checkDuplicates) {
-                    let deduplicated = [];
-                    // TODO also deduplicate/de-overlap among the ranges that ought to be inserted?
-                    for (const range of ranges) {
-                        if (!(await hasCoveringRange(this.replicationIndex, range))) {
-                            deduplicated.push(range);
-                        }
+                ranges = deduplicated;
+            }
+            let existingMap = new Map();
+            for (const result of existing) {
+                existingMap.set(result.value.idString, result.value);
+            }
+            let changes = ranges
+                .map((x) => {
+                const prev = existingMap.get(x.idString);
+                if (prev) {
+                    if (prev.equalRange(x)) {
+                        return undefined;
                     }
-                    ranges = deduplicated;
+                    return { range: x, prev, type: "updated" };
                 }
-                let existingMap = new Map();
-                for (const result of existing) {
-                    existingMap.set(result.value.idString, result.value);
+                else {
+                    return { range: x, type: "added" };
                 }
-                let changes = ranges
-                    .map((x) => {
-                    const prev = existingMap.get(x.idString);
-                    if (prev) {
-                        if (prev.equalRange(x)) {
-                            return undefined;
+            })
+                .filter((x) => x != null);
+            diffs = changes;
+        }
+        this.uniqueReplicators.add(from.hashcode());
+        let now = +new Date();
+        let minRoleAge = await this.getDefaultMinRoleAge();
+        let isAllMature = true;
+        for (const diff of diffs) {
+            if (diff.type === "added" || diff.type === "updated") {
+                /* if (this.closed) {
+                    return;
+                } */
+                await this.replicationIndex.put(diff.range);
+                if (!reset) {
+                    this.oldestOpenTime = Math.min(Number(diff.range.timestamp), this.oldestOpenTime);
+                }
+                const isMature = isMatured(diff.range, now, minRoleAge);
+                if (!isMature /* && diff.range.hash !== this.node.identity.publicKey.hashcode() */) {
+                    // second condition is to avoid the case where we are adding a range that we own
+                    isAllMature = false;
+                    let pendingRanges = this.pendingMaturity.get(diff.range.hash);
+                    if (!pendingRanges) {
+                        pendingRanges = new Map();
+                        this.pendingMaturity.set(diff.range.hash, pendingRanges);
+                    }
+                    let waitForMaturityTime = Math.max(minRoleAge - (now - Number(diff.range.timestamp)), 0);
+                    const setupTimeout = () => setTimeout(async () => {
+                        this.events.dispatchEvent(new CustomEvent("replicator:mature", {
+                            detail: { publicKey: from },
+                        }));
+                        this.replicationChangeDebounceFn.add(diff); // we need to call this here because the outcom of findLeaders will be different when some ranges become mature, i.e. some of data we own might be prunable!
+                        pendingRanges.delete(diff.range.idString);
+                        if (pendingRanges.size === 0) {
+                            this.pendingMaturity.delete(diff.range.hash);
                         }
-                        return { range: x, prev, type: "updated" };
+                    }, waitForMaturityTime);
+                    let prevPendingMaturity = pendingRanges.get(diff.range.idString);
+                    if (prevPendingMaturity) {
+                        // only reset the timer if the new range is older than the previous one, this means that waitForMaturityTime less than the previous one
+                        clearTimeout(prevPendingMaturity.timeout);
+                        prevPendingMaturity.timeout = setupTimeout();
                     }
                     else {
-                        return { range: x, type: "added" };
-                    }
-                })
-                    .filter((x) => x != null);
-                diffs = changes;
-            }
-            let now = +new Date();
-            let minRoleAge = await this.getDefaultMinRoleAge();
-            let isAllMature = true;
-            for (const diff of diffs) {
-                if (diff.type === "added" || diff.type === "updated") {
-                    await this.replicationIndex.put(diff.range);
-                    if (!reset) {
-                        this.oldestOpenTime = Math.min(Number(diff.range.timestamp), this.oldestOpenTime);
-                    }
-                    const isMature = isMatured(diff.range, now, minRoleAge);
-                    if (!isMature /* && diff.range.hash !== this.node.identity.publicKey.hashcode() */) {
-                        // second condition is to avoid the case where we are adding a range that we own
-                        isAllMature = false;
-                        let prevPendingMaturity = this.pendingMaturity.get(diff.range.hash);
-                        let map;
-                        let waitForMaturityTime = Math.max(minRoleAge - (now - Number(diff.range.timestamp)), 0);
-                        if (prevPendingMaturity) {
-                            map = prevPendingMaturity.ranges;
-                            if (prevPendingMaturity.timestamp < diff.range.timestamp) {
-                                // something has changed so we need to reset the timeout
-                                clearTimeout(prevPendingMaturity.timeout);
-                                prevPendingMaturity.timestamp = diff.range.timestamp;
-                                prevPendingMaturity.timeout = setTimeout(() => {
-                                    this.events.dispatchEvent(new CustomEvent("replicator:mature", {
-                                        detail: { publicKey: from },
-                                    }));
-                                    for (const value of map.values()) {
-                                        this.replicationChangeDebounceFn.add(value); // we need to call this here because the outcom of findLeaders will be different when some ranges become mature, i.e. some of data we own might be prunable!
-                                    }
-                                }, waitForMaturityTime);
-                            }
-                        }
-                        else {
-                            map = new Map();
-                            this.pendingMaturity.set(diff.range.hash, {
-                                timestamp: diff.range.timestamp,
-                                ranges: map,
-                                timeout: setTimeout(() => {
-                                    this.events.dispatchEvent(new CustomEvent("replicator:mature", {
-                                        detail: { publicKey: from },
-                                    }));
-                                    for (const value of map.values()) {
-                                        this.replicationChangeDebounceFn.add(value); // we need to call this here because the outcom of findLeaders will be different when some ranges become mature, i.e. some of data we own might be prunable!
-                                    }
-                                }, waitForMaturityTime),
-                            });
-                        }
-                        map.set(diff.range.idString, diff);
+                        pendingRanges.set(diff.range.idString, {
+                            range: diff,
+                            timeout: setupTimeout(),
+                        });
                     }
                 }
-                else {
-                    const prev = this.pendingMaturity.get(diff.range.hash);
+            }
+            else {
+                const pendingFromPeer = this.pendingMaturity.get(diff.range.hash);
+                if (pendingFromPeer) {
+                    const prev = pendingFromPeer.get(diff.range.idString);
                     if (prev) {
-                        prev.ranges.delete(diff.range.idString);
+                        clearTimeout(prev.timeout);
+                        pendingFromPeer.delete(diff.range.idString);
+                    }
+                    if (pendingFromPeer.size === 0) {
+                        this.pendingMaturity.delete(diff.range.hash);
                     }
                 }
             }
-            if (reset) {
-                await this.updateOldestTimestampFromIndex();
-            }
-            this.events.dispatchEvent(new CustomEvent("replication:change", {
+        }
+        if (reset) {
+            await this.updateOldestTimestampFromIndex();
+        }
+        this.events.dispatchEvent(new CustomEvent("replication:change", {
+            detail: { publicKey: from },
+        }));
+        if (isNewReplicator) {
+            this.events.dispatchEvent(new CustomEvent("replicator:join", {
                 detail: { publicKey: from },
             }));
-            if (isNewReplicator) {
-                this.events.dispatchEvent(new CustomEvent("replicator:join", {
+            if (isAllMature) {
+                this.events.dispatchEvent(new CustomEvent("replicator:mature", {
                     detail: { publicKey: from },
                 }));
-                if (isAllMature) {
-                    this.events.dispatchEvent(new CustomEvent("replicator:mature", {
-                        detail: { publicKey: from },
-                    }));
-                }
             }
-            diffs.length > 0 &&
-                diffs.map((x) => this.replicationChangeDebounceFn.add(x));
-            if (!from.equals(this.node.identity.publicKey)) {
-                this.rebalanceParticipationDebounced?.();
+        }
+        if (diffs.length > 0) {
+            for (const diff of diffs) {
+                this.replicationChangeDebounceFn.add(diff);
             }
-            return diffs;
-        };
-        // we sequialize this because we are going to queries to check wether to add or not
-        // if two processes do the same this both process might add a range while only one in practice should
-        return fn();
+        }
+        if (!from.equals(this.node.identity.publicKey)) {
+            this.rebalanceParticipationDebounced?.();
+        }
+        return diffs;
     }
     async startAnnounceReplicating(range, options = {}) {
         const change = await this.addReplicationRange(range, this.node.identity.publicKey, options);
@@ -670,15 +715,47 @@ let SharedLog = class SharedLog extends Program {
             }
         }
     }
+    removePeerFromGidPeerHistory(publicKeyHash, gid) {
+        if (gid) {
+            const gidMap = this._gidPeersHistory.get(gid);
+            if (gidMap) {
+                gidMap.delete(publicKeyHash);
+                if (gidMap.size === 0) {
+                    this._gidPeersHistory.delete(gid);
+                }
+            }
+        }
+        else {
+            for (const key of this._gidPeersHistory.keys()) {
+                this.removePeerFromGidPeerHistory(publicKeyHash, key);
+            }
+        }
+    }
+    addPeersToGidPeerHistory(gid, publicKeys, reset) {
+        let set = this._gidPeersHistory.get(gid);
+        if (!set) {
+            set = new Set();
+            this._gidPeersHistory.set(gid, set);
+        }
+        else {
+            if (reset) {
+                set.clear();
+            }
+        }
+        for (const key of publicKeys) {
+            set.add(key);
+        }
+        return set;
+    }
     async append(data, options) {
         const appendOptions = { ...options };
-        const minReplicas = options?.replicas
+        const minReplicas = this.getClampedReplicas(options?.replicas
             ? typeof options.replicas === "number"
                 ? new AbsoluteReplicas(options.replicas)
                 : options.replicas
-            : this.replicas.min;
-        const minReplicasValue = minReplicas.getValue(this);
+            : undefined);
         const minReplicasData = encodeReplicas(minReplicas);
+        const minReplicasValue = minReplicas.getValue(this);
         checkMinReplicasLimit(minReplicasValue);
         if (!appendOptions.meta) {
             appendOptions.meta = {
@@ -707,10 +784,14 @@ let SharedLog = class SharedLog extends Program {
         if (options?.replicate) {
             await this.replicate(result.entry, { checkDuplicates: true });
         }
-        let { leaders, isLeader } = await this.findLeadersPersist({
-            entry: result.entry,
-            minReplicas: minReplicas.getValue(this),
-        }, result.entry, { persist: {} });
+        const coordinates = await this.createCoordinates(result.entry, minReplicasValue);
+        let isLeader = false;
+        let leaders = await this.findLeaders(coordinates, result.entry, {
+            persist: {},
+            onLeader: (key) => {
+                isLeader = isLeader || this.node.identity.publicKey.hashcode() === key;
+            },
+        });
         // --------------
         if (options?.target !== "none") {
             for await (const message of createExchangeHeadsMessages(this.log, [
@@ -718,35 +799,25 @@ let SharedLog = class SharedLog extends Program {
             ])) {
                 if (options?.target === "replicators" || !options?.target) {
                     if (message.heads[0].gidRefrences.length > 0) {
-                        const newAndOldLeaders = new Map(leaders);
                         for (const ref of message.heads[0].gidRefrences) {
                             const entryFromGid = this.log.entryIndex.getHeads(ref, false);
                             for (const entry of await entryFromGid.all()) {
-                                let coordinate = await this.getCoordinates(entry);
-                                if (coordinate == null) {
-                                    coordinate = await this.createCoordinates(entry, minReplicasValue);
+                                let coordinates = await this.getCoordinates(entry);
+                                if (coordinates == null) {
+                                    coordinates = await this.createCoordinates(entry, minReplicasValue);
                                     // TODO are we every to come here?
                                 }
-                                for (const [hash, features] of await this.findLeaders(coordinate)) {
-                                    newAndOldLeaders.set(hash, features);
+                                const result = await this._findLeaders(coordinates);
+                                for (const [k, v] of result) {
+                                    leaders.set(k, v);
                                 }
                             }
                         }
-                        leaders = newAndOldLeaders;
-                    }
-                    let set = this._gidPeersHistory.get(result.entry.meta.gid);
-                    if (!set) {
-                        set = new Set(leaders.keys());
-                        this._gidPeersHistory.set(result.entry.meta.gid, set);
-                    }
-                    else {
-                        for (const [receiver, _features] of leaders) {
-                            set.add(receiver);
-                        }
                     }
+                    const set = this.addPeersToGidPeerHistory(result.entry.meta.gid, leaders.keys());
                     mode = isLeader
-                        ? new SilentDelivery({ redundancy: 1, to: leaders.keys() })
-                        : new AcknowledgeDelivery({ redundancy: 1, to: leaders.keys() });
+                        ? new SilentDelivery({ redundancy: 1, to: set })
+                        : new AcknowledgeDelivery({ redundancy: 1, to: set });
                 }
                 // TODO add options for waiting ?
                 this.rpc.send(message, {
@@ -757,7 +828,7 @@ let SharedLog = class SharedLog extends Program {
         if (!isLeader) {
             this.pruneDebouncedFn.add({
                 key: result.entry.hash,
-                value: result.entry,
+                value: { entry: result.entry, leaders },
             });
         }
         this.rebalanceParticipationDebounced?.();
@@ -765,7 +836,7 @@ let SharedLog = class SharedLog extends Program {
     }
     async open(options) {
         this.replicas = {
-            min: options?.replicas?.min
+            min: options?.replicas?.min != null
                 ? typeof options?.replicas?.min === "number"
                     ? new AbsoluteReplicas(options?.replicas?.min)
                     : options?.replicas?.min
@@ -776,14 +847,18 @@ let SharedLog = class SharedLog extends Program {
                     : options.replicas.max
                 : undefined,
         };
-        this.domain = options?.domain ?? createReplicationDomainHash();
+        this._logProperties = options;
+        // TODO types
+        this.domain = options?.domain
+            ? options.domain
+            : createReplicationDomainHash(options?.compatibility && options?.compatibility < 10 ? "u32" : "u64");
+        this.indexableDomain = createIndexableDomainFromResolution(this.domain.resolution);
         this._respondToIHaveTimeout = options?.respondToIHaveTimeout ?? 2e4;
         this._pendingDeletes = new Map();
         this._pendingIHave = new Map();
         this.latestReplicationInfoMessage = new Map();
-        this.syncInFlightQueue = new Map();
-        this.syncInFlightQueueInverted = new Map();
-        this.syncInFlight = new Map();
+        this.coordinateToHash = new Cache({ max: 1e6, ttl: 1e4 });
+        this.uniqueReplicators = new Set();
         this.openTime = +new Date();
         this.oldestOpenTime = this.openTime;
         this.distributionDebounceTime =
@@ -792,10 +867,13 @@ let SharedLog = class SharedLog extends Program {
             options?.timeUntilRoleMaturity ?? WAIT_FOR_ROLE_MATURITY;
         this.waitForReplicatorTimeout =
             options?.waitForReplicatorTimeout || WAIT_FOR_REPLICATOR_TIMEOUT;
+        this.waitForPruneDelay = options?.waitForPruneDelay || WAIT_FOR_PRUNE_DELAY;
+        if (this.waitForReplicatorTimeout < this.timeUntilRoleMaturity) {
+            this.waitForReplicatorTimeout = this.timeUntilRoleMaturity; // does not makes sense to expect a replicator to mature faster than it is reachable
+        }
         this._closeController = new AbortController();
         this._isTrustedReplicator = options?.canReplicate;
         this.sync = options?.sync;
-        this._logProperties = options;
         this.pendingMaturity = new Map();
         const id = sha256Base64Sync(this.log.id);
         const storage = await this.node.storage.sublevel(id);
@@ -810,14 +888,13 @@ let SharedLog = class SharedLog extends Program {
             waitFor: this.rpc.waitFor.bind(this.rpc),
         });
         await this.remoteBlocks.start();
-        /* this._totalParticipation = 0; */
         const logScope = await this.node.indexer.scope(id);
         const replicationIndex = await logScope.scope("replication");
         this._replicationRangeIndex = await replicationIndex.init({
-            schema: ReplicationRangeIndexable,
+            schema: this.indexableDomain.constructorRange,
         });
         this._entryCoordinatesIndex = await replicationIndex.init({
-            schema: EntryReplicated,
+            schema: this.indexableDomain.constructorEntry,
         });
         const logIndex = await logScope.scope("log");
         await this.node.indexer.start(); // TODO why do we need to start the indexer here?
@@ -829,13 +906,21 @@ let SharedLog = class SharedLog extends Program {
                 }),
             ],
         })) > 0;
-        /* this._totalParticipation = await this.calculateTotalParticipation(); */
         this._gidPeersHistory = new Map();
+        this._requestIPruneSent = new Map();
+        this._requestIPruneResponseReplicatorSet = new Map();
         this.replicationChangeDebounceFn = debounceAggregationChanges((change) => this.onReplicationChange(change).then(() => this.rebalanceParticipationDebounced?.()), this.distributionDebounceTime);
         this.pruneDebouncedFn = debouncedAccumulatorMap((map) => {
             this.prune(map);
-        }, PRUNE_DEBOUNCE_INTERVAL);
-        this.responseToPruneDebouncedFn = debounceAcculmulator((result) => {
+        }, PRUNE_DEBOUNCE_INTERVAL, // TODO make this dynamic on the number of replicators
+        (into, from) => {
+            for (const [k, v] of from.leaders) {
+                if (!into.leaders.has(k)) {
+                    into.leaders.set(k, v);
+                }
+            }
+        });
+        this.responseToPruneDebouncedFn = debounceAccumulator((result) => {
             let allRequestingPeers = new Set();
             let hashes = [];
             for (const [hash, requestingPeers] of result) {
@@ -896,6 +981,40 @@ let SharedLog = class SharedLog extends Program {
             },
             indexer: logIndex,
         });
+        if (options?.syncronizer) {
+            this.syncronizer = new options.syncronizer({
+                numbers: this.indexableDomain.numbers,
+                entryIndex: this.entryCoordinatesIndex,
+                log: this.log,
+                rangeIndex: this._replicationRangeIndex,
+                rpc: this.rpc,
+                coordinateToHash: this.coordinateToHash,
+            });
+        }
+        else {
+            if (this._logProperties?.compatibility &&
+                this._logProperties.compatibility < 10) {
+                this.syncronizer = new SimpleSyncronizer({
+                    log: this.log,
+                    rpc: this.rpc,
+                    entryIndex: this.entryCoordinatesIndex,
+                    coordinateToHash: this.coordinateToHash,
+                });
+            }
+            else {
+                if (this.domain.resolution === "u32") {
+                    logger.warn("u32 resolution is not recommended for RatelessIBLTSynchronizer");
+                }
+                this.syncronizer = new RatelessIBLTSynchronizer({
+                    numbers: this.indexableDomain.numbers,
+                    entryIndex: this.entryCoordinatesIndex,
+                    log: this.log,
+                    rangeIndex: this._replicationRangeIndex,
+                    rpc: this.rpc,
+                    coordinateToHash: this.coordinateToHash,
+                });
+            }
+        }
         // Open for communcation
         await this.rpc.open({
             queryType: TransportMessage,
@@ -910,59 +1029,6 @@ let SharedLog = class SharedLog extends Program {
             this._onUnsubscriptionFn || this._onUnsubscription.bind(this);
         await this.node.services.pubsub.addEventListener("unsubscribe", this._onUnsubscriptionFn);
         await this.rpc.subscribe();
-        const requestSync = async () => {
-            /**
-             * This method fetches entries that we potentially want.
-             * In a case in which we become replicator of a segment,
-             * multiple remote peers might want to send us entries
-             * This method makes sure that we only request on entry from the remotes at a time
-             * so we don't get flooded with the same entry
-             */
-            const requestHashes = [];
-            const from = new Set();
-            for (const [key, value] of this.syncInFlightQueue) {
-                if (!(await this.log.has(key))) {
-                    // TODO test that this if statement actually does anymeaningfull
-                    if (value.length > 0) {
-                        requestHashes.push(key);
-                        const publicKeyHash = value.shift().hashcode();
-                        from.add(publicKeyHash);
-                        const invertedSet = this.syncInFlightQueueInverted.get(publicKeyHash);
-                        if (invertedSet) {
-                            if (invertedSet.delete(key)) {
-                                if (invertedSet.size === 0) {
-                                    this.syncInFlightQueueInverted.delete(publicKeyHash);
-                                }
-                            }
-                        }
-                    }
-                    if (value.length === 0) {
-                        this.syncInFlightQueue.delete(key); // no-one more to ask for this entry
-                    }
-                }
-                else {
-                    this.syncInFlightQueue.delete(key);
-                }
-            }
-            const nowMin10s = +new Date() - 1e4;
-            for (const [key, map] of this.syncInFlight) {
-                // cleanup "old" missing syncs
-                for (const [hash, { timestamp }] of map) {
-                    if (timestamp < nowMin10s) {
-                        map.delete(hash);
-                    }
-                }
-                if (map.size === 0) {
-                    this.syncInFlight.delete(key);
-                }
-            }
-            this.requestSync(requestHashes, from).finally(() => {
-                if (this.closed) {
-                    return;
-                }
-                this.syncMoreInterval = setTimeout(requestSync, 3e3);
-            });
-        };
         // if we had a previous session with replication info, and new replication info dictates that we unreplicate
         // we should do that. Otherwise if options is a unreplication we dont need to do anything because
         // we are already unreplicated (as we are just opening)
@@ -980,7 +1046,7 @@ let SharedLog = class SharedLog extends Program {
                 reset: true,
             });
         }
-        requestSync();
+        await this.syncronizer.open();
         this.interval = setInterval(() => {
             this.rebalanceParticipationDebounced?.();
         }, RECALCULATE_PARTICIPATION_DEBOUNCE_INTERVAL);
@@ -1005,14 +1071,14 @@ let SharedLog = class SharedLog extends Program {
         // go through all segments and for waitForAll replicators to become reachable if not prune them away
         const promises = [];
         const iterator = this.replicationIndex.iterate();
-        let checked = new Set();
+        let checkedIsAlive = new Set();
         while (!iterator.done()) {
             for (const segment of await iterator.next(1000)) {
-                if (checked.has(segment.value.hash) ||
+                if (checkedIsAlive.has(segment.value.hash) ||
                     this.node.identity.publicKey.hashcode() === segment.value.hash) {
                     continue;
                 }
-                checked.add(segment.value.hash);
+                checkedIsAlive.add(segment.value.hash);
                 promises.push(this.waitFor(segment.value.hash, {
                     timeout: this.waitForReplicatorTimeout,
                     signal: this._closeController.signal,
@@ -1090,24 +1156,28 @@ let SharedLog = class SharedLog extends Program {
             peers: this.replicationIndex,
             start: range.offset,
             widthToCoverScaled: range.length ??
-                (await minimumWidthToCover(this.replicas.min.getValue(this))),
+                (await minimumWidthToCover(this.replicas.min.getValue(this), this.indexableDomain.numbers)),
             roleAge,
             eager,
-            intervalWidth: MAX_U32,
+            numbers: this.indexableDomain.numbers,
         });
         // add all in flight
-        for (const [key, _] of this.syncInFlight) {
+        for (const [key, _] of this.syncronizer.syncInFlight) {
             set.add(key);
         }
         return [...set];
     }
     async _close() {
-        clearTimeout(this.syncMoreInterval);
-        for (const [_key, value] of this.pendingMaturity) {
-            clearTimeout(value.timeout);
+        await this.syncronizer.close();
+        for (const [_key, peerMap] of this.pendingMaturity) {
+            for (const [_key2, info] of peerMap) {
+                clearTimeout(info.timeout);
+            }
         }
         this.pendingMaturity.clear();
         this.distributeQueue?.clear();
+        this.coordinateToHash.clear();
+        this.uniqueReplicators.clear();
         this._closeController.abort();
         clearInterval(this.interval);
         this.node.services.pubsub.removeEventListener("subscribe", this._onSubscriptionFn);
@@ -1122,13 +1192,14 @@ let SharedLog = class SharedLog extends Program {
         await this.remoteBlocks.stop();
         this._pendingDeletes.clear();
         this._pendingIHave.clear();
-        this.syncInFlightQueue.clear();
-        this.syncInFlightQueueInverted.clear();
-        this.syncInFlight.clear();
         this.latestReplicationInfoMessage.clear();
         this._gidPeersHistory.clear();
+        this._requestIPruneSent.clear();
+        this._requestIPruneResponseReplicatorSet.clear();
         this.pruneDebouncedFn = undefined;
         this.rebalanceParticipationDebounced = undefined;
+        this._replicationRangeIndex.stop();
+        this._entryCoordinatesIndex.stop();
         this._replicationRangeIndex = undefined;
         this._entryCoordinatesIndex = undefined;
         this.cpuUsage?.stop?.();
@@ -1148,6 +1219,8 @@ let SharedLog = class SharedLog extends Program {
         if (!superDropped) {
             return superDropped;
         }
+        await this._entryCoordinatesIndex.drop();
+        await this._replicationRangeIndex.drop();
         await this.log.drop();
         await this._close();
         return true;
@@ -1190,6 +1263,11 @@ let SharedLog = class SharedLog extends Program {
                     const promises = [];
                     for (const [gid, entries] of groupedByGid) {
                         const fn = async () => {
+                            /// we clear sync in flight here because we want to join before that, so that entries are totally accounted for
+                            await this.syncronizer.onReceivedEntries({
+                                entries,
+                                from: context.from,
+                            });
                             const headsWithGid = await this.log.entryIndex
                                 .getHeads(gid)
                                 .all();
@@ -1200,16 +1278,40 @@ let SharedLog = class SharedLog extends Program {
                             const maxReplicasFromNewEntries = maxReplicas(this, entries.map((x) => x.entry));
                             const maxMaxReplicas = Math.max(maxReplicasFromHead, maxReplicasFromNewEntries);
                             const cursor = await this.createCoordinates(latestEntry, maxMaxReplicas);
-                            const isReplicating = await this.isReplicating();
-                            let isLeader;
+                            const isReplicating = this._isReplicating;
+                            let isLeader = false;
+                            let fromIsLeader = false;
+                            let leaders;
                             if (isReplicating) {
-                                isLeader = await this.waitForIsLeader(cursor, this.node.identity.publicKey.hashcode());
+                                leaders = await this._waitForReplicators(cursor, latestEntry, [
+                                    {
+                                        key: this.node.identity.publicKey.hashcode(),
+                                        replicator: true,
+                                    },
+                                ], {
+                                    // we do this here so that we quickly assume leader role (and also so that 'from' is also assumed to be leader)
+                                    // TODO potential side effects?
+                                    roleAge: 0,
+                                    timeout: 2e4,
+                                    onLeader: (key) => {
+                                        isLeader =
+                                            isLeader ||
+                                                this.node.identity.publicKey.hashcode() === key;
+                                        fromIsLeader =
+                                            fromIsLeader || context.from.hashcode() === key;
+                                    },
+                                });
                             }
                             else {
-                                isLeader = await this.findLeaders(cursor);
-                                isLeader = isLeader.has(this.node.identity.publicKey.hashcode())
-                                    ? isLeader
-                                    : false;
+                                leaders = await this.findLeaders(cursor, latestEntry, {
+                                    onLeader: (key) => {
+                                        fromIsLeader =
+                                            fromIsLeader || context.from.hashcode() === key;
+                                        isLeader =
+                                            isLeader ||
+                                                this.node.identity.publicKey.hashcode() === key;
+                                    },
+                                });
                             }
                             if (this.closed) {
                                 return;
@@ -1220,22 +1322,13 @@ let SharedLog = class SharedLog extends Program {
                             if (isLeader) {
                                 for (const entry of entries) {
                                     this.pruneDebouncedFn.delete(entry.entry.hash);
-                                }
-                                for (const entry of entries) {
-                                    await this.persistCoordinate({
-                                        leaders: isLeader,
-                                        coordinates: cursor,
-                                        entry: entry.entry,
-                                    });
-                                }
-                                const fromIsLeader = isLeader.get(context.from.hashcode());
-                                if (fromIsLeader) {
-                                    let peerSet = this._gidPeersHistory.get(gid);
-                                    if (!peerSet) {
-                                        peerSet = new Set();
-                                        this._gidPeersHistory.set(gid, peerSet);
+                                    this._requestIPruneSent.delete(entry.entry.hash);
+                                    this._requestIPruneResponseReplicatorSet.delete(entry.entry.hash);
+                                    if (fromIsLeader) {
+                                        this.addPeersToGidPeerHistory(gid, [
+                                            context.from.hashcode(),
+                                        ]);
                                     }
-                                    peerSet.add(context.from.hashcode());
                                 }
                                 if (maxReplicasFromNewEntries < maxReplicasFromHead) {
                                     (maybeDelete || (maybeDelete = [])).push(entries);
@@ -1262,19 +1355,14 @@ let SharedLog = class SharedLog extends Program {
                             }
                             if (toMerge.length > 0) {
                                 await this.log.join(toMerge);
-                                toDelete?.map((x) => this.pruneDebouncedFn.add({ key: x.hash, value: x }));
+                                toDelete?.map((x) => 
+                                // TODO types
+                                this.pruneDebouncedFn.add({
+                                    key: x.hash,
+                                    value: { entry: x, leaders: leaders },
+                                }));
                                 this.rebalanceParticipationDebounced?.();
                             }
-                            /// we clear sync in flight here because we want to join before that, so that entries are totally accounted for
-                            for (const entry of entries) {
-                                const set = this.syncInFlight.get(context.from.hashcode());
-                                if (set) {
-                                    set.delete(entry.entry.hash);
-                                    if (set?.size === 0) {
-                                        this.syncInFlight.delete(context.from.hashcode());
-                                    }
-                                }
-                            }
                             if (maybeDelete) {
                                 for (const entries of maybeDelete) {
                                     const headsWithGid = await this.log.entryIndex
@@ -1287,10 +1375,16 @@ let SharedLog = class SharedLog extends Program {
                                             replicas: minReplicas,
                                         });
                                         if (!isLeader) {
-                                            entries.map((x) => this.pruneDebouncedFn.add({
-                                                key: x.entry.hash,
-                                                value: x.entry,
-                                            }));
+                                            for (const x of entries) {
+                                                this.pruneDebouncedFn.add({
+                                                    key: x.entry.hash,
+                                                    // TODO types
+                                                    value: {
+                                                        entry: x.entry,
+                                                        leaders: leaders,
+                                                    },
+                                                });
+                                            }
                                         }
                                     }
                                 }
@@ -1303,17 +1397,30 @@ let SharedLog = class SharedLog extends Program {
             }
             else if (msg instanceof RequestIPrune) {
                 const hasAndIsLeader = [];
-                //	await delay(3000)
                 for (const hash of msg.hashes) {
+                    // if we expect the remote to be owner of this entry because we are to prune ourselves, then we need to remove the remote
+                    // this is due to that the remote has previously indicated to be a replicator to help us prune but now has changed their mind
+                    const outGoingPrunes = this._requestIPruneResponseReplicatorSet.get(hash);
+                    if (outGoingPrunes) {
+                        outGoingPrunes.delete(context.from.hashcode());
+                    }
                     const indexedEntry = await this.log.entryIndex.getShallow(hash);
-                    if (indexedEntry &&
-                        (await this.findLeadersPersist({
-                            entry: indexedEntry.value,
-                            minReplicas: decodeReplicas(indexedEntry.value).getValue(this),
-                        }, indexedEntry.value)).isLeader) {
-                        this._gidPeersHistory
-                            .get(indexedEntry.value.meta.gid)
-                            ?.delete(context.from.hashcode());
+                    let isLeader = false;
+                    if (indexedEntry) {
+                        this.removePeerFromGidPeerHistory(context.from.hashcode(), indexedEntry.value.meta.gid);
+                        await this._waitForReplicators(await this.createCoordinates(indexedEntry.value, decodeReplicas(indexedEntry.value).getValue(this)), indexedEntry.value, [
+                            {
+                                key: this.node.identity.publicKey.hashcode(),
+                                replicator: true,
+                            },
+                        ], {
+                            onLeader: (key) => {
+                                isLeader =
+                                    isLeader || key === this.node.identity.publicKey.hashcode();
+                            },
+                        });
+                    }
+                    if (isLeader) {
                         hasAndIsLeader.push(hash);
                         hasAndIsLeader.length > 0 &&
                             this.responseToPruneDebouncedFn.add({
@@ -1344,13 +1451,16 @@ let SharedLog = class SharedLog extends Program {
                                     clearTimeout(timeout);
                                 },
                                 callback: async (entry) => {
-                                    if ((await this.findLeadersPersist({
-                                        entry,
-                                        minReplicas: decodeReplicas(entry).getValue(this),
-                                    }, entry)).isLeader) {
-                                        for (const peer of requesting) {
-                                            this._gidPeersHistory.get(entry.meta.gid)?.delete(peer);
-                                        }
+                                    this.removePeerFromGidPeerHistory(context.from.hashcode(), entry.meta.gid);
+                                    let isLeader = false;
+                                    await this.findLeaders(await this.createCoordinates(entry, decodeReplicas(entry).getValue(this)), entry, {
+                                        onLeader: (key) => {
+                                            isLeader =
+                                                isLeader ||
+                                                    key === this.node.identity.publicKey.hashcode();
+                                        },
+                                    });
+                                    if (isLeader) {
                                         this.responseToPruneDebouncedFn.add({
                                             hashes: [entry.hash],
                                             peers: requesting,
@@ -1369,38 +1479,49 @@ let SharedLog = class SharedLog extends Program {
                     this._pendingDeletes.get(hash)?.resolve(context.from.hashcode());
                 }
             }
-            else if (msg instanceof RequestMaybeSync) {
-                const requestHashes = [];
+            else if (await this.syncronizer.onMessage(msg, context)) {
+                return; // the syncronizer has handled the message
+            } /* else if (msg instanceof RequestMaybeSync) {
+                const requestHashes: string[] = [];
+
                 for (const hash of msg.hashes) {
                     const inFlight = this.syncInFlightQueue.get(hash);
                     if (inFlight) {
-                        if (!inFlight.find((x) => x.hashcode() === context.from.hashcode())) {
+                        if (
+                            !inFlight.find((x) => x.hashcode() === context.from!.hashcode())
+                        ) {
                             inFlight.push(context.from);
-                            let inverted = this.syncInFlightQueueInverted.get(context.from.hashcode());
+                            let inverted = this.syncInFlightQueueInverted.get(
+                                context.from.hashcode(),
+                            );
                             if (!inverted) {
                                 inverted = new Set();
-                                this.syncInFlightQueueInverted.set(context.from.hashcode(), inverted);
+                                this.syncInFlightQueueInverted.set(
+                                    context.from.hashcode(),
+                                    inverted,
+                                );
                             }
                             inverted.add(hash);
                         }
-                    }
-                    else if (!(await this.log.has(hash))) {
+                    } else if (!(await this.log.has(hash))) {
                         this.syncInFlightQueue.set(hash, []);
                         requestHashes.push(hash); // request immediately (first time we have seen this hash)
                     }
                 }
                 requestHashes.length > 0 &&
                     (await this.requestSync(requestHashes, [context.from.hashcode()]));
-            }
-            else if (msg instanceof ResponseMaybeSync) {
+            } else if (msg instanceof ResponseMaybeSync) {
                 // TODO perhaps send less messages to more receivers for performance reasons?
                 // TODO wait for previous send to target before trying to send more?
-                for await (const message of createExchangeHeadsMessages(this.log, msg.hashes)) {
+                for await (const message of createExchangeHeadsMessages(
+                    this.log,
+                    msg.hashes,
+                )) {
                     await this.rpc.send(message, {
-                        mode: new SilentDelivery({ to: [context.from], redundancy: 1 }),
+                        mode: new SilentDelivery({ to: [context.from!], redundancy: 1 }),
                     });
                 }
-            }
+            }  */
             else if (msg instanceof BlocksMessage) {
                 await this.remoteBlocks.onMessage(msg.message);
             }
@@ -1453,6 +1574,9 @@ let SharedLog = class SharedLog extends Program {
                     }
                     this.latestReplicationInfoMessage.set(context.from.hashcode(), context.timestamp);
                     let reset = msg instanceof AllReplicatingSegmentsMessage;
+                    if (this.closed) {
+                        return;
+                    }
                     await this.addReplicationRange(replicationInfoMessage.segments.map((x) => x.toReplicationRangeIndexable(context.from)), context.from, { reset, checkDuplicates: true });
                     /* await this._modifyReplicators(msg.role, context.from!); */
                 })
@@ -1463,6 +1587,9 @@ let SharedLog = class SharedLog extends Program {
                     if (e instanceof NotStartedError) {
                         return;
                     }
+                    if (e instanceof IndexNotStartedError) {
+                        return;
+                    }
                     logger.error("Failed to find peer who updated replication settings: " +
                         e?.message);
                 });
@@ -1478,7 +1605,9 @@ let SharedLog = class SharedLog extends Program {
             }
         }
         catch (e) {
-            if (e instanceof AbortError) {
+            if (e instanceof AbortError ||
+                e instanceof NotStartedError ||
+                e instanceof IndexNotStartedError) {
                 return;
             }
             if (e instanceof BorshError) {
@@ -1492,6 +1621,39 @@ let SharedLog = class SharedLog extends Program {
             logger.error(e);
         }
     }
+    async calculateTotalParticipation(options) {
+        if (options?.sum) {
+            const ranges = await this.replicationIndex.iterate().all();
+            let sum = 0;
+            for (const range of ranges) {
+                sum += range.value.widthNormalized;
+            }
+            return sum;
+        }
+        return appromixateCoverage({
+            peers: this._replicationRangeIndex,
+            numbers: this.indexableDomain.numbers,
+            samples: 25,
+        });
+    }
+    /* async calculateTotalParticipation() {
+        const sum = await this.replicationIndex.sum({ key: "width" });
+        return Number(sum) / MAX_U32;
+    }
+ */
+    async countReplicationSegments() {
+        const count = await this.replicationIndex.count({
+            query: new StringMatch({
+                key: "hash",
+                value: this.node.identity.publicKey.hashcode(),
+            }),
+        });
+        return count;
+    }
+    async getAllReplicationSegments() {
+        const ranges = await this.replicationIndex.iterate().all();
+        return ranges.map((x) => x.value);
+    }
     async getMyReplicationSegments() {
         const ranges = await this.replicationIndex
             .iterate({
@@ -1503,7 +1665,7 @@ let SharedLog = class SharedLog extends Program {
             .all();
         return ranges.map((x) => x.value);
     }
-    async getMyTotalParticipation() {
+    async calculateMyTotalParticipation() {
         // sum all of my replicator rects
         return (await this.getMyReplicationSegments()).reduce((acc, { widthNormalized }) => acc + widthNormalized, 0);
     }
@@ -1535,9 +1697,9 @@ let SharedLog = class SharedLog extends Program {
     async waitForReplicator(...keys) {
         const check = async () => {
             for (const k of keys) {
-                const rects = await this.replicationIndex
-                    ?.iterate({ query: new StringMatch({ key: "hash", value: k.hashcode() }) }, { reference: true })
-                    .all();
+                const iterator = this.replicationIndex?.iterate({ query: new StringMatch({ key: "hash", value: k.hashcode() }) }, { reference: true });
+                const rects = await iterator?.next(1);
+                await iterator.close();
                 const rect = rects[0]?.value;
                 if (!rect ||
                     !isMatured(rect, +new Date(), await this.getDefaultMinRoleAge())) {
@@ -1546,6 +1708,7 @@ let SharedLog = class SharedLog extends Program {
             }
             return true;
         };
+        // TODO do event based
         return waitFor(() => check(), {
             signal: this._closeController.signal,
         }).catch((e) => {
@@ -1557,119 +1720,137 @@ let SharedLog = class SharedLog extends Program {
         });
     }
     async join(entries, options) {
-        let messageToSend = undefined;
+        let entriesToReplicate = [];
         if (options?.replicate) {
             // TODO this block should perhaps be called from a callback on the this.log.join method on all the ignored element because already joined, like "onAlreadyJoined"
             // check which entrise we already have but not are replicating, and replicate them
-            let alreadyJoined = [];
+            // we can not just do the 'join' call because it will ignore the already joined entries
             for (const element of entries) {
                 if (typeof element === "string") {
                     const entry = await this.log.get(element);
                     if (entry) {
-                        alreadyJoined.push(entry);
+                        entriesToReplicate.push(entry);
                     }
                 }
                 else if (element instanceof Entry) {
                     if (await this.log.has(element.hash)) {
-                        alreadyJoined.push(element);
+                        entriesToReplicate.push(element);
                     }
                 }
                 else {
                     const entry = await this.log.get(element.hash);
                     if (entry) {
-                        alreadyJoined.push(entry);
+                        entriesToReplicate.push(entry);
                     }
                 }
             }
-            // assume is heads
-            await this.replicate(alreadyJoined, {
-                checkDuplicates: true,
-                announce: (msg) => {
-                    messageToSend = msg;
-                },
-            });
         }
-        let joinOptions = options?.replicate
-            ? {
-                ...options,
-                onChange: async (change) => {
-                    if (change.added) {
-                        for (const entry of change.added) {
-                            if (entry.head) {
-                                await this.replicate(entry.entry, {
-                                    checkDuplicates: true,
-                                    // we override the announce step here to make sure we announce all new replication info
-                                    // in one large message instead
-                                    announce: (msg) => {
-                                        if (msg instanceof AllReplicatingSegmentsMessage) {
-                                            throw new Error("Unexpected");
-                                        }
-                                        if (messageToSend) {
-                                            // merge segments to make it into one messages
-                                            for (const segment of msg.segments) {
-                                                messageToSend.segments.push(segment);
-                                            }
-                                        }
-                                        else {
-                                            messageToSend = msg;
-                                        }
-                                    },
-                                });
-                            }
+        const onChangeForReplication = options?.replicate
+            ? async (change) => {
+                if (change.added) {
+                    for (const entry of change.added) {
+                        if (entry.head) {
+                            entriesToReplicate.push(entry.entry);
                         }
                     }
-                },
+                }
             }
-            : options;
+            : undefined;
+        const persistCoordinate = async (entry) => {
+            const minReplicas = decodeReplicas(entry).getValue(this);
+            await this.findLeaders(await this.createCoordinates(entry, minReplicas), entry, { persist: {} });
+        };
+        let entriesToPersist = [];
+        let joinOptions = {
+            ...options,
+            onChange: async (change) => {
+                await onChangeForReplication?.(change);
+                for (const entry of change.added) {
+                    if (!entry.head) {
+                        continue;
+                    }
+                    if (!options?.replicate) {
+                        // we persist coordinates for all added entries here
+                        await persistCoordinate(entry.entry);
+                    }
+                    else {
+                        // else we persist after replication range update has been done so that
+                        // the indexed info becomes up to date
+                        entriesToPersist.push(entry.entry);
+                    }
+                }
+            },
+        };
         await this.log.join(entries, joinOptions);
-        if (messageToSend) {
-            await this.rpc.send(messageToSend, {
-                priority: 1,
-            });
-        }
-    }
-    async findLeadersPersist(cursor, entry, options) {
-        const coordinates = Array.isArray(cursor)
-            ? cursor
-            : await this.createCoordinates(cursor.entry, cursor.minReplicas);
-        const minReplicas = coordinates.length;
-        const leaders = await this.findLeaders(coordinates, options);
-        const isLeader = leaders.has(this.node.identity.publicKey.hashcode());
-        if (isLeader || options?.persist) {
-            let assignToRangeBoundary = undefined;
-            if (options?.persist?.prev) {
-                assignToRangeBoundary = shouldAssigneToRangeBoundary(leaders, minReplicas);
-                const prev = options.persist.prev;
-                // dont do anthing if nothing has changed
-                if (prev.length > 0) {
-                    let allTheSame = true;
-                    for (const element of prev) {
-                        if (element.assignedToRangeBoundary !== assignToRangeBoundary) {
-                            allTheSame = false;
-                            break;
+        if (options?.replicate) {
+            let messageToSend = undefined;
+            await this.replicate(entriesToReplicate, {
+                checkDuplicates: true,
+                mergeSegments: typeof options.replicate !== "boolean" && options.replicate
+                    ? options.replicate.mergeSegments
+                    : false,
+                // we override the announce step here to make sure we announce all new replication info
+                // in one large message instead
+                announce: (msg) => {
+                    if (msg instanceof AllReplicatingSegmentsMessage) {
+                        throw new Error("Unexpected");
+                    }
+                    if (messageToSend) {
+                        // merge segments to make it into one messages
+                        for (const segment of msg.segments) {
+                            messageToSend.segments.push(segment);
                         }
                     }
-                    if (allTheSame) {
-                        return { leaders, isLeader };
+                    else {
+                        messageToSend = msg;
                     }
-                }
+                },
+            });
+            for (const entry of entriesToPersist) {
+                await persistCoordinate(entry);
+            }
+            if (messageToSend) {
+                await this.rpc.send(messageToSend, {
+                    priority: 1,
+                });
             }
-            !this.closed &&
-                (await this.persistCoordinate({
-                    leaders,
-                    coordinates,
-                    entry,
-                }, {
-                    assignToRangeBoundary,
-                }));
         }
-        return { leaders, isLeader };
     }
-    async isLeader(cursor, options) {
-        const leaders = await this.findLeaders(cursor, options);
-        return leaders.has(this.node.identity.publicKey.hashcode());
-    }
-    async waitForIsLeader(cursor, hash, options = { timeout: this.waitForReplicatorTimeout }) {
+    /*
+        private async updateLeaders(
+            cursor: NumberFromType<R>,
+            prev: EntryReplicated<R>,
+            options?: {
+                roleAge?: number;
+            },
+        ): Promise<{
+            isLeader: boolean;
+            leaders: Map<string, { intersecting: boolean }>;
+        }> {
+            // we consume a list of coordinates in this method since if we are leader of one coordinate we want to persist all of them
+            const leaders = await this._findLeaders(cursor, options);
+            const isLeader = leaders.has(this.node.identity.publicKey.hashcode());
+            const isAtRangeBoundary = shouldAssignToRangeBoundary(leaders, 1);
+    
+            // dont do anthing if nothing has changed
+            if (prev.assignedToRangeBoundary !== isAtRangeBoundary) {
+                return { isLeader, leaders };
+            }
+    
+            await this.entryCoordinatesIndex.put(
+                new this.indexableDomain.constructorEntry({
+                    assignedToRangeBoundary: isAtRangeBoundary,
+                    coordinate: cursor,
+                    meta: prev.meta,
+                    hash: prev.hash,
+                }),
+            );
+    
+            return { isLeader, leaders };
+        }
+     */
+    async _waitForReplicators(cursors, entry, waitFor, options = { timeout: this.waitForReplicatorTimeout }) {
+        const timeout = options.timeout ?? this.waitForReplicatorTimeout;
         return new Promise((resolve, reject) => {
             const removeListeners = () => {
                 this.events.removeEventListener("replication:change", roleListener);
@@ -1681,18 +1862,32 @@ let SharedLog = class SharedLog extends Program {
                 clearTimeout(timer);
                 resolve(false);
             };
-            const timer = setTimeout(() => {
+            const timer = setTimeout(async () => {
                 removeListeners();
                 resolve(false);
-            }, options.timeout);
-            const check = () => this.findLeaders(cursor).then((leaders) => {
-                const isLeader = leaders.has(hash);
-                if (isLeader) {
-                    removeListeners();
-                    clearTimeout(timer);
-                    resolve(leaders);
+            }, timeout);
+            const check = async () => {
+                let leaderKeys = new Set();
+                const leaders = await this.findLeaders(cursors, entry, {
+                    ...options,
+                    onLeader: (key) => {
+                        options?.onLeader && options.onLeader(key);
+                        leaderKeys.add(key);
+                    },
+                });
+                for (const waitForKey of waitFor) {
+                    if (waitForKey.replicator && !leaderKeys.has(waitForKey.key)) {
+                        return;
+                    }
+                    if (!waitForKey.replicator && leaderKeys.has(waitForKey.key)) {
+                        return;
+                    }
                 }
-            });
+                options?.onLeader && leaderKeys.forEach(options.onLeader);
+                removeListeners();
+                clearTimeout(timer);
+                resolve(leaders);
+            };
             const roleListener = () => {
                 check();
             };
@@ -1702,51 +1897,135 @@ let SharedLog = class SharedLog extends Program {
             check();
         });
     }
-    async findLeaders(cursor, options) {
+    /*
+        private async waitForIsLeader(
+            cursors: NumberFromType<R>[],
+            hash: string,
+            options: {
+                timeout: number;
+            } = { timeout: this.waitForReplicatorTimeout },
+        ): Promise<Map<string, { intersecting: boolean }> | false> {
+            return new Promise((resolve, reject) => {
+                const removeListeners = () => {
+                    this.events.removeEventListener("replication:change", roleListener);
+                    this.events.removeEventListener("replicator:mature", roleListener); // TODO replication:change event  ?
+                    this._closeController.signal.removeEventListener(
+                        "abort",
+                        abortListener,
+                    );
+                };
+                const abortListener = () => {
+                    removeListeners();
+                    clearTimeout(timer);
+                    resolve(false);
+                };
+    
+                const timer = setTimeout(() => {
+                    removeListeners();
+                    resolve(false);
+                }, options.timeout);
+    
+                const check = async () => {
+                    const leaders = await this.mergeLeadersMap(await Promise.all(cursors.map(x => this.findLeaders(x))));
+                    const isLeader = leaders.has(hash);
+                    if (isLeader) {
+                        removeListeners();
+                        clearTimeout(timer);
+                        resolve(leaders);
+                    }
+                }
+    
+                const roleListener = () => {
+                    check();
+                };
+    
+                this.events.addEventListener("replication:change", roleListener); // TODO replication:change event  ?
+                this.events.addEventListener("replicator:mature", roleListener); // TODO replication:change event  ?
+                this._closeController.signal.addEventListener("abort", abortListener);
+                check();
+            });
+        } */
+    /* async findLeaders(
+        cursor:
+            | NumberFromType<R>[]
+            | {
+                entry: ShallowOrFullEntry<any> | EntryReplicated<R>;
+                replicas: number;
+            },
+        options?: {
+            roleAge?: number;
+        },
+    ): Promise<Map<string, { intersecting: boolean }>> {
         if (this.closed) {
             const map = new Map(); // Assumption: if the store is closed, always assume we have responsibility over the data
             map.set(this.node.identity.publicKey.hashcode(), { intersecting: false });
             return map;
         }
+
         const coordinates = Array.isArray(cursor)
             ? cursor
             : await this.createCoordinates(cursor.entry, cursor.replicas);
-        const leaders = await this.findLeadersFromU32(coordinates, options);
+        const leaders = await this.findLeadersFromN(coordinates, options);
+
         return leaders;
-    }
-    async groupByLeaders(cursors, options) {
-        const leaders = await Promise.all(cursors.map((x) => this.findLeaders(x, options)));
-        const map = new Map();
-        leaders.forEach((leader, i) => {
-            for (const [hash] of leader) {
-                const arr = map.get(hash) ?? [];
-                arr.push(i);
-                map.set(hash, arr);
+    } */
+    /* private async groupByLeaders(
+        entries: (ShallowOrFullEntry<any> | EntryReplicated<R>)[],
+        options?: {
+            roleAge?: number;
+        },
+    ) {
+        try {
+            const leaders = await Promise.all(
+                entries.map(async (x) => {
+                    return this.findLeadersFromEntry(x, decodeReplicas(x).getValue(this), options);
+                }),
+            );
+            const map = new Map<string, number[]>();
+            leaders.forEach((leader, i) => {
+                for (const [hash] of leader) {
+                    const arr = map.get(hash) ?? [];
+                    arr.push(i);
+                    map.set(hash, arr);
+                }
+            });
+            return map;
+        } catch (error) {
+            if (error instanceof NotStartedError || error instanceof IndexNotStartedError) {
+                // ignore because we are shutting down
+                return new Map<string, number[]>();
+            } else {
+                throw error;
             }
-        });
-        return map;
-    }
+        }
+    } */
     async createCoordinates(entry, minReplicas) {
-        const cursor = await this.domain.fromEntry(entry);
-        const out = getEvenlySpacedU32(cursor, minReplicas);
+        const cursor = typeof entry === "number" || typeof entry === "bigint"
+            ? entry
+            : await this.domain.fromEntry(entry);
+        const out = this.indexableDomain.numbers.getGrid(cursor, minReplicas);
         return out;
     }
     async getCoordinates(entry) {
         const result = await this.entryCoordinatesIndex
             .iterate({ query: { hash: entry.hash } })
             .all();
-        return result.map((x) => x.value.coordinate);
-    }
-    async persistCoordinate(properties, options) {
-        let assignedToRangeBoundary = options?.assignToRangeBoundary ??
-            shouldAssigneToRangeBoundary(properties.leaders, properties.coordinates.length);
+        return result[0].value.coordinates;
+    }
+    async persistCoordinate(properties) {
+        let assignedToRangeBoundary = shouldAssignToRangeBoundary(properties.leaders, properties.replicas);
+        if (properties.prev &&
+            properties.prev.assignedToRangeBoundary === assignedToRangeBoundary) {
+            return; // no change
+        }
+        await this.entryCoordinatesIndex.put(new this.indexableDomain.constructorEntry({
+            assignedToRangeBoundary,
+            coordinates: properties.coordinates,
+            meta: properties.entry.meta,
+            hash: properties.entry.hash,
+        }));
         for (const coordinate of properties.coordinates) {
-            await this.entryCoordinatesIndex.put(new EntryReplicated({
-                assignedToRangeBoundary,
-                coordinate,
-                meta: properties.entry.meta,
-                hash: properties.entry.hash,
-            }));
+            this.coordinateToHash.add(coordinate, properties.entry.hash);
         }
         if (properties.entry.meta.next.length > 0) {
             await this.entryCoordinatesIndex.del({
@@ -1758,30 +2037,85 @@ let SharedLog = class SharedLog extends Program {
         await this.entryCoordinatesIndex.del({ query: properties });
     }
     async getDefaultMinRoleAge() {
-        if ((await this.isReplicating()) === false) {
+        if (this._isReplicating === false) {
             return 0;
         }
         const now = +new Date();
-        const replLength = await this.replicationIndex.getSize();
-        const diffToOldest = replLength > 1 ? now - this.oldestOpenTime - 1 : Number.MAX_SAFE_INTEGER;
-        return Math.min(this.timeUntilRoleMaturity, Math.max(diffToOldest, this.timeUntilRoleMaturity), Math.max(Math.round((this.timeUntilRoleMaturity * Math.log(replLength + 1)) / 3), this.timeUntilRoleMaturity)); // / 3 so that if 2 replicators and timeUntilRoleMaturity = 1e4 the result will be 1
+        const subscribers = (await this.node.services.pubsub.getSubscribers(this.rpc.topic))
+            ?.length ?? 1;
+        const diffToOldest = subscribers > 1 ? now - this.oldestOpenTime - 1 : Number.MAX_SAFE_INTEGER;
+        const result = Math.min(this.timeUntilRoleMaturity, Math.max(diffToOldest, this.timeUntilRoleMaturity), Math.max(Math.round((this.timeUntilRoleMaturity * Math.log(subscribers + 1)) / 3), this.timeUntilRoleMaturity)); // / 3 so that if 2 replicators and timeUntilRoleMaturity = 1e4 the result will be 1
+        return result;
+        /* return Math.min(1e3, this.timeUntilRoleMaturity); */
+    }
+    async findLeaders(cursors, entry, options) {
+        // we consume a list of coordinates in this method since if we are leader of one coordinate we want to persist all of them
+        let isLeader = false;
+        const set = await this._findLeaders(cursors, options);
+        for (const key of set.keys()) {
+            if (options?.onLeader) {
+                options.onLeader(key);
+                isLeader = isLeader || key === this.node.identity.publicKey.hashcode();
+            }
+        }
+        if (options?.persist !== false) {
+            if (isLeader || options?.persist) {
+                !this.closed &&
+                    (await this.persistCoordinate({
+                        leaders: set,
+                        coordinates: cursors,
+                        replicas: cursors.length,
+                        entry,
+                        prev: options?.persist?.prev,
+                    }));
+            }
+        }
+        return set;
+    }
+    async isLeader(properties, options) {
+        let cursors = await this.createCoordinates(properties.entry, properties.replicas);
+        const leaders = await this.findLeaders(cursors, properties.entry, options);
+        if (leaders.has(this.node.identity.publicKey.hashcode())) {
+            return true;
+        }
+        return false;
     }
-    async findLeadersFromU32(cursor, options) {
+    async _findLeaders(cursors, options) {
         const roleAge = options?.roleAge ?? (await this.getDefaultMinRoleAge()); // TODO -500 as is added so that i f someone else is just as new as us, then we treat them as mature as us. without -500 we might be slower syncing if two nodes starts almost at the same time
-        return getSamples(cursor, this.replicationIndex, roleAge);
+        return getSamples(cursors, this.replicationIndex, roleAge, this.indexableDomain.numbers, {
+            uniqueReplicators: this.uniqueReplicators,
+        });
+    }
+    async findLeadersFromEntry(entry, replicas, options) {
+        const coordinates = await this.createCoordinates(entry, replicas);
+        const result = await this._findLeaders(coordinates, options);
+        return result;
     }
     async isReplicator(entry, options) {
-        return this.isLeader({ entry, replicas: decodeReplicas(entry).getValue(this) }, options);
+        return this.isLeader({
+            entry,
+            replicas: maxReplicas(this, [entry]),
+        }, options);
     }
     async handleSubscriptionChange(publicKey, topics, subscribed) {
         if (!topics.includes(this.topic)) {
             return;
         }
         if (!subscribed) {
-            for (const [_a, b] of this._gidPeersHistory) {
-                b.delete(publicKey.hashcode());
+            this.removePeerFromGidPeerHistory(publicKey.hashcode());
+            for (const [k, v] of this._requestIPruneSent) {
+                v.delete(publicKey.hashcode());
+                if (v.size === 0) {
+                    this._requestIPruneSent.delete(k);
+                }
             }
-            this.clearSyncProcessPublicKey(publicKey);
+            for (const [k, v] of this._requestIPruneResponseReplicatorSet) {
+                v.delete(publicKey.hashcode());
+                if (v.size === 0) {
+                    this._requestIPruneSent.delete(k);
+                }
+            }
+            this.syncronizer.onPeerDisconnected(publicKey);
             (await this.replicationIndex.count({
                 query: { hash: publicKey.hashcode() },
             })) > 0 &&
@@ -1796,14 +2130,14 @@ let SharedLog = class SharedLog extends Program {
                     .send(new AllReplicatingSegmentsMessage({
                     segments: replicationSegments.map((x) => x.toReplicationRange()),
                 }), {
-                    mode: new SilentDelivery({ redundancy: 1, to: [publicKey] }),
+                    mode: new SeekDelivery({ redundancy: 1, to: [publicKey] }),
                 })
                     .catch((e) => logger.error(e.toString()));
                 if (this.v8Behaviour) {
                     // for backwards compatibility
                     this.rpc
                         .send(new ResponseRoleMessage({ role: await this.getRole() }), {
-                        mode: new SilentDelivery({ redundancy: 1, to: [publicKey] }),
+                        mode: new SeekDelivery({ redundancy: 1, to: [publicKey] }),
                     })
                         .catch((e) => logger.error(e.toString()));
                 }
@@ -1813,11 +2147,24 @@ let SharedLog = class SharedLog extends Program {
             await this.removeReplicator(publicKey);
         }
     }
+    getClampedReplicas(customValue) {
+        if (!customValue) {
+            return this.replicas.min;
+        }
+        const min = customValue.getValue(this);
+        const maxValue = Math.max(this.replicas.min.getValue(this), min);
+        if (this.replicas.max) {
+            return new AbsoluteReplicas(Math.min(maxValue, this.replicas.max.getValue(this)));
+        }
+        return new AbsoluteReplicas(maxValue);
+    }
     prune(entries, options) {
         if (options?.unchecked) {
             return [...entries.values()].map((x) => {
-                this._gidPeersHistory.delete(x.meta.gid);
-                return this.log.remove(x, {
+                this._gidPeersHistory.delete(x.entry.meta.gid);
+                this._requestIPruneSent.delete(x.entry.hash);
+                this._requestIPruneResponseReplicatorSet.delete(x.entry.hash);
+                return this.log.remove(x.entry, {
                     recursively: true,
                 });
             });
@@ -1829,16 +2176,22 @@ let SharedLog = class SharedLog extends Program {
         // - An entry is joined, where min replicas is lower than before (for all heads for this particular gid) and therefore we are not replicating anymore for this particular gid
         // - Peers join and leave, which means we might not be a replicator anymore
         const promises = [];
-        const filteredEntries = [];
-        const deleted = new Set();
-        for (const entry of entries.values()) {
+        let peerToEntries = new Map();
+        let cleanupTimer = [];
+        for (const { entry, leaders } of entries.values()) {
+            for (const leader of leaders.keys()) {
+                let set = peerToEntries.get(leader);
+                if (!set) {
+                    set = [];
+                    peerToEntries.set(leader, set);
+                }
+                set.push(entry.hash);
+            }
             const pendingPrev = this._pendingDeletes.get(entry.hash);
             if (pendingPrev) {
                 promises.push(pendingPrev.promise.promise);
                 continue;
             }
-            filteredEntries.push(entry);
-            const existCounter = new Set();
             const minReplicas = decodeReplicas(entry);
             const deferredPromise = pDefer();
             const clear = () => {
@@ -1851,10 +2204,45 @@ let SharedLog = class SharedLog extends Program {
             };
             const resolve = () => {
                 clear();
-                deferredPromise.resolve();
+                cleanupTimer.push(setTimeout(async () => {
+                    if (await this.isLeader({
+                        entry,
+                        replicas: minReplicas.getValue(this),
+                    })) {
+                        deferredPromise.reject(new Error("Failed to delete, is leader again"));
+                        return;
+                    }
+                    this._gidPeersHistory.delete(entry.meta.gid);
+                    this._requestIPruneSent.delete(entry.hash);
+                    this._requestIPruneResponseReplicatorSet.delete(entry.hash);
+                    return this.log
+                        .remove(entry, {
+                        recursively: true,
+                    })
+                        .then(() => {
+                        deferredPromise.resolve();
+                    })
+                        .catch((e) => {
+                        deferredPromise.reject(e);
+                    })
+                        .finally(async () => {
+                        this._gidPeersHistory.delete(entry.meta.gid);
+                        this._requestIPruneSent.delete(entry.hash);
+                        this._requestIPruneResponseReplicatorSet.delete(entry.hash);
+                        // TODO in the case we become leader again here we need to re-add the entry
+                        if (await this.isLeader({
+                            entry,
+                            replicas: minReplicas.getValue(this),
+                        })) {
+                            logger.error("Unexpected: Is leader after delete");
+                        }
+                    });
+                }, this.waitForPruneDelay));
             };
             const reject = (e) => {
                 clear();
+                this._requestIPruneSent.delete(entry.hash);
+                this._requestIPruneResponseReplicatorSet.delete(entry.hash);
                 deferredPromise.reject(e);
             };
             let cursor = undefined;
@@ -1868,106 +2256,143 @@ let SharedLog = class SharedLog extends Program {
                 },
                 reject,
                 resolve: async (publicKeyHash) => {
-                    const minReplicasValue = minReplicas.getValue(this);
-                    const minMinReplicasValue = this.replicas.max
-                        ? Math.min(minReplicasValue, this.replicas.max.getValue(this))
-                        : minReplicasValue;
-                    const leaders = await this.waitForIsLeader(cursor ??
-                        (cursor = await this.createCoordinates(entry, minMinReplicasValue)), publicKeyHash);
-                    if (leaders) {
-                        if (leaders.has(this.node.identity.publicKey.hashcode())) {
-                            reject(new Error("Failed to delete, is leader"));
-                            return;
-                        }
-                        existCounter.add(publicKeyHash);
-                        if (minMinReplicasValue <= existCounter.size) {
-                            clear();
-                            this._gidPeersHistory.delete(entry.meta.gid);
-                            this.log
-                                .remove(entry, {
-                                recursively: true,
-                            })
-                                .then(() => {
-                                deleted.add(entry.hash);
-                                return resolve();
-                            })
-                                .catch((e) => {
-                                reject(new Error("Failed to delete entry: " + e.toString()));
-                            });
-                        }
+                    const minReplicasObj = this.getClampedReplicas(minReplicas);
+                    const minReplicasValue = minReplicasObj.getValue(this);
+                    // TODO is this check necessary
+                    if (!(await this._waitForReplicators(cursor ??
+                        (cursor = await this.createCoordinates(entry, minReplicasValue)), entry, [
+                        { key: publicKeyHash, replicator: true },
+                        {
+                            key: this.node.identity.publicKey.hashcode(),
+                            replicator: false,
+                        },
+                    ], {
+                        persist: false,
+                    }))) {
+                        return;
+                    }
+                    let existCounter = this._requestIPruneResponseReplicatorSet.get(entry.hash);
+                    if (!existCounter) {
+                        existCounter = new Set();
+                        this._requestIPruneResponseReplicatorSet.set(entry.hash, existCounter);
+                    }
+                    existCounter.add(publicKeyHash);
+                    if (minReplicasValue <= existCounter.size) {
+                        resolve();
                     }
                 },
             });
             promises.push(deferredPromise.promise);
         }
-        if (filteredEntries.length === 0) {
-            return promises;
+        const emitMessages = async (entries, to) => {
+            const filteredSet = [];
+            for (const entry of entries) {
+                let set = this._requestIPruneSent.get(entry);
+                if (!set) {
+                    set = new Set();
+                    this._requestIPruneSent.set(entry, set);
+                }
+                /* if (set.has(to)) {
+                    continue;
+                } */
+                set.add(to);
+                filteredSet.push(entry);
+            }
+            if (filteredSet.length > 0) {
+                return this.rpc.send(new RequestIPrune({
+                    hashes: filteredSet,
+                }), {
+                    mode: new SilentDelivery({
+                        to: [to], // TODO group by peers?
+                        redundancy: 1,
+                    }),
+                    priority: 1,
+                });
+            }
+        };
+        for (const [k, v] of peerToEntries) {
+            emitMessages(v, k);
         }
-        const emitMessages = (entries, to) => {
-            this.rpc.send(new RequestIPrune({
-                hashes: entries,
-            }), {
-                mode: new SilentDelivery({
-                    to: [to], // TODO group by peers?
-                    redundancy: 1,
+        /* const fn = async () => {
+            this.rpc.send(
+                new RequestIPrune({
+                    hashes: filteredEntries.map(x => x.hash),
                 }),
-                priority: 1,
-            });
+                {
+                    mode: new SilentDelivery({
+                        to: [...await this.getReplicators()],
+                        redundancy: 1,
+                    }),
+                    priority: 1,
+                },
+            )
         };
-        const maxReplicasValue = maxReplicas(this, filteredEntries);
-        this.groupByLeaders(filteredEntries.map((x) => {
-            return { entry: x, replicas: maxReplicasValue }; // TODO choose right maxReplicasValue, should it really be for all entries combined?
-        })).then((map) => {
-            for (const [peer, idx] of map) {
-                emitMessages(idx.map((i) => filteredEntries[i].hash), peer);
-            }
-        });
-        const onPeersChange = async (e) => {
-            if (e.detail.publicKey.equals(this.node.identity.publicKey) === false) {
-                const peerEntries = (await this.groupByLeaders(filteredEntries
-                    .filter((x) => !deleted.has(x.hash))
-                    .map((x) => {
-                    return { entry: x, replicas: maxReplicasValue }; // TODO choose right maxReplicasValue, should it really be for all entries combined?
-                }))).get(e.detail.publicKey.hashcode());
-                if (peerEntries && peerEntries.length > 0) {
-                    emitMessages(peerEntries.map((x) => filteredEntries[x].hash), e.detail.publicKey.hashcode());
+        fn() */
+        /* const onPeersChange = async (
+            e?: CustomEvent<ReplicatorJoinEvent>,
+            reason?: string,
+        ) => {
+            if (
+                true  // e.detail.publicKey.equals(this.node.identity.publicKey) === false  // TODO proper condition
+            ) {
+                
+                const peerToEntryMap = await this.groupByLeaders(
+                    filteredEntries
+                        .filter((x) => !readyToDelete.has(x.hash))
+                        .map((x) => {
+                            return { entry: x, replicas: maxReplicasValue }; // TODO choose right maxReplicasValue, should it really be for all entries combined?
+                        }),
+                );
+                for (const receiver of peerToEntryMap.keys()) {
+                    if (receiver === this.node.identity.publicKey.hashcode()) {
+                        continue;
+                    }
+                    const peerEntries = peerToEntryMap.get(receiver);
+                    if (peerEntries && peerEntries.length > 0) {
+                        emitMessages(
+                            peerEntries.map((x) => filteredEntries[x].hash),
+                            receiver,
+                        );
+                    }
                 }
             }
-        };
+        }; */
         // check joining peers
+        /* this.events.addEventListener("replication:change", onPeersChange);
         this.events.addEventListener("replicator:mature", onPeersChange);
-        this.events.addEventListener("replicator:join", onPeersChange);
-        Promise.allSettled(promises).finally(() => {
+        this.events.addEventListener("replicator:join", onPeersChange); */
+        let cleanup = () => {
+            for (const timer of cleanupTimer) {
+                clearTimeout(timer);
+            }
+            /* this.events.removeEventListener("replication:change", onPeersChange);
             this.events.removeEventListener("replicator:mature", onPeersChange);
-            this.events.removeEventListener("replicator:join", onPeersChange);
-        });
+            this.events.removeEventListener("replicator:join", onPeersChange); */
+            this._closeController.signal.removeEventListener("abort", cleanup);
+        };
+        Promise.allSettled(promises).finally(cleanup);
+        this._closeController.signal.addEventListener("abort", cleanup);
         return promises;
     }
     /**
      * For debugging
      */
-    async getPrunable() {
+    async getPrunable(roleAge) {
         const heads = await this.log.getHeads(true).all();
         let prunable = [];
         for (const head of heads) {
-            const isLeader = await this.isLeader({
-                entry: head,
-                replicas: maxReplicas(this, [head]),
-            });
+            const isLeader = await this.isLeader({ entry: head, replicas: maxReplicas(this, [head]) }, { roleAge });
             if (!isLeader) {
                 prunable.push(head);
             }
         }
         return prunable;
     }
-    async getNonPrunable() {
+    async getNonPrunable(roleAge) {
         const heads = await this.log.getHeads(true).all();
         let nonPrunable = [];
         for (const head of heads) {
-            const isLeader = await this.isLeader({
-                entry: head,
-                replicas: maxReplicas(this, [head]),
-            });
+            const isLeader = await this.isLeader({ entry: head, replicas: maxReplicas(this, [head]) }, { roleAge });
             if (isLeader) {
                 nonPrunable.push(head);
             }
@@ -1978,7 +2403,7 @@ let SharedLog = class SharedLog extends Program {
         if (options?.clearCache) {
             this._gidPeersHistory.clear();
         }
-        this.onReplicationChange((await this.getMyReplicationSegments()).map((x) => {
+        this.onReplicationChange((await this.getAllReplicationSegments()).map((x) => {
             return { range: x, type: "added" };
         }));
     }
@@ -1993,75 +2418,70 @@ let SharedLog = class SharedLog extends Program {
         if (this.closed) {
             return;
         }
+        await this.log.trim();
         const change = mergeReplicationChanges(changeOrChanges);
         const changed = false;
         try {
-            await this.log.trim();
             const uncheckedDeliver = new Map();
-            const allEntriesToDelete = [];
-            for await (const { gid, entries: coordinates } of toRebalance(change, this.entryCoordinatesIndex)) {
+            for await (const entryReplicated of toRebalance(change, this.entryCoordinatesIndex)) {
                 if (this.closed) {
                     break;
                 }
-                const oldPeersSet = this._gidPeersHistory.get(gid);
-                if (this.closed) {
-                    return;
-                }
-                let { isLeader, leaders: currentPeers } = await this.findLeadersPersist(coordinates.map((x) => x.coordinate), coordinates[0], {
+                let oldPeersSet = this._gidPeersHistory.get(entryReplicated.gid);
+                let isLeader = false;
+                let currentPeers = await this.findLeaders(entryReplicated.coordinates, entryReplicated, {
+                    // we do this to make sure new replicators get data even though they are not mature so they can figure out if they want to replicate more or less
+                    // TODO make this smarter because if a new replicator is not mature and want to replicate too much data the syncing overhead can be bad
                     roleAge: 0,
-                    persist: {
-                        prev: coordinates,
-                    },
                 });
-                if (isLeader) {
-                    for (const entry of coordinates) {
-                        this.pruneDebouncedFn.delete(entry.hash);
-                    }
-                }
-                const currentPeersSet = new Set(currentPeers.keys());
-                this._gidPeersHistory.set(gid, currentPeersSet);
                 for (const [currentPeer] of currentPeers) {
                     if (currentPeer === this.node.identity.publicKey.hashcode()) {
+                        isLeader = true;
                         continue;
                     }
                     if (!oldPeersSet?.has(currentPeer)) {
                         let set = uncheckedDeliver.get(currentPeer);
                         if (!set) {
-                            set = new Set();
+                            set = new Map();
                             uncheckedDeliver.set(currentPeer, set);
                         }
-                        for (const entry of coordinates) {
-                            set.add(entry.hash);
+                        if (!set.has(entryReplicated.hash)) {
+                            set.set(entryReplicated.hash, entryReplicated);
                         }
+                        /* for (const entry of coordinates) {
+                            let arr = set.get(entry.hash);
+                            if (!arr) {
+                                arr = [];
+                                set.set(entry.hash, arr);
+                            }
+                            arr.push(entry);
+                        } */
                     }
                 }
+                this.addPeersToGidPeerHistory(entryReplicated.gid, currentPeers.keys(), true);
                 if (!isLeader) {
-                    if (currentPeers.size > 0) {
-                        // If we are observer, never prune locally created entries, since we dont really know who can store them
-                        // if we are replicator, we will always persist entries that we need to so filtering on createdLocally will not make a difference
-                        let entriesToDelete = coordinates;
-                        if (this.sync) {
-                            entriesToDelete = entriesToDelete.filter((entry) => this.sync(entry) === false);
-                        }
-                        allEntriesToDelete.push(...entriesToDelete);
+                    if (!this.sync || this.sync(entryReplicated) === false) {
+                        this.pruneDebouncedFn.add({
+                            key: entryReplicated.hash,
+                            value: { entry: entryReplicated, leaders: currentPeers },
+                        });
                     }
+                    this.responseToPruneDebouncedFn.delete(entryReplicated.hash); // don't allow others to prune because of expecting me to replicating this entry
                 }
                 else {
-                    for (const entry of coordinates) {
-                        await this._pendingDeletes
-                            .get(entry.hash)
-                            ?.reject(new Error("Failed to delete, is leader again. Closed: " + this.closed));
-                    }
+                    this.pruneDebouncedFn.delete(entryReplicated.hash);
+                    await this._pendingDeletes
+                        .get(entryReplicated.hash)
+                        ?.reject(new Error("Failed to delete, is leader again"));
+                    this._requestIPruneSent.delete(entryReplicated.hash);
                 }
             }
             for (const [target, entries] of uncheckedDeliver) {
-                this.rpc.send(new RequestMaybeSync({ hashes: [...entries] }), {
-                    mode: new SilentDelivery({ to: [target], redundancy: 1 }),
+                this.syncronizer.onMaybeMissingEntries({
+                    entries,
+                    targets: [target],
                 });
             }
-            if (allEntriesToDelete.length > 0) {
-                allEntriesToDelete.map((x) => this.pruneDebouncedFn.add({ key: x.hash, value: x }));
-            }
             return changed;
         }
         catch (error) {
@@ -2069,58 +2489,16 @@ let SharedLog = class SharedLog extends Program {
             throw error;
         }
     }
-    async requestSync(hashes, to) {
-        const now = +new Date();
-        for (const node of to) {
-            let map = this.syncInFlight.get(node);
-            if (!map) {
-                map = new Map();
-                this.syncInFlight.set(node, map);
-            }
-            for (const hash of hashes) {
-                map.set(hash, { timestamp: now });
-            }
-        }
-        await this.rpc.send(new ResponseMaybeSync({
-            hashes: hashes,
-        }), {
-            mode: new SilentDelivery({ to, redundancy: 1 }),
-        });
-    }
     async _onUnsubscription(evt) {
-        logger.debug(`Peer disconnected '${evt.detail.from.hashcode()}' from '${JSON.stringify(evt.detail.unsubscriptions.map((x) => x))} '`);
+        logger.trace(`Peer disconnected '${evt.detail.from.hashcode()}' from '${JSON.stringify(evt.detail.unsubscriptions.map((x) => x))} '`);
         this.latestReplicationInfoMessage.delete(evt.detail.from.hashcode());
         return this.handleSubscriptionChange(evt.detail.from, evt.detail.unsubscriptions, false);
     }
     async _onSubscription(evt) {
-        logger.debug(`New peer '${evt.detail.from.hashcode()}' connected to '${JSON.stringify(evt.detail.subscriptions.map((x) => x))}'`);
+        logger.trace(`New peer '${evt.detail.from.hashcode()}' connected to '${JSON.stringify(evt.detail.subscriptions.map((x) => x))}'`);
         this.remoteBlocks.onReachable(evt.detail.from);
         return this.handleSubscriptionChange(evt.detail.from, evt.detail.subscriptions, true);
     }
-    async addToHistory(usedMemory, factor) {
-        (this.history || (this.history = [])).push({ usedMemory, factor });
-        // Keep only the last N entries in the history array (you can adjust N based on your needs)
-        const maxHistoryLength = 10;
-        if (this.history.length > maxHistoryLength) {
-            this.history.shift();
-        }
-    }
-    async calculateTrend() {
-        // Calculate the average change in factor per unit change in memory usage
-        const factorChanges = this.history.map((entry, index) => {
-            if (index > 0) {
-                const memoryChange = entry.usedMemory - this.history[index - 1].usedMemory;
-                if (memoryChange !== 0) {
-                    const factorChange = entry.factor - this.history[index - 1].factor;
-                    return factorChange / memoryChange;
-                }
-            }
-            return 0;
-        });
-        // Return the average factor change per unit memory change
-        return (factorChanges.reduce((sum, change) => sum + change, 0) /
-            factorChanges.length);
-    }
     async rebalanceParticipation() {
         // update more participation rate to converge to the average expected rate or bounded by
         // resources such as memory and or cpu
@@ -2152,9 +2530,9 @@ let SharedLog = class SharedLog extends Program {
                     dynamicRange.widthNormalized;
                 if (relativeDifference > 0.0001) {
                     // TODO can not reuse old range, since it will (potentially) affect the index because of sideeffects
-                    dynamicRange = new ReplicationRangeIndexable({
-                        offset: hashToU32(this.node.identity.publicKey.bytes),
-                        length: scaleToU32(newFactor),
+                    dynamicRange = new this.indexableDomain.constructorRange({
+                        offset: dynamicRange.start1,
+                        length: this.indexableDomain.numbers.denormalize(newFactor),
                         publicKeyHash: dynamicRange.hash,
                         id: dynamicRange.id,
                         mode: dynamicRange.mode,
@@ -2183,6 +2561,17 @@ let SharedLog = class SharedLog extends Program {
         const resp = await fn();
         return resp;
     }
+    getDynamicRangeOffset() {
+        const options = this._logProperties
+            ?.replicate;
+        if (options?.offset != null) {
+            const normalized = options.normalized ?? true;
+            return (normalized
+                ? this.indexableDomain.numbers.denormalize(Number(options.offset))
+                : options.offset);
+        }
+        return this.indexableDomain.numbers.bytesToNumber(this.node.identity.publicKey.bytes);
+    }
     async getDynamicRange() {
         let dynamicRangeId = getIdForDynamicRange(this.node.identity.publicKey);
         let range = (await this.replicationIndex
@@ -2196,10 +2585,9 @@ let SharedLog = class SharedLog extends Program {
         })
             .all())?.[0]?.value;
         if (!range) {
-            range = new ReplicationRangeIndexable({
-                normalized: true,
-                offset: Math.random(),
-                length: 0,
+            range = new this.indexableDomain.constructorRange({
+                offset: this.getDynamicRangeOffset(),
+                length: this.indexableDomain.numbers.zero,
                 publicKeyHash: this.node.identity.publicKey.hashcode(),
                 mode: ReplicationIntent.NonStrict,
                 timestamp: BigInt(+new Date()),
@@ -2213,50 +2601,16 @@ let SharedLog = class SharedLog extends Program {
         }
         return range;
     }
-    clearSyncProcess(hash) {
-        const inflight = this.syncInFlightQueue.get(hash);
-        if (inflight) {
-            for (const key of inflight) {
-                const map = this.syncInFlightQueueInverted.get(key.hashcode());
-                if (map) {
-                    map.delete(hash);
-                    if (map.size === 0) {
-                        this.syncInFlightQueueInverted.delete(key.hashcode());
-                    }
-                }
-            }
-            this.syncInFlightQueue.delete(hash);
-        }
-    }
-    clearSyncProcessPublicKey(publicKey) {
-        this.syncInFlight.delete(publicKey.hashcode());
-        const map = this.syncInFlightQueueInverted.get(publicKey.hashcode());
-        if (map) {
-            for (const hash of map) {
-                const arr = this.syncInFlightQueue.get(hash);
-                if (arr) {
-                    const filtered = arr.filter((x) => !x.equals(publicKey));
-                    if (filtered.length > 0) {
-                        this.syncInFlightQueue.set(hash, filtered);
-                    }
-                    else {
-                        this.syncInFlightQueue.delete(hash);
-                    }
-                }
-            }
-            this.syncInFlightQueueInverted.delete(publicKey.hashcode());
-        }
-    }
     async onEntryAdded(entry) {
         const ih = this._pendingIHave.get(entry.hash);
         if (ih) {
             ih.clear();
             ih.callback(entry);
         }
-        this.clearSyncProcess(entry.hash);
+        this.syncronizer.onEntryAdded(entry);
     }
     onEntryRemoved(hash) {
-        this.clearSyncProcess(hash);
+        this.syncronizer.onEntryRemoved(hash);
     }
 };
 __decorate([