@peerbit/shared-log 9.1.2 → 9.2.0

package/dist/src/index.js

@@ -8,36 +8,39 @@ var __metadata = (this && this.__metadata) || function (k, v) {
     if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
 };
 import { BorshError, field, variant } from "@dao-xyz/borsh";
-import { CustomEvent } from "@libp2p/interface";
 import { AnyBlockStore, RemoteBlocks } from "@peerbit/blocks";
 import { Cache } from "@peerbit/cache";
 import { AccessError, PublicSignKey, sha256Base64Sync, sha256Sync, } from "@peerbit/crypto";
-import { And, ByteMatchQuery, CountRequest, DeleteRequest, Or, SearchRequest, Sort, StringMatch, SumRequest, } from "@peerbit/indexer-interface";
+import { And, ByteMatchQuery, Or, Sort, StringMatch, } from "@peerbit/indexer-interface";
 import { Entry, Log, ShallowEntry, } from "@peerbit/log";
 import { logger as loggerFn } from "@peerbit/logger";
 import { ClosedError, Program } from "@peerbit/program";
 import { SubscriptionEvent, UnsubcriptionEvent, } from "@peerbit/pubsub-interface";
 import { RPC } from "@peerbit/rpc";
 import { AcknowledgeDelivery, DeliveryMode, NotStartedError, SilentDelivery, } from "@peerbit/stream-interface";
-import { AbortError, delay, waitFor } from "@peerbit/time";
-import debounce from "p-debounce";
+import { AbortError, 
+/* delay, */
+waitFor, } from "@peerbit/time";
 import pDefer, {} from "p-defer";
 import PQueue from "p-queue";
 import { concat } from "uint8arrays";
 import { BlocksMessage } from "./blocks.js";
 import { CPUUsageIntervalLag } from "./cpu.js";
+import { debounceAcculmulator, debounceFixedInterval, debouncedAccumulatorMap, } from "./debounce.js";
 import { EntryWithRefs, ExchangeHeadsMessage, RequestIPrune, RequestMaybeSync, ResponseIPrune, ResponseMaybeSync, createExchangeHeadsMessages, } from "./exchange-heads.js";
 import { TransportMessage } from "./message.js";
 import { PIDReplicationController } from "./pid.js";
-import { getCoverSet, getSamples, hasCoveringRange, isMatured, minimumWidthToCover, } from "./ranges.js";
+import { EntryReplicated, ReplicationIntent, ReplicationRange, ReplicationRangeIndexable, getCoverSet, getEvenlySpacedU32, getSamples, hasCoveringRange, isMatured, minimumWidthToCover, shouldAssigneToRangeBoundary, toRebalance, } from "./ranges.js";
 import { createReplicationDomainHash, hashToU32, } from "./replication-domain-hash.js";
 import { createReplicationDomainTime, } from "./replication-domain-time.js";
-import {} from "./replication-domain.js";
-import { AbsoluteReplicas, AddedReplicationSegmentMessage, AllReplicatingSegmentsMessage, ReplicationError, ReplicationIntent, ReplicationRange, ReplicationRangeIndexable, RequestReplicationInfoMessage, ResponseRoleMessage, StoppedReplicating, decodeReplicas, encodeReplicas, maxReplicas, } from "./replication.js";
+import { debounceAggregationChanges, mergeReplicationChanges, } from "./replication-domain.js";
+import { AbsoluteReplicas, AddedReplicationSegmentMessage, AllReplicatingSegmentsMessage, ReplicationError, RequestReplicationInfoMessage, ResponseRoleMessage, StoppedReplicating, decodeReplicas, encodeReplicas, maxReplicas, } from "./replication.js";
 import { MAX_U32, Observer, Replicator, scaleToU32 } from "./role.js";
+import { groupByGid } from "./utils.js";
 export { createReplicationDomainHash, createReplicationDomainTime, };
 export { CPUUsageIntervalLag };
 export * from "./replication.js";
+export { EntryReplicated };
 export const logger = loggerFn({ module: "shared-log" });
 const getLatestEntry = (entries) => {
     let latest = undefined;
@@ -50,23 +53,6 @@ const getLatestEntry = (entries) => {
     }
     return latest;
 };
-const groupByGid = async (entries) => {
-    const groupByGid = new Map();
-    for (const head of entries) {
-        const gid = await (head instanceof Entry
-            ? (await head.getMeta()).gid
-            : head instanceof ShallowEntry
-                ? head.meta.gid
-                : (await head.entry.getMeta()).gid);
-        let value = groupByGid.get(gid);
-        if (!value) {
-            value = [];
-            groupByGid.set(gid, value);
-        }
-        value.push(head);
-    }
-    return groupByGid;
-};
 const isAdaptiveReplicatorOption = (options) => {
     if (typeof options === "number") {
         return false;
@@ -86,11 +72,35 @@ const isUnreplicationOptions = (options) => options === false ||
     options === 0 ||
     (options?.offset === undefined &&
         options?.factor === 0);
+const isReplicationOptionsDependentOnPreviousState = (options) => {
+    if (options === true) {
+        return true;
+    }
+    if (options == null) {
+        // when not providing options, we assume previous behaviour
+        return true;
+    }
+    // if empty object but with no keys
+    if (typeof options === "object" && Object.keys(options).length === 0) {
+        return true;
+    }
+    return false;
+};
 export const DEFAULT_MIN_REPLICAS = 2;
 export const WAIT_FOR_REPLICATOR_TIMEOUT = 9000;
 export const WAIT_FOR_ROLE_MATURITY = 5000;
-const REBALANCE_DEBOUNCE_INTERVAL = 100;
+const PRUNE_DEBOUNCE_INTERVAL = 500;
+// DONT SET THIS ANY LOWER, because it will make the pid controller unstable as the system responses are not fast enough to updates from the pid controller
+const RECALCULATE_PARTICIPATION_DEBOUNCE_INTERVAL = 1000;
 const DEFAULT_DISTRIBUTION_DEBOUNCE_TIME = 500;
+const getIdForDynamicRange = (publicKey) => {
+    return sha256Sync(concat([publicKey.bytes, new TextEncoder().encode("dynamic")]));
+};
+const checkMinReplicasLimit = (minReplicas) => {
+    if (minReplicas > 100) {
+        throw new Error("Higher replication degree than 100 is not recommended for performance reasons");
+    }
+};
 let SharedLog = class SharedLog extends Program {
     log;
     rpc;
@@ -98,6 +108,7 @@ let SharedLog = class SharedLog extends Program {
     _isReplicating;
     _isAdaptiveReplicating;
     _replicationRangeIndex;
+    _entryCoordinatesIndex;
     /* private _totalParticipation!: number; */
     _gidPeersHistory;
     _onSubscriptionFn;
@@ -109,6 +120,7 @@ let SharedLog = class SharedLog extends Program {
     _respondToIHaveTimeout;
     _pendingDeletes;
     _pendingIHave;
+    pendingMaturity; // map of peerId to timeout
     latestReplicationInfoMessage;
     remoteBlocks;
     openTime;
@@ -116,8 +128,11 @@ let SharedLog = class SharedLog extends Program {
     sync;
     // A fn that we can call many times that recalculates the participation role
     rebalanceParticipationDebounced;
+    // A fn for debouncing the calls for pruning
+    pruneDebouncedFn;
+    responseToPruneDebouncedFn;
+    replicationChangeDebounceFn;
     // regular distribution checks
-    distributeInterval;
     distributeQueue;
     // Syncing and dedeplucation work
     syncMoreInterval;
@@ -134,7 +149,7 @@ let SharedLog = class SharedLog extends Program {
     replicationController;
     history;
     domain;
-    pq;
+    interval;
     constructor(properties) {
         super();
         this.log = new Log(properties);
@@ -180,21 +195,29 @@ let SharedLog = class SharedLog extends Program {
         return this._totalParticipation;
     } */
     async calculateTotalParticipation() {
-        const sum = await this.replicationIndex.sum(new SumRequest({ key: "width" }));
+        const sum = await this.replicationIndex.sum({ key: "width" });
         return Number(sum) / MAX_U32;
     }
     async countReplicationSegments() {
-        const count = await this.replicationIndex.count(new CountRequest({
+        const count = await this.replicationIndex.count({
             query: new StringMatch({
                 key: "hash",
                 value: this.node.identity.publicKey.hashcode(),
             }),
-        }));
+        });
         return count;
     }
-    setupRebalanceDebounceFunction() {
+    setupRebalanceDebounceFunction(interval = RECALCULATE_PARTICIPATION_DEBOUNCE_INTERVAL) {
         this.rebalanceParticipationDebounced = undefined;
-        this.rebalanceParticipationDebounced = debounce(() => this.rebalanceParticipation(), 
+        // make the rebalancing to respect warmup time
+        let intervalTime = interval * 2;
+        let timeout = setTimeout(() => {
+            intervalTime = interval;
+        }, this.timeUntilRoleMaturity);
+        this._closeController.signal.addEventListener("abort", () => {
+            clearTimeout(timeout);
+        });
+        this.rebalanceParticipationDebounced = debounceFixedInterval(() => this.rebalanceParticipation(), 
         /* Math.max(
             REBALANCE_DEBOUNCE_INTERVAL,
             Math.log(
@@ -202,7 +225,7 @@ let SharedLog = class SharedLog extends Program {
                 REBALANCE_DEBOUNCE_INTERVAL
             )
         ) */
-        REBALANCE_DEBOUNCE_INTERVAL);
+        () => intervalTime);
     }
     async _replicate(options, { reset, checkDuplicates, announce, } = {}) {
         let offsetWasProvided = false;
@@ -261,6 +284,7 @@ let SharedLog = class SharedLog extends Program {
                     let factor = rangeArg.factor;
                     let width = normalized ? 1 : scaleToU32(1);
                     ranges.push(new ReplicationRangeIndexable({
+                        id: rangeArg.id,
                         normalized,
                         offset: offset,
                         length: typeof factor === "number"
@@ -291,6 +315,7 @@ let SharedLog = class SharedLog extends Program {
                 checkDuplicates,
                 announce,
             });
+            return ranges;
         }
     }
     setupDebouncedRebalancing(options) {
@@ -312,7 +337,7 @@ let SharedLog = class SharedLog extends Program {
                 ? options?.limits?.cpu?.monitor || new CPUUsageIntervalLag()
                 : new CPUUsageIntervalLag();
         this.cpuUsage?.start?.();
-        this.setupRebalanceDebounceFunction();
+        this.setupRebalanceDebounceFunction(options?.limits?.interval);
     }
     async replicate(rangeOrEntry, options) {
         let range = undefined;
@@ -345,10 +370,7 @@ let SharedLog = class SharedLog extends Program {
         else {
             range = rangeOrEntry ?? true;
         }
-        const newRanges = await this._replicate(range, options);
-        // assume new role
-        await this.distribute();
-        return newRanges;
+        return this._replicate(range, options);
     }
     async unreplicate(rangeOrEntry) {
         let range;
@@ -372,13 +394,13 @@ let SharedLog = class SharedLog extends Program {
             // TODO support this by never deleting the range with the segment id that is generated by the dynamic replication method
             throw new Error("Unsupported when adaptive replicating");
         }
-        const indexed = await this.replicationIndex.query(new SearchRequest({
+        const indexed = this.replicationIndex.iterate({
             query: {
                 width: 1,
                 start1: range.offset,
             },
-        }));
-        const segmentIds = indexed.results.map((x) => x.id.key);
+        });
+        const segmentIds = (await indexed.all()).map((x) => x.id.key);
         await this.removeReplicationRange(segmentIds, this.node.identity.publicKey);
         await this.rpc.send(new StoppedReplicating({ segmentIds }), {
             priority: 1,
@@ -386,7 +408,12 @@ let SharedLog = class SharedLog extends Program {
     }
     async removeReplicator(key) {
         const fn = async () => {
-            await this.replicationIndex.del(new DeleteRequest({ query: { hash: key.hashcode() } }));
+            const deleted = await this.replicationIndex
+                .iterate({
+                query: { hash: key.hashcode() },
+            })
+                .all();
+            await this.replicationIndex.del({ query: { hash: key.hashcode() } });
             await this.updateOldestTimestampFromIndex();
             if (this.node.identity.publicKey.equals(key)) {
                 // announce that we are no longer replicating
@@ -395,32 +422,54 @@ let SharedLog = class SharedLog extends Program {
             this.events.dispatchEvent(new CustomEvent("replication:change", {
                 detail: { publicKey: key },
             }));
+            deleted.forEach((x) => {
+                return this.replicationChangeDebounceFn.add({
+                    range: x.value,
+                    type: "removed",
+                });
+            });
+            const pendingMaturity = this.pendingMaturity.get(key.hashcode());
+            if (pendingMaturity) {
+                clearTimeout(pendingMaturity.timeout);
+                this.pendingMaturity.delete(key.hashcode());
+            }
             if (!key.equals(this.node.identity.publicKey)) {
                 this.rebalanceParticipationDebounced?.();
             }
         };
-        return this.pq.add(fn);
+        return fn();
     }
     async updateOldestTimestampFromIndex() {
-        const oldestTimestampFromDB = (await this.replicationIndex.query(new SearchRequest({
-            fetch: 1,
+        const iterator = await this.replicationIndex.iterate({
             sort: [new Sort({ key: "timestamp", direction: "asc" })],
-        }), { reference: true })).results[0]?.value.timestamp;
+        }, { reference: true });
+        const oldestTimestampFromDB = (await iterator.next(1))[0]?.value.timestamp;
+        await iterator.close();
         this.oldestOpenTime =
             oldestTimestampFromDB != null
                 ? Number(oldestTimestampFromDB)
                 : +new Date();
     }
-    async removeReplicationRange(id, from) {
+    async removeReplicationRange(ids, from) {
         const fn = async () => {
-            let idMatcher = new Or(id.map((x) => new ByteMatchQuery({ key: "id", value: x })));
+            let idMatcher = new Or(ids.map((x) => new ByteMatchQuery({ key: "id", value: x })));
             // make sure we are not removing something that is owned by the replicator
             let identityMatcher = new StringMatch({
                 key: "hash",
                 value: from.hashcode(),
             });
             let query = new And([idMatcher, identityMatcher]);
-            await this.replicationIndex.del(new DeleteRequest({ query }));
+            const pendingMaturity = this.pendingMaturity.get(from.hashcode());
+            if (pendingMaturity) {
+                for (const id of ids) {
+                    pendingMaturity.ranges.delete(id.toString());
+                }
+                if (pendingMaturity.ranges.size === 0) {
+                    clearTimeout(pendingMaturity.timeout);
+                    this.pendingMaturity.delete(from.hashcode());
+                }
+            }
+            await this.replicationIndex.del({ query });
             await this.updateOldestTimestampFromIndex();
             this.events.dispatchEvent(new CustomEvent("replication:change", {
                 detail: { publicKey: from },
@@ -429,35 +478,137 @@ let SharedLog = class SharedLog extends Program {
                 this.rebalanceParticipationDebounced?.();
             }
         };
-        return this.pq.add(fn);
+        return fn();
     }
     async addReplicationRange(ranges, from, { reset, checkDuplicates, } = {}) {
         const fn = async () => {
             if (this._isTrustedReplicator &&
                 !(await this._isTrustedReplicator(from))) {
-                return false;
+                return undefined;
             }
-            let prevCount = await this.replicationIndex.count(new CountRequest({
-                query: new StringMatch({ key: "hash", value: from.hashcode() }),
-            }));
-            const isNewReplicator = prevCount === 0;
+            let isNewReplicator = false;
+            let diffs;
+            let deleted = undefined;
             if (reset) {
-                await this.replicationIndex.del(new DeleteRequest({ query: { hash: from.hashcode() } }));
-            }
-            else if (checkDuplicates) {
-                let deduplicated = [];
-                // TODO also deduplicate/de-overlap among the ranges that ought to be inserted?
-                for (const range of ranges) {
-                    if (!(await hasCoveringRange(this.replicationIndex, range))) {
-                        deduplicated.push(range);
+                deleted = (await this.replicationIndex
+                    .iterate({
+                    query: { hash: from.hashcode() },
+                })
+                    .all()).map((x) => x.value);
+                await this.replicationIndex.del({ query: { hash: from.hashcode() } });
+                diffs = [
+                    ...deleted.map((x) => {
+                        return { range: x, type: "removed" };
+                    }),
+                    ...ranges.map((x) => {
+                        return { range: x, type: "added" };
+                    }),
+                ];
+                let prevCount = await this.replicationIndex.count({
+                    query: new StringMatch({ key: "hash", value: from.hashcode() }),
+                });
+                isNewReplicator = prevCount === 0;
+            }
+            else {
+                let existing = await this.replicationIndex
+                    .iterate({
+                    query: ranges.map((x) => new ByteMatchQuery({ key: "id", value: x.id })),
+                }, { reference: true })
+                    .all();
+                if (existing.length === 0) {
+                    let prevCount = await this.replicationIndex.count({
+                        query: new StringMatch({ key: "hash", value: from.hashcode() }),
+                    });
+                    isNewReplicator = prevCount === 0;
+                }
+                else {
+                    isNewReplicator = false;
+                }
+                if (checkDuplicates) {
+                    let deduplicated = [];
+                    // TODO also deduplicate/de-overlap among the ranges that ought to be inserted?
+                    for (const range of ranges) {
+                        if (!(await hasCoveringRange(this.replicationIndex, range))) {
+                            deduplicated.push(range);
+                        }
                     }
+                    ranges = deduplicated;
                 }
-                ranges = deduplicated;
-            }
-            for (const range of ranges) {
-                await this.replicationIndex.put(range);
-                if (!reset) {
-                    this.oldestOpenTime = Math.min(Number(range.timestamp), this.oldestOpenTime);
+                let existingMap = new Map();
+                for (const result of existing) {
+                    existingMap.set(result.value.idString, result.value);
+                }
+                let changes = ranges
+                    .map((x) => {
+                    const prev = existingMap.get(x.idString);
+                    if (prev) {
+                        if (prev.equalRange(x)) {
+                            return undefined;
+                        }
+                        return { range: x, prev, type: "updated" };
+                    }
+                    else {
+                        return { range: x, type: "added" };
+                    }
+                })
+                    .filter((x) => x != null);
+                diffs = changes;
+            }
+            let now = +new Date();
+            let minRoleAge = await this.getDefaultMinRoleAge();
+            let isAllMature = true;
+            for (const diff of diffs) {
+                if (diff.type === "added" || diff.type === "updated") {
+                    await this.replicationIndex.put(diff.range);
+                    if (!reset) {
+                        this.oldestOpenTime = Math.min(Number(diff.range.timestamp), this.oldestOpenTime);
+                    }
+                    const isMature = isMatured(diff.range, now, minRoleAge);
+                    if (!isMature /* && diff.range.hash !== this.node.identity.publicKey.hashcode() */) {
+                        // second condition is to avoid the case where we are adding a range that we own
+                        isAllMature = false;
+                        let prevPendingMaturity = this.pendingMaturity.get(diff.range.hash);
+                        let map;
+                        let waitForMaturityTime = Math.max(minRoleAge - (now - Number(diff.range.timestamp)), 0);
+                        if (prevPendingMaturity) {
+                            map = prevPendingMaturity.ranges;
+                            if (prevPendingMaturity.timestamp < diff.range.timestamp) {
+                                // something has changed so we need to reset the timeout
+                                clearTimeout(prevPendingMaturity.timeout);
+                                prevPendingMaturity.timestamp = diff.range.timestamp;
+                                prevPendingMaturity.timeout = setTimeout(() => {
+                                    this.events.dispatchEvent(new CustomEvent("replicator:mature", {
+                                        detail: { publicKey: from },
+                                    }));
+                                    for (const value of map.values()) {
+                                        this.replicationChangeDebounceFn.add(value); // we need to call this here because the outcom of findLeaders will be different when some ranges become mature, i.e. some of data we own might be prunable!
+                                    }
+                                }, waitForMaturityTime);
+                            }
+                        }
+                        else {
+                            map = new Map();
+                            this.pendingMaturity.set(diff.range.hash, {
+                                timestamp: diff.range.timestamp,
+                                ranges: map,
+                                timeout: setTimeout(() => {
+                                    this.events.dispatchEvent(new CustomEvent("replicator:mature", {
+                                        detail: { publicKey: from },
+                                    }));
+                                    for (const value of map.values()) {
+                                        this.replicationChangeDebounceFn.add(value); // we need to call this here because the outcom of findLeaders will be different when some ranges become mature, i.e. some of data we own might be prunable!
+                                    }
+                                }, waitForMaturityTime),
+                            });
+                        }
+                        map.set(diff.range.idString, diff);
+                    }
+                }
+                else {
+                    const prev = this.pendingMaturity.get(diff.range.hash);
+                    if (prev) {
+                        prev.ranges.delete(diff.range.idString);
+                    }
                 }
             }
             if (reset) {
@@ -470,23 +621,33 @@ let SharedLog = class SharedLog extends Program {
                 this.events.dispatchEvent(new CustomEvent("replicator:join", {
                     detail: { publicKey: from },
                 }));
+                if (isAllMature) {
+                    this.events.dispatchEvent(new CustomEvent("replicator:mature", {
+                        detail: { publicKey: from },
+                    }));
+                }
             }
+            if (ranges.length === 0 && deleted?.length === 0) {
+                throw new Error("Unexpected");
+            }
+            diffs.length > 0 &&
+                diffs.map((x) => this.replicationChangeDebounceFn.add(x));
             if (!from.equals(this.node.identity.publicKey)) {
                 this.rebalanceParticipationDebounced?.();
             }
-            return true;
+            return diffs;
         };
         // we sequialize this because we are going to queries to check wether to add or not
         // if two processes do the same this both process might add a range while only one in practice should
-        return this.pq.add(fn);
+        return fn();
     }
     async startAnnounceReplicating(range, options = {}) {
-        const added = await this.addReplicationRange(range, this.node.identity.publicKey, options);
-        if (!added) {
+        const change = await this.addReplicationRange(range, this.node.identity.publicKey, options);
+        if (!change) {
             logger.warn("Not allowed to replicate by canReplicate");
         }
         let message;
-        if (added) {
+        if (change) {
             if (options.reset) {
                 message = new AllReplicatingSegmentsMessage({
                     segments: range.map((x) => x.toReplicationRange()),
@@ -509,11 +670,14 @@ let SharedLog = class SharedLog extends Program {
     }
     async append(data, options) {
         const appendOptions = { ...options };
-        const minReplicasData = encodeReplicas(options?.replicas
+        const minReplicas = options?.replicas
             ? typeof options.replicas === "number"
                 ? new AbsoluteReplicas(options.replicas)
                 : options.replicas
-            : this.replicas.min);
+            : this.replicas.min;
+        const minReplicasValue = minReplicas.getValue(this);
+        const minReplicasData = encodeReplicas(minReplicas);
+        checkMinReplicasLimit(minReplicasValue);
         if (!appendOptions.meta) {
             appendOptions.meta = {
                 data: minReplicasData,
@@ -541,40 +705,55 @@ let SharedLog = class SharedLog extends Program {
         if (options?.replicate) {
             await this.replicate(result.entry, { checkDuplicates: true });
         }
-        for (const message of await createExchangeHeadsMessages(this.log, [result.entry], this._gidParentCache)) {
-            if (options?.target === "replicators" || !options?.target) {
-                const minReplicas = decodeReplicas(result.entry).getValue(this);
-                let leaders = await this.findLeaders(result.entry, minReplicas);
-                const isLeader = leaders.includes(this.node.identity.publicKey.hashcode());
-                if (message.heads[0].gidRefrences.length > 0) {
-                    const newAndOldLeaders = new Set(leaders);
-                    for (const ref of message.heads[0].gidRefrences) {
-                        const entryFromGid = this.log.entryIndex.getHeads(ref, false);
-                        for (const entry of await entryFromGid.all()) {
-                            for (const hash of await this.findLeaders(entry, minReplicas)) {
-                                newAndOldLeaders.add(hash);
+        let { leaders, isLeader } = await this.findLeadersPersist({
+            entry: result.entry,
+            minReplicas: minReplicas.getValue(this),
+        }, result.entry, { persist: {} });
+        // --------------
+        if (options?.target !== "none") {
+            for (const message of await createExchangeHeadsMessages(this.log, [result.entry], this._gidParentCache)) {
+                if (options?.target === "replicators" || !options?.target) {
+                    if (message.heads[0].gidRefrences.length > 0) {
+                        const newAndOldLeaders = new Map(leaders);
+                        for (const ref of message.heads[0].gidRefrences) {
+                            const entryFromGid = this.log.entryIndex.getHeads(ref, false);
+                            for (const entry of await entryFromGid.all()) {
+                                let coordinate = await this.getCoordinates(entry);
+                                if (coordinate == null) {
+                                    coordinate = await this.createCoordinates(entry, minReplicasValue);
+                                    // TODO are we every to come here?
+                                }
+                                for (const [hash, features] of await this.findLeaders(coordinate)) {
+                                    newAndOldLeaders.set(hash, features);
+                                }
                             }
                         }
+                        leaders = newAndOldLeaders;
                     }
-                    leaders = newAndOldLeaders;
-                }
-                let set = this._gidPeersHistory.get(result.entry.meta.gid);
-                if (!set) {
-                    set = new Set(leaders);
-                    this._gidPeersHistory.set(result.entry.meta.gid, set);
-                }
-                else {
-                    for (const receiver of leaders) {
-                        set.add(receiver);
+                    let set = this._gidPeersHistory.get(result.entry.meta.gid);
+                    if (!set) {
+                        set = new Set(leaders.keys());
+                        this._gidPeersHistory.set(result.entry.meta.gid, set);
+                    }
+                    else {
+                        for (const [receiver, _features] of leaders) {
+                            set.add(receiver);
+                        }
                     }
+                    mode = isLeader
+                        ? new SilentDelivery({ redundancy: 1, to: leaders.keys() })
+                        : new AcknowledgeDelivery({ redundancy: 1, to: leaders.keys() });
                 }
-                mode = isLeader
-                    ? new SilentDelivery({ redundancy: 1, to: leaders })
-                    : new AcknowledgeDelivery({ redundancy: 1, to: leaders });
+                // TODO add options for waiting ?
+                this.rpc.send(message, {
+                    mode,
+                });
             }
-            // TODO add options for waiting ?
-            this.rpc.send(message, {
-                mode,
+        }
+        if (!isLeader) {
+            this.pruneDebouncedFn.add({
+                key: result.entry.hash,
+                value: result.entry,
             });
         }
         this.rebalanceParticipationDebounced?.();
@@ -594,7 +773,7 @@ let SharedLog = class SharedLog extends Program {
                 : undefined,
         };
         this.domain = options?.domain ?? createReplicationDomainHash();
-        this._respondToIHaveTimeout = options?.respondToIHaveTimeout ?? 10 * 1000; // TODO make into arg
+        this._respondToIHaveTimeout = options?.respondToIHaveTimeout ?? 2e4;
         this._pendingDeletes = new Map();
         this._pendingIHave = new Map();
         this.latestReplicationInfoMessage = new Map();
@@ -614,7 +793,7 @@ let SharedLog = class SharedLog extends Program {
         this._isTrustedReplicator = options?.canReplicate;
         this.sync = options?.sync;
         this._logProperties = options;
-        this.pq = new PQueue({ concurrency: 1000 });
+        this.pendingMaturity = new Map();
         const id = sha256Base64Sync(this.log.id);
         const storage = await this.node.storage.sublevel(id);
         const localBlocks = await new AnyBlockStore(await storage.sublevel("blocks"));
@@ -634,16 +813,73 @@ let SharedLog = class SharedLog extends Program {
         this._replicationRangeIndex = await replicationIndex.init({
             schema: ReplicationRangeIndexable,
         });
+        this._entryCoordinatesIndex = await replicationIndex.init({
+            schema: EntryReplicated,
+        });
         const logIndex = await logScope.scope("log");
         await this.node.indexer.start(); // TODO why do we need to start the indexer here?
-        const hasIndexedReplicationInfo = (await this.replicationIndex.getSize()) > 0;
+        const hasIndexedReplicationInfo = (await this.replicationIndex.count({
+            query: [
+                new StringMatch({
+                    key: "hash",
+                    value: this.node.identity.publicKey.hashcode(),
+                }),
+            ],
+        })) > 0;
         /* this._totalParticipation = await this.calculateTotalParticipation(); */
         this._gidPeersHistory = new Map();
+        this.replicationChangeDebounceFn = debounceAggregationChanges((change) => this.onReplicationChange(change).then(() => this.rebalanceParticipationDebounced?.()), this.distributionDebounceTime);
+        this.pruneDebouncedFn = debouncedAccumulatorMap((map) => {
+            this.prune(map);
+        }, PRUNE_DEBOUNCE_INTERVAL);
+        this.responseToPruneDebouncedFn = debounceAcculmulator((result) => {
+            let allRequestingPeers = new Set();
+            let hashes = [];
+            for (const [hash, requestingPeers] of result) {
+                for (const peer of requestingPeers) {
+                    allRequestingPeers.add(peer);
+                }
+                hashes.push(hash);
+            }
+            hashes.length > 0 &&
+                this.rpc.send(new ResponseIPrune({ hashes }), {
+                    mode: new SilentDelivery({
+                        to: allRequestingPeers,
+                        redundancy: 1,
+                    }),
+                    priority: 1,
+                });
+        }, () => {
+            let accumulator = new Map();
+            return {
+                add: (props) => {
+                    for (const hash of props.hashes) {
+                        let prev = accumulator.get(hash);
+                        if (!prev) {
+                            prev = new Set();
+                            accumulator.set(hash, prev);
+                        }
+                        for (const peer of props.peers) {
+                            prev.add(peer);
+                        }
+                    }
+                },
+                delete: (hash) => {
+                    accumulator.delete(hash);
+                },
+                finalize: () => {
+                    return undefined;
+                },
+                size: () => accumulator.size,
+                clear: () => accumulator.clear(),
+                value: accumulator,
+            };
+        }, PRUNE_DEBOUNCE_INTERVAL);
         await this.log.open(this.remoteBlocks, this.node.identity, {
             keychain: this.node.services.keychain,
             ...this._logProperties,
-            onChange: (change) => {
-                this.onChange(change);
+            onChange: async (change) => {
+                await this.onChange(change);
                 return this._logProperties?.onChange?.(change);
             },
             canAppend: async (entry) => {
@@ -661,7 +897,7 @@ let SharedLog = class SharedLog extends Program {
         await this.rpc.open({
             queryType: TransportMessage,
             responseType: TransportMessage,
-            responseHandler: this._onMessage.bind(this),
+            responseHandler: (query, context) => this._onMessage(query, context),
             topic: this.topic,
         });
         this._onSubscriptionFn =
@@ -671,13 +907,6 @@ let SharedLog = class SharedLog extends Program {
             this._onUnsubscriptionFn || this._onUnsubscription.bind(this);
         await this.node.services.pubsub.addEventListener("unsubscribe", this._onUnsubscriptionFn);
         await this.rpc.subscribe();
-        // await this.log.load();
-        // TODO (do better)
-        // we do this distribution interval to eliminate the sideeffects arriving from updating roles and joining entries continously.
-        // an alternative to this would be to call distribute/maybe prune after every join if our role has changed
-        this.distributeInterval = setInterval(() => {
-            this.distribute();
-        }, 7.5 * 1000);
         const requestSync = async () => {
             /**
              * This method fetches entries that we potentially want.
@@ -693,7 +922,16 @@ let SharedLog = class SharedLog extends Program {
                     // TODO test that this if statement actually does anymeaningfull
                     if (value.length > 0) {
                         requestHashes.push(key);
-                        from.add(value.shift().hashcode());
+                        const publicKeyHash = value.shift().hashcode();
+                        from.add(publicKeyHash);
+                        const invertedSet = this.syncInFlightQueueInverted.get(publicKeyHash);
+                        if (invertedSet) {
+                            if (invertedSet.delete(key)) {
+                                if (invertedSet.size === 0) {
+                                    this.syncInFlightQueueInverted.delete(publicKeyHash);
+                                }
+                            }
+                        }
                     }
                     if (value.length === 0) {
                         this.syncInFlightQueue.delete(key); // no-one more to ask for this entry
@@ -719,17 +957,30 @@ let SharedLog = class SharedLog extends Program {
                 if (this.closed) {
                     return;
                 }
-                this.syncMoreInterval = setTimeout(requestSync, 1e4);
+                this.syncMoreInterval = setTimeout(requestSync, 3e3);
             });
         };
         // if we had a previous session with replication info, and new replication info dictates that we unreplicate
         // we should do that. Otherwise if options is a unreplication we dont need to do anything because
         // we are already unreplicated (as we are just opening)
-        if (hasIndexedReplicationInfo ||
-            isUnreplicationOptions(options?.replicate) === false) {
+        let isUnreplicationOptionsDefined = isUnreplicationOptions(options?.replicate);
+        if (hasIndexedReplicationInfo && isUnreplicationOptionsDefined) {
             await this.replicate(options?.replicate, { checkDuplicates: true });
         }
+        else if (isReplicationOptionsDependentOnPreviousState(options?.replicate) &&
+            hasIndexedReplicationInfo) {
+            // dont do anthing since we are alread replicating stuff
+        }
+        else {
+            await this.replicate(options?.replicate, {
+                checkDuplicates: true,
+                reset: true,
+            });
+        }
         requestSync();
+        this.interval = setInterval(() => {
+            this.rebalanceParticipationDebounced?.();
+        }, RECALCULATE_PARTICIPATION_DEBOUNCE_INTERVAL);
     }
     async afterOpen() {
         await super.afterOpen();
@@ -759,6 +1010,7 @@ let SharedLog = class SharedLog extends Program {
             this.onEntryAdded(added.entry);
         }
         for (const removed of change.removed) {
+            await this.deleteCoordinates({ hash: removed.hash });
             this.onEntryRemoved(removed.hash);
         }
     }
@@ -772,6 +1024,7 @@ let SharedLog = class SharedLog extends Program {
             if (Number.isFinite(replicas) === false) {
                 return false;
             }
+            checkMinReplicasLimit(replicas);
             // Don't verify entries that we have created (TODO should we? perf impact?)
             if (!entry.createdLocally && !(await entry.verifySignatures())) {
                 return false;
@@ -807,9 +1060,13 @@ let SharedLog = class SharedLog extends Program {
     }
     async _close() {
         clearTimeout(this.syncMoreInterval);
-        clearInterval(this.distributeInterval);
+        for (const [_key, value] of this.pendingMaturity) {
+            clearTimeout(value.timeout);
+        }
+        this.pendingMaturity.clear();
         this.distributeQueue?.clear();
         this._closeController.abort();
+        clearInterval(this.interval);
         this.node.services.pubsub.removeEventListener("subscribe", this._onSubscriptionFn);
         this.node.services.pubsub.removeEventListener("unsubscribe", this._onUnsubscriptionFn);
         for (const [_k, v] of this._pendingDeletes) {
@@ -828,10 +1085,12 @@ let SharedLog = class SharedLog extends Program {
         this.syncInFlight.clear();
         this.latestReplicationInfoMessage.clear();
         this._gidPeersHistory.clear();
+        this.pruneDebouncedFn = undefined;
+        this.rebalanceParticipationDebounced = undefined;
         this._replicationRangeIndex = undefined;
+        this._entryCoordinatesIndex = undefined;
         this.cpuUsage?.stop?.();
         /* this._totalParticipation = 0; */
-        this.pq.clear();
     }
     async close(from) {
         const superClosed = await super.close(from);
@@ -885,9 +1144,6 @@ let SharedLog = class SharedLog extends Program {
                     if (filteredHeads.length === 0) {
                         return;
                     }
-                    const toMerge = [];
-                    let toDelete = undefined;
-                    let maybeDelete = undefined;
                     const groupedByGid = await groupByGid(filteredHeads);
                     const promises = [];
                     for (const [gid, entries] of groupedByGid) {
@@ -900,19 +1156,38 @@ let SharedLog = class SharedLog extends Program {
                                 ? maxReplicas(this, [...headsWithGid.values()])
                                 : this.replicas.min.getValue(this);
                             const maxReplicasFromNewEntries = maxReplicas(this, entries.map((x) => x.entry));
+                            const maxMaxReplicas = Math.max(maxReplicasFromHead, maxReplicasFromNewEntries);
+                            const cursor = await this.createCoordinates(latestEntry, maxMaxReplicas);
                             const isReplicating = await this.isReplicating();
                             let isLeader;
                             if (isReplicating) {
-                                isLeader = await this.waitForIsLeader(latestEntry, Math.max(maxReplicasFromHead, maxReplicasFromNewEntries));
+                                isLeader = await this.waitForIsLeader(cursor, this.node.identity.publicKey.hashcode());
                             }
                             else {
-                                isLeader = await this.findLeaders(latestEntry, Math.max(maxReplicasFromHead, maxReplicasFromNewEntries));
-                                isLeader = isLeader.includes(this.node.identity.publicKey.hashcode())
+                                isLeader = await this.findLeaders(cursor);
+                                isLeader = isLeader.has(this.node.identity.publicKey.hashcode())
                                     ? isLeader
                                     : false;
                             }
+                            if (this.closed) {
+                                return;
+                            }
+                            let maybeDelete;
+                            let toMerge = [];
+                            let toDelete;
                             if (isLeader) {
-                                if (isLeader.find((x) => x === context.from.hashcode())) {
+                                for (const entry of entries) {
+                                    this.pruneDebouncedFn.delete(entry.entry.hash);
+                                }
+                                for (const entry of entries) {
+                                    await this.persistCoordinate({
+                                        leaders: isLeader,
+                                        coordinates: cursor,
+                                        entry: entry.entry,
+                                    });
+                                }
+                                const fromIsLeader = isLeader.get(context.from.hashcode());
+                                if (fromIsLeader) {
                                     let peerSet = this._gidPeersHistory.get(gid);
                                     if (!peerSet) {
                                         peerSet = new Set();
@@ -940,95 +1215,112 @@ let SharedLog = class SharedLog extends Program {
                                 }
                                 logger.debug(`${this.node.identity.publicKey.hashcode()}: Dropping heads with gid: ${entry.entry.meta.gid}. Because not leader`);
                             }
-                        };
-                        promises.push(fn());
-                    }
-                    await Promise.all(promises);
-                    if (this.closed) {
-                        return;
-                    }
-                    if (toMerge.length > 0) {
-                        await this.log.join(toMerge);
-                        toDelete &&
-                            Promise.all(this.prune(toDelete)).catch((e) => {
-                                logger.info(e.toString());
-                            });
-                        this.rebalanceParticipationDebounced?.();
-                    }
-                    /// we clear sync in flight here because we want to join before that, so that entries are totally accounted for
-                    for (const head of heads) {
-                        const set = this.syncInFlight.get(context.from.hashcode());
-                        if (set) {
-                            set.delete(head.entry.hash);
-                            if (set?.size === 0) {
-                                this.syncInFlight.delete(context.from.hashcode());
+                            if (this.closed) {
+                                return;
                             }
-                        }
-                    }
-                    if (maybeDelete) {
-                        for (const entries of maybeDelete) {
-                            const headsWithGid = await this.log.entryIndex
-                                .getHeads(entries[0].entry.meta.gid)
-                                .all();
-                            if (headsWithGid && headsWithGid.length > 0) {
-                                const minReplicas = maxReplicas(this, headsWithGid.values());
-                                const isLeader = await this.isLeader(entries[0].entry, minReplicas);
-                                if (!isLeader) {
-                                    Promise.all(this.prune(entries.map((x) => x.entry))).catch((e) => {
-                                        logger.info(e.toString());
-                                    });
+                            if (toMerge.length > 0) {
+                                await this.log.join(toMerge);
+                                toDelete?.map((x) => this.pruneDebouncedFn.add({ key: x.hash, value: x }));
+                                this.rebalanceParticipationDebounced?.();
+                            }
+                            /// we clear sync in flight here because we want to join before that, so that entries are totally accounted for
+                            for (const entry of entries) {
+                                const set = this.syncInFlight.get(context.from.hashcode());
+                                if (set) {
+                                    set.delete(entry.entry.hash);
+                                    if (set?.size === 0) {
+                                        this.syncInFlight.delete(context.from.hashcode());
+                                    }
                                 }
                             }
-                        }
+                            if (maybeDelete) {
+                                for (const entries of maybeDelete) {
+                                    const headsWithGid = await this.log.entryIndex
+                                        .getHeads(entries[0].entry.meta.gid)
+                                        .all();
+                                    if (headsWithGid && headsWithGid.length > 0) {
+                                        const minReplicas = maxReplicas(this, headsWithGid.values());
+                                        const isLeader = await this.isLeader({
+                                            entry: entries[0].entry,
+                                            replicas: minReplicas,
+                                        });
+                                        if (!isLeader) {
+                                            entries.map((x) => this.pruneDebouncedFn.add({
+                                                key: x.entry.hash,
+                                                value: x.entry,
+                                            }));
+                                        }
+                                    }
+                                }
+                            }
+                        };
+                        promises.push(fn()); // we do this concurrently since waitForIsLeader might be a blocking operation for some entries
                     }
+                    await Promise.all(promises);
                 }
             }
             else if (msg instanceof RequestIPrune) {
                 const hasAndIsLeader = [];
+                //	await delay(3000)
                 for (const hash of msg.hashes) {
                     const indexedEntry = await this.log.entryIndex.getShallow(hash);
                     if (indexedEntry &&
-                        (await this.isLeader(indexedEntry.value, decodeReplicas(indexedEntry.value).getValue(this)))) {
+                        (await this.findLeadersPersist({
+                            entry: indexedEntry.value,
+                            minReplicas: decodeReplicas(indexedEntry.value).getValue(this),
+                        }, indexedEntry.value)).isLeader) {
                         this._gidPeersHistory
                             .get(indexedEntry.value.meta.gid)
                             ?.delete(context.from.hashcode());
                         hasAndIsLeader.push(hash);
+                        hasAndIsLeader.length > 0 &&
+                            this.responseToPruneDebouncedFn.add({
+                                hashes: hasAndIsLeader,
+                                peers: [context.from.hashcode()],
+                            });
                     }
                     else {
                         const prevPendingIHave = this._pendingIHave.get(hash);
-                        const pendingIHave = {
-                            clear: () => {
-                                clearTimeout(timeout);
-                                prevPendingIHave?.clear();
-                            },
-                            callback: async (entry) => {
-                                if (await this.isLeader(entry, decodeReplicas(entry).getValue(this))) {
-                                    this._gidPeersHistory
-                                        .get(entry.meta.gid)
-                                        ?.delete(context.from.hashcode());
-                                    this.rpc.send(new ResponseIPrune({ hashes: [entry.hash] }), {
-                                        mode: new SilentDelivery({
-                                            to: [context.from],
-                                            redundancy: 1,
-                                        }),
-                                    });
-                                }
-                                prevPendingIHave && prevPendingIHave.callback(entry);
-                                this._pendingIHave.delete(entry.hash);
-                            },
-                        };
-                        const timeout = setTimeout(() => {
-                            const pendingIHaveRef = this._pendingIHave.get(hash);
-                            if (pendingIHave === pendingIHaveRef) {
+                        if (prevPendingIHave) {
+                            prevPendingIHave.requesting.add(context.from.hashcode());
+                            prevPendingIHave.resetTimeout();
+                        }
+                        else {
+                            const requesting = new Set([context.from.hashcode()]);
+                            let timeout = setTimeout(() => {
                                 this._pendingIHave.delete(hash);
-                            }
-                        }, this._respondToIHaveTimeout);
-                        this._pendingIHave.set(hash, pendingIHave);
+                            }, this._respondToIHaveTimeout);
+                            const pendingIHave = {
+                                requesting,
+                                resetTimeout: () => {
+                                    clearTimeout(timeout);
+                                    timeout = setTimeout(() => {
+                                        this._pendingIHave.delete(hash);
+                                    }, this._respondToIHaveTimeout);
+                                },
+                                clear: () => {
+                                    clearTimeout(timeout);
+                                },
+                                callback: async (entry) => {
+                                    if ((await this.findLeadersPersist({
+                                        entry,
+                                        minReplicas: decodeReplicas(entry).getValue(this),
+                                    }, entry)).isLeader) {
+                                        for (const peer of requesting) {
+                                            this._gidPeersHistory.get(entry.meta.gid)?.delete(peer);
+                                        }
+                                        this.responseToPruneDebouncedFn.add({
+                                            hashes: [entry.hash],
+                                            peers: requesting,
+                                        });
+                                        this._pendingIHave.delete(hash);
+                                    }
+                                },
+                            };
+                            this._pendingIHave.set(hash, pendingIHave);
+                        }
                     }
                 }
-                await this.rpc.send(new ResponseIPrune({ hashes: hasAndIsLeader }), {
-                    mode: new SilentDelivery({ to: [context.from], redundancy: 1 }),
-                });
             }
             else if (msg instanceof ResponseIPrune) {
                 for (const hash of msg.hashes) {
@@ -1040,20 +1332,23 @@ let SharedLog = class SharedLog extends Program {
                 for (const hash of msg.hashes) {
                     const inFlight = this.syncInFlightQueue.get(hash);
                     if (inFlight) {
-                        inFlight.push(context.from);
-                        let inverted = this.syncInFlightQueueInverted.get(context.from.hashcode());
-                        if (!inverted) {
-                            inverted = new Set();
-                            this.syncInFlightQueueInverted.set(context.from.hashcode(), inverted);
+                        if (!inFlight.find((x) => x.hashcode() === context.from.hashcode())) {
+                            inFlight.push(context.from);
+                            let inverted = this.syncInFlightQueueInverted.get(context.from.hashcode());
+                            if (!inverted) {
+                                inverted = new Set();
+                                this.syncInFlightQueueInverted.set(context.from.hashcode(), inverted);
+                            }
+                            inverted.add(hash);
                         }
-                        inverted.add(hash);
                     }
                     else if (!(await this.log.has(hash))) {
                         this.syncInFlightQueue.set(hash, []);
                         requestHashes.push(hash); // request immediately (first time we have seen this hash)
                     }
                 }
-                await this.requestSync(requestHashes, [context.from.hashcode()]);
+                requestHashes.length > 0 &&
+                    (await this.requestSync(requestHashes, [context.from.hashcode()]));
             }
             else if (msg instanceof ResponseMaybeSync) {
                 // TODO better choice of step size
@@ -1120,8 +1415,7 @@ let SharedLog = class SharedLog extends Program {
                     }
                     this.latestReplicationInfoMessage.set(context.from.hashcode(), context.timestamp);
                     let reset = msg instanceof AllReplicatingSegmentsMessage;
-                    const added = await this.addReplicationRange(replicationInfoMessage.segments.map((x) => x.toReplicationRangeIndexable(context.from)), context.from, { reset, checkDuplicates: true });
-                    added && (await this.distribute());
+                    await this.addReplicationRange(replicationInfoMessage.segments.map((x) => x.toReplicationRangeIndexable(context.from)), context.from, { reset, checkDuplicates: true });
                     /* await this._modifyReplicators(msg.role, context.from!); */
                 })
                     .catch((e) => {
@@ -1161,16 +1455,15 @@ let SharedLog = class SharedLog extends Program {
         }
     }
     async getMyReplicationSegments() {
-        const ranges = await this.replicationIndex.query(new SearchRequest({
-            query: [
-                new StringMatch({
-                    key: "hash",
-                    value: this.node.identity.publicKey.hashcode(),
-                }),
-            ],
-            fetch: 0xffffffff,
-        }));
-        return ranges.results.map((x) => x.value);
+        const ranges = await this.replicationIndex
+            .iterate({
+            query: new StringMatch({
+                key: "hash",
+                value: this.node.identity.publicKey.hashcode(),
+            }),
+        })
+            .all();
+        return ranges.map((x) => x.value);
     }
     async getMyTotalParticipation() {
         // sum all of my replicator rects
@@ -1182,13 +1475,21 @@ let SharedLog = class SharedLog extends Program {
         }
         return this._replicationRangeIndex;
     }
+    get entryCoordinatesIndex() {
+        if (!this._entryCoordinatesIndex) {
+            throw new ClosedError();
+        }
+        return this._entryCoordinatesIndex;
+    }
     /**
      * TODO improve efficiency
      */
     async getReplicators() {
         let set = new Set();
-        const results = await this.replicationIndex.query(new SearchRequest({ fetch: 0xfffffff }), { reference: true, shape: { hash: true } });
-        results.results.forEach((result) => {
+        const results = await this.replicationIndex
+            .iterate({}, { reference: true, shape: { hash: true } })
+            .all();
+        results.forEach((result) => {
             set.add(result.value.hash);
         });
         return set;
@@ -1196,10 +1497,10 @@ let SharedLog = class SharedLog extends Program {
     async waitForReplicator(...keys) {
         const check = async () => {
             for (const k of keys) {
-                const rects = await this.replicationIndex?.query(new SearchRequest({
-                    query: [new StringMatch({ key: "hash", value: k.hashcode() })],
-                }), { reference: true });
-                const rect = await rects.results[0]?.value;
+                const rects = await this.replicationIndex
+                    ?.iterate({ query: new StringMatch({ key: "hash", value: k.hashcode() }) }, { reference: true })
+                    .all();
+                const rect = rects[0]?.value;
                 if (!rect ||
                     !isMatured(rect, +new Date(), await this.getDefaultMinRoleAge())) {
                     return false;
@@ -1292,15 +1593,52 @@ let SharedLog = class SharedLog extends Program {
             });
         }
     }
-    async isLeader(entry, numberOfLeaders, options) {
-        const isLeader = (await this.findLeaders(entry, numberOfLeaders, options)).find((l) => l === this.node.identity.publicKey.hashcode());
-        return !!isLeader;
+    async findLeadersPersist(cursor, entry, options) {
+        const coordinates = Array.isArray(cursor)
+            ? cursor
+            : await this.createCoordinates(cursor.entry, cursor.minReplicas);
+        const leaders = await this.findLeaders(coordinates, options);
+        const isLeader = leaders.has(this.node.identity.publicKey.hashcode());
+        if (isLeader || options?.persist) {
+            let assignToRangeBoundary = undefined;
+            if (options?.persist?.prev) {
+                assignToRangeBoundary = shouldAssigneToRangeBoundary(leaders);
+                const prev = options.persist.prev;
+                // dont do anthing if nothing has changed
+                if (prev.length > 0) {
+                    let allTheSame = true;
+                    for (const element of prev) {
+                        if (element.assignedToRangeBoundary !== assignToRangeBoundary) {
+                            allTheSame = false;
+                            break;
+                        }
+                    }
+                    if (allTheSame) {
+                        return { leaders, isLeader };
+                    }
+                }
+            }
+            !this.closed &&
+                (await this.persistCoordinate({
+                    leaders,
+                    coordinates,
+                    entry,
+                }, {
+                    assignToRangeBoundary: assignToRangeBoundary,
+                }));
+        }
+        return { leaders, isLeader };
+    }
+    async isLeader(cursor, options) {
+        const leaders = await this.findLeaders(cursor, options);
+        return leaders.has(this.node.identity.publicKey.hashcode());
     }
-    async waitForIsLeader(entry, numberOfLeaders, timeout = this.waitForReplicatorTimeout) {
+    async waitForIsLeader(cursor, hash, options = { timeout: this.waitForReplicatorTimeout }) {
         return new Promise((resolve, reject) => {
             const removeListeners = () => {
                 this.events.removeEventListener("replication:change", roleListener);
-                this._closeController.signal.addEventListener("abort", abortListener);
+                this.events.removeEventListener("replicator:mature", roleListener); // TODO replication:change event  ?
+                this._closeController.signal.removeEventListener("abort", abortListener);
             };
             const abortListener = () => {
                 removeListeners();
@@ -1310,9 +1648,9 @@ let SharedLog = class SharedLog extends Program {
             const timer = setTimeout(() => {
                 removeListeners();
                 resolve(false);
-            }, timeout);
-            const check = () => this.findLeaders(entry, numberOfLeaders).then((leaders) => {
-                const isLeader = leaders.find((l) => l === this.node.identity.publicKey.hashcode());
+            }, options.timeout);
+            const check = () => this.findLeaders(cursor).then((leaders) => {
+                const isLeader = leaders.has(hash);
                 if (isLeader) {
                     removeListeners();
                     clearTimeout(timer);
@@ -1323,16 +1661,65 @@ let SharedLog = class SharedLog extends Program {
                 check();
             };
             this.events.addEventListener("replication:change", roleListener); // TODO replication:change event  ?
+            this.events.addEventListener("replicator:mature", roleListener); // TODO replication:change event  ?
             this._closeController.signal.addEventListener("abort", abortListener);
             check();
         });
     }
-    async findLeaders(entry, numberOfLeaders, options) {
+    async findLeaders(cursor, options) {
         if (this.closed) {
-            return [this.node.identity.publicKey.hashcode()]; // Assumption: if the store is closed, always assume we have responsibility over the data
+            const map = new Map(); // Assumption: if the store is closed, always assume we have responsibility over the data
+            map.set(this.node.identity.publicKey.hashcode(), { intersecting: false });
+            return map;
         }
+        const coordinates = Array.isArray(cursor)
+            ? cursor
+            : await this.createCoordinates(cursor.entry, cursor.replicas);
+        const leaders = await this.findLeadersFromU32(coordinates, options);
+        return leaders;
+    }
+    async groupByLeaders(cursors, options) {
+        const leaders = await Promise.all(cursors.map((x) => this.findLeaders(x, options)));
+        const map = new Map();
+        leaders.forEach((leader, i) => {
+            for (const [hash] of leader) {
+                const arr = map.get(hash) ?? [];
+                arr.push(i);
+                map.set(hash, arr);
+            }
+        });
+        return map;
+    }
+    async createCoordinates(entry, minReplicas) {
         const cursor = await this.domain.fromEntry(entry);
-        return this.findLeadersFromU32(cursor, numberOfLeaders, options);
+        const out = getEvenlySpacedU32(cursor, minReplicas);
+        return out;
+    }
+    async getCoordinates(entry) {
+        const result = await this.entryCoordinatesIndex
+            .iterate({ query: { hash: entry.hash } })
+            .all();
+        return result.map((x) => x.value.coordinate);
+    }
+    async persistCoordinate(properties, options) {
+        let assignedToRangeBoundary = options?.assignToRangeBoundary ??
+            shouldAssigneToRangeBoundary(properties.leaders);
+        for (const coordinate of properties.coordinates) {
+            await this.entryCoordinatesIndex.put(new EntryReplicated({
+                assignedToRangeBoundary,
+                coordinate,
+                meta: properties.entry.meta,
+                hash: properties.entry.hash,
+            }));
+        }
+        if (properties.entry.meta.next.length > 0) {
+            await this.entryCoordinatesIndex.del({
+                query: new Or(properties.entry.meta.next.map((x) => new StringMatch({ key: "hash", value: x }))),
+            });
+        }
+    }
+    async deleteCoordinates(properties) {
+        await this.entryCoordinatesIndex.del({ query: properties });
     }
     async getDefaultMinRoleAge() {
         if ((await this.isReplicating()) === false) {
@@ -1343,12 +1730,12 @@ let SharedLog = class SharedLog extends Program {
         const diffToOldest = replLength > 1 ? now - this.oldestOpenTime - 1 : Number.MAX_SAFE_INTEGER;
         return Math.min(this.timeUntilRoleMaturity, Math.max(diffToOldest, this.timeUntilRoleMaturity), Math.max(Math.round((this.timeUntilRoleMaturity * Math.log(replLength + 1)) / 3), this.timeUntilRoleMaturity)); // / 3 so that if 2 replicators and timeUntilRoleMaturity = 1e4 the result will be 1
     }
-    async findLeadersFromU32(cursor, numberOfLeaders, options) {
+    async findLeadersFromU32(cursor, options) {
         const roleAge = options?.roleAge ?? (await this.getDefaultMinRoleAge()); // TODO -500 as is added so that i f someone else is just as new as us, then we treat them as mature as us. without -500 we might be slower syncing if two nodes starts almost at the same time
-        return getSamples(cursor, this.replicationIndex, numberOfLeaders, roleAge);
+        return getSamples(cursor, this.replicationIndex, roleAge);
     }
     async isReplicator(entry, options) {
-        return this.isLeader(entry, decodeReplicas(entry).getValue(this), options);
+        return this.isLeader({ entry, replicas: decodeReplicas(entry).getValue(this) }, options);
     }
     async handleSubscriptionChange(publicKey, topics, subscribed) {
         for (const topic of topics) {
@@ -1360,20 +1747,7 @@ let SharedLog = class SharedLog extends Program {
             for (const [_a, b] of this._gidPeersHistory) {
                 b.delete(publicKey.hashcode());
             }
-            this.syncInFlight.delete(publicKey.hashcode());
-            const waitingHashes = this.syncInFlightQueueInverted.get(publicKey.hashcode());
-            if (waitingHashes) {
-                for (const hash of waitingHashes) {
-                    let arr = this.syncInFlightQueue.get(hash);
-                    if (arr) {
-                        arr = arr.filter((x) => !x.equals(publicKey));
-                    }
-                    if (this.syncInFlightQueue.size === 0) {
-                        this.syncInFlightQueue.delete(hash);
-                    }
-                }
-            }
-            this.syncInFlightQueueInverted.delete(publicKey.hashcode());
+            this.clearSyncProcessPublicKey(publicKey);
         }
         if (subscribed) {
             const replicationSegments = await this.getMyReplicationSegments();
@@ -1401,7 +1775,7 @@ let SharedLog = class SharedLog extends Program {
     }
     prune(entries, options) {
         if (options?.unchecked) {
-            return entries.map((x) => {
+            return [...entries.values()].map((x) => {
                 this._gidPeersHistory.delete(x.meta.gid);
                 return this.log.remove(x, {
                     recursively: true,
@@ -1416,7 +1790,8 @@ let SharedLog = class SharedLog extends Program {
         // - Peers join and leave, which means we might not be a replicator anymore
         const promises = [];
         const filteredEntries = [];
-        for (const entry of entries) {
+        const deleted = new Set();
+        for (const entry of entries.values()) {
             const pendingPrev = this._pendingDeletes.get(entry.hash);
             if (pendingPrev) {
                 promises.push(pendingPrev.promise.promise);
@@ -1442,9 +1817,10 @@ let SharedLog = class SharedLog extends Program {
                 clear();
                 deferredPromise.reject(e);
             };
-            const timeout = setTimeout(() => {
+            let cursor = undefined;
+            const timeout = setTimeout(async () => {
                 reject(new Error("Timeout for checked pruning: Closed: " + this.closed));
-            }, options?.timeout ?? 10 * 1000);
+            }, options?.timeout ?? 1e4);
             this._pendingDeletes.set(entry.hash, {
                 promise: deferredPromise,
                 clear: () => {
@@ -1456,12 +1832,13 @@ let SharedLog = class SharedLog extends Program {
                     const minMinReplicasValue = this.replicas.max
                         ? Math.min(minReplicasValue, this.replicas.max.getValue(this))
                         : minReplicasValue;
-                    const leaders = await this.findLeaders(entry, minMinReplicasValue);
-                    if (leaders.find((x) => x === this.node.identity.publicKey.hashcode())) {
-                        reject(new Error("Failed to delete, is leader"));
-                        return;
-                    }
-                    if (leaders.find((x) => x === publicKeyHash)) {
+                    const leaders = await this.waitForIsLeader(cursor ??
+                        (cursor = await this.createCoordinates(entry, minMinReplicasValue)), publicKeyHash);
+                    if (leaders) {
+                        if (leaders.has(this.node.identity.publicKey.hashcode())) {
+                            reject(new Error("Failed to delete, is leader"));
+                            return;
+                        }
                         existCounter.add(publicKeyHash);
                         if (minMinReplicasValue <= existCounter.size) {
                             clear();
@@ -1471,7 +1848,8 @@ let SharedLog = class SharedLog extends Program {
                                 recursively: true,
                             })
                                 .then(() => {
-                                resolve();
+                                deleted.add(entry.hash);
+                                return resolve();
                             })
                                 .catch((e) => {
                                 reject(new Error("Failed to delete entry: " + e.toString()));
@@ -1485,23 +1863,47 @@ let SharedLog = class SharedLog extends Program {
         if (filteredEntries.length === 0) {
             return promises;
         }
-        this.rpc.send(new RequestIPrune({ hashes: filteredEntries.map((x) => x.hash) }));
-        const onNewPeer = async (e) => {
+        const emitMessages = (entries, to) => {
+            this.rpc.send(new RequestIPrune({
+                hashes: entries,
+            }), {
+                mode: new SilentDelivery({
+                    to: [to], // TODO group by peers?
+                    redundancy: 1,
+                }),
+                priority: 1,
+            });
+        };
+        const maxReplicasValue = maxReplicas(this, filteredEntries);
+        this.groupByLeaders(filteredEntries.map((x) => {
+            return { entry: x, replicas: maxReplicasValue }; // TODO choose right maxReplicasValue, should it really be for all entries combined?
+        })).then((map) => {
+            for (const [peer, idx] of map) {
+                emitMessages(idx.map((i) => filteredEntries[i].hash), peer);
+            }
+        });
+        const onPeersChange = async (e) => {
             if (e.detail.publicKey.equals(this.node.identity.publicKey) === false) {
-                await this.rpc.send(new RequestIPrune({ hashes: filteredEntries.map((x) => x.hash) }), {
-                    mode: new SilentDelivery({
-                        to: [e.detail.publicKey.hashcode()],
-                        redundancy: 1,
-                    }),
-                });
+                const peerEntries = (await this.groupByLeaders(filteredEntries
+                    .filter((x) => !deleted.has(x.hash))
+                    .map((x) => {
+                    return { entry: x, replicas: maxReplicasValue }; // TODO choose right maxReplicasValue, should it really be for all entries combined?
+                }))).get(e.detail.publicKey.hashcode());
+                if (peerEntries && peerEntries.length > 0) {
+                    emitMessages(peerEntries.map((x) => filteredEntries[x].hash), e.detail.publicKey.hashcode());
+                }
             }
         };
         // check joining peers
-        this.events.addEventListener("replicator:join", onNewPeer);
-        Promise.allSettled(promises).finally(() => this.events.removeEventListener("replicator:join", onNewPeer));
+        this.events.addEventListener("replicator:mature", onPeersChange);
+        this.events.addEventListener("replicator:join", onPeersChange);
+        Promise.allSettled(promises).finally(() => {
+            this.events.removeEventListener("replicator:mature", onPeersChange);
+            this.events.removeEventListener("replicator:join", onPeersChange);
+        });
         return promises;
     }
-    async distribute() {
+    /* async distribute() {
         // if there is one or more items waiting for run, don't bother adding a new item just wait for the queue to empty
         if (this.distributeQueue && this.distributeQueue?.size > 0) {
             return this.distributeQueue.onEmpty();
@@ -1509,15 +1911,57 @@ let SharedLog = class SharedLog extends Program {
         if (this.closed) {
             return;
         }
-        const queue = this.distributeQueue ||
+        const queue =
+            this.distributeQueue ||
             (this.distributeQueue = new PQueue({ concurrency: 1 }));
         return queue
-            .add(() => delay(Math.min(this.log.length, this.distributionDebounceTime), {
-            signal: this._closeController.signal,
-        }).then(() => this._distribute()))
+            .add(() =>
+                delay(Math.min(this.log.length, this.distributionDebounceTime), {
+                    signal: this._closeController.signal,
+                }).then(() => this._distribute()),
+            )
             .catch(() => { }); // catch ignore delay abort errror
+    } */
+    /**
+     * For debugging
+     */
+    async getPrunable() {
+        const heads = await this.log.getHeads(true).all();
+        let prunable = [];
+        for (const head of heads) {
+            const isLeader = await this.isLeader({
+                entry: head,
+                replicas: maxReplicas(this, [head]),
+            });
+            if (!isLeader) {
+                prunable.push(head);
+            }
+        }
+        return prunable;
+    }
+    async getNonPrunable() {
+        const heads = await this.log.getHeads(true).all();
+        let nonPrunable = [];
+        for (const head of heads) {
+            const isLeader = await this.isLeader({
+                entry: head,
+                replicas: maxReplicas(this, [head]),
+            });
+            if (isLeader) {
+                nonPrunable.push(head);
+            }
+        }
+        return nonPrunable;
     }
-    async _distribute() {
+    async rebalanceAll() {
+        this.onReplicationChange((await this.getMyReplicationSegments()).map((x) => {
+            return { range: x, type: "added" };
+        }));
+    }
+    async waitForPruned() {
+        await waitFor(() => this._pendingDeletes.size === 0);
+    }
+    async onReplicationChange(changeOrChanges) {
         /**
          * TODO use information of new joined/leaving peer to create a subset of heads
          * that we potentially need to share with other peers
@@ -1525,69 +1969,81 @@ let SharedLog = class SharedLog extends Program {
         if (this.closed) {
             return;
         }
+        const change = mergeReplicationChanges(changeOrChanges);
         const changed = false;
-        await this.log.trim();
-        const heads = await this.log.getHeads().all();
-        const groupedByGid = await groupByGid(heads);
-        const uncheckedDeliver = new Map();
-        const allEntriesToDelete = [];
-        for (const [gid, entries] of groupedByGid) {
-            if (this.closed) {
-                break;
-            }
-            if (entries.length === 0) {
-                continue; // TODO maybe close store?
-            }
-            const oldPeersSet = this._gidPeersHistory.get(gid);
-            const currentPeers = await this.findLeaders(getLatestEntry(entries), maxReplicas(this, entries));
-            const isLeader = currentPeers.find((x) => x === this.node.identity.publicKey.hashcode());
-            const currentPeersSet = new Set(currentPeers);
-            this._gidPeersHistory.set(gid, currentPeersSet);
-            for (const currentPeer of currentPeers) {
-                if (currentPeer === this.node.identity.publicKey.hashcode()) {
-                    continue;
+        try {
+            await this.log.trim();
+            const uncheckedDeliver = new Map();
+            const allEntriesToDelete = [];
+            for await (const { gid, entries: coordinates } of toRebalance(change, this.entryCoordinatesIndex)) {
+                if (this.closed) {
+                    break;
+                }
+                const oldPeersSet = this._gidPeersHistory.get(gid);
+                if (this.closed) {
+                    return;
                 }
-                if (!oldPeersSet?.has(currentPeer)) {
-                    let arr = uncheckedDeliver.get(currentPeer);
-                    if (!arr) {
-                        arr = [];
-                        uncheckedDeliver.set(currentPeer, arr);
+                let { isLeader, leaders: currentPeers } = await this.findLeadersPersist(coordinates.map((x) => x.coordinate), coordinates[0], {
+                    roleAge: 0,
+                    persist: {
+                        prev: coordinates,
+                    },
+                });
+                if (isLeader) {
+                    for (const entry of coordinates) {
+                        this.pruneDebouncedFn.delete(entry.hash);
                     }
-                    for (const entry of entries) {
-                        arr.push(entry);
+                }
+                const currentPeersSet = new Set(currentPeers.keys());
+                this._gidPeersHistory.set(gid, currentPeersSet);
+                for (const [currentPeer] of currentPeers) {
+                    if (currentPeer === this.node.identity.publicKey.hashcode()) {
+                        continue;
+                    }
+                    if (!oldPeersSet?.has(currentPeer)) {
+                        let arr = uncheckedDeliver.get(currentPeer);
+                        if (!arr) {
+                            arr = [];
+                            uncheckedDeliver.set(currentPeer, arr);
+                        }
+                        for (const entry of coordinates) {
+                            arr.push(entry);
+                        }
                     }
                 }
-            }
-            if (!isLeader) {
-                if (currentPeers.length > 0) {
-                    // If we are observer, never prune locally created entries, since we dont really know who can store them
-                    // if we are replicator, we will always persist entries that we need to so filtering on createdLocally will not make a difference
-                    let entriesToDelete = entries;
-                    if (this.sync) {
-                        entriesToDelete = entriesToDelete.filter((entry) => this.sync(entry) === false);
+                if (!isLeader) {
+                    if (currentPeers.size > 0) {
+                        // If we are observer, never prune locally created entries, since we dont really know who can store them
+                        // if we are replicator, we will always persist entries that we need to so filtering on createdLocally will not make a difference
+                        let entriesToDelete = coordinates;
+                        if (this.sync) {
+                            entriesToDelete = entriesToDelete.filter((entry) => this.sync(entry) === false);
+                        }
+                        allEntriesToDelete.push(...entriesToDelete);
                     }
-                    allEntriesToDelete.push(...entriesToDelete);
                 }
-            }
-            else {
-                for (const entry of entries) {
-                    await this._pendingDeletes
-                        .get(entry.hash)
-                        ?.reject(new Error("Failed to delete, is leader again. Closed: " + this.closed));
+                else {
+                    for (const entry of coordinates) {
+                        await this._pendingDeletes
+                            .get(entry.hash)
+                            ?.reject(new Error("Failed to delete, is leader again. Closed: " + this.closed));
+                    }
                 }
             }
+            for (const [target, entries] of uncheckedDeliver) {
+                this.rpc.send(new RequestMaybeSync({ hashes: entries.map((x) => x.hash) }), {
+                    mode: new SilentDelivery({ to: [target], redundancy: 1 }),
+                });
+            }
+            if (allEntriesToDelete.length > 0) {
+                allEntriesToDelete.map((x) => this.pruneDebouncedFn.add({ key: x.hash, value: x }));
+            }
+            return changed;
         }
-        for (const [target, entries] of uncheckedDeliver) {
-            this.rpc.send(new RequestMaybeSync({ hashes: entries.map((x) => x.hash) }), {
-                mode: new SilentDelivery({ to: [target], redundancy: 1 }),
-            });
-        }
-        if (allEntriesToDelete.length > 0) {
-            Promise.allSettled(this.prune(allEntriesToDelete)).catch((e) => {
-                logger.info(e.toString());
-            });
+        catch (error) {
+            logger.error(error.toString());
+            throw error;
         }
-        return changed;
     }
     async requestSync(hashes, to) {
         const now = +new Date();
@@ -1608,7 +2064,7 @@ let SharedLog = class SharedLog extends Program {
         });
     }
     async _onUnsubscription(evt) {
-        logger.debug(`Peer disconnected '${evt.detail.from.hashcode()}' from '${JSON.stringify(evt.detail.unsubscriptions.map((x) => x))}'`);
+        logger.debug(`Peer disconnected '${evt.detail.from.hashcode()}' from '${JSON.stringify(evt.detail.unsubscriptions.map((x) => x))} '`);
         this.latestReplicationInfoMessage.delete(evt.detail.from.hashcode());
         // TODO only emit this if the peer is actually replicating anything
         this.events.dispatchEvent(new CustomEvent("replicator:leave", {
@@ -1645,78 +2101,80 @@ let SharedLog = class SharedLog extends Program {
         return (factorChanges.reduce((sum, change) => sum + change, 0) /
             factorChanges.length);
     }
-    async rebalanceParticipation(onRoleChange = true) {
+    async rebalanceParticipation() {
         // update more participation rate to converge to the average expected rate or bounded by
         // resources such as memory and or cpu
-        if (this.closed) {
-            return false;
-        }
-        // The role is fixed (no changes depending on memory usage or peer count etc)
-        if (!this._isReplicating) {
-            return false;
-        }
-        if (this._isAdaptiveReplicating) {
-            const peers = this.replicationIndex;
-            const usedMemory = await this.getMemoryUsage();
-            let dynamicRange = await this.getDynamicRange();
-            if (!dynamicRange) {
-                return; // not allowed to replicate
-            }
-            const peersSize = (await peers.getSize()) || 1;
-            const totalParticipation = await this.calculateTotalParticipation();
-            const newFactor = this.replicationController.step({
-                memoryUsage: usedMemory,
-                currentFactor: dynamicRange.widthNormalized,
-                totalFactor: totalParticipation, // TODO use this._totalParticipation when flakiness is fixed
-                peerCount: peersSize,
-                cpuUsage: this.cpuUsage?.value(),
-            });
-            const relativeDifference = Math.abs(dynamicRange.widthNormalized - newFactor) /
-                dynamicRange.widthNormalized;
-            if (relativeDifference > 0.0001) {
-                // TODO can not reuse old range, since it will (potentially) affect the index because of sideeffects
-                dynamicRange = new ReplicationRangeIndexable({
-                    offset: hashToU32(this.node.identity.publicKey.bytes),
-                    length: scaleToU32(newFactor),
-                    publicKeyHash: dynamicRange.hash,
-                    id: dynamicRange.id,
-                    mode: dynamicRange.mode,
-                    timestamp: dynamicRange.timestamp,
-                });
-                const canReplicate = !this._isTrustedReplicator ||
-                    (await this._isTrustedReplicator(this.node.identity.publicKey));
-                if (!canReplicate) {
-                    return false;
+        const fn = async () => {
+            if (this.closed) {
+                return false;
+            }
+            // The role is fixed (no changes depending on memory usage or peer count etc)
+            if (!this._isReplicating) {
+                return false;
+            }
+            if (this._isAdaptiveReplicating) {
+                const peers = this.replicationIndex;
+                const usedMemory = await this.getMemoryUsage();
+                let dynamicRange = await this.getDynamicRange();
+                if (!dynamicRange) {
+                    return; // not allowed to replicate
                 }
-                await this.startAnnounceReplicating([dynamicRange], {
-                    checkDuplicates: false,
-                    reset: false,
+                const peersSize = (await peers.getSize()) || 1;
+                const totalParticipation = await this.calculateTotalParticipation();
+                const newFactor = this.replicationController.step({
+                    memoryUsage: usedMemory,
+                    currentFactor: dynamicRange.widthNormalized,
+                    totalFactor: totalParticipation, // TODO use this._totalParticipation when flakiness is fixed
+                    peerCount: peersSize,
+                    cpuUsage: this.cpuUsage?.value(),
                 });
-                /* await this._updateRole(newRole, onRoleChange); */
-                this.rebalanceParticipationDebounced?.();
-                return true;
-            }
-            else {
-                this.rebalanceParticipationDebounced?.();
+                const relativeDifference = Math.abs(dynamicRange.widthNormalized - newFactor) /
+                    dynamicRange.widthNormalized;
+                if (relativeDifference > 0.0001) {
+                    // TODO can not reuse old range, since it will (potentially) affect the index because of sideeffects
+                    dynamicRange = new ReplicationRangeIndexable({
+                        offset: hashToU32(this.node.identity.publicKey.bytes),
+                        length: scaleToU32(newFactor),
+                        publicKeyHash: dynamicRange.hash,
+                        id: dynamicRange.id,
+                        mode: dynamicRange.mode,
+                        timestamp: dynamicRange.timestamp,
+                    });
+                    const canReplicate = !this._isTrustedReplicator ||
+                        (await this._isTrustedReplicator(this.node.identity.publicKey));
+                    if (!canReplicate) {
+                        return false;
+                    }
+                    await this.startAnnounceReplicating([dynamicRange], {
+                        checkDuplicates: false,
+                        reset: false,
+                    });
+                    /* await this._updateRole(newRole, onRoleChange); */
+                    this.rebalanceParticipationDebounced?.();
+                    return true;
+                }
+                else {
+                    this.rebalanceParticipationDebounced?.();
+                }
+                return false;
             }
             return false;
-        }
-        return false;
+        };
+        const resp = await fn();
+        return resp;
     }
     async getDynamicRange() {
-        let dynamicRangeId = sha256Sync(concat([
-            this.node.identity.publicKey.bytes,
-            new TextEncoder().encode("dynamic"),
-        ]));
-        let range = (await this.replicationIndex.query(new SearchRequest({
+        let dynamicRangeId = getIdForDynamicRange(this.node.identity.publicKey);
+        let range = (await this.replicationIndex
+            .iterate({
             query: [
                 new ByteMatchQuery({
                     key: "id",
                     value: dynamicRangeId,
                 }),
             ],
-            fetch: 1,
-        })))?.results[0]?.value;
+        })
+            .all())?.[0]?.value;
         if (!range) {
             range = new ReplicationRangeIndexable({
                 normalized: true,
@@ -1750,7 +2208,26 @@ let SharedLog = class SharedLog extends Program {
             this.syncInFlightQueue.delete(hash);
         }
     }
-    onEntryAdded(entry) {
+    clearSyncProcessPublicKey(publicKey) {
+        this.syncInFlight.delete(publicKey.hashcode());
+        const map = this.syncInFlightQueueInverted.get(publicKey.hashcode());
+        if (map) {
+            for (const hash of map) {
+                const arr = this.syncInFlightQueue.get(hash);
+                if (arr) {
+                    const filtered = arr.filter((x) => !x.equals(publicKey));
+                    if (filtered.length > 0) {
+                        this.syncInFlightQueue.set(hash, filtered);
+                    }
+                    else {
+                        this.syncInFlightQueue.delete(hash);
+                    }
+                }
+            }
+            this.syncInFlightQueueInverted.delete(publicKey.hashcode());
+        }
+    }
+    async onEntryAdded(entry) {
         const ih = this._pendingIHave.get(entry.hash);
         if (ih) {
             ih.clear();