@peerbit/shared-log 9.0.10 → 9.1.0

package/dist/src/index.js

@@ -7,12 +7,12 @@ var __decorate = (this && this.__decorate) || function (decorators, target, key,
 var __metadata = (this && this.__metadata) || function (k, v) {
     if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
 };
-import { BinaryWriter, BorshError, field, variant } from "@dao-xyz/borsh";
+import { BorshError, field, variant } from "@dao-xyz/borsh";
 import { CustomEvent } from "@libp2p/interface";
 import { AnyBlockStore, RemoteBlocks } from "@peerbit/blocks";
 import { Cache } from "@peerbit/cache";
-import { AccessError, PublicSignKey, sha256, sha256Base64Sync, sha256Sync, } from "@peerbit/crypto";
-import { And, ByteMatchQuery, CountRequest, DeleteRequest, IntegerCompare, Or, SearchRequest, Sort, StringMatch, SumRequest, toId, } from "@peerbit/indexer-interface";
+import { AccessError, PublicSignKey, sha256Base64Sync, sha256Sync, } from "@peerbit/crypto";
+import { And, ByteMatchQuery, CountRequest, DeleteRequest, Or, SearchRequest, Sort, StringMatch, SumRequest, } from "@peerbit/indexer-interface";
 import { Entry, Log, ShallowEntry, } from "@peerbit/log";
 import { logger as loggerFn } from "@peerbit/logger";
 import { ClosedError, Program } from "@peerbit/program";
@@ -23,17 +23,33 @@ import { AbortError, delay, waitFor } from "@peerbit/time";
 import debounce from "p-debounce";
 import pDefer, {} from "p-defer";
 import PQueue from "p-queue";
+import { concat } from "uint8arrays";
 import { BlocksMessage } from "./blocks.js";
 import { CPUUsageIntervalLag } from "./cpu.js";
 import { EntryWithRefs, ExchangeHeadsMessage, RequestIPrune, RequestMaybeSync, ResponseIPrune, ResponseMaybeSync, createExchangeHeadsMessages, } from "./exchange-heads.js";
 import { TransportMessage } from "./message.js";
 import { PIDReplicationController } from "./pid.js";
-import { getCoverSet, getSamples, isMatured } from "./ranges.js";
-import { AbsoluteReplicas, ReplicationError, ReplicationIntent, ReplicationRange, ReplicationRangeIndexable, RequestReplicationInfoMessage, ResponseReplicationInfoMessage, ResponseRoleMessage, StartedReplicating, StoppedReplicating, decodeReplicas, encodeReplicas, hashToUniformNumber, maxReplicas, } from "./replication.js";
-import { Observer, Replicator, SEGMENT_COORDINATE_SCALE } from "./role.js";
-export * from "./replication.js";
+import { getCoverSet, getSamples, hasCoveringRange, isMatured, minimumWidthToCover, } from "./ranges.js";
+import { createReplicationDomainHash, hashToU32, } from "./replication-domain-hash.js";
+import { createReplicationDomainTime, } from "./replication-domain-time.js";
+import {} from "./replication-domain.js";
+import { AbsoluteReplicas, AddedReplicationSegmentMessage, AllReplicatingSegmentsMessage, ReplicationError, ReplicationIntent, ReplicationRange, ReplicationRangeIndexable, RequestReplicationInfoMessage, ResponseRoleMessage, StoppedReplicating, decodeReplicas, encodeReplicas, maxReplicas, } from "./replication.js";
+import { MAX_U32, Observer, Replicator, scaleToU32 } from "./role.js";
+export { createReplicationDomainHash, createReplicationDomainTime, };
 export { CPUUsageIntervalLag };
+export * from "./replication.js";
 export const logger = loggerFn({ module: "shared-log" });
+const getLatestEntry = (entries) => {
+    let latest = undefined;
+    for (const element of entries) {
+        let entry = element instanceof EntryWithRefs ? element.entry : element;
+        if (!latest ||
+            entry.meta.clock.timestamp.compare(latest.meta.clock.timestamp) > 0) {
+            latest = entry;
+        }
+    }
+    return latest;
+};
 const groupByGid = async (entries) => {
     const groupByGid = new Map();
     for (const head of entries) {
@@ -61,8 +77,15 @@ const isAdaptiveReplicatorOption = (options) => {
     if (options.factor != null) {
         return false;
     }
+    if (Array.isArray(options)) {
+        return false;
+    }
     return true;
 };
+const isUnreplicationOptions = (options) => options === false ||
+    options === 0 ||
+    (options?.offset === undefined &&
+        options?.factor === 0);
 export const DEFAULT_MIN_REPLICAS = 2;
 export const WAIT_FOR_REPLICATOR_TIMEOUT = 9000;
 export const WAIT_FOR_ROLE_MATURITY = 5000;
@@ -72,7 +95,8 @@ let SharedLog = class SharedLog extends Program {
     log;
     rpc;
     // options
-    _replicationSettings;
+    _isReplicating;
+    _isAdaptiveReplicating;
     _replicationRangeIndex;
     /* private _totalParticipation!: number; */
     _gidPeersHistory;
@@ -85,7 +109,7 @@ let SharedLog = class SharedLog extends Program {
     _respondToIHaveTimeout;
     _pendingDeletes;
     _pendingIHave;
-    latestRoleMessages;
+    latestReplicationInfoMessage;
     remoteBlocks;
     openTime;
     oldestOpenTime;
@@ -109,18 +133,13 @@ let SharedLog = class SharedLog extends Program {
     distributionDebounceTime;
     replicationController;
     history;
+    domain;
     pq;
     constructor(properties) {
         super();
         this.log = new Log(properties);
         this.rpc = new RPC();
     }
-    /**
-     * Return the
-     */
-    get replicationSettings() {
-        return this._replicationSettings;
-    }
     get compatibility() {
         return this._logProperties?.compatibility;
     }
@@ -128,30 +147,33 @@ let SharedLog = class SharedLog extends Program {
         return (this.compatibility ?? Number.MAX_VALUE) < 9;
     }
     // @deprecated
-    getRole() {
-        let isFixedReplicationSettings = this._replicationSettings.factor !==
-            undefined;
-        if (isFixedReplicationSettings) {
-            const fixedSettings = this
-                ._replicationSettings;
+    async getRole() {
+        const segments = await this.getMyReplicationSegments();
+        if (segments.length > 1) {
+            throw new Error("More than one replication segment found. Can only use one segment for compatbility with v8");
+        }
+        if (segments.length > 0) {
+            const segment = segments[0].toReplicationRange();
             return new Replicator({
-                factor: fixedSettings.factor,
-                offset: fixedSettings.offset ?? 0,
+                factor: segment.factor / MAX_U32,
+                offset: segment.offset / MAX_U32,
             });
         }
         // TODO this is not accurate but might be good enough
         return new Observer();
     }
     async isReplicating() {
-        if (!this._replicationSettings) {
+        if (!this._isReplicating) {
             return false;
         }
+        /*
         if (isAdaptiveReplicatorOption(this._replicationSettings)) {
             return true;
         }
-        if (this.replicationSettings.factor > 0) {
+
+        if ((this.replicationSettings as FixedReplicationOptions).factor !== 0) {
             return true;
-        }
+        } */
         return (await this.countReplicationSegments()) > 0;
     }
     /* get totalParticipation(): number {
@@ -159,7 +181,7 @@ let SharedLog = class SharedLog extends Program {
     } */
     async calculateTotalParticipation() {
         const sum = await this.replicationIndex.sum(new SumRequest({ key: "width" }));
-        return Number(sum) / SEGMENT_COORDINATE_SCALE;
+        return Number(sum) / MAX_U32;
     }
     async countReplicationSegments() {
         const count = await this.replicationIndex.count(new CountRequest({
@@ -171,6 +193,7 @@ let SharedLog = class SharedLog extends Program {
         return count;
     }
     setupRebalanceDebounceFunction() {
+        this.rebalanceParticipationDebounced = undefined;
         this.rebalanceParticipationDebounced = debounce(() => this.rebalanceParticipation(), 
         /* Math.max(
             REBALANCE_DEBOUNCE_INTERVAL,
@@ -181,107 +204,194 @@ let SharedLog = class SharedLog extends Program {
         ) */
         REBALANCE_DEBOUNCE_INTERVAL);
     }
-    async setupReplicationSettings(options) {
-        this.rebalanceParticipationDebounced = undefined;
-        const setupDebouncedRebalancing = (options) => {
-            this.cpuUsage?.stop?.();
-            this.replicationController = new PIDReplicationController(this.node.identity.publicKey.hashcode(), {
-                storage: options?.limits?.storage != null
-                    ? { max: options?.limits?.storage }
-                    : undefined,
-                cpu: options?.limits?.cpu != null
-                    ? {
-                        max: typeof options?.limits?.cpu === "object"
-                            ? options.limits.cpu.max
-                            : options?.limits?.cpu,
-                    }
-                    : undefined,
-            });
-            this.cpuUsage =
-                options?.limits?.cpu && typeof options?.limits?.cpu === "object"
-                    ? options?.limits?.cpu?.monitor || new CPUUsageIntervalLag()
-                    : new CPUUsageIntervalLag();
-            this.cpuUsage?.start?.();
-            this.setupRebalanceDebounceFunction();
-        };
-        if (options) {
+    async _replicate(options, { reset, checkDuplicates, announce, } = {}) {
+        let offsetWasProvided = false;
+        if (isUnreplicationOptions(options)) {
+            await this.unreplicate();
+        }
+        else {
+            let ranges = [];
+            if (options == null) {
+                options = {};
+            }
+            else if (options === true) {
+                options = {};
+            }
+            this._isReplicating = true;
+            this._isAdaptiveReplicating = false;
             if (isAdaptiveReplicatorOption(options)) {
-                this._replicationSettings = options;
-                setupDebouncedRebalancing(this._replicationSettings);
+                this._isAdaptiveReplicating = true;
+                this.setupDebouncedRebalancing(options);
+                // initial role in a dynamic setup
+                const maybeRange = await this.getDynamicRange();
+                if (!maybeRange) {
+                    // not allowed
+                    return;
+                }
+                ranges = [maybeRange];
+                offsetWasProvided = true;
             }
-            else if (options === true ||
-                (options && Object.keys(options).length === 0)) {
-                this._replicationSettings = {};
-                setupDebouncedRebalancing(this._replicationSettings);
+            else if (options instanceof ReplicationRange) {
+                ranges = [
+                    options.toReplicationRangeIndexable(this.node.identity.publicKey),
+                ];
+                offsetWasProvided = true;
             }
             else {
+                let rangeArgs;
                 if (typeof options === "number") {
-                    this._replicationSettings = {
-                        factor: options,
-                    };
+                    rangeArgs = [
+                        {
+                            factor: options,
+                        },
+                    ];
                 }
                 else {
-                    this._replicationSettings = { ...options };
+                    rangeArgs = (Array.isArray(options) ? options : [{ ...options }]);
+                }
+                if (rangeArgs.length === 0) {
+                    // nothing to do
+                    return;
                 }
+                for (const rangeArg of rangeArgs) {
+                    const normalized = rangeArg.normalized ?? true;
+                    offsetWasProvided = rangeArg.offset != null;
+                    const offset = rangeArg.offset ??
+                        (normalized ? Math.random() : scaleToU32(Math.random()));
+                    let factor = rangeArg.factor;
+                    let width = normalized ? 1 : scaleToU32(1);
+                    ranges.push(new ReplicationRangeIndexable({
+                        normalized,
+                        offset: offset,
+                        length: typeof factor === "number"
+                            ? factor
+                            : factor === "all"
+                                ? width
+                                : width - offset,
+                        publicKeyHash: this.node.identity.publicKey.hashcode(),
+                        mode: rangeArg.strict
+                            ? ReplicationIntent.Strict
+                            : ReplicationIntent.NonStrict, // automatic means that this range might be reused later for dynamic replication behaviour
+                        timestamp: BigInt(+new Date()),
+                    }));
+                }
+            }
+            for (const range of ranges) {
+                this.oldestOpenTime = Math.min(Number(range.timestamp), this.oldestOpenTime);
             }
+            let resetRanges = reset;
+            if (!resetRanges && !offsetWasProvided) {
+                resetRanges = true;
+                // because if we do something like replicate ({ factor: 0.5 }) it means that we want to replicate 50%
+                // but ({ replicate: 0.5, offset: 0.5 }) means that we want to add a range
+                // TODO make behaviour more clear
+            }
+            await this.startAnnounceReplicating(ranges, {
+                reset: resetRanges ?? false,
+                checkDuplicates,
+                announce,
+            });
         }
-        else {
-            return;
+    }
+    setupDebouncedRebalancing(options) {
+        this.cpuUsage?.stop?.();
+        this.replicationController = new PIDReplicationController(this.node.identity.publicKey.hashcode(), {
+            storage: options?.limits?.storage != null
+                ? { max: options?.limits?.storage }
+                : undefined,
+            cpu: options?.limits?.cpu != null
+                ? {
+                    max: typeof options?.limits?.cpu === "object"
+                        ? options.limits.cpu.max
+                        : options?.limits?.cpu,
+                }
+                : undefined,
+        });
+        this.cpuUsage =
+            options?.limits?.cpu && typeof options?.limits?.cpu === "object"
+                ? options?.limits?.cpu?.monitor || new CPUUsageIntervalLag()
+                : new CPUUsageIntervalLag();
+        this.cpuUsage?.start?.();
+        this.setupRebalanceDebounceFunction();
+    }
+    async replicate(rangeOrEntry, options) {
+        let range = undefined;
+        if (rangeOrEntry instanceof ReplicationRange) {
+            range = rangeOrEntry;
         }
-        if (isAdaptiveReplicatorOption(this._replicationSettings)) {
-            // initial role in a dynamic setup
-            await this.getDynamicRange();
+        else if (rangeOrEntry instanceof Entry) {
+            range = {
+                factor: 1,
+                offset: await this.domain.fromEntry(rangeOrEntry),
+                normalized: false,
+            };
+        }
+        else if (Array.isArray(rangeOrEntry)) {
+            let ranges = [];
+            for (const entry of rangeOrEntry) {
+                if (entry instanceof Entry) {
+                    ranges.push({
+                        factor: 1,
+                        offset: await this.domain.fromEntry(entry),
+                        normalized: false,
+                    });
+                }
+                else {
+                    ranges.push(entry);
+                }
+            }
+            range = ranges;
         }
         else {
-            // fixed
-            const range = new ReplicationRangeIndexable({
-                offset: this._replicationSettings.offset ??
-                    Math.random(),
-                length: this._replicationSettings.factor,
-                publicKeyHash: this.node.identity.publicKey.hashcode(),
-                replicationIntent: ReplicationIntent.Explicit, // automatic means that this range might be reused later for dynamic replication behaviour
-                timestamp: BigInt(+new Date()),
-                id: sha256Sync(this.node.identity.publicKey.bytes),
-            });
-            await this.startAnnounceReplicating(range);
+            range = rangeOrEntry ?? true;
         }
+        const newRanges = await this._replicate(range, options);
+        // assume new role
+        await this.distribute();
+        return newRanges;
     }
-    async replicate(range) {
-        if (range === false || range === 0) {
-            this._replicationSettings = undefined;
-            await this.removeReplicator(this.node.identity.publicKey);
+    async unreplicate(rangeOrEntry) {
+        let range;
+        if (rangeOrEntry instanceof Entry) {
+            range = {
+                factor: 1,
+                offset: await this.domain.fromEntry(rangeOrEntry),
+            };
+        }
+        else if (rangeOrEntry instanceof ReplicationRange) {
+            range = rangeOrEntry;
         }
         else {
-            await this.rpc.subscribe();
-            if (range instanceof ReplicationRange) {
-                this.oldestOpenTime = Math.min(Number(range.timestamp), this.oldestOpenTime);
-                await this.startAnnounceReplicating(range.toReplicationRangeIndexable(this.node.identity.publicKey));
-            }
-            else {
-                await this.setupReplicationSettings(range ?? true);
-            }
+            this._isReplicating = false;
+            this._isAdaptiveReplicating = false;
+            await this.removeReplicator(this.node.identity.publicKey);
+            return;
         }
-        // assume new role
-        await this.distribute();
+        if (this._isAdaptiveReplicating) {
+            // we can not unreplicate individual ranges when dynamically replicating (yet)
+            // TODO support this by never deleting the range with the segment id that is generated by the dynamic replication method
+            throw new Error("Unsupported when adaptive replicating");
+        }
+        const indexed = await this.replicationIndex.query(new SearchRequest({
+            query: {
+                width: 1,
+                start1: range.offset,
+            },
+        }));
+        const segmentIds = indexed.results.map((x) => x.id.key);
+        await this.removeReplicationRange(segmentIds, this.node.identity.publicKey);
+        await this.rpc.send(new StoppedReplicating({ segmentIds }), {
+            priority: 1,
+        });
     }
     async removeReplicator(key) {
         const fn = async () => {
-            let prev = await this.replicationIndex.query(new SearchRequest({
-                query: { hash: key.hashcode() },
-                fetch: 0xffffffff,
-            }), { reference: true });
-            if (prev.results.length === 0) {
-                return;
-            }
-            /* let sumWidth = prev.results.reduce(
-                (acc, x) => acc + x.value.widthNormalized,
-                0,
-            );
-            this._totalParticipation -= sumWidth;
- */
-            let idMatcher = new Or(prev.results.map((x) => new ByteMatchQuery({ key: "id", value: x.value.id })));
-            await this.replicationIndex.del(new DeleteRequest({ query: idMatcher }));
+            await this.replicationIndex.del(new DeleteRequest({ query: { hash: key.hashcode() } }));
             await this.updateOldestTimestampFromIndex();
+            if (this.node.identity.publicKey.equals(key)) {
+                // announce that we are no longer replicating
+                await this.rpc.send(new AllReplicatingSegmentsMessage({ segments: [] }), { priority: 1 });
+            }
             this.events.dispatchEvent(new CustomEvent("replication:change", {
                 detail: { publicKey: key },
             }));
@@ -310,11 +420,6 @@ let SharedLog = class SharedLog extends Program {
                 value: from.hashcode(),
             });
             let query = new And([idMatcher, identityMatcher]);
-            /* const prevSum = await this.replicationIndex.sum(
-                new SumRequest({ query, key: "width" }),
-            );
-            const prevSumNormalized = Number(prevSum) / SEGMENT_COORDINATE_SCALE;
-            this._totalParticipation -= prevSumNormalized; */
             await this.replicationIndex.del(new DeleteRequest({ query }));
             await this.updateOldestTimestampFromIndex();
             this.events.dispatchEvent(new CustomEvent("replication:change", {
@@ -326,27 +431,38 @@ let SharedLog = class SharedLog extends Program {
         };
         return this.pq.add(fn);
     }
-    async addReplicationRange(range, from) {
+    async addReplicationRange(ranges, from, { reset, checkDuplicates, } = {}) {
         const fn = async () => {
             if (this._isTrustedReplicator &&
                 !(await this._isTrustedReplicator(from))) {
                 return false;
             }
-            range.id = new Uint8Array(range.id);
             let prevCount = await this.replicationIndex.count(new CountRequest({
                 query: new StringMatch({ key: "hash", value: from.hashcode() }),
             }));
             const isNewReplicator = prevCount === 0;
-            let prev = await this.replicationIndex.get(toId(range.id));
-            if (prev) {
-                if (prev.value.equals(range)) {
-                    return false;
+            if (reset) {
+                await this.replicationIndex.del(new DeleteRequest({ query: { hash: from.hashcode() } }));
+            }
+            else if (checkDuplicates) {
+                let deduplicated = [];
+                // TODO also deduplicate/de-overlap among the ranges that ought to be inserted?
+                for (const range of ranges) {
+                    if (!(await hasCoveringRange(this.replicationIndex, range))) {
+                        deduplicated.push(range);
+                    }
+                }
+                ranges = deduplicated;
+            }
+            for (const range of ranges) {
+                await this.replicationIndex.put(range);
+                if (!reset) {
+                    this.oldestOpenTime = Math.min(Number(range.timestamp), this.oldestOpenTime);
                 }
-                /* this._totalParticipation -= prev.value.widthNormalized; */
             }
-            await this.replicationIndex.put(range);
-            /* 	this._totalParticipation += range.widthNormalized; */
-            this.oldestOpenTime = Math.min(Number(range.timestamp), this.oldestOpenTime);
+            if (reset) {
+                await this.updateOldestTimestampFromIndex();
+            }
             this.events.dispatchEvent(new CustomEvent("replication:change", {
                 detail: { publicKey: from },
             }));
@@ -360,17 +476,35 @@ let SharedLog = class SharedLog extends Program {
             }
             return true;
         };
+        // we sequialize this because we are going to queries to check wether to add or not
+        // if two processes do the same this both process might add a range while only one in practice should
         return this.pq.add(fn);
     }
-    async startAnnounceReplicating(range) {
-        const added = await this.addReplicationRange(range, this.node.identity.publicKey);
+    async startAnnounceReplicating(range, options = {}) {
+        const added = await this.addReplicationRange(range, this.node.identity.publicKey, options);
         if (!added) {
             logger.warn("Not allowed to replicate by canReplicate");
         }
+        let message;
         if (added) {
-            await this.rpc.send(new StartedReplicating({ segments: [range.toReplicationRange()] }), {
-                priority: 1,
-            });
+            if (options.reset) {
+                message = new AllReplicatingSegmentsMessage({
+                    segments: range.map((x) => x.toReplicationRange()),
+                });
+            }
+            else {
+                message = new AddedReplicationSegmentMessage({
+                    segments: range.map((x) => x.toReplicationRange()),
+                });
+            }
+            if (options.announce) {
+                return options.announce(message);
+            }
+            else {
+                await this.rpc.send(message, {
+                    priority: 1,
+                });
+            }
         }
     }
     async append(data, options) {
@@ -404,16 +538,22 @@ let SharedLog = class SharedLog extends Program {
         }
         const result = await this.log.append(data, appendOptions);
         let mode = undefined;
+        if (options?.replicate) {
+            await this.replicate(result.entry, { checkDuplicates: true });
+        }
         for (const message of await createExchangeHeadsMessages(this.log, [result.entry], this._gidParentCache)) {
             if (options?.target === "replicators" || !options?.target) {
                 const minReplicas = decodeReplicas(result.entry).getValue(this);
-                let leaders = await this.findLeaders(result.entry.meta.gid, minReplicas);
+                let leaders = await this.findLeaders(result.entry, minReplicas);
                 const isLeader = leaders.includes(this.node.identity.publicKey.hashcode());
                 if (message.heads[0].gidRefrences.length > 0) {
                     const newAndOldLeaders = new Set(leaders);
                     for (const ref of message.heads[0].gidRefrences) {
-                        for (const hash of await this.findLeaders(ref, minReplicas)) {
-                            newAndOldLeaders.add(hash);
+                        const entryFromGid = this.log.entryIndex.getHeads(ref, false);
+                        for (const entry of await entryFromGid.all()) {
+                            for (const hash of await this.findLeaders(entry, minReplicas)) {
+                                newAndOldLeaders.add(hash);
+                            }
                         }
                     }
                     leaders = newAndOldLeaders;
@@ -453,10 +593,11 @@ let SharedLog = class SharedLog extends Program {
                     : options.replicas.max
                 : undefined,
         };
+        this.domain = options?.domain ?? createReplicationDomainHash();
         this._respondToIHaveTimeout = options?.respondToIHaveTimeout ?? 10 * 1000; // TODO make into arg
         this._pendingDeletes = new Map();
         this._pendingIHave = new Map();
-        this.latestRoleMessages = new Map();
+        this.latestReplicationInfoMessage = new Map();
         this.syncInFlightQueue = new Map();
         this.syncInFlightQueueInverted = new Map();
         this.syncInFlight = new Map();
@@ -495,6 +636,7 @@ let SharedLog = class SharedLog extends Program {
         });
         const logIndex = await logScope.scope("log");
         await this.node.indexer.start(); // TODO why do we need to start the indexer here?
+        const hasIndexedReplicationInfo = (await this.replicationIndex.getSize()) > 0;
         /* this._totalParticipation = await this.calculateTotalParticipation(); */
         this._gidPeersHistory = new Map();
         await this.log.open(this.remoteBlocks, this.node.identity, {
@@ -528,6 +670,7 @@ let SharedLog = class SharedLog extends Program {
         this._onUnsubscriptionFn =
             this._onUnsubscriptionFn || this._onUnsubscription.bind(this);
         await this.node.services.pubsub.addEventListener("unsubscribe", this._onUnsubscriptionFn);
+        await this.rpc.subscribe();
         // await this.log.load();
         // TODO (do better)
         // we do this distribution interval to eliminate the sideeffects arriving from updating roles and joining entries continously.
@@ -579,7 +722,13 @@ let SharedLog = class SharedLog extends Program {
                 this.syncMoreInterval = setTimeout(requestSync, 1e4);
             });
         };
-        await this.replicate(options?.replicate);
+        // if we had a previous session with replication info, and new replication info dictates that we unreplicate
+        // we should do that. Otherwise if options is a unreplication we dont need to do anything because
+        // we are already unreplicated (as we are just opening)
+        if (hasIndexedReplicationInfo ||
+            isUnreplicationOptions(options?.replicate) === false) {
+            await this.replicate(options?.replicate, { checkDuplicates: true });
+        }
         requestSync();
     }
     async afterOpen() {
@@ -607,7 +756,7 @@ let SharedLog = class SharedLog extends Program {
     }
     async onChange(change) {
         for (const added of change.added) {
-            this.onEntryAdded(added);
+            this.onEntryAdded(added.entry);
         }
         for (const removed of change.removed) {
             this.onEntryRemoved(removed.hash);
@@ -637,6 +786,17 @@ let SharedLog = class SharedLog extends Program {
             throw error;
         }
     }
+    async getCover(args, roleAge) {
+        roleAge = roleAge ?? (await this.getDefaultMinRoleAge());
+        const range = await this.domain.fromArgs(args, this);
+        const set = await getCoverSet(this.replicationIndex, roleAge, range.offset, range.length ??
+            (await minimumWidthToCover(this.replicas.min.getValue(this))), MAX_U32);
+        // add all in flight
+        for (const [key, _] of this.syncInFlight) {
+            set.add(key);
+        }
+        return [...set];
+    }
     async _close() {
         clearTimeout(this.syncMoreInterval);
         clearInterval(this.distributeInterval);
@@ -658,7 +818,7 @@ let SharedLog = class SharedLog extends Program {
         this.syncInFlightQueue.clear();
         this.syncInFlightQueueInverted.clear();
         this.syncInFlight.clear();
-        this.latestRoleMessages.clear();
+        this.latestReplicationInfoMessage.clear();
         this._gidPeersHistory.clear();
         this._replicationRangeIndex = undefined;
         this.cpuUsage?.stop?.();
@@ -727,6 +887,7 @@ let SharedLog = class SharedLog extends Program {
                             const headsWithGid = await this.log.entryIndex
                                 .getHeads(gid)
                                 .all();
+                            const latestEntry = getLatestEntry(entries);
                             const maxReplicasFromHead = headsWithGid && headsWithGid.length > 0
                                 ? maxReplicas(this, [...headsWithGid.values()])
                                 : this.replicas.min.getValue(this);
@@ -734,10 +895,10 @@ let SharedLog = class SharedLog extends Program {
                             const isReplicating = await this.isReplicating();
                             let isLeader;
                             if (isReplicating) {
-                                isLeader = await this.waitForIsLeader(gid, Math.max(maxReplicasFromHead, maxReplicasFromNewEntries));
+                                isLeader = await this.waitForIsLeader(latestEntry, Math.max(maxReplicasFromHead, maxReplicasFromNewEntries));
                             }
                             else {
-                                isLeader = await this.findLeaders(gid, Math.max(maxReplicasFromHead, maxReplicasFromNewEntries));
+                                isLeader = await this.findLeaders(latestEntry, Math.max(maxReplicasFromHead, maxReplicasFromNewEntries));
                                 isLeader = isLeader.includes(this.node.identity.publicKey.hashcode())
                                     ? isLeader
                                     : false;
@@ -803,7 +964,7 @@ let SharedLog = class SharedLog extends Program {
                                 .all();
                             if (headsWithGid && headsWithGid.length > 0) {
                                 const minReplicas = maxReplicas(this, headsWithGid.values());
-                                const isLeader = await this.isLeader(entries[0].entry.meta.gid, minReplicas);
+                                const isLeader = await this.isLeader(entries[0].entry, minReplicas);
                                 if (!isLeader) {
                                     Promise.all(this.prune(entries.map((x) => x.entry))).catch((e) => {
                                         logger.info(e.toString());
@@ -819,7 +980,7 @@ let SharedLog = class SharedLog extends Program {
                 for (const hash of msg.hashes) {
                     const indexedEntry = await this.log.entryIndex.getShallow(hash);
                     if (indexedEntry &&
-                        (await this.isLeader(indexedEntry.value.meta.gid, decodeReplicas(indexedEntry.value).getValue(this)))) {
+                        (await this.isLeader(indexedEntry.value, decodeReplicas(indexedEntry.value).getValue(this)))) {
                         this._gidPeersHistory
                             .get(indexedEntry.value.meta.gid)
                             ?.delete(context.from.hashcode());
@@ -833,7 +994,7 @@ let SharedLog = class SharedLog extends Program {
                                 prevPendingIHave?.clear();
                             },
                             callback: async (entry) => {
-                                if (await this.isLeader(entry.meta.gid, decodeReplicas(entry).getValue(this))) {
+                                if (await this.isLeader(entry, decodeReplicas(entry).getValue(this))) {
                                     this._gidPeersHistory
                                         .get(entry.meta.gid)
                                         ?.delete(context.from.hashcode());
@@ -907,7 +1068,7 @@ let SharedLog = class SharedLog extends Program {
                 if (context.from.equals(this.node.identity.publicKey)) {
                     return;
                 }
-                await this.rpc.send(new ResponseReplicationInfoMessage({
+                await this.rpc.send(new AllReplicatingSegmentsMessage({
                     segments: (await this.getMyReplicationSegments()).map((x) => x.toReplicationRange()),
                 }), {
                     mode: new SilentDelivery({ to: [context.from], redundancy: 1 }),
@@ -916,9 +1077,8 @@ let SharedLog = class SharedLog extends Program {
                 if (this.v8Behaviour) {
                     const role = this.getRole();
                     if (role instanceof Replicator) {
-                        const fixedSettings = this
-                            ._replicationSettings;
-                        if (fixedSettings.factor === 1) {
+                        const fixedSettings = !this._isAdaptiveReplicating;
+                        if (fixedSettings) {
                             await this.rpc.send(new ResponseRoleMessage({
                                 role,
                             }), {
@@ -931,8 +1091,8 @@ let SharedLog = class SharedLog extends Program {
                     }
                 }
             }
-            else if (msg instanceof ResponseReplicationInfoMessage ||
-                msg instanceof StartedReplicating) {
+            else if (msg instanceof AllReplicatingSegmentsMessage ||
+                msg instanceof AddedReplicationSegmentMessage) {
                 if (context.from.equals(this.node.identity.publicKey)) {
                     return;
                 }
@@ -944,23 +1104,16 @@ let SharedLog = class SharedLog extends Program {
                     timeout: this.waitForReplicatorTimeout,
                 })
                     .then(async () => {
-                    // peer should not be online (for us)
-                    const prev = this.latestRoleMessages.get(context.from.hashcode());
+                    // do use an operation log here, because we want to make sure that we don't miss any updates
+                    // and do them in the right order
+                    const prev = this.latestReplicationInfoMessage.get(context.from.hashcode());
                     if (prev && prev > context.timestamp) {
                         return;
                     }
-                    this.latestRoleMessages.set(context.from.hashcode(), context.timestamp);
-                    if (msg instanceof ResponseReplicationInfoMessage) {
-                        await this.removeReplicator(context.from);
-                    }
-                    let addedOnce = false;
-                    for (const segment of replicationInfoMessage.segments) {
-                        const added = await this.addReplicationRange(segment.toReplicationRangeIndexable(context.from), context.from);
-                        if (typeof added === "boolean") {
-                            addedOnce = addedOnce || added;
-                        }
-                    }
-                    addedOnce && (await this.distribute());
+                    this.latestReplicationInfoMessage.set(context.from.hashcode(), context.timestamp);
+                    let reset = msg instanceof AllReplicatingSegmentsMessage;
+                    const added = await this.addReplicationRange(replicationInfoMessage.segments.map((x) => x.toReplicationRangeIndexable(context.from)), context.from, { reset, checkDuplicates: true });
+                    added && (await this.distribute());
                     /* await this._modifyReplicators(msg.role, context.from!); */
                 })
                     .catch((e) => {
@@ -1011,7 +1164,7 @@ let SharedLog = class SharedLog extends Program {
         }));
         return ranges.results.map((x) => x.value);
     }
-    async getTotalParticipation() {
+    async getMyTotalParticipation() {
         // sum all of my replicator rects
         return (await this.getMyReplicationSegments()).reduce((acc, { widthNormalized }) => acc + widthNormalized, 0);
     }
@@ -1056,11 +1209,86 @@ let SharedLog = class SharedLog extends Program {
             throw e;
         });
     }
-    async isLeader(slot, numberOfLeaders, options) {
-        const isLeader = (await this.findLeaders(slot, numberOfLeaders, options)).find((l) => l === this.node.identity.publicKey.hashcode());
+    async join(entries, options) {
+        let messageToSend = undefined;
+        if (options?.replicate) {
+            // TODO this block should perhaps be called from a callback on the this.log.join method on all the ignored element because already joined, like "onAlreadyJoined"
+            // check which entrise we already have but not are replicating, and replicate them
+            let alreadyJoined = [];
+            for (const element of entries) {
+                if (typeof element === "string") {
+                    const entry = await this.log.get(element);
+                    if (entry) {
+                        alreadyJoined.push(entry);
+                    }
+                }
+                else if (element instanceof Entry) {
+                    if (await this.log.has(element.hash)) {
+                        alreadyJoined.push(element);
+                    }
+                }
+                else {
+                    const entry = await this.log.get(element.hash);
+                    if (entry) {
+                        alreadyJoined.push(entry);
+                    }
+                }
+            }
+            // assume is heads
+            await this.replicate(alreadyJoined, {
+                checkDuplicates: true,
+                announce: (msg) => {
+                    if (msg instanceof AllReplicatingSegmentsMessage) {
+                        throw new Error("Unexpected");
+                    }
+                    messageToSend = msg;
+                },
+            });
+        }
+        let joinOptions = options?.replicate
+            ? {
+                ...options,
+                onChange: async (change) => {
+                    if (change.added) {
+                        for (const entry of change.added) {
+                            if (entry.head) {
+                                await this.replicate(entry.entry, {
+                                    checkDuplicates: true,
+                                    // we override the announce step here to make sure we announce all new replication info
+                                    // in one large message instead
+                                    announce: (msg) => {
+                                        if (msg instanceof AllReplicatingSegmentsMessage) {
+                                            throw new Error("Unexpected");
+                                        }
+                                        if (messageToSend) {
+                                            // merge segments to make it into one messages
+                                            for (const segment of msg.segments) {
+                                                messageToSend.segments.push(segment);
+                                            }
+                                        }
+                                        else {
+                                            messageToSend = msg;
+                                        }
+                                    },
+                                });
+                            }
+                        }
+                    }
+                },
+            }
+            : options;
+        await this.log.join(entries, joinOptions);
+        if (messageToSend) {
+            await this.rpc.send(messageToSend, {
+                priority: 1,
+            });
+        }
+    }
+    async isLeader(entry, numberOfLeaders, options) {
+        const isLeader = (await this.findLeaders(entry, numberOfLeaders, options)).find((l) => l === this.node.identity.publicKey.hashcode());
         return !!isLeader;
     }
-    async waitForIsLeader(slot, numberOfLeaders, timeout = this.waitForReplicatorTimeout) {
+    async waitForIsLeader(entry, numberOfLeaders, timeout = this.waitForReplicatorTimeout) {
         return new Promise((resolve, reject) => {
             const removeListeners = () => {
                 this.events.removeEventListener("replication:change", roleListener);
@@ -1075,7 +1303,7 @@ let SharedLog = class SharedLog extends Program {
                 removeListeners();
                 resolve(false);
             }, timeout);
-            const check = () => this.findLeaders(slot, numberOfLeaders).then((leaders) => {
+            const check = () => this.findLeaders(entry, numberOfLeaders).then((leaders) => {
                 const isLeader = leaders.find((l) => l === this.node.identity.publicKey.hashcode());
                 if (isLeader) {
                     removeListeners();
@@ -1091,20 +1319,12 @@ let SharedLog = class SharedLog extends Program {
             check();
         });
     }
-    async findLeaders(subject, numberOfLeaders, options) {
+    async findLeaders(entry, numberOfLeaders, options) {
         if (this.closed) {
             return [this.node.identity.publicKey.hashcode()]; // Assumption: if the store is closed, always assume we have responsibility over the data
         }
-        // For a fixed set or members, the choosen leaders will always be the same (address invariant)
-        // This allows for that same content is always chosen to be distributed to same peers, to remove unecessary copies
-        // Convert this thing we wan't to distribute to 8 bytes so we get can convert it into a u64
-        // modulus into an index
-        const utf8writer = new BinaryWriter();
-        utf8writer.string(subject.toString());
-        const seed = await sha256(utf8writer.finalize());
-        // convert hash of slot to a number
-        const cursor = hashToUniformNumber(seed); // bounded between 0 and 1
-        return this.findLeadersFromUniformNumber(cursor, numberOfLeaders, options);
+        const cursor = await this.domain.fromEntry(entry);
+        return this.findLeadersFromU32(cursor, numberOfLeaders, options);
     }
     async getDefaultMinRoleAge() {
         if ((await this.isReplicating()) === false) {
@@ -1115,39 +1335,12 @@ let SharedLog = class SharedLog extends Program {
         const diffToOldest = replLength > 1 ? now - this.oldestOpenTime - 1 : Number.MAX_SAFE_INTEGER;
         return Math.min(this.timeUntilRoleMaturity, diffToOldest, Math.round((this.timeUntilRoleMaturity * Math.log(replLength + 1)) / 3)); // / 3 so that if 2 replicators and timeUntilRoleMaturity = 1e4 the result will be 1
     }
-    async findLeadersFromUniformNumber(cursor, numberOfLeaders, options) {
+    async findLeadersFromU32(cursor, numberOfLeaders, options) {
         const roleAge = options?.roleAge ?? (await this.getDefaultMinRoleAge()); // TODO -500 as is added so that i f someone else is just as new as us, then we treat them as mature as us. without -500 we might be slower syncing if two nodes starts almost at the same time
-        const samples = await getSamples(cursor, this.replicationIndex, numberOfLeaders, roleAge);
-        return samples;
-    }
-    /**
-     *
-     * @returns groups where at least one in any group will have the entry you are looking for
-     */
-    async getReplicatorUnion(roleAge) {
-        roleAge = roleAge ?? (await this.getDefaultMinRoleAge());
-        if (this.closed === true) {
-            throw new ClosedError();
-        }
-        // Total replication "width"
-        const width = 1;
-        // How much width you need to "query" to
-        const peers = this.replicationIndex; // TODO types
-        const minReplicas = Math.min(await peers.getSize(), this.replicas.min.getValue(this));
-        // If min replicas = 2
-        // then we need to make sure we cover 0.5 of the total 'width' of the replication space
-        // to make sure we reach sufficient amount of nodes such that at least one one has
-        // the entry we are looking for
-        const coveringWidth = width / minReplicas;
-        const set = await getCoverSet(coveringWidth, peers, roleAge, this.node.identity.publicKey);
-        // add all in flight
-        for (const [key, _] of this.syncInFlight) {
-            set.add(key);
-        }
-        return [...set];
+        return getSamples(cursor, this.replicationIndex, numberOfLeaders, roleAge);
     }
     async isReplicator(entry, options) {
-        return this.isLeader(entry.meta.gid, decodeReplicas(entry).getValue(this), options);
+        return this.isLeader(entry, decodeReplicas(entry).getValue(this), options);
     }
     async handleSubscriptionChange(publicKey, topics, subscribed) {
         for (const topic of topics) {
@@ -1178,7 +1371,7 @@ let SharedLog = class SharedLog extends Program {
             const replicationSegments = await this.getMyReplicationSegments();
             if (replicationSegments.length > 0) {
                 this.rpc
-                    .send(new ResponseReplicationInfoMessage({
+                    .send(new AllReplicatingSegmentsMessage({
                     segments: replicationSegments.map((x) => x.toReplicationRange()),
                 }), {
                     mode: new SilentDelivery({ redundancy: 1, to: [publicKey] }),
@@ -1187,7 +1380,7 @@ let SharedLog = class SharedLog extends Program {
                 if (this.v8Behaviour) {
                     // for backwards compatibility
                     this.rpc
-                        .send(new ResponseRoleMessage({ role: this.getRole() }), {
+                        .send(new ResponseRoleMessage({ role: await this.getRole() }), {
                         mode: new SilentDelivery({ redundancy: 1, to: [publicKey] }),
                     })
                         .catch((e) => logger.error(e.toString()));
@@ -1255,7 +1448,7 @@ let SharedLog = class SharedLog extends Program {
                     const minMinReplicasValue = this.replicas.max
                         ? Math.min(minReplicasValue, this.replicas.max.getValue(this))
                         : minReplicasValue;
-                    const leaders = await this.findLeaders(entry.meta.gid, minMinReplicasValue);
+                    const leaders = await this.findLeaders(entry, minMinReplicasValue);
                     if (leaders.find((x) => x === this.node.identity.publicKey.hashcode())) {
                         reject(new Error("Failed to delete, is leader"));
                         return;
@@ -1338,7 +1531,7 @@ let SharedLog = class SharedLog extends Program {
                 continue; // TODO maybe close store?
             }
             const oldPeersSet = this._gidPeersHistory.get(gid);
-            const currentPeers = await this.findLeaders(gid, maxReplicas(this, entries));
+            const currentPeers = await this.findLeaders(getLatestEntry(entries), maxReplicas(this, entries));
             const isLeader = currentPeers.find((x) => x === this.node.identity.publicKey.hashcode());
             const currentPeersSet = new Set(currentPeers);
             this._gidPeersHistory.set(gid, currentPeersSet);
@@ -1408,7 +1601,8 @@ let SharedLog = class SharedLog extends Program {
     }
     async _onUnsubscription(evt) {
         logger.debug(`Peer disconnected '${evt.detail.from.hashcode()}' from '${JSON.stringify(evt.detail.unsubscriptions.map((x) => x))}'`);
-        this.latestRoleMessages.delete(evt.detail.from.hashcode());
+        this.latestReplicationInfoMessage.delete(evt.detail.from.hashcode());
+        // TODO only emit this if the peer is actually replicating anything
         this.events.dispatchEvent(new CustomEvent("replicator:leave", {
             detail: { publicKey: evt.detail.from },
         }));
@@ -1450,10 +1644,10 @@ let SharedLog = class SharedLog extends Program {
             return false;
         }
         // The role is fixed (no changes depending on memory usage or peer count etc)
-        if (!this._replicationSettings) {
+        if (!this._isReplicating) {
             return false;
         }
-        if (isAdaptiveReplicatorOption(this._replicationSettings)) {
+        if (this._isAdaptiveReplicating) {
             const peers = this.replicationIndex;
             const usedMemory = await this.getMemoryUsage();
             let dynamicRange = await this.getDynamicRange();
@@ -1461,10 +1655,11 @@ let SharedLog = class SharedLog extends Program {
                 return; // not allowed to replicate
             }
             const peersSize = (await peers.getSize()) || 1;
+            const totalParticipation = await this.calculateTotalParticipation();
             const newFactor = this.replicationController.step({
                 memoryUsage: usedMemory,
                 currentFactor: dynamicRange.widthNormalized,
-                totalFactor: await this.calculateTotalParticipation(), // TODO use this._totalParticipation when flakiness is fixed
+                totalFactor: totalParticipation, // TODO use this._totalParticipation when flakiness is fixed
                 peerCount: peersSize,
                 cpuUsage: this.cpuUsage?.value(),
             });
@@ -1473,11 +1668,11 @@ let SharedLog = class SharedLog extends Program {
             if (relativeDifference > 0.0001) {
                 // TODO can not reuse old range, since it will (potentially) affect the index because of sideeffects
                 dynamicRange = new ReplicationRangeIndexable({
-                    offset: hashToUniformNumber(this.node.identity.publicKey.bytes),
-                    length: newFactor,
+                    offset: hashToU32(this.node.identity.publicKey.bytes),
+                    length: scaleToU32(newFactor),
                     publicKeyHash: dynamicRange.hash,
                     id: dynamicRange.id,
-                    replicationIntent: dynamicRange.replicationIntent,
+                    mode: dynamicRange.mode,
                     timestamp: dynamicRange.timestamp,
                 });
                 const canReplicate = !this._isTrustedReplicator ||
@@ -1485,7 +1680,10 @@ let SharedLog = class SharedLog extends Program {
                 if (!canReplicate) {
                     return false;
                 }
-                await this.startAnnounceReplicating(dynamicRange);
+                await this.startAnnounceReplicating([dynamicRange], {
+                    checkDuplicates: false,
+                    reset: false,
+                });
                 /* await this._updateRole(newRole, onRoleChange); */
                 this.rebalanceParticipationDebounced?.();
                 return true;
@@ -1498,31 +1696,30 @@ let SharedLog = class SharedLog extends Program {
         return false;
     }
     async getDynamicRange() {
+        let dynamicRangeId = sha256Sync(concat([
+            this.node.identity.publicKey.bytes,
+            new TextEncoder().encode("dynamic"),
+        ]));
         let range = (await this.replicationIndex.query(new SearchRequest({
             query: [
-                new StringMatch({
-                    key: "hash",
-                    value: this.node.identity.publicKey.hashcode(),
-                }),
-                new IntegerCompare({
-                    key: "replicationIntent",
-                    value: ReplicationIntent.Automatic,
-                    compare: "eq",
+                new ByteMatchQuery({
+                    key: "id",
+                    value: dynamicRangeId,
                 }),
             ],
             fetch: 1,
         })))?.results[0]?.value;
         if (!range) {
-            let seed = Math.random();
             range = new ReplicationRangeIndexable({
-                offset: seed,
+                normalized: true,
+                offset: Math.random(),
                 length: 0,
                 publicKeyHash: this.node.identity.publicKey.hashcode(),
-                replicationIntent: ReplicationIntent.Automatic,
+                mode: ReplicationIntent.NonStrict,
                 timestamp: BigInt(+new Date()),
-                id: sha256Sync(this.node.identity.publicKey.bytes),
+                id: dynamicRangeId,
             });
-            const added = await this.addReplicationRange(range, this.node.identity.publicKey);
+            const added = await this.addReplicationRange([range], this.node.identity.publicKey, { reset: false, checkDuplicates: false });
             if (!added) {
                 logger.warn("Not allowed to replicate by canReplicate");
                 return;