@peerbit/shared-log 8.0.7 → 9.0.0-55cebfe

package/{lib/esm → dist/src}/index.js

@@ -7,39 +7,41 @@ var __decorate = (this && this.__decorate) || function (decorators, target, key,
 var __metadata = (this && this.__metadata) || function (k, v) {
     if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
 };
-import { RPC } from "@peerbit/rpc";
-import { TransportMessage } from "./message.js";
-import { Entry, Log } from "@peerbit/log";
-import { Program } from "@peerbit/program";
 import { BinaryWriter, BorshError, field, variant } from "@dao-xyz/borsh";
-import { AccessError, sha256, sha256Base64Sync } from "@peerbit/crypto";
-import { logger as loggerFn } from "@peerbit/logger";
-import { ExchangeHeadsMessage, RequestIPrune, RequestMaybeSync, ResponseIPrune, ResponseMaybeSync, createExchangeHeadsMessages } from "./exchange-heads.js";
-import { AbortError, delay, waitFor } from "@peerbit/time";
-import { Observer, Replicator, Role } from "./role.js";
-import { AbsoluteReplicas, ReplicationError, RequestRoleMessage, ResponseRoleMessage, decodeReplicas, encodeReplicas, hashToUniformNumber, maxReplicas } from "./replication.js";
-import pDefer from "p-defer";
-import { Cache } from "@peerbit/cache";
 import { CustomEvent } from "@libp2p/interface";
-import yallist from "yallist";
-import { AcknowledgeDelivery, SilentDelivery, NotStartedError } from "@peerbit/stream-interface";
 import { AnyBlockStore, RemoteBlocks } from "@peerbit/blocks";
-import { BlocksMessage } from "./blocks.js";
+import { Cache } from "@peerbit/cache";
+import { AccessError, PublicSignKey, sha256, sha256Base64Sync, sha256Sync, } from "@peerbit/crypto";
+import { And, ByteMatchQuery, CountRequest, DeleteRequest, IntegerCompare, Or, SearchRequest, Sort, StringMatch, SumRequest, toId, } from "@peerbit/indexer-interface";
+import { Entry, Log, ShallowEntry, } from "@peerbit/log";
+import { logger as loggerFn } from "@peerbit/logger";
+import { Program } from "@peerbit/program";
+import { SubscriptionEvent, UnsubcriptionEvent, } from "@peerbit/pubsub-interface";
+import { RPC } from "@peerbit/rpc";
+import { AcknowledgeDelivery, DeliveryMode, NotStartedError, SilentDelivery, } from "@peerbit/stream-interface";
+import { AbortError, delay, waitFor } from "@peerbit/time";
 import debounce from "p-debounce";
-import { PIDReplicationController } from "./pid.js";
-export * from "./replication.js";
+import pDefer, {} from "p-defer";
 import PQueue from "p-queue";
+import { BlocksMessage } from "./blocks.js";
 import { CPUUsageIntervalLag } from "./cpu.js";
+import { EntryWithRefs, ExchangeHeadsMessage, RequestIPrune, RequestMaybeSync, ResponseIPrune, ResponseMaybeSync, createExchangeHeadsMessages, } from "./exchange-heads.js";
+import { TransportMessage } from "./message.js";
+import { PIDReplicationController } from "./pid.js";
 import { getCoverSet, getSamples, isMatured } from "./ranges.js";
+import { AbsoluteReplicas, ReplicationError, ReplicationIntent, ReplicationRange, ReplicationRangeIndexable, RequestReplicationInfoMessage, ResponseReplicationInfoMessage, StartedReplicating, StoppedReplicating, decodeReplicas, encodeReplicas, hashToUniformNumber, maxReplicas, } from "./replication.js";
+import { SEGMENT_COORDINATE_SCALE } from "./role.js";
+export * from "./replication.js";
 export { CPUUsageIntervalLag };
-export { Observer, Replicator, Role };
 export const logger = loggerFn({ module: "shared-log" });
 const groupByGid = async (entries) => {
     const groupByGid = new Map();
     for (const head of entries) {
         const gid = await (head instanceof Entry
             ? head.getGid()
-            : head.entry.getGid());
+            : head instanceof ShallowEntry
+                ? head.meta.gid
+                : head.entry.getGid());
         let value = groupByGid.get(gid);
         if (!value) {
             value = [];
@@ -50,11 +52,16 @@ const groupByGid = async (entries) => {
     return groupByGid;
 };
 const isAdaptiveReplicatorOption = (options) => {
-    if (options.limits ||
-        options.factor == null) {
-        return true;
+    if (typeof options === "number") {
+        return false;
+    }
+    if (typeof options === "boolean") {
+        return false;
     }
-    return false;
+    if (options.factor != null) {
+        return false;
+    }
+    return true;
 };
 export const DEFAULT_MIN_REPLICAS = 2;
 export const WAIT_FOR_REPLICATOR_TIMEOUT = 9000;
@@ -65,14 +72,13 @@ let SharedLog = class SharedLog extends Program {
     log;
     rpc;
     // options
-    _role;
-    _roleConfig;
-    _sortedPeersCache;
+    _replicationSettings;
+    _replicationRangeIndex;
     _totalParticipation;
     _gidPeersHistory;
     _onSubscriptionFn;
     _onUnsubscriptionFn;
-    _canReplicate;
+    _isTrustedReplicator;
     _logProperties;
     _closeController;
     _gidParentCache;
@@ -91,39 +97,70 @@ let SharedLog = class SharedLog extends Program {
     distributeQueue;
     // Syncing and dedeplucation work
     syncMoreInterval;
+    // map of hash to public keys that we can ask for entries
     syncInFlightQueue;
     syncInFlightQueueInverted;
+    // map of hash to public keys that we have asked for entries
     syncInFlight;
     replicas;
     cpuUsage;
     timeUntilRoleMaturity;
     waitForReplicatorTimeout;
     distributionDebounceTime;
+    replicationController;
+    history;
+    pq;
     constructor(properties) {
         super();
         this.log = new Log(properties);
         this.rpc = new RPC();
     }
-    /**
-     * Returns the current role
-     */
-    get role() {
-        return this._role;
-    }
     /**
      * Return the
      */
-    get roleConfig() {
-        return this._roleConfig;
+    get replicationSettings() {
+        return this._replicationSettings;
+    }
+    async isReplicating() {
+        if (!this._replicationSettings) {
+            return false;
+        }
+        if (isAdaptiveReplicatorOption(this._replicationSettings)) {
+            return true;
+        }
+        if (this.replicationSettings.factor > 0) {
+            return true;
+        }
+        return (await this.countReplicationSegments()) > 0;
     }
     get totalParticipation() {
         return this._totalParticipation;
     }
+    async calculateTotalParticipation() {
+        const sum = await this.replicationIndex.sum(new SumRequest({ key: "width" }));
+        return Number(sum) / SEGMENT_COORDINATE_SCALE;
+    }
+    async countReplicationSegments() {
+        const count = await this.replicationIndex.count(new CountRequest({
+            query: new StringMatch({
+                key: "hash",
+                value: this.node.identity.publicKey.hashcode(),
+            }),
+        }));
+        return count;
+    }
     setupRebalanceDebounceFunction() {
-        this.rebalanceParticipationDebounced = debounce(() => this.rebalanceParticipation(), Math.max(REBALANCE_DEBOUNCE_INTERVAL, Math.log((this.getReplicatorsSorted()?.length || 0) *
-            REBALANCE_DEBOUNCE_INTERVAL)));
+        this.rebalanceParticipationDebounced = debounce(() => this.rebalanceParticipation(), 
+        /* Math.max(
+            REBALANCE_DEBOUNCE_INTERVAL,
+            Math.log(
+                (this.getReplicatorsSorted()?.getSize() || 0) *
+                REBALANCE_DEBOUNCE_INTERVAL
+            )
+        ) */
+        REBALANCE_DEBOUNCE_INTERVAL);
     }
-    setupRole(options) {
+    async setupReplicationSettings(options) {
         this.rebalanceParticipationDebounced = undefined;
         const setupDebouncedRebalancing = (options) => {
             this.cpuUsage?.stop?.();
@@ -135,9 +172,9 @@ let SharedLog = class SharedLog extends Program {
                     ? {
                         max: typeof options?.limits?.cpu === "object"
                             ? options.limits.cpu.max
-                            : options?.limits?.cpu
+                            : options?.limits?.cpu,
                     }
-                    : undefined
+                    : undefined,
             });
             this.cpuUsage =
                 options?.limits?.cpu && typeof options?.limits?.cpu === "object"
@@ -146,75 +183,216 @@ let SharedLog = class SharedLog extends Program {
             this.cpuUsage?.start?.();
             this.setupRebalanceDebounceFunction();
         };
-        if (options instanceof Observer || options instanceof Replicator) {
-            throw new Error("Unsupported role option type");
-        }
-        else if (options === "observer") {
-            this._roleConfig = new Observer();
-        }
-        else if (options === "replicator") {
-            setupDebouncedRebalancing();
-            this._roleConfig = { type: options };
-        }
-        else if (options) {
-            if (options.type === "replicator") {
-                if (isAdaptiveReplicatorOption(options)) {
-                    setupDebouncedRebalancing(options);
-                    this._roleConfig = options;
+        if (options) {
+            if (isAdaptiveReplicatorOption(options)) {
+                this._replicationSettings = options;
+                setupDebouncedRebalancing(this._replicationSettings);
+            }
+            else if (options === true ||
+                (options && Object.keys(options).length === 0)) {
+                this._replicationSettings = {};
+                setupDebouncedRebalancing(this._replicationSettings);
+            }
+            else {
+                if (typeof options === "number") {
+                    this._replicationSettings = {
+                        factor: options,
+                    };
                 }
                 else {
-                    this._roleConfig = new Replicator({
-                        factor: options.factor,
-                        offset: this.getReplicationOffset()
-                    });
+                    this._replicationSettings = { ...options };
                 }
             }
-            else {
-                this._roleConfig = new Observer();
-            }
         }
         else {
-            // Default option
-            setupDebouncedRebalancing();
-            this._roleConfig = { type: "replicator" };
+            return;
         }
-        // setup the initial role
-        if (this._roleConfig instanceof Replicator ||
-            this._roleConfig instanceof Observer) {
-            this._role = this._roleConfig;
+        if (isAdaptiveReplicatorOption(this._replicationSettings)) {
+            // initial role in a dynamic setup
+            await this.getDynamicRange();
         }
         else {
-            // initial role in a dynamic setup
-            if (this._roleConfig?.limits) {
-                this._role = new Replicator({
-                    factor: this._role instanceof Replicator ? this._role.factor : 0,
-                    offset: this.getReplicationOffset()
-                });
+            // fixed
+            const range = new ReplicationRangeIndexable({
+                offset: this._replicationSettings.offset ??
+                    Math.random(),
+                length: this._replicationSettings.factor,
+                publicKeyHash: this.node.identity.publicKey.hashcode(),
+                replicationIntent: ReplicationIntent.Explicit, // automatic means that this range might be reused later for dynamic replication behaviour
+                timestamp: BigInt(+new Date()),
+                id: sha256Sync(this.node.identity.publicKey.bytes),
+            });
+            await this.startAnnounceReplicating(range);
+        }
+    }
+    async replicate(range) {
+        if (range === false || range === 0) {
+            this._replicationSettings = undefined;
+            await this.removeReplicator(this.node.identity.publicKey);
+        }
+        else {
+            await this.rpc.subscribe();
+            if (range instanceof ReplicationRange) {
+                this.oldestOpenTime = Math.min(Number(range.timestamp), this.oldestOpenTime);
+                await this.startAnnounceReplicating(range.toReplicationRangeIndexable(this.node.identity.publicKey));
             }
             else {
-                this._role = new Replicator({
-                    factor: this._role instanceof Replicator ? this._role.factor : 1,
-                    offset: this.getReplicationOffset()
-                });
+                await this.setupReplicationSettings(range ?? true);
             }
         }
-        return this._role;
+        // assume new role
+        await this.distribute();
     }
-    async updateRole(role, onRoleChange = true) {
-        return this._updateRole(this.setupRole(role), onRoleChange);
+    async removeReplicator(key) {
+        const fn = async () => {
+            let prev = await this.replicationIndex.query(new SearchRequest({
+                query: { hash: key.hashcode() },
+                fetch: 0xffffffff,
+            }), { reference: true });
+            if (prev.results.length === 0) {
+                return;
+            }
+            let sumWidth = prev.results.reduce((acc, x) => acc + x.value.widthNormalized, 0);
+            this._totalParticipation -= sumWidth;
+            let idMatcher = new Or(prev.results.map((x) => new ByteMatchQuery({ key: "id", value: x.value.id })));
+            await this.replicationIndex.del(new DeleteRequest({ query: idMatcher }));
+            const calculated = await this.calculateTotalParticipation();
+            if (Math.abs(this._totalParticipation - calculated) > 0.001) {
+                throw new Error("Total participation is out of sync");
+            }
+            await this.updateOldestTimestampFromIndex();
+            this.events.dispatchEvent(new CustomEvent("replication:change", {
+                detail: { publicKey: key },
+            }));
+            if (!key.equals(this.node.identity.publicKey)) {
+                this.rebalanceParticipationDebounced?.();
+            }
+        };
+        return this.pq.add(fn);
+    }
+    async updateOldestTimestampFromIndex() {
+        const oldestTimestampFromDB = (await this.replicationIndex.query(new SearchRequest({
+            fetch: 1,
+            sort: [new Sort({ key: "timestamp", direction: "asc" })],
+        }), { reference: true })).results[0]?.value.timestamp;
+        this.oldestOpenTime =
+            oldestTimestampFromDB != null
+                ? Number(oldestTimestampFromDB)
+                : +new Date();
+    }
+    async removeReplicationRange(id, from) {
+        const fn = async () => {
+            let idMatcher = new Or(id.map((x) => new ByteMatchQuery({ key: "id", value: x })));
+            // make sure we are not removing something that is owned by the replicator
+            let identityMatcher = new StringMatch({
+                key: "hash",
+                value: from.hashcode(),
+            });
+            let query = new And([idMatcher, identityMatcher]);
+            const prevSum = await this.replicationIndex.sum(new SumRequest({ query, key: "width" }));
+            const prevSumNormalized = Number(prevSum) / SEGMENT_COORDINATE_SCALE;
+            this._totalParticipation -= prevSumNormalized;
+            await this.replicationIndex.del(new DeleteRequest({ query }));
+            const calculated = await this.calculateTotalParticipation();
+            if (Math.abs(this._totalParticipation - calculated) > 0.001) {
+                throw new Error("Total participation is out of sync");
+            }
+            await this.updateOldestTimestampFromIndex();
+            this.events.dispatchEvent(new CustomEvent("replication:change", {
+                detail: { publicKey: from },
+            }));
+            if (!from.equals(this.node.identity.publicKey)) {
+                this.rebalanceParticipationDebounced?.();
+            }
+        };
+        return this.pq.add(fn);
     }
-    async _updateRole(role = this._role, onRoleChange = true) {
+    async addReplicationRange(range, from) {
+        const fn = async () => {
+            if (this._isTrustedReplicator &&
+                !(await this._isTrustedReplicator(from))) {
+                if (this.node.identity.publicKey.equals(from)) {
+                    if (range.replicationIntent === ReplicationIntent.Automatic) {
+                        return false; // we dont want to replicate automatic ranges if not allowed by others
+                    }
+                }
+                else {
+                    return false;
+                }
+            }
+            range.id = new Uint8Array(range.id);
+            let prevCount = await this.replicationIndex.count(new CountRequest({
+                query: new StringMatch({ key: "hash", value: from.hashcode() }),
+            }));
+            const isNewReplicator = prevCount === 0;
+            let prev = await this.replicationIndex.get(toId(range.id));
+            if (prev) {
+                if (prev.value.equals(range)) {
+                    return false;
+                }
+                this._totalParticipation -= prev.value.widthNormalized;
+            }
+            await this.replicationIndex.put(range);
+            let inserted = await this.replicationIndex.get(toId(range.id));
+            if (!inserted?.value.equals(range)) {
+                throw new Error("Failed to insert range");
+            }
+            this._totalParticipation += range.widthNormalized;
+            const calculated = await this.calculateTotalParticipation();
+            if (Math.abs(this._totalParticipation - calculated) > 0.001) {
+                throw new Error("Total participation is out of sync");
+            }
+            this.oldestOpenTime = Math.min(Number(range.timestamp), this.oldestOpenTime);
+            this.events.dispatchEvent(new CustomEvent("replication:change", {
+                detail: { publicKey: from },
+            }));
+            if (isNewReplicator) {
+                this.events.dispatchEvent(new CustomEvent("replicator:join", {
+                    detail: { publicKey: from },
+                }));
+            }
+            if (!from.equals(this.node.identity.publicKey)) {
+                this.rebalanceParticipationDebounced?.();
+            }
+            return true;
+        };
+        return this.pq.add(fn);
+    }
+    async startAnnounceReplicating(range) {
+        const added = await this.addReplicationRange(range, this.node.identity.publicKey);
+        if (!added) {
+            logger.warn("Not allowed to replicate by canReplicate");
+        }
+        added &&
+            (await this.rpc.send(new StartedReplicating({ segments: [range.toReplicationRange()] }), {
+                priority: 1,
+            }));
+    }
+    /* async updateRole(role: InitialReplicationOptions, onRoleChange = true) {
+        await this.setupReplicationSettings(role)
+        return this._updateRole(, onRoleChange);
+    } */
+    /* private async _updateRole(
+        role: Observer | Replicator = this._role,
+        onRoleChange = true
+    ) {
         this._role = role;
-        const { changed } = await this._modifyReplicators(this.role, this.node.identity.publicKey);
+        const { changed } = await this._modifyReplicators(
+            this.role,
+            this.node.identity.publicKey
+        );
+
         await this.rpc.subscribe();
-        await this.rpc.send(new ResponseRoleMessage({ role: this._role }), {
+        await this.rpc.send(new ResponseReplicationInfoMessage({ segments: await  }), {
             priority: 1
         });
+
         if (onRoleChange && changed !== "none") {
-            this.onRoleChange(this._role, this.node.identity.publicKey);
+            await this.onRoleChange(this._role, this.node.identity.publicKey);
         }
+
         return changed;
-    }
+    } */
     async append(data, options) {
         const appendOptions = { ...options };
         const minReplicasData = encodeReplicas(options?.replicas
@@ -224,12 +402,24 @@ let SharedLog = class SharedLog extends Program {
             : this.replicas.min);
         if (!appendOptions.meta) {
             appendOptions.meta = {
-                data: minReplicasData
+                data: minReplicasData,
             };
         }
         else {
             appendOptions.meta.data = minReplicasData;
         }
+        if (options?.canAppend) {
+            appendOptions.canAppend = async (entry) => {
+                await this.canAppend(entry);
+                return options.canAppend(entry);
+            };
+        }
+        if (options?.onChange) {
+            appendOptions.onChange = async (change) => {
+                await this.onChange(change);
+                return options.onChange(change);
+            };
+        }
         const result = await this.log.append(data, appendOptions);
         let mode = undefined;
         for (const message of await createExchangeHeadsMessages(this.log, [result.entry], this._gidParentCache)) {
@@ -262,7 +452,7 @@ let SharedLog = class SharedLog extends Program {
             }
             // TODO add options for waiting ?
             this.rpc.send(message, {
-                mode
+                mode,
             });
         }
         this.rebalanceParticipationDebounced?.();
@@ -279,7 +469,7 @@ let SharedLog = class SharedLog extends Program {
                 ? typeof options?.replicas?.max === "number"
                     ? new AbsoluteReplicas(options?.replicas?.max)
                     : options.replicas.max
-                : undefined
+                : undefined,
         };
         this._respondToIHaveTimeout = options?.respondToIHaveTimeout ?? 10 * 1000; // TODO make into arg
         this._pendingDeletes = new Map();
@@ -291,17 +481,17 @@ let SharedLog = class SharedLog extends Program {
         this.openTime = +new Date();
         this.oldestOpenTime = this.openTime;
         this.distributionDebounceTime =
-            options?.distributionDebounceTime || DEFAULT_DISTRIBUTION_DEBOUNCE_TIME;
+            options?.distributionDebounceTime || DEFAULT_DISTRIBUTION_DEBOUNCE_TIME; // expect > 0
         this.timeUntilRoleMaturity =
-            options?.timeUntilRoleMaturity || WAIT_FOR_ROLE_MATURITY;
+            options?.timeUntilRoleMaturity ?? WAIT_FOR_ROLE_MATURITY;
         this.waitForReplicatorTimeout =
             options?.waitForReplicatorTimeout || WAIT_FOR_REPLICATOR_TIMEOUT;
-        this._gidParentCache = new Cache({ max: 1000 });
+        this._gidParentCache = new Cache({ max: 100 }); // TODO choose a good number
         this._closeController = new AbortController();
-        this._canReplicate = options?.canReplicate;
+        this._isTrustedReplicator = options?.canReplicate;
         this.sync = options?.sync;
         this._logProperties = options;
-        this.setupRole(options?.role);
+        this.pq = new PQueue({ concurrency: 1000 });
         const id = sha256Base64Sync(this.log.id);
         const storage = await this.node.storage.sublevel(id);
         const localBlocks = await new AnyBlockStore(await storage.sublevel("blocks"));
@@ -310,76 +500,60 @@ let SharedLog = class SharedLog extends Program {
             publish: (message, options) => this.rpc.send(new BlocksMessage(message), {
                 mode: options?.to
                     ? new SilentDelivery({ to: options.to, redundancy: 1 })
-                    : undefined
+                    : undefined,
             }),
-            waitFor: this.rpc.waitFor.bind(this.rpc)
+            waitFor: this.rpc.waitFor.bind(this.rpc),
         });
         await this.remoteBlocks.start();
-        this._onSubscriptionFn = this._onSubscription.bind(this);
         this._totalParticipation = 0;
-        this._sortedPeersCache = yallist.create();
+        const logScope = await this.node.indexer.scope(id);
+        const replicationIndex = await logScope.scope("replication");
+        this._replicationRangeIndex = await replicationIndex.init({
+            schema: ReplicationRangeIndexable,
+        });
+        const logIndex = await logScope.scope("log");
+        await this.node.indexer.start(); // TODO why do we need to start the indexer here?
+        this._totalParticipation = await this.calculateTotalParticipation();
         this._gidPeersHistory = new Map();
-        const cache = await storage.sublevel("cache");
         await this.log.open(this.remoteBlocks, this.node.identity, {
             keychain: this.node.services.keychain,
             ...this._logProperties,
             onChange: (change) => {
-                for (const added of change.added) {
-                    this.onEntryAdded(added);
-                }
-                for (const removed of change.removed) {
-                    this.onEntryRemoved(removed.hash);
-                }
+                this.onChange(change);
                 return this._logProperties?.onChange?.(change);
             },
             canAppend: async (entry) => {
-                try {
-                    if (!entry.meta.data) {
-                        logger.warn("Received entry without meta data, skipping");
-                        return false;
-                    }
-                    const replicas = decodeReplicas(entry).getValue(this);
-                    if (Number.isFinite(replicas) === false) {
-                        return false;
-                    }
-                    // Don't verify entries that we have created (TODO should we? perf impact?)
-                    if (!entry.createdLocally && !(await entry.verifySignatures())) {
-                        return false;
-                    }
-                }
-                catch (error) {
-                    if (error instanceof BorshError ||
-                        error instanceof ReplicationError) {
-                        logger.warn("Received payload that could not be decoded, skipping");
-                        return false;
-                    }
-                    throw error;
+                if (!(await this.canAppend(entry))) {
+                    return false;
                 }
                 return this._logProperties?.canAppend?.(entry) ?? true;
             },
             trim: this._logProperties?.trim && {
-                ...this._logProperties?.trim
+                ...this._logProperties?.trim,
             },
-            cache: cache
+            indexer: logIndex,
         });
         // Open for communcation
         await this.rpc.open({
             queryType: TransportMessage,
             responseType: TransportMessage,
             responseHandler: this._onMessage.bind(this),
-            topic: this.topic
+            topic: this.topic,
         });
+        this._onSubscriptionFn =
+            this._onSubscriptionFn || this._onSubscription.bind(this);
         await this.node.services.pubsub.addEventListener("subscribe", this._onSubscriptionFn);
-        this._onUnsubscriptionFn = this._onUnsubscription.bind(this);
+        this._onUnsubscriptionFn =
+            this._onUnsubscriptionFn || this._onUnsubscription.bind(this);
         await this.node.services.pubsub.addEventListener("unsubscribe", this._onUnsubscriptionFn);
-        await this.log.load();
+        // await this.log.load();
         // TODO (do better)
         // we do this distribution interval to eliminate the sideeffects arriving from updating roles and joining entries continously.
         // an alternative to this would be to call distribute/maybe prune after every join if our role has changed
         this.distributeInterval = setInterval(() => {
             this.distribute();
         }, 7.5 * 1000);
-        const requestSync = () => {
+        const requestSync = async () => {
             /**
              * This method fetches entries that we potentially want.
              * In a case in which we become replicator of a segment,
@@ -390,7 +564,7 @@ let SharedLog = class SharedLog extends Program {
             const requestHashes = [];
             const from = new Set();
             for (const [key, value] of this.syncInFlightQueue) {
-                if (!this.log.has(key)) {
+                if (!(await this.log.has(key))) {
                     // TODO test that this if statement actually does anymeaningfull
                     if (value.length > 0) {
                         requestHashes.push(key);
@@ -423,12 +597,13 @@ let SharedLog = class SharedLog extends Program {
                 this.syncMoreInterval = setTimeout(requestSync, 1e4);
             });
         };
+        await this.replicate(options?.replicate);
         requestSync();
     }
     async afterOpen() {
         await super.afterOpen();
         // We do this here, because these calls requires this.closed == false
-        await this._updateRole();
+        /* await this._updateRole(); */
         await this.rebalanceParticipation();
         // Take into account existing subscription
         (await this.node.services.pubsub.getSubscribers(this.topic))?.forEach((v, k) => {
@@ -438,25 +613,60 @@ let SharedLog = class SharedLog extends Program {
             this.handleSubscriptionChange(v, [this.topic], true);
         });
     }
+    async reload() {
+        await this.log.load({ reset: true, reload: true });
+    }
     async getMemoryUsage() {
-        return (((await this.log.memory?.size()) || 0) + (await this.log.blocks.size()));
+        return this.log.blocks.size();
+        /* ((await this.log.entryIndex?.getMemoryUsage()) || 0) */ // + (await this.log.blocks.size())
     }
     get topic() {
         return this.log.idString;
     }
+    async onChange(change) {
+        for (const added of change.added) {
+            this.onEntryAdded(added);
+        }
+        for (const removed of change.removed) {
+            this.onEntryRemoved(removed.hash);
+        }
+    }
+    async canAppend(entry) {
+        try {
+            if (!entry.meta.data) {
+                logger.warn("Received entry without meta data, skipping");
+                return false;
+            }
+            const replicas = decodeReplicas(entry).getValue(this);
+            if (Number.isFinite(replicas) === false) {
+                return false;
+            }
+            // Don't verify entries that we have created (TODO should we? perf impact?)
+            if (!entry.createdLocally && !(await entry.verifySignatures())) {
+                return false;
+            }
+            return true;
+        }
+        catch (error) {
+            if (error instanceof BorshError || error instanceof ReplicationError) {
+                logger.warn("Received payload that could not be decoded, skipping");
+                return false;
+            }
+            throw error;
+        }
+    }
     async _close() {
         clearTimeout(this.syncMoreInterval);
         clearInterval(this.distributeInterval);
         this.distributeQueue?.clear();
         this._closeController.abort();
         this.node.services.pubsub.removeEventListener("subscribe", this._onSubscriptionFn);
-        this._onUnsubscriptionFn = this._onUnsubscription.bind(this);
         this.node.services.pubsub.removeEventListener("unsubscribe", this._onUnsubscriptionFn);
-        for (const [k, v] of this._pendingDeletes) {
+        for (const [_k, v] of this._pendingDeletes) {
             v.clear();
             v.promise.resolve(); // TODO or reject?
         }
-        for (const [k, v] of this._pendingIHave) {
+        for (const [_k, v] of this._pendingIHave) {
             v.clear();
         }
         await this.remoteBlocks.stop();
@@ -468,8 +678,10 @@ let SharedLog = class SharedLog extends Program {
         this.syncInFlight.clear();
         this.latestRoleMessages.clear();
         this._gidPeersHistory.clear();
-        this._sortedPeersCache = undefined;
+        this._replicationRangeIndex = undefined;
         this.cpuUsage?.stop?.();
+        this._totalParticipation = 0;
+        this.pq.clear();
     }
     async close(from) {
         const superClosed = await super.close(from);
@@ -508,11 +720,11 @@ let SharedLog = class SharedLog extends Program {
                 if (heads) {
                     const filteredHeads = [];
                     for (const head of heads) {
-                        if (!this.log.has(head.entry.hash)) {
+                        if (!(await this.log.has(head.entry.hash))) {
                             head.entry.init({
                                 // we need to init because we perhaps need to decrypt gid
                                 keychain: this.log.keychain,
-                                encoding: this.log.encoding
+                                encoding: this.log.encoding,
                             });
                             filteredHeads.push(head);
                         }
@@ -527,17 +739,26 @@ let SharedLog = class SharedLog extends Program {
                     const promises = [];
                     for (const [gid, entries] of groupedByGid) {
                         const fn = async () => {
-                            const headsWithGid = this.log.headsIndex.gids.get(gid);
-                            const maxReplicasFromHead = headsWithGid && headsWithGid.size > 0
+                            const headsWithGid = await this.log.entryIndex
+                                .getHeads(gid)
+                                .all();
+                            const maxReplicasFromHead = headsWithGid && headsWithGid.length > 0
                                 ? maxReplicas(this, [...headsWithGid.values()])
                                 : this.replicas.min.getValue(this);
                             const maxReplicasFromNewEntries = maxReplicas(this, entries.map((x) => x.entry));
-                            const leaders = await (this.role instanceof Observer
-                                ? this.findLeaders(gid, Math.max(maxReplicasFromHead, maxReplicasFromNewEntries))
-                                : this.waitForIsLeader(gid, Math.max(maxReplicasFromHead, maxReplicasFromNewEntries)));
-                            const isLeader = !!leaders;
+                            const isReplicating = await this.isReplicating();
+                            let isLeader;
+                            if (isReplicating) {
+                                isLeader = await this.waitForIsLeader(gid, Math.max(maxReplicasFromHead, maxReplicasFromNewEntries));
+                            }
+                            else {
+                                isLeader = await this.findLeaders(gid, Math.max(maxReplicasFromHead, maxReplicasFromNewEntries));
+                                isLeader = isLeader.includes(this.node.identity.publicKey.hashcode())
+                                    ? isLeader
+                                    : false;
+                            }
                             if (isLeader) {
-                                if (leaders.find((x) => x === context.from.hashcode())) {
+                                if (isLeader.find((x) => x === context.from.hashcode())) {
                                     let peerSet = this._gidPeersHistory.get(gid);
                                     if (!peerSet) {
                                         peerSet = new Set();
@@ -555,8 +776,8 @@ let SharedLog = class SharedLog extends Program {
                                 }
                                 else {
                                     for (const ref of entry.gidRefrences) {
-                                        const map = this.log.headsIndex.gids.get(ref);
-                                        if (map && map.size > 0) {
+                                        const map = await this.log.entryIndex.getHeads(ref).all();
+                                        if (map && map.length > 0) {
                                             toMerge.push(entry.entry);
                                             (toDelete || (toDelete = [])).push(entry.entry);
                                             continue outer;
@@ -592,8 +813,10 @@ let SharedLog = class SharedLog extends Program {
                     }
                     if (maybeDelete) {
                         for (const entries of maybeDelete) {
-                            const headsWithGid = this.log.headsIndex.gids.get(entries[0].entry.meta.gid);
-                            if (headsWithGid && headsWithGid.size > 0) {
+                            const headsWithGid = await this.log.entryIndex
+                                .getHeads(entries[0].entry.meta.gid)
+                                .all();
+                            if (headsWithGid && headsWithGid.length > 0) {
                                 const minReplicas = maxReplicas(this, headsWithGid.values());
                                 const isLeader = await this.isLeader(entries[0].entry.meta.gid, minReplicas);
                                 if (!isLeader) {
@@ -609,11 +832,11 @@ let SharedLog = class SharedLog extends Program {
             else if (msg instanceof RequestIPrune) {
                 const hasAndIsLeader = [];
                 for (const hash of msg.hashes) {
-                    const indexedEntry = this.log.entryIndex.getShallow(hash);
+                    const indexedEntry = await this.log.entryIndex.getShallow(hash);
                     if (indexedEntry &&
-                        (await this.isLeader(indexedEntry.meta.gid, decodeReplicas(indexedEntry).getValue(this)))) {
+                        (await this.isLeader(indexedEntry.value.meta.gid, decodeReplicas(indexedEntry.value).getValue(this)))) {
                         this._gidPeersHistory
-                            .get(indexedEntry.meta.gid)
+                            .get(indexedEntry.value.meta.gid)
                             ?.delete(context.from.hashcode());
                         hasAndIsLeader.push(hash);
                     }
@@ -632,13 +855,13 @@ let SharedLog = class SharedLog extends Program {
                                     this.rpc.send(new ResponseIPrune({ hashes: [entry.hash] }), {
                                         mode: new SilentDelivery({
                                             to: [context.from],
-                                            redundancy: 1
-                                        })
+                                            redundancy: 1,
+                                        }),
                                     });
                                 }
                                 prevPendingIHave && prevPendingIHave.callback(entry);
                                 this._pendingIHave.delete(entry.hash);
-                            }
+                            },
                         };
                         const timeout = setTimeout(() => {
                             const pendingIHaveRef = this._pendingIHave.get(hash);
@@ -650,7 +873,7 @@ let SharedLog = class SharedLog extends Program {
                     }
                 }
                 await this.rpc.send(new ResponseIPrune({ hashes: hasAndIsLeader }), {
-                    mode: new SilentDelivery({ to: [context.from], redundancy: 1 })
+                    mode: new SilentDelivery({ to: [context.from], redundancy: 1 }),
                 });
             }
             else if (msg instanceof ResponseIPrune) {
@@ -671,7 +894,7 @@ let SharedLog = class SharedLog extends Program {
                         }
                         inverted.add(hash);
                     }
-                    else if (!this.log.has(hash)) {
+                    else if (!(await this.log.has(hash))) {
                         this.syncInFlightQueue.set(hash, []);
                         requestHashes.push(hash); // request immediately (first time we have seen this hash)
                     }
@@ -687,22 +910,25 @@ let SharedLog = class SharedLog extends Program {
                 let p = Promise.resolve();
                 for (const message of messages) {
                     p = p.then(() => this.rpc.send(message, {
-                        mode: new SilentDelivery({ to: [context.from], redundancy: 1 })
+                        mode: new SilentDelivery({ to: [context.from], redundancy: 1 }),
                     })); // push in series, if one fails, then we should just stop
                 }
             }
             else if (msg instanceof BlocksMessage) {
                 await this.remoteBlocks.onMessage(msg.message);
             }
-            else if (msg instanceof RequestRoleMessage) {
+            else if (msg instanceof RequestReplicationInfoMessage) {
                 if (context.from.equals(this.node.identity.publicKey)) {
                     return;
                 }
-                await this.rpc.send(new ResponseRoleMessage({ role: this.role }), {
-                    mode: new SilentDelivery({ to: [context.from], redundancy: 1 })
+                await this.rpc.send(new ResponseReplicationInfoMessage({
+                    segments: (await this.getMyReplicationSegments()).map((x) => x.toReplicationRange()),
+                }), {
+                    mode: new SilentDelivery({ to: [context.from], redundancy: 1 }),
                 });
             }
-            else if (msg instanceof ResponseRoleMessage) {
+            else if (msg instanceof ResponseReplicationInfoMessage ||
+                msg instanceof StartedReplicating) {
                 if (context.from.equals(this.node.identity.publicKey)) {
                     return;
                 }
@@ -710,7 +936,7 @@ let SharedLog = class SharedLog extends Program {
                 // but we don't know them as "subscribers" yet. i.e. they are not online
                 this.waitFor(context.from, {
                     signal: this._closeController.signal,
-                    timeout: this.waitForReplicatorTimeout
+                    timeout: this.waitForReplicatorTimeout,
                 })
                     .then(async () => {
                     // peer should not be online (for us)
@@ -719,7 +945,18 @@ let SharedLog = class SharedLog extends Program {
                         return;
                     }
                     this.latestRoleMessages.set(context.from.hashcode(), context.timestamp);
-                    await this.modifyReplicators(msg.role, context.from);
+                    if (msg instanceof ResponseReplicationInfoMessage) {
+                        await this.removeReplicator(context.from);
+                    }
+                    let addedOnce = false;
+                    for (const segment of msg.segments) {
+                        const added = await this.addReplicationRange(segment.toReplicationRangeIndexable(context.from), context.from);
+                        if (typeof added === "boolean") {
+                            addedOnce = addedOnce || added;
+                        }
+                    }
+                    addedOnce && (await this.distribute());
+                    /* await this._modifyReplicators(msg.role, context.from!); */
                 })
                     .catch((e) => {
                     if (e instanceof AbortError) {
@@ -728,9 +965,16 @@ let SharedLog = class SharedLog extends Program {
                     if (e instanceof NotStartedError) {
                         return;
                     }
-                    logger.error("Failed to find peer who updated their role: " + e?.message);
+                    logger.error("Failed to find peer who updated replication settings: " +
+                        e?.message);
                 });
             }
+            else if (msg instanceof StoppedReplicating) {
+                if (context.from.equals(this.node.identity.publicKey)) {
+                    return;
+                }
+                await this.removeReplicationRange(msg.segmentIds, context.from);
+            }
             else {
                 throw new Error("Unexpected message");
             }
@@ -750,24 +994,55 @@ let SharedLog = class SharedLog extends Program {
             logger.error(e);
         }
     }
-    getReplicatorsSorted() {
-        return this._sortedPeersCache;
+    async getMyReplicationSegments() {
+        const ranges = await this.replicationIndex.query(new SearchRequest({
+            query: [
+                new StringMatch({
+                    key: "hash",
+                    value: this.node.identity.publicKey.hashcode(),
+                }),
+            ],
+            fetch: 0xffffffff,
+        }));
+        return ranges.results.map((x) => x.value);
+    }
+    async getTotalParticipation() {
+        // sum all of my replicator rects
+        return (await this.getMyReplicationSegments()).reduce((acc, { widthNormalized }) => acc + widthNormalized, 0);
+    }
+    get replicationIndex() {
+        if (!this._replicationRangeIndex) {
+            throw new Error("Not open");
+        }
+        return this._replicationRangeIndex;
+    }
+    /**
+     * TODO improve efficiency
+     */
+    async getReplicators() {
+        let set = new Set();
+        const results = await this.replicationIndex.query(new SearchRequest({ fetch: 0xfffffff }), { reference: true, shape: { hash: true } });
+        results.results.forEach((result) => {
+            set.add(result.value.hash);
+        });
+        return set;
     }
     async waitForReplicator(...keys) {
-        const check = () => {
+        const check = async () => {
             for (const k of keys) {
-                const rect = this.getReplicatorsSorted()
-                    ?.toArray()
-                    ?.find((x) => x.publicKey.equals(k));
+                const rects = await this.replicationIndex?.query(new SearchRequest({
+                    query: [new StringMatch({ key: "hash", value: k.hashcode() })],
+                }), { reference: true });
+                const rect = await rects.results[0]?.value;
                 if (!rect ||
-                    !isMatured(rect.role, +new Date(), this.getDefaultMinRoleAge())) {
+                    !isMatured(rect, +new Date(), await this.getDefaultMinRoleAge())) {
                     return false;
                 }
             }
             return true;
         };
         return waitFor(() => check(), {
-            signal: this._closeController.signal
+            signal: this._closeController.signal,
         }).catch((e) => {
             if (e instanceof AbortError) {
                 // ignore error
@@ -780,36 +1055,33 @@ let SharedLog = class SharedLog extends Program {
         const isLeader = (await this.findLeaders(slot, numberOfLeaders, options)).find((l) => l === this.node.identity.publicKey.hashcode());
         return !!isLeader;
     }
-    getReplicationOffset() {
-        return hashToUniformNumber(this.node.identity.publicKey.bytes);
-    }
     async waitForIsLeader(slot, numberOfLeaders, timeout = this.waitForReplicatorTimeout) {
-        return new Promise((res, rej) => {
+        return new Promise((resolve, reject) => {
             const removeListeners = () => {
-                this.events.removeEventListener("role", roleListener);
+                this.events.removeEventListener("replication:change", roleListener);
                 this._closeController.signal.addEventListener("abort", abortListener);
             };
             const abortListener = () => {
                 removeListeners();
                 clearTimeout(timer);
-                res(false);
+                resolve(false);
             };
             const timer = setTimeout(() => {
                 removeListeners();
-                res(false);
+                resolve(false);
             }, timeout);
             const check = () => this.findLeaders(slot, numberOfLeaders).then((leaders) => {
                 const isLeader = leaders.find((l) => l === this.node.identity.publicKey.hashcode());
                 if (isLeader) {
                     removeListeners();
                     clearTimeout(timer);
-                    res(leaders);
+                    resolve(leaders);
                 }
             });
             const roleListener = () => {
                 check();
             };
-            this.events.addEventListener("role", roleListener);
+            this.events.addEventListener("replication:change", roleListener); // TODO replication:change event  ?
             this._closeController.signal.addEventListener("abort", abortListener);
             check();
         });
@@ -829,179 +1101,51 @@ let SharedLog = class SharedLog extends Program {
         const cursor = hashToUniformNumber(seed); // bounded between 0 and 1
         return this.findLeadersFromUniformNumber(cursor, numberOfLeaders, options);
     }
-    getDefaultMinRoleAge() {
+    async getDefaultMinRoleAge() {
+        if ((await this.isReplicating()) === false) {
+            return 0;
+        }
         const now = +new Date();
-        const replLength = this.getReplicatorsSorted().length;
+        const replLength = await this.replicationIndex.getSize();
         const diffToOldest = replLength > 1 ? now - this.oldestOpenTime - 1 : Number.MAX_SAFE_INTEGER;
-        return Math.min(this.timeUntilRoleMaturity, diffToOldest, (this.timeUntilRoleMaturity * Math.log(replLength)) / 3); // / 3 so that if 2 replicators and timeUntilRoleMaturity = 1e4 the result will be 1
+        return Math.min(this.timeUntilRoleMaturity, diffToOldest, Math.round((this.timeUntilRoleMaturity * Math.log(replLength + 1)) / 3)); // / 3 so that if 2 replicators and timeUntilRoleMaturity = 1e4 the result will be 1
     }
-    findLeadersFromUniformNumber(cursor, numberOfLeaders, options) {
-        const roleAge = options?.roleAge ?? this.getDefaultMinRoleAge(); // TODO -500 as is added so that i f someone else is just as new as us, then we treat them as mature as us. without -500 we might be slower syncing if two nodes starts almost at the same time
-        const sampes = getSamples(cursor, this.getReplicatorsSorted(), numberOfLeaders, roleAge, this.role instanceof Replicator && this.role.factor === 0
-            ? this.node.identity.publicKey.hashcode()
-            : undefined);
-        return sampes;
+    async findLeadersFromUniformNumber(cursor, numberOfLeaders, options) {
+        const roleAge = options?.roleAge ?? (await this.getDefaultMinRoleAge()); // TODO -500 as is added so that i f someone else is just as new as us, then we treat them as mature as us. without -500 we might be slower syncing if two nodes starts almost at the same time
+        const samples = await getSamples(cursor, this.replicationIndex, numberOfLeaders, roleAge);
+        return samples;
     }
     /**
      *
      * @returns groups where at least one in any group will have the entry you are looking for
      */
-    getReplicatorUnion(roleAge = this.getDefaultMinRoleAge()) {
+    async getReplicatorUnion(roleAge) {
+        roleAge = roleAge ?? (await this.getDefaultMinRoleAge());
         if (this.closed === true) {
             throw new Error("Closed");
         }
         // Total replication "width"
-        const width = 1; //this.getParticipationSum(roleAge);
+        const width = 1;
         // How much width you need to "query" to
-        const peers = this.getReplicatorsSorted(); // TODO types
-        const minReplicas = Math.min(peers.length, this.replicas.min.getValue(this));
+        const peers = this.replicationIndex; // TODO types
+        const minReplicas = Math.min(await peers.getSize(), this.replicas.min.getValue(this));
         // If min replicas = 2
         // then we need to make sure we cover 0.5 of the total 'width' of the replication space
         // to make sure we reach sufficient amount of nodes such that at least one one has
         // the entry we are looking for
         const coveringWidth = width / minReplicas;
-        const set = getCoverSet(coveringWidth, peers, roleAge, this.role instanceof Replicator ? this.node.identity.publicKey : undefined);
+        const set = await getCoverSet(coveringWidth, peers, roleAge, this.node.identity.publicKey);
         // add all in flight
         for (const [key, _] of this.syncInFlight) {
             set.add(key);
         }
         return [...set];
     }
-    async replicator(entry, options) {
+    async isReplicator(entry, options) {
         return this.isLeader(entry.gid, decodeReplicas(entry).getValue(this), options);
     }
-    onRoleChange(role, publicKey) {
-        if (this.closed) {
-            return;
-        }
-        this.distribute();
-        if (role instanceof Replicator) {
-            const timer = setTimeout(async () => {
-                this._closeController.signal.removeEventListener("abort", listener);
-                await this.rebalanceParticipationDebounced?.();
-                this.distribute();
-            }, this.getDefaultMinRoleAge() + 100);
-            const listener = () => {
-                clearTimeout(timer);
-                this._closeController.signal.removeEventListener("abort", listener);
-            };
-            this._closeController.signal.addEventListener("abort", listener);
-        }
-        this.events.dispatchEvent(new CustomEvent("role", {
-            detail: { publicKey, role }
-        }));
-    }
-    async modifyReplicators(role, publicKey) {
-        const update = await this._modifyReplicators(role, publicKey);
-        if (update.changed !== "none") {
-            if (update.changed === "added" || update.changed === "removed") {
-                this.setupRebalanceDebounceFunction();
-            }
-            await this.rebalanceParticipationDebounced?.(); /* await this.rebalanceParticipation(false); */
-            if (update.changed === "added") {
-                // TODO this message can be redudant, only send this when necessary (see conditions when rebalanceParticipation sends messages)
-                await this.rpc.send(new ResponseRoleMessage({ role: this._role }), {
-                    mode: new SilentDelivery({
-                        to: [publicKey.hashcode()],
-                        redundancy: 1
-                    }),
-                    priority: 1
-                });
-            }
-            this.onRoleChange(role, publicKey);
-            return true;
-        }
-        return false;
-    }
-    async _modifyReplicators(role, publicKey) {
-        // TODO can this call create race condition? _modifyReplicators might have to be queued
-        // TODO should we remove replicators if they are already added?
-        if (role instanceof Replicator &&
-            this._canReplicate &&
-            !(await this._canReplicate(publicKey, role))) {
-            return { changed: "none" };
-        }
-        const sortedPeer = this._sortedPeersCache;
-        if (!sortedPeer) {
-            if (this.closed === false) {
-                throw new Error("Unexpected, sortedPeersCache is undefined");
-            }
-            return { changed: "none" };
-        }
-        if (role instanceof Replicator) {
-            // TODO use Set + list for fast lookup
-            // check also that peer is online
-            const isOnline = this.node.identity.publicKey.equals(publicKey) ||
-                (await this.getReady()).has(publicKey.hashcode());
-            if (!isOnline) {
-                // TODO should we remove replicators if they are already added?
-                return { changed: "none" };
-            }
-            this.oldestOpenTime = Math.min(this.oldestOpenTime, Number(role.timestamp));
-            // insert or if already there do nothing
-            const rect = {
-                publicKey,
-                role
-            };
-            let currentNode = sortedPeer.head;
-            if (!currentNode) {
-                sortedPeer.push(rect);
-                this._totalParticipation += rect.role.factor;
-                return { changed: "added" };
-            }
-            else {
-                while (currentNode) {
-                    if (currentNode.value.publicKey.equals(publicKey)) {
-                        // update the value
-                        // rect.timestamp = currentNode.value.timestamp;
-                        const prev = currentNode.value;
-                        currentNode.value = rect;
-                        this._totalParticipation += rect.role.factor;
-                        this._totalParticipation -= prev.role.factor;
-                        // TODO change detection and only do change stuff if diff?
-                        return { prev: prev.role, changed: "updated" };
-                    }
-                    if (role.offset > currentNode.value.role.offset) {
-                        const next = currentNode?.next;
-                        if (next) {
-                            currentNode = next;
-                            continue;
-                        }
-                        else {
-                            break;
-                        }
-                    }
-                    else {
-                        currentNode = currentNode.prev;
-                        break;
-                    }
-                }
-                const prev = currentNode;
-                if (!prev?.next?.value.publicKey.equals(publicKey)) {
-                    this._totalParticipation += rect.role.factor;
-                    _insertAfter(sortedPeer, prev || undefined, rect);
-                }
-                else {
-                    throw new Error("Unexpected");
-                }
-                return { changed: "added" };
-            }
-        }
-        else {
-            let currentNode = sortedPeer.head;
-            while (currentNode) {
-                if (currentNode.value.publicKey.equals(publicKey)) {
-                    sortedPeer.removeNode(currentNode);
-                    this._totalParticipation -= currentNode.value.role.factor;
-                    return { prev: currentNode.value.role, changed: "removed" };
-                }
-                currentNode = currentNode.next;
-            }
-            return { changed: "none" };
-        }
-    }
-    async handleSubscriptionChange(publicKey, changes, subscribed) {
-        for (const topic of changes) {
+    async handleSubscriptionChange(publicKey, topics, subscribed) {
+        for (const topic of topics) {
             if (this.log.idString !== topic) {
                 continue;
             }
@@ -1026,16 +1170,19 @@ let SharedLog = class SharedLog extends Program {
             this.syncInFlightQueueInverted.delete(publicKey.hashcode());
         }
         if (subscribed) {
-            if (this.role instanceof Replicator) {
+            const replicationSegments = await this.getMyReplicationSegments();
+            if (replicationSegments.length > 0) {
                 this.rpc
-                    .send(new ResponseRoleMessage({ role: this._role }), {
-                    mode: new SilentDelivery({ redundancy: 1, to: [publicKey] })
+                    .send(new ResponseReplicationInfoMessage({
+                    segments: replicationSegments.map((x) => x.toReplicationRange()),
+                }), {
+                    mode: new SilentDelivery({ redundancy: 1, to: [publicKey] }),
                 })
                     .catch((e) => logger.error(e.toString()));
             }
         }
         else {
-            await this.modifyReplicators(new Observer(), publicKey);
+            await this.removeReplicator(publicKey);
         }
     }
     prune(entries, options) {
@@ -1043,7 +1190,7 @@ let SharedLog = class SharedLog extends Program {
             return entries.map((x) => {
                 this._gidPeersHistory.delete(x.meta.gid);
                 return this.log.remove(x, {
-                    recursively: true
+                    recursively: true,
                 });
             });
         }
@@ -1068,7 +1215,7 @@ let SharedLog = class SharedLog extends Program {
             const clear = () => {
                 //pendingPrev?.clear();
                 const pending = this._pendingDeletes.get(entry.hash);
-                if (pending?.promise == deferredPromise) {
+                if (pending?.promise === deferredPromise) {
                     this._pendingDeletes.delete(entry.hash);
                 }
                 clearTimeout(timeout);
@@ -1082,7 +1229,7 @@ let SharedLog = class SharedLog extends Program {
                 deferredPromise.reject(e);
             };
             const timeout = setTimeout(() => {
-                reject(new Error("Timeout for checked pruning"));
+                reject(new Error("Timeout for checked pruning: Closed: " + this.closed));
             }, options?.timeout ?? 10 * 1000);
             this._pendingDeletes.set(entry.hash, {
                 promise: deferredPromise,
@@ -1095,7 +1242,7 @@ let SharedLog = class SharedLog extends Program {
                     const minMinReplicasValue = this.replicas.max
                         ? Math.min(minReplicasValue, this.replicas.max.getValue(this))
                         : minReplicasValue;
-                    const leaders = await this.findLeaders(entry.gid, minMinReplicasValue);
+                    const leaders = await this.findLeaders(entry.meta.gid, minMinReplicasValue);
                     if (leaders.find((x) => x === this.node.identity.publicKey.hashcode())) {
                         reject(new Error("Failed to delete, is leader"));
                         return;
@@ -1103,10 +1250,11 @@ let SharedLog = class SharedLog extends Program {
                     if (leaders.find((x) => x === publicKeyHash)) {
                         existCounter.add(publicKeyHash);
                         if (minMinReplicasValue <= existCounter.size) {
+                            clear();
                             this._gidPeersHistory.delete(entry.meta.gid);
                             this.log
                                 .remove(entry, {
-                                recursively: true
+                                recursively: true,
                             })
                                 .then(() => {
                                 resolve();
@@ -1116,27 +1264,27 @@ let SharedLog = class SharedLog extends Program {
                             });
                         }
                     }
-                }
+                },
             });
             promises.push(deferredPromise.promise);
         }
-        if (filteredEntries.length == 0) {
-            return [];
+        if (filteredEntries.length === 0) {
+            return promises;
         }
         this.rpc.send(new RequestIPrune({ hashes: filteredEntries.map((x) => x.hash) }));
         const onNewPeer = async (e) => {
-            if (e.detail.role instanceof Replicator) {
+            if (e.detail.publicKey.equals(this.node.identity.publicKey) === false) {
                 await this.rpc.send(new RequestIPrune({ hashes: filteredEntries.map((x) => x.hash) }), {
                     mode: new SilentDelivery({
                         to: [e.detail.publicKey.hashcode()],
-                        redundancy: 1
-                    })
+                        redundancy: 1,
+                    }),
                 });
             }
         };
         // check joining peers
-        this.events.addEventListener("role", onNewPeer);
-        Promise.allSettled(promises).finally(() => this.events.removeEventListener("role", onNewPeer));
+        this.events.addEventListener("replicator:join", onNewPeer);
+        Promise.allSettled(promises).finally(() => this.events.removeEventListener("replicator:join", onNewPeer));
         return promises;
     }
     async distribute() {
@@ -1151,7 +1299,7 @@ let SharedLog = class SharedLog extends Program {
             (this.distributeQueue = new PQueue({ concurrency: 1 }));
         return queue
             .add(() => delay(Math.min(this.log.length, this.distributionDebounceTime), {
-            signal: this._closeController.signal
+            signal: this._closeController.signal,
         }).then(() => this._distribute()))
             .catch(() => { }); // catch ignore delay abort errror
     }
@@ -1165,7 +1313,7 @@ let SharedLog = class SharedLog extends Program {
         }
         const changed = false;
         await this.log.trim();
-        const heads = await this.log.getHeads();
+        const heads = await this.log.getHeads().all();
         const groupedByGid = await groupByGid(heads);
         const uncheckedDeliver = new Map();
         const allEntriesToDelete = [];
@@ -1177,13 +1325,12 @@ let SharedLog = class SharedLog extends Program {
                 continue; // TODO maybe close store?
             }
             const oldPeersSet = this._gidPeersHistory.get(gid);
-            const currentPeers = await this.findLeaders(gid, maxReplicas(this, entries) // pick max replication policy of all entries, so all information is treated equally important as the most important
-            );
+            const currentPeers = await this.findLeaders(gid, maxReplicas(this, entries));
             const isLeader = currentPeers.find((x) => x === this.node.identity.publicKey.hashcode());
             const currentPeersSet = new Set(currentPeers);
             this._gidPeersHistory.set(gid, currentPeersSet);
             for (const currentPeer of currentPeers) {
-                if (currentPeer == this.node.identity.publicKey.hashcode()) {
+                if (currentPeer === this.node.identity.publicKey.hashcode()) {
                     continue;
                 }
                 if (!oldPeersSet?.has(currentPeer)) {
@@ -1201,9 +1348,7 @@ let SharedLog = class SharedLog extends Program {
                 if (currentPeers.length > 0) {
                     // If we are observer, never prune locally created entries, since we dont really know who can store them
                     // if we are replicator, we will always persist entries that we need to so filtering on createdLocally will not make a difference
-                    let entriesToDelete = this._role instanceof Observer
-                        ? entries.filter((e) => !e.createdLocally)
-                        : entries;
+                    let entriesToDelete = entries;
                     if (this.sync) {
                         entriesToDelete = entriesToDelete.filter((entry) => this.sync(entry) === false);
                     }
@@ -1212,20 +1357,20 @@ let SharedLog = class SharedLog extends Program {
             }
             else {
                 for (const entry of entries) {
-                    this._pendingDeletes
+                    await this._pendingDeletes
                         .get(entry.hash)
-                        ?.reject(new Error("Failed to delete, is leader again"));
+                        ?.reject(new Error("Failed to delete, is leader again. Closed: " + this.closed));
                 }
             }
         }
         for (const [target, entries] of uncheckedDeliver) {
             this.rpc.send(new RequestMaybeSync({ hashes: entries.map((x) => x.hash) }), {
-                mode: new SilentDelivery({ to: [target], redundancy: 1 })
+                mode: new SilentDelivery({ to: [target], redundancy: 1 }),
             });
         }
         if (allEntriesToDelete.length > 0) {
             Promise.allSettled(this.prune(allEntriesToDelete)).catch((e) => {
-                logger.error(e.toString());
+                logger.info(e.toString());
             });
         }
         return changed;
@@ -1243,16 +1388,16 @@ let SharedLog = class SharedLog extends Program {
             }
         }
         await this.rpc.send(new ResponseMaybeSync({
-            hashes: hashes
+            hashes: hashes,
         }), {
-            mode: new SilentDelivery({ to, redundancy: 1 })
+            mode: new SilentDelivery({ to, redundancy: 1 }),
         });
     }
     async _onUnsubscription(evt) {
         logger.debug(`Peer disconnected '${evt.detail.from.hashcode()}' from '${JSON.stringify(evt.detail.unsubscriptions.map((x) => x))}'`);
         this.latestRoleMessages.delete(evt.detail.from.hashcode());
-        this.events.dispatchEvent(new CustomEvent("role", {
-            detail: { publicKey: evt.detail.from, role: new Observer() }
+        this.events.dispatchEvent(new CustomEvent("replicator:leave", {
+            detail: { publicKey: evt.detail.from },
         }));
         return this.handleSubscriptionChange(evt.detail.from, evt.detail.unsubscriptions, false);
     }
@@ -1261,8 +1406,6 @@ let SharedLog = class SharedLog extends Program {
         this.remoteBlocks.onReachable(evt.detail.from);
         return this.handleSubscriptionChange(evt.detail.from, evt.detail.subscriptions, true);
     }
-    replicationController;
-    history;
     async addToHistory(usedMemory, factor) {
         (this.history || (this.history = [])).push({ usedMemory, factor });
         // Keep only the last N entries in the history array (you can adjust N based on your needs)
@@ -1294,34 +1437,43 @@ let SharedLog = class SharedLog extends Program {
             return false;
         }
         // The role is fixed (no changes depending on memory usage or peer count etc)
-        if (this._roleConfig instanceof Role) {
+        if (!this._replicationSettings) {
             return false;
         }
-        // TODO second condition: what if the current role is Observer?
-        if (this._roleConfig.type == "replicator" &&
-            this._role instanceof Replicator) {
-            const peers = this.getReplicatorsSorted();
+        if (isAdaptiveReplicatorOption(this._replicationSettings)) {
+            const peers = this.replicationIndex;
             const usedMemory = await this.getMemoryUsage();
+            let dynamicRange = await this.getDynamicRange();
+            if (!dynamicRange) {
+                return; // not allowed to replicate
+            }
+            const peersSize = (await peers.getSize()) || 1;
             const newFactor = this.replicationController.step({
                 memoryUsage: usedMemory,
-                currentFactor: this._role.factor,
+                currentFactor: dynamicRange.widthNormalized,
                 totalFactor: this._totalParticipation,
-                peerCount: peers?.length || 1,
-                cpuUsage: this.cpuUsage?.value()
+                peerCount: peersSize,
+                cpuUsage: this.cpuUsage?.value(),
             });
-            const relativeDifference = Math.abs(this._role.factor - newFactor) / this._role.factor;
+            const relativeDifference = Math.abs(dynamicRange.widthNormalized - newFactor) /
+                dynamicRange.widthNormalized;
             if (relativeDifference > 0.0001) {
-                const newRole = new Replicator({
-                    factor: newFactor,
-                    timestamp: this._role.timestamp,
-                    offset: hashToUniformNumber(this.node.identity.publicKey.bytes)
+                // TODO can not reuse old range, since it will (potentially) affect the index because of sideeffects
+                dynamicRange = new ReplicationRangeIndexable({
+                    offset: hashToUniformNumber(this.node.identity.publicKey.bytes),
+                    length: newFactor,
+                    publicKeyHash: dynamicRange.hash,
+                    id: dynamicRange.id,
+                    replicationIntent: dynamicRange.replicationIntent,
+                    timestamp: dynamicRange.timestamp,
                 });
-                const canReplicate = !this._canReplicate ||
-                    (await this._canReplicate(this.node.identity.publicKey, newRole));
+                const canReplicate = !this._isTrustedReplicator ||
+                    (await this._isTrustedReplicator(this.node.identity.publicKey));
                 if (!canReplicate) {
                     return false;
                 }
-                await this._updateRole(newRole, onRoleChange);
+                await this.startAnnounceReplicating(dynamicRange);
+                /* await this._updateRole(newRole, onRoleChange); */
                 this.rebalanceParticipationDebounced?.();
                 return true;
             }
@@ -1332,6 +1484,39 @@ let SharedLog = class SharedLog extends Program {
         }
         return false;
     }
+    async getDynamicRange() {
+        let range = (await this.replicationIndex.query(new SearchRequest({
+            query: [
+                new StringMatch({
+                    key: "hash",
+                    value: this.node.identity.publicKey.hashcode(),
+                }),
+                new IntegerCompare({
+                    key: "replicationIntent",
+                    value: ReplicationIntent.Automatic,
+                    compare: "eq",
+                }),
+            ],
+            fetch: 1,
+        })))?.results[0]?.value;
+        if (!range) {
+            let seed = Math.random();
+            range = new ReplicationRangeIndexable({
+                offset: seed,
+                length: 0,
+                publicKeyHash: this.node.identity.publicKey.hashcode(),
+                replicationIntent: ReplicationIntent.Automatic,
+                timestamp: BigInt(+new Date()),
+                id: sha256Sync(this.node.identity.publicKey.bytes),
+            });
+            const added = await this.addReplicationRange(range, this.node.identity.publicKey);
+            if (!added) {
+                logger.warn("Not allowed to replicate by canReplicate");
+                return;
+            }
+        }
+        return range;
+    }
     clearSyncProcess(hash) {
         const inflight = this.syncInFlightQueue.get(hash);
         if (inflight) {
@@ -1372,19 +1557,4 @@ SharedLog = __decorate([
     __metadata("design:paramtypes", [Object])
 ], SharedLog);
 export { SharedLog };
-function _insertAfter(self, node, value) {
-    const inserted = !node
-        ? new yallist.Node(value, null, self.head, self)
-        : new yallist.Node(value, node, node.next, self);
-    // is tail
-    if (inserted.next === null) {
-        self.tail = inserted;
-    }
-    // is head
-    if (inserted.prev === null) {
-        self.head = inserted;
-    }
-    self.length++;
-    return inserted;
-}
 //# sourceMappingURL=index.js.map