@peerbit/shared-log 3.1.10 → 4.0.2

package/lib/esm/index.js

@@ -11,19 +11,25 @@ import { RPC } from "@peerbit/rpc";
 import { TransportMessage } from "./message.js";
 import { Entry, Log } from "@peerbit/log";
 import { Program } from "@peerbit/program";
-import { BinaryReader, BinaryWriter, BorshError, deserialize, field, serialize, variant } from "@dao-xyz/borsh";
-import { AccessError, getPublicKeyFromPeerId, sha256, sha256Base64Sync } from "@peerbit/crypto";
+import { BinaryWriter, BorshError, field, variant } from "@dao-xyz/borsh";
+import { AccessError, sha256, sha256Base64Sync } from "@peerbit/crypto";
 import { logger as loggerFn } from "@peerbit/logger";
 import { ExchangeHeadsMessage, RequestIHave, ResponseIHave, createExchangeHeadsMessage } from "./exchange-heads.js";
-import { startsWith } from "@peerbit/uint8arrays";
-import { TimeoutError } from "@peerbit/time";
-import { REPLICATOR_TYPE_VARIANT, Observer, Replicator, Role } from "./role.js";
-import { AbsoluteReplicas, ReplicationError, decodeReplicas, encodeReplicas, maxReplicas } from "./replication.js";
+import { AbortError, waitFor } from "@peerbit/time";
+import { Observer, Replicator, Role } from "./role.js";
+import { AbsoluteReplicas, ReplicationError, RequestRoleMessage, ResponseRoleMessage, decodeReplicas, encodeReplicas, hashToUniformNumber, maxReplicas } from "./replication.js";
 import pDefer from "p-defer";
 import { Cache } from "@peerbit/cache";
+import { CustomEvent } from "@libp2p/interface";
+import yallist from "yallist";
+import { AcknowledgeDelivery, SilentDelivery } from "@peerbit/stream-interface";
+import { AnyBlockStore, RemoteBlocks } from "@peerbit/blocks";
+import { BlocksMessage } from "./blocks.js";
+import debounce from "p-debounce";
+import { PIDReplicationController } from "./pid.js";
 export * from "./replication.js";
 export { Observer, Replicator, Role };
-export const logger = loggerFn({ module: "peer" });
+export const logger = loggerFn({ module: "shared-log" });
 const groupByGid = async (entries) => {
     const groupByGid = new Map();
     for (const head of entries) {
@@ -39,25 +45,41 @@ const groupByGid = async (entries) => {
     }
     return groupByGid;
 };
+const isAdaptiveReplicatorOption = (options) => {
+    if (options.limits ||
+        options.error ||
+        options.factor == null) {
+        return true;
+    }
+    return false;
+};
 export const DEFAULT_MIN_REPLICAS = 2;
+export const WAIT_FOR_REPLICATOR_TIMEOUT = 9000;
+export const WAIT_FOR_ROLE_MATURITY = 5000;
+const REBALANCE_DEBOUNCE_INTERAVAL = 50;
 let SharedLog = class SharedLog extends Program {
     log;
     rpc;
     // options
-    _sync;
     _role;
+    _roleOptions;
     _sortedPeersCache;
-    _lastSubscriptionMessageId;
+    _totalParticipation;
     _gidPeersHistory;
     _onSubscriptionFn;
     _onUnsubscriptionFn;
     _canReplicate;
     _logProperties;
+    _closeController;
     _loadedOnce = false;
     _gidParentCache;
     _respondToIHaveTimeout;
     _pendingDeletes;
     _pendingIHave;
+    latestRoleMessages;
+    remoteBlocks;
+    openTime;
+    rebalanceParticipationDebounced;
     replicas;
     constructor(properties) {
         super();
@@ -67,31 +89,78 @@ let SharedLog = class SharedLog extends Program {
     get role() {
         return this._role;
     }
-    async updateRole(role) {
-        const wasRepicators = this._role instanceof Replicator;
-        this._role = role;
-        await this.initializeWithRole();
-        await this.rpc.subscribe(serialize(this._role));
-        if (wasRepicators) {
-            await this.replicationReorganization();
-        }
+    get totalParticipation() {
+        return this._totalParticipation;
     }
-    async initializeWithRole() {
-        try {
-            await this.modifySortedSubscriptionCache(this._role instanceof Replicator ? true : false, getPublicKeyFromPeerId(this.node.peerId));
-            if (!this._loadedOnce) {
-                await this.log.load();
-                this._loadedOnce = true;
-            }
+    setupRole(options) {
+        this.rebalanceParticipationDebounced = undefined;
+        const setupDebouncedRebalancing = (options) => {
+            this.replicationController = new PIDReplicationController({
+                targetMemoryLimit: options?.limits?.memory,
+                errorFunction: options?.error
+            });
+            this.rebalanceParticipationDebounced = debounce(() => this.rebalanceParticipation(), REBALANCE_DEBOUNCE_INTERAVAL // TODO make dynamic
+            );
+        };
+        if (options instanceof Observer || options instanceof Replicator) {
+            throw new Error("Unsupported role option type");
         }
-        catch (error) {
-            if (error instanceof AccessError) {
-                logger.error("Failed to load all entries due to access error, make sure you are opening the program with approate keychain configuration");
+        else if (options === "observer") {
+            this._roleOptions = new Observer();
+        }
+        else if (options === "replicator") {
+            setupDebouncedRebalancing();
+            this._roleOptions = { type: options };
+        }
+        else if (options) {
+            if (options.type === "replicator") {
+                if (isAdaptiveReplicatorOption(options)) {
+                    setupDebouncedRebalancing(options);
+                    this._roleOptions = options;
+                }
+                else {
+                    this._roleOptions = new Replicator({ factor: options.factor });
+                }
             }
             else {
-                throw error;
+                this._roleOptions = new Observer();
             }
         }
+        else {
+            // Default option
+            setupDebouncedRebalancing();
+            this._roleOptions = { type: "replicator" };
+        }
+        // setup the initial role
+        if (this._roleOptions instanceof Replicator ||
+            this._roleOptions instanceof Observer) {
+            this._role = this._roleOptions; // Fixed
+        }
+        else {
+            this._role = new Replicator({
+                // initial role in a dynamic setup
+                factor: 1,
+                timestamp: BigInt(+new Date())
+            });
+        }
+        return this._role;
+    }
+    async updateRole(role, onRoleChange = true) {
+        return this._updateRole(this.setupRole(role), onRoleChange);
+    }
+    async _updateRole(role = this._role, onRoleChange = true) {
+        this._role = role;
+        const { changed } = await this._modifyReplicators(this.role, this.node.identity.publicKey);
+        if (!this._loadedOnce) {
+            await this.log.load();
+            this._loadedOnce = true;
+        }
+        await this.rpc.subscribe();
+        await this.rpc.send(new ResponseRoleMessage(role));
+        if (onRoleChange && changed) {
+            this.onRoleChange(undefined, this._role, this.node.identity.publicKey);
+        }
+        return changed;
     }
     async append(data, options) {
         const appendOptions = { ...options };
@@ -109,7 +178,14 @@ let SharedLog = class SharedLog extends Program {
             appendOptions.meta.data = minReplicasData;
         }
         const result = await this.log.append(data, appendOptions);
-        await this.rpc.send(await createExchangeHeadsMessage(this.log, [result.entry], this._gidParentCache));
+        const leaders = await this.findLeaders(result.entry.meta.gid, decodeReplicas(result.entry).getValue(this));
+        const isLeader = leaders.includes(this.node.identity.publicKey.hashcode());
+        await this.rpc.send(await createExchangeHeadsMessage(this.log, [result.entry], this._gidParentCache), {
+            mode: isLeader
+                ? new SilentDelivery({ redundancy: 1, to: leaders })
+                : new AcknowledgeDelivery({ redundancy: 1, to: leaders })
+        });
+        this.rebalanceParticipationDebounced?.();
         return result;
     }
     async open(options) {
@@ -128,20 +204,34 @@ let SharedLog = class SharedLog extends Program {
         this._respondToIHaveTimeout = options?.respondToIHaveTimeout ?? 10 * 1000; // TODO make into arg
         this._pendingDeletes = new Map();
         this._pendingIHave = new Map();
+        this.latestRoleMessages = new Map();
+        this.openTime = +new Date();
         this._gidParentCache = new Cache({ max: 1000 });
+        this._closeController = new AbortController();
         this._canReplicate = options?.canReplicate;
-        this._sync = options?.sync;
         this._logProperties = options;
-        this._role = options?.role || new Replicator();
-        this._lastSubscriptionMessageId = 0;
+        this.setupRole(options?.role);
         this._onSubscriptionFn = this._onSubscription.bind(this);
-        this._sortedPeersCache = [];
+        this._totalParticipation = 0;
+        this._sortedPeersCache = yallist.create();
         this._gidPeersHistory = new Map();
         await this.node.services.pubsub.addEventListener("subscribe", this._onSubscriptionFn);
         this._onUnsubscriptionFn = this._onUnsubscription.bind(this);
         await this.node.services.pubsub.addEventListener("unsubscribe", this._onUnsubscriptionFn);
-        await this.log.open(this.node.services.blocks, this.node.identity, {
-            keychain: this.node.keychain,
+        const id = sha256Base64Sync(this.log.id);
+        const storage = await this.node.memory.sublevel(id);
+        const localBlocks = await new AnyBlockStore(await storage.sublevel("blocks"));
+        const cache = await storage.sublevel("cache");
+        this.remoteBlocks = new RemoteBlocks({
+            local: localBlocks,
+            publish: (message, options) => this.rpc.send(new BlocksMessage(message), {
+                to: options?.to
+            }),
+            waitFor: this.rpc.waitFor.bind(this.rpc)
+        });
+        await this.remoteBlocks.start();
+        await this.log.open(this.remoteBlocks, this.node.identity, {
+            keychain: this.node.services.keychain,
             ...this._logProperties,
             onChange: (change) => {
                 if (this._pendingIHave.size > 0) {
@@ -149,7 +239,7 @@ let SharedLog = class SharedLog extends Program {
                         const ih = this._pendingIHave.get(added.hash);
                         if (ih) {
                             ih.clear();
-                            ih.callback();
+                            ih.callback(added);
                         }
                     }
                 }
@@ -181,33 +271,38 @@ let SharedLog = class SharedLog extends Program {
                 return this._logProperties?.canAppend?.(entry) ?? true;
             },
             trim: this._logProperties?.trim && {
-                ...this._logProperties?.trim,
-                filter: {
-                    canTrim: async (entry) => !(await this.isLeader(entry.meta.gid, decodeReplicas(entry).getValue(this))),
-                    cacheId: () => this._lastSubscriptionMessageId
-                }
+                ...this._logProperties?.trim
             },
-            cache: this.node.memory &&
-                (await this.node.memory.sublevel(sha256Base64Sync(this.log.id)))
-        });
-        await this.initializeWithRole();
-        // Take into account existing subscription
-        (await this.node.services.pubsub.getSubscribers(this.topic))?.forEach((v, k) => {
-            this.handleSubscriptionChange(v.publicKey, [{ topic: this.topic, data: v.data }], true);
+            cache: cache
         });
         // Open for communcation
         await this.rpc.open({
             queryType: TransportMessage,
             responseType: TransportMessage,
             responseHandler: this._onMessage.bind(this),
-            topic: this.topic,
-            subscriptionData: serialize(this.role)
+            topic: this.topic
         });
+        await this._updateRole();
+        await this.rebalanceParticipation();
+        // Take into account existing subscription
+        (await this.node.services.pubsub.getSubscribers(this.topic))?.forEach((v, k) => {
+            if (v.equals(this.node.identity.publicKey)) {
+                return;
+            }
+            this.handleSubscriptionChange(v, [this.topic], true);
+        });
+    }
+    async getMemoryUsage() {
+        return (((await this.log.memory?.size()) || 0) + (await this.log.blocks.size()));
     }
     get topic() {
         return this.log.idString;
     }
     async _close() {
+        this._closeController.abort();
+        this.node.services.pubsub.removeEventListener("subscribe", this._onSubscriptionFn);
+        this._onUnsubscriptionFn = this._onUnsubscription.bind(this);
+        this.node.services.pubsub.removeEventListener("unsubscribe", this._onUnsubscriptionFn);
         for (const [k, v] of this._pendingDeletes) {
             v.clear();
             v.promise.resolve(); // TODO or reject?
@@ -215,15 +310,14 @@ let SharedLog = class SharedLog extends Program {
         for (const [k, v] of this._pendingIHave) {
             v.clear();
         }
+        await this.remoteBlocks.stop();
         this._gidParentCache.clear();
         this._pendingDeletes = new Map();
         this._pendingIHave = new Map();
+        this.latestRoleMessages.clear();
         this._gidPeersHistory = new Map();
         this._sortedPeersCache = undefined;
         this._loadedOnce = false;
-        this.node.services.pubsub.removeEventListener("subscribe", this._onSubscriptionFn);
-        this._onUnsubscriptionFn = this._onUnsubscription.bind(this);
-        this.node.services.pubsub.removeEventListener("unsubscribe", this._onUnsubscriptionFn);
     }
     async close(from) {
         const superClosed = await super.close(from);
@@ -239,8 +333,8 @@ let SharedLog = class SharedLog extends Program {
         if (!superDropped) {
             return superDropped;
         }
-        await this._close();
         await this.log.drop();
+        await this._close();
         return true;
     }
     async recover() {
@@ -255,7 +349,6 @@ let SharedLog = class SharedLog extends Program {
                  * I can use them to load associated logs and join/sync them with the data stores I own
                  */
                 const { heads } = msg;
-                // replication topic === trustedNetwork address
                 logger.debug(`${this.node.identity.publicKey.hashcode()}: Recieved heads: ${heads.length === 1 ? heads[0].entry.hash : "#" + heads.length}, logId: ${this.log.idString}`);
                 if (heads) {
                     const filteredHeads = [];
@@ -269,20 +362,22 @@ let SharedLog = class SharedLog extends Program {
                             filteredHeads.push(head);
                         }
                     }
-                    if (!this._sync) {
-                        const toMerge = [];
-                        let toDelete = undefined;
-                        let maybeDelete = undefined;
-                        const groupedByGid = await groupByGid(filteredHeads);
-                        for (const [gid, entries] of groupedByGid) {
+                    if (filteredHeads.length === 0) {
+                        return;
+                    }
+                    const toMerge = [];
+                    let toDelete = undefined;
+                    let maybeDelete = undefined;
+                    const groupedByGid = await groupByGid(filteredHeads);
+                    const promises = [];
+                    for (const [gid, entries] of groupedByGid) {
+                        const fn = async () => {
                             const headsWithGid = this.log.headsIndex.gids.get(gid);
                             const maxReplicasFromHead = headsWithGid && headsWithGid.size > 0
                                 ? maxReplicas(this, [...headsWithGid.values()])
                                 : this.replicas.min.getValue(this);
-                            const maxReplicasFromNewEntries = maxReplicas(this, [
-                                ...entries.map((x) => x.entry)
-                            ]);
-                            const isLeader = await this.isLeader(gid, Math.max(maxReplicasFromHead, maxReplicasFromNewEntries));
+                            const maxReplicasFromNewEntries = maxReplicas(this, entries.map((x) => x.entry));
+                            const isLeader = await this.waitForIsLeader(gid, Math.max(maxReplicasFromHead, maxReplicasFromNewEntries));
                             if (maxReplicasFromNewEntries < maxReplicasFromHead && isLeader) {
                                 (maybeDelete || (maybeDelete = [])).push(entries);
                             }
@@ -302,34 +397,35 @@ let SharedLog = class SharedLog extends Program {
                                 }
                                 logger.debug(`${this.node.identity.publicKey.hashcode()}: Dropping heads with gid: ${entry.entry.gid}. Because not leader`);
                             }
-                        }
-                        if (toMerge.length > 0) {
-                            await this.log.join(toMerge);
-                            toDelete &&
-                                Promise.all(this.pruneSafely(toDelete)).catch((e) => {
-                                    logger.error(e.toString());
-                                });
-                        }
-                        if (maybeDelete) {
-                            for (const entries of maybeDelete) {
-                                const headsWithGid = this.log.headsIndex.gids.get(entries[0].entry.meta.gid);
-                                if (headsWithGid && headsWithGid.size > 0) {
-                                    const minReplicas = maxReplicas(this, [
-                                        ...headsWithGid.values()
-                                    ]);
-                                    const isLeader = await this.isLeader(entries[0].entry.meta.gid, minReplicas);
-                                    if (!isLeader) {
-                                        Promise.all(this.pruneSafely(entries.map((x) => x.entry))).catch((e) => {
-                                            logger.error(e.toString());
-                                        });
-                                    }
+                        };
+                        promises.push(fn());
+                    }
+                    await Promise.all(promises);
+                    if (this.closed) {
+                        return;
+                    }
+                    if (toMerge.length > 0) {
+                        await this.log.join(toMerge);
+                        toDelete &&
+                            this.prune(toDelete).catch((e) => {
+                                logger.error(e.toString());
+                            });
+                        this.rebalanceParticipationDebounced?.();
+                    }
+                    if (maybeDelete) {
+                        for (const entries of maybeDelete) {
+                            const headsWithGid = this.log.headsIndex.gids.get(entries[0].entry.meta.gid);
+                            if (headsWithGid && headsWithGid.size > 0) {
+                                const minReplicas = maxReplicas(this, headsWithGid.values());
+                                const isLeader = await this.isLeader(entries[0].entry.meta.gid, minReplicas);
+                                if (!isLeader) {
+                                    this.prune(entries.map((x) => x.entry)).catch((e) => {
+                                        logger.error(e.toString());
+                                    });
                                 }
                             }
                         }
                     }
-                    else {
-                        await this.log.join(await Promise.all(filteredHeads.map((x) => this._sync(x.entry))).then((filter) => filteredHeads.filter((v, ix) => filter[ix])));
-                    }
                 }
             }
             else if (msg instanceof RequestIHave) {
@@ -347,12 +443,14 @@ let SharedLog = class SharedLog extends Program {
                                 clearTimeout(timeout);
                                 prevPendingIHave?.clear();
                             },
-                            callback: () => {
-                                prevPendingIHave && prevPendingIHave.callback();
-                                this.rpc.send(new ResponseIHave({ hashes: [hash] }), {
-                                    to: [context.from]
-                                });
-                                this._pendingIHave.delete(hash);
+                            callback: async (entry) => {
+                                if (await this.isLeader(entry.meta.gid, decodeReplicas(entry).getValue(this))) {
+                                    this.rpc.send(new ResponseIHave({ hashes: [entry.hash] }), {
+                                        to: [context.from]
+                                    });
+                                }
+                                prevPendingIHave && prevPendingIHave.callback(entry);
+                                this._pendingIHave.delete(entry.hash);
                             }
                         };
                         const timeout = setTimeout(() => {
@@ -364,7 +462,7 @@ let SharedLog = class SharedLog extends Program {
                         this._pendingIHave.set(hash, pendingIHave);
                     }
                 }
-                this.rpc.send(new ResponseIHave({ hashes: hasAndIsLeader }), {
+                await this.rpc.send(new ResponseIHave({ hashes: hasAndIsLeader }), {
                     to: [context.from]
                 });
             }
@@ -373,11 +471,55 @@ let SharedLog = class SharedLog extends Program {
                     this._pendingDeletes.get(hash)?.callback(context.from.hashcode());
                 }
             }
+            else if (msg instanceof BlocksMessage) {
+                await this.remoteBlocks.onMessage(msg.message);
+            }
+            else if (msg instanceof RequestRoleMessage) {
+                if (!context.from) {
+                    throw new Error("Missing form in update role message");
+                }
+                if (context.from.equals(this.node.identity.publicKey)) {
+                    return;
+                }
+                await this.rpc.send(new ResponseRoleMessage({ role: this.role }), {
+                    to: [context.from]
+                });
+            }
+            else if (msg instanceof ResponseRoleMessage) {
+                if (!context.from) {
+                    throw new Error("Missing form in update role message");
+                }
+                if (context.from.equals(this.node.identity.publicKey)) {
+                    return;
+                }
+                this.waitFor(context.from, {
+                    signal: this._closeController.signal,
+                    timeout: WAIT_FOR_REPLICATOR_TIMEOUT
+                })
+                    .then(async () => {
+                    /* await delay(1000 * Math.random()) */
+                    const prev = this.latestRoleMessages.get(context.from.hashcode());
+                    if (prev && prev > context.timestamp) {
+                        return;
+                    }
+                    this.latestRoleMessages.set(context.from.hashcode(), context.timestamp);
+                    await this.modifyReplicators(msg.role, context.from);
+                })
+                    .catch((e) => {
+                    if (e instanceof AbortError) {
+                        return;
+                    }
+                    logger.error("Failed to find peer who updated their role: " + e?.message);
+                });
+            }
             else {
                 throw new Error("Unexpected message");
             }
         }
         catch (e) {
+            if (e instanceof AbortError) {
+                return;
+            }
             if (e instanceof BorshError) {
                 logger.trace(`${this.node.identity.publicKey.hashcode()}: Failed to handle message on topic: ${JSON.stringify(this.log.idString)}: Got message for a different namespace`);
                 return;
@@ -392,114 +534,327 @@ let SharedLog = class SharedLog extends Program {
     getReplicatorsSorted() {
         return this._sortedPeersCache;
     }
-    async isLeader(slot, numberOfLeaders) {
-        const isLeader = (await this.findLeaders(slot, numberOfLeaders)).find((l) => l === this.node.identity.publicKey.hashcode());
+    async waitForReplicator(...keys) {
+        const check = () => {
+            for (const k of keys) {
+                if (!this.getReplicatorsSorted()
+                    ?.toArray()
+                    ?.find((x) => x.publicKey.equals(k))) {
+                    return false;
+                }
+            }
+            return true;
+        };
+        return waitFor(() => check(), { signal: this._closeController.signal });
+    }
+    async isLeader(slot, numberOfLeaders, options) {
+        const isLeader = (await this.findLeaders(slot, numberOfLeaders, options)).find((l) => l === this.node.identity.publicKey.hashcode());
         return !!isLeader;
     }
-    async findLeaders(subject, numberOfLeadersUnbounded) {
-        const lower = this.replicas.min.getValue(this);
-        const higher = this.replicas.max?.getValue(this) ?? Number.MAX_SAFE_INTEGER;
-        let numberOfLeaders = Math.max(Math.min(higher, numberOfLeadersUnbounded), lower);
+    async waitForIsLeader(slot, numberOfLeaders, timeout = WAIT_FOR_REPLICATOR_TIMEOUT) {
+        return new Promise((res, rej) => {
+            const removeListeners = () => {
+                this.events.removeEventListener("role", roleListener);
+                this._closeController.signal.addEventListener("abort", abortListener);
+            };
+            const abortListener = () => {
+                removeListeners();
+                clearTimeout(timer);
+                res(false);
+            };
+            const timer = setTimeout(() => {
+                removeListeners();
+                res(false);
+            }, timeout);
+            const check = () => this.isLeader(slot, numberOfLeaders).then((isLeader) => {
+                if (isLeader) {
+                    removeListeners();
+                    clearTimeout(timer);
+                    res(isLeader);
+                }
+            });
+            const roleListener = () => {
+                check();
+            };
+            this.events.addEventListener("role", roleListener);
+            this._closeController.signal.addEventListener("abort", abortListener);
+            check();
+        });
+    }
+    async findLeaders(subject, numberOfLeaders, options) {
         // For a fixed set or members, the choosen leaders will always be the same (address invariant)
         // This allows for that same content is always chosen to be distributed to same peers, to remove unecessary copies
-        const peers = this.getReplicatorsSorted() || [];
-        if (peers.length === 0) {
-            return [];
-        }
-        numberOfLeaders = Math.min(numberOfLeaders, peers.length);
         // Convert this thing we wan't to distribute to 8 bytes so we get can convert it into a u64
         // modulus into an index
         const utf8writer = new BinaryWriter();
         utf8writer.string(subject.toString());
         const seed = await sha256(utf8writer.finalize());
         // convert hash of slot to a number
-        const seedNumber = new BinaryReader(seed.subarray(seed.length - 8, seed.length)).u64();
-        const startIndex = Number(seedNumber % BigInt(peers.length));
-        // we only step forward 1 step (ignoring that step backward 1 could be 'closer')
-        // This does not matter, we only have to make sure all nodes running the code comes to somewhat the
-        // same conclusion (are running the same leader selection algorithm)
-        const leaders = new Array(numberOfLeaders);
+        const cursor = hashToUniformNumber(seed); // bounded between 0 and 1
+        return this.findLeadersFromUniformNumber(cursor, numberOfLeaders, options);
+    }
+    findLeadersFromUniformNumber(cursor, numberOfLeaders, options) {
+        const leaders = new Set();
+        const width = 1; // this.getParticipationSum(roleAge);
+        const peers = this.getReplicatorsSorted();
+        if (!peers || peers?.length === 0) {
+            return [];
+        }
+        numberOfLeaders = Math.min(numberOfLeaders, peers.length);
+        const t = +new Date();
+        const roleAge = options?.roleAge ??
+            Math.min(WAIT_FOR_ROLE_MATURITY, +new Date() - this.openTime);
         for (let i = 0; i < numberOfLeaders; i++) {
-            leaders[i] = peers[(i + startIndex) % peers.length].hash;
+            let matured = 0;
+            const maybeIncrementMatured = (role) => {
+                if (t - Number(role.timestamp) > roleAge) {
+                    matured++;
+                }
+            };
+            const x = ((cursor + i / numberOfLeaders) % 1) * width;
+            let currentNode = peers.head;
+            const diffs = [];
+            while (currentNode) {
+                const start = currentNode.value.offset % width;
+                const absDelta = Math.abs(start - x);
+                const diff = Math.min(absDelta, width - absDelta);
+                if (diff < currentNode.value.role.factor / 2 + 0.00001) {
+                    leaders.add(currentNode.value.publicKey.hashcode());
+                    maybeIncrementMatured(currentNode.value.role);
+                }
+                else {
+                    diffs.push({
+                        diff: currentNode.value.role.factor > 0
+                            ? diff / currentNode.value.role.factor
+                            : Number.MAX_SAFE_INTEGER,
+                        rect: currentNode.value
+                    });
+                }
+                currentNode = currentNode.next;
+            }
+            if (matured === 0) {
+                diffs.sort((x, y) => x.diff - y.diff);
+                for (const node of diffs) {
+                    leaders.add(node.rect.publicKey.hashcode());
+                    maybeIncrementMatured(node.rect.role);
+                    if (matured > 0) {
+                        break;
+                    }
+                }
+            }
         }
-        return leaders;
+        return [...leaders];
     }
-    async modifySortedSubscriptionCache(subscribed, publicKey) {
-        if (subscribed &&
+    /**
+     *
+     * @returns groups where at least one in any group will have the entry you are looking for
+     */
+    getReplicatorUnion(roleAge = WAIT_FOR_ROLE_MATURITY) {
+        // Total replication "width"
+        const width = 1; //this.getParticipationSum(roleAge);
+        // How much width you need to "query" to
+        const peers = this.getReplicatorsSorted(); // TODO types
+        const minReplicas = Math.min(peers.length, this.replicas.min.getValue(this));
+        const coveringWidth = width / minReplicas;
+        let walker = peers.head;
+        if (this.role instanceof Replicator) {
+            // start at our node (local first)
+            while (walker) {
+                if (walker.value.publicKey.equals(this.node.identity.publicKey)) {
+                    break;
+                }
+                walker = walker.next;
+            }
+        }
+        else {
+            const seed = Math.round(peers.length * Math.random()); // start at a random point
+            for (let i = 0; i < seed - 1; i++) {
+                if (walker?.next == null) {
+                    break;
+                }
+                walker = walker.next;
+            }
+        }
+        const set = [];
+        let distance = 0;
+        const startNode = walker;
+        if (!startNode) {
+            return [];
+        }
+        let nextPoint = startNode.value.offset;
+        const t = +new Date();
+        while (walker && distance < coveringWidth) {
+            const absDelta = Math.abs(walker.value.offset - nextPoint);
+            const diff = Math.min(absDelta, width - absDelta);
+            if (diff < walker.value.role.factor / 2 + 0.00001) {
+                set.push(walker.value.publicKey.hashcode());
+                if (t - Number(walker.value.role.timestamp) >
+                    roleAge /* ||
+                walker!.value.publicKey.equals(this.node.identity.publicKey)) */) {
+                    nextPoint = (nextPoint + walker.value.role.factor) % 1;
+                    distance += walker.value.role.factor;
+                }
+            }
+            walker = walker.next || peers.head;
+            if (walker?.value.publicKey &&
+                startNode?.value.publicKey.equals(walker?.value.publicKey)) {
+                break; // TODO throw error for failing to fetch ffull width
+            }
+        }
+        return set;
+    }
+    async replicator(entry, options) {
+        return this.isLeader(entry.gid, decodeReplicas(entry).getValue(this), options);
+    }
+    onRoleChange(prev, role, publicKey) {
+        if (this.closed) {
+            return;
+        }
+        this.distribute();
+        if (role instanceof Replicator) {
+            const timer = setTimeout(async () => {
+                this._closeController.signal.removeEventListener("abort", listener);
+                await this.rebalanceParticipationDebounced?.();
+                this.distribute();
+            }, WAIT_FOR_ROLE_MATURITY + 2000);
+            const listener = () => {
+                clearTimeout(timer);
+            };
+            this._closeController.signal.addEventListener("abort", listener);
+        }
+        this.events.dispatchEvent(new CustomEvent("role", {
+            detail: { publicKey, role }
+        }));
+    }
+    async modifyReplicators(role, publicKey) {
+        const { prev, changed } = await this._modifyReplicators(role, publicKey);
+        if (changed) {
+            await this.rebalanceParticipationDebounced?.(); // await this.rebalanceParticipation(false);
+            this.onRoleChange(prev, role, publicKey);
+            return true;
+        }
+        return false;
+    }
+    async _modifyReplicators(role, publicKey) {
+        if (role instanceof Replicator &&
             this._canReplicate &&
-            !(await this._canReplicate(publicKey))) {
-            return false;
+            !(await this._canReplicate(publicKey, role))) {
+            return { changed: false };
         }
         const sortedPeer = this._sortedPeersCache;
         if (!sortedPeer) {
             if (this.closed === false) {
                 throw new Error("Unexpected, sortedPeersCache is undefined");
             }
-            return false;
+            return { changed: false };
         }
-        const code = publicKey.hashcode();
-        if (subscribed) {
+        if (role instanceof Replicator && role.factor > 0) {
             // TODO use Set + list for fast lookup
-            if (!sortedPeer.find((x) => x.hash === code)) {
-                sortedPeer.push({ hash: code, timestamp: +new Date() });
-                sortedPeer.sort((a, b) => a.hash.localeCompare(b.hash));
-                return true;
+            // check also that peer is online
+            const isOnline = this.node.identity.publicKey.equals(publicKey) ||
+                (await this.waitFor(publicKey, { signal: this._closeController.signal })
+                    .then(() => true)
+                    .catch(() => false));
+            if (isOnline) {
+                // insert or if already there do nothing
+                const code = hashToUniformNumber(publicKey.bytes);
+                const rect = {
+                    publicKey,
+                    offset: code,
+                    role
+                };
+                let currentNode = sortedPeer.head;
+                if (!currentNode) {
+                    sortedPeer.push(rect);
+                    this._totalParticipation += rect.role.factor;
+                    return { changed: true };
+                }
+                else {
+                    while (currentNode) {
+                        if (currentNode.value.publicKey.equals(publicKey)) {
+                            // update the value
+                            // rect.timestamp = currentNode.value.timestamp;
+                            const prev = currentNode.value;
+                            currentNode.value = rect;
+                            this._totalParticipation += rect.role.factor;
+                            this._totalParticipation -= prev.role.factor;
+                            // TODO change detection and only do change stuff if diff?
+                            return { prev: prev.role, changed: true };
+                        }
+                        if (code > currentNode.value.offset) {
+                            const next = currentNode?.next;
+                            if (next) {
+                                currentNode = next;
+                                continue;
+                            }
+                            else {
+                                break;
+                            }
+                        }
+                        else {
+                            currentNode = currentNode.prev;
+                            break;
+                        }
+                    }
+                    const prev = currentNode;
+                    if (!prev?.next?.value.publicKey.equals(publicKey)) {
+                        this._totalParticipation += rect.role.factor;
+                        _insertAfter(sortedPeer, prev || undefined, rect);
+                    }
+                    else {
+                        throw new Error("Unexpected");
+                    }
+                    return { changed: true };
+                }
             }
             else {
-                return false;
+                return { changed: false };
             }
         }
         else {
-            const deleteIndex = sortedPeer.findIndex((x) => x.hash === code);
-            if (deleteIndex >= 0) {
-                sortedPeer.splice(deleteIndex, 1);
-                return true;
-            }
-            else {
-                return false;
+            let currentNode = sortedPeer.head;
+            while (currentNode) {
+                if (currentNode.value.publicKey.equals(publicKey)) {
+                    sortedPeer.removeNode(currentNode);
+                    this._totalParticipation -= currentNode.value.role.factor;
+                    return { prev: currentNode.value.role, changed: true };
+                }
+                currentNode = currentNode.next;
             }
+            return { changed: false };
         }
     }
     async handleSubscriptionChange(publicKey, changes, subscribed) {
-        // TODO why are we doing two loops?
-        const prev = [];
-        for (const subscription of changes) {
-            if (this.log.idString !== subscription.topic) {
-                continue;
-            }
-            if (!subscription.data ||
-                !startsWith(subscription.data, REPLICATOR_TYPE_VARIANT)) {
-                prev.push(await this.modifySortedSubscriptionCache(false, publicKey));
-                continue;
-            }
-            else {
-                this._lastSubscriptionMessageId += 1;
-                prev.push(await this.modifySortedSubscriptionCache(subscribed, publicKey));
-            }
-        }
-        // TODO don't do this i fnot is replicator?
-        for (const [i, subscription] of changes.entries()) {
-            if (this.log.idString !== subscription.topic) {
-                continue;
-            }
-            if (subscription.data) {
-                try {
-                    const type = deserialize(subscription.data, Role);
-                    // Reorganize if the new subscriber is a replicator, or observers AND was replicator
-                    if (type instanceof Replicator || prev[i]) {
-                        await this.replicationReorganization();
+        if (subscribed) {
+            if (this.role instanceof Replicator) {
+                for (const subscription of changes) {
+                    if (this.log.idString !== subscription) {
+                        continue;
                     }
+                    this.rpc
+                        .send(new ResponseRoleMessage(this.role), {
+                        mode: new AcknowledgeDelivery({ redundancy: 1, to: [publicKey] })
+                    })
+                        .catch((e) => logger.error(e.toString()));
                 }
-                catch (error) {
-                    logger.warn("Recieved subscription with invalid data on topic: " +
-                        subscription.topic +
-                        ". Error: " +
-                        error?.message);
+            }
+            //if(evt.detail.subscriptions.map((x) => x.topic).includes())
+        }
+        else {
+            for (const topic of changes) {
+                if (this.log.idString !== topic) {
+                    continue;
                 }
+                await this.modifyReplicators(new Observer(), publicKey);
             }
         }
     }
-    pruneSafely(entries, options) {
+    async prune(entries, options) {
+        if (options?.unchecked) {
+            return Promise.all(entries.map((x) => this.log.remove(x, {
+                recursively: true
+            })));
+        }
         // ask network if they have they entry,
         // so I can delete it
         // There is a few reasons why we might end up here
@@ -510,12 +865,16 @@ let SharedLog = class SharedLog extends Program {
         const filteredEntries = [];
         for (const entry of entries) {
             const pendingPrev = this._pendingDeletes.get(entry.hash);
+            if (pendingPrev) {
+                promises.push(pendingPrev.promise.promise);
+                continue;
+            }
             filteredEntries.push(entry);
             const existCounter = new Set();
             const minReplicas = decodeReplicas(entry);
             const deferredPromise = pDefer();
             const clear = () => {
-                pendingPrev?.clear();
+                //pendingPrev?.clear();
                 const pending = this._pendingDeletes.get(entry.hash);
                 if (pending?.promise == deferredPromise) {
                     this._pendingDeletes.delete(entry.hash);
@@ -531,7 +890,7 @@ let SharedLog = class SharedLog extends Program {
                 deferredPromise.reject(e);
             };
             const timeout = setTimeout(() => {
-                reject(new Error("Timeout"));
+                reject(new Error("Timeout for checked pruning"));
             }, options?.timeout ?? 10 * 1000);
             this._pendingDeletes.set(entry.hash, {
                 promise: deferredPromise,
@@ -540,10 +899,17 @@ let SharedLog = class SharedLog extends Program {
                 },
                 callback: async (publicKeyHash) => {
                     const minReplicasValue = minReplicas.getValue(this);
-                    const l = await this.findLeaders(entry.gid, minReplicasValue);
-                    if (l.find((x) => x === publicKeyHash)) {
+                    const minMinReplicasValue = this.replicas.max
+                        ? Math.min(minReplicasValue, this.replicas.max.getValue(this))
+                        : minReplicasValue;
+                    const leaders = await this.findLeaders(entry.gid, minMinReplicasValue);
+                    if (leaders.find((x) => x === this.node.identity.publicKey.hashcode())) {
+                        reject(new Error("Failed to delete, is leader"));
+                        return;
+                    }
+                    if (leaders.find((x) => x === publicKeyHash)) {
                         existCounter.add(publicKeyHash);
-                        if (minReplicas.getValue(this) <= existCounter.size) {
+                        if (minMinReplicasValue <= existCounter.size) {
                             this.log
                                 .remove(entry, {
                                 recursively: true
@@ -560,118 +926,173 @@ let SharedLog = class SharedLog extends Program {
             });
             promises.push(deferredPromise.promise);
         }
-        if (filteredEntries.length > 0) {
-            this.rpc.send(new RequestIHave({ hashes: filteredEntries.map((x) => x.hash) }));
+        if (filteredEntries.length == 0) {
+            return;
         }
-        return promises;
+        this.rpc.send(new RequestIHave({ hashes: filteredEntries.map((x) => x.hash) }));
+        const onNewPeer = async (e) => {
+            if (e.detail.role instanceof Replicator) {
+                await this.rpc.send(new RequestIHave({ hashes: filteredEntries.map((x) => x.hash) }), {
+                    to: [e.detail.publicKey.hashcode()]
+                });
+            }
+        };
+        // check joining peers
+        this.events.addEventListener("role", onNewPeer);
+        return Promise.all(promises).finally(() => this.events.removeEventListener("role", onNewPeer));
     }
-    /**
-     * When a peers join the networkk and want to participate the leaders for particular log subgraphs might change, hence some might start replicating, might some stop
-     * This method will go through my owned entries, and see whether I should share them with a new leader, and/or I should stop care about specific entries
-     * @param channel
-     */
-    async replicationReorganization() {
+    async distribute() {
+        /**
+         * TODO use information of new joined/leaving peer to create a subset of heads
+         * that we potentially need to share with other peers
+         */
+        if (this.closed) {
+            return;
+        }
         const changed = false;
+        await this.log.trim();
         const heads = await this.log.getHeads();
         const groupedByGid = await groupByGid(heads);
-        let storeChanged = false;
+        const toDeliver = new Map();
+        const allEntriesToDelete = [];
         for (const [gid, entries] of groupedByGid) {
-            const toSend = new Map();
-            const newPeers = [];
+            if (this.closed) {
+                break;
+            }
             if (entries.length === 0) {
                 continue; // TODO maybe close store?
             }
             const oldPeersSet = this._gidPeersHistory.get(gid);
             const currentPeers = await this.findLeaders(gid, maxReplicas(this, entries) // pick max replication policy of all entries, so all information is treated equally important as the most important
             );
+            const currentPeersSet = new Set(currentPeers);
+            this._gidPeersHistory.set(gid, currentPeersSet);
             for (const currentPeer of currentPeers) {
-                if (!oldPeersSet?.has(currentPeer) &&
-                    currentPeer !== this.node.identity.publicKey.hashcode()) {
-                    storeChanged = true;
+                if (currentPeer == this.node.identity.publicKey.hashcode()) {
+                    continue;
+                }
+                if (!oldPeersSet?.has(currentPeer)) {
                     // second condition means that if the new peer is us, we should not do anything, since we are expecting to receive heads, not send
-                    newPeers.push(currentPeer);
-                    // send heads to the new peer
-                    // console.log('new gid for peer', newPeers.length, this.id.toString(), newPeer, gid, entries.length, newPeers)
-                    try {
-                        logger.debug(`${this.node.identity.publicKey.hashcode()}: Exchange heads ${entries.length === 1 ? entries[0].hash : "#" + entries.length}  on rebalance`);
-                        for (const entry of entries) {
-                            toSend.set(entry.hash, entry);
-                        }
+                    let arr = toDeliver.get(currentPeer);
+                    if (!arr) {
+                        arr = [];
+                        toDeliver.set(currentPeer, arr);
                     }
-                    catch (error) {
-                        if (error instanceof TimeoutError) {
-                            logger.error("Missing channel when reorg to peer: " + currentPeer.toString());
-                            continue;
-                        }
-                        throw error;
+                    for (const entry of entries) {
+                        arr.push(entry);
                     }
                 }
             }
-            // We don't need this clause anymore because we got the trim option!
             if (!currentPeers.find((x) => x === this.node.identity.publicKey.hashcode())) {
-                let entriesToDelete = entries.filter((e) => !e.createdLocally);
-                if (this._sync) {
-                    // dont delete entries which we wish to keep
-                    entriesToDelete = await Promise.all(entriesToDelete.map((x) => this._sync(x))).then((filter) => entriesToDelete.filter((v, ix) => !filter[ix]));
-                }
-                // delete entries since we are not suppose to replicate this anymore
-                // TODO add delay? freeze time? (to ensure resiliance for bad io)
-                if (entriesToDelete.length > 0) {
-                    Promise.all(this.pruneSafely(entriesToDelete)).catch((e) => {
-                        logger.error(e.toString());
-                    });
+                if (currentPeers.length > 0) {
+                    // If we are observer, never prune locally created entries, since we dont really know who can store them
+                    // if we are replicator, we will always persist entries that we need to so filtering on createdLocally will not make a difference
+                    const entriesToDelete = this._role instanceof Observer
+                        ? entries.filter((e) => !e.createdLocally)
+                        : entries;
+                    entriesToDelete.map((x) => this._gidPeersHistory.delete(x.meta.gid));
+                    allEntriesToDelete.push(...entriesToDelete);
                 }
-                // TODO if length === 0 maybe close store?
             }
-            this._gidPeersHistory.set(gid, new Set(currentPeers));
-            if (toSend.size === 0) {
-                continue;
+            else {
+                for (const entry of entries) {
+                    this._pendingDeletes
+                        .get(entry.hash)
+                        ?.promise.reject(new Error("Failed to delete, is leader: " +
+                        this.role.constructor.name +
+                        ". " +
+                        this.node.identity.publicKey.hashcode()));
+                }
             }
-            const message = await createExchangeHeadsMessage(this.log, [...toSend.values()], // TODO send to peers directly
+        }
+        for (const [target, entries] of toDeliver) {
+            const message = await createExchangeHeadsMessage(this.log, entries, // TODO send to peers directly
             this._gidParentCache);
             // TODO perhaps send less messages to more receivers for performance reasons?
             await this.rpc.send(message, {
-                to: newPeers,
-                strict: true
+                to: [target]
             });
         }
-        if (storeChanged) {
-            await this.log.trim(); // because for entries createdLocally,we can have trim options that still allow us to delete them
-        }
-        return storeChanged || changed;
-    }
-    /**
-     *
-     * @returns groups where at least one in any group will have the entry you are looking for
-     */
-    getDiscoveryGroups() {
-        // TODO Optimize this so we don't have to recreate the array all the time!
-        const minReplicas = this.replicas.min.getValue(this);
-        const replicators = this.getReplicatorsSorted();
-        if (!replicators) {
-            return []; // No subscribers and we are not replicating
-        }
-        const numberOfGroups = Math.min(Math.ceil(replicators.length / minReplicas));
-        const groups = new Array(numberOfGroups);
-        for (let i = 0; i < groups.length; i++) {
-            groups[i] = [];
-        }
-        for (let i = 0; i < replicators.length; i++) {
-            groups[i % numberOfGroups].push(replicators[i]);
+        if (allEntriesToDelete.length > 0) {
+            this.prune(allEntriesToDelete).catch((e) => {
+                logger.error(e.toString());
+            });
         }
-        return groups;
-    }
-    async replicator(entry) {
-        return this.isLeader(entry.gid, decodeReplicas(entry).getValue(this));
+        return changed;
     }
     async _onUnsubscription(evt) {
-        logger.debug(`Peer disconnected '${evt.detail.from.hashcode()}' from '${JSON.stringify(evt.detail.unsubscriptions.map((x) => x.topic))}'`);
+        logger.debug(`Peer disconnected '${evt.detail.from.hashcode()}' from '${JSON.stringify(evt.detail.unsubscriptions.map((x) => x))}'`);
+        this.latestRoleMessages.delete(evt.detail.from.hashcode());
+        this.events.dispatchEvent(new CustomEvent("role", {
+            detail: { publicKey: evt.detail.from, role: new Observer() }
+        }));
         return this.handleSubscriptionChange(evt.detail.from, evt.detail.unsubscriptions, false);
     }
     async _onSubscription(evt) {
-        logger.debug(`New peer '${evt.detail.from.hashcode()}' connected to '${JSON.stringify(evt.detail.subscriptions.map((x) => x.topic))}'`);
+        logger.debug(`New peer '${evt.detail.from.hashcode()}' connected to '${JSON.stringify(evt.detail.subscriptions.map((x) => x))}'`);
+        this.remoteBlocks.onReachable(evt.detail.from);
         return this.handleSubscriptionChange(evt.detail.from, evt.detail.subscriptions, true);
     }
+    replicationController;
+    history;
+    async addToHistory(usedMemory, factor) {
+        (this.history || (this.history = [])).push({ usedMemory, factor });
+        // Keep only the last N entries in the history array (you can adjust N based on your needs)
+        const maxHistoryLength = 10;
+        if (this.history.length > maxHistoryLength) {
+            this.history.shift();
+        }
+    }
+    async calculateTrend() {
+        // Calculate the average change in factor per unit change in memory usage
+        const factorChanges = this.history.map((entry, index) => {
+            if (index > 0) {
+                const memoryChange = entry.usedMemory - this.history[index - 1].usedMemory;
+                if (memoryChange !== 0) {
+                    const factorChange = entry.factor - this.history[index - 1].factor;
+                    return factorChange / memoryChange;
+                }
+            }
+            return 0;
+        });
+        // Return the average factor change per unit memory change
+        return (factorChanges.reduce((sum, change) => sum + change, 0) /
+            factorChanges.length);
+    }
+    async rebalanceParticipation(onRoleChange = true) {
+        // update more participation rate to converge to the average expected rate or bounded by
+        // resources such as memory and or cpu
+        if (this.closed) {
+            return false;
+        }
+        // The role is fixed (no changes depending on memory usage or peer count etc)
+        if (this._roleOptions instanceof Role) {
+            return false;
+        }
+        // TODO second condition: what if the current role is Observer?
+        if (this._roleOptions.type == "replicator" &&
+            this._role instanceof Replicator) {
+            const peers = this.getReplicatorsSorted();
+            const usedMemory = await this.getMemoryUsage();
+            const newFactor = await this.replicationController.adjustReplicationFactor(usedMemory, this._role.factor, this._totalParticipation, peers?.length || 1);
+            const newRole = new Replicator({
+                factor: newFactor,
+                timestamp: this._role.timestamp
+            });
+            const relativeDifference = Math.abs(this._role.factor - newRole.factor) / this._role.factor;
+            if (relativeDifference > 0.0001) {
+                const canReplicate = !this._canReplicate ||
+                    (await this._canReplicate(this.node.identity.publicKey, newRole));
+                if (!canReplicate) {
+                    return false;
+                }
+                await this._updateRole(newRole, onRoleChange);
+                return true;
+            }
+            return false;
+        }
+        return false;
+    }
 };
 __decorate([
     field({ type: Log }),
@@ -686,4 +1107,19 @@ SharedLog = __decorate([
     __metadata("design:paramtypes", [Object])
 ], SharedLog);
 export { SharedLog };
+function _insertAfter(self, node, value) {
+    const inserted = !node
+        ? new yallist.Node(value, null, self.head, self)
+        : new yallist.Node(value, node, node.next, self);
+    // is tail
+    if (inserted.next === null) {
+        self.tail = inserted;
+    }
+    // is head
+    if (inserted.prev === null) {
+        self.head = inserted;
+    }
+    self.length++;
+    return inserted;
+}
 //# sourceMappingURL=index.js.map