@peerbit/shared-log 2.0.0 → 3.0.0

package/lib/esm/index.js

@@ -14,10 +14,14 @@ import { Program } from "@peerbit/program";
 import { BinaryReader, BinaryWriter, BorshError, deserialize, field, serialize, variant, } from "@dao-xyz/borsh";
 import { AccessError, getPublicKeyFromPeerId, sha256, sha256Base64Sync, } from "@peerbit/crypto";
 import { logger as loggerFn } from "@peerbit/logger";
-import { AbsolutMinReplicas, ExchangeHeadsMessage, createExchangeHeadsMessage, } from "./exchange-heads.js";
+import { ExchangeHeadsMessage, RequestIHave, ResponseIHave, createExchangeHeadsMessage, } from "./exchange-heads.js";
 import { startsWith } from "@peerbit/uint8arrays";
 import { TimeoutError } from "@peerbit/time";
 import { REPLICATOR_TYPE_VARIANT, Observer, Replicator, Role } from "./role.js";
+import { AbsoluteReplicas, decodeReplicas, encodeReplicas, maxReplicas, } from "./replication.js";
+import pDefer from "p-defer";
+import { Cache } from "@peerbit/cache";
+export * from "./replication.js";
 export { Observer, Replicator, Role };
 export const logger = loggerFn({ module: "peer" });
 const groupByGid = async (entries) => {
@@ -40,7 +44,6 @@ export let SharedLog = class SharedLog extends Program {
     log;
     rpc;
     // options
-    _minReplicas;
     _sync;
     _role;
     _sortedPeersCache;
@@ -48,31 +51,88 @@ export let SharedLog = class SharedLog extends Program {
     _gidPeersHistory;
     _onSubscriptionFn;
     _onUnsubscriptionFn;
+    _canReplicate;
     _logProperties;
+    _loadedOnce = false;
+    _gidParentCache;
+    _respondToIHaveTimeout;
+    _pendingDeletes;
+    _pendingIHave;
+    replicas;
     constructor(properties) {
         super();
         this.log = new Log(properties);
         this.rpc = new RPC();
     }
-    get minReplicas() {
-        return this._minReplicas;
-    }
-    set minReplicas(minReplicas) {
-        this._minReplicas = minReplicas;
-    }
     get role() {
         return this._role;
     }
+    async updateRole(role) {
+        const wasRepicators = this._role instanceof Replicator;
+        this._role = role;
+        await this.initializeWithRole();
+        await this.rpc.subscribe(serialize(this._role));
+        if (wasRepicators) {
+            await this.replicationReorganization();
+        }
+    }
+    async initializeWithRole() {
+        try {
+            await this.modifySortedSubscriptionCache(this._role instanceof Replicator ? true : false, getPublicKeyFromPeerId(this.node.peerId));
+            if (!this._loadedOnce) {
+                await this.log.load();
+                this._loadedOnce = true;
+            }
+        }
+        catch (error) {
+            if (error instanceof AccessError) {
+                logger.error("Failed to load all entries due to access error, make sure you are opening the program with approate keychain configuration");
+            }
+            else {
+                throw error;
+            }
+        }
+    }
     async append(data, options) {
-        const result = await this.log.append(data, options);
-        await this.rpc.send(await createExchangeHeadsMessage(this.log, [result.entry], true));
+        const appendOptions = { ...options };
+        const minReplicasData = encodeReplicas(options?.replicas
+            ? typeof options.replicas === "number"
+                ? new AbsoluteReplicas(options.replicas)
+                : options.replicas
+            : this.replicas.min);
+        if (!appendOptions.meta) {
+            appendOptions.meta = {
+                data: minReplicasData,
+            };
+        }
+        else {
+            appendOptions.meta.data = minReplicasData;
+        }
+        const result = await this.log.append(data, appendOptions);
+        await this.rpc.send(await createExchangeHeadsMessage(this.log, [result.entry], this._gidParentCache));
         return result;
     }
     async open(options) {
-        this._minReplicas = new AbsolutMinReplicas(options?.minReplicas || 2);
+        this.replicas = {
+            min: options?.replicas?.min
+                ? typeof options?.replicas?.min === "number"
+                    ? new AbsoluteReplicas(options?.replicas?.min)
+                    : options?.replicas?.min
+                : new AbsoluteReplicas(DEFAULT_MIN_REPLICAS),
+            max: options?.replicas?.max
+                ? typeof options?.replicas?.max === "number"
+                    ? new AbsoluteReplicas(options?.replicas?.max)
+                    : options.replicas.max
+                : undefined,
+        };
+        this._respondToIHaveTimeout = options?.respondToIHaveTimeout ?? 10 * 1000; // TODO make into arg
+        this._pendingDeletes = new Map();
+        this._pendingIHave = new Map();
+        this._gidParentCache = new Cache({ max: 1000 });
+        this._canReplicate = options?.canReplicate;
         this._sync = options?.sync;
-        this._role = options?.role || new Replicator();
         this._logProperties = options;
+        this._role = options?.role || new Replicator();
         this._lastSubscriptionMessageId = 0;
         this._onSubscriptionFn = this._onSubscription.bind(this);
         this._sortedPeersCache = [];
@@ -83,36 +143,43 @@ export let SharedLog = class SharedLog extends Program {
         await this.log.open(this.node.services.blocks, this.node.identity, {
             keychain: this.node.keychain,
             ...this._logProperties,
+            onChange: (change) => {
+                if (this._pendingIHave.size > 0) {
+                    for (const added of change.added) {
+                        const ih = this._pendingIHave.get(added.hash);
+                        if (ih) {
+                            ih.clear();
+                            ih.callback();
+                        }
+                    }
+                }
+                return this._logProperties?.onChange?.(change);
+            },
+            canAppend: async (entry) => {
+                const replicas = decodeReplicas(entry).getValue(this);
+                if (Number.isFinite(replicas) === false) {
+                    return false;
+                }
+                // Don't verify entries that we have created (TODO should we? perf impact?)
+                if (!entry.createdLocally && !(await entry.verifySignatures())) {
+                    return false;
+                }
+                return this._logProperties?.canAppend?.(entry) ?? true;
+            },
             trim: this._logProperties?.trim && {
                 ...this._logProperties?.trim,
                 filter: {
-                    canTrim: async (gid) => !(await this.isLeader(gid)),
+                    canTrim: async (entry) => !(await this.isLeader(entry.meta.gid, decodeReplicas(entry).getValue(this))),
                     cacheId: () => this._lastSubscriptionMessageId,
                 },
             },
             cache: this.node.memory &&
                 (await this.node.memory.sublevel(sha256Base64Sync(this.log.id))),
         });
-        try {
-            if (this._role instanceof Replicator) {
-                this.modifySortedSubscriptionCache(true, getPublicKeyFromPeerId(this.node.peerId).hashcode());
-                await this.log.load();
-            }
-            else {
-                await this.log.load({ heads: true, reload: true });
-            }
-        }
-        catch (error) {
-            if (error instanceof AccessError) {
-                logger.error("Failed to load all entries due to access error, make sure you are opening the program with approate keychain configuration");
-            }
-            else {
-                throw error;
-            }
-        }
+        await this.initializeWithRole();
         // Take into account existing subscription
         (await this.node.services.pubsub.getSubscribers(this.topic))?.forEach((v, k) => {
-            this.handleSubscriptionChange(k, [{ topic: this.topic, data: v.data }], true);
+            this.handleSubscriptionChange(v.publicKey, [{ topic: this.topic, data: v.data }], true);
         });
         // Open for communcation
         await this.rpc.open({
@@ -127,8 +194,19 @@ export let SharedLog = class SharedLog extends Program {
         return this.log.idString;
     }
     async _close() {
+        for (const [k, v] of this._pendingDeletes) {
+            v.clear();
+            v.promise.resolve(); // TODO or reject?
+        }
+        for (const [k, v] of this._pendingIHave) {
+            v.clear();
+        }
+        this._gidParentCache.clear();
+        this._pendingDeletes = new Map();
+        this._pendingIHave = new Map();
         this._gidPeersHistory = new Map();
         this._sortedPeersCache = undefined;
+        this._loadedOnce = false;
         this.node.services.pubsub.removeEventListener("subscribe", this._onSubscriptionFn);
         this._onUnsubscriptionFn = this._onUnsubscription.bind(this);
         this.node.services.pubsub.removeEventListener("unsubscribe", this._onUnsubscriptionFn);
@@ -156,7 +234,7 @@ export let SharedLog = class SharedLog extends Program {
         try {
             if (msg instanceof ExchangeHeadsMessage) {
                 /**
-                 * I have recieved heads from someone else.
+                 * I have received heads from someone else.
                  * I can use them to load associated logs and join/sync them with the data stores I own
                  */
                 const { heads } = msg;
@@ -174,26 +252,109 @@ export let SharedLog = class SharedLog extends Program {
                             filteredHeads.push(head);
                         }
                     }
-                    let toMerge;
                     if (!this._sync) {
-                        toMerge = [];
-                        for (const [gid, value] of await groupByGid(filteredHeads)) {
-                            if (!(await this.isLeader(gid, this._minReplicas.value))) {
-                                logger.debug(`${this.node.identity.publicKey.hashcode()}: Dropping heads with gid: ${gid}. Because not leader`);
-                                continue;
+                        const toMerge = [];
+                        let toDelete = undefined;
+                        let maybeDelete = undefined;
+                        const groupedByGid = await groupByGid(filteredHeads);
+                        for (const [gid, entries] of groupedByGid) {
+                            const headsWithGid = this.log.headsIndex.gids.get(gid);
+                            const maxReplicasFromHead = headsWithGid && headsWithGid.size > 0
+                                ? maxReplicas(this, [...headsWithGid.values()])
+                                : this.replicas.min.getValue(this);
+                            const maxReplicasFromNewEntries = maxReplicas(this, [
+                                ...entries.map((x) => x.entry),
+                            ]);
+                            const isLeader = await this.isLeader(gid, Math.max(maxReplicasFromHead, maxReplicasFromNewEntries));
+                            if (maxReplicasFromNewEntries < maxReplicasFromHead && isLeader) {
+                                (maybeDelete || (maybeDelete = [])).push(entries);
+                            }
+                            outer: for (const entry of entries) {
+                                if (isLeader) {
+                                    toMerge.push(entry);
+                                }
+                                else {
+                                    for (const ref of entry.references) {
+                                        const map = this.log.headsIndex.gids.get(await ref.getGid());
+                                        if (map && map.size > 0) {
+                                            toMerge.push(entry);
+                                            (toDelete || (toDelete = [])).push(entry.entry);
+                                            continue outer;
+                                        }
+                                    }
+                                }
+                                logger.debug(`${this.node.identity.publicKey.hashcode()}: Dropping heads with gid: ${entry.entry.gid}. Because not leader`);
                             }
-                            for (const head of value) {
-                                toMerge.push(head);
+                        }
+                        if (toMerge.length > 0) {
+                            await this.log.join(toMerge);
+                            toDelete &&
+                                Promise.all(this.pruneSafely(toDelete)).catch((e) => {
+                                    logger.error(e.toString());
+                                });
+                        }
+                        if (maybeDelete) {
+                            for (const entries of maybeDelete) {
+                                const headsWithGid = this.log.headsIndex.gids.get(entries[0].entry.meta.gid);
+                                if (headsWithGid && headsWithGid.size > 0) {
+                                    const minReplicas = maxReplicas(this, [
+                                        ...headsWithGid.values(),
+                                    ]);
+                                    const isLeader = await this.isLeader(entries[0].entry.meta.gid, minReplicas);
+                                    if (!isLeader) {
+                                        Promise.all(this.pruneSafely(entries.map((x) => x.entry))).catch((e) => {
+                                            logger.error(e.toString());
+                                        });
+                                    }
+                                }
                             }
                         }
                     }
                     else {
-                        toMerge = await Promise.all(filteredHeads.map((x) => this._sync(x.entry))).then((filter) => filteredHeads.filter((v, ix) => filter[ix]));
+                        await this.log.join(await Promise.all(filteredHeads.map((x) => this._sync(x.entry))).then((filter) => filteredHeads.filter((v, ix) => filter[ix])));
                     }
-                    if (toMerge.length > 0) {
-                        await this.log.join(toMerge);
+                }
+            }
+            else if (msg instanceof RequestIHave) {
+                const hasAndIsLeader = [];
+                for (const hash of msg.hashes) {
+                    const indexedEntry = this.log.entryIndex.getShallow(hash);
+                    if (indexedEntry &&
+                        (await this.isLeader(indexedEntry.meta.gid, decodeReplicas(indexedEntry).getValue(this)))) {
+                        hasAndIsLeader.push(hash);
+                    }
+                    else {
+                        const prevPendingIHave = this._pendingIHave.get(hash);
+                        const pendingIHave = {
+                            clear: () => {
+                                clearTimeout(timeout);
+                                prevPendingIHave?.clear();
+                            },
+                            callback: () => {
+                                prevPendingIHave && prevPendingIHave.callback();
+                                this.rpc.send(new ResponseIHave({ hashes: [hash] }), {
+                                    to: [context.from],
+                                });
+                                this._pendingIHave.delete(hash);
+                            },
+                        };
+                        const timeout = setTimeout(() => {
+                            const pendingIHaveRef = this._pendingIHave.get(hash);
+                            if (pendingIHave === pendingIHaveRef) {
+                                this._pendingIHave.delete(hash);
+                            }
+                        }, this._respondToIHaveTimeout);
+                        this._pendingIHave.set(hash, pendingIHave);
                     }
                 }
+                this.rpc.send(new ResponseIHave({ hashes: hasAndIsLeader }), {
+                    to: [context.from],
+                });
+            }
+            else if (msg instanceof ResponseIHave) {
+                for (const hash of msg.hashes) {
+                    this._pendingDeletes.get(hash)?.callback(context.from.hashcode());
+                }
             }
             else {
                 throw new Error("Unexpected message");
@@ -214,11 +375,14 @@ export let SharedLog = class SharedLog extends Program {
     getReplicatorsSorted() {
         return this._sortedPeersCache;
     }
-    async isLeader(slot, numberOfLeaders = this.minReplicas.value) {
+    async isLeader(slot, numberOfLeaders) {
         const isLeader = (await this.findLeaders(slot, numberOfLeaders)).find((l) => l === this.node.identity.publicKey.hashcode());
         return !!isLeader;
     }
-    async findLeaders(subject, numberOfLeaders = this.minReplicas.value) {
+    async findLeaders(subject, numberOfLeadersUnbounded) {
+        const lower = this.replicas.min.getValue(this);
+        const higher = this.replicas.max?.getValue(this) ?? Number.MAX_SAFE_INTEGER;
+        let numberOfLeaders = Math.max(Math.min(higher, numberOfLeadersUnbounded), lower);
         // For a fixed set or members, the choosen leaders will always be the same (address invariant)
         // This allows for that same content is always chosen to be distributed to same peers, to remove unecessary copies
         const peers = this.getReplicatorsSorted() || [];
@@ -243,48 +407,69 @@ export let SharedLog = class SharedLog extends Program {
         }
         return leaders;
     }
-    modifySortedSubscriptionCache(subscribed, fromHash) {
+    async modifySortedSubscriptionCache(subscribed, publicKey) {
+        if (subscribed &&
+            this._canReplicate &&
+            !(await this._canReplicate(publicKey))) {
+            return false;
+        }
         const sortedPeer = this._sortedPeersCache;
         if (!sortedPeer) {
             if (this.closed === false) {
                 throw new Error("Unexpected, sortedPeersCache is undefined");
             }
-            return;
+            return false;
         }
-        const code = fromHash;
+        const code = publicKey.hashcode();
         if (subscribed) {
             // TODO use Set + list for fast lookup
             if (!sortedPeer.find((x) => x.hash === code)) {
                 sortedPeer.push({ hash: code, timestamp: +new Date() });
                 sortedPeer.sort((a, b) => a.hash.localeCompare(b.hash));
+                return true;
+            }
+            else {
+                return false;
             }
         }
         else {
             const deleteIndex = sortedPeer.findIndex((x) => x.hash === code);
-            sortedPeer.splice(deleteIndex, 1);
+            if (deleteIndex >= 0) {
+                sortedPeer.splice(deleteIndex, 1);
+                return true;
+            }
+            else {
+                return false;
+            }
         }
     }
-    async handleSubscriptionChange(fromHash, changes, subscribed) {
+    async handleSubscriptionChange(publicKey, changes, subscribed) {
         // TODO why are we doing two loops?
+        const prev = [];
         for (const subscription of changes) {
             if (this.log.idString !== subscription.topic) {
                 continue;
             }
             if (!subscription.data ||
                 !startsWith(subscription.data, REPLICATOR_TYPE_VARIANT)) {
+                prev.push(await this.modifySortedSubscriptionCache(false, publicKey));
                 continue;
             }
-            this._lastSubscriptionMessageId += 1;
-            this.modifySortedSubscriptionCache(subscribed, fromHash);
+            else {
+                this._lastSubscriptionMessageId += 1;
+                prev.push(await this.modifySortedSubscriptionCache(subscribed, publicKey));
+            }
         }
-        for (const subscription of changes) {
+        // TODO don't do this i fnot is replicator?
+        for (const [i, subscription] of changes.entries()) {
             if (this.log.idString !== subscription.topic) {
                 continue;
             }
             if (subscription.data) {
                 try {
                     const type = deserialize(subscription.data, Role);
-                    if (type instanceof Replicator) {
+                    // Reorganize if the new subscriber is a replicator, or observers AND was replicator
+                    if (type instanceof Replicator || prev[i]) {
                         await this.replicationReorganization();
                     }
                 }
@@ -297,6 +482,72 @@ export let SharedLog = class SharedLog extends Program {
             }
         }
     }
+    pruneSafely(entries, options) {
+        // ask network if they have they entry,
+        // so I can delete it
+        // There is a few reasons why we might end up here
+        // - Two logs merge, and we should not anymore keep the joined log replicated (because we are not responsible for the resulting gid)
+        // - An entry is joined, where min replicas is lower than before (for all heads for this particular gid) and therefore we are not replicating anymore for this particular gid
+        // - Peers join and leave, which means we might not be a replicator anymore
+        const promises = [];
+        const filteredEntries = [];
+        for (const entry of entries) {
+            const pendingPrev = this._pendingDeletes.get(entry.hash);
+            filteredEntries.push(entry);
+            const existCounter = new Set();
+            const minReplicas = decodeReplicas(entry);
+            const deferredPromise = pDefer();
+            const clear = () => {
+                pendingPrev?.clear();
+                const pending = this._pendingDeletes.get(entry.hash);
+                if (pending?.promise == deferredPromise) {
+                    this._pendingDeletes.delete(entry.hash);
+                }
+                clearTimeout(timeout);
+            };
+            const resolve = () => {
+                clear();
+                deferredPromise.resolve();
+            };
+            const reject = (e) => {
+                clear();
+                deferredPromise.reject(e);
+            };
+            const timeout = setTimeout(() => {
+                reject(new Error("Timeout"));
+            }, options?.timeout ?? 10 * 1000);
+            this._pendingDeletes.set(entry.hash, {
+                promise: deferredPromise,
+                clear: () => {
+                    clear();
+                },
+                callback: async (publicKeyHash) => {
+                    const minReplicasValue = minReplicas.getValue(this);
+                    const l = await this.findLeaders(entry.gid, minReplicasValue);
+                    if (l.find((x) => x === publicKeyHash)) {
+                        existCounter.add(publicKeyHash);
+                        if (minReplicas.getValue(this) <= existCounter.size) {
+                            this.log
+                                .remove(entry, {
+                                recursively: true,
+                            })
+                                .then(() => {
+                                resolve();
+                            })
+                                .catch((e) => {
+                                reject(new Error("Failed to delete entry: " + e.toString()));
+                            });
+                        }
+                    }
+                },
+            });
+            promises.push(deferredPromise.promise);
+        }
+        if (filteredEntries.length > 0) {
+            this.rpc.send(new RequestIHave({ hashes: filteredEntries.map((x) => x.hash) }));
+        }
+        return promises;
+    }
     /**
      * When a peers join the networkk and want to participate the leaders for particular log subgraphs might change, hence some might start replicating, might some stop
      * This method will go through my owned entries, and see whether I should share them with a new leader, and/or I should stop care about specific entries
@@ -314,12 +565,13 @@ export let SharedLog = class SharedLog extends Program {
                 continue; // TODO maybe close store?
             }
             const oldPeersSet = this._gidPeersHistory.get(gid);
-            const currentPeers = await this.findLeaders(gid);
+            const currentPeers = await this.findLeaders(gid, maxReplicas(this, entries) // pick max replication policy of all entries, so all information is treated equally important as the most important
+            );
             for (const currentPeer of currentPeers) {
                 if (!oldPeersSet?.has(currentPeer) &&
                     currentPeer !== this.node.identity.publicKey.hashcode()) {
                     storeChanged = true;
-                    // second condition means that if the new peer is us, we should not do anything, since we are expecting to recieve heads, not send
+                    // second condition means that if the new peer is us, we should not do anything, since we are expecting to receive heads, not send
                     newPeers.push(currentPeer);
                     // send heads to the new peer
                     // console.log('new gid for peer', newPeers.length, this.id.toString(), newPeer, gid, entries.length, newPeers)
@@ -348,8 +600,8 @@ export let SharedLog = class SharedLog extends Program {
                 // delete entries since we are not suppose to replicate this anymore
                 // TODO add delay? freeze time? (to ensure resiliance for bad io)
                 if (entriesToDelete.length > 0) {
-                    await this.log.remove(entriesToDelete, {
-                        recursively: true,
+                    Promise.all(this.pruneSafely(entriesToDelete)).catch((e) => {
+                        logger.error(e.toString());
                     });
                 }
                 // TODO if length === 0 maybe close store?
@@ -359,8 +611,8 @@ export let SharedLog = class SharedLog extends Program {
                 continue;
             }
             const message = await createExchangeHeadsMessage(this.log, [...toSend.values()], // TODO send to peers directly
-            true);
-            // TODO perhaps send less messages to more recievers for performance reasons?
+            this._gidParentCache);
+            // TODO perhaps send less messages to more receivers for performance reasons?
             await this.rpc.send(message, {
                 to: newPeers,
                 strict: true,
@@ -371,9 +623,13 @@ export let SharedLog = class SharedLog extends Program {
         }
         return storeChanged || changed;
     }
-    replicators() {
+    /**
+     *
+     * @returns groups where at least one in any group will have the entry you are looking for
+     */
+    getDiscoveryGroups() {
         // TODO Optimize this so we don't have to recreate the array all the time!
-        const minReplicas = this.minReplicas.value;
+        const minReplicas = this.replicas.min.getValue(this);
         const replicators = this.getReplicatorsSorted();
         if (!replicators) {
             return []; // No subscribers and we are not replicating
@@ -388,16 +644,16 @@ export let SharedLog = class SharedLog extends Program {
         }
         return groups;
     }
-    async replicator(gid) {
-        return this.isLeader(gid);
+    async replicator(entry) {
+        return this.isLeader(entry.gid, decodeReplicas(entry).getValue(this));
     }
     async _onUnsubscription(evt) {
         logger.debug(`Peer disconnected '${evt.detail.from.hashcode()}' from '${JSON.stringify(evt.detail.unsubscriptions.map((x) => x.topic))}'`);
-        return this.handleSubscriptionChange(evt.detail.from.hashcode(), evt.detail.unsubscriptions, false);
+        return this.handleSubscriptionChange(evt.detail.from, evt.detail.unsubscriptions, false);
     }
     async _onSubscription(evt) {
         logger.debug(`New peer '${evt.detail.from.hashcode()}' connected to '${JSON.stringify(evt.detail.subscriptions.map((x) => x.topic))}'`);
-        return this.handleSubscriptionChange(evt.detail.from.hashcode(), evt.detail.subscriptions, true);
+        return this.handleSubscriptionChange(evt.detail.from, evt.detail.subscriptions, true);
     }
 };
 __decorate([