@peerbit/shared-log 13.0.8 → 13.0.10

package/dist/src/index.js

@@ -38,13 +38,13 @@ import { cidifyString } from "@peerbit/blocks-interface";
 import { Cache } from "@peerbit/cache";
 import { AccessError, PublicSignKey, getPublicKeyFromPeerId, sha256Base64Sync, sha256Sync, } from "@peerbit/crypto";
 import { And, ByteMatchQuery, NotStartedError as IndexNotStartedError, Or, Sort, StringMatch, toId, } from "@peerbit/indexer-interface";
-import { Entry, Log, Meta, ShallowEntry, } from "@peerbit/log";
+import { Entry, EntryType, Log, Meta, ShallowEntry, } from "@peerbit/log";
 import { logger as loggerFn } from "@peerbit/logger";
 import { ClosedError, Program } from "@peerbit/program";
 import { FanoutChannel, waitForSubscribers, } from "@peerbit/pubsub";
 import { SubscriptionEvent, UnsubcriptionEvent, } from "@peerbit/pubsub-interface";
 import { RPC } from "@peerbit/rpc";
-import { AcknowledgeDelivery, AnyWhere, createRequestTransportContext, DataMessage, MessageHeader, NotStartedError, SilentDelivery, } from "@peerbit/stream-interface";
+import { ACK_CONTROL_PRIORITY, AcknowledgeDelivery, AnyWhere, createRequestTransportContext, DataMessage, MessageHeader, NotStartedError, SilentDelivery, } from "@peerbit/stream-interface";
 import { AbortError, TimeoutError, debounceAccumulator, debounceFixedInterval, waitFor, } from "@peerbit/time";
 import pDefer, {} from "p-defer";
 import PQueue from "p-queue";
@@ -215,6 +215,8 @@ const RECALCULATE_PARTICIPATION_MIN_RELATIVE_CHANGE = 0.01;
 const RECALCULATE_PARTICIPATION_MIN_RELATIVE_CHANGE_WITH_CPU_LIMIT = 0.005;
 const RECALCULATE_PARTICIPATION_MIN_RELATIVE_CHANGE_WITH_MEMORY_LIMIT = 0.001;
 const RECALCULATE_PARTICIPATION_RELATIVE_DENOMINATOR_FLOOR = 1e-3;
+const ADAPTIVE_REBALANCE_IDLE_INTERVAL_MULTIPLIER = 5;
+const ADAPTIVE_REBALANCE_MIN_IDLE_AFTER_LOCAL_APPEND_MS = 10_000;
 const DEFAULT_DISTRIBUTION_DEBOUNCE_TIME = 500;
 const RECENT_REPAIR_DISPATCH_TTL_MS = 5_000;
 const REPAIR_SWEEP_ENTRY_BATCH_SIZE = 1_000;
@@ -348,6 +350,8 @@ let SharedLog = (() => {
         syncronizer;
         replicas;
         cpuUsage;
+        _lastLocalAppendAt;
+        adaptiveRebalanceIdleMs;
         timeUntilRoleMaturity;
         waitForReplicatorTimeout;
         waitForReplicatorRequestIntervalMs;
@@ -966,6 +970,51 @@ let SharedLog = (() => {
             ) */
             interval);
         }
+        markLocalAppendActivity(timestamp = Date.now()) {
+            this._lastLocalAppendAt = Math.max(this._lastLocalAppendAt ?? 0, timestamp);
+        }
+        shouldDelayAdaptiveRebalance(now = Date.now()) {
+            return (this._isAdaptiveReplicating &&
+                this._lastLocalAppendAt > 0 &&
+                now - this._lastLocalAppendAt < this.adaptiveRebalanceIdleMs);
+        }
+        shouldDeferHeadCoordinatePersistence(options) {
+            return (!this._isReplicating &&
+                options?.replicate === false &&
+                options?.target === "none");
+        }
+        async deleteCoordinatesForHashes(hashes) {
+            const values = [...new Set([...hashes].filter(Boolean))];
+            if (values.length === 0) {
+                return;
+            }
+            await this.entryCoordinatesIndex.del({
+                query: values.length === 1
+                    ? { hash: values[0] }
+                    : new Or(values.map((hash) => new StringMatch({ key: "hash", value: hash }))),
+            });
+        }
+        async ensureCurrentHeadCoordinatesIndexed() {
+            const heads = await this.log.getHeads(true).all();
+            const headsByHash = new Map(heads.map((head) => [head.hash, head]));
+            const indexedHeads = await this.entryCoordinatesIndex
+                .iterate({}, { shape: { hash: true } })
+                .all();
+            const indexedHashes = new Set(indexedHeads.map((entry) => entry.value.hash));
+            const staleHashes = indexedHeads
+                .map((entry) => entry.value.hash)
+                .filter((hash) => !headsByHash.has(hash));
+            if (staleHashes.length > 0) {
+                await this.deleteCoordinatesForHashes(staleHashes);
+            }
+            for (const head of heads) {
+                if (indexedHashes.has(head.hash)) {
+                    continue;
+                }
+                const minReplicas = decodeReplicas(head).getValue(this);
+                await this.findLeaders(await this.createCoordinates(head, minReplicas), head, { persist: {} });
+            }
+        }
         async _replicate(options, { reset, checkDuplicates, announce, mergeSegments, rebalance, } = {}) {
             let offsetWasProvided = false;
             if (isUnreplicationOptions(options)) {
@@ -1564,6 +1613,7 @@ let SharedLog = (() => {
             return diffs;
         }
         async startAnnounceReplicating(range, options = {}) {
+            await this.ensureCurrentHeadCoordinatesIndexed();
             const change = await this.addReplicationRange(range, this.node.identity.publicKey, options);
             if (!change) {
                 warn("Not allowed to replicate by canReplicate");
@@ -1889,6 +1939,9 @@ let SharedLog = (() => {
             this._checkedPruneRetries.set(hash, state);
         }
         async append(data, options) {
+            if (this._isAdaptiveReplicating) {
+                this.markLocalAppendActivity();
+            }
             const appendOptions = { ...options };
             const minReplicas = this.getClampedReplicas(options?.replicas
                 ? typeof options.replicas === "number"
@@ -1921,9 +1974,18 @@ let SharedLog = (() => {
                 };
             }
             const result = await this.log.append(data, appendOptions);
+            const deferHeadCoordinatePersistence = result.entry.meta.type !== EntryType.CUT &&
+                this.shouldDeferHeadCoordinatePersistence(options);
             if (options?.replicate) {
                 await this.replicate(result.entry, { checkDuplicates: true });
             }
+            if (deferHeadCoordinatePersistence) {
+                await this.deleteCoordinatesForHashes([
+                    ...result.entry.meta.next,
+                    ...result.removed.map((entry) => entry.hash),
+                ]);
+                return result;
+            }
             const coordinates = await this.createCoordinates(result.entry, minReplicasValue);
             const selfHash = this.node.identity.publicKey.hashcode();
             let isLeader = false;
@@ -1950,13 +2012,15 @@ let SharedLog = (() => {
                     await this._appendDeliverToReplicators(result.entry, minReplicasValue, leaders, selfHash, isLeader, deliveryArg);
                 }
             }
-            if (!isLeader) {
+            if (!isLeader && !this.shouldDelayAdaptiveRebalance()) {
                 this.pruneDebouncedFnAddIfNotKeeping({
                     key: result.entry.hash,
                     value: { entry: result.entry, leaders },
                 });
             }
-            this.rebalanceParticipationDebounced?.call();
+            if (!this._isAdaptiveReplicating) {
+                this.rebalanceParticipationDebounced?.call();
+            }
             return result;
         }
         async open(options) {
@@ -2002,6 +2066,13 @@ let SharedLog = (() => {
             this._replicatorLivenessCursor = 0;
             this._replicatorLivenessFailures = new Map();
             this._replicatorLastActivityAt = new Map();
+            this._lastLocalAppendAt = 0;
+            const adaptiveReplicateOptions = options?.replicate && isAdaptiveReplicatorOption(options.replicate)
+                ? options.replicate
+                : undefined;
+            this.adaptiveRebalanceIdleMs = Math.max(ADAPTIVE_REBALANCE_MIN_IDLE_AFTER_LOCAL_APPEND_MS, (adaptiveReplicateOptions?.limits?.interval ??
+                RECALCULATE_PARTICIPATION_DEBOUNCE_INTERVAL) *
+                ADAPTIVE_REBALANCE_IDLE_INTERVAL_MULTIPLIER);
             this.openTime = +new Date();
             this.oldestOpenTime = this.openTime;
             this.distributionDebounceTime =
@@ -2595,7 +2666,8 @@ let SharedLog = (() => {
                 // triggering large segment snapshots just to prove liveness.
                 await this.rpc.send(new ReplicationPingMessage(), {
                     mode: new AcknowledgeDelivery({ redundancy: 1, to: [publicKey] }),
-                    priority: 1,
+                    priority: ACK_CONTROL_PRIORITY,
+                    responsePriority: ACK_CONTROL_PRIORITY,
                 });
                 this.markReplicatorActivity(peerHash);
                 this._replicatorLivenessFailures.delete(peerHash);
@@ -2606,6 +2678,14 @@ let SharedLog = (() => {
                     return;
                 }
             }
+            // Relay-backed prod paths can keep a peer subscribed/reachable even if an
+            // ACKed liveness ping gets delayed or dropped under load. Treat observed
+            // topic presence as a positive liveness signal before evicting the peer.
+            if (await this.confirmReplicatorSubscriberPresence(peerHash)) {
+                this.markReplicatorActivity(peerHash);
+                this._replicatorLivenessFailures.delete(peerHash);
+                return;
+            }
             const failures = (this._replicatorLivenessFailures.get(peerHash) ?? 0) + 1;
             this._replicatorLivenessFailures.set(peerHash, failures);
             this.scheduleReplicationInfoRequests(publicKey);
@@ -2618,6 +2698,21 @@ let SharedLog = (() => {
             }
             await this.evictReplicatorFromLiveness(peerHash, publicKey);
         }
+        async confirmReplicatorSubscriberPresence(peerHash) {
+            try {
+                await waitForSubscribers(this.node, peerHash, this.rpc.topic, {
+                    signal: this._closeController.signal,
+                    timeout: Math.max(1_000, Math.min(5_000, Math.floor(this.waitForReplicatorTimeout / 4))),
+                });
+                return true;
+            }
+            catch (error) {
+                if (isNotStartedError(error)) {
+                    return false;
+                }
+                return false;
+            }
+        }
         async getMemoryUsage() {
             return this.log.blocks.size();
             /* ((await this.log.entryIndex?.getMemoryUsage()) || 0) */ // + (await this.log.blocks.size())
@@ -3416,13 +3511,16 @@ let SharedLog = (() => {
         }
         async join(entries, options) {
             let entriesToReplicate = [];
+            const localHashes = options?.replicate && this.log.length > 0
+                ? await this.log.entryIndex.hasMany(entries.map((element) => typeof element === "string" ? element : element.hash))
+                : new Set();
             if (options?.replicate && this.log.length > 0) {
                 // TODO this block should perhaps be called from a callback on the this.log.join method on all the ignored element because already joined, like "onAlreadyJoined"
                 // check which entrise we already have but not are replicating, and replicate them
                 // we can not just do the 'join' call because it will ignore the already joined entries
                 for (const element of entries) {
                     if (typeof element === "string") {
-                        if (await this.log.has(element)) {
+                        if (localHashes.has(element)) {
                             const entry = await this.log.get(element);
                             if (entry) {
                                 entriesToReplicate.push(entry);
@@ -3430,12 +3528,12 @@ let SharedLog = (() => {
                         }
                     }
                     else if (element instanceof Entry) {
-                        if (await this.log.has(element.hash)) {
+                        if (localHashes.has(element.hash)) {
                             entriesToReplicate.push(element);
                         }
                     }
                     else {
-                        if (await this.log.has(element.hash)) {
+                        if (localHashes.has(element.hash)) {
                             const entry = await this.log.get(element.hash);
                             if (entry) {
                                 entriesToReplicate.push(entry);
@@ -4569,6 +4667,13 @@ let SharedLog = (() => {
         async rebalanceParticipation() {
             // update more participation rate to converge to the average expected rate or bounded by
             // resources such as memory and or cpu
+            const isClosedStoreRace = (error) => {
+                const message = typeof error?.message === "string" ? error.message : String(error);
+                return (this.closed ||
+                    message.includes("Iterator is not open") ||
+                    message.includes("cannot read after close()") ||
+                    message.includes("Database is not open"));
+            };
             const fn = async () => {
                 if (this.closed) {
                     return false;
@@ -4578,6 +4683,10 @@ let SharedLog = (() => {
                     return false;
                 }
                 if (this._isAdaptiveReplicating) {
+                    if (this.shouldDelayAdaptiveRebalance()) {
+                        this.rebalanceParticipationDebounced?.call();
+                        return false;
+                    }
                     const peers = this.replicationIndex;
                     const usedMemory = await this.getMemoryUsage();
                     let dynamicRange = await this.getDynamicRange();
@@ -4635,7 +4744,12 @@ let SharedLog = (() => {
                 }
                 return false;
             };
-            const resp = await fn();
+            const resp = await fn().catch((error) => {
+                if (isNotStartedError(error) || isClosedStoreRace(error)) {
+                    return false;
+                }
+                throw error;
+            });
             return resp;
         }
         getDynamicRangeOffset() {