@peerbit/shared-log 16.0.28 → 16.0.30

package/dist/src/index.js

@@ -186,7 +186,7 @@ const emitAdvisorySyncProfileDuration = (profile, startedAt, event) => {
         emitSyncProfileDuration(profile, startedAt, event);
     }
     catch {
-        // Diagnostics must not change open or provider-resolution correctness.
+        // Advisory diagnostics must not change replication or lifecycle behavior.
     }
 };
 const canUseOptionalNativeModuleImports = () => {
@@ -1964,7 +1964,7 @@ let SharedLog = (() => {
                 bumpSimpleFallbackPasses: () => {
                     this._repairMetrics["join-warmup"].simpleFallbackPasses += 1;
                 },
-                sendEntriesSimple: (target, entries, options) => this.sendRepairEntriesWithTransport(target, entries, "simple", options),
+                sendEntriesSimple: (target, entries, options) => this.sendRepairEntriesWithTransport(target, entries, "simple", options, "join-warmup"),
                 logError: (error) => logger.error(error),
             });
         }
@@ -3836,6 +3836,23 @@ let SharedLog = (() => {
                                         current.peerSession === captured.peerSession);
                                 };
                                 try {
+                                    if (!isPeerRoundCurrent())
+                                        return;
+                                    if (!this._v2Send.hasCurrentStateForPeer({
+                                        peerHash: peer,
+                                        peerSession: captured.peerSession,
+                                        receiverTransportSession: captured.capabilitySession,
+                                    })) {
+                                        // Preflight can precede replacement or loss of the sender
+                                        // stream. Delivery must recover its own exact current
+                                        // binding before waiting for application confirmation.
+                                        this._v2Receive.reAdvertiseLocalCapabilityForRemoteFull({
+                                            peerHash: peer,
+                                            peerSession: captured.peerSession,
+                                            receiveEpoch: captured.receiveEpoch,
+                                            signal: roundSignal,
+                                        });
+                                    }
                                     await this._v2Send.confirmLatestForPeer({
                                         peerHash: peer,
                                         peerSession: captured.peerSession,
@@ -6840,47 +6857,88 @@ let SharedLog = (() => {
                 signal,
             });
         }
-        async sendRepairEntriesWithTransport(target, entries, transport, options) {
+        async sendRepairEntriesWithTransport(target, entries, transport, options, mode) {
             const isStillCurrent = options?.isStillCurrent ?? (() => true);
             if (!isStillCurrent()) {
                 return;
             }
             const unknownEntries = new Map();
             const knownHashes = [];
-            for (const [hash, entry] of entries) {
-                if ((options?.bypassRecentKnownPeers ||
-                    !this.isEntryRecentlyKnownByPeer(hash, target, RECENT_KNOWN_REPAIR_SUPPRESSION_MS)) &&
-                    (options?.bypassKnownPeers || !this.isEntryKnownByPeer(hash, target))) {
-                    unknownEntries.set(hash, entry);
+            const profile = this._logProperties?.sync?.profile;
+            const startedAt = syncProfileStart(profile);
+            const inputEntries = profile ? entries.size : 0;
+            let selectedEntries = 0;
+            let lastObservedCurrent = true;
+            let outcome = "stale";
+            try {
+                for (const [hash, entry] of entries) {
+                    if ((options?.bypassRecentKnownPeers ||
+                        !this.isEntryRecentlyKnownByPeer(hash, target, RECENT_KNOWN_REPAIR_SUPPRESSION_MS)) &&
+                        (options?.bypassKnownPeers || !this.isEntryKnownByPeer(hash, target))) {
+                        unknownEntries.set(hash, entry);
+                    }
+                    else {
+                        knownHashes.push(hash);
+                    }
+                }
+                // A custom synchronizer may mutate the Map once it receives it.
+                if (profile)
+                    selectedEntries = unknownEntries.size;
+                if (!isStillCurrent())
+                    return;
+                this.clearRepairFrontierHashes(target, knownHashes);
+                if (unknownEntries.size === 0) {
+                    outcome = "known-suppressed";
+                    return;
+                }
+                if (transport === "simple") {
+                    // Observe only checks the lower path already makes, without adding
+                    // lifecycle decisions or wrapping the disabled-profiling path.
+                    const dispatchIsStillCurrent = profile
+                        ? () => (lastObservedCurrent = isStillCurrent())
+                        : isStillCurrent;
+                    // Fallback repair does not wait for the maybe-sync round trip.
+                    await this.pushRepairEntries(target, unknownEntries, dispatchIsStillCurrent, options?.signal);
                 }
                 else {
-                    knownHashes.push(hash);
+                    const syncEntries = this._logProperties?.sync?.priority
+                        ? this._coordinates.materializeRepairDispatchEntries(unknownEntries)
+                        : unknownEntries;
+                    if (!isStillCurrent())
+                        return;
+                    await this.syncronizer.onMaybeMissingEntries({
+                        entries: syncEntries,
+                        targets: [target],
+                        signal: options?.signal,
+                    });
                 }
+                outcome = !lastObservedCurrent
+                    ? "stale"
+                    : options?.signal?.aborted
+                        ? "cancelled"
+                        : "dispatched";
             }
-            if (!isStillCurrent()) {
-                return;
-            }
-            this.clearRepairFrontierHashes(target, knownHashes);
-            if (unknownEntries.size === 0) {
-                return;
-            }
-            if (transport === "simple") {
-                // Fallback repair should not depend on the target completing the
-                // RequestMaybeSync -> ResponseMaybeSync round trip.
-                await this.pushRepairEntries(target, unknownEntries, isStillCurrent, options?.signal);
-                return;
+            catch (error) {
+                outcome = "error";
+                throw error;
             }
-            const syncEntries = this._logProperties?.sync?.priority
-                ? this._coordinates.materializeRepairDispatchEntries(unknownEntries)
-                : unknownEntries;
-            if (!isStillCurrent()) {
-                return;
+            finally {
+                if (profile) {
+                    emitAdvisorySyncProfileDuration(profile, startedAt, {
+                        name: "sharedLog.repair.dispatch",
+                        component: "shared-log",
+                        entries: inputEntries,
+                        count: selectedEntries,
+                        targets: 1,
+                        details: {
+                            mode,
+                            transport,
+                            outcome,
+                            knownSuppressedEntries: knownHashes.length,
+                        },
+                    });
+                }
             }
-            await this.syncronizer.onMaybeMissingEntries({
-                entries: syncEntries,
-                targets: [target],
-                signal: options?.signal,
-            });
         }
         async sendMaybeMissingEntriesNow(target, entries, options, repairLifecycleController = this._instanceLifecycle
             ?.ownershipLifecycleController) {
@@ -6938,7 +6996,7 @@ let SharedLog = (() => {
                 bypassRecentKnownPeers: bypassKnownPeerHints,
                 isStillCurrent: () => this.isRepairLifecycleActive(repairLifecycleController),
                 signal: repairLifecycleController.signal,
-            })).catch((error) => logger.error(error));
+            }, options.mode)).catch((error) => logger.error(error));
         }
         ensureRepairFrontierRunner(mode, target, retryScheduleMs, repairLifecycleController = this._instanceLifecycle
             ?.ownershipLifecycleController) {
@@ -7198,7 +7256,7 @@ let SharedLog = (() => {
                     bypassRecentKnownPeers: bypassKnownPeerHints,
                     isStillCurrent: () => this.isRepairLifecycleActive(repairLifecycleController),
                     signal: repairLifecycleController.signal,
-                })).catch((error) => logger.error(error));
+                }, options.mode)).catch((error) => logger.error(error));
             };
             const delayedJoinWarmupRetries = [];
             retrySchedule.forEach((delayMs, index) => {
@@ -7303,6 +7361,18 @@ let SharedLog = (() => {
         }
         async runRepairSweep(repairLifecycleController = this._instanceLifecycle
             ?.ownershipLifecycleController) {
+            const profile = this._logProperties?.sync?.profile;
+            const startedAt = syncProfileStart(profile);
+            const profileCounts = profile
+                ? {
+                    passes: 0,
+                    inputEntries: 0,
+                    nativePasses: 0,
+                    repairCandidates: 0,
+                    repairBatches: 0,
+                    outcome: "stale",
+                }
+                : undefined;
             try {
                 while (this.isRepairLifecycleActive(repairLifecycleController)) {
                     if (!this.isRepairLifecycleActive(repairLifecycleController)) {
@@ -7334,8 +7404,12 @@ let SharedLog = (() => {
                     };
                     pruneStaleJoinWarmupPeers();
                     if (pendingModes.size === 0) {
+                        if (profileCounts)
+                            profileCounts.outcome = "completed";
                         return;
                     }
+                    if (profileCounts)
+                        profileCounts.passes += 1;
                     const optimisticGidPeersByMode = new Map();
                     const optimisticGidPeersConsumedByMode = new Map();
                     for (const mode of pendingModes) {
@@ -7406,6 +7480,10 @@ let SharedLog = (() => {
                             }
                             return;
                         }
+                        if (profileCounts) {
+                            profileCounts.repairCandidates += entries.size;
+                            profileCounts.repairBatches += 1;
+                        }
                         this.dispatchMaybeMissingEntries(target, entries, {
                             bypassRecentDedupe: true,
                             bypassKnownPeerHints: mode === "churn" ||
@@ -7456,6 +7534,10 @@ let SharedLog = (() => {
                     if ((this._nativeBackbone ?? this._nativeSharedLogState) &&
                         residentEntriesByHash &&
                         !this.hasCustomFindLeaders()) {
+                        if (profileCounts) {
+                            profileCounts.nativePasses += 1;
+                            profileCounts.inputEntries += residentEntriesByHash.size;
+                        }
                         const repairDispatchPlan = pruneStaleJoinWarmupPeers()
                             ? await this.planResidentRepairDispatchBatch({
                                 pendingModes,
@@ -7488,6 +7570,8 @@ let SharedLog = (() => {
                                 !iterator.done() &&
                                 pruneStaleJoinWarmupPeers()) {
                                 const entries = await iterator.next(REPAIR_SWEEP_ENTRY_BATCH_SIZE);
+                                if (profileCounts)
+                                    profileCounts.inputEntries += entries.length;
                                 if (!this.isRepairLifecycleActive(repairLifecycleController)) {
                                     return;
                                 }
@@ -7602,6 +7686,8 @@ let SharedLog = (() => {
                 }
             }
             catch (error) {
+                if (profileCounts)
+                    profileCounts.outcome = "error";
                 if (this.isRepairLifecycleActive(repairLifecycleController) &&
                     !isNotStartedError(error)) {
                     logger.error(`Repair sweep failed: ${error?.message ?? error}`);
@@ -7618,6 +7704,21 @@ let SharedLog = (() => {
                         void this.runRepairSweep(repairLifecycleController);
                     }
                 }
+                if (profileCounts) {
+                    emitAdvisorySyncProfileDuration(profile, startedAt, {
+                        name: "sharedLog.placement.pass",
+                        component: "shared-log",
+                        entries: profileCounts.inputEntries,
+                        count: profileCounts.repairCandidates,
+                        details: {
+                            phase: "repair-sweep",
+                            outcome: profileCounts.outcome,
+                            passes: profileCounts.passes,
+                            nativePasses: profileCounts.nativePasses,
+                            repairBatches: profileCounts.repairBatches,
+                        },
+                    });
+                }
             }
         }
         async pruneDebouncedFnAddIfNotKeeping(args, ownershipLifecycleController = this.captureReplicationOwnershipLifecycle(), additionalCurrentCheck) {
@@ -14374,10 +14475,11 @@ let SharedLog = (() => {
             const rawExistingStartedAt = syncProfileStart(syncProfile);
             const rawExistingHashes = await this.log.hasMany(msg.heads.map((head) => head.hash));
             if (syncProfile) {
-                emitSyncProfileDuration(syncProfile, rawExistingStartedAt, {
+                emitAdvisorySyncProfileDuration(syncProfile, rawExistingStartedAt, {
                     name: "sharedLog.rawReceive.existingHeads",
                     component: "shared-log",
                     entries: msg.heads.length,
+                    count: rawExistingHashes.size,
                     messages: 1,
                 });
             }
@@ -14694,10 +14796,11 @@ let SharedLog = (() => {
                             ? undefined
                             : await this.log.hasMany(headHashes);
                         if (syncProfile) {
-                            emitSyncProfileDuration(syncProfile, existingStartedAt, {
+                            emitAdvisorySyncProfileDuration(syncProfile, existingStartedAt, {
                                 name: "sharedLog.receive.existingHeads",
                                 component: "shared-log",
                                 entries: heads.length,
+                                count: existingHashes?.size,
                                 messages: 1,
                                 details: { rawMaterializedKnownMissing },
                             });
@@ -17264,6 +17367,23 @@ let SharedLog = (() => {
                     if (!continueWait())
                         return;
                     const recoveryTarget = confirmationRecoveryTarget;
+                    if (confirmationController && recoveryTarget) {
+                        if (this._peerSessions.current(peerHash) !==
+                            recoveryTarget.peerSession ||
+                            this._peerSessions.receiveEpoch(peerHash) !==
+                                recoveryTarget.receiveEpoch ||
+                            this._peerSyncCapabilitySessions.get(peerHash) !==
+                                recoveryTarget.capabilitySession ||
+                            !this.uniqueReplicators.has(peerHash)) {
+                            // Events can be coalesced while application confirmation waits.
+                            // Release the obsolete target on the recovery tick as well,
+                            // including loss of replicator eligibility within one session.
+                            // A temporary receive reservation keeps this same target alive;
+                            // the final readiness inspection still checks every admission gate.
+                            confirmationRecoveryTarget = undefined;
+                            confirmationController.abort(new AbortError("Persisted-receipt readiness target changed"));
+                        }
+                    }
                     this.nudgePersistedReceiptPeerReadiness(key, operationSignal, recoveryTarget && Date.now() >= recoveryTarget.notBefore
                         ? recoveryTarget
                         : undefined);
@@ -20395,6 +20515,17 @@ let SharedLog = (() => {
                     warmupSessions.get(target);
             const areJoinWarmupGenerationsCurrent = () => isOwnershipLifecycleCurrent() &&
                 [...warmupPeers].every(isCurrentJoinWarmupTarget);
+            const profile = this._logProperties?.sync?.profile;
+            const profileStartedAt = syncProfileStart(profile);
+            const profileCounts = profile
+                ? {
+                    examinedEntries: 0,
+                    repairCandidates: 0,
+                    repairBatches: 0,
+                    pruneScan: false,
+                    outcome: "stale",
+                }
+                : undefined;
             try {
                 const uncheckedDeliver = new Map();
                 const flushUncheckedDeliverTarget = (target) => {
@@ -20415,6 +20546,10 @@ let SharedLog = (() => {
                         : isWarmupTarget
                             ? "join-warmup"
                             : "join-authoritative";
+                    if (profileCounts) {
+                        profileCounts.repairCandidates += entries.size;
+                        profileCounts.repairBatches += 1;
+                    }
                     this.dispatchMaybeMissingEntries(target, entries, {
                         bypassRecentDedupe: isWarmupTarget || forceFreshDelivery,
                         bypassKnownPeerHints: forceFreshDelivery ||
@@ -20453,6 +20588,8 @@ let SharedLog = (() => {
                     for await (const entryReplicated of toRebalance(immediateRebalanceChanges, this.entryCoordinatesIndex, this.recentlyRebalanced, {
                         forceFresh: forceFreshDelivery || useJoinWarmupFastPath,
                     })) {
+                        if (profileCounts)
+                            profileCounts.examinedEntries += 1;
                         if (!isOwnershipLifecycleCurrent() ||
                             (useJoinWarmupFastPath && !areJoinWarmupGenerationsCurrent())) {
                             break;
@@ -20649,6 +20786,8 @@ let SharedLog = (() => {
                             change.type === "removed" ||
                             change.type === "replaced"));
                 if (shouldRunLocalPruneScan) {
+                    if (profileCounts)
+                        profileCounts.pruneScan = true;
                     throwIfOwnershipLifecycleInactive();
                     // Adaptive range changes and fixed zero-width updates can make already-indexed
                     // local heads prunable even when the incremental rebalance scan misses them
@@ -20667,6 +20806,8 @@ let SharedLog = (() => {
                         return false;
                     }
                 }
+                if (profileCounts)
+                    profileCounts.outcome = "completed";
                 return changed;
             }
             catch (error) {
@@ -20676,9 +20817,30 @@ let SharedLog = (() => {
                 if (isNotStartedError(error)) {
                     return false; // we are not started yet, so no changes
                 }
+                if (profileCounts)
+                    profileCounts.outcome = "error";
                 logger.error(error.toString());
                 throw error;
             }
+            finally {
+                if (profileCounts) {
+                    emitAdvisorySyncProfileDuration(profile, profileStartedAt, {
+                        name: "sharedLog.placement.pass",
+                        component: "shared-log",
+                        entries: profileCounts.examinedEntries,
+                        count: profileCounts.repairCandidates,
+                        details: {
+                            phase: "range-change",
+                            outcome: profileCounts.outcome,
+                            changes: changes.length,
+                            repairBatches: profileCounts.repairBatches,
+                            pruneScan: profileCounts.pruneScan,
+                            forceFreshDelivery,
+                            joinWarmupFastPath: useJoinWarmupFastPath,
+                        },
+                    });
+                }
+            }
         }
         async _onUnsubscription(evt) {
             logger.trace(`Peer disconnected '${evt.detail.from.hashcode()}' from '${JSON.stringify(evt.detail.topics.map((x) => x))} '`);
@@ -20707,6 +20869,13 @@ let SharedLog = (() => {
             await this.handleSubscriptionChange(evt.detail.from, evt.detail.topics, true, subscriptionEpoch, evt.detail.session);
         }
         async rebalanceParticipation(ownershipLifecycleController = this.captureReplicationOwnershipLifecycle(), rebalanceParticipationDebounced = this.rebalanceParticipationDebounced) {
+            const profile = this._isAdaptiveReplicating
+                ? this._logProperties?.sync?.profile
+                : undefined;
+            const profileStartedAt = syncProfileStart(profile);
+            const profileDetails = profile
+                ? { outcome: "stale", idleRemainingMs: 0 }
+                : undefined;
             // Stage 3: the lifecycle owns all three identity terms. `lifecycle` may
             // go stale later; its deps late-bind to the host, so the debouncer term
             // still reads the current host field, and the role term can disagree
@@ -20738,6 +20907,11 @@ let SharedLog = (() => {
                 }
                 if (this._isAdaptiveReplicating) {
                     if (this.shouldDelayAdaptiveRebalance()) {
+                        if (profileDetails) {
+                            profileDetails.outcome = "idle-deferred";
+                            profileDetails.idleRemainingMs = Math.max(0, this.adaptiveRebalanceIdleMs -
+                                (Date.now() - this._lastLocalAppendAt));
+                        }
                         if (isCurrent()) {
                             void rebalanceParticipationDebounced?.call();
                         }
@@ -20747,11 +20921,18 @@ let SharedLog = (() => {
                     const usedMemory = await this.getMemoryUsage();
                     if (!isCurrent())
                         return false;
+                    if (profileDetails) {
+                        profileDetails.storageUsedBytes = usedMemory;
+                        profileDetails.storageObjectiveBytes =
+                            this.replicationController.maxMemoryLimit;
+                    }
                     this.scheduleReplicationStatusRefreshForStorage(usedMemory);
                     let dynamicRange = await this.getDynamicRange();
                     if (!isCurrent())
                         return false;
                     if (!dynamicRange) {
+                        if (profileDetails)
+                            profileDetails.outcome = "not-permitted";
                         return; // not allowed to replicate
                     }
                     if (this.replicationController.maxMemoryLimit != null &&
@@ -20771,13 +20952,24 @@ let SharedLog = (() => {
                     const totalParticipation = await this.calculateTotalParticipation();
                     if (!isCurrent())
                         return false;
+                    const cpuUsage = this.cpuUsage?.value();
+                    const stepStartedAt = syncProfileStart(profile);
                     const newFactor = this.replicationController.step({
                         memoryUsage: usedMemory,
                         currentFactor: dynamicRange.widthNormalized,
                         totalFactor: totalParticipation, // TODO use this._totalParticipation when flakiness is fixed
                         peerCount: peersSize,
-                        cpuUsage: this.cpuUsage?.value(),
+                        cpuUsage,
                     });
+                    if (profileDetails) {
+                        profileDetails.preStepMs = stepStartedAt - profileStartedAt;
+                        profileDetails.stepMs = syncProfileStart(profile) - stepStartedAt;
+                        profileDetails.currentFactor = dynamicRange.widthNormalized;
+                        profileDetails.proposedFactor = newFactor;
+                        profileDetails.totalFactor = totalParticipation;
+                        profileDetails.controllerPeerCount = peersSize;
+                        profileDetails.cpuUsage = cpuUsage;
+                    }
                     const absoluteDifference = Math.abs(dynamicRange.widthNormalized - newFactor);
                     const relativeDifference = absoluteDifference /
                         Math.max(dynamicRange.widthNormalized, RECALCULATE_PARTICIPATION_RELATIVE_DENOMINATOR_FLOOR);
@@ -20805,8 +20997,11 @@ let SharedLog = (() => {
                         if (!isCurrent())
                             return false;
                         if (!canReplicate) {
+                            if (profileDetails)
+                                profileDetails.outcome = "not-permitted";
                             return false;
                         }
+                        const applyStartedAt = syncProfileStart(profile);
                         await this.startAnnounceReplicating([dynamicRange], {
                             checkDuplicates: false,
                             reset: false,
@@ -20814,6 +21009,10 @@ let SharedLog = (() => {
                         }, ownershipLifecycleController);
                         if (!isCurrent())
                             return false;
+                        if (profileDetails) {
+                            profileDetails.outcome = "apply-settled";
+                            profileDetails.applyMs = syncProfileStart(profile) - applyStartedAt;
+                        }
                         /* await this._updateRole(newRole, onRoleChange); */
                         if (isCurrent()) {
                             void rebalanceParticipationDebounced?.call();
@@ -20821,6 +21020,8 @@ let SharedLog = (() => {
                         return true;
                     }
                     else {
+                        if (profileDetails)
+                            profileDetails.outcome = "unchanged";
                         if (isCurrent()) {
                             void rebalanceParticipationDebounced?.call();
                         }
@@ -20829,13 +21030,27 @@ let SharedLog = (() => {
                 }
                 return false;
             };
-            const resp = await fn().catch((error) => {
-                if (isNotStartedError(error) || isClosedStoreRace(error)) {
-                    return false;
+            try {
+                return await fn().catch((error) => {
+                    if (isNotStartedError(error) || isClosedStoreRace(error)) {
+                        if (profileDetails)
+                            profileDetails.outcome = "stale";
+                        return false;
+                    }
+                    if (profileDetails)
+                        profileDetails.outcome = "error";
+                    throw error;
+                });
+            }
+            finally {
+                if (profileDetails) {
+                    emitAdvisorySyncProfileDuration(profile, profileStartedAt, {
+                        name: "sharedLog.adaptive.rebalance",
+                        component: "shared-log",
+                        details: profileDetails,
+                    });
                 }
-                throw error;
-            });
-            return resp;
+            }
         }
         getDynamicRangeOffset() {
             const options = this._logProperties