@agentunion/fastaun 0.5.21 → 0.5.22

package/dist/client/delivery.js

@@ -2,6 +2,7 @@ import { _INTERNAL_MESSAGE_FLOW_LIMITS, slotIsolationKey } from '../config.js';
 import { parseFileFrontmatter } from '../agent-md-schema.js';
 import { AsyncLocalStorage } from 'node:async_hooks';
 import { trafficLogContext, withLogContext } from '../logger.js';
+import { normalizeGroupAid } from '../group-id.js';
 import { ConnectionState, isJsonObject } from '../types.js';
 import { validateGroupAIDFormat } from '../validators.js';
 import { hasGroupMentionModeField, normalizeGroupMentionMode } from './mention-mode.js';
@@ -12,6 +13,24 @@ const GROUP_RECALL_SEEN_LIMIT = 10_000;
 const VOLATILE_SEEN_LIMIT = 10_000;
 const SEQ_TRACKER_PERSIST_FLUSH_DELAY_MS = 200;
 const BACKGROUND_PULL_RETRY_DELAY_MS = 250;
+function isTerminalGroupPullError(error) {
+    if (!error || typeof error !== 'object')
+        return false;
+    const failure = error;
+    if (typeof failure.data?.retryable === 'boolean')
+        return !failure.data.retryable;
+    if (failure.retryable === true)
+        return false;
+    const message = String(failure.message ?? '').trim().toLowerCase();
+    if (['timeout', 'deadline exceeded', 'gateway service degraded', 'certificate not loaded',
+        'long_connection_already_exists', 'long connection already exists'].some((text) => message.includes(text)))
+        return false;
+    return [-32001, -32003, -32004, -32600, -32601, -32602].includes(failure.code ?? 0)
+        || (typeof failure.code === 'number' && failure.code >= -33009 && failure.code <= -33001)
+        || ['banned members', 'blocked members', 'member is banned', 'member is blocked',
+            'group is not active', 'group is dissolved', 'group is suspended',
+            'group not found', 'member not found', 'not a member'].some((text) => message.includes(text));
+}
 const DELIVERY_CHANGED_MESSAGE_EVENTS = new Set([
     'message.received',
     'group.message_created',
@@ -115,8 +134,12 @@ export class MessageDeliveryEngine {
     seqTrackerNamespaceGenerations = new Map();
     peerRecoveryTasks = new Map();
     peerRecoveryNeeds = new Map();
+    peerForwardNeeds = new Map();
+    peerPushVersions = new Map();
+    peerCatalogVersion = null;
     peerPullHints = new Map();
     peerPullConfirmedThrough = new Map();
+    peerServerConfirmedAcks = new Map();
     peerVolatileCursors = new Map();
     settledDeliveries = new Map();
     deliveryInflight = new Map();
@@ -124,9 +147,100 @@ export class MessageDeliveryEngine {
     applicationClaims = new Map();
     volatileCompletions = new Map();
     peerPageChains = new Map();
+    automaticPullFailures = new Map();
+    terminalGroupPulls = new Map();
+    automaticPullOperations = new WeakSet();
+    automaticPullContext = new AsyncLocalStorage();
+    automaticPullAbort = new AbortController();
     constructor(runtime) {
         this.runtime = runtime;
     }
+    resetAutomaticPullBackoff() {
+        this.automaticPullAbort.abort();
+        this.automaticPullAbort = new AbortController();
+        this.automaticPullFailures.clear();
+        this.terminalGroupPulls.clear();
+    }
+    groupPullNamespace(namespace) {
+        const groupId = namespace.startsWith('group_event:') ? namespace.slice('group_event:'.length)
+            : namespace.startsWith('group:') ? namespace.slice('group:'.length) : '';
+        return groupId ? `group:${normalizeGroupAid(groupId)}` : '';
+    }
+    automaticGroupPullStopped(namespace) {
+        return this.terminalGroupPulls.has(this.groupPullNamespace(namespace));
+    }
+    observeSuccessfulGroupPull(namespace) {
+        if (!this.automaticPullContext.getStore())
+            this.terminalGroupPulls.delete(this.groupPullNamespace(namespace));
+    }
+    stopAutomaticGroupPull(namespace, error) {
+        const canonical = this.groupPullNamespace(namespace);
+        if (!canonical)
+            return;
+        this.terminalGroupPulls.set(canonical, error);
+        for (const pending of [this.automaticPullFailures, this.backgroundPullNeeds, this.pendingGroupPullUpper, this.onlineUnreadHintTargets]) {
+            for (const ns of pending?.keys() ?? []) {
+                if (this.groupPullNamespace(ns) === canonical)
+                    pending.delete(ns);
+            }
+        }
+    }
+    automaticPullRetryAt(namespace, operation) {
+        return this.automaticPullOperations.has(operation) ? this.automaticPullFailures.get(namespace)?.retryAt ?? 0 : 0;
+    }
+    runPeerPullAttempt(namespace, operation, schedule) {
+        return this.automaticPullContext.getStore() ? this.runAutomaticPull(namespace, operation, schedule) : schedule(operation);
+    }
+    async runAutomaticPull(namespace, operation, schedule) {
+        this.ensurePeerContext();
+        if (this.automaticGroupPullStopped(namespace))
+            throw this.terminalGroupPulls.get(this.groupPullNamespace(namespace));
+        const nested = this.automaticPullContext.getStore() === namespace;
+        const generation = this.captureInlineAckGeneration();
+        const signal = this.automaticPullAbort.signal;
+        const client = this.runtime.client;
+        while ((this.automaticPullFailures.get(namespace)?.retryAt ?? 0) > Date.now()) {
+            const delay = this.automaticPullFailures.get(namespace).retryAt - Date.now();
+            await new Promise((resolve, reject) => {
+                const abort = () => {
+                    clearTimeout(timer);
+                    reject(new Error('pull invalidated'));
+                };
+                const timer = setTimeout(() => { signal.removeEventListener('abort', abort); resolve(); }, delay);
+                signal.addEventListener('abort', abort, { once: true });
+            });
+        }
+        if (signal.aborted || generation !== this.captureInlineAckGeneration() || client._closing || client.state !== ConnectionState.READY) {
+            throw new Error('pull invalidated');
+        }
+        const attempt = async () => {
+            if (this.automaticGroupPullStopped(namespace))
+                throw this.terminalGroupPulls.get(this.groupPullNamespace(namespace));
+            const previousFailure = this.automaticPullFailures.get(namespace);
+            try {
+                const result = await this.automaticPullContext.run(namespace, operation);
+                if (!nested && !signal.aborted && generation === this.captureInlineAckGeneration())
+                    this.automaticPullFailures.delete(namespace);
+                return result;
+            }
+            catch (error) {
+                if (!signal.aborted && generation === this.captureInlineAckGeneration()
+                    && this.groupPullNamespace(namespace) && isTerminalGroupPullError(error)) {
+                    this.stopAutomaticGroupPull(namespace, error);
+                    throw error;
+                }
+                const currentFailure = this.automaticPullFailures.get(namespace);
+                if ((!currentFailure || currentFailure === previousFailure)
+                    && !signal.aborted && generation === this.captureInlineAckGeneration() && !client._closing && client.state === ConnectionState.READY) {
+                    const delay = Math.min(30_000, (this.automaticPullFailures.get(namespace)?.delay ?? 500) * 2);
+                    this.automaticPullFailures.set(namespace, { delay, retryAt: Date.now() + delay });
+                }
+                throw error;
+            }
+        };
+        this.automaticPullOperations.add(attempt);
+        return await (schedule ? schedule(attempt) : attempt());
+    }
     currentApplicationPage() {
         const page = this.applicationPage.getStore();
         return page?.collecting && page.batching ? page : undefined;
@@ -143,6 +257,10 @@ export class MessageDeliveryEngine {
     snapshotApplicationHandlers() {
         return this.runtime.client._dispatcher._snapshotApplicationHandlers?.(['message.batch', 'group.message_batch', 'group.event_batch']);
     }
+    hasP2pApplicationSubscription() {
+        const handlers = this.runtime.client._dispatcher._snapshotApplicationHandlers?.(['message.received', 'message.recalled', 'message.batch']);
+        return !handlers || [...handlers.values()].some((selected) => selected.length > 0);
+    }
     async runApplicationPage(namespace, source, operation, reservation, snapshot) {
         const dispatcher = this.runtime.client._dispatcher;
         if (typeof dispatcher._snapshotApplicationHandlers !== 'function')
@@ -288,6 +406,7 @@ export class MessageDeliveryEngine {
         const client = this.runtime.client;
         const context = JSON.stringify([client._aid ?? '', client._deviceId ?? '', slotIsolationKey(client._slotId)]);
         if (context !== this.peerContext || client._seqTracker !== this.peerTracker) {
+            this.resetAutomaticPullBackoff();
             this.peerContext = context;
             this.peerTracker = client._seqTracker;
             this.peerContextGeneration += 1;
@@ -297,8 +416,12 @@ export class MessageDeliveryEngine {
             this.seqTrackerNamespaceGenerations.clear();
             this.peerVolatileCursors.clear();
             this.peerRecoveryNeeds.clear();
+            this.peerForwardNeeds.clear();
+            this.peerPushVersions.clear();
+            this.peerCatalogVersion = null;
             this.peerPullHints.clear();
             this.peerPullConfirmedThrough.clear();
+            this.peerServerConfirmedAcks.clear();
             this.peerRecoveryTasks.clear();
             this.settledDeliveries.clear();
             this.deliveryInflight.clear();
@@ -373,7 +496,14 @@ export class MessageDeliveryEngine {
     }
     peerCommittedAck(namespace) {
         this.ensurePeerContext();
-        return Math.min(this.committedSeqTrackerStates.get(namespace)?.ack ?? 0, this.syncState(namespace).ack);
+        const committed = this.committedSeqTrackerStates.get(namespace);
+        return committed ? Math.min(committed.ack, this.syncState(namespace).ack) : 0;
+    }
+    observePeerServerAck(namespace, sequence) {
+        this.ensurePeerContext();
+        if (!namespace.startsWith('p2p.peer.v1/') || !Number.isSafeInteger(sequence) || sequence < 0)
+            return;
+        this.peerServerConfirmedAcks.set(namespace, Math.max(this.peerServerConfirmedAcks.get(namespace) ?? 0, sequence));
     }
     rememberCommittedSeqState(namespace) {
         this.ensurePeerContext();
@@ -461,7 +591,16 @@ export class MessageDeliveryEngine {
         }
         for (const row of rows) {
             const ns = this.peerNamespace(String(row.sender_aid));
+            if (row.volatile_has_more === true || (nonNegativeSafeSequenceHint(row.head_seq) ?? 0) > Math.max(this.syncState(ns).head, client._seqTracker.getMaxSeenSeq(ns))) {
+                this.peerPushVersions.set(ns, {});
+            }
+            const head = nonNegativeSafeSequenceHint(row.head_seq);
+            if (head !== null)
+                client._seqTracker.updateMaxSeen(ns, head);
             const hint = nonNegativeSafeSequenceHint(row.next_pending_seq);
+            const serverAck = nonNegativeSafeSequenceHint(row.server_ack_seq);
+            if (serverAck !== null)
+                this.observePeerServerAck(ns, serverAck);
             const confirmed = Math.max(this.peerPullConfirmedThrough.get(ns) ?? 0, nonNegativeSafeSequenceHint(row.server_ack_seq) ?? nonNegativeSafeSequenceHint(row.initial_after_seq) ?? 0);
             this.peerPullConfirmedThrough.set(ns, confirmed);
             const current = this.peerPullHints.get(ns);
@@ -478,8 +617,40 @@ export class MessageDeliveryEngine {
             client._dispatcher.enqueue('message.peers_snapshot', snapshot, () => context === this.ensurePeerContext() && generation === this.captureInlineAckGeneration());
         return snapshot;
     }
+    capturePeerTailVersion(senderAid) {
+        const context = this.ensurePeerContext();
+        return { context, push: this.peerPushVersions.get(this.peerNamespace(senderAid)), catalog: this.peerCatalogVersion };
+    }
+    isPeerTailCurrent(senderAid, version, committedAck) {
+        const namespace = this.peerNamespace(senderAid);
+        return version.context === this.ensurePeerContext() && this.peerPushVersions.get(namespace) === version.push
+            && this.peerCatalogVersion === version.catalog && this.runtime.client._seqTracker.getMaxSeenSeq(namespace) <= committedAck
+            && this.peerPullPriority(namespace) <= committedAck && this.syncState(namespace).head <= committedAck;
+    }
+    shouldTailPeer(namespace, head, limit, expectedForwardNeed) {
+        const state = this.syncState(namespace);
+        const knownHead = Math.max(head, state.head, this.runtime.client._seqTracker.getMaxSeenSeq(namespace), this.peerPullPriority(namespace));
+        return this.peerOwnType() === 'human' && (!this.peerForwardNeeds.has(namespace) || this.peerForwardNeeds.get(namespace) === expectedForwardNeed) && state.ack >= state.head
+            && Number.isSafeInteger(limit) && limit > 0 && knownHead - this.peerCommittedAck(namespace) > limit;
+    }
+    canClosePeerTail(namespace, tail, pushVersion, catalogVersion, expectedNeed) {
+        const head = nonNegativeSafeSequenceHint(tail.head_seq);
+        const committedAck = this.peerCommittedAck(namespace);
+        const need = this.peerRecoveryNeeds.get(this.peerSender(namespace));
+        return tail.complete === true && tail.volatile_has_more === false && tail.pending_ack_seq === 0
+            && head !== null && committedAck >= head && this.syncState(namespace).head <= committedAck
+            && (this.peerServerConfirmedAcks.get(namespace) ?? 0) >= committedAck
+            && this.runtime.client._seqTracker.getMaxSeenSeq(namespace) <= committedAck
+            && this.peerPullPriority(namespace) <= committedAck
+            && this.peerPushVersions.get(namespace) === pushVersion
+            && this.peerCatalogVersion === catalogVersion
+            && (!need || (need === expectedNeed && !need.pending && need.head <= committedAck));
+    }
     async pullPeers(params = {}, background = false) {
         const client = this.runtime.client;
+        const context = this.ensurePeerContext();
+        const generation = this.captureInlineAckGeneration();
+        const isCurrent = () => context === this.ensurePeerContext() && generation === this.captureInlineAckGeneration();
         const sender = String(params.sender_aid ?? '').trim();
         if (!sender && ('after_seq' in params || 'after_volatile_cursor' in params || 'window_mode' in params || params.force === true))
             throw new Error('explicit peer is required for after_seq, after_volatile_cursor, window_mode or force');
@@ -488,30 +659,88 @@ export class MessageDeliveryEngine {
         const budget = { remaining: Math.max(1, Math.trunc(Number(params.max_pages ?? 100) || 100)) };
         const results = await Promise.all(rows.map(async (row) => {
             const peer = String(row.sender_aid);
+            const peerPullVersion = this.capturePeerTailVersion(peer);
             const ns = await this.initializePeer(peer);
             try {
-                const state = this.syncState(ns);
-                if (!params.window_mode && params.force !== true && this.peerOwnType() === 'human' && state.head <= state.ack && budget.remaining > 0) {
-                    const tail = await client._v2E2EE.pullPeerV2(peer, { ...params, window_mode: 'tail', budget, background });
+                if (!isCurrent())
+                    throw new Error('pull invalidated');
+                let forwardNeed = this.peerForwardNeeds.get(ns);
+                if (!params.window_mode && params.force !== true && this.shouldTailPeer(ns, Number(row.head_seq ?? 0), Number(params.limit ?? _INTERNAL_MESSAGE_FLOW_LIMITS.pullPageSize)) && budget.remaining > 0) {
+                    forwardNeed = {};
+                    this.peerForwardNeeds.set(ns, forwardNeed);
+                    const pushVersion = this.peerPushVersions.get(ns);
+                    const catalogVersion = this.peerCatalogVersion;
+                    const autoTail = { head: Number(row.head_seq ?? 0), forwardNeed, forward: false };
+                    const tail = await client._v2E2EE.pullPeerV2(peer, { ...params, window_mode: 'tail', budget, background, peerPullVersion, autoTail });
+                    if (!isCurrent())
+                        throw new Error('pull invalidated');
+                    if (autoTail.forward) {
+                        const notified = this.peerPushVersions.get(ns) !== pushVersion || this.peerCatalogVersion !== catalogVersion;
+                        if (notified)
+                            this.peerForwardNeeds.set(ns, {});
+                        else if (tail.complete === true && this.peerForwardNeeds.get(ns) === forwardNeed)
+                            this.peerForwardNeeds.delete(ns);
+                        return { ...tail, complete: tail.complete === true && !notified };
+                    }
+                    if (this.canClosePeerTail(ns, tail, pushVersion, catalogVersion)) {
+                        if (this.peerForwardNeeds.get(ns) === forwardNeed)
+                            this.peerForwardNeeds.delete(ns);
+                        return tail;
+                    }
                     if (!budget.remaining) {
-                        if (row.volatile_has_more)
-                            this.schedulePeerRecovery(peer, Number(tail.head_seq ?? 0), true);
-                        return { ...tail, complete: tail.complete && !row.volatile_has_more };
+                        this.schedulePeerRecovery(peer, Number(tail.head_seq ?? 0), true);
+                        return { ...tail, complete: false };
                     }
+                    forwardNeed = this.peerForwardNeeds.get(ns);
+                    const forwardPushVersion = this.peerPushVersions.get(ns);
+                    const forwardCatalogVersion = this.peerCatalogVersion;
                     const forward = await client._v2E2EE.pullPeerV2(peer, { ...params, budget, background });
+                    if (!isCurrent())
+                        throw new Error('pull invalidated');
+                    const notified = this.peerPushVersions.get(ns) !== forwardPushVersion || this.peerCatalogVersion !== forwardCatalogVersion;
+                    if (notified)
+                        this.peerForwardNeeds.set(ns, {});
+                    else if (forward.complete === true && this.peerForwardNeeds.get(ns) === forwardNeed)
+                        this.peerForwardNeeds.delete(ns);
                     const messages = [...new Map([...tail.messages, ...forward.messages].map((message) => [message.seq ?? message.volatile_cursor, message])).values()];
-                    return { ...forward, messages };
+                    return { ...forward, messages, complete: forward.complete === true && !notified };
                 }
-                return await client._v2E2EE.pullPeerV2(peer, { ...params, budget, background });
+                const pushVersion = this.peerPushVersions.get(ns);
+                const catalogVersion = this.peerCatalogVersion;
+                const result = await client._v2E2EE.pullPeerV2(peer, { ...params, budget, background, peerPullVersion });
+                if (!isCurrent())
+                    throw new Error('pull invalidated');
+                if (params.window_mode === 'tail') {
+                    const complete = this.canClosePeerTail(ns, result, pushVersion, catalogVersion);
+                    if (!complete) {
+                        this.peerForwardNeeds.set(ns, {});
+                        this.schedulePeerRecovery(peer, Number(result.head_seq ?? 0), true);
+                    }
+                    else if (this.peerForwardNeeds.get(ns) === forwardNeed)
+                        this.peerForwardNeeds.delete(ns);
+                    return { ...result, complete };
+                }
+                if (params.force !== true && (this.peerPushVersions.get(ns) !== pushVersion || this.peerCatalogVersion !== catalogVersion)) {
+                    this.peerForwardNeeds.set(ns, {});
+                    return { ...result, complete: false };
+                }
+                if (params.force !== true && result.complete === true && this.peerForwardNeeds.get(ns) === forwardNeed)
+                    this.peerForwardNeeds.delete(ns);
+                return result;
             }
             catch (error) {
-                this.schedulePeerRecovery(peer, Number(row.head_seq ?? 0), Boolean(row.volatile_has_more));
-                return { sender_aid: peer, messages: [], complete: false, ack_seq: this.syncState(ns).ack, error: error instanceof Error ? error.message : String(error) };
+                if (isCurrent())
+                    this.schedulePeerRecovery(peer, Number(row.head_seq ?? 0), Boolean(row.volatile_has_more));
+                return { sender_aid: peer, messages: [], complete: false, ack_seq: isCurrent() ? this.syncState(ns).ack : 0, error: error instanceof Error ? error.message : String(error) };
             }
         }));
         return { cursor_scope: 'peer', peers: results, messages: results.flatMap((row) => row.messages), partial: results.some((row) => !row.complete) };
     }
     refreshPendingPeers() {
+        this.ensurePeerContext();
+        this.peerCatalogVersion = {};
+        if (!this.hasP2pApplicationSubscription())
+            return;
         const client = this.runtime.client;
         if (client._sessionOptions?.background_sync === false || !client._aid || client._closing || client.state !== ConnectionState.READY)
             return;
@@ -521,9 +750,13 @@ export class MessageDeliveryEngine {
             return;
         const generation = this.captureInlineAckGeneration();
         const task = (async () => {
-            while (this.peerCatalogDirty && generation === this.captureInlineAckGeneration()) {
+            while (this.peerCatalogDirty && generation === this.captureInlineAckGeneration() && this.hasP2pApplicationSubscription()) {
                 this.peerCatalogDirty = false;
-                const result = await this.peers({}, true);
+                const result = await this.runAutomaticPull('message.peers', () => this.peers({}, true));
+                if (!this.hasP2pApplicationSubscription()) {
+                    this.peerCatalogDirty = true;
+                    return;
+                }
                 const rows = [...result.peers];
                 const hint = (row) => this.peerPullPriority(this.peerNamespace(String(row.sender_aid)));
                 rows.sort((left, right) => hint(left) - hint(right));
@@ -543,9 +776,29 @@ export class MessageDeliveryEngine {
     }
     schedulePeerRecovery(sender, head = 0, volatile = false) {
         const client = this.runtime.client;
-        if (client._sessionOptions?.background_sync === false || !sender || client._closing)
+        if (!sender || client._closing)
             return;
         this.ensurePeerContext();
+        if (Number.isSafeInteger(head) && head > 0 && !volatile) {
+            const namespace = this.peerNamespace(sender);
+            const committedAck = this.peerCommittedAck(namespace);
+            const serverAck = this.peerServerConfirmedAcks.get(namespace) ?? 0;
+            if (head <= committedAck && serverAck >= committedAck) {
+                const existing = this.peerRecoveryNeeds.get(sender);
+                if (!existing || this.peerRecoveryTasks.has(sender))
+                    return;
+                if (Number.isSafeInteger(existing.head) && existing.head > 0 && !existing.volatile && existing.head <= committedAck) {
+                    this.peerRecoveryNeeds.delete(sender);
+                    return;
+                }
+            }
+        }
+        const namespace = this.peerNamespace(sender);
+        this.peerPushVersions.set(namespace, {});
+        if (!this.hasP2pApplicationSubscription() || client._sessionOptions?.background_sync === false) {
+            this.peerForwardNeeds.set(namespace, {});
+            return;
+        }
         const need = this.peerRecoveryNeeds.get(sender) ?? { head: 0, volatile: false, pending: false };
         need.head = Math.max(need.head, head);
         need.volatile ||= volatile;
@@ -555,15 +808,67 @@ export class MessageDeliveryEngine {
             return;
         const generation = this.captureInlineAckGeneration();
         const task = (async () => {
-            await this.initializePeer(sender);
-            const state = this.syncState(this.peerNamespace(sender));
-            if (this.peerOwnType() === 'human' && need.head > state.head && state.ack >= state.tail - 1) {
-                await client._v2E2EE.pullPeerV2(sender, { window_mode: 'tail', background: true, retainMessages: false, budget: { remaining: 1 } });
+            await this.runAutomaticPull(`peer-init:${sender}`, () => this.initializePeer(sender));
+            if (!this.hasP2pApplicationSubscription())
+                return;
+            const namespace = this.peerNamespace(sender);
+            if (this.shouldTailPeer(namespace, need.head, _INTERNAL_MESSAGE_FLOW_LIMITS.pullPageSize)) {
+                const forwardNeed = {};
+                this.peerForwardNeeds.set(namespace, forwardNeed);
+                need.pending = false;
+                const pushVersion = this.peerPushVersions.get(namespace);
+                const catalogVersion = this.peerCatalogVersion;
+                const autoTail = { head: need.head, forwardNeed, forward: false };
+                const tail = await this.runAutomaticPull(namespace, async () => {
+                    const beforeAck = this.peerCommittedAck(namespace);
+                    const beforeVolatile = this.peerVolatilePullCursor(sender);
+                    const result = await client._v2E2EE.pullPeerV2(sender, { window_mode: 'tail', background: true, retainMessages: false, budget: { remaining: 1 }, autoTail });
+                    if (autoTail.forward && !result.complete && this.peerCommittedAck(namespace) <= beforeAck && this.peerVolatilePullCursor(sender) === beforeVolatile) {
+                        throw new Error('peer recovery made no progress');
+                    }
+                    return result;
+                });
+                if (generation !== this.captureInlineAckGeneration() || this.peerRecoveryNeeds.get(sender) !== need)
+                    return;
+                if (autoTail.forward) {
+                    if (tail.complete && this.peerForwardNeeds.get(namespace) === forwardNeed
+                        && this.peerPushVersions.get(namespace) === pushVersion && this.peerCatalogVersion === catalogVersion)
+                        this.peerForwardNeeds.delete(namespace);
+                    if (tail.complete && Number(tail.ack_seq) >= need.head && !need.pending
+                        && this.peerPushVersions.get(namespace) === pushVersion && this.peerCatalogVersion === catalogVersion) {
+                        this.peerRecoveryNeeds.delete(sender);
+                        return;
+                    }
+                    await new Promise((resolve) => setTimeout(resolve, BACKGROUND_PULL_RETRY_DELAY_MS));
+                }
+                else if (this.canClosePeerTail(namespace, tail, pushVersion, catalogVersion, need)) {
+                    this.peerRecoveryNeeds.delete(sender);
+                    if (this.peerForwardNeeds.get(namespace) === forwardNeed)
+                        this.peerForwardNeeds.delete(namespace);
+                    return;
+                }
             }
-            while (generation === this.captureInlineAckGeneration() && this.peerRecoveryNeeds.has(sender)) {
+            while (generation === this.captureInlineAckGeneration() && this.peerRecoveryNeeds.has(sender) && this.hasP2pApplicationSubscription()) {
                 need.pending = false;
-                const result = await client._v2E2EE.pullPeerV2(sender, { background: true, retainMessages: false, budget: { remaining: 100 } });
-                if (result.complete && result.ack_seq >= need.head && !need.pending) {
+                const forwardNeed = this.peerForwardNeeds.get(namespace);
+                const pushVersion = this.peerPushVersions.get(namespace);
+                const catalogVersion = this.peerCatalogVersion;
+                const result = await this.runAutomaticPull(namespace, async () => {
+                    const beforeAck = this.peerCommittedAck(namespace);
+                    const beforeVolatile = this.peerVolatilePullCursor(sender);
+                    const result = await client._v2E2EE.pullPeerV2(sender, { background: true, retainMessages: false, budget: { remaining: 100 } });
+                    if (!result.complete && this.peerCommittedAck(namespace) <= beforeAck && this.peerVolatilePullCursor(sender) === beforeVolatile) {
+                        throw new Error('peer recovery made no progress');
+                    }
+                    return result;
+                });
+                if (generation !== this.captureInlineAckGeneration() || this.peerRecoveryNeeds.get(sender) !== need)
+                    return;
+                if (result.complete && this.peerForwardNeeds.get(namespace) === forwardNeed
+                    && this.peerPushVersions.get(namespace) === pushVersion && this.peerCatalogVersion === catalogVersion)
+                    this.peerForwardNeeds.delete(namespace);
+                if (result.complete && result.ack_seq >= need.head && !need.pending
+                    && this.peerPushVersions.get(namespace) === pushVersion && this.peerCatalogVersion === catalogVersion) {
                     this.peerRecoveryNeeds.delete(sender);
                     break;
                 }
@@ -979,6 +1284,7 @@ export class MessageDeliveryEngine {
         return processed.map((message) => this.publicVolatileMessage(message));
     }
     resetInlineAckState() {
+        this.resetAutomaticPullBackoff();
         this.syncStopped('aborted');
         this.deliveryChangedGeneration += 1;
         this.pendingDeliveryChanges = null;
@@ -1003,8 +1309,12 @@ export class MessageDeliveryEngine {
         this.peerCatalogTask = null;
         this.peerCatalogDirty = false;
         this.peerRecoveryNeeds.clear();
+        this.peerForwardNeeds.clear();
+        this.peerPushVersions.clear();
+        this.peerCatalogVersion = null;
         this.peerPullHints.clear();
         this.peerPullConfirmedThrough.clear();
+        this.peerServerConfirmedAcks.clear();
         this.peerRecoveryTasks.clear();
     }
     deliveryChangedNamespace(event, payload) {
@@ -1295,6 +1605,8 @@ export class MessageDeliveryEngine {
         this.schedulePendingGroupPullIfNeeded(ns, 'pull-gate-idle');
     }
     recordBackgroundPullNeed(ns, throughSeq = 0, probe = false) {
+        if (this.automaticGroupPullStopped(ns))
+            return;
         const client = this.runtime.client;
         if (client._sessionOptions?.background_sync === false || !client._aid
             || !(ns === `p2p:${client._aid}` || ns.startsWith('group:') || ns.startsWith('group_event:')))
@@ -1329,6 +1641,8 @@ export class MessageDeliveryEngine {
     finishBackgroundPull(ns, generation, afterSeq, failed) {
         if (generation !== this.captureInlineAckGeneration())
             return;
+        if (this.automaticGroupPullStopped(ns))
+            return;
         const need = this.backgroundPullNeeds.get(ns);
         if (need) {
             if (failed)
@@ -1362,6 +1676,12 @@ export class MessageDeliveryEngine {
             }
             let retryAt = Infinity;
             for (const [ns, need] of this.backgroundPullNeeds) {
+                if (this.automaticGroupPullStopped(ns)) {
+                    this.backgroundPullNeeds.delete(ns);
+                    continue;
+                }
+                if (ns.startsWith('p2p:') && !this.hasP2pApplicationSubscription())
+                    continue;
                 if (this.backgroundPullTasks.has(ns) || this.realtimeSyncing?.has(ns) || this.onlineUnreadHintTasks?.has(ns))
                     continue;
                 const eventScope = ns.startsWith('group_event:');
@@ -1428,6 +1748,8 @@ export class MessageDeliveryEngine {
         client._safeAsync(this.fillP2pGap());
     }
     recordPendingGroupPull(ns, seq) {
+        if (this.automaticGroupPullStopped(ns))
+            return;
         if (!ns || !Number.isSafeInteger(seq) || seq <= 0)
             return;
         const pending = this.pendingGroupPullUpper ?? new Map();
@@ -1435,6 +1757,8 @@ export class MessageDeliveryEngine {
         pending.set(ns, Math.max(pending.get(ns) ?? 0, seq));
     }
     schedulePendingGroupPullIfNeeded(ns, reason) {
+        if (this.automaticGroupPullStopped(ns))
+            return false;
         const pending = this.pendingGroupPullUpper;
         if (!pending)
             return false;
@@ -2580,13 +2904,15 @@ export class MessageDeliveryEngine {
         };
         if (client._rpcPipeline) {
             const key = client._rpcPipeline.pullGateKeyForCall('group.pull', { group_id: groupId, after_seq: afterSeq, limit: 50 });
-            await client._rpcPipeline.runPullNamespaceSerialized(key, operation, true);
+            await this.runAutomaticPull(ns, operation, (attempt) => client._rpcPipeline.runPullNamespaceSerialized(key, attempt, true));
         }
         else {
-            await client._withBackgroundRpc(operation);
+            await this.runAutomaticPull(ns, operation, (attempt) => client._withBackgroundRpc(attempt));
         }
     }
     async fillP2pGap() {
+        if (!this.hasP2pApplicationSubscription())
+            return;
         const client = this.runtime.client;
         if (client._p2pPeerPullEnabled) {
             this.refreshPendingPeers();
@@ -2611,9 +2937,9 @@ export class MessageDeliveryEngine {
             // Gap Fill 本身就是 Forward Pull，必须沿用协调器的完整事务：
             // 原始页提交成功后由协调器负责 Piggyback/终端 ACK。
             const operation = () => client._pullV2(afterSeq, 50, { gateLocked: true });
-            const messages = client._rpcPipeline
-                ? await client._rpcPipeline.runPullSerialized(client._rpcPipeline.pullGateKeyForCall('message.pull', { after_seq: afterSeq, limit: 50 }), operation, true)
-                : await client._withBackgroundRpc(operation);
+            const messages = await this.runAutomaticPull(ns, operation, (attempt) => client._rpcPipeline
+                ? client._rpcPipeline.runPullSerialized(client._rpcPipeline.pullGateKeyForCall('message.pull', { after_seq: afterSeq, limit: 50 }), attempt, true)
+                : client._withBackgroundRpc(attempt));
             filled = messages.length;
             this.prunePushedSeqs(ns);
             client._clientLog.debug(`P2P message gap fill done: after_seq=${afterSeq}, filled=${filled}`);
@@ -2638,6 +2964,8 @@ export class MessageDeliveryEngine {
         if (!groupId)
             return;
         const ns = `group:${groupId}`;
+        if (this.automaticGroupPullStopped(ns))
+            return;
         const afterSeq = client._seqTracker.getContiguousSeq(ns);
         const generation = this.captureInlineAckGeneration();
         const dedupKey = `group_msg:${groupId}:${afterSeq}`;
@@ -2650,9 +2978,9 @@ export class MessageDeliveryEngine {
         let failed = false;
         try {
             const operation = () => client._pullGroupV2(groupId, afterSeq, 50, { gateLocked: true, background: true });
-            const messages = client._rpcPipeline
-                ? await client._rpcPipeline.runPullNamespaceSerialized(client._rpcPipeline.pullGateKeyForCall('group.pull', { group_id: groupId, after_seq: afterSeq, limit: 50 }), operation, true)
-                : await client._withBackgroundRpc(operation);
+            const messages = await this.runAutomaticPull(ns, operation, (attempt) => client._rpcPipeline
+                ? client._rpcPipeline.runPullNamespaceSerialized(client._rpcPipeline.pullGateKeyForCall('group.pull', { group_id: groupId, after_seq: afterSeq, limit: 50 }), attempt, true)
+                : client._withBackgroundRpc(attempt));
             filled = messages.length;
             this.prunePushedSeqs(ns);
             client._clientLog.debug(`group message gap fill done: group=${groupId}, after_seq=${afterSeq}, filled=${filled}`);
@@ -2708,6 +3036,8 @@ export class MessageDeliveryEngine {
         if (!groupId)
             return;
         const ns = `group_event:${groupId}`;
+        if (this.automaticGroupPullStopped(ns))
+            return;
         const generation = this.captureInlineAckGeneration();
         const requestedAfterSeq = Number(options.afterSeq ?? 0);
         const afterSeq = Number.isSafeInteger(requestedAfterSeq) && requestedAfterSeq > 0
@@ -2751,10 +3081,10 @@ export class MessageDeliveryEngine {
                 let result;
                 if (client._rpcPipeline) {
                     const pullKey = client._rpcPipeline.pullGateKeyForCall('group.pull_events', request);
-                    result = await client._rpcPipeline.runPullSerialized(pullKey, () => client.call('group.pull_events', request), true);
+                    result = await this.runAutomaticPull(ns, () => client.call('group.pull_events', request), (attempt) => client._rpcPipeline.runPullSerialized(pullKey, attempt, true));
                 }
                 else {
-                    result = await client.call('group.pull_events', request);
+                    result = await this.runAutomaticPull(ns, () => client.call('group.pull_events', request));
                 }
                 if (!isJsonObject(result))
                     return;
@@ -2923,6 +3253,8 @@ export class MessageDeliveryEngine {
     enqueueOnlineUnreadEventHint(data) {
         const client = this.runtime.client;
         const groupId = String(data.group_id ?? '').trim();
+        if (this.automaticGroupPullStopped(`group_event:${groupId}`))
+            return;
         if (!groupId || client._sessionOptions?.background_sync === false)
             return;
         this.recordBackgroundPullNeed(`group_event:${groupId}`, positiveSafeSequenceHint(data.event_seq), true);
@@ -3252,9 +3584,9 @@ export class MessageDeliveryEngine {
                 return { messages, raw_count: messages.length };
             };
             if (!client._rpcPipeline)
-                return await invoke();
+                return await this.runAutomaticPull(ns, invoke);
             const key = client._rpcPipeline.pullGateKeyForCall('message.pull', request);
-            return await client._rpcPipeline.runPullSerialized(key, invoke, background);
+            return await this.runAutomaticPull(ns, invoke, (attempt) => client._rpcPipeline.runPullSerialized(key, attempt, background));
         };
         if (order.length === 1 && order[0] === 'tail') {
             result = await run({ window_mode: 'tail', after_seq: this.syncState(ns).ack, limit: pageLimit }, false);
@@ -3330,9 +3662,9 @@ export class MessageDeliveryEngine {
                 return { messages, raw_count: messages.length };
             };
             if (!client._rpcPipeline)
-                return await invoke();
+                return await this.runAutomaticPull(ns, invoke);
             const key = client._rpcPipeline.pullGateKeyForCall('group.pull', { group_id: groupId, ...request });
-            return await client._rpcPipeline.runPullNamespaceSerialized(key, invoke, background);
+            return await this.runAutomaticPull(ns, invoke, (attempt) => client._rpcPipeline.runPullNamespaceSerialized(key, attempt, background));
         };
         if (order.length === 1 && order[0] === 'tail') {
             result = await run({ window_mode: 'tail', after_seq: this.syncState(ns).ack, limit: pageLimit }, false);
@@ -3373,9 +3705,9 @@ export class MessageDeliveryEngine {
             return { messages, raw_count: messages.length };
         };
         if (!client._rpcPipeline)
-            return await invoke();
+            return await this.runAutomaticPull(ns, invoke);
         const key = client._rpcPipeline.pullGateKeyForCall('message.pull', request);
-        return await client._rpcPipeline.runPullSerialized(key, invoke, background);
+        return await this.runAutomaticPull(ns, invoke, (attempt) => client._rpcPipeline.runPullSerialized(key, attempt, background));
     }
     async runGroupForwardRecovery(groupId, ns, pageLimit, background = false, throughSeq = 0) {
         const client = this.runtime.client;
@@ -3387,9 +3719,9 @@ export class MessageDeliveryEngine {
             return { messages, raw_count: messages.length };
         };
         if (!client._rpcPipeline)
-            return await invoke();
+            return await this.runAutomaticPull(ns, invoke);
         const key = client._rpcPipeline.pullGateKeyForCall('group.pull', request);
-        return await client._rpcPipeline.runPullNamespaceSerialized(key, invoke, background);
+        return await this.runAutomaticPull(ns, invoke, (attempt) => client._rpcPipeline.runPullNamespaceSerialized(key, attempt, background));
     }
     syncState(ns) {
         const tracker = this.runtime.client._seqTracker;
@@ -4018,15 +4350,31 @@ export class MessageDeliveryEngine {
             return;
         }
         const sender = String(head.sender_aid);
-        const ns = await this.initializePeer(sender);
+        this.ensurePeerContext();
+        this.peerPushVersions.set(this.peerNamespace(sender), {});
+        if (!hasNonEmptyInlineMessage(head.inline_message) && !this.isVolatileMessage(head) && !this.hasP2pApplicationSubscription())
+            return;
         const generation = this.captureInlineAckGeneration();
+        let ns;
+        try {
+            ns = await this.runAutomaticPull(`peer-init:${sender}`, () => this.initializePeer(sender));
+        }
+        catch (error) {
+            if (generation === this.captureInlineAckGeneration())
+                this.schedulePeerRecovery(sender, positiveSafeSequenceHint(head.seq) ?? 0, this.isVolatileMessage(head));
+            throw error;
+        }
         const seq = positiveSafeSequenceHint(head.seq);
         if (this.isVolatileMessage(head)) {
             await client._v2E2EE.publishV2VolatileMessages([head], 'push', false);
             return;
         }
         if (!seq) {
-            this.refreshPendingPeers();
+            if (client._sessionOptions?.background_sync === false && this.hasP2pApplicationSubscription()) {
+                await this.runAutomaticPull(ns, () => client._v2E2EE.pullPeerV2(sender, { limit: _INTERNAL_MESSAGE_FLOW_LIMITS.pullPageSize, budget: { remaining: 1 } }));
+            }
+            else
+                this.refreshPendingPeers();
             return;
         }
         client._seqTracker.updateMaxSeen(ns, seq);
@@ -4065,14 +4413,14 @@ export class MessageDeliveryEngine {
             if (consumed?.acquired && consumed.value)
                 return;
         }
-        if (seq <= this.syncState(ns).ack)
+        if (seq <= this.syncState(ns).ack || !this.hasP2pApplicationSubscription())
             return;
         if (client._sessionOptions?.background_sync !== false) {
             this.schedulePeerRecovery(sender, seq);
         }
         else {
-            await client._v2E2EE.pullPeerV2(sender, { limit: _INTERNAL_MESSAGE_FLOW_LIMITS.pullPageSize, budget: { remaining: 1 },
-                ...(this.peerOwnType() === 'human' ? { window_mode: 'tail' } : {}) });
+            await this.runAutomaticPull(ns, () => client._v2E2EE.pullPeerV2(sender, { limit: _INTERNAL_MESSAGE_FLOW_LIMITS.pullPageSize, budget: { remaining: 1 },
+                ...(this.shouldTailPeer(ns, seq, _INTERNAL_MESSAGE_FLOW_LIMITS.pullPageSize) ? { window_mode: 'tail', autoTail: { head: seq, forward: false } } : {}) }));
         }
     }
     async onV2PushNotification(data, expectedVersion, inlineGeneration) {
@@ -4136,6 +4484,8 @@ export class MessageDeliveryEngine {
                 this.retryPendingP2PInlineAck(ns, pushSeq, 'covered inline push');
             return;
         }
+        if (!this.hasP2pApplicationSubscription())
+            return;
         this.recordBackgroundPullNeed(ns, pushSeq);
         const pullActive = client._rpcPipeline?.hasPullActivity?.(ns) === true;
         if (pullActive || !this.beginRealtimeSync(ns)) {
@@ -4167,6 +4517,8 @@ export class MessageDeliveryEngine {
         if (!groupId)
             return;
         const ns = `group:${groupId}`;
+        if (this.automaticGroupPullStopped(ns))
+            return;
         const generation = this.captureInlineAckGeneration();
         const afterSeq = this.syncState(ns).ack;
         this.recordBackgroundPullNeed(ns, positiveSafeSequenceHint(data.seq));
@@ -4240,6 +4592,8 @@ export class MessageDeliveryEngine {
             return;
         }
         const ns = `group:${groupId}`;
+        if (this.automaticGroupPullStopped(ns))
+            return;
         let inlineConsumed = false;
         const pendingInlineAck = this.pendingGroupInlineAcks?.get(ns);
         const inlineBlockedByPendingAck = Boolean(pendingInlineAck && pendingInlineAck.seq !== seq);
@@ -4649,6 +5003,7 @@ export class MessageDeliveryEngine {
             this.seqTrackerNamespaceGenerations.set(ns, (this.seqTrackerNamespaceGenerations.get(ns) ?? 0) + 1);
             this.committedSeqTrackerStates.delete(ns);
             this.initializedPeers.delete(ns);
+            this.peerServerConfirmedAcks.delete(ns);
         }
         this.dropSeqTrackerPending(ns);
         const saveSeq = client._tokenStore.saveSeq;