@agentunion/fastaun 0.5.19 → 0.5.20

package/dist/client/delivery.js

@@ -9,6 +9,7 @@ const MESSAGE_RECALL_SEEN_LIMIT = 10_000;
 const GROUP_RECALL_SEEN_LIMIT = 10_000;
 const VOLATILE_SEEN_LIMIT = 10_000;
 const SEQ_TRACKER_PERSIST_FLUSH_DELAY_MS = 200;
+const BACKGROUND_PULL_RETRY_DELAY_MS = 250;
 const DELIVERY_CHANGED_MESSAGE_EVENTS = new Set([
     'message.received',
     'group.message_created',
@@ -80,6 +81,11 @@ export class MessageDeliveryEngine {
     pendingGroupPullUpper = null;
     onlineUnreadHintTargets = null;
     onlineUnreadHintTasks = null;
+    backgroundPullNeeds = new Map();
+    backgroundPullTasks = new Set();
+    backgroundPullCheckQueued = false;
+    backgroundPullTimer = null;
+    backgroundPullTimerAt = 0;
     pendingP2PInlineAcks = null;
     pendingGroupInlineAcks = null;
     pendingGroupInlineAckNamespaces = null;
@@ -422,6 +428,13 @@ export class MessageDeliveryEngine {
         this.deliveryChangedGeneration += 1;
         this.pendingDeliveryChanges = null;
         this.inlineAckGeneration += 1;
+        if (this.backgroundPullTimer)
+            clearTimeout(this.backgroundPullTimer);
+        this.backgroundPullTimer = null;
+        this.backgroundPullTimerAt = 0;
+        this.backgroundPullNeeds.clear();
+        this.backgroundPullTasks.clear();
+        this.backgroundPullCheckQueued = false;
         void this.runtime.client._rpcPipeline?.invalidatePulls?.();
         this.pendingP2PInlineAcks = null;
         this.pendingGroupInlineAcks = null;
@@ -693,6 +706,7 @@ export class MessageDeliveryEngine {
     }
     endRealtimeSync(ns) {
         this.realtimeSyncing?.delete(ns);
+        this.requestBackgroundPullRecovery();
         this.runtime.client._rpcPipeline?.requestPullDrainedCheck?.();
     }
     hasPendingPull(ns) {
@@ -703,6 +717,7 @@ export class MessageDeliveryEngine {
     }
     hasAnyPendingPull() {
         return [...(this.pendingGroupPullUpper?.values() ?? [])].some((seq) => seq > 0)
+            || this.backgroundPullNeeds.size > 0
             || Boolean(this.onlineUnreadHintTasks?.size)
             || Boolean(this.realtimeSyncing?.size)
             || Boolean(this.runtime.client._v2E2EE?.hasPendingSenderIKWork?.());
@@ -710,12 +725,124 @@ export class MessageDeliveryEngine {
     onPullGateIdle(ns) {
         if (!ns)
             return;
+        this.requestBackgroundPullRecovery();
         if (ns.startsWith('p2p:')) {
             this.runtime.client._schedulePendingP2pPullIfNeeded?.(ns, 'pull-gate-idle');
             return;
         }
         this.schedulePendingGroupPullIfNeeded(ns, 'pull-gate-idle');
     }
+    recordBackgroundPullNeed(ns, throughSeq = 0, probe = false) {
+        const client = this.runtime.client;
+        if (client._sessionOptions?.background_sync === false || !client._aid
+            || !(ns === `p2p:${client._aid}` || ns.startsWith('group:') || ns.startsWith('group_event:')))
+            return;
+        const need = this.backgroundPullNeeds.get(ns) ?? {
+            throughSeq: 0, hasMore: false, probe, lastAfter: -1, retryAt: 0,
+        };
+        need.throughSeq = Math.max(need.throughSeq, throughSeq);
+        need.probe ||= probe;
+        this.backgroundPullNeeds.set(ns, need);
+    }
+    observeBackgroundPullResponse(ns, afterSeq, result, limit = 50) {
+        if (!isJsonObject(result))
+            return;
+        const response = result;
+        const rows = ns.startsWith('group_event:') ? response.events : response.messages;
+        if (!Array.isArray(rows))
+            return;
+        const cursor = isJsonObject(response.cursor)
+            ? response.cursor : null;
+        const throughSeq = Math.max(positiveSafeSequenceHint(cursor?.latest_seq), positiveSafeSequenceHint(response.latest_seq), positiveSafeSequenceHint(response.latest_message_seq), positiveSafeSequenceHint(response.latest_event_seq), ...rows.map((row) => row && typeof row === 'object' && row.storage !== 'volatile'
+            ? positiveSafeSequenceHint(row.seq ?? row.event_seq) : 0));
+        this.recordBackgroundPullNeed(ns, throughSeq);
+        const need = this.backgroundPullNeeds.get(ns);
+        if (!need || afterSeq < need.lastAfter)
+            return;
+        need.lastAfter = afterSeq;
+        need.hasMore = response.has_more === true || positiveSafeSequenceHint(response.remaining) > 0
+            || (!('has_more' in response) && limit > 0 && rows.length >= limit);
+        need.probe = false;
+    }
+    finishBackgroundPull(ns, generation, afterSeq, failed) {
+        if (generation !== this.captureInlineAckGeneration())
+            return;
+        const need = this.backgroundPullNeeds.get(ns);
+        if (need) {
+            if (failed)
+                need.probe = true;
+            else
+                need.probe = false;
+            need.retryAt = failed || this.syncState(ns).ack <= afterSeq
+                ? Date.now() + BACKGROUND_PULL_RETRY_DELAY_MS : 0;
+        }
+        this.requestBackgroundPullRecovery();
+    }
+    requestBackgroundPullRecovery() {
+        if (this.backgroundPullCheckQueued || this.backgroundPullNeeds.size === 0)
+            return;
+        this.backgroundPullCheckQueued = true;
+        const generation = this.captureInlineAckGeneration();
+        queueMicrotask(() => {
+            this.backgroundPullCheckQueued = false;
+            if (generation !== this.captureInlineAckGeneration())
+                return;
+            const client = this.runtime.client;
+            if (client._sessionOptions?.background_sync === false || client.state !== ConnectionState.READY || client._closing)
+                return;
+            let retryAt = Infinity;
+            for (const [ns, need] of this.backgroundPullNeeds) {
+                if (this.backgroundPullTasks.has(ns) || this.realtimeSyncing?.has(ns) || this.onlineUnreadHintTasks?.has(ns))
+                    continue;
+                const eventScope = ns.startsWith('group_event:');
+                const filling = [...(client._gapFillDone?.keys?.() ?? [])].some((key) => eventScope
+                    ? String(key) === `group_event_pull:${ns}`
+                    : ns.startsWith('p2p:') ? String(key).startsWith('p2p:') : String(key).startsWith(`group_msg:${ns.slice(6)}:`));
+                if (filling || (!eventScope && client._rpcPipeline?.hasPullActivity?.(ns, false))
+                    || (eventScope && ['pulling', 'processing', 'pulled'].includes(client._rpcPipeline?.getPullLifecycle?.(ns))))
+                    continue;
+                if (this.syncState(ns).ack >= need.throughSeq && !need.hasMore && !need.probe) {
+                    this.backgroundPullNeeds.delete(ns);
+                    continue;
+                }
+                if (need.retryAt > Date.now()) {
+                    retryAt = Math.min(retryAt, need.retryAt);
+                    continue;
+                }
+                this.backgroundPullTasks.add(ns);
+                const task = Promise.resolve().then(() => {
+                    if (generation !== this.captureInlineAckGeneration())
+                        return;
+                    return eventScope ? this.fillGroupEventGap(ns.slice('group_event:'.length))
+                        : ns.startsWith('group:') ? this.fillGroupGap(ns.slice(6)) : this.fillP2pGap();
+                }).finally(() => {
+                    if (generation !== this.captureInlineAckGeneration())
+                        return;
+                    this.backgroundPullTasks.delete(ns);
+                    this.requestBackgroundPullRecovery();
+                });
+                client._safeAsync(task);
+            }
+            if (this.backgroundPullNeeds.size === 0) {
+                if (this.backgroundPullTimer)
+                    clearTimeout(this.backgroundPullTimer);
+                this.backgroundPullTimer = null;
+                this.backgroundPullTimerAt = 0;
+                client._rpcPipeline?.requestPullDrainedCheck?.();
+            }
+            if (Number.isFinite(retryAt) && (!this.backgroundPullTimer || retryAt < this.backgroundPullTimerAt)) {
+                if (this.backgroundPullTimer)
+                    clearTimeout(this.backgroundPullTimer);
+                this.backgroundPullTimerAt = retryAt;
+                this.backgroundPullTimer = setTimeout(() => {
+                    this.backgroundPullTimer = null;
+                    this.backgroundPullTimerAt = 0;
+                    this.requestBackgroundPullRecovery();
+                }, Math.max(0, retryAt - Date.now()));
+                this.backgroundPullTimer.unref?.();
+            }
+        });
+    }
     onForegroundPullsDrained() {
         const client = this.runtime.client;
         if (!client._aid || client._sessionOptions?.background_sync === false
@@ -1340,6 +1467,7 @@ export class MessageDeliveryEngine {
         if (seqNum > client._seqTracker.getContiguousSeq(ns) && client._sessionOptions?.background_sync !== false) {
             this.fillGroupGap(groupId).catch((exc) => client._clientLog.warn(`background recall gap fill failed: ${formatDeliveryError(exc)}`));
         }
+        await this.publishGroupRecallTombstone(groupId, seq, wrapped);
     }
     publishAppEvent(event, payload, source = 'direct', deliveryContext) {
         const client = this.runtime.client;
@@ -1625,7 +1753,6 @@ export class MessageDeliveryEngine {
             if (Number(seq) > client._seqTracker.getContiguousSeq(ns) && client._sessionOptions?.background_sync !== false) {
                 this.fillP2pGap().catch((exc) => client._clientLog.warn(`background recall gap fill failed: ${formatDeliveryError(exc)}`));
             }
-            return;
         }
         await this.publishMessageRecallTombstone(seq, msg);
     }
@@ -1826,12 +1953,15 @@ export class MessageDeliveryEngine {
             return;
         const ns = `p2p:${client._aid}`;
         const afterSeq = client._seqTracker.getContiguousSeq(ns);
+        const generation = this.captureInlineAckGeneration();
         const dedupKey = `p2p:${afterSeq}`;
         if (client._gapFillDone.has(dedupKey))
             return;
         client._gapFillDone.set(dedupKey, Date.now());
+        this.recordBackgroundPullNeed(ns, 0, true);
         client._clientLog.debug(`P2P message gap fill start: after_seq=${afterSeq}`);
         let filled = 0;
+        let failed = false;
         try {
             // Gap Fill 本身就是 Forward Pull，必须沿用协调器的完整事务：
             // 原始页提交成功后由协调器负责 Piggyback/终端 ACK。
@@ -1844,11 +1974,15 @@ export class MessageDeliveryEngine {
             client._clientLog.debug(`P2P message gap fill done: after_seq=${afterSeq}, filled=${filled}`);
         }
         catch (exc) {
+            failed = true;
             client._clientLog.warn(`P2P message gap fill failed: ${formatDeliveryError(exc)}`);
         }
         finally {
+            if (generation !== this.captureInlineAckGeneration())
+                return;
             client._gapFillDone.delete(dedupKey);
             client._schedulePendingP2pPullIfNeeded?.(ns, 'gap-fill-complete');
+            this.finishBackgroundPull(ns, generation, afterSeq, failed);
         }
     }
     async fillGroupGap(groupId) {
@@ -1860,12 +1994,15 @@ export class MessageDeliveryEngine {
             return;
         const ns = `group:${groupId}`;
         const afterSeq = client._seqTracker.getContiguousSeq(ns);
+        const generation = this.captureInlineAckGeneration();
         const dedupKey = `group_msg:${groupId}:${afterSeq}`;
         if (client._gapFillDone.has(dedupKey))
             return;
         client._gapFillDone.set(dedupKey, Date.now());
+        this.recordBackgroundPullNeed(ns, 0, true);
         client._clientLog.debug(`group message gap fill start: group=${groupId}, after_seq=${afterSeq}`);
         let filled = 0;
+        let failed = false;
         try {
             const operation = () => client._pullGroupV2(groupId, afterSeq, 50, { gateLocked: true });
             const messages = client._rpcPipeline
@@ -1876,14 +2013,15 @@ export class MessageDeliveryEngine {
             client._clientLog.debug(`group message gap fill done: group=${groupId}, after_seq=${afterSeq}, filled=${filled}`);
         }
         catch (exc) {
+            failed = true;
             client._clientLog.warn(`group message gap fill failed: ${formatDeliveryError(exc)}`);
         }
         finally {
+            if (generation !== this.captureInlineAckGeneration())
+                return;
             client._gapFillDone.delete(dedupKey);
-            if (filled > 0 && client._seqTracker.getContiguousSeq(ns) > afterSeq) {
-                void this.fillGroupGap(groupId);
-            }
             this.schedulePendingGroupPullIfNeeded(ns, 'gap-fill-complete');
+            this.finishBackgroundPull(ns, generation, afterSeq, failed);
         }
     }
     scheduleRealtimeWindowForward(ns, groupId = '') {
@@ -1944,6 +2082,7 @@ export class MessageDeliveryEngine {
             }
         }
         client._gapFillDone.set(dedupKey, Date.now());
+        this.recordBackgroundPullNeed(ns, client._seqTracker.getMaxSeenSeq(ns), true);
         this.syncStarted(ns);
         let filled = 0;
         let continuationAfterSeq = 0;
@@ -1974,6 +2113,9 @@ export class MessageDeliveryEngine {
                 }
                 if (!isJsonObject(result))
                     return;
+                if (generation !== this.captureInlineAckGeneration())
+                    return;
+                this.observeBackgroundPullResponse(ns, nextAfterSeq, result);
                 const events = result.events;
                 if (!Array.isArray(events))
                     return;
@@ -2085,6 +2227,7 @@ export class MessageDeliveryEngine {
                 const nextAfter = Math.max(eventSeqs.length > 0 ? Math.max(...eventSeqs) : nextAfterSeq, nextAfterSeq);
                 const pageAdvanced = eventObjects.length > 0 && nextAfter > nextAfterSeq;
                 const needsMore = result.has_more === true
+                    || positiveSafeSequenceHint(result.remaining) > 0
                     || client._seqTracker.getMaxSeenSeq(ns) > nextAfter;
                 if (singlePage && pageAdvanced && needsMore) {
                     continuationAfterSeq = nextAfter;
@@ -2103,19 +2246,22 @@ export class MessageDeliveryEngine {
             client._clientLog.warn(`group event gap fill failed: ${formatDeliveryError(exc)}`);
         }
         finally {
-            client._gapFillDone.delete(dedupKey);
+            if (generation === this.captureInlineAckGeneration())
+                client._gapFillDone.delete(dedupKey);
             if (legacyGateToken !== null && typeof client._releasePullGate === 'function') {
                 client._releasePullGate(ns, legacyGateToken);
             }
+            if (generation !== this.captureInlineAckGeneration())
+                return;
+            if (dissolved)
+                this.backgroundPullNeeds.delete(ns);
             if (!dissolved && singlePage && continuationAfterSeq > afterSeq) {
                 this.enqueueOnlineUnreadEventHint({
                     group_id: groupId,
                     _online_hint_after_event_seq: continuationAfterSeq,
                 });
             }
-            else if (!dissolved && !singlePage && filled > 0 && client._seqTracker.getContiguousSeq(ns) > afterSeq) {
-                void this.fillGroupEventGap(groupId);
-            }
+            this.finishBackgroundPull(ns, generation, afterSeq, failed);
             if (!client._rpcPipeline)
                 this.syncStopped(failed ? 'aborted' : 'pull_drained');
         }
@@ -2125,6 +2271,7 @@ export class MessageDeliveryEngine {
         const groupId = String(data.group_id ?? '').trim();
         if (!groupId || client._sessionOptions?.background_sync === false)
             return;
+        this.recordBackgroundPullNeed(`group_event:${groupId}`, positiveSafeSequenceHint(data.event_seq), true);
         const rawAfter = Number(data._online_hint_after_event_seq ?? 0);
         const afterSeq = Number.isSafeInteger(rawAfter) && rawAfter > 0 ? rawAfter : 0;
         client._safeAsync(this.fillGroupEventGap(groupId, {
@@ -3197,6 +3344,7 @@ export class MessageDeliveryEngine {
     }
     async onV2PushNotification(data, expectedVersion, inlineGeneration) {
         const client = this.runtime.client;
+        const syncGeneration = this.captureInlineAckGeneration();
         if (inlineGeneration !== undefined && !this.inlineGenerationIsCurrent(inlineGeneration))
             return;
         const pushData = isJsonObject(data) ? data : null;
@@ -3251,6 +3399,7 @@ export class MessageDeliveryEngine {
                 this.retryPendingP2PInlineAck(ns, pushSeq, 'covered inline push');
             return;
         }
+        this.recordBackgroundPullNeed(ns, pushSeq);
         const pullActive = client._rpcPipeline?.hasPullActivity?.(ns) === true;
         if (pullActive || !this.beginRealtimeSync(ns)) {
             client._recordPendingP2pPull(ns, pushSeq);
@@ -3267,7 +3416,9 @@ export class MessageDeliveryEngine {
             client._clientLog.warn(`V2 P2P push auto-sync failed: ${formatDeliveryError(exc)}`);
         }
         finally {
-            if (inlineGeneration === undefined || this.inlineGenerationIsCurrent(inlineGeneration)) {
+            if (syncGeneration === this.captureInlineAckGeneration()
+                && (inlineGeneration === undefined || this.inlineGenerationIsCurrent(inlineGeneration))) {
+                this.finishBackgroundPull(ns, syncGeneration, this.syncState(ns).ack, false);
                 this.endRealtimeSync(ns);
                 client._schedulePendingP2pPullIfNeeded?.(ns, 'push-auto-pull-complete');
             }
@@ -3279,6 +3430,9 @@ export class MessageDeliveryEngine {
         if (!groupId)
             return;
         const ns = `group:${groupId}`;
+        const generation = this.captureInlineAckGeneration();
+        const afterSeq = this.syncState(ns).ack;
+        this.recordBackgroundPullNeed(ns, positiveSafeSequenceHint(data.seq));
         const targets = this.onlineUnreadHintTargets ?? new Map();
         const tasks = this.onlineUnreadHintTasks ?? new Map();
         this.onlineUnreadHintTargets = targets;
@@ -3286,10 +3440,13 @@ export class MessageDeliveryEngine {
         targets.set(ns, Math.max(targets.get(ns) ?? 0, positiveSafeSequenceHint(data.seq)));
         if (tasks.has(ns))
             return;
+        let attemptAfterSeq = afterSeq;
         const task = Promise.resolve().then(async () => {
+            let failed = false;
             try {
                 while (true) {
                     const ackBefore = this.syncState(ns).ack;
+                    attemptAfterSeq = ackBefore;
                     await this.runGroupForwardRecovery(groupId, ns, 50, true, targets.get(ns) ?? 0);
                     const ackAfter = this.syncState(ns).ack;
                     if (ackAfter >= (targets.get(ns) ?? 0) || ackAfter <= ackBefore)
@@ -3297,12 +3454,14 @@ export class MessageDeliveryEngine {
                 }
             }
             catch (exc) {
+                failed = true;
                 client._clientLog.debug(`online unread hint background Forward failed: ns=${ns} err=${formatDeliveryError(exc)}`);
             }
             finally {
                 targets.delete(ns);
                 if (tasks.get(ns) === task)
                     tasks.delete(ns);
+                this.finishBackgroundPull(ns, generation, attemptAfterSeq, failed);
                 client._rpcPipeline?.requestPullDrainedCheck?.();
             }
         });
@@ -3311,6 +3470,7 @@ export class MessageDeliveryEngine {
     }
     async onRawGroupV2MessageCreated(data, expectedVersion, inlineGeneration) {
         const client = this.runtime.client;
+        const syncGeneration = this.captureInlineAckGeneration();
         if (inlineGeneration !== undefined && !this.inlineGenerationIsCurrent(inlineGeneration))
             return;
         if (!isJsonObject(data)) {
@@ -3373,6 +3533,7 @@ export class MessageDeliveryEngine {
             }
             return;
         }
+        this.recordBackgroundPullNeed(ns, seq);
         const pullActive = client._rpcPipeline?.hasPullActivity?.(ns) === true;
         if (pullActive || !this.beginRealtimeSync(ns)) {
             this.recordPendingGroupPull(ns, seq);
@@ -3388,7 +3549,9 @@ export class MessageDeliveryEngine {
                 return;
             client._clientLog.warn(`V2 group push auto-sync failed: group=${groupId} err=${formatDeliveryError(exc)}`);
         }).finally(() => {
-            if (inlineGeneration === undefined || this.inlineGenerationIsCurrent(inlineGeneration)) {
+            if (syncGeneration === this.captureInlineAckGeneration()
+                && (inlineGeneration === undefined || this.inlineGenerationIsCurrent(inlineGeneration))) {
+                this.finishBackgroundPull(ns, syncGeneration, this.syncState(ns).ack, false);
                 this.endRealtimeSync(ns);
                 this.schedulePendingGroupPullIfNeeded(ns, 'push-auto-pull-complete');
             }
@@ -3450,9 +3613,14 @@ export class MessageDeliveryEngine {
         if (!client._aid || !client._v2Session)
             return;
         const ns = `p2p:${client._aid}`;
+        const generation = this.captureInlineAckGeneration();
+        const afterSeq = this.syncState(ns).ack;
+        this.recordBackgroundPullNeed(ns, positiveSafeSequenceHint(data.seq), true);
+        let failed = false;
         client._safeAsync(this.runP2PForwardRecovery(ns, 50, true).catch((exc) => {
+            failed = true;
             client._clientLog.debug(`P2P online unread hint background Forward failed: ns=${ns} err=${formatDeliveryError(exc)}`);
-        }));
+        }).finally(() => this.finishBackgroundPull(ns, generation, afterSeq, failed)));
     }
     async saveSeqTrackerState() {
         const client = this.runtime.client;