@bobfrankston/rmfmail 1.0.570 → 1.0.590

package/packages/mailx-imap/index.ts

@@ -28,7 +28,12 @@ type OpsTask = () => Promise<void>;
 interface OpsQueue {
     fast: OpsTask[];
     slow: OpsTask[];
-    running: boolean;
+    /** Whether the fast lane is currently draining a task. Independent of
+     *  `runningSlow` — both lanes can drain concurrently because each uses
+     *  its own connection (fastClient vs. opsClient). C123: the second
+     *  connection is lazy-allocated on the first fast-lane task. */
+    runningFast: boolean;
+    runningSlow: boolean;
 }
 interface HostSemaphore {
     permits: number;
@@ -389,11 +394,18 @@ export class ImapManager extends EventEmitter {
     // No semaphore, no pool, no per-operation connect/disconnect.
     // IDLE uses a separate connection (see startWatching).
 
-    /** Persistent operational connections — one per account, reused for all operations.
-     *  Body fetch, sync, prefetch, outbox-append, flag/move all serialize through
-     *  this single client per account via withConnection(). The priority lane
-     *  in the queue lets interactive clicks jump ahead of background prefetch. */
+    /** Persistent slow-lane operational connection per account. Used by sync,
+     *  prefetch, outbox-append, large backfills, and any other "this might
+     *  take a while" operation. */
     private opsClients = new Map<string, any>();
+    /** Lazy-allocated fast-lane client per account (C123). Lives alongside
+     *  `opsClients` so click-time body fetches and flag toggles don't have
+     *  to wait behind a multi-minute prefetch / backfill on the slow client.
+     *  Created on the first fast-lane `withConnection` call; reused while
+     *  alive; recreated on stale-detect or error-discard like opsClients.
+     *  Costs +1 IMAP socket per active account, well under any reasonable
+     *  per-user-IP cap (Dovecot's default is 20). */
+    private fastClients = new Map<string, any>();
     /** Two-lane operation queue per account — interactive ops (body fetch on
      *  click, flag toggle) drain before background ops (sync, prefetch). FIFO
      *  within each lane. The single ops connection means there's never a race
@@ -407,10 +419,15 @@ export class ImapManager extends EventEmitter {
     private hostSemaphores = new Map<string, HostSemaphore>();
     private static readonly HOST_PERMITS = 4;
 
-    /** Get (or create) the persistent operational connection for an account.
-     *  logout() is wrapped as a no-op so legacy callers don't close it. */
-    private async getOpsClient(accountId: string): Promise<any> {
-        let client = this.opsClients.get(accountId);
+    /** Get (or create) a persistent connection for an account on the named
+     *  lane. Two lanes today: `slow` (the original `opsClients` map — sync,
+     *  prefetch, backfill, outbox) and `fast` (the C123 `fastClients` map —
+     *  click-time body fetch, flag toggles, anything user-driven). Same
+     *  stale-detect + reconnect semantics on both. logout() is wrapped as a
+     *  no-op so legacy callers don't close the persistent client. */
+    private async getLaneClient(accountId: string, lane: "fast" | "slow"): Promise<any> {
+        const map = lane === "fast" ? this.fastClients : this.opsClients;
+        let client = map.get(accountId);
         if (client) {
             // C38: health-check the cached client before returning. If the
             // underlying socket is dead (Dovecot silently dropped IDLE after
@@ -422,20 +439,27 @@ export class ImapManager extends EventEmitter {
             const dead = sock?.destroyed || sock?.readyState === "closed" || client?._dead;
             if (!dead) return client;
             try { await (client._realLogout || client.logout)(); } catch { /* */ }
-            this.opsClients.delete(accountId);
-            console.log(`  [conn] ${accountId}: stale ops client detected in getOpsClient — reconnecting`);
+            map.delete(accountId);
+            console.log(`  [conn] ${accountId}: stale ${lane} client detected — reconnecting`);
             client = undefined as any;
         }
-        client = await this.newClient(accountId, "ops");
+        client = await this.newClient(accountId, lane === "fast" ? "fast" : "ops");
         // Wrap logout as no-op — this is a persistent connection. The
         // newClient wrapper's close-counter runs on `_realLogout`.
         const realLogout = client.logout.bind(client);
         client.logout = async () => { /* no-op — persistent connection */ };
         client._realLogout = realLogout;
-        this.opsClients.set(accountId, client);
+        map.set(accountId, client);
         return client;
     }
 
+    /** Backwards-compat shim — callers outside the queue still ask for the
+     *  ops client by historical name. New code should prefer withConnection
+     *  with `slow:true` for slow operations. */
+    private async getOpsClient(accountId: string): Promise<any> {
+        return this.getLaneClient(accountId, "slow");
+    }
+
     /** Run an operation on the account's connection — queued, sequential, no concurrency */
     /** Run an operation against the account's single ops connection. Tasks
      *  queue strictly sequentially per account — only one IMAP command in
@@ -457,12 +481,13 @@ export class ImapManager extends EventEmitter {
     ): Promise<T> {
         let queue = this.opsQueues.get(accountId);
         if (!queue) {
-            queue = { fast: [], slow: [], running: false };
+            queue = { fast: [], slow: [], runningFast: false, runningSlow: false };
             this.opsQueues.set(accountId, queue);
         }
+        const lane: "fast" | "slow" = opts.slow ? "slow" : "fast";
         // Per-task wall-clock cap. Without one, a wedged IMAP command (TCP
         // half-open, server stalled mid-FETCH) keeps the queue's running flag
-        // set forever and every subsequent fast-lane task — including the
+        // set forever and every subsequent same-lane task — including the
         // retry button the user just hit — waits behind it. Default is
         // generous; callers driving user-visible reads pass a tighter value.
         const timeoutMs = opts.timeoutMs ?? 90_000;
@@ -470,23 +495,26 @@ export class ImapManager extends EventEmitter {
             const task: OpsTask = async () => {
                 let timer: any;
                 try {
-                    const client = await this.getOpsClient(accountId);
+                    const client = await this.getLaneClient(accountId, lane);
                     const result = await Promise.race<T>([
                         fn(client),
                         new Promise<T>((_, rej) => {
-                            timer = setTimeout(() => rej(new Error(`ops timeout after ${Math.round(timeoutMs / 1000)}s — discarding client`)), timeoutMs);
+                            timer = setTimeout(() => rej(new Error(`${lane}-ops timeout after ${Math.round(timeoutMs / 1000)}s — discarding client`)), timeoutMs);
                         }),
                     ]);
                     clearTimeout(timer);
                     resolve(result);
                 } catch (e: any) {
                     clearTimeout(timer);
-                    // Discard client on any error — keeping a half-broken
-                    // socket poisoned every subsequent request. Destroy
-                    // synchronously kills the in-flight command's socket so
-                    // the underlying promise rejects and stops holding state.
-                    const stale = this.opsClients.get(accountId);
-                    this.opsClients.delete(accountId);
+                    // Discard the lane's client on any error — a half-broken
+                    // socket poisons every subsequent request on that lane.
+                    // Don't touch the OTHER lane's client; the lanes are
+                    // independent. Destroy synchronously kills the in-flight
+                    // command's socket so the underlying promise rejects and
+                    // stops holding state.
+                    const map = lane === "fast" ? this.fastClients : this.opsClients;
+                    const stale = map.get(accountId);
+                    map.delete(accountId);
                     if (stale) {
                         try { await (stale._realLogout || stale.logout)(); } catch { /* */ }
                         try { stale.destroy?.(); } catch { /* */ }
@@ -494,24 +522,31 @@ export class ImapManager extends EventEmitter {
                     reject(e);
                 }
             };
-            (opts.slow ? queue!.slow : queue!.fast).push(task);
+            queue![lane].push(task);
             this.drainOpsQueue(accountId);
         });
     }
 
-    /** Run the next queued task. Fast lane drains before slow.
-     *  Idempotent — safe to call after each task completes; the running
-     *  flag prevents reentrant draining. */
+    /** Run the next queued task on each lane. Fast and slow lanes drain
+     *  CONCURRENTLY — each on its own connection (C123). FIFO within each
+     *  lane. The running flag per lane prevents reentrant draining of the
+     *  same lane. */
     private drainOpsQueue(accountId: string): void {
         const queue = this.opsQueues.get(accountId);
-        if (!queue || queue.running) return;
-        const next = queue.fast.shift() || queue.slow.shift();
-        if (!next) return;
-        queue.running = true;
-        next().finally(() => {
-            queue.running = false;
-            this.drainOpsQueue(accountId);
-        });
+        if (!queue) return;
+        const drainLane = (lane: "fast" | "slow"): void => {
+            const runningKey = lane === "fast" ? "runningFast" : "runningSlow";
+            if (queue[runningKey]) return;
+            const next = queue[lane].shift();
+            if (!next) return;
+            queue[runningKey] = true;
+            next().finally(() => {
+                queue[runningKey] = false;
+                drainLane(lane);
+            });
+        };
+        drainLane("fast");
+        drainLane("slow");
     }
 
     /** Acquire one slot of the per-host connection semaphore. Returns a release
@@ -559,7 +594,15 @@ export class ImapManager extends EventEmitter {
         const releaseHostSlot = await this.acquireHostSlot(host);
         let client: any;
         try {
-            client = new CompatImapClient(config, this.transportFactory);
+            // Verbose IMAP wire trace for ops connections only — that's the
+            // lane where commands have been hanging silently with no
+            // heartbeat / wall-clock fire / reject. Need to see the actual
+            // commands sent and bytes received to pinpoint where the
+            // pendingCommand is getting lost. Fast lane (C123) shares the
+            // verbose treatment so click-time wedges show too. Other lanes
+            // (idle, quickCheck) stay quiet so the log doesn't drown.
+            const cfgWithVerbose = (purpose === "ops" || purpose === "fast") ? { ...config, verbose: true } : config;
+            client = new CompatImapClient(cfgWithVerbose, this.transportFactory);
         } catch (e) {
             releaseHostSlot();
             throw e;
@@ -593,14 +636,17 @@ export class ImapManager extends EventEmitter {
         return client;
     }
 
-    /** Force-close every IMAP socket for an account — ops + any lingering
-     *  ones in openClients (e.g. an IDLE watcher in flight). Used during
-     *  account removal and disconnectOps so the server's connection slots
-     *  free immediately rather than waiting for socket idle timeouts. */
+    /** Force-close every IMAP socket for an account — both lane clients
+     *  (ops + fast) plus any lingering ones in openClients (e.g. an IDLE
+     *  watcher in flight). Used during account removal and disconnectOps
+     *  so the server's connection slots free immediately rather than
+     *  waiting for socket idle timeouts. */
     async closeAllClients(accountId: string): Promise<void> {
-        const ops = this.opsClients.get(accountId);
-        this.opsClients.delete(accountId);
-        if (ops) { try { await (ops._realLogout || ops.logout)(); } catch { /* */ } try { ops.destroy?.(); } catch { /* */ } }
+        for (const map of [this.opsClients, this.fastClients]) {
+            const c = map.get(accountId);
+            map.delete(accountId);
+            if (c) { try { await (c._realLogout || c.logout)(); } catch { /* */ } try { c.destroy?.(); } catch { /* */ } }
+        }
         const open = this.openClients.get(accountId);
         if (open) {
             for (const c of Array.from(open)) {
@@ -611,19 +657,22 @@ export class ImapManager extends EventEmitter {
         }
     }
 
-    /** Disconnect the persistent operational connection for an account */
+    /** Disconnect the persistent operational connections (both lanes) for
+     *  an account. */
     async disconnectOps(accountId: string): Promise<void> {
-        const client = this.opsClients.get(accountId);
-        this.opsClients.delete(accountId);
-        if (client) {
-            // Force-close: don't wait for LOGOUT on a possibly dead socket
-            try {
-                const timeout = new Promise(r => setTimeout(r, 2000));
-                await Promise.race([(client._realLogout || client.logout)(), timeout]);
-            } catch { /* */ }
-            // Destroy underlying socket if still open
-            try { client.destroy?.(); } catch { /* */ }
-            console.log(`  [conn] ${accountId}: disconnected`);
+        for (const [name, map] of [["ops", this.opsClients], ["fast", this.fastClients]] as const) {
+            const client = map.get(accountId);
+            map.delete(accountId);
+            if (client) {
+                // Force-close: don't wait for LOGOUT on a possibly dead socket
+                try {
+                    const timeout = new Promise(r => setTimeout(r, 2000));
+                    await Promise.race([(client._realLogout || client.logout)(), timeout]);
+                } catch { /* */ }
+                // Destroy underlying socket if still open
+                try { client.destroy?.(); } catch { /* */ }
+                console.log(`  [conn] ${accountId} (${name}): disconnected`);
+            }
         }
     }
 
@@ -915,6 +964,54 @@ export class ImapManager extends EventEmitter {
         return stored;
     }
 
+    /** Insert a local row for a message we just APPENDed (Sent / Drafts).
+     *  Uses the source bytes we already have plus the UID returned by the
+     *  server's APPENDUID response to build the local row directly — no
+     *  IMAP SELECT/SEARCH/FETCH round-trip required. This is the user-
+     *  visible fix for "I sent a message but it doesn't show up in mailx
+     *  Sent until the broad sync finally completes" (which on slow servers
+     *  can be minutes — and which Thunderbird sidesteps entirely because
+     *  it inserts its own row at APPEND time too).
+     *
+     *  Fires the same emits as a normal sync so the UI updates. */
+    async insertLocalRowFromSource(
+        accountId: string,
+        folder: { id: number; path: string },
+        uid: number,
+        source: string,
+        flags: string[],
+    ): Promise<void> {
+        const { simpleParser } = await import("mailparser");
+        const parsed = await simpleParser(source);
+        // Coerce mailparser AddressObject(s) into the flat `{name, address}[]`
+        // shape storeMessages's downstream toEmailAddresses expects.
+        const flat = (a: any): { name: string; address: string }[] => {
+            if (!a) return [];
+            if (Array.isArray(a)) return a.flatMap(x => x?.value || []);
+            return a.value || [];
+        };
+        const msg: any = {
+            uid,
+            messageId: parsed.messageId || "",
+            inReplyTo: parsed.inReplyTo || "",
+            date: parsed.date || new Date(),
+            subject: parsed.subject || "",
+            from: flat(parsed.from),
+            to: flat(parsed.to),
+            cc: flat(parsed.cc),
+            seen: flags.includes("\\Seen"),
+            flagged: flags.includes("\\Flagged"),
+            answered: flags.includes("\\Answered"),
+            draft: flags.includes("\\Draft"),
+            source,
+            size: source.length,
+        };
+        await this.storeMessages(accountId, folder.id, folder, [msg], 0);
+        this.db.recalcFolderCounts(folder.id);
+        this.emit("folderCountsChanged", accountId, {});
+        console.log(`  [local-insert] ${folder.path} UID ${uid}: ${parsed.subject || "(no subject)"} (no IMAP roundtrip)`);
+    }
+
     /** Sync messages for a specific folder */
     async syncFolder(accountId: string, folderId: number, client?: any): Promise<number> {
         if (!client) client = await this.getOpsClient(accountId);
@@ -926,19 +1023,75 @@ export class ImapManager extends EventEmitter {
 
         this.emit("syncProgress", accountId, `sync:${folder.path}`, 0);
 
+        // Top-of-syncFolder breadcrumb — fires BEFORE any async I/O so a
+        // hang in STATUS or SELECT can be located by its absence of a
+        // following completion log. Without this, "Sent never showed up
+        // in the log" was indistinguishable from "Sent hadn't started"
+        // versus "Sent's STATUS call is hung."
+        const __sfStart = Date.now();
+        console.log(`  [sync-enter] ${accountId}/${folder.path} (folderId=${folderId})`);
+
         // Get the highest UID we already have for this folder. IMAP UIDs are
         // monotonically increasing within a UIDVALIDITY (RFC 3501); a
         // high-water mark is the right anchor for incremental fetch.
         const highestUid = this.db.getHighestUid(accountId, folderId);
+        const localCount = this.db.getMessageCount(accountId, folderId);
+
+        // STATUS-before-SELECT — fast O(1) command that returns UIDNEXT and
+        // MESSAGES without touching the mailbox SELECT state. RFC 3501 §6.3.10.
+        // This is the difference between "INBOX works, Sent doesn't" on this
+        // server: INBOX's quickImapCheck uses STATUS first; non-INBOX folders
+        // were going straight to SELECT every cycle, and SELECT on Sent has
+        // been observed wedging this server's ops connection for minutes
+        // (Thunderbird, which uses STATUS-before-SELECT, doesn't see this).
+        // When the server's UIDNEXT-1 ≤ our highestUid AND MESSAGES count
+        // matches our local count, NOTHING changed — skip SELECT entirely.
+        // For first sync (highestUid = 0) skip the STATUS check and let the
+        // existing first-sync path (sequence-FETCH the latest N) run.
+        if (highestUid > 0) {
+            try {
+                console.log(`  [sync-status] ${accountId}/${folder.path}: calling STATUS...`);
+                const __statusT0 = Date.now();
+                const status: any = await (client as any).native?.getStatus?.(folder.path)
+                    ?? await (client as any).getStatus?.(folder.path);
+                console.log(`  [sync-status] ${accountId}/${folder.path}: STATUS returned in ${Date.now() - __statusT0}ms`);
+                if (status && typeof status.uidNext === "number") {
+                    const serverHighest = status.uidNext - 1;
+                    const noNewUids = serverHighest <= highestUid;
+                    const countsMatch = typeof status.messages === "number" && status.messages === localCount;
+                    if (noNewUids && countsMatch) {
+                        // Nothing new server-side AND counts match — full
+                        // sync would be a no-op. Skip SELECT/SEARCH/FETCH
+                        // entirely. This is the path Sent takes most of the
+                        // time after the optimistic-APPEND-insert (1.0.573)
+                        // already filed the just-sent row locally.
+                        console.log(`  [sync] ${accountId}/${folder.path}: STATUS clean (uidNext=${status.uidNext} messages=${status.messages}) — skipping SELECT`);
+                        this.emit("syncProgress", accountId, `sync:${folder.path}`, 100);
+                        return 0;
+                    }
+                    // Counts differ or server has new UIDs — fall through to
+                    // the full sync path below, which knows how to reconcile.
+                    console.log(`  [sync] ${accountId}/${folder.path}: STATUS uidNext=${status.uidNext} messages=${status.messages} local=${localCount}/${highestUid} — proceeding to SELECT`);
+                }
+            } catch (statusErr: any) {
+                // STATUS failed — fall through to SELECT (some servers don't
+                // support STATUS on the currently-SELECTed mailbox; per
+                // RFC 3501 §6.3.10 it's actually allowed everywhere but a
+                // misbehaving server might refuse). The fallback is the
+                // pre-1.0.574 behavior, so worst case we lose the speedup.
+                console.log(`  [sync] ${accountId}/${folder.path}: STATUS failed (${statusErr?.message || statusErr}) — falling back to SELECT`);
+            }
+        }
 
-        // STATUS-before-SELECT was here. Removed — added a round-trip per
-        // folder with no measured benefit on Bob's link, and the speculative
-        // "skip SELECT when nothing changed" optimization didn't actually
-        // skip anything in practice (server count vs local count nearly
-        // always differs slightly because of in-flight deletes/moves).
         console.log(`  [sync] ${accountId}/${folder.path}: highestUid=${highestUid}, fetching...`);
 
         let messages: any[];
+        // Cache the server-UID list across set-diff and deletion-recon so we
+        // don't pay for two `UID SEARCH` round-trips (the second one was
+        // `UID SEARCH ALL` which on a 134k-message INBOX hangs for minutes
+        // on this server). Set-diff populates this; deletion-recon reuses.
+        let serverUidsCached: number[] | null = null;
+        let serverUidsAreDateBounded = false;
         const firstSync = highestUid === 0;
         const historyDays = getHistoryDays(accountId);
         // historyDays=0 means "all". On first sync we still cap at 30 days
@@ -957,58 +1110,130 @@ export class ImapManager extends EventEmitter {
             // Filter out the last known message (IMAP * always returns at least one)
             messages = fetched.filter((m: any) => m.uid > highestUid);
 
-            // Gap detection: check for missing UIDs within the range we've already synced
-            // Only reconcile between our lowest and highest UID — don't try to fetch the entire folder history
+            // Reconciliation by SET DIFFERENCE, not high-water-mark.
+            //
+            // The high-water-mark approach (fetch UIDs > highestUid) only
+            // catches NEW arrivals. It cannot see anything below the local
+            // lowest UID — so messages that exist on the server but predate
+            // the user's first sync (or any other gap) are invisible forever.
+            // The previous gap-fill clamped reconciliation to [lowestUid,
+            // highestUid] which made that gap permanent.
+            //
+            // The right model: server UID set is the truth. Anything on
+            // the server that's not local → fetch. Anything local that's
+            // not on the server → reconcile-delete (handled separately by
+            // the deferred-delete path, with grace + 50% safeguards).
+            //
+            // Bounded by a 5000-cap so a misbehaving server response or
+            // brand-new account doesn't try to pull tens of thousands of
+            // historical messages in one cycle. Hit the cap and the next
+            // sync picks up the rest.
             const existingUids = this.db.getUidsForFolder(accountId, folderId);
-            if (existingUids.length > 0) {
-                try {
-                    const lowestUid = Math.min(...existingUids);
-                    // Fetch UIDs in our known range from IMAP
-                    const rangeUids = await (client as any).getUids(folder.path);
-                    const rangeInScope = rangeUids.filter((uid: number) => uid >= lowestUid && uid <= highestUid);
-                    const existingSet = new Set(existingUids);
-                    const newSet = new Set(messages.map(m => m.uid));
-                    const missingUids = rangeInScope.filter((uid: number) => !existingSet.has(uid) && !newSet.has(uid));
-                    if (missingUids.length > 0 && missingUids.length <= 5000) {
-                        console.log(`  ${folder.path}: gap detected — ${missingUids.length} missing UIDs in range ${lowestUid}..${highestUid}`);
-                        const chunkSize = 500;
-                        let recoveredTotal = 0;
-                        for (let i = 0; i < missingUids.length; i += chunkSize) {
-                            const chunk = missingUids.slice(i, i + chunkSize);
-                            const range = chunk.join(",");
-                            const recovered = await (client as any).fetchMessages(folder.path, range, { source: false });
-                            messages.push(...recovered);
-                            recoveredTotal += recovered.length;
-                            console.log(`  ${folder.path}: gap-fill ${recoveredTotal}/${missingUids.length}`);
-                        }
-                    } else if (missingUids.length > 5000) {
-                        console.log(`  ${folder.path}: ${missingUids.length} missing UIDs — too many, skipping reconciliation (delete DB to force full re-sync)`);
+            try {
+                // Date-bound the server UID query when we have a history
+                // window. UID SEARCH ALL on a 134k-message INBOX returns
+                // 134k integers and triggers a heavyweight full-folder
+                // scan on the server; UID SEARCH SINCE <date> returns only
+                // UIDs in the window we actually care about. For
+                // historyDays=0 ("keep everything") we still bound to the
+                // last 5 years on incremental syncs — enough that we never
+                // miss an in-flight message, without enumerating decades of
+                // archive every cycle. First sync gets all UIDs (no anchor
+                // yet so we have to compare against the empty local set
+                // anyway).
+                const SINCE_DAYS = effectiveDays > 0 ? effectiveDays : 1825;
+                const sinceDate = new Date(Date.now() - SINCE_DAYS * 86400000);
+                console.log(`  [sync-uids] ${accountId}/${folder.path}: getUidsSince ${sinceDate.toISOString().slice(0,10)}...`);
+                const __uidsT0 = Date.now();
+                const allServerUids = typeof (client as any).getUidsSince === "function"
+                    ? await (client as any).getUidsSince(folder.path, sinceDate)
+                    : await (client as any).getUids(folder.path);
+                console.log(`  [sync-uids] ${accountId}/${folder.path}: ${allServerUids.length} UIDs in ${Date.now() - __uidsT0}ms`);
+                // Stash for the deletion-reconciliation block below — we
+                // already have the date-bounded server UID list, no point
+                // hitting the server a second time with UID SEARCH ALL.
+                serverUidsCached = allServerUids;
+                serverUidsAreDateBounded = true;
+                const existingSet = new Set(existingUids);
+                const newSet = new Set(messages.map(m => m.uid));
+                const missingUids = allServerUids.filter((uid: number) =>
+                    !existingSet.has(uid) && !newSet.has(uid)
+                );
+                // Backfill chunk size: ops queue yields between chunks so
+                // a click-time body fetch can interleave. 100 UIDs per
+                // chunk balances throughput (one FETCH command per chunk)
+                // against latency (a chunk takes 1-3s on Dovecot, which
+                // is the worst-case wait an interactive click endures).
+                // The 500-chunk version held the ops queue for the
+                // entire backfill — Bob 2026-05-08 saw a click-to-render
+                // wait of 100+ minutes on a busy backfill of the IP
+                // folder.
+                const BACKFILL_CHUNK_SIZE = 100;
+                if (missingUids.length > 0 && missingUids.length <= 5000) {
+                    // For the log line we report a count; computing
+                    // min/max via a spread (`Math.min(...arr)`) blows V8's
+                    // argument limit on folders with tens of thousands of
+                    // UIDs. Use a manual reduce.
+                    let minU = existingUids[0] ?? 0;
+                    for (let i = 1; i < existingUids.length; i++) if (existingUids[i] < minU) minU = existingUids[i];
+                    console.log(`  ${folder.path}: ${missingUids.length} server-only UIDs (local lowest=${minU}, highest=${highestUid}) — fetching`);
+                    let recoveredTotal = 0;
+                    for (let i = 0; i < missingUids.length; i += BACKFILL_CHUNK_SIZE) {
+                        const chunk = missingUids.slice(i, i + BACKFILL_CHUNK_SIZE);
+                        const range = chunk.join(",");
+                        // Each chunk gets its own withConnection slow-lane
+                        // turn so any fast-lane click queued in the
+                        // meantime gets serviced between chunks. The
+                        // outer `client` param is bypassed here; the
+                        // queue-managed client is the same persistent
+                        // ops client (getOpsClient).
+                        const recovered = await this.withConnection(accountId, async (c) =>
+                            await (c as any).fetchMessages(folder.path, range, { source: false }),
+                            { slow: true },
+                        );
+                        messages.push(...recovered);
+                        recoveredTotal += recovered.length;
+                        console.log(`  ${folder.path}: fetch ${recoveredTotal}/${missingUids.length}`);
                     }
-                } catch (e: any) {
-                    console.error(`  ${folder.path}: gap detection failed: ${e.message}`);
-                }
-            }
-
-            // Backfill: if the history window reaches further back than our
-            // oldest local message, fetch the gap. Chunk 90 days per sync
-            // cycle so historyDays=0 catches up incrementally instead of
-            // asking Dovecot for SEARCH SINCE 1970 in one go.
-            const oldestDate = this.db.getOldestDate(accountId, folderId);
-            if (oldestDate > 0 && startDate.getTime() < oldestDate) {
-                try {
-                    const CHUNK_MS = 90 * 86400000;
-                    const chunkStart = new Date(Math.max(startDate.getTime(), oldestDate - CHUNK_MS));
-                    const existingUids = new Set(this.db.getUidsForFolder(accountId, folderId));
-                    const backfill = await client.fetchMessageByDate(folder.path, chunkStart, new Date(oldestDate), { source: false });
-                    const newBackfill = backfill.filter((m: any) => !existingUids.has(m.uid));
-                    if (newBackfill.length > 0) {
-                        console.log(`  ${folder.path}: backfilling ${newBackfill.length} older messages (${chunkStart.toISOString().slice(0,10)} → ${new Date(oldestDate).toISOString().slice(0,10)})`);
-                        messages.push(...newBackfill);
+                } else if (missingUids.length > 5000) {
+                    console.log(`  ${folder.path}: ${missingUids.length} server-only UIDs — capped; will resume next cycle`);
+                    // ASSUMPTION: vanilla IMAP under stable UIDVALIDITY,
+                    // higher UID = later assignment ≈ more recent message.
+                    // True for Dovecot / Cyrus / standard IMAP, which is the
+                    // only place this code path runs (Gmail-API mode goes
+                    // through syncAccountViaApi and doesn't reach here —
+                    // its synthesized hash-UIDs have no temporal meaning).
+                    // If we ever wire this for a non-monotonic UID source,
+                    // sort by date instead — but that means an extra
+                    // fetch-of-INTERNALDATE round-trip, which we avoid for
+                    // free here under the IMAP guarantee.
+                    const cappedSlice = missingUids.sort((a: number, b: number) => b - a).slice(0, 5000);
+                    let recoveredTotal = 0;
+                    for (let i = 0; i < cappedSlice.length; i += BACKFILL_CHUNK_SIZE) {
+                        const chunk = cappedSlice.slice(i, i + BACKFILL_CHUNK_SIZE);
+                        const recovered = await this.withConnection(accountId, async (c) =>
+                            await (c as any).fetchMessages(folder.path, chunk.join(","), { source: false }),
+                            { slow: true },
+                        );
+                        messages.push(...recovered);
+                        recoveredTotal += recovered.length;
+                        console.log(`  ${folder.path}: fetch ${recoveredTotal}/5000 (capped)`);
                     }
-                } catch (e: any) {
-                    console.error(`  ${folder.path}: backfill failed: ${e.message}`);
                 }
+            } catch (e: any) {
+                console.error(`  ${folder.path}: reconciliation failed: ${e.message}`);
             }
+
+            // Date-based backfill via SEARCH SINCE was here. Removed —
+            // set-diff above already covers the same case (any server UID
+            // not in our local set gets fetched, regardless of whether it's
+            // above or below our local highest/lowest). SEARCH SINCE on a
+            // large Dovecot folder walks INTERNALDATE on every message and
+            // can take many minutes, which was wedging the whole syncFolder
+            // call AFTER set-diff had succeeded — preventing syncAccount
+            // from ever moving past INBOX to Sent / other folders. Don't
+            // need both, and date-based-with-high-water-mark is exactly
+            // the brittle model the set-diff replaces.
         } else {
             // First sync: fetch in chunks, store each chunk immediately for instant UI
             let totalStored = 0;
@@ -1134,6 +1359,16 @@ export class ImapManager extends EventEmitter {
                     [folderId]: { total: folderInfo?.totalCount || 0, unread: folderInfo?.unreadCount || 0 }
                 });
             }
+
+            // Yield to the event loop between batches so a pending IPC
+            // (e.g. user clicked a message) gets dispatched. Without this
+            // yield, a 5000-message store loop blocked Node's event loop
+            // for ~5s of synchronous SQLite work — clicks fired during
+            // that window felt frozen because the getMessage IPC sat in
+            // stdin until the loop finished. setImmediate runs after I/O
+            // callbacks, which means stdin readable events fire first and
+            // get serviced.
+            await new Promise<void>(r => setImmediate(r));
         }
         if (newCount > 0) console.log(`    stored ${newCount} new messages`);
 
@@ -1150,12 +1385,30 @@ export class ImapManager extends EventEmitter {
         let deletedCount = 0;
         if (!firstSync) {
             try {
-                const serverUidsArr = await client.getUids(folder.path);
+                // Reuse the server UID list set-diff already fetched.
+                // Without this we made TWO `UID SEARCH` calls per folder
+                // per sync — the first date-bounded (set-diff), the second
+                // `UID SEARCH ALL` (this block). The second call hung
+                // indefinitely on Bob's 134k-message INBOX, blocking the
+                // ops worker and preventing Sent / other folders from
+                // ever getting their turn.
+                const serverUidsArr = serverUidsCached ?? await client.getUids(folder.path);
                 const serverUids = new Set(serverUidsArr);
-                const localUids = this.db.getUidsForFolder(accountId, folderId);
+                const localUidsAll = this.db.getUidsForFolder(accountId, folderId);
+                // When the server-UID list is date-bounded, we can only
+                // reason about deletions for local UIDs in the same window.
+                // UIDs older than the window's lowest server UID may or may
+                // not still be on the server — we never asked. Treat them
+                // as out-of-scope rather than deletion candidates.
+                let localUids = localUidsAll;
+                if (serverUidsAreDateBounded && serverUidsArr.length > 0) {
+                    let minServerUid = serverUidsArr[0];
+                    for (let i = 1; i < serverUidsArr.length; i++) if (serverUidsArr[i] < minServerUid) minServerUid = serverUidsArr[i];
+                    localUids = localUidsAll.filter(u => u >= minServerUid);
+                }
                 const toDelete = localUids.filter(uid => !serverUids.has(uid));
-                if (serverUidsArr.length === 0 && localUids.length > 0) {
-                    console.log(`  [sync] ${accountId}/${folder.path}: reconcile skipped — server UID list empty but local has ${localUids.length} (treating as transient)`);
+                if (serverUidsArr.length === 0 && localUidsAll.length > 0) {
+                    console.log(`  [sync] ${accountId}/${folder.path}: reconcile skipped — server UID list empty but local has ${localUidsAll.length} (treating as transient)`);
                 } else if (localUids.length > 0 && toDelete.length / localUids.length > 0.5) {
                     console.log(`  [sync] ${accountId}/${folder.path}: reconcile REFUSED — would delete ${toDelete.length}/${localUids.length} (${Math.round(toDelete.length / localUids.length * 100)}%) — probably a sync bug, skipping`);
                 } else if (toDelete.length > 0) {
@@ -1357,14 +1610,24 @@ export class ImapManager extends EventEmitter {
                 const isTrashChild = folder.path.includes("/") && folder.path.toLowerCase().startsWith("trash");
                 const highestUid = this.db.getHighestUid(accountId, folder.id);
                 if (isTrashChild && highestUid === 0) return;
+                let fresh: any = null;
                 try {
-                    const fresh = await this.getOpsClient(accountId);
+                    fresh = await this.getOpsClient(accountId);
                     await Promise.race([
                         this.syncFolder(accountId, folder.id, fresh),
-                        new Promise((_, reject) => setTimeout(
-                            () => reject(new Error(`per-folder timeout (${PER_FOLDER_TIMEOUT_MS / 1000}s): ${folder.path}`)),
-                            PER_FOLDER_TIMEOUT_MS
-                        )),
+                        new Promise((_, reject) => setTimeout(() => {
+                            // C120: pull TCP transport diagnostics into the
+                            // per-folder timeout error so the [sync] log
+                            // distinguishes "server stopped responding"
+                            // (sinceLastRead high) from "we never finished
+                            // writing" (writes climbing without reads). Same
+                            // shape iflow-direct emits on its own timeouts.
+                            const d = (fresh as any)?.transport?.diagnostics;
+                            const diag = d
+                                ? ` [conn#${d.connId} r=${d.bytesRead}B w=${d.bytesWritten}B writes=${d.writeCount} sinceLastRead=${d.lastReadAt ? Date.now() - d.lastReadAt : -1}ms]`
+                                : "";
+                            reject(new Error(`per-folder timeout (${PER_FOLDER_TIMEOUT_MS / 1000}s): ${folder.path}${diag}`));
+                        }, PER_FOLDER_TIMEOUT_MS)),
                     ]);
                 } catch (e: any) {
                     if (e.responseText?.includes("doesn't exist")) {
@@ -2255,13 +2518,18 @@ export class ImapManager extends EventEmitter {
                             }
                         }
                         await Promise.all(pending);
-                        // CRITICAL: only prune as "server-deleted" when the batch
-                        // actually completed. If the batch threw (403, 429, network
-                        // error, etc.) NOTHING was received, and treating every
-                        // requested UID as deleted silently wipes 100 messages per
-                        // batch. That's a data-loss bug. Earlier version did this
-                        // and pruned 296 messages on a 403 auth error.
-                        if (batchSucceeded) {
+                        // Prune missing UIDs only when the batch actually
+                        // completed AND returned at least one body. Earlier
+                        // guards (don't-prune-on-thrown-batch) handled 403 /
+                        // 429 / network errors but not the "successful but
+                        // empty" case — a Gmail batch endpoint that returns
+                        // an HTTP 200 with no inner messages, a parser miss,
+                        // or a transient that didn't bubble. Same data-loss
+                        // pattern: 0 received → all UIDs pruned. The
+                        // set-diff reconcile in syncFolder/syncAccountViaApi
+                        // owns deletion via a 30-min grace; defer to it.
+                        const someReceived = received.size > 0;
+                        if (batchSucceeded && someReceived) {
                             for (const uid of uidsInFolder) {
                                 if (received.has(uid)) continue;
                                 try {
@@ -2271,6 +2539,8 @@ export class ImapManager extends EventEmitter {
                                     madeProgress = true;
                                 } catch { /* ignore */ }
                             }
+                        } else if (batchSucceeded && !someReceived) {
+                            console.error(`  [prefetch] ${accountId}/${folder.path}: Gmail batch returned 0/${uidsInFolder.length} bodies — NOT pruning (set-diff reconcile owns deletion). UIDs: ${uidsInFolder.slice(0, 5).join(",")}${uidsInFolder.length > 5 ? "..." : ""}`);
                         }
                         if (counters.errors >= ERROR_BUDGET) break;
                     }
@@ -2303,6 +2573,17 @@ export class ImapManager extends EventEmitter {
                     const bi = bf?.specialUse === "inbox" ? 0 : 1;
                     return ai - bi;
                 });
+                // PREFETCH_CHUNK_SIZE: how many UIDs prefetch holds the ops
+                // connection for before yielding. Smaller = interactive
+                // clicks see less wait when prefetch is busy; larger =
+                // fewer round-trips and less per-chunk SELECT overhead.
+                // 25 sits at the knee of that curve for Dovecot — one
+                // FETCH command per chunk (iflow's `fetchChunkSize: 10`
+                // makes that 3 sub-FETCHes), then we release the queue.
+                // Bob 2026-05-08: a 33-UID prefetch held the queue for
+                // 100 minutes during which every click waited; chunking
+                // gives clicks a window roughly every 5-30 seconds.
+                const PREFETCH_CHUNK_SIZE = 25;
                 for (const [folderId, uids] of orderedFolders) {
                     const folder = folders.find(f => f.id === folderId);
                     if (!folder) continue;
@@ -2310,51 +2591,72 @@ export class ImapManager extends EventEmitter {
                         console.log(`  [prefetch] ${accountId}: skipping ${folder.path} (recent timeouts — cooling down)`);
                         continue;
                     }
-                    const received = new Set<number>();
-                    let batchSucceeded = false;
-                    try {
-                        // Slow lane: prefetch is the textbook "this might take
-                        // a while" case — let interactive ops slip ahead.
-                        await this.withConnection(accountId, async (client) => {
-                            const pending: Promise<void>[] = [];
-                            await client.fetchBodiesBatch(folder.path, uids, (uid: number, source: string) => {
-                                received.add(uid);
-                                pending.push((async () => {
-                                    try {
-                                        const raw = Buffer.from(source, "utf-8");
-                                        const bodyPath = await this.bodyStore.putMessage(accountId, folderId, uid, raw);
-                                        this.db.updateBodyPath(accountId, uid, bodyPath);
-                                        this.emit("bodyCached", accountId, uid);
-                                        counters.totalFetched++;
-                                        madeProgress = true;
-                                    } catch (e: any) {
-                                        console.error(`  [prefetch] ${accountId}/${uid}: store write failed: ${e.message}`);
-                                    }
-                                })());
-                            });
-                            await Promise.all(pending);
-                        }, { slow: true });
-                        batchSucceeded = true;
-                        this.clearFolderErrors(accountId, folder.path);
-                    } catch (e: any) {
-                        const msg = String(e?.message || "");
-                        console.error(`  [prefetch] ${accountId} folder ${folder.path}: batch fetch failed: ${msg}`);
-                        counters.errors++;
-                        this.recordFolderError(accountId, folder.path);
-                        if (counters.errors >= ERROR_BUDGET) break;
-                    }
-                    // CRITICAL: only prune when the batch actually completed.
-                    // A thrown batch means NOTHING was received; treating
-                    // absence as server-deletion lost 296 messages once.
-                    if (batchSucceeded) for (const uid of uids) {
-                        if (received.has(uid)) continue;
+                    for (let chunkStart = 0; chunkStart < uids.length; chunkStart += PREFETCH_CHUNK_SIZE) {
+                        const chunk = uids.slice(chunkStart, chunkStart + PREFETCH_CHUNK_SIZE);
+                        const received = new Set<number>();
+                        let batchSucceeded = false;
                         try {
-                            this.unlinkBodyFile(accountId, uid, folderId).catch(() => {});
-                            this.db.deleteMessage(accountId, uid, "prefetch batch: server didn't return body for queued UID — assumed deleted", "mailx-imap prefetchBodies (IMAP batch)");
-                            counters.deleted++;
-                            madeProgress = true;
-                        } catch { /* ignore */ }
+                            // Slow lane: prefetch is the textbook "this
+                            // might take a while" case — let interactive
+                            // ops slip ahead. Each chunk is its own
+                            // withConnection so the queue drains the
+                            // fast lane between chunks instead of holding
+                            // the connection through the whole folder.
+                            await this.withConnection(accountId, async (client) => {
+                                const pending: Promise<void>[] = [];
+                                await client.fetchBodiesBatch(folder.path, chunk, (uid: number, source: string) => {
+                                    received.add(uid);
+                                    pending.push((async () => {
+                                        try {
+                                            const raw = Buffer.from(source, "utf-8");
+                                            const bodyPath = await this.bodyStore.putMessage(accountId, folderId, uid, raw);
+                                            this.db.updateBodyPath(accountId, uid, bodyPath);
+                                            this.emit("bodyCached", accountId, uid);
+                                            counters.totalFetched++;
+                                            madeProgress = true;
+                                        } catch (e: any) {
+                                            console.error(`  [prefetch] ${accountId}/${uid}: store write failed: ${e.message}`);
+                                        }
+                                    })());
+                                });
+                                await Promise.all(pending);
+                            }, { slow: true });
+                            batchSucceeded = true;
+                            this.clearFolderErrors(accountId, folder.path);
+                        } catch (e: any) {
+                            const msg = String(e?.message || "");
+                            console.error(`  [prefetch] ${accountId} folder ${folder.path} chunk ${chunkStart / PREFETCH_CHUNK_SIZE}: batch fetch failed: ${msg}`);
+                            counters.errors++;
+                            this.recordFolderError(accountId, folder.path);
+                            if (counters.errors >= ERROR_BUDGET) break;
+                        }
+                        // Prune missing UIDs only when the batch actually
+                        // completed AND returned at least one body. A
+                        // "successful but empty" batch (FETCH parser
+                        // missed every literal, wrong-folder selected,
+                        // mid-stream connection hiccup that didn't
+                        // throw) is indistinguishable from "all UIDs
+                        // were deleted server-side" without that signal
+                        // — and erring toward delete cost Bob ~66
+                        // messages on 2026-05-08 (`prefetch] bobma: 0
+                        // bodies cached, 66 stale rows pruned`). The
+                        // set-diff reconcile in syncFolder is the
+                        // authoritative deletion path with a 30-min
+                        // grace window; prefetch defers to it.
+                        const someReceived = received.size > 0;
+                        if (batchSucceeded && someReceived) for (const uid of chunk) {
+                            if (received.has(uid)) continue;
+                            try {
+                                this.unlinkBodyFile(accountId, uid, folderId).catch(() => {});
+                                this.db.deleteMessage(accountId, uid, "prefetch batch: server didn't return body for queued UID — assumed deleted", "mailx-imap prefetchBodies (IMAP batch)");
+                                counters.deleted++;
+                                madeProgress = true;
+                            } catch { /* ignore */ }
+                        } else if (batchSucceeded && !someReceived) {
+                            console.error(`  [prefetch] ${accountId}/${folder.path}: chunk ${chunkStart}-${chunkStart + chunk.length - 1} returned 0/${chunk.length} bodies — NOT pruning (set-diff reconcile owns deletion). UIDs: ${chunk.slice(0, 5).join(",")}${chunk.length > 5 ? "..." : ""}`);
+                        }
                     }
+                    if (counters.errors >= ERROR_BUDGET) break;
                 }
                 if (counters.errors >= ERROR_BUDGET) {
                     console.error(`  [prefetch] ${accountId}: stopping after ${counters.errors} errors (${counters.totalFetched} cached, ${counters.deleted} pruned)`);
@@ -3019,6 +3321,20 @@ export class ImapManager extends EventEmitter {
         // retry. Foreign hosts are left alone — we have no way to know if their
         // process is alive. Cross-host stale recovery is the IMAP-folder path's
         // job (sweeper looks at server-side claim flags, not local files).
+        // Stale-claim recovery. A claim is "stale" if any of:
+        //  (a) the PID is dead — original owner crashed mid-send
+        //  (b) the PID is alive BUT it's not us, and the file mtime is
+        //      older than STALE_CLAIM_MS — the OS recycled the PID for
+        //      some other process. `process.kill(pid, 0)` returning success
+        //      only proves *some* process owns that PID, not that it's
+        //      our long-dead mailx daemon. Without the age guard, a
+        //      claim survives forever as soon as any other Node process
+        //      (statusline, msger, npm) gets the recycled PID. Bob saw
+        //      this exact case: `.sending-rmf39-63196` sat in the queue
+        //      for 7+ hours because PID 63196 was now an unrelated Node.
+        //  (c) it's our PID — never sweep our own claim.
+        const STALE_CLAIM_MS = 3600_000;
+        const myPid = process.pid;
         for (const dir of [outboxDir, queuedDir]) {
             if (!fs.existsSync(dir)) continue;
             for (const f of fs.readdirSync(dir)) {
@@ -3027,12 +3343,19 @@ export class ImapManager extends EventEmitter {
                 const [, original, host, pidStr] = m;
                 if (host !== this.hostname) continue;
                 const pid = parseInt(pidStr);
+                if (pid === myPid) continue; // it's us
                 let alive = false;
                 try { process.kill(pid, 0); alive = true; } catch { /* dead */ }
-                if (alive) continue; // live claim — owner (sibling or self) still has it
+                let ageMs = Infinity;
+                try { ageMs = Date.now() - fs.statSync(path.join(dir, f)).mtimeMs; } catch { /* */ }
+                // Live PID + recent mtime → assume genuine sibling owner.
+                // Live PID + ancient mtime → PID got recycled, sweep it.
+                // Dead PID → sweep regardless of age.
+                if (alive && ageMs < STALE_CLAIM_MS) continue;
                 try {
                     fs.renameSync(path.join(dir, f), path.join(dir, original));
-                    console.log(`  [outbox] Recovered stale claim ${f} → ${original}`);
+                    const reason = alive ? `recycled PID, mtime ${Math.round(ageMs / 60_000)}m old` : "dead PID";
+                    console.log(`  [outbox] Recovered stale claim ${f} → ${original} (${reason})`);
                 } catch { /* ignore */ }
             }
         }
@@ -3324,14 +3647,31 @@ export class ImapManager extends EventEmitter {
                     await client.deleteMessageByUid(outboxFolder.path, uid);
                 }, { slow: true });
                 if (sentFolder) {
+                    let appendedSentUid: number | null = null;
                     try {
                         await this.withConnection(accountId, async (client) => {
-                            await client.appendMessage(sentFolder.path, source, ["\\Seen"]);
+                            appendedSentUid = await client.appendMessage(sentFolder.path, source, ["\\Seen"]);
                         }, { slow: true });
-                        this.syncFolder(accountId, sentFolder.id).catch(() => {});
                     } catch (sentErr: any) {
                         console.error(`  [outbox] Failed to copy to Sent: ${sentErr.message} — message was sent successfully`);
                     }
+                    if (appendedSentUid != null) {
+                        // The server's APPENDUID response gave us the exact UID
+                        // the message landed at in Sent. Insert the local row
+                        // directly from the source we already have — no IMAP
+                        // round-trip, no SELECT-then-FETCH dance. The Sent
+                        // folder view shows the message immediately. The next
+                        // periodic sync will see this UID already present and
+                        // no-op. Critical: this means a slow/stuck Sent SELECT
+                        // never blocks "show me what I just sent" — that was
+                        // the user-visible "where's my sent message?" bug.
+                        await this.insertLocalRowFromSource(accountId, sentFolder, appendedSentUid, source, ["\\Seen"])
+                            .catch((e: any) => console.error(`  [outbox] Local Sent row insert failed: ${e?.message || e} — falling back to broad sync`));
+                    } else {
+                        // No APPENDUID — server doesn't support UIDPLUS, or
+                        // APPEND itself failed. Fall back to a broad sync.
+                        this.syncFolder(accountId, sentFolder.id).catch(() => {});
+                    }
                     this.syncFolder(accountId, outboxFolder.id).catch(() => {});
                 }
             } catch (e: any) {
@@ -3683,8 +4023,14 @@ export class ImapManager extends EventEmitter {
         this.stopPeriodicSync();
         this.stopOutboxWorker();
         await this.stopWatching();
-        // Disconnect all persistent operational connections
-        for (const [accountId] of this.opsClients) {
+        // Disconnect persistent connections on both lanes. Use a Set
+        // because fastClients can hold an entry an account doesn't have
+        // in opsClients (e.g. body fetches happened but no sync did).
+        const accountIds = new Set<string>([
+            ...this.opsClients.keys(),
+            ...this.fastClients.keys(),
+        ]);
+        for (const accountId of accountIds) {
             await this.disconnectOps(accountId);
         }
     }