@bobfrankston/mailx-imap 0.1.42 → 0.1.44

package/index.d.ts

@@ -67,6 +67,11 @@ export declare class ImapManager extends EventEmitter {
     private accountErrorShown;
     private syncing;
     private inboxSyncing;
+    /** Wall-clock of ImapManager construction. Used by the lazy-folder-sync
+     *  gate so non-priority folders defer their first-sync until ~3 min
+     *  past startup, when the event loop has quieted down. Each restart
+     *  resets this; the gate auto-lifts as time passes. */
+    private _startupAt;
     /** Use native IMAP client instead of imapflow. Set to true to enable. */
     useNativeClient: boolean;
     /** Per-account health counters. Incremented when the server misbehaves

package/index.js

@@ -142,6 +142,11 @@ export class ImapManager extends EventEmitter {
     accountErrorShown = new Set();
     syncing = false;
     inboxSyncing = false;
+    /** Wall-clock of ImapManager construction. Used by the lazy-folder-sync
+     *  gate so non-priority folders defer their first-sync until ~3 min
+     *  past startup, when the event loop has quieted down. Each restart
+     *  resets this; the gate auto-lifts as time passes. */
+    _startupAt = Date.now();
     /** Use native IMAP client instead of imapflow. Set to true to enable. */
     useNativeClient = false;
     // Connection management: see withConnection() below.
@@ -899,12 +904,15 @@ export class ImapManager extends EventEmitter {
                 // upsert turns into an UPDATE (cheap); if it doesn't, the
                 // insert proceeds. Trust the constraint — don't second-
                 // guess it on a stale highestUid snapshot.
-                // Tombstone check: if the user locally deleted this Message-ID,
-                // don't re-import it. Server-side EXPUNGE may lag, or reconcile
-                // may find the message in an old list snapshot. Without this,
-                // deleted messages reappear on the next sync pass.
+                // Tombstone check: skip re-import while a local delete/move
+                // is in flight to the server. Cleared on action success
+                // (server EXPUNGED → next sync's set-diff drops the row
+                // anyway) or on permanent failure (clearTombstoneForUid in
+                // failSyncAction → row reappears, user sees the action
+                // didn't take). QRESYNC servers bypass this entirely
+                // because their VANISHED responses are authoritative.
                 if (msg.messageId && this.db.hasTombstone(accountId, msg.messageId)) {
-                    console.log(`  [tombstone] ${accountId}: skipping ${msg.messageId} (locally deleted)`);
+                    console.log(`  [tombstone] ${accountId}: skipping ${msg.messageId} (locally deleted, awaiting server confirm)`);
                     continue;
                 }
                 const source = msg.source || "";
@@ -1119,6 +1127,93 @@ export class ImapManager extends EventEmitter {
             }
         }
         console.log(`  [sync] ${accountId}/${folder.path}: highestUid=${highestUid}, fetching...`);
+        // ── QRESYNC fast path (RFC 7162) ─────────────────────────────────
+        // When the server supports QRESYNC AND we have a saved (uidValidity,
+        // modSeq) watermark for this folder, ask the server "what changed
+        // since modSeq?". The server replies with an authoritative VANISHED
+        // list (no client-side diffing needed, no tombstones needed) plus
+        // unsolicited FETCH for flag/state changes. New UIDs > our prior
+        // highest still go through the normal fetchMessagesSinceUid path
+        // afterwards. Skips the heavy UID SEARCH set-diff below entirely.
+        const folderSyncState = this.db.getFolderSync(folderId);
+        const prevModSeq = parseInt(folderSyncState.highestModseq || "0", 10);
+        const prevUidValidity = folderSyncState.uidvalidity || 0;
+        const caps = (typeof client.getCapabilities === "function")
+            ? client.getCapabilities()
+            : new Set();
+        const qresyncSupported = caps.has("QRESYNC");
+        const haveWatermark = highestUid > 0 && prevModSeq > 0 && prevUidValidity > 0;
+        if (qresyncSupported && haveWatermark && typeof client.resyncFolder === "function") {
+            try {
+                // Ensure ENABLE QRESYNC has been issued on this connection.
+                // enableQresync is idempotent + capability-gated so this is a
+                // no-op on the second-and-subsequent call per connection.
+                if (typeof client.enableQresync === "function") {
+                    await client.enableQresync();
+                }
+                const __qrT0 = Date.now();
+                const qr = await client.resyncFolder(folder.path, prevUidValidity, prevModSeq);
+                console.log(`  [qresync] ${accountId}/${folder.path}: vanished=${qr.vanishedUids.length} changed=${qr.changedMessages.length} newModSeq=${qr.newHighestModSeq} in ${Date.now() - __qrT0}ms`);
+                if (qr.uidValidityChanged) {
+                    // UIDVALIDITY rolled — our local UIDs are stale. Fall through
+                    // to the full set-diff path; that'll discover the new state
+                    // and the deletion safeguards (50% threshold) will keep us
+                    // from wiping anything we shouldn't.
+                    console.log(`  [qresync] ${accountId}/${folder.path}: UIDVALIDITY changed (was ${prevUidValidity}, now ${qr.exists}); falling back to full sync`);
+                }
+                else {
+                    // Apply VANISHED — server says these UIDs are gone. No
+                    // tombstone, no diff, just delete the local rows.
+                    let vanishedApplied = 0;
+                    for (const uid of qr.vanishedUids) {
+                        const env = this.db.getMessageByUid(accountId, uid, folderId);
+                        if (env) {
+                            this.db.deleteMessage(accountId, uid, "server VANISHED via QRESYNC", "mailx-imap syncFolder/qresync");
+                            vanishedApplied++;
+                        }
+                    }
+                    if (vanishedApplied > 0) {
+                        this.db.recalcFolderCounts(folderId);
+                        this.emit("folderCountsChanged", accountId, {});
+                    }
+                    // Apply changed-state FETCH — flag updates since prior modSeq.
+                    for (const m of qr.changedMessages) {
+                        try {
+                            const flagsArr = Array.from(m.flags || []).map(f => String(f));
+                            this.db.updateMessageFlags(accountId, m.uid, flagsArr);
+                        }
+                        catch { /* row may have just been VANISHED */ }
+                    }
+                    // Fetch genuinely new messages — UID > our prior highest.
+                    // resyncFolder doesn't return these directly; the server
+                    // emits them only via state-change FETCH if their flags
+                    // changed since modSeq, which is unreliable. Pull them
+                    // explicitly via the existing incremental path.
+                    const fetched = await client.fetchMessagesSinceUid(folder.path, highestUid, { source: false });
+                    const newOnes = fetched.filter((m) => m.uid > highestUid);
+                    if (newOnes.length > 0) {
+                        await this.storeMessages(accountId, folderId, folder, newOnes, highestUid);
+                    }
+                    // Persist new watermark — next resync starts here.
+                    if (qr.newHighestModSeq !== undefined) {
+                        this.db.updateFolderSync(folderId, qr.exists ? prevUidValidity : prevUidValidity, String(qr.newHighestModSeq));
+                    }
+                    this.emit("syncProgress", accountId, `sync:${folder.path}`, 100);
+                    console.log(`  [qresync] ${accountId}/${folder.path}: done in ${Date.now() - __sfStart}ms (path: QRESYNC)`);
+                    return newOnes.length;
+                }
+            }
+            catch (qrErr) {
+                // QRESYNC failed (server quirk, network glitch, etc.) — log
+                // and fall through to the legacy set-diff path. We do NOT
+                // clear the modSeq watermark here; the next attempt will
+                // re-try QRESYNC, and if that also fails enough times the
+                // operator can manually clear `highest_modseq` to force a
+                // full resync.
+                console.warn(`  [qresync] ${accountId}/${folder.path}: failed (${qrErr?.message || qrErr}) — falling back to set-diff`);
+            }
+        }
+        // ────────────────────────────────────────────────────────────────
         let messages;
         // Cache the server-UID list across set-diff and deletion-recon so we
         // don't pay for two `UID SEARCH` round-trips (the second one was
@@ -1315,10 +1410,9 @@ export class ImapManager extends EventEmitter {
                     // UNIQUE constraint to dedupe — if mailx truly has the
                     // row, the upsert becomes an UPDATE that refreshes
                     // flags too, all in one path.
-                    // Tombstone check — same reason as the streamy onChunk path
-                    // at storeMessages: a locally-deleted message that the server
-                    // hasn't EXPUNGEd yet would otherwise reappear on next sync.
-                    // User-visible symptom: "I deleted it but it came back."
+                    // Tombstone-skip: in-flight local delete/move waiting
+                    // for server confirmation. Cleared on action success or
+                    // permanent failure (see clearTombstoneForUid).
                     if (msg.messageId && this.db.hasTombstone(accountId, msg.messageId)) {
                         console.log(`  [tombstone] ${accountId}: skipping ${msg.messageId} in syncFolder (locally deleted)`);
                         continue;
@@ -1484,6 +1578,22 @@ export class ImapManager extends EventEmitter {
         }
         this.emit("folderSynced", accountId, folderId, syncedAt);
         this.db.updateLastSync(accountId, syncedAt);
+        // Seed the QRESYNC watermark from the SELECT response of whatever
+        // legacy operation just ran (fetchMessagesSinceUid, getUids, etc.).
+        // Without this, a CONDSTORE/QRESYNC-capable server's `HIGHESTMODSEQ`
+        // is captured by the native layer but never persisted, and the
+        // QRESYNC fast path above stays permanently disabled because
+        // `prevModSeq` is 0. Captures on every sync (cheap) so the watermark
+        // converges to the latest known modSeq.
+        try {
+            if (typeof client.getCurrentMailboxInfo === "function") {
+                const info = client.getCurrentMailboxInfo();
+                if (info && info.highestModSeq && info.uidValidity) {
+                    this.db.updateFolderSync(folderId, info.uidValidity, String(info.highestModSeq));
+                }
+            }
+        }
+        catch { /* non-fatal — QRESYNC just stays disabled */ }
         return newCount;
     }
     /** Sync all folders for all accounts */
@@ -1611,12 +1721,37 @@ export class ImapManager extends EventEmitter {
             // timeout abandons a stalled command instead of waiting out
             // Dovecot's 300s server-side inactivity timer; the next sync tick
             // retries on a fresh socket.
-            const remaining = folders.filter(f => f.specialUse !== "inbox");
+            // 2026-05-13: defer first-sync of non-priority folders by default
+            // (C119 lazy folder sync). On bobma with 90+ folders, doing a
+            // first-sync of every folder on every startup hammers the IMAP
+            // socket buffer and pegs the daemon's event loop processing
+            // FETCH literals — user clicks get a 20+ second IPC delay
+            // because no IPC can squeeze in between FETCH chunks. Now: only
+            // special-use folders (Sent/Drafts/Archive/Junk/Trash) + folders
+            // mailx has previously seen (highestUid > 0, i.e. tracked
+            // across the prior session) sync automatically. First-sync of
+            // a "never-touched" non-special folder is deferred until the
+            // user opens that folder (on-demand syncFolder call) or until
+            // ~3 minutes after startup when the event loop is quiet.
+            const STARTUP_LAZY_DELAY_MS = 3 * 60 * 1000;
+            const startupQuietPoint = (this._startupAt || 0) + STARTUP_LAZY_DELAY_MS;
+            const isLazyEligible = (f) => {
+                if (f.specialUse && priorityOrder.includes(f.specialUse))
+                    return false;
+                if (this.db.getHighestUid(accountId, f.id) > 0)
+                    return false;
+                return true;
+            };
+            const remaining = folders.filter(f => f.specialUse !== "inbox" && !(isLazyEligible(f) && Date.now() < startupQuietPoint));
             remaining.sort((a, b) => {
                 const pa = priorityOrder.indexOf(a.specialUse || "") >= 0 ? priorityOrder.indexOf(a.specialUse || "") : 5;
                 const pb = priorityOrder.indexOf(b.specialUse || "") >= 0 ? priorityOrder.indexOf(b.specialUse || "") : 5;
                 return pa - pb;
             });
+            const deferredCount = folders.filter(f => f.specialUse !== "inbox" && isLazyEligible(f) && Date.now() < startupQuietPoint).length;
+            if (deferredCount > 0) {
+                console.log(`  [sync] ${accountId}: deferring first-sync of ${deferredCount} non-priority folder(s) until ${new Date(startupQuietPoint).toLocaleTimeString()} or user opens them`);
+            }
             const CONCURRENCY = 2;
             // First-sync of a fresh account on a cold Dovecot is dominated by
             // `UID SEARCH SINCE 30-days-ago`, which can take 5+ minutes on a
@@ -1892,10 +2027,9 @@ export class ImapManager extends EventEmitter {
         // standalone — bad rows are logged and skipped.
         for (const msg of msgs) {
             try {
-                // Tombstone check — Gmail API sync was missing this so a
-                // locally-trashed Gmail message would reappear on the next
-                // listMessages tick if Gmail's eventual-consistency hadn't
-                // promoted the trash yet. Symmetric with the IMAP paths.
+                // Tombstone-skip: in-flight local delete/move on Gmail.
+                // Cleared by clearTombstoneForUid on failSyncAction so a
+                // refused trash doesn't permanently hide the message.
                 if (msg.messageId && this.db.hasTombstone(accountId, msg.messageId)) {
                     continue;
                 }
@@ -2799,6 +2933,14 @@ export class ImapManager extends EventEmitter {
         if (messages.length === 0)
             return;
         const trash = this.findFolder(accountId, "trash");
+        // Tombstone each Message-ID so sync won't re-import the source-folder
+        // row before the server-side MOVE completes. Cleared on permanent
+        // failure (clearTombstoneForUid in processSyncActions).
+        for (const msg of messages) {
+            const env = this.db.getMessageByUid(accountId, msg.uid, msg.folderId);
+            if (env?.messageId)
+                this.db.addTombstone(accountId, env.messageId, env.subject || "");
+        }
         // Local first — move to trash folder locally so the row stays
         // visible in Trash and Ctrl+Z can restore it. Body file stays in
         // its original folder dir; the next sync rebinds path on
@@ -2865,6 +3007,14 @@ export class ImapManager extends EventEmitter {
     /** Move a message to Trash (delete) — local-first, queues IMAP sync */
     async trashMessage(accountId, folderId, uid) {
         const trash = this.findFolder(accountId, "trash");
+        // Tombstone the Message-ID so sync won't re-import the row in the
+        // source folder before the server-side move completes. Cleared on
+        // permanent failure of the queued sync_action (see processSyncActions
+        // catch block, where clearTombstoneForUid runs after attempts >= 5)
+        // so the user sees the row reappear when their action didn't take.
+        const env = this.db.getMessageByUid(accountId, uid, folderId);
+        if (env?.messageId)
+            this.db.addTombstone(accountId, env.messageId, env.subject || "");
         // Local first — move to trash folder so the row stays visible in
         // Trash and Ctrl+Z can restore. Body file retained for undelete.
         // If we're already in trash (or no trash configured), fall through
@@ -3034,8 +3184,15 @@ export class ImapManager extends EventEmitter {
                     catch (e) {
                         console.error(`  [api] ${accountId}: flag sync failed UID ${action.uid}: ${e.message}`);
                         this.db.failSyncAction(action.id, e.message);
-                        if (action.attempts >= 5)
+                        if (action.attempts >= 5) {
+                            // Terminal failure on delete/move → clear tombstone
+                            // so the row reappears on next sync (server still
+                            // has it). Same rationale as the IMAP branch below.
+                            if (action.action === "delete" || action.action === "move") {
+                                this.db.clearTombstoneForUid(accountId, action.uid, action.folderId);
+                            }
                             this.db.completeSyncAction(action.id);
+                        }
                     }
                 }
             }
@@ -3115,6 +3272,17 @@ export class ImapManager extends EventEmitter {
                     this.emit("syncActionFailed", accountId, action.action, action.uid, e.message);
                     if (action.attempts >= 5) {
                         console.error(`  [sync] Giving up on action ${action.id} after 5 attempts`);
+                        // 2026-05-13: terminal failure on a delete/move
+                        // means the server still has the message — so any
+                        // tombstone we set for it would otherwise hide it
+                        // from the user forever. Clear the tombstone so
+                        // the next sync re-imports the row, accurately
+                        // reflecting "your action didn't take, here it is
+                        // again." Applies to delete + move; flags/append
+                        // never tombstone.
+                        if (action.action === "delete" || action.action === "move") {
+                            this.db.clearTombstoneForUid(accountId, action.uid, action.folderId);
+                        }
                         this.db.completeSyncAction(action.id);
                         this.emit("syncActionFailed", accountId, action.action, action.uid, `Gave up after 5 attempts: ${e.message}`);
                     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bobfrankston/mailx-imap",
-  "version": "0.1.42",
+  "version": "0.1.44",
   "type": "module",
   "main": "index.js",
   "types": "index.d.ts",
@@ -11,8 +11,8 @@
   "dependencies": {
     "@bobfrankston/mailx-types": "^0.1.11",
     "@bobfrankston/mailx-settings": "^0.1.16",
-    "@bobfrankston/mailx-store": "^0.1.22",
-    "@bobfrankston/iflow-direct": "^0.1.42",
+    "@bobfrankston/mailx-store": "^0.1.24",
+    "@bobfrankston/iflow-direct": "^0.1.44",
     "@bobfrankston/tcp-transport": "^0.1.6",
     "@bobfrankston/smtp-direct": "^0.1.8",
     "@bobfrankston/mailx-sync": "^0.1.16",
@@ -39,8 +39,8 @@
     "dependencies": {
       "@bobfrankston/mailx-types": "^0.1.11",
       "@bobfrankston/mailx-settings": "^0.1.16",
-      "@bobfrankston/mailx-store": "^0.1.22",
-      "@bobfrankston/iflow-direct": "^0.1.42",
+      "@bobfrankston/mailx-store": "^0.1.24",
+      "@bobfrankston/iflow-direct": "^0.1.44",
       "@bobfrankston/tcp-transport": "^0.1.6",
       "@bobfrankston/smtp-direct": "^0.1.8",
       "@bobfrankston/mailx-sync": "^0.1.16",