@bobfrankston/rmfmail 1.0.697 → 1.0.699

package/packages/mailx-imap/index.js

@@ -98,7 +98,7 @@ function decodeEntities(text) {
 /** Extract a plain-text preview from message source */
 async function extractPreview(source) {
     try {
-        const parsed = await parseSerial(source);
+        const parsed = await parseSerial(source, "background");
         const bodyText = parsed.text || "";
         const bodyHtml = parsed.html || "";
         // Use text part; fall back to stripping HTML tags if text is empty
@@ -899,12 +899,15 @@ export class ImapManager extends EventEmitter {
                 // upsert turns into an UPDATE (cheap); if it doesn't, the
                 // insert proceeds. Trust the constraint — don't second-
                 // guess it on a stale highestUid snapshot.
-                // Tombstone check: if the user locally deleted this Message-ID,
-                // don't re-import it. Server-side EXPUNGE may lag, or reconcile
-                // may find the message in an old list snapshot. Without this,
-                // deleted messages reappear on the next sync pass.
+                // Tombstone check: skip re-import while a local delete/move
+                // is in flight to the server. Cleared on action success
+                // (server EXPUNGED → next sync's set-diff drops the row
+                // anyway) or on permanent failure (clearTombstoneForUid in
+                // failSyncAction → row reappears, user sees the action
+                // didn't take). QRESYNC servers bypass this entirely
+                // because their VANISHED responses are authoritative.
                 if (msg.messageId && this.db.hasTombstone(accountId, msg.messageId)) {
-                    console.log(`  [tombstone] ${accountId}: skipping ${msg.messageId} (locally deleted)`);
+                    console.log(`  [tombstone] ${accountId}: skipping ${msg.messageId} (locally deleted, awaiting server confirm)`);
                     continue;
                 }
                 const source = msg.source || "";
@@ -971,7 +974,11 @@ export class ImapManager extends EventEmitter {
      *
      *  Fires the same emits as a normal sync so the UI updates. */
     async insertLocalRowFromSource(accountId, folder, uid, source, flags) {
-        const parsed = await parseSerial(source);
+        // insertLocalRowFromSource runs right after sendMessage — that's a
+        // user-initiated path but the parse cost is on the post-send
+        // background work, not the click-through. Tag as background so a
+        // concurrent user-click foreground parse jumps ahead.
+        const parsed = await parseSerial(source, "background");
         // Coerce mailparser AddressObject(s) into the flat `{name, address}[]`
         // shape storeMessages's downstream toEmailAddresses expects.
         // RFC 2047 encoded-word decoding (incl. inside quoted-strings) is
@@ -1115,6 +1122,93 @@ export class ImapManager extends EventEmitter {
             }
         }
         console.log(`  [sync] ${accountId}/${folder.path}: highestUid=${highestUid}, fetching...`);
+        // ── QRESYNC fast path (RFC 7162) ─────────────────────────────────
+        // When the server supports QRESYNC AND we have a saved (uidValidity,
+        // modSeq) watermark for this folder, ask the server "what changed
+        // since modSeq?". The server replies with an authoritative VANISHED
+        // list (no client-side diffing needed, no tombstones needed) plus
+        // unsolicited FETCH for flag/state changes. New UIDs > our prior
+        // highest still go through the normal fetchMessagesSinceUid path
+        // afterwards. Skips the heavy UID SEARCH set-diff below entirely.
+        const folderSyncState = this.db.getFolderSync(folderId);
+        const prevModSeq = parseInt(folderSyncState.highestModseq || "0", 10);
+        const prevUidValidity = folderSyncState.uidvalidity || 0;
+        const caps = (typeof client.getCapabilities === "function")
+            ? client.getCapabilities()
+            : new Set();
+        const qresyncSupported = caps.has("QRESYNC");
+        const haveWatermark = highestUid > 0 && prevModSeq > 0 && prevUidValidity > 0;
+        if (qresyncSupported && haveWatermark && typeof client.resyncFolder === "function") {
+            try {
+                // Ensure ENABLE QRESYNC has been issued on this connection.
+                // enableQresync is idempotent + capability-gated so this is a
+                // no-op on the second-and-subsequent call per connection.
+                if (typeof client.enableQresync === "function") {
+                    await client.enableQresync();
+                }
+                const __qrT0 = Date.now();
+                const qr = await client.resyncFolder(folder.path, prevUidValidity, prevModSeq);
+                console.log(`  [qresync] ${accountId}/${folder.path}: vanished=${qr.vanishedUids.length} changed=${qr.changedMessages.length} newModSeq=${qr.newHighestModSeq} in ${Date.now() - __qrT0}ms`);
+                if (qr.uidValidityChanged) {
+                    // UIDVALIDITY rolled — our local UIDs are stale. Fall through
+                    // to the full set-diff path; that'll discover the new state
+                    // and the deletion safeguards (50% threshold) will keep us
+                    // from wiping anything we shouldn't.
+                    console.log(`  [qresync] ${accountId}/${folder.path}: UIDVALIDITY changed (was ${prevUidValidity}, now ${qr.exists}); falling back to full sync`);
+                }
+                else {
+                    // Apply VANISHED — server says these UIDs are gone. No
+                    // tombstone, no diff, just delete the local rows.
+                    let vanishedApplied = 0;
+                    for (const uid of qr.vanishedUids) {
+                        const env = this.db.getMessageByUid(accountId, uid, folderId);
+                        if (env) {
+                            this.db.deleteMessage(accountId, uid, "server VANISHED via QRESYNC", "mailx-imap syncFolder/qresync");
+                            vanishedApplied++;
+                        }
+                    }
+                    if (vanishedApplied > 0) {
+                        this.db.recalcFolderCounts(folderId);
+                        this.emit("folderCountsChanged", accountId, {});
+                    }
+                    // Apply changed-state FETCH — flag updates since prior modSeq.
+                    for (const m of qr.changedMessages) {
+                        try {
+                            const flagsArr = Array.from(m.flags || []).map(f => String(f));
+                            this.db.updateMessageFlags(accountId, m.uid, flagsArr);
+                        }
+                        catch { /* row may have just been VANISHED */ }
+                    }
+                    // Fetch genuinely new messages — UID > our prior highest.
+                    // resyncFolder doesn't return these directly; the server
+                    // emits them only via state-change FETCH if their flags
+                    // changed since modSeq, which is unreliable. Pull them
+                    // explicitly via the existing incremental path.
+                    const fetched = await client.fetchMessagesSinceUid(folder.path, highestUid, { source: false });
+                    const newOnes = fetched.filter((m) => m.uid > highestUid);
+                    if (newOnes.length > 0) {
+                        await this.storeMessages(accountId, folderId, folder, newOnes, highestUid);
+                    }
+                    // Persist new watermark — next resync starts here.
+                    if (qr.newHighestModSeq !== undefined) {
+                        this.db.updateFolderSync(folderId, qr.exists ? prevUidValidity : prevUidValidity, String(qr.newHighestModSeq));
+                    }
+                    this.emit("syncProgress", accountId, `sync:${folder.path}`, 100);
+                    console.log(`  [qresync] ${accountId}/${folder.path}: done in ${Date.now() - __sfStart}ms (path: QRESYNC)`);
+                    return newOnes.length;
+                }
+            }
+            catch (qrErr) {
+                // QRESYNC failed (server quirk, network glitch, etc.) — log
+                // and fall through to the legacy set-diff path. We do NOT
+                // clear the modSeq watermark here; the next attempt will
+                // re-try QRESYNC, and if that also fails enough times the
+                // operator can manually clear `highest_modseq` to force a
+                // full resync.
+                console.warn(`  [qresync] ${accountId}/${folder.path}: failed (${qrErr?.message || qrErr}) — falling back to set-diff`);
+            }
+        }
+        // ────────────────────────────────────────────────────────────────
         let messages;
         // Cache the server-UID list across set-diff and deletion-recon so we
         // don't pay for two `UID SEARCH` round-trips (the second one was
@@ -1311,10 +1405,9 @@ export class ImapManager extends EventEmitter {
                     // UNIQUE constraint to dedupe — if mailx truly has the
                     // row, the upsert becomes an UPDATE that refreshes
                     // flags too, all in one path.
-                    // Tombstone check — same reason as the streamy onChunk path
-                    // at storeMessages: a locally-deleted message that the server
-                    // hasn't EXPUNGEd yet would otherwise reappear on next sync.
-                    // User-visible symptom: "I deleted it but it came back."
+                    // Tombstone-skip: in-flight local delete/move waiting
+                    // for server confirmation. Cleared on action success or
+                    // permanent failure (see clearTombstoneForUid).
                     if (msg.messageId && this.db.hasTombstone(accountId, msg.messageId)) {
                         console.log(`  [tombstone] ${accountId}: skipping ${msg.messageId} in syncFolder (locally deleted)`);
                         continue;
@@ -1480,6 +1573,22 @@ export class ImapManager extends EventEmitter {
         }
         this.emit("folderSynced", accountId, folderId, syncedAt);
         this.db.updateLastSync(accountId, syncedAt);
+        // Seed the QRESYNC watermark from the SELECT response of whatever
+        // legacy operation just ran (fetchMessagesSinceUid, getUids, etc.).
+        // Without this, a CONDSTORE/QRESYNC-capable server's `HIGHESTMODSEQ`
+        // is captured by the native layer but never persisted, and the
+        // QRESYNC fast path above stays permanently disabled because
+        // `prevModSeq` is 0. Captures on every sync (cheap) so the watermark
+        // converges to the latest known modSeq.
+        try {
+            if (typeof client.getCurrentMailboxInfo === "function") {
+                const info = client.getCurrentMailboxInfo();
+                if (info && info.highestModSeq && info.uidValidity) {
+                    this.db.updateFolderSync(folderId, info.uidValidity, String(info.highestModSeq));
+                }
+            }
+        }
+        catch { /* non-fatal — QRESYNC just stays disabled */ }
         return newCount;
     }
     /** Sync all folders for all accounts */
@@ -1888,10 +1997,9 @@ export class ImapManager extends EventEmitter {
         // standalone — bad rows are logged and skipped.
         for (const msg of msgs) {
             try {
-                // Tombstone check — Gmail API sync was missing this so a
-                // locally-trashed Gmail message would reappear on the next
-                // listMessages tick if Gmail's eventual-consistency hadn't
-                // promoted the trash yet. Symmetric with the IMAP paths.
+                // Tombstone-skip: in-flight local delete/move on Gmail.
+                // Cleared by clearTombstoneForUid on failSyncAction so a
+                // refused trash doesn't permanently hide the message.
                 if (msg.messageId && this.db.hasTombstone(accountId, msg.messageId)) {
                     continue;
                 }
@@ -2795,6 +2903,14 @@ export class ImapManager extends EventEmitter {
         if (messages.length === 0)
             return;
         const trash = this.findFolder(accountId, "trash");
+        // Tombstone each Message-ID so sync won't re-import the source-folder
+        // row before the server-side MOVE completes. Cleared on permanent
+        // failure (clearTombstoneForUid in processSyncActions).
+        for (const msg of messages) {
+            const env = this.db.getMessageByUid(accountId, msg.uid, msg.folderId);
+            if (env?.messageId)
+                this.db.addTombstone(accountId, env.messageId, env.subject || "");
+        }
         // Local first — move to trash folder locally so the row stays
         // visible in Trash and Ctrl+Z can restore it. Body file stays in
         // its original folder dir; the next sync rebinds path on
@@ -2861,6 +2977,14 @@ export class ImapManager extends EventEmitter {
     /** Move a message to Trash (delete) — local-first, queues IMAP sync */
     async trashMessage(accountId, folderId, uid) {
         const trash = this.findFolder(accountId, "trash");
+        // Tombstone the Message-ID so sync won't re-import the row in the
+        // source folder before the server-side move completes. Cleared on
+        // permanent failure of the queued sync_action (see processSyncActions
+        // catch block, where clearTombstoneForUid runs after attempts >= 5)
+        // so the user sees the row reappear when their action didn't take.
+        const env = this.db.getMessageByUid(accountId, uid, folderId);
+        if (env?.messageId)
+            this.db.addTombstone(accountId, env.messageId, env.subject || "");
         // Local first — move to trash folder so the row stays visible in
         // Trash and Ctrl+Z can restore. Body file retained for undelete.
         // If we're already in trash (or no trash configured), fall through
@@ -3030,8 +3154,15 @@ export class ImapManager extends EventEmitter {
                     catch (e) {
                         console.error(`  [api] ${accountId}: flag sync failed UID ${action.uid}: ${e.message}`);
                         this.db.failSyncAction(action.id, e.message);
-                        if (action.attempts >= 5)
+                        if (action.attempts >= 5) {
+                            // Terminal failure on delete/move → clear tombstone
+                            // so the row reappears on next sync (server still
+                            // has it). Same rationale as the IMAP branch below.
+                            if (action.action === "delete" || action.action === "move") {
+                                this.db.clearTombstoneForUid(accountId, action.uid, action.folderId);
+                            }
                             this.db.completeSyncAction(action.id);
+                        }
                     }
                 }
             }
@@ -3111,6 +3242,17 @@ export class ImapManager extends EventEmitter {
                     this.emit("syncActionFailed", accountId, action.action, action.uid, e.message);
                     if (action.attempts >= 5) {
                         console.error(`  [sync] Giving up on action ${action.id} after 5 attempts`);
+                        // 2026-05-13: terminal failure on a delete/move
+                        // means the server still has the message — so any
+                        // tombstone we set for it would otherwise hide it
+                        // from the user forever. Clear the tombstone so
+                        // the next sync re-imports the row, accurately
+                        // reflecting "your action didn't take, here it is
+                        // again." Applies to delete + move; flags/append
+                        // never tombstone.
+                        if (action.action === "delete" || action.action === "move") {
+                            this.db.clearTombstoneForUid(accountId, action.uid, action.folderId);
+                        }
                         this.db.completeSyncAction(action.id);
                         this.emit("syncActionFailed", accountId, action.action, action.uid, `Gave up after 5 attempts: ${e.message}`);
                     }