@bobfrankston/mailx-imap 0.1.100 → 0.1.102

package/index.d.ts

@@ -271,6 +271,26 @@ export declare class ImapManager extends EventEmitter {
      *  the first caller finishes — important for outbox flushes that
      *  expect their syncFolder for `Sent` to actually run. */
     private syncFolderLocks;
+    /** Per-folder timestamp of the last set-difference reconcile. Drives the
+     *  throttle for the QRESYNC-path gap backfill (see fetchServerOnlyUids /
+     *  syncFolder QRESYNC branch). Empty at boot, so the first sync of each
+     *  folder after a restart always runs a reconcile — which is exactly when
+     *  a daemon-was-down gap needs to be healed. */
+    private lastReconcileMs;
+    /**
+     * Set-difference backfill. Fetch every server UID (within the history
+     * window) that we don't already have locally and aren't fetching this
+     * cycle. This is the ONLY mechanism that recovers a message which is on
+     * the server but missing locally — the high-water-mark fetch
+     * (fetchMessagesSinceUid) cannot, because it only looks above the local
+     * highest UID. Gaps it heals: a message that arrived while the daemon was
+     * down (so it sits below the watermark once the next poll advances it), or
+     * one dropped mid-stream by a response desync. Returns the recovered
+     * messages (caller stores them) plus the server UID list it fetched, so the
+     * caller's deletion-reconcile can reuse it instead of paying for a second
+     * UID SEARCH round-trip.
+     */
+    private fetchServerOnlyUids;
     syncFolder(accountId: string, folderId: number, client?: any): Promise<number>;
     private _syncFolderImpl;
     /** Sync all folders for all accounts */
@@ -379,7 +399,7 @@ export declare class ImapManager extends EventEmitter {
      *  Server fetch goes through the unified ops queue on the fast lane —
      *  the user clicked, they're waiting, this jumps ahead of any background
      *  prefetch sitting in the slow lane. */
-    fetchMessageBody(accountId: string, folderId: number, uid: number): Promise<Buffer | null>;
+    fetchMessageBody(accountId: string, folderId: number, uid: number, force?: boolean): Promise<Buffer | null>;
     /** Fetch message body via Gmail/Outlook API.
      *  Throws `MessageNotFoundError` when the server says the message is gone
      *  (deleted from another device, for example). The caller uses that to

package/index.js

@@ -21,6 +21,11 @@ const SMTP_PORT_IMPLICIT_TLS = 465;
  *  the same file is retried. Gives the server time to settle so a retry after a
  *  lost-ack doesn't arrive while the first copy is still being processed. */
 const OUTBOX_RETRY_DELAY_MS = 60000;
+/** How often the QRESYNC fast path runs a full set-difference reconcile to
+ *  heal gaps the high-water-mark fetch can't see. The first sync of a folder
+ *  after boot always runs one (the map is empty); thereafter it's throttled to
+ *  this interval so the QRESYNC speedup is preserved in steady state. */
+const RECONCILE_THROTTLE_MS = 15 * 60 * 1000;
 /** Parse X-Mailx-Retry* tracking headers from a raw RFC822 message. */
 function parseRetryInfo(raw) {
     const headerEnd = raw.search(/\r?\n\r?\n/);
@@ -1307,6 +1312,77 @@ export class ImapManager extends EventEmitter {
      *  the first caller finishes — important for outbox flushes that
      *  expect their syncFolder for `Sent` to actually run. */
     syncFolderLocks = new Map();
+    /** Per-folder timestamp of the last set-difference reconcile. Drives the
+     *  throttle for the QRESYNC-path gap backfill (see fetchServerOnlyUids /
+     *  syncFolder QRESYNC branch). Empty at boot, so the first sync of each
+     *  folder after a restart always runs a reconcile — which is exactly when
+     *  a daemon-was-down gap needs to be healed. */
+    lastReconcileMs = new Map();
+    /**
+     * Set-difference backfill. Fetch every server UID (within the history
+     * window) that we don't already have locally and aren't fetching this
+     * cycle. This is the ONLY mechanism that recovers a message which is on
+     * the server but missing locally — the high-water-mark fetch
+     * (fetchMessagesSinceUid) cannot, because it only looks above the local
+     * highest UID. Gaps it heals: a message that arrived while the daemon was
+     * down (so it sits below the watermark once the next poll advances it), or
+     * one dropped mid-stream by a response desync. Returns the recovered
+     * messages (caller stores them) plus the server UID list it fetched, so the
+     * caller's deletion-reconcile can reuse it instead of paying for a second
+     * UID SEARCH round-trip.
+     */
+    async fetchServerOnlyUids(client, accountId, folderId, folderPath, statusMessageCount, excludeUids) {
+        const existingUids = this.db.getUidsForFolder(accountId, folderId);
+        // Small folders (Drafts, Sent, …) get a FULL UID fetch so server-side
+        // deletions of OLDER messages are reflected; large folders (134k INBOX)
+        // get a date-bounded query so we don't enumerate the whole mailbox.
+        const SMALL_FOLDER_THRESHOLD = 500;
+        const isSmallFolder = statusMessageCount !== null && statusMessageCount <= SMALL_FOLDER_THRESHOLD;
+        const historyDays = getHistoryDays(accountId);
+        const SINCE_DAYS = historyDays > 0 ? historyDays : 1825;
+        const sinceDate = new Date(Date.now() - SINCE_DAYS * 86400000);
+        let serverUids;
+        let dateBounded;
+        const __t0 = Date.now();
+        if (isSmallFolder) {
+            console.log(`  [sync-uids] ${accountId}/${folderPath}: small folder (server=${statusMessageCount}) — getUids ALL`);
+            serverUids = await client.getUids(folderPath);
+            dateBounded = false;
+        }
+        else {
+            console.log(`  [sync-uids] ${accountId}/${folderPath}: getUidsSince ${sinceDate.toISOString().slice(0, 10)}...`);
+            serverUids = typeof client.getUidsSince === "function"
+                ? await client.getUidsSince(folderPath, sinceDate)
+                : await client.getUids(folderPath);
+            dateBounded = true;
+        }
+        console.log(`  [sync-uids] ${accountId}/${folderPath}: ${serverUids.length} UIDs in ${Date.now() - __t0}ms`);
+        const existingSet = new Set(existingUids);
+        const missingUids = serverUids.filter((uid) => !existingSet.has(uid) && !excludeUids.has(uid));
+        const recovered = [];
+        if (missingUids.length === 0)
+            return { recovered, serverUids, dateBounded };
+        // Cap so a misbehaving response / brand-new account doesn't pull tens
+        // of thousands at once; the rest resumes next cycle. Higher UID ≈ more
+        // recent under stable UIDVALIDITY, so prefer newest when capping.
+        const BACKFILL_CHUNK_SIZE = 100;
+        const RECONCILE_FETCH_CAP = 5000;
+        let toFetch = missingUids;
+        if (missingUids.length > RECONCILE_FETCH_CAP) {
+            console.log(`  ${folderPath}: ${missingUids.length} server-only UIDs — capped at ${RECONCILE_FETCH_CAP}; will resume next cycle`);
+            toFetch = missingUids.slice().sort((a, b) => b - a).slice(0, RECONCILE_FETCH_CAP);
+        }
+        else {
+            console.log(`  ${folderPath}: ${missingUids.length} server-only UIDs — fetching`);
+        }
+        for (let i = 0; i < toFetch.length; i += BACKFILL_CHUNK_SIZE) {
+            const chunk = toFetch.slice(i, i + BACKFILL_CHUNK_SIZE);
+            const got = await client.fetchMessages(folderPath, chunk.join(","), { source: false });
+            recovered.push(...got);
+            console.log(`  ${folderPath}: backfill ${recovered.length}/${toFetch.length}`);
+        }
+        return { recovered, serverUids, dateBounded };
+    }
     async syncFolder(accountId, folderId, client) {
         const lockKey = `${accountId}:${folderId}`;
         const inflight = this.syncFolderLocks.get(lockKey);
@@ -1501,6 +1577,35 @@ export class ImapManager extends EventEmitter {
                         await this.storeMessages(accountId, folderId, folder, newOnes, highestUid);
                         console.log(`  [qr-phase] ${folder.path}: storeMessages(${newOnes.length}) in ${Date.now() - __t2}ms`);
                     }
+                    // SET-DIFF GAP RECOVERY (bounded + throttled). The
+                    // fetchMessagesSinceUid above is a high-water-mark fetch —
+                    // it CANNOT recover a server UID that's missing below our
+                    // highest (a message that arrived while the daemon was down,
+                    // now under the watermark the next poll advances) or one
+                    // dropped mid-stream by a response desync. QRESYNC otherwise
+                    // returns here, making such gaps permanent (Bob 2026-06-20:
+                    // "messages before 7am are missing" — daemon crashed
+                    // overnight; the catch-up since-fetch lost the low UIDs of
+                    // the range and QRESYNC never reconciled). Run a full
+                    // set-diff on the first sync after boot, then at most once
+                    // per RECONCILE_THROTTLE_MS so the fast path stays fast.
+                    let backfilledCount = 0;
+                    const lastRecon = this.lastReconcileMs.get(folderId) ?? 0;
+                    if (Date.now() - lastRecon > RECONCILE_THROTTLE_MS) {
+                        this.lastReconcileMs.set(folderId, Date.now());
+                        try {
+                            const sd = await this.fetchServerOnlyUids(client, accountId, folderId, folder.path, statusMessageCount, new Set(newOnes.map((m) => m.uid)));
+                            if (sd.recovered.length > 0) {
+                                backfilledCount = await this.storeMessages(accountId, folderId, folder, sd.recovered, highestUid);
+                                console.log(`  [qresync] ${accountId}/${folder.path}: set-diff backfilled ${backfilledCount} server-only UID(s)`);
+                                this.db.recalcFolderCounts(folderId);
+                                this.emit("folderCountsChanged", accountId, {});
+                            }
+                        }
+                        catch (e) {
+                            console.error(`  [qresync] ${accountId}/${folder.path}: set-diff backfill failed: ${e?.message || e}`);
+                        }
+                    }
                     // Persist new watermark — next resync starts here.
                     if (qr.newHighestModSeq !== undefined) {
                         this.db.updateFolderSync(folderId, qr.exists ? prevUidValidity : prevUidValidity, String(qr.newHighestModSeq));
@@ -1515,7 +1620,7 @@ export class ImapManager extends EventEmitter {
                     this.sweepPhantomRows(accountId, folderId, folder.path, serverUidNext);
                     this.emit("syncProgress", accountId, `sync:${folder.path}`, 100);
                     console.log(`  [qresync] ${accountId}/${folder.path}: done in ${Date.now() - __sfStart}ms (path: QRESYNC)`);
-                    return newOnes.length;
+                    return newOnes.length + backfilledCount;
                 }
             }
             catch (qrErr) {
@@ -1571,96 +1676,22 @@ export class ImapManager extends EventEmitter {
             // brand-new account doesn't try to pull tens of thousands of
             // historical messages in one cycle. Hit the cap and the next
             // sync picks up the rest.
-            const existingUids = this.db.getUidsForFolder(accountId, folderId);
             try {
-                // Date-bound the server UID query when we have a history
-                // window. UID SEARCH ALL on a 134k-message INBOX returns
-                // 134k integers and triggers a heavyweight full-folder
-                // scan on the server; UID SEARCH SINCE <date> returns only
-                // UIDs in the window we actually care about. For
-                // historyDays=0 ("keep everything") we still bound to the
-                // last 5 years on incremental syncs — enough that we never
-                // miss an in-flight message, without enumerating decades of
-                // archive every cycle. First sync gets all UIDs (no anchor
-                // yet so we have to compare against the empty local set
-                // anyway).
-                // Small folders (Drafts, Sent, Outbox, Trash, …) get a FULL
-                // UID fetch instead of date-bounded. Without this, server-side
-                // deletions of OLDER messages (e.g., user emptied Trash from
-                // Thunderbird) are never reflected — date-bound filters them
-                // out of the comparison entirely. Bob 2026-05-12: "I deleted
-                // my drafts and Thunderbird shows that but rmfmail is not
-                // acknowledging the deletions." Threshold tuned conservatively
-                // — UID SEARCH ALL on a 500-message folder is sub-second; the
-                // pathology being avoided is the same call on 130k+ INBOX.
-                const SMALL_FOLDER_THRESHOLD = 500;
-                const isSmallFolder = statusMessageCount !== null && statusMessageCount <= SMALL_FOLDER_THRESHOLD;
-                const SINCE_DAYS = effectiveDays > 0 ? effectiveDays : 1825;
-                const sinceDate = new Date(Date.now() - SINCE_DAYS * 86400000);
-                let allServerUids;
-                const __uidsT0 = Date.now();
-                if (isSmallFolder) {
-                    console.log(`  [sync-uids] ${accountId}/${folder.path}: small folder (server=${statusMessageCount}) — getUids ALL`);
-                    allServerUids = await client.getUids(folder.path);
-                    serverUidsAreDateBounded = false;
-                }
-                else {
-                    console.log(`  [sync-uids] ${accountId}/${folder.path}: getUidsSince ${sinceDate.toISOString().slice(0, 10)}...`);
-                    allServerUids = typeof client.getUidsSince === "function"
-                        ? await client.getUidsSince(folder.path, sinceDate)
-                        : await client.getUids(folder.path);
-                    serverUidsAreDateBounded = true;
-                }
-                console.log(`  [sync-uids] ${accountId}/${folder.path}: ${allServerUids.length} UIDs in ${Date.now() - __uidsT0}ms`);
-                // Stash for the deletion-reconciliation block below — we
-                // already have the server UID list, no point hitting the
-                // server a second time.
-                serverUidsCached = allServerUids;
-                const existingSet = new Set(existingUids);
-                const newSet = new Set(messages.map(m => m.uid));
-                const missingUids = allServerUids.filter((uid) => !existingSet.has(uid) && !newSet.has(uid));
-                // Backfill chunk size. Use the passed-in `client` directly
-                // (NOT a nested withConnection) — syncFolder is now wrapped
-                // in withConnection at the call site, so the slow lane is
-                // already locked for our duration. A nested withConnection
-                // would deadlock waiting for the slot we hold.
-                const BACKFILL_CHUNK_SIZE = 100;
-                if (missingUids.length > 0 && missingUids.length <= 5000) {
-                    let minU = existingUids[0] ?? 0;
-                    for (let i = 1; i < existingUids.length; i++)
-                        if (existingUids[i] < minU)
-                            minU = existingUids[i];
-                    console.log(`  ${folder.path}: ${missingUids.length} server-only UIDs (local lowest=${minU}, highest=${highestUid}) — fetching`);
-                    let recoveredTotal = 0;
-                    for (let i = 0; i < missingUids.length; i += BACKFILL_CHUNK_SIZE) {
-                        const chunk = missingUids.slice(i, i + BACKFILL_CHUNK_SIZE);
-                        const range = chunk.join(",");
-                        const recovered = await client.fetchMessages(folder.path, range, { source: false });
-                        messages.push(...recovered);
-                        recoveredTotal += recovered.length;
-                        console.log(`  ${folder.path}: fetch ${recoveredTotal}/${missingUids.length}`);
-                    }
-                }
-                else if (missingUids.length > 5000) {
-                    console.log(`  ${folder.path}: ${missingUids.length} server-only UIDs — capped; will resume next cycle`);
-                    // Vanilla IMAP under stable UIDVALIDITY: higher UID =
-                    // later assignment ≈ more recent message (Dovecot/
-                    // Cyrus). Gmail-API path is separate (no temporal
-                    // meaning to its hash-UIDs).
-                    const cappedSlice = missingUids.sort((a, b) => b - a).slice(0, 5000);
-                    let recoveredTotal = 0;
-                    for (let i = 0; i < cappedSlice.length; i += BACKFILL_CHUNK_SIZE) {
-                        const chunk = cappedSlice.slice(i, i + BACKFILL_CHUNK_SIZE);
-                        const recovered = await client.fetchMessages(folder.path, chunk.join(","), { source: false });
-                        messages.push(...recovered);
-                        recoveredTotal += recovered.length;
-                        console.log(`  ${folder.path}: fetch ${recoveredTotal}/5000 (capped)`);
-                    }
-                }
+                // Shared with the QRESYNC fast path (fetchServerOnlyUids).
+                // Exclude the UIDs we just fetched via fetchMessagesSinceUid —
+                // they're in `messages` but not yet in the DB, so they'd
+                // otherwise look "missing" and get re-fetched.
+                const sd = await this.fetchServerOnlyUids(client, accountId, folderId, folder.path, statusMessageCount, new Set(messages.map(m => m.uid)));
+                messages.push(...sd.recovered);
+                // Stash the server UID list for the deletion-reconciliation
+                // block below — no point hitting the server a second time.
+                serverUidsCached = sd.serverUids;
+                serverUidsAreDateBounded = sd.dateBounded;
             }
             catch (e) {
                 console.error(`  ${folder.path}: reconciliation failed: ${e.message}`);
             }
+            this.lastReconcileMs.set(folderId, Date.now());
             // Date-based backfill via SEARCH SINCE was here. Removed —
             // set-diff above already covers the same case (any server UID
             // not in our local set gets fetched, regardless of whether it's
@@ -3017,7 +3048,7 @@ export class ImapManager extends EventEmitter {
      *  Server fetch goes through the unified ops queue on the fast lane —
      *  the user clicked, they're waiting, this jumps ahead of any background
      *  prefetch sitting in the slow lane. */
-    async fetchMessageBody(accountId, folderId, uid) {
+    async fetchMessageBody(accountId, folderId, uid, force = false) {
         // Belt-and-braces against `UID FETCH 0`. The IMAP server rejects it as
         // BAD "Invalid uidset" and the connection slot is consumed for the
         // round-trip. The enqueue path now guards too — this catches direct
@@ -3042,7 +3073,17 @@ export class ImapManager extends EventEmitter {
         // delayed until the client timed out. The 5min→12h backoff (shared with
         // prefetch via isPrefetchEmpty) stops the tight re-fetch loop; the body
         // simply shows as unavailable until the backoff lapses and one retry runs.
-        if (this.isPrefetchEmpty(accountId, folderId, uid))
+        // EXCEPT an interactive click (`force`): the user is staring at a blank
+        // preview demanding THIS body NOW. Honoring a *background* prefetch
+        // failure's backoff for an explicit user action is a local-first
+        // violation — it left a real INBOX message (bobma uid 4967554,
+        // 2026-06-19) showing an eternal "getting body" spinner for up to 12 h
+        // because one earlier prefetch chunk returned 0 bodies (an iflow parser
+        // miss, not a real expunge — siblings recovered on retry). A forced
+        // single-UID fetch re-attempts immediately; if it genuinely fails it
+        // re-arms the backoff below and the reconciler emits bodyFetchError so
+        // the viewer shows a banner instead of spinning forever.
+        if (!force && this.isPrefetchEmpty(accountId, folderId, uid))
             return null;
         const folder = this.db.getFolders(accountId).find(f => f.id === folderId);
         if (!folder)
@@ -3067,6 +3108,9 @@ export class ImapManager extends EventEmitter {
                 return null;
             const bodyPath = await this.bodyStore.putMessage(accountId, folderId, uid, raw);
             this.db.updateBodyPath(accountId, folderId, uid, bodyPath);
+            // A forced fetch that beat a stale backoff just healed the row;
+            // drop the prefetch-empty record so it doesn't suppress the next tick.
+            this.clearPrefetchEmpty(accountId, folderId, uid);
             this.emit("bodyCached", accountId, uid);
             return raw;
         }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bobfrankston/mailx-imap",
-  "version": "0.1.100",
+  "version": "0.1.102",
   "type": "module",
   "main": "index.js",
   "types": "index.d.ts",