@bobfrankston/mailx-imap 0.1.100 → 0.1.103

package/index.d.ts

@@ -271,6 +271,26 @@ export declare class ImapManager extends EventEmitter {
      *  the first caller finishes — important for outbox flushes that
      *  expect their syncFolder for `Sent` to actually run. */
     private syncFolderLocks;
+    /** Per-folder timestamp of the last set-difference reconcile. Drives the
+     *  throttle for the QRESYNC-path gap backfill (see fetchServerOnlyUids /
+     *  syncFolder QRESYNC branch). Empty at boot, so the first sync of each
+     *  folder after a restart always runs a reconcile — which is exactly when
+     *  a daemon-was-down gap needs to be healed. */
+    private lastReconcileMs;
+    /**
+     * Set-difference backfill. Fetch every server UID (within the history
+     * window) that we don't already have locally and aren't fetching this
+     * cycle. This is the ONLY mechanism that recovers a message which is on
+     * the server but missing locally — the high-water-mark fetch
+     * (fetchMessagesSinceUid) cannot, because it only looks above the local
+     * highest UID. Gaps it heals: a message that arrived while the daemon was
+     * down (so it sits below the watermark once the next poll advances it), or
+     * one dropped mid-stream by a response desync. Returns the recovered
+     * messages (caller stores them) plus the server UID list it fetched, so the
+     * caller's deletion-reconcile can reuse it instead of paying for a second
+     * UID SEARCH round-trip.
+     */
+    private fetchServerOnlyUids;
     syncFolder(accountId: string, folderId: number, client?: any): Promise<number>;
     private _syncFolderImpl;
     /** Sync all folders for all accounts */
@@ -379,7 +399,7 @@ export declare class ImapManager extends EventEmitter {
      *  Server fetch goes through the unified ops queue on the fast lane —
      *  the user clicked, they're waiting, this jumps ahead of any background
      *  prefetch sitting in the slow lane. */
-    fetchMessageBody(accountId: string, folderId: number, uid: number): Promise<Buffer | null>;
+    fetchMessageBody(accountId: string, folderId: number, uid: number, force?: boolean): Promise<Buffer | null>;
     /** Fetch message body via Gmail/Outlook API.
      *  Throws `MessageNotFoundError` when the server says the message is gone
      *  (deleted from another device, for example). The caller uses that to

package/index.js

@@ -21,6 +21,11 @@ const SMTP_PORT_IMPLICIT_TLS = 465;
  *  the same file is retried. Gives the server time to settle so a retry after a
  *  lost-ack doesn't arrive while the first copy is still being processed. */
 const OUTBOX_RETRY_DELAY_MS = 60000;
+/** How often the QRESYNC fast path runs a full set-difference reconcile to
+ *  heal gaps the high-water-mark fetch can't see. The first sync of a folder
+ *  after boot always runs one (the map is empty); thereafter it's throttled to
+ *  this interval so the QRESYNC speedup is preserved in steady state. */
+const RECONCILE_THROTTLE_MS = 15 * 60 * 1000;
 /** Parse X-Mailx-Retry* tracking headers from a raw RFC822 message. */
 function parseRetryInfo(raw) {
     const headerEnd = raw.search(/\r?\n\r?\n/);
@@ -1307,6 +1312,94 @@ export class ImapManager extends EventEmitter {
      *  the first caller finishes — important for outbox flushes that
      *  expect their syncFolder for `Sent` to actually run. */
     syncFolderLocks = new Map();
+    /** Per-folder timestamp of the last set-difference reconcile. Drives the
+     *  throttle for the QRESYNC-path gap backfill (see fetchServerOnlyUids /
+     *  syncFolder QRESYNC branch). Empty at boot, so the first sync of each
+     *  folder after a restart always runs a reconcile — which is exactly when
+     *  a daemon-was-down gap needs to be healed. */
+    lastReconcileMs = new Map();
+    /**
+     * Set-difference backfill. Fetch every server UID (within the history
+     * window) that we don't already have locally and aren't fetching this
+     * cycle. This is the ONLY mechanism that recovers a message which is on
+     * the server but missing locally — the high-water-mark fetch
+     * (fetchMessagesSinceUid) cannot, because it only looks above the local
+     * highest UID. Gaps it heals: a message that arrived while the daemon was
+     * down (so it sits below the watermark once the next poll advances it), or
+     * one dropped mid-stream by a response desync. Returns the recovered
+     * messages (caller stores them) plus the server UID list it fetched, so the
+     * caller's deletion-reconcile can reuse it instead of paying for a second
+     * UID SEARCH round-trip.
+     */
+    async fetchServerOnlyUids(client, accountId, folderId, folder, statusMessageCount, excludeUids) {
+        const folderPath = folder.path;
+        const existingUids = this.db.getUidsForFolder(accountId, folderId);
+        // Small folders (Drafts, Sent, …) get a FULL UID fetch so server-side
+        // deletions of OLDER messages are reflected; large folders (134k INBOX)
+        // get a date-bounded query so we don't enumerate the whole mailbox.
+        const SMALL_FOLDER_THRESHOLD = 500;
+        const isSmallFolder = statusMessageCount !== null && statusMessageCount <= SMALL_FOLDER_THRESHOLD;
+        const historyDays = getHistoryDays(accountId);
+        const SINCE_DAYS = historyDays > 0 ? historyDays : 1825;
+        const sinceDate = new Date(Date.now() - SINCE_DAYS * 86400000);
+        let serverUids;
+        let dateBounded;
+        const __t0 = Date.now();
+        if (isSmallFolder) {
+            console.log(`  [sync-uids] ${accountId}/${folderPath}: small folder (server=${statusMessageCount}) — getUids ALL`);
+            serverUids = await client.getUids(folderPath);
+            dateBounded = false;
+        }
+        else {
+            console.log(`  [sync-uids] ${accountId}/${folderPath}: getUidsSince ${sinceDate.toISOString().slice(0, 10)}...`);
+            serverUids = typeof client.getUidsSince === "function"
+                ? await client.getUidsSince(folderPath, sinceDate)
+                : await client.getUids(folderPath);
+            dateBounded = true;
+        }
+        console.log(`  [sync-uids] ${accountId}/${folderPath}: ${serverUids.length} UIDs in ${Date.now() - __t0}ms`);
+        const existingSet = new Set(existingUids);
+        const missingUids = serverUids.filter((uid) => !existingSet.has(uid) && !excludeUids.has(uid));
+        if (missingUids.length === 0)
+            return { recoveredCount: 0, serverUids, dateBounded };
+        // Cap so a misbehaving response / brand-new account doesn't pull tens
+        // of thousands at once; the rest resumes next cycle. Higher UID ≈ more
+        // recent under stable UIDVALIDITY, so prefer newest when capping — the
+        // user cares most about recent mail and the deficit gate keeps calling
+        // us until the backlog is drained.
+        const BACKFILL_CHUNK_SIZE = 100;
+        const RECONCILE_FETCH_CAP = 5000;
+        let toFetch = missingUids;
+        if (missingUids.length > RECONCILE_FETCH_CAP) {
+            console.log(`  ${folderPath}: ${missingUids.length} server-only UIDs — capped at ${RECONCILE_FETCH_CAP}; resumes next cycle`);
+            toFetch = missingUids.slice().sort((a, b) => b - a).slice(0, RECONCILE_FETCH_CAP);
+        }
+        else {
+            console.log(`  ${folderPath}: ${missingUids.length} server-only UIDs — fetching`);
+        }
+        // Store each chunk as it arrives so a mid-stream connection drop (the
+        // "Not connected" / desync class on this server) keeps the progress it
+        // already made instead of losing the whole batch. Stop on the first
+        // chunk error; the deficit gate calls us again next cycle to resume.
+        let recoveredCount = 0;
+        for (let i = 0; i < toFetch.length; i += BACKFILL_CHUNK_SIZE) {
+            const chunk = toFetch.slice(i, i + BACKFILL_CHUNK_SIZE);
+            try {
+                const got = await client.fetchMessages(folderPath, chunk.join(","), { source: false });
+                recoveredCount += await this.storeMessages(accountId, folderId, folder, got, 0);
+            }
+            catch (e) {
+                console.error(`  ${folderPath}: backfill chunk ${i}-${i + chunk.length} failed (${e?.message || e}) — keeping ${recoveredCount} so far, resuming next cycle`);
+                break;
+            }
+            console.log(`  ${folderPath}: backfill ${Math.min(i + BACKFILL_CHUNK_SIZE, toFetch.length)}/${toFetch.length} (stored ${recoveredCount})`);
+        }
+        if (recoveredCount > 0) {
+            this.db.recalcFolderCounts(folderId);
+            this.emit("folderCountsChanged", accountId, {});
+        }
+        return { recoveredCount, serverUids, dateBounded };
+    }
     async syncFolder(accountId, folderId, client) {
         const lockKey = `${accountId}:${folderId}`;
         const inflight = this.syncFolderLocks.get(lockKey);
@@ -1501,6 +1594,46 @@ export class ImapManager extends EventEmitter {
                         await this.storeMessages(accountId, folderId, folder, newOnes, highestUid);
                         console.log(`  [qr-phase] ${folder.path}: storeMessages(${newOnes.length}) in ${Date.now() - __t2}ms`);
                     }
+                    // SET-DIFF GAP RECOVERY (bounded + throttled). The
+                    // fetchMessagesSinceUid above is a high-water-mark fetch —
+                    // it CANNOT recover a server UID that's missing below our
+                    // highest (a message that arrived while the daemon was down,
+                    // now under the watermark the next poll advances) or one
+                    // dropped mid-stream by a response desync. QRESYNC otherwise
+                    // returns here, making such gaps permanent (Bob 2026-06-20:
+                    // "messages before 7am are missing" — daemon crashed
+                    // overnight; the catch-up since-fetch lost the low UIDs of
+                    // the range and QRESYNC never reconciled). Run a full
+                    // set-diff on the first sync after boot, then at most once
+                    // per RECONCILE_THROTTLE_MS so the fast path stays fast.
+                    let backfilledCount = 0;
+                    // Run the set-diff when EITHER the folder has a known
+                    // deficit (local count < server count → messages are
+                    // missing, keep reconciling every cycle until caught up) OR
+                    // the throttle window elapsed (periodic safety sweep for
+                    // gaps that don't move the count, e.g. a desync-dropped
+                    // message replaced by a later arrival). On success advance
+                    // the throttle; on FAILURE leave it so the next cycle
+                    // retries instead of locking out for RECONCILE_THROTTLE_MS
+                    // (Bob 2026-06-20: boot-time "Not connected" failure locked
+                    // INBOX recovery out for 15 min).
+                    const localCount = this.db.getMessageCount(accountId, folderId);
+                    const hasDeficit = statusMessageCount !== null && localCount < statusMessageCount;
+                    const lastRecon = this.lastReconcileMs.get(folderId) ?? 0;
+                    const throttleElapsed = Date.now() - lastRecon > RECONCILE_THROTTLE_MS;
+                    if (hasDeficit || throttleElapsed) {
+                        try {
+                            const sd = await this.fetchServerOnlyUids(client, accountId, folderId, folder, statusMessageCount, new Set(newOnes.map((m) => m.uid)));
+                            backfilledCount = sd.recoveredCount;
+                            if (backfilledCount > 0) {
+                                console.log(`  [qresync] ${accountId}/${folder.path}: set-diff backfilled ${backfilledCount} server-only UID(s)`);
+                            }
+                            this.lastReconcileMs.set(folderId, Date.now());
+                        }
+                        catch (e) {
+                            console.error(`  [qresync] ${accountId}/${folder.path}: set-diff backfill failed: ${e?.message || e}`);
+                        }
+                    }
                     // Persist new watermark — next resync starts here.
                     if (qr.newHighestModSeq !== undefined) {
                         this.db.updateFolderSync(folderId, qr.exists ? prevUidValidity : prevUidValidity, String(qr.newHighestModSeq));
@@ -1515,7 +1648,7 @@ export class ImapManager extends EventEmitter {
                     this.sweepPhantomRows(accountId, folderId, folder.path, serverUidNext);
                     this.emit("syncProgress", accountId, `sync:${folder.path}`, 100);
                     console.log(`  [qresync] ${accountId}/${folder.path}: done in ${Date.now() - __sfStart}ms (path: QRESYNC)`);
-                    return newOnes.length;
+                    return newOnes.length + backfilledCount;
                 }
             }
             catch (qrErr) {
@@ -1571,92 +1704,19 @@ export class ImapManager extends EventEmitter {
             // brand-new account doesn't try to pull tens of thousands of
             // historical messages in one cycle. Hit the cap and the next
             // sync picks up the rest.
-            const existingUids = this.db.getUidsForFolder(accountId, folderId);
             try {
-                // Date-bound the server UID query when we have a history
-                // window. UID SEARCH ALL on a 134k-message INBOX returns
-                // 134k integers and triggers a heavyweight full-folder
-                // scan on the server; UID SEARCH SINCE <date> returns only
-                // UIDs in the window we actually care about. For
-                // historyDays=0 ("keep everything") we still bound to the
-                // last 5 years on incremental syncs — enough that we never
-                // miss an in-flight message, without enumerating decades of
-                // archive every cycle. First sync gets all UIDs (no anchor
-                // yet so we have to compare against the empty local set
-                // anyway).
-                // Small folders (Drafts, Sent, Outbox, Trash, …) get a FULL
-                // UID fetch instead of date-bounded. Without this, server-side
-                // deletions of OLDER messages (e.g., user emptied Trash from
-                // Thunderbird) are never reflected — date-bound filters them
-                // out of the comparison entirely. Bob 2026-05-12: "I deleted
-                // my drafts and Thunderbird shows that but rmfmail is not
-                // acknowledging the deletions." Threshold tuned conservatively
-                // — UID SEARCH ALL on a 500-message folder is sub-second; the
-                // pathology being avoided is the same call on 130k+ INBOX.
-                const SMALL_FOLDER_THRESHOLD = 500;
-                const isSmallFolder = statusMessageCount !== null && statusMessageCount <= SMALL_FOLDER_THRESHOLD;
-                const SINCE_DAYS = effectiveDays > 0 ? effectiveDays : 1825;
-                const sinceDate = new Date(Date.now() - SINCE_DAYS * 86400000);
-                let allServerUids;
-                const __uidsT0 = Date.now();
-                if (isSmallFolder) {
-                    console.log(`  [sync-uids] ${accountId}/${folder.path}: small folder (server=${statusMessageCount}) — getUids ALL`);
-                    allServerUids = await client.getUids(folder.path);
-                    serverUidsAreDateBounded = false;
-                }
-                else {
-                    console.log(`  [sync-uids] ${accountId}/${folder.path}: getUidsSince ${sinceDate.toISOString().slice(0, 10)}...`);
-                    allServerUids = typeof client.getUidsSince === "function"
-                        ? await client.getUidsSince(folder.path, sinceDate)
-                        : await client.getUids(folder.path);
-                    serverUidsAreDateBounded = true;
-                }
-                console.log(`  [sync-uids] ${accountId}/${folder.path}: ${allServerUids.length} UIDs in ${Date.now() - __uidsT0}ms`);
-                // Stash for the deletion-reconciliation block below — we
-                // already have the server UID list, no point hitting the
-                // server a second time.
-                serverUidsCached = allServerUids;
-                const existingSet = new Set(existingUids);
-                const newSet = new Set(messages.map(m => m.uid));
-                const missingUids = allServerUids.filter((uid) => !existingSet.has(uid) && !newSet.has(uid));
-                // Backfill chunk size. Use the passed-in `client` directly
-                // (NOT a nested withConnection) — syncFolder is now wrapped
-                // in withConnection at the call site, so the slow lane is
-                // already locked for our duration. A nested withConnection
-                // would deadlock waiting for the slot we hold.
-                const BACKFILL_CHUNK_SIZE = 100;
-                if (missingUids.length > 0 && missingUids.length <= 5000) {
-                    let minU = existingUids[0] ?? 0;
-                    for (let i = 1; i < existingUids.length; i++)
-                        if (existingUids[i] < minU)
-                            minU = existingUids[i];
-                    console.log(`  ${folder.path}: ${missingUids.length} server-only UIDs (local lowest=${minU}, highest=${highestUid}) — fetching`);
-                    let recoveredTotal = 0;
-                    for (let i = 0; i < missingUids.length; i += BACKFILL_CHUNK_SIZE) {
-                        const chunk = missingUids.slice(i, i + BACKFILL_CHUNK_SIZE);
-                        const range = chunk.join(",");
-                        const recovered = await client.fetchMessages(folder.path, range, { source: false });
-                        messages.push(...recovered);
-                        recoveredTotal += recovered.length;
-                        console.log(`  ${folder.path}: fetch ${recoveredTotal}/${missingUids.length}`);
-                    }
-                }
-                else if (missingUids.length > 5000) {
-                    console.log(`  ${folder.path}: ${missingUids.length} server-only UIDs — capped; will resume next cycle`);
-                    // Vanilla IMAP under stable UIDVALIDITY: higher UID =
-                    // later assignment ≈ more recent message (Dovecot/
-                    // Cyrus). Gmail-API path is separate (no temporal
-                    // meaning to its hash-UIDs).
-                    const cappedSlice = missingUids.sort((a, b) => b - a).slice(0, 5000);
-                    let recoveredTotal = 0;
-                    for (let i = 0; i < cappedSlice.length; i += BACKFILL_CHUNK_SIZE) {
-                        const chunk = cappedSlice.slice(i, i + BACKFILL_CHUNK_SIZE);
-                        const recovered = await client.fetchMessages(folder.path, chunk.join(","), { source: false });
-                        messages.push(...recovered);
-                        recoveredTotal += recovered.length;
-                        console.log(`  ${folder.path}: fetch ${recoveredTotal}/5000 (capped)`);
-                    }
-                }
+                // Shared with the QRESYNC fast path (fetchServerOnlyUids). It
+                // stores recovered messages itself (chunk-at-a-time, so a
+                // dropped connection keeps partial progress). Exclude the UIDs
+                // we just fetched via fetchMessagesSinceUid — they're in
+                // `messages`, about to be stored by the batch loop below, so
+                // they'd otherwise look "missing" and get re-fetched.
+                const sd = await this.fetchServerOnlyUids(client, accountId, folderId, folder, statusMessageCount, new Set(messages.map(m => m.uid)));
+                // Stash the server UID list for the deletion-reconciliation
+                // block below — no point hitting the server a second time.
+                serverUidsCached = sd.serverUids;
+                serverUidsAreDateBounded = sd.dateBounded;
+                this.lastReconcileMs.set(folderId, Date.now());
             }
             catch (e) {
                 console.error(`  ${folder.path}: reconciliation failed: ${e.message}`);
@@ -3017,7 +3077,7 @@ export class ImapManager extends EventEmitter {
      *  Server fetch goes through the unified ops queue on the fast lane —
      *  the user clicked, they're waiting, this jumps ahead of any background
      *  prefetch sitting in the slow lane. */
-    async fetchMessageBody(accountId, folderId, uid) {
+    async fetchMessageBody(accountId, folderId, uid, force = false) {
         // Belt-and-braces against `UID FETCH 0`. The IMAP server rejects it as
         // BAD "Invalid uidset" and the connection slot is consumed for the
         // round-trip. The enqueue path now guards too — this catches direct
@@ -3042,7 +3102,17 @@ export class ImapManager extends EventEmitter {
         // delayed until the client timed out. The 5min→12h backoff (shared with
         // prefetch via isPrefetchEmpty) stops the tight re-fetch loop; the body
         // simply shows as unavailable until the backoff lapses and one retry runs.
-        if (this.isPrefetchEmpty(accountId, folderId, uid))
+        // EXCEPT an interactive click (`force`): the user is staring at a blank
+        // preview demanding THIS body NOW. Honoring a *background* prefetch
+        // failure's backoff for an explicit user action is a local-first
+        // violation — it left a real INBOX message (bobma uid 4967554,
+        // 2026-06-19) showing an eternal "getting body" spinner for up to 12 h
+        // because one earlier prefetch chunk returned 0 bodies (an iflow parser
+        // miss, not a real expunge — siblings recovered on retry). A forced
+        // single-UID fetch re-attempts immediately; if it genuinely fails it
+        // re-arms the backoff below and the reconciler emits bodyFetchError so
+        // the viewer shows a banner instead of spinning forever.
+        if (!force && this.isPrefetchEmpty(accountId, folderId, uid))
             return null;
         const folder = this.db.getFolders(accountId).find(f => f.id === folderId);
         if (!folder)
@@ -3067,6 +3137,9 @@ export class ImapManager extends EventEmitter {
                 return null;
             const bodyPath = await this.bodyStore.putMessage(accountId, folderId, uid, raw);
             this.db.updateBodyPath(accountId, folderId, uid, bodyPath);
+            // A forced fetch that beat a stale backoff just healed the row;
+            // drop the prefetch-empty record so it doesn't suppress the next tick.
+            this.clearPrefetchEmpty(accountId, folderId, uid);
             this.emit("bodyCached", accountId, uid);
             return raw;
         }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bobfrankston/mailx-imap",
-  "version": "0.1.100",
+  "version": "0.1.103",
   "type": "module",
   "main": "index.js",
   "types": "index.d.ts",
@@ -10,7 +10,7 @@
   "license": "ISC",
   "dependencies": {
     "@bobfrankston/mailx-types": "^0.1.19",
-    "@bobfrankston/mailx-settings": "^0.1.28",
+    "@bobfrankston/mailx-settings": "^0.1.29",
     "@bobfrankston/mailx-store": "^0.1.54",
     "@bobfrankston/iflow-direct": "^0.1.55",
     "@bobfrankston/tcp-transport": "^0.1.7",
@@ -38,7 +38,7 @@
   ".transformedSnapshot": {
     "dependencies": {
       "@bobfrankston/mailx-types": "^0.1.19",
-      "@bobfrankston/mailx-settings": "^0.1.28",
+      "@bobfrankston/mailx-settings": "^0.1.29",
       "@bobfrankston/mailx-store": "^0.1.54",
       "@bobfrankston/iflow-direct": "^0.1.55",
       "@bobfrankston/tcp-transport": "^0.1.7",