@bobfrankston/rmfmail 1.1.99 → 1.1.102

package/packages/mailx-imap/index.js

@@ -2717,6 +2717,36 @@ export class ImapManager extends EventEmitter {
     clearFolderErrors(accountId, folderPath) {
         this.folderErrorCooldown.delete(`${accountId}:${folderPath}`);
     }
+    /** UIDs that returned 0 bodies on a successful batch — either stale DB rows
+     *  whose UIDVALIDITY has rotated, or a transient iflow parser miss that
+     *  dropped the body literal mid-stream. The "NOT pruning" safety guard in
+     *  `_prefetchBodies` refuses to delete them so a transient miss can't
+     *  trigger data loss; the trade-off is they sit in `getMessagesWithoutBody`
+     *  and would otherwise dominate the size-asc query, starving live folders.
+     *
+     *  Backoff schedule on repeated failure: 5 min → 30 min → 2 h → 12 h.
+     *  Without a TTL the very first transient miss permanently blocklists the
+     *  UID for the session and the row never shows the cached-body dot
+     *  (the symptom Bob reported 2026-05-19: document/large messages with
+     *  no teal mark). With TTL, transient misses self-heal in 5 minutes;
+     *  genuine zombies back off to once-per-12-hour retries — bounded load.
+     *  Key shape `${accountId}:${folderId}:${uid}`. */
+    prefetchFailures = new Map();
+    markPrefetchEmpty(accountId, folderId, uid) {
+        const key = `${accountId}:${folderId}:${uid}`;
+        const prev = this.prefetchFailures.get(key);
+        this.prefetchFailures.set(key, { count: (prev?.count ?? 0) + 1, lastTried: Date.now() });
+    }
+    isPrefetchEmpty(accountId, folderId, uid) {
+        const f = this.prefetchFailures.get(`${accountId}:${folderId}:${uid}`);
+        if (!f)
+            return false;
+        const backoffMs = f.count <= 1 ? 5 * 60_000
+            : f.count <= 2 ? 30 * 60_000
+                : f.count <= 3 ? 2 * 3600_000
+                    : 12 * 3600_000;
+        return Date.now() - f.lastTried < backoffMs;
+    }
     /** Background body-cache backfill. Public so the Reconciler can schedule
      *  the periodic tick under its priority/back-pressure rules; existing
      *  in-method post-sync nudges (sync, fetchSince, fetchOne) call this
@@ -2752,8 +2782,21 @@ export class ImapManager extends EventEmitter {
         // governing unit). IMAP uses the batched `fetchBodiesBatch` path via iflow-direct —
         // one SELECT + one UID FETCH per folder per tick instead of N round trips.
         let announced = false;
-        while (true) {
-            const missing = this.db.getMessagesWithoutBody(accountId, BATCH_SIZE);
+        // Cap iterations so a fully-blocked queue can't loop forever. With
+        // BATCH_SIZE=100 and the blocklist filter below, each iteration
+        // either advances or terminates; this bound is a belt-and-braces
+        // safety net for unexpected DB states.
+        let iterationsRemaining = 50;
+        while (iterationsRemaining-- > 0) {
+            // Pull more than BATCH_SIZE from the DB so the empty-uid filter
+            // can drop blocklisted rows and still hand a useful batch to the
+            // fetcher. Without this, a queue dominated by stale UIDs returns
+            // BATCH_SIZE rows that all get filtered to zero, and the loop
+            // terminates without ever trying live messages further down the
+            // size-asc list.
+            const raw = this.db.getMessagesWithoutBody(accountId, BATCH_SIZE * 4);
+            const missing = raw.filter(m => !this.isPrefetchEmpty(accountId, m.folderId, m.uid))
+                .slice(0, BATCH_SIZE);
             if (missing.length === 0)
                 break;
             if (!announced) {
@@ -2794,7 +2837,8 @@ export class ImapManager extends EventEmitter {
                                     try {
                                         const raw = Buffer.from(source, "utf-8");
                                         const bodyPath = await this.bodyStore.putMessage(accountId, folderId, uid, raw);
-                                        this.db.updateBodyPath(accountId, uid, bodyPath);
+                                        const parsed = await extractPreview(source);
+                                        this.db.updateBodyMeta(accountId, uid, bodyPath, parsed.hasAttachments, parsed.preview);
                                         this.emit("bodyCached", accountId, uid);
                                         counters.totalFetched++;
                                         madeProgress = true;
@@ -2830,20 +2874,22 @@ export class ImapManager extends EventEmitter {
                         // owns deletion via a 30-min grace; defer to it.
                         const someReceived = received.size > 0;
                         if (batchSucceeded && someReceived) {
+                            // Mirror of the IMAP-path fix: never delete on a
+                            // partial batch — Gmail API has its own transient
+                            // miss modes (rate-limit retry losing a message,
+                            // /batch response parse error) that look exactly
+                            // like server-side expunge. Set-diff reconcile in
+                            // syncAccountViaApi owns deletion.
                             for (const uid of uidsInFolder) {
                                 if (received.has(uid))
                                     continue;
-                                try {
-                                    this.unlinkBodyFile(accountId, uid, folderId).catch(() => { });
-                                    this.db.deleteMessage(accountId, uid, "prefetch batch: server didn't return body for queued UID — assumed deleted", "mailx-imap prefetchBodies (Gmail batch)");
-                                    counters.deleted++;
-                                    madeProgress = true;
-                                }
-                                catch { /* ignore */ }
+                                this.markPrefetchEmpty(accountId, folderId, uid);
                             }
                         }
                         else if (batchSucceeded && !someReceived) {
                             console.error(`  [prefetch] ${accountId}/${folder.path}: Gmail batch returned 0/${uidsInFolder.length} bodies — NOT pruning (set-diff reconcile owns deletion). UIDs: ${uidsInFolder.slice(0, 5).join(",")}${uidsInFolder.length > 5 ? "..." : ""}`);
+                            for (const uid of uidsInFolder)
+                                this.markPrefetchEmpty(accountId, folderId, uid);
                         }
                         if (counters.errors >= ERROR_BUDGET)
                             break;
@@ -2923,7 +2969,8 @@ export class ImapManager extends EventEmitter {
                                         try {
                                             const raw = Buffer.from(source, "utf-8");
                                             const bodyPath = await this.bodyStore.putMessage(accountId, folderId, uid, raw);
-                                            this.db.updateBodyPath(accountId, uid, bodyPath);
+                                            const parsed = await extractPreview(source);
+                                            this.db.updateBodyMeta(accountId, uid, bodyPath, parsed.hasAttachments, parsed.preview);
                                             this.emit("bodyCached", accountId, uid);
                                             counters.totalFetched++;
                                             madeProgress = true;
@@ -2960,20 +3007,28 @@ export class ImapManager extends EventEmitter {
                         // authoritative deletion path with a 30-min
                         // grace window; prefetch defers to it.
                         const someReceived = received.size > 0;
-                        if (batchSucceeded && someReceived)
+                        if (batchSucceeded && someReceived) {
+                            // DO NOT DELETE missing UIDs. A partial response is
+                            // an iflow parser miss / mid-stream hiccup MUCH more
+                            // often than a real server expunge — and deleting on
+                            // that signal cost Bob a Bambu Labs verification
+                            // code (audit id 3785, 2026-05-20), plus dozens of
+                            // other valid messages over the day. Set-diff
+                            // reconcile in syncFolder is the authoritative
+                            // deletion path with a 30-min grace window; prefetch
+                            // defers to it. Mark the UIDs as prefetch-empty so
+                            // the TTL backoff (5min → 12h) retries them — that
+                            // path is non-destructive.
                             for (const uid of chunk) {
                                 if (received.has(uid))
                                     continue;
-                                try {
-                                    this.unlinkBodyFile(accountId, uid, folderId).catch(() => { });
-                                    this.db.deleteMessage(accountId, uid, "prefetch batch: server didn't return body for queued UID — assumed deleted", "mailx-imap prefetchBodies (IMAP batch)");
-                                    counters.deleted++;
-                                    madeProgress = true;
-                                }
-                                catch { /* ignore */ }
+                                this.markPrefetchEmpty(accountId, folderId, uid);
                             }
+                        }
                         else if (batchSucceeded && !someReceived) {
                             console.error(`  [prefetch] ${accountId}/${folder.path}: chunk ${chunkStart}-${chunkStart + chunk.length - 1} returned 0/${chunk.length} bodies — NOT pruning (set-diff reconcile owns deletion). UIDs: ${chunk.slice(0, 5).join(",")}${chunk.length > 5 ? "..." : ""}`);
+                            for (const uid of chunk)
+                                this.markPrefetchEmpty(accountId, folderId, uid);
                         }
                     }
                     if (counters.errors >= ERROR_BUDGET)
@@ -3000,6 +3055,49 @@ export class ImapManager extends EventEmitter {
     getBodyStore() {
         return this.bodyStore;
     }
+    /** Re-parse existing .eml files from disk to backfill has_attachments
+     *  + preview for messages whose body landed before the prefetch-reparse
+     *  fix shipped. No IMAP traffic — pure local-disk work. Throttled by the
+     *  reconciler the same way as prefetch (back off when interactive lane
+     *  has clicks pending) so it doesn't fight with the user. */
+    backfillingAccounts = new Set();
+    async backfillBodyMeta(accountId, limit = 100) {
+        if (this.backfillingAccounts.has(accountId))
+            return;
+        if (!this.configs.has(accountId))
+            return;
+        this.backfillingAccounts.add(accountId);
+        try {
+            const rows = this.db.getMessagesNeedingBodyReparse(accountId, limit);
+            if (rows.length === 0)
+                return;
+            let ok = 0, fail = 0;
+            for (const row of rows) {
+                try {
+                    if (!(await this.bodyStore.hasByPath(row.bodyPath))) {
+                        // Body file vanished — stamp so we don't keep retrying.
+                        this.db.updateBodyMetaById(row.id, false, "");
+                        continue;
+                    }
+                    const buf = await this.bodyStore.readByPath(row.bodyPath);
+                    const parsed = await extractPreview(buf.toString("utf-8"));
+                    this.db.updateBodyMetaById(row.id, parsed.hasAttachments, parsed.preview);
+                    ok++;
+                }
+                catch (e) {
+                    fail++;
+                    console.error(`  [backfill] ${accountId}/${row.uid}: ${e?.message || e}`);
+                }
+            }
+            if (ok > 0 || fail > 0) {
+                console.log(`  [backfill] ${accountId}: reparsed ${ok}, failed ${fail}`);
+                this.emit("folderCountsChanged", accountId, {});
+            }
+        }
+        finally {
+            this.backfillingAccounts.delete(accountId);
+        }
+    }
     /** Move message across accounts using iflow's moveMessageToServer */
     async moveMessageCrossAccount(fromAccountId, uid, fromFolderId, toAccountId, toFolderId) {
         const fromFolders = this.db.getFolders(fromAccountId);