@bobfrankston/mailx-imap 0.1.51 → 0.1.53

package/index.d.ts

@@ -380,6 +380,23 @@ export declare class ImapManager extends EventEmitter {
     private shouldSkipFolder;
     private recordFolderError;
     private clearFolderErrors;
+    /** UIDs that returned 0 bodies on a successful batch — either stale DB rows
+     *  whose UIDVALIDITY has rotated, or a transient iflow parser miss that
+     *  dropped the body literal mid-stream. The "NOT pruning" safety guard in
+     *  `_prefetchBodies` refuses to delete them so a transient miss can't
+     *  trigger data loss; the trade-off is they sit in `getMessagesWithoutBody`
+     *  and would otherwise dominate the size-asc query, starving live folders.
+     *
+     *  Backoff schedule on repeated failure: 5 min → 30 min → 2 h → 12 h.
+     *  Without a TTL the very first transient miss permanently blocklists the
+     *  UID for the session and the row never shows the cached-body dot
+     *  (the symptom Bob reported 2026-05-19: document/large messages with
+     *  no teal mark). With TTL, transient misses self-heal in 5 minutes;
+     *  genuine zombies back off to once-per-12-hour retries — bounded load.
+     *  Key shape `${accountId}:${folderId}:${uid}`. */
+    private prefetchFailures;
+    private markPrefetchEmpty;
+    private isPrefetchEmpty;
     /** Background body-cache backfill. Public so the Reconciler can schedule
      *  the periodic tick under its priority/back-pressure rules; existing
      *  in-method post-sync nudges (sync, fetchSince, fetchOne) call this
@@ -388,6 +405,13 @@ export declare class ImapManager extends EventEmitter {
     private _prefetchBodies;
     /** Get the body store for direct access */
     getBodyStore(): FileMessageStore;
+    /** Re-parse existing .eml files from disk to backfill has_attachments
+     *  + preview for messages whose body landed before the prefetch-reparse
+     *  fix shipped. No IMAP traffic — pure local-disk work. Throttled by the
+     *  reconciler the same way as prefetch (back off when interactive lane
+     *  has clicks pending) so it doesn't fight with the user. */
+    private backfillingAccounts;
+    backfillBodyMeta(accountId: string, limit?: number): Promise<void>;
     /** Move message across accounts using iflow's moveMessageToServer */
     moveMessageCrossAccount(fromAccountId: string, uid: number, fromFolderId: number, toAccountId: string, toFolderId: number): Promise<void>;
     /** Process pending sync actions for an account */

package/index.js

@@ -951,11 +951,13 @@ export class ImapManager extends EventEmitter {
                 let bodyPath = "";
                 let preview = "";
                 let hasAttachments = false;
+                let ftsBody = "";
                 if (source) {
                     bodyPath = await this.bodyStore.putMessage(accountId, folderId, msg.uid, Buffer.from(source, "utf-8"));
                     const parsed = await extractPreview(source);
                     preview = parsed.preview;
                     hasAttachments = parsed.hasAttachments;
+                    ftsBody = parsed.bodyText || "";
                 }
                 const flags = [];
                 if (msg.seen)
@@ -966,7 +968,7 @@ export class ImapManager extends EventEmitter {
                     flags.push("\\Answered");
                 if (msg.draft)
                     flags.push("\\Draft");
-                this.db.upsertMessage({
+                const ftsRowId = this.db.upsertMessage({
                     accountId, folderId, uid: msg.uid,
                     messageId: msg.messageId || "",
                     inReplyTo: msg.inReplyTo || "",
@@ -978,6 +980,12 @@ export class ImapManager extends EventEmitter {
                     cc: toEmailAddresses(msg.cc || []),
                     flags, size: msg.size || 0, hasAttachments, preview, bodyPath
                 });
+                // Index the full body for search. extractPreview already
+                // parsed `source`, so the body text is free here — and
+                // indexing it at sync time means a word buried in the body
+                // is searchable without the user first opening the message.
+                if (ftsRowId && ftsBody)
+                    this.db.updateFtsBody(ftsRowId, ftsBody);
                 stored++;
                 batchCount++;
                 if (batchCount >= BATCH_SIZE) {
@@ -1476,7 +1484,7 @@ export class ImapManager extends EventEmitter {
                     if (msg.draft)
                         flags.push("\\Draft");
                     // Store metadata
-                    this.db.upsertMessage({
+                    const ftsRowId = this.db.upsertMessage({
                         accountId,
                         folderId,
                         uid: msg.uid,
@@ -1494,6 +1502,11 @@ export class ImapManager extends EventEmitter {
                         preview: parsed.preview,
                         bodyPath
                     });
+                    // Full body into the FTS index — the parse above already
+                    // produced the text, so search covers body content even
+                    // for messages the user hasn't opened.
+                    if (ftsRowId && parsed.bodyText)
+                        this.db.updateFtsBody(ftsRowId, parsed.bodyText);
                     newCount++;
                 }
                 this.db.commitTransaction();
@@ -2704,6 +2717,36 @@ export class ImapManager extends EventEmitter {
     clearFolderErrors(accountId, folderPath) {
         this.folderErrorCooldown.delete(`${accountId}:${folderPath}`);
     }
+    /** UIDs that returned 0 bodies on a successful batch — either stale DB rows
+     *  whose UIDVALIDITY has rotated, or a transient iflow parser miss that
+     *  dropped the body literal mid-stream. The "NOT pruning" safety guard in
+     *  `_prefetchBodies` refuses to delete them so a transient miss can't
+     *  trigger data loss; the trade-off is they sit in `getMessagesWithoutBody`
+     *  and would otherwise dominate the size-asc query, starving live folders.
+     *
+     *  Backoff schedule on repeated failure: 5 min → 30 min → 2 h → 12 h.
+     *  Without a TTL the very first transient miss permanently blocklists the
+     *  UID for the session and the row never shows the cached-body dot
+     *  (the symptom Bob reported 2026-05-19: document/large messages with
+     *  no teal mark). With TTL, transient misses self-heal in 5 minutes;
+     *  genuine zombies back off to once-per-12-hour retries — bounded load.
+     *  Key shape `${accountId}:${folderId}:${uid}`. */
+    prefetchFailures = new Map();
+    markPrefetchEmpty(accountId, folderId, uid) {
+        const key = `${accountId}:${folderId}:${uid}`;
+        const prev = this.prefetchFailures.get(key);
+        this.prefetchFailures.set(key, { count: (prev?.count ?? 0) + 1, lastTried: Date.now() });
+    }
+    isPrefetchEmpty(accountId, folderId, uid) {
+        const f = this.prefetchFailures.get(`${accountId}:${folderId}:${uid}`);
+        if (!f)
+            return false;
+        const backoffMs = f.count <= 1 ? 5 * 60_000
+            : f.count <= 2 ? 30 * 60_000
+                : f.count <= 3 ? 2 * 3600_000
+                    : 12 * 3600_000;
+        return Date.now() - f.lastTried < backoffMs;
+    }
     /** Background body-cache backfill. Public so the Reconciler can schedule
      *  the periodic tick under its priority/back-pressure rules; existing
      *  in-method post-sync nudges (sync, fetchSince, fetchOne) call this
@@ -2739,8 +2782,21 @@ export class ImapManager extends EventEmitter {
         // governing unit). IMAP uses the batched `fetchBodiesBatch` path via iflow-direct —
         // one SELECT + one UID FETCH per folder per tick instead of N round trips.
         let announced = false;
-        while (true) {
-            const missing = this.db.getMessagesWithoutBody(accountId, BATCH_SIZE);
+        // Cap iterations so a fully-blocked queue can't loop forever. With
+        // BATCH_SIZE=100 and the blocklist filter below, each iteration
+        // either advances or terminates; this bound is a belt-and-braces
+        // safety net for unexpected DB states.
+        let iterationsRemaining = 50;
+        while (iterationsRemaining-- > 0) {
+            // Pull more than BATCH_SIZE from the DB so the empty-uid filter
+            // can drop blocklisted rows and still hand a useful batch to the
+            // fetcher. Without this, a queue dominated by stale UIDs returns
+            // BATCH_SIZE rows that all get filtered to zero, and the loop
+            // terminates without ever trying live messages further down the
+            // size-asc list.
+            const raw = this.db.getMessagesWithoutBody(accountId, BATCH_SIZE * 4);
+            const missing = raw.filter(m => !this.isPrefetchEmpty(accountId, m.folderId, m.uid))
+                .slice(0, BATCH_SIZE);
             if (missing.length === 0)
                 break;
             if (!announced) {
@@ -2781,7 +2837,8 @@ export class ImapManager extends EventEmitter {
                                     try {
                                         const raw = Buffer.from(source, "utf-8");
                                         const bodyPath = await this.bodyStore.putMessage(accountId, folderId, uid, raw);
-                                        this.db.updateBodyPath(accountId, uid, bodyPath);
+                                        const parsed = await extractPreview(source);
+                                        this.db.updateBodyMeta(accountId, uid, bodyPath, parsed.hasAttachments, parsed.preview);
                                         this.emit("bodyCached", accountId, uid);
                                         counters.totalFetched++;
                                         madeProgress = true;
@@ -2831,6 +2888,8 @@ export class ImapManager extends EventEmitter {
                         }
                         else if (batchSucceeded && !someReceived) {
                             console.error(`  [prefetch] ${accountId}/${folder.path}: Gmail batch returned 0/${uidsInFolder.length} bodies — NOT pruning (set-diff reconcile owns deletion). UIDs: ${uidsInFolder.slice(0, 5).join(",")}${uidsInFolder.length > 5 ? "..." : ""}`);
+                            for (const uid of uidsInFolder)
+                                this.markPrefetchEmpty(accountId, folderId, uid);
                         }
                         if (counters.errors >= ERROR_BUDGET)
                             break;
@@ -2910,7 +2969,8 @@ export class ImapManager extends EventEmitter {
                                         try {
                                             const raw = Buffer.from(source, "utf-8");
                                             const bodyPath = await this.bodyStore.putMessage(accountId, folderId, uid, raw);
-                                            this.db.updateBodyPath(accountId, uid, bodyPath);
+                                            const parsed = await extractPreview(source);
+                                            this.db.updateBodyMeta(accountId, uid, bodyPath, parsed.hasAttachments, parsed.preview);
                                             this.emit("bodyCached", accountId, uid);
                                             counters.totalFetched++;
                                             madeProgress = true;
@@ -2961,6 +3021,8 @@ export class ImapManager extends EventEmitter {
                             }
                         else if (batchSucceeded && !someReceived) {
                             console.error(`  [prefetch] ${accountId}/${folder.path}: chunk ${chunkStart}-${chunkStart + chunk.length - 1} returned 0/${chunk.length} bodies — NOT pruning (set-diff reconcile owns deletion). UIDs: ${chunk.slice(0, 5).join(",")}${chunk.length > 5 ? "..." : ""}`);
+                            for (const uid of chunk)
+                                this.markPrefetchEmpty(accountId, folderId, uid);
                         }
                     }
                     if (counters.errors >= ERROR_BUDGET)
@@ -2987,6 +3049,49 @@ export class ImapManager extends EventEmitter {
     getBodyStore() {
         return this.bodyStore;
     }
+    /** Re-parse existing .eml files from disk to backfill has_attachments
+     *  + preview for messages whose body landed before the prefetch-reparse
+     *  fix shipped. No IMAP traffic — pure local-disk work. Throttled by the
+     *  reconciler the same way as prefetch (back off when interactive lane
+     *  has clicks pending) so it doesn't fight with the user. */
+    backfillingAccounts = new Set();
+    async backfillBodyMeta(accountId, limit = 100) {
+        if (this.backfillingAccounts.has(accountId))
+            return;
+        if (!this.configs.has(accountId))
+            return;
+        this.backfillingAccounts.add(accountId);
+        try {
+            const rows = this.db.getMessagesNeedingBodyReparse(accountId, limit);
+            if (rows.length === 0)
+                return;
+            let ok = 0, fail = 0;
+            for (const row of rows) {
+                try {
+                    if (!(await this.bodyStore.hasByPath(row.bodyPath))) {
+                        // Body file vanished — stamp so we don't keep retrying.
+                        this.db.updateBodyMetaById(row.id, false, "");
+                        continue;
+                    }
+                    const buf = await this.bodyStore.readByPath(row.bodyPath);
+                    const parsed = await extractPreview(buf.toString("utf-8"));
+                    this.db.updateBodyMetaById(row.id, parsed.hasAttachments, parsed.preview);
+                    ok++;
+                }
+                catch (e) {
+                    fail++;
+                    console.error(`  [backfill] ${accountId}/${row.uid}: ${e?.message || e}`);
+                }
+            }
+            if (ok > 0 || fail > 0) {
+                console.log(`  [backfill] ${accountId}: reparsed ${ok}, failed ${fail}`);
+                this.emit("folderCountsChanged", accountId, {});
+            }
+        }
+        finally {
+            this.backfillingAccounts.delete(accountId);
+        }
+    }
     /** Move message across accounts using iflow's moveMessageToServer */
     async moveMessageCrossAccount(fromAccountId, uid, fromFolderId, toAccountId, toFolderId) {
         const fromFolders = this.db.getFolders(fromAccountId);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bobfrankston/mailx-imap",
-  "version": "0.1.51",
+  "version": "0.1.53",
   "type": "module",
   "main": "index.js",
   "types": "index.d.ts",
@@ -9,14 +9,14 @@
   },
   "license": "ISC",
   "dependencies": {
-    "@bobfrankston/mailx-types": "^0.1.15",
-    "@bobfrankston/mailx-settings": "^0.1.17",
-    "@bobfrankston/mailx-store": "^0.1.28",
+    "@bobfrankston/mailx-types": "^0.1.18",
+    "@bobfrankston/mailx-settings": "^0.1.20",
+    "@bobfrankston/mailx-store": "^0.1.30",
     "@bobfrankston/iflow-direct": "^0.1.50",
     "@bobfrankston/tcp-transport": "^0.1.6",
     "@bobfrankston/smtp-direct": "^0.1.8",
-    "@bobfrankston/mailx-sync": "^0.1.17",
-    "@bobfrankston/oauthsupport": "^1.0.26"
+    "@bobfrankston/mailx-sync": "^0.1.19",
+    "@bobfrankston/oauthsupport": "^1.0.27"
   },
   "repository": {
     "type": "git",
@@ -37,14 +37,14 @@
   },
   ".transformedSnapshot": {
     "dependencies": {
-      "@bobfrankston/mailx-types": "^0.1.15",
-      "@bobfrankston/mailx-settings": "^0.1.17",
-      "@bobfrankston/mailx-store": "^0.1.28",
+      "@bobfrankston/mailx-types": "^0.1.18",
+      "@bobfrankston/mailx-settings": "^0.1.20",
+      "@bobfrankston/mailx-store": "^0.1.30",
       "@bobfrankston/iflow-direct": "^0.1.50",
       "@bobfrankston/tcp-transport": "^0.1.6",
       "@bobfrankston/smtp-direct": "^0.1.8",
-      "@bobfrankston/mailx-sync": "^0.1.17",
-      "@bobfrankston/oauthsupport": "^1.0.26"
+      "@bobfrankston/mailx-sync": "^0.1.19",
+      "@bobfrankston/oauthsupport": "^1.0.27"
     }
   }
 }