@bobfrankston/mailx 1.0.253 → 1.0.260

package/packages/mailx-imap/index.js

@@ -1259,11 +1259,15 @@ export class ImapManager extends EventEmitter {
             // which gives instant push — the STATUS poll is just a fallback
             // in case IDLE silently dropped.
             const isGmail = this.isGmailAccount(accountId);
-            // Both Gmail API and IMAP accounts have IDLE running for instant
-            // push. The STATUS poll is just a safety net for silent IDLE drops.
-            // Keep it infrequent to avoid hammering — Gmail 429 rate limits
-            // and Dovecot connection limits are both real constraints.
-            const interval = isGmail ? 120000 : 300000; // Gmail: 2min; IMAP: 5min
+            // IMAP accounts: IDLE gives instant push; STATUS poll is just a
+            // safety net for silent IDLE drops — keep it infrequent.
+            // Gmail accounts: no IDLE (Gmail API doesn't expose it), so the
+            // quick poll IS the primary path to new-mail latency. Drop to 30s
+            // so Gmail mail appears in ~15s average. Gmail quota budget is
+            // huge (250 units/sec per user, 1.2B/day) — 120 polls/hour × 5
+            // units ≈ 600/hour, trivial. Dovecot accounts stay at 5min to
+            // respect connection limits (each poll = fresh connection).
+            const interval = isGmail ? 30000 : 300000; // Gmail: 30s; IMAP: 5min
             const timer = setInterval(() => {
                 this.quickInboxCheckAccount(accountId).catch(() => { });
             }, interval);
@@ -1422,8 +1426,18 @@ export class ImapManager extends EventEmitter {
                 // fetchOne returned null — message doesn't exist on the server anymore
                 throw makeNotFoundError(accountId, folderId, uid);
             }
-            if (!msg.source)
+            if (!msg.source) {
+                // Gmail returned a message object but no raw bytes. Seen when:
+                //  (a) the message exists but is larger than the format=raw cap (~10MB),
+                //  (b) UID→Gmail-ID resolution picked a collision and the target
+                //      exists only as a stub, or (c) the listMessageIds top-1000
+                //      didn't include our UID and fetchOne returned null above —
+                //      wait, that would hit the !msg branch. So (a)/(b) remain.
+                // Log enough to distinguish; surface the reason up via a non-null
+                // return so the UI stops showing a generic "fetch returned nothing".
+                console.error(`  [api] Body fetch empty source (${accountId}/${uid}): Gmail returned no raw body — likely too-large-for-format-raw or UID hash collision`);
                 return null;
+            }
             const raw = Buffer.from(msg.source, "utf-8");
             const bodyPath = await this.bodyStore.putMessage(accountId, folderId, uid, raw);
             this.db.updateBodyPath(accountId, uid, bodyPath);
@@ -1443,77 +1457,223 @@ export class ImapManager extends EventEmitter {
      *  stale row locally and keep going. Only unrelated errors (network,
      *  auth, rate limits) count against the error budget, and the budget is
      *  generous so a few transient failures don't kill the whole run. */
+    /** Guard against concurrent prefetchBodies for the same account — mirror of
+     *  `sendingAccounts`. Without this, every periodic-sync tick spawns a new
+     *  prefetch session alongside any still in flight, blowing through Gmail's
+     *  per-minute quota and racing on disk writes. One prefetch per account. */
+    prefetchingAccounts = new Set();
     async prefetchBodies(accountId) {
-        let totalFetched = 0;
-        let deleted = 0;
-        let errors = 0;
-        let rateLimited = false;
+        if (this.prefetchingAccounts.has(accountId))
+            return;
+        this.prefetchingAccounts.add(accountId);
+        try {
+            await this._prefetchBodies(accountId);
+        }
+        finally {
+            this.prefetchingAccounts.delete(accountId);
+        }
+    }
+    async _prefetchBodies(accountId) {
+        const counters = { totalFetched: 0, deleted: 0, errors: 0 };
         const ERROR_BUDGET = 20;
-        // Pace body fetches to avoid slamming Gmail's rate limit. Without a
-        // delay, 500+ body-fetch API calls fire in a burst, every one hits 429,
-        // and the error budget drains before any bodies land.
-        const FETCH_DELAY_MS = this.isGmailAccount(accountId) ? 1000 : 200;
         const RATE_LIMIT_PAUSE_MS = 30000;
+        const BATCH_SIZE = 100;
+        const isGmail = this.isGmailAccount(accountId);
+        // Gmail still uses per-message fetch (HTTP /batch is a separate TODO in this file's
+        // governing unit). IMAP uses the batched `fetchBodiesBatch` path via iflow-direct —
+        // one SELECT + one UID FETCH per folder per tick instead of N round trips.
+        let announced = false;
         while (true) {
-            const missing = this.db.getMessagesWithoutBody(accountId, 100);
+            const missing = this.db.getMessagesWithoutBody(accountId, BATCH_SIZE);
             if (missing.length === 0)
                 break;
-            if (totalFetched === 0 && deleted === 0)
+            if (!announced) {
                 console.log(`  [prefetch] ${accountId}: ${missing.length}+ bodies to fetch`);
+                announced = true;
+            }
             let madeProgress = false;
-            for (const msg of missing) {
-                // If we hit a rate limit, pause before the next fetch
-                if (rateLimited) {
-                    console.log(`  [prefetch] ${accountId}: rate-limited — pausing ${RATE_LIMIT_PAUSE_MS / 1000}s`);
-                    await new Promise(r => setTimeout(r, RATE_LIMIT_PAUSE_MS));
-                    rateLimited = false;
+            if (isGmail) {
+                // Gmail batch path: group by label (what mailx calls "folder"),
+                // list once per label, bounded-concurrency fetch. Far fewer
+                // HTTP round trips than the old one-listMessageIds-per-body path.
+                // Note on the model: Gmail has labels, not folders. A message in
+                // multiple labels gets fetched twice under current grouping. A
+                // deeper label-native redesign is tracked as a separate TODO.
+                const byFolder = new Map();
+                for (const m of missing) {
+                    let arr = byFolder.get(m.folderId);
+                    if (!arr) {
+                        arr = [];
+                        byFolder.set(m.folderId, arr);
+                    }
+                    arr.push(m.uid);
+                }
+                const folders = this.db.getFolders(accountId);
+                const api = this.getGmailProvider(accountId);
+                try {
+                    for (const [folderId, uidsInFolder] of byFolder) {
+                        const folder = folders.find(f => f.id === folderId);
+                        if (!folder)
+                            continue;
+                        const received = new Set();
+                        const pending = [];
+                        let batchSucceeded = false;
+                        try {
+                            await api.fetchBodiesBatch(folder.path, uidsInFolder, (uid, source) => {
+                                received.add(uid);
+                                pending.push((async () => {
+                                    try {
+                                        const raw = Buffer.from(source, "utf-8");
+                                        const bodyPath = await this.bodyStore.putMessage(accountId, folderId, uid, raw);
+                                        this.db.updateBodyPath(accountId, uid, bodyPath);
+                                        counters.totalFetched++;
+                                        madeProgress = true;
+                                    }
+                                    catch (e) {
+                                        console.error(`  [prefetch] ${accountId}/${uid}: store write failed: ${e.message}`);
+                                    }
+                                })());
+                            });
+                            batchSucceeded = true;
+                        }
+                        catch (e) {
+                            const isRate = /429|rate|too many/i.test(String(e?.message || ""));
+                            if (isRate) {
+                                console.log(`  [prefetch] ${accountId}: rate-limited — pausing ${RATE_LIMIT_PAUSE_MS / 1000}s`);
+                                await new Promise(r => setTimeout(r, RATE_LIMIT_PAUSE_MS));
+                            }
+                            else {
+                                console.error(`  [prefetch] ${accountId} folder ${folder.path}: Gmail batch fetch failed: ${e.message}`);
+                                counters.errors++;
+                            }
+                        }
+                        await Promise.all(pending);
+                        // CRITICAL: only prune as "server-deleted" when the batch
+                        // actually completed. If the batch threw (403, 429, network
+                        // error, etc.) NOTHING was received, and treating every
+                        // requested UID as deleted silently wipes 100 messages per
+                        // batch. That's a data-loss bug. Earlier version did this
+                        // and pruned 296 messages on a 403 auth error.
+                        if (batchSucceeded) {
+                            for (const uid of uidsInFolder) {
+                                if (received.has(uid))
+                                    continue;
+                                try {
+                                    this.db.deleteMessage(accountId, uid);
+                                    this.bodyStore.deleteMessage(accountId, folderId, uid).catch(() => { });
+                                    counters.deleted++;
+                                    madeProgress = true;
+                                }
+                                catch { /* ignore */ }
+                            }
+                        }
+                        if (counters.errors >= ERROR_BUDGET)
+                            break;
+                    }
+                }
+                finally {
+                    try {
+                        await api.close();
+                    }
+                    catch { /* ignore */ }
+                }
+                if (counters.errors >= ERROR_BUDGET) {
+                    console.error(`  [prefetch] ${accountId}: stopping after ${counters.errors} errors (${counters.totalFetched} cached, ${counters.deleted} pruned)`);
+                    return;
                 }
-                else if (FETCH_DELAY_MS > 0) {
-                    await new Promise(r => setTimeout(r, FETCH_DELAY_MS));
+            }
+            else {
+                // IMAP batch path: group by folder, one UID FETCH per folder.
+                const byFolder = new Map();
+                for (const m of missing) {
+                    let arr = byFolder.get(m.folderId);
+                    if (!arr) {
+                        arr = [];
+                        byFolder.set(m.folderId, arr);
+                    }
+                    arr.push(m.uid);
                 }
+                const folders = this.db.getFolders(accountId);
+                let client = null;
                 try {
-                    const result = await this.fetchMessageBody(accountId, msg.folderId, msg.uid);
-                    if (result) {
-                        totalFetched++;
-                        madeProgress = true;
+                    client = await this.createClientWithLimit(accountId);
+                    for (const [folderId, uids] of byFolder) {
+                        const folder = folders.find(f => f.id === folderId);
+                        if (!folder)
+                            continue;
+                        const received = new Set();
+                        // onBody fires synchronously as each message streams in from the server.
+                        // Disk/DB writes are kicked off fire-and-forget; we await them after the
+                        // batch command finishes. This keeps streaming throughput high while
+                        // still giving us a single await point for progress accounting.
+                        const pending = [];
+                        let batchSucceeded = false;
+                        try {
+                            await client.fetchBodiesBatch(folder.path, uids, (uid, source) => {
+                                received.add(uid);
+                                pending.push((async () => {
+                                    try {
+                                        const raw = Buffer.from(source, "utf-8");
+                                        const bodyPath = await this.bodyStore.putMessage(accountId, folderId, uid, raw);
+                                        this.db.updateBodyPath(accountId, uid, bodyPath);
+                                        counters.totalFetched++;
+                                        madeProgress = true;
+                                    }
+                                    catch (e) {
+                                        // EBUSY / disk error — non-fatal per message
+                                        console.error(`  [prefetch] ${accountId}/${uid}: store write failed: ${e.message}`);
+                                    }
+                                })());
+                            });
+                            batchSucceeded = true;
+                        }
+                        catch (e) {
+                            console.error(`  [prefetch] ${accountId} folder ${folder.path}: batch fetch failed: ${e.message}`);
+                            counters.errors++;
+                            if (counters.errors >= ERROR_BUDGET)
+                                break;
+                        }
+                        await Promise.all(pending);
+                        // CRITICAL: only prune when the batch actually completed.
+                        // A thrown batch means NOTHING was received and we must
+                        // not treat absence-from-received as server-deletion.
+                        if (batchSucceeded)
+                            for (const uid of uids) {
+                                if (received.has(uid))
+                                    continue;
+                                try {
+                                    this.db.deleteMessage(accountId, uid);
+                                    this.bodyStore.deleteMessage(accountId, folderId, uid).catch(() => { });
+                                    counters.deleted++;
+                                    madeProgress = true;
+                                }
+                                catch { /* ignore */ }
+                            }
                     }
                 }
-                catch (e) {
-                    if (e?.isNotFound) {
-                        // Message deleted on the server — drop the stale row so
-                        // we stop re-asking. This also moves the loop forward
-                        // (next getMessagesWithoutBody call won't return it).
+                finally {
+                    if (client) {
                         try {
-                            this.db.deleteMessage(accountId, msg.uid);
-                            this.bodyStore.deleteMessage(accountId, msg.folderId, msg.uid).catch(() => { });
-                            deleted++;
-                            madeProgress = true;
+                            await client.logout();
                         }
                         catch { /* ignore */ }
-                        continue;
-                    }
-                    // If the error is a rate limit (429), don't count against
-                    // the budget — just slow down. The API will accept requests
-                    // again after a brief pause.
-                    if (/429|rate|too many/i.test(String(e?.message || ""))) {
-                        rateLimited = true;
-                    }
-                    else {
-                        errors++;
-                    }
-                    if (errors >= ERROR_BUDGET) {
-                        console.error(`  [prefetch] ${accountId}: stopping after ${errors} errors (${totalFetched} cached, ${deleted} pruned)`);
-                        return;
                     }
                 }
+                if (counters.errors >= ERROR_BUDGET) {
+                    console.error(`  [prefetch] ${accountId}: stopping after ${counters.errors} errors (${counters.totalFetched} cached, ${counters.deleted} pruned)`);
+                    return;
+                }
             }
-            // Safety: if we made zero progress this iteration, bail — otherwise
-            // we'd loop forever on rows that keep failing without isNotFound.
+            // Safety: zero progress this tick → bail rather than loop forever.
             if (!madeProgress)
                 break;
+            // Emit so the UI refreshes the open-circle → filled-teal indicator
+            // without waiting for the next sync cycle.
+            this.emit("folderCountsChanged", accountId, {});
         }
-        if (totalFetched > 0 || deleted > 0) {
-            console.log(`  [prefetch] ${accountId}: ${totalFetched} bodies cached, ${deleted} stale rows pruned (done)`);
+        if (counters.totalFetched > 0 || counters.deleted > 0) {
+            console.log(`  [prefetch] ${accountId}: ${counters.totalFetched} bodies cached, ${counters.deleted} stale rows pruned (done)`);
+            this.emit("folderCountsChanged", accountId, {});
         }
     }
     /** Get the body store for direct access */
@@ -1781,8 +1941,35 @@ export class ImapManager extends EventEmitter {
                     console.error(`  [drafts] searchByHeader for ${draftId} failed: ${e.message}`);
                 }
             }
-            // Append new draft
-            const result = await client.appendMessage(drafts.path, rawMessage, ["\\Draft", "\\Seen"]);
+            // Append new draft. If the server returns [TRYCREATE] (RFC 3501 §7.1),
+            // the folder doesn't exist on the server even though mailx's DB has
+            // it — happens when the folder was never created, or when the local
+            // special-folder detection latched onto a path that doesn't match
+            // the server's actual name. Create it then retry. Logs the path so
+            // we can diagnose a mis-detected Drafts folder.
+            let result;
+            try {
+                result = await client.appendMessage(drafts.path, rawMessage, ["\\Draft", "\\Seen"]);
+            }
+            catch (e) {
+                const msg = String(e?.message || e);
+                if (/TRYCREATE/i.test(msg)) {
+                    console.log(`  [drafts] APPEND got TRYCREATE for "${drafts.path}" — creating folder and retrying`);
+                    try {
+                        await client.createmailbox(drafts.path);
+                    }
+                    catch (ce) {
+                        // "already exists" is benign; others we surface
+                        if (!/already exists/i.test(String(ce?.message || ""))) {
+                            console.error(`  [drafts] Folder create failed for "${drafts.path}": ${ce.message}`);
+                        }
+                    }
+                    result = await client.appendMessage(drafts.path, rawMessage, ["\\Draft", "\\Seen"]);
+                }
+                else {
+                    throw e;
+                }
+            }
             // APPENDUID returns the UID directly; imapflow returns { destination, uid }
             const uid = typeof result === "number" ? result : result?.uid || null;
             return uid;

package/packages/mailx-imap/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bobfrankston/mailx-imap",
-  "version": "0.1.5",
+  "version": "0.1.11",
   "type": "module",
   "main": "index.js",
   "types": "index.d.ts",
@@ -15,6 +15,7 @@
     "@bobfrankston/iflow-direct": "file:../../../MailApps/iflow-direct",
     "@bobfrankston/tcp-transport": "file:../../../MailApps/tcp-transport",
     "@bobfrankston/smtp-direct": "file:../../../MailApps/smtp-direct",
+    "@bobfrankston/mailx-sync": "file:../../../MailApps/mailx-sync",
     "@bobfrankston/oauthsupport": "file:../../../../projects/oauth/oauthsupport"
   },
   "repository": {

package/packages/mailx-imap/providers/gmail-api.d.ts

@@ -1,32 +1,8 @@
 /**
- * Gmail API provider — replaces IMAP for Gmail accounts.
- * Uses REST API for fast, reliable sync without connection limit issues.
+ * Back-compat re-export. The canonical Gmail provider lives in
+ * @bobfrankston/mailx-sync. mailx-imap re-exports it under its old name so
+ * call sites here keep compiling. Both desktop (this package) and Android
+ * (mailx-store-web) consume the same single implementation now.
  */
-import type { MailProvider, ProviderFolder, ProviderMessage, FetchOptions } from "./types.js";
-export declare class GmailApiProvider implements MailProvider {
-    private tokenProvider;
-    constructor(tokenProvider: () => Promise<string>);
-    private fetch;
-    listFolders(): Promise<ProviderFolder[]>;
-    /** List message IDs matching a query, handling pagination.
-     *  IMPORTANT: on any error we throw — do NOT return a partial list, because
-     *  callers use this for sync reconciliation and a short list would delete
-     *  real messages from the local DB. Returning [] silently caused the
-     *  "INBOX empty in mailx" bug when a rate-limit hit mid-pagination. */
-    private listMessageIds;
-    /** Batch-fetch message metadata or full content */
-    private batchFetch;
-    /** Parse a Gmail API message response into ProviderMessage */
-    private parseMessage;
-    fetchSince(folder: string, sinceUid: number, options?: FetchOptions): Promise<ProviderMessage[]>;
-    fetchByDate(folder: string, since: Date, before: Date, options?: FetchOptions, onChunk?: (msgs: ProviderMessage[]) => void): Promise<ProviderMessage[]>;
-    fetchByUids(folder: string, uids: number[], options?: FetchOptions): Promise<ProviderMessage[]>;
-    fetchOne(folder: string, uid: number, options?: FetchOptions): Promise<ProviderMessage | null>;
-    getUids(folder: string): Promise<number[]>;
-    close(): Promise<void>;
-    /** Map folder path to Gmail label query term */
-    private folderToLabel;
-    /** Format date for Gmail query (YYYY/MM/DD) */
-    private formatDate;
-}
+export { GmailApiProvider } from "@bobfrankston/mailx-sync";
 //# sourceMappingURL=gmail-api.d.ts.map