@bobfrankston/mailx-imap 0.1.24 → 0.1.26

package/index.d.ts

@@ -5,7 +5,7 @@
  */
 import type { TransportFactory } from "@bobfrankston/tcp-transport";
 import { MailxDB, FileMessageStore } from "@bobfrankston/mailx-store";
-import type { AccountConfig, MessageEnvelope, EmailAddress, Folder } from "@bobfrankston/mailx-types";
+import type { AccountConfig, MessageEnvelope, Folder } from "@bobfrankston/mailx-types";
 import { EventEmitter } from "node:events";
 /** Events emitted by the IMAP manager */
 export interface ImapManagerEvents {
@@ -247,7 +247,10 @@ export declare class ImapManager extends EventEmitter {
     stopPeriodicSync(): void;
     /** Check if an account is OAuth (Gmail/Outlook — generous connection limits) */
     isOAuthAccount(accountId: string): boolean;
-    /** Start IMAP IDLE watchers for INBOX on each account */
+    /** Start an IMAP IDLE watcher per account on INBOX. Other folders are
+     *  not special-cased here — sync runs uniformly on every folder via the
+     *  periodic full-sync timer (STATUS-before-SELECT optimization is the
+     *  right way to make that cheap, not piling per-folder IDLE sockets). */
     startWatching(): Promise<void>;
     /** Stop all IDLE watchers */
     stopWatching(): Promise<void>;
@@ -336,31 +339,6 @@ export declare class ImapManager extends EventEmitter {
     processSyncActions(accountId: string): Promise<void>;
     /** Find a folder by specialUse, case-insensitive */
     private findFolder;
-    /** Optimistic local-first Sent insert: write a row into the local DB's
-     *  Sent folder the moment the user hits Send, so the list reflects it
-     *  immediately instead of waiting for SMTP + IMAP-APPEND + syncFolder
-     *  (five server round-trips against a Dovecot that caps at 20 conns).
-     *
-     *  Uses a synthetic negative UID so it can't collide with a real APPENDUID
-     *  (which is always positive). When the real sync eventually picks the
-     *  message up in Sent with the server's UID, `db.upsertMessage` spots
-     *  the Message-ID match and rebinds the existing row's UID — no duplicate.
-     *  Negative UID also makes the row render pink (getMessages flags uid<0
-     *  as pending) so the user sees it's not-yet-reconciled.
-     *
-     *  Best-effort — any failure path (no Sent folder yet, parse error, store
-     *  write error) is logged and swallowed; the send itself is unaffected. */
-    insertOptimisticSentRow(accountId: string, envelope: {
-        messageId: string;
-        inReplyTo: string;
-        references: string[];
-        subject: string;
-        from: EmailAddress;
-        to: EmailAddress[];
-        cc: EmailAddress[];
-        bcc: EmailAddress[];
-        date: number;
-    }, rawMessage: string): Promise<void>;
     /** Copy sent message to the Sent folder via IMAP APPEND */
     copyToSent(accountId: string, rawMessage: string | Buffer): Promise<void>;
     /** Save a draft to the Drafts folder via IMAP APPEND.

package/index.js

@@ -874,8 +874,40 @@ export class ImapManager extends EventEmitter {
         if (!folder)
             throw new Error(`Folder ${folderId} not found`);
         this.emit("syncProgress", accountId, `sync:${folder.path}`, 0);
-        // Get the highest UID we already have for this folder
+        // Get the highest UID we already have for this folder. IMAP UIDs are
+        // monotonically increasing within a UIDVALIDITY (RFC 3501); a
+        // high-water mark is the right anchor for incremental fetch.
         const highestUid = this.db.getHighestUid(accountId, folderId);
+        // STATUS-before-SELECT: ask the server cheaply whether anything has
+        // changed (UIDNEXT moved or message count differs). STATUS doesn't
+        // load the mailbox index, doesn't lock the mailbox, doesn't risk the
+        // SELECT-wedge that's been hammering bobma. Only when STATUS says
+        // there's actual work to do do we fall through to SELECT+FETCH.
+        // Apply uniformly to every folder — no special-casing.
+        try {
+            if (typeof client.getStatus === "function") {
+                const status = await client.getStatus(folder.path);
+                const serverHighest = (status.uidNext || 1) - 1;
+                const serverCount = status.messages ?? -1;
+                const localCount = this.db.getMessageCount(accountId, folderId);
+                const isUpToDate = highestUid > 0
+                    && serverHighest <= highestUid
+                    && serverCount >= 0
+                    && serverCount === localCount;
+                if (isUpToDate) {
+                    console.log(`  [sync] ${accountId}/${folder.path}: STATUS up-to-date (uidNext=${status.uidNext}, server=${serverCount}, local=${localCount}) — skipping SELECT`);
+                    this.emit("syncProgress", accountId, `sync:${folder.path}`, 100);
+                    return 0;
+                }
+                console.log(`  [sync] ${accountId}/${folder.path}: STATUS uidNext=${status.uidNext} server=${serverCount} local=${localCount} highestUid=${highestUid} — needs SELECT`);
+            }
+        }
+        catch (e) {
+            // STATUS shouldn't fail, but don't make it load-bearing — fall
+            // through to the SELECT path on any error and let that path's
+            // existing handling deal with it.
+            console.log(`  [sync] ${accountId}/${folder.path}: STATUS failed (${e?.message || e}) — falling through to SELECT`);
+        }
         console.log(`  [sync] ${accountId}/${folder.path}: highestUid=${highestUid}, fetching...`);
         let messages;
         const firstSync = highestUid === 0;
@@ -1106,7 +1138,7 @@ export class ImapManager extends EventEmitter {
                         const tag = env ? `msgid=${env.messageId || "?"} subj="${(env.subject || "").slice(0, 60)}"` : "unknown";
                         console.log(`  [reconcile-delete] ${accountId}/${folder.path} uid=${uid} ${tag}`);
                         this.unlinkBodyFile(accountId, uid, folderId).catch(() => { });
-                        this.db.deleteMessage(accountId, uid);
+                        this.db.deleteMessage(accountId, uid, "reconcile: server returned UID list without this row", `mailx-imap syncFolder reconcile (${folder.path})`);
                         deletedCount++;
                     }
                     if (deletedCount > 0)
@@ -1482,7 +1514,7 @@ export class ImapManager extends EventEmitter {
                         const tag = env ? `msgid=${env.messageId || "?"} subj="${(env.subject || "").slice(0, 60)}"` : "unknown";
                         console.log(`  [reconcile-delete] ${accountId}/${folder.path} uid=${uid} ${tag}`);
                         this.unlinkBodyFile(accountId, uid, folder.id).catch(() => { });
-                        this.db.deleteMessage(accountId, uid);
+                        this.db.deleteMessage(accountId, uid, "Gmail-API reconcile: server list missing this UID", `mailx-imap Gmail reconcile (${folder.path})`);
                     }
                     if (toDelete.length > 0)
                         console.log(`  [api] ${accountId}/${folder.path}: ${toDelete.length} deleted`);
@@ -1861,7 +1893,10 @@ export class ImapManager extends EventEmitter {
         const config = this.configs.get(accountId);
         return !!config?.tokenProvider;
     }
-    /** Start IMAP IDLE watchers for INBOX on each account */
+    /** Start an IMAP IDLE watcher per account on INBOX. Other folders are
+     *  not special-cased here — sync runs uniformly on every folder via the
+     *  periodic full-sync timer (STATUS-before-SELECT optimization is the
+     *  right way to make that cheap, not piling per-folder IDLE sockets). */
     async startWatching() {
         for (const [accountId] of this.configs) {
             if (this.watchers.has(accountId))
@@ -1871,70 +1906,16 @@ export class ImapManager extends EventEmitter {
                 // is parked in IDLE, it's unusable for any other command, so
                 // it can't share the ops queue. Counts against the per-host
                 // semaphore (one slot for the IDLE socket).
-                //
-                // We watch INBOX (incoming mail) and Sent + Drafts (outgoing
-                // changes from another client — Thunderbird, phone, web).
-                // Without IDLE on Sent, a message just sent from another
-                // device would only show in mailx after the next periodic
-                // full sync.
-                const stops = [];
-                const clients = [];
-                const watchOne = async (mailboxLabel, path) => {
-                    const client = await this.createClient(accountId, "idle");
-                    clients.push(client);
-                    const stop = await client.watchMailbox(path, (newCount) => {
-                        console.log(`  [idle] ${accountId} ${path}: ${newCount} new message(s)`);
-                        if (mailboxLabel === "inbox") {
-                            // Fast path: incremental fetch of NEW UIDs only.
-                            // Heavy reconcile runs on the 5-minute STATUS poll.
-                            this.syncInboxNewOnly(accountId).catch(e => console.error(`  [idle] inbox sync error: ${e.message}`));
-                        }
-                        else {
-                            // Sent / Drafts changed elsewhere. Use the
-                            // standard folder sync — picks up the new UID,
-                            // rebinds any optimistic local row by Message-ID.
-                            const folder = this.findFolder(accountId, mailboxLabel);
-                            if (folder) {
-                                this.syncFolder(accountId, folder.id).catch(e => console.error(`  [idle] ${path} sync error: ${e.message}`));
-                            }
-                        }
-                    });
-                    stops.push(stop);
-                };
-                await watchOne("inbox", "INBOX");
-                const sent = this.findFolder(accountId, "sent");
-                if (sent) {
-                    try {
-                        await watchOne("sent", sent.path);
-                    }
-                    catch (e) {
-                        console.error(`  [idle] Failed to watch ${sent.path}: ${e.message}`);
-                    }
-                }
-                const drafts = this.findFolder(accountId, "drafts");
-                if (drafts) {
-                    try {
-                        await watchOne("drafts", drafts.path);
-                    }
-                    catch (e) {
-                        console.error(`  [idle] Failed to watch ${drafts.path}: ${e.message}`);
-                    }
-                }
+                const watchClient = await this.createClient(accountId, "idle");
+                const stop = await watchClient.watchMailbox("INBOX", (newCount) => {
+                    console.log(`  [idle] ${accountId}: ${newCount} new message(s)`);
+                    this.syncInboxNewOnly(accountId).catch(e => console.error(`  [idle] sync error: ${e.message}`));
+                });
                 this.watchers.set(accountId, async () => {
-                    for (const stop of stops) {
-                        try {
-                            await stop();
-                        }
-                        catch { /* ignore */ }
-                    }
-                    for (const c of clients) {
-                        try {
-                            await c.logout();
-                        }
-                        catch { /* ignore */ }
-                    }
+                    await stop();
+                    await watchClient.logout();
                 });
-                console.log(`  [idle] Watching INBOX${sent ? "+Sent" : ""}${drafts ? "+Drafts" : ""} for ${accountId}`);
+                console.log(`  [idle] Watching INBOX for ${accountId}`);
             }
             catch (e) {
                 console.error(`  [idle] Failed to watch ${accountId}: ${e.message}`);
@@ -2202,7 +2183,7 @@ export class ImapManager extends EventEmitter {
                                     continue;
                                 try {
                                     this.unlinkBodyFile(accountId, uid, folderId).catch(() => { });
-                                    this.db.deleteMessage(accountId, uid);
+                                    this.db.deleteMessage(accountId, uid, "prefetch batch: server didn't return body for queued UID — assumed deleted", "mailx-imap prefetchBodies (Gmail batch)");
                                     counters.deleted++;
                                     madeProgress = true;
                                 }
@@ -2302,7 +2283,7 @@ export class ImapManager extends EventEmitter {
                                 continue;
                             try {
                                 this.unlinkBodyFile(accountId, uid, folderId).catch(() => { });
-                                this.db.deleteMessage(accountId, uid);
+                                this.db.deleteMessage(accountId, uid, "prefetch batch: server didn't return body for queued UID — assumed deleted", "mailx-imap prefetchBodies (IMAP batch)");
                                 counters.deleted++;
                                 madeProgress = true;
                             }
@@ -2338,7 +2319,7 @@ export class ImapManager extends EventEmitter {
         // Local first — remove all from DB immediately
         for (const msg of messages) {
             this.unlinkBodyFile(accountId, msg.uid, msg.folderId).catch(() => { });
-            this.db.deleteMessage(accountId, msg.uid);
+            this.db.deleteMessage(accountId, msg.uid, "user-initiated delete (bulk)", "mailx-imap deleteMessages");
         }
         console.log(`  Deleted ${messages.length} messages locally`);
         // Queue IMAP actions
@@ -2391,7 +2372,7 @@ export class ImapManager extends EventEmitter {
         const trash = this.findFolder(accountId, "trash");
         // Local first — remove from DB immediately
         this.unlinkBodyFile(accountId, uid, folderId).catch(() => { });
-        this.db.deleteMessage(accountId, uid);
+        this.db.deleteMessage(accountId, uid, "user-initiated trash", "mailx-imap trashMessage");
         // Queue IMAP action + log the resolution so "I deleted a message and
         // now it's in neither trash nor deleted" is diagnosable from the log.
         if (trash && trash.id !== folderId) {
@@ -2433,7 +2414,7 @@ export class ImapManager extends EventEmitter {
                 if (!msg)
                     throw new Error(`Message UID ${uid} not found in ${fromFolder.path}`);
                 await sourceClient.moveMessageToServer(msg, fromFolder.path, targetClient, toFolder.path);
-                this.db.deleteMessage(fromAccountId, uid);
+                this.db.deleteMessage(fromAccountId, uid, `cross-account move to ${toAccountId}/${toFolder.path}`, "mailx-imap moveBetweenAccounts");
                 console.log(`  Cross-account move: ${fromAccountId}/${fromFolder.path} UID ${uid} → ${toAccountId}/${toFolder.path}`);
             });
         });
@@ -2616,67 +2597,10 @@ export class ImapManager extends EventEmitter {
         return folders.find(f => f.specialUse === specialUse ||
             f.path.toLowerCase() === specialUse.toLowerCase()) || null;
     }
-    /** Optimistic local-first Sent insert: write a row into the local DB's
-     *  Sent folder the moment the user hits Send, so the list reflects it
-     *  immediately instead of waiting for SMTP + IMAP-APPEND + syncFolder
-     *  (five server round-trips against a Dovecot that caps at 20 conns).
-     *
-     *  Uses a synthetic negative UID so it can't collide with a real APPENDUID
-     *  (which is always positive). When the real sync eventually picks the
-     *  message up in Sent with the server's UID, `db.upsertMessage` spots
-     *  the Message-ID match and rebinds the existing row's UID — no duplicate.
-     *  Negative UID also makes the row render pink (getMessages flags uid<0
-     *  as pending) so the user sees it's not-yet-reconciled.
-     *
-     *  Best-effort — any failure path (no Sent folder yet, parse error, store
-     *  write error) is logged and swallowed; the send itself is unaffected. */
-    async insertOptimisticSentRow(accountId, envelope, rawMessage) {
-        try {
-            const sent = this.findFolder(accountId, "sent");
-            if (!sent) {
-                console.log(`  [sent-local] no Sent folder for ${accountId}; skipping optimistic row`);
-                return;
-            }
-            // Synthetic UID — negative ms timestamp is monotonic + won't
-            // collide with server UIDs. When the real APPENDUID returns via
-            // sync, upsertMessage's Message-ID rebind swaps this for the
-            // real positive value.
-            const synthUid = -Date.now();
-            const bodyPath = await this.bodyStore.putMessage(accountId, sent.id, synthUid, Buffer.from(rawMessage, "utf-8"));
-            const parsed = await extractPreview(rawMessage);
-            this.db.upsertMessage({
-                accountId,
-                folderId: sent.id,
-                uid: synthUid,
-                messageId: envelope.messageId,
-                inReplyTo: envelope.inReplyTo,
-                references: envelope.references,
-                date: envelope.date,
-                subject: envelope.subject,
-                from: envelope.from,
-                to: envelope.to,
-                cc: envelope.cc,
-                flags: ["\\Seen"],
-                size: rawMessage.length,
-                hasAttachments: parsed.hasAttachments,
-                preview: parsed.preview,
-                bodyPath,
-            });
-            // Folder-tree badge refresh + message-list reload if the user
-            // is currently on Sent — same event shape the sync path emits.
-            // (Was sending {accountId,folderId} as a single arg, which the
-            // IPC forwarder + UI handler decoded as garbage — the optimistic
-            // row landed in the DB but never appeared in the list.)
-            this.db.recalcFolderCounts(sent.id);
-            this.emit("folderCountsChanged", accountId, {});
-            console.log(`  [sent-local] wrote optimistic row in Sent (uid=${synthUid}) for ${accountId}: ${envelope.subject}`);
-        }
-        catch (e) {
-            // Non-fatal — send continues, Sent folder just won't show the
-            // row until the real APPEND-then-sync cycle completes.
-            console.error(`  [sent-local] optimistic insert failed: ${e?.message || e}`);
-        }
-    }
+    // insertOptimisticSentRow removed — the synthetic-negative-UID hack
+    // wedged Sent's high-water-mark sync (a synthetic value polluted MAX(uid)
+    // and made every Sent fetch take a stale code path). Sent is now what
+    // the server has, period. Pending sends live in the Outbox view.
     /** Copy sent message to the Sent folder via IMAP APPEND */
     async copyToSent(accountId, rawMessage) {
         const sent = this.findFolder(accountId, "sent");

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bobfrankston/mailx-imap",
-  "version": "0.1.24",
+  "version": "0.1.26",
   "type": "module",
   "main": "index.js",
   "types": "index.d.ts",
@@ -11,8 +11,8 @@
   "dependencies": {
     "@bobfrankston/mailx-types": "^0.1.10",
     "@bobfrankston/mailx-settings": "^0.1.13",
-    "@bobfrankston/mailx-store": "^0.1.10",
-    "@bobfrankston/iflow-direct": "^0.1.30",
+    "@bobfrankston/mailx-store": "^0.1.11",
+    "@bobfrankston/iflow-direct": "^0.1.32",
     "@bobfrankston/tcp-transport": "^0.1.5",
     "@bobfrankston/smtp-direct": "^0.1.5",
     "@bobfrankston/mailx-sync": "^0.1.15",
@@ -39,8 +39,8 @@
     "dependencies": {
       "@bobfrankston/mailx-types": "^0.1.10",
       "@bobfrankston/mailx-settings": "^0.1.13",
-      "@bobfrankston/mailx-store": "^0.1.10",
-      "@bobfrankston/iflow-direct": "^0.1.30",
+      "@bobfrankston/mailx-store": "^0.1.11",
+      "@bobfrankston/iflow-direct": "^0.1.32",
       "@bobfrankston/tcp-transport": "^0.1.5",
       "@bobfrankston/smtp-direct": "^0.1.5",
       "@bobfrankston/mailx-sync": "^0.1.15",