@bobfrankston/mailx-imap 0.1.25 → 0.1.27

package/index.d.ts

@@ -247,13 +247,33 @@ export declare class ImapManager extends EventEmitter {
     stopPeriodicSync(): void;
     /** Check if an account is OAuth (Gmail/Outlook — generous connection limits) */
     isOAuthAccount(accountId: string): boolean;
-    /** Start IMAP IDLE watchers for INBOX on each account */
+    /** Start an IMAP IDLE watcher per account on INBOX. Other folders are
+     *  not special-cased here — sync runs uniformly on every folder via the
+     *  periodic full-sync timer (STATUS-before-SELECT optimization is the
+     *  right way to make that cheap, not piling per-folder IDLE sockets). */
     startWatching(): Promise<void>;
     /** Stop all IDLE watchers */
     stopWatching(): Promise<void>;
     /** Unlink the on-disk body file for a message by reading its `body_path`
      *  from the DB. Safe to call either before or after `db.deleteMessage`
      *  — read body_path first, store it, then unlink whenever. */
+    /** Per-(account, folder, uid) deferred-delete timer. Reconcile populates
+     *  this; the timer fires 60s later and re-checks whether the row at that
+     *  exact (folder, uid) still exists. If yes → really delete (server
+     *  expunged it). If no → skip (move-detect rebound it elsewhere; the
+     *  row already moved to a new folder/uid in the same DB row, so the
+     *  original key no longer matches anything). Net effect: server-side
+     *  moves preserve UUID + body cache + flags. */
+    private deferredDeletes;
+    /** Grace window for reconcile-delete. Set to 30 minutes because the
+     *  full-folder sync loop walks ~96 folders sequentially and each one
+     *  takes 1-30 seconds; a server-side move from INBOX to a folder that
+     *  syncs late (alphabetically far from INBOX) can take 10+ minutes
+     *  before move-detect fires in the destination. 60s was way too short
+     *  in production — the grace expired and rows were committed-deleted
+     *  before _Spam / Sent / archive folders had a chance to rebind. */
+    private static readonly RECONCILE_DELETE_GRACE_MS;
+    private scheduleDeferredReconcileDelete;
     private unlinkBodyFile;
     /** Fetch a single message body on demand, caching in the store.
      *

package/index.js

@@ -199,6 +199,19 @@ export class ImapManager extends EventEmitter {
         this.transportFactory = transportFactory;
         const storePath = getStorePath();
         this.bodyStore = new FileMessageStore(storePath);
+        // Cancel pending deferred-delete when move-detect rebinds a row.
+        // Without this, the source folder's reconcile-delete grace timer
+        // would fire 30 minutes after detection for a row that's already
+        // moved elsewhere — the user would see the message vanish.
+        this.db.setOnMoveDetected((info) => {
+            const key = `${info.accountId}:${info.fromFolderId}:${info.fromUid}`;
+            const t = this.deferredDeletes.get(key);
+            if (t) {
+                clearTimeout(t);
+                this.deferredDeletes.delete(key);
+                console.log(`  [reconcile-cancel] ${info.accountId} ${info.fromFolderId}/${info.fromUid}: deferred delete cancelled (move-detect rebound to ${info.toFolderId}/${info.toUid})`);
+            }
+        });
     }
     /** Get OAuth access token for an account (for SMTP auth) */
     async getOAuthToken(accountId) {
@@ -813,8 +826,15 @@ export class ImapManager extends EventEmitter {
                     const hex = Buffer.from(msg.subject, "utf-8").subarray(0, 40).toString("hex");
                     console.log(`  [encoding] subject: "${msg.subject.substring(0, 60)}" hex: ${hex}`);
                 }
-                if (msg.uid <= highestUid)
-                    continue; // already have it
+                // CRITICAL: do NOT skip on `uid <= highestUid`. That check
+                // was a major bug — it silently dropped every gap-filled
+                // message (gap-fill specifically recovers UIDs in the
+                // already-scanned range, all of which are <= highestUid).
+                // upsertMessage's UNIQUE(account, folder, uid) constraint
+                // already deduplicates: if mailx truly has the row, the
+                // upsert turns into an UPDATE (cheap); if it doesn't, the
+                // insert proceeds. Trust the constraint — don't second-
+                // guess it on a stale highestUid snapshot.
                 // Tombstone check: if the user locally deleted this Message-ID,
                 // don't re-import it. Server-side EXPUNGE may lag, or reconcile
                 // may find the message in an old list snapshot. Without this,
@@ -878,6 +898,11 @@ export class ImapManager extends EventEmitter {
         // monotonically increasing within a UIDVALIDITY (RFC 3501); a
         // high-water mark is the right anchor for incremental fetch.
         const highestUid = this.db.getHighestUid(accountId, folderId);
+        // STATUS-before-SELECT was here. Removed — added a round-trip per
+        // folder with no measured benefit on Bob's link, and the speculative
+        // "skip SELECT when nothing changed" optimization didn't actually
+        // skip anything in practice (server count vs local count nearly
+        // always differs slightly because of in-flight deletes/moves).
         console.log(`  [sync] ${accountId}/${folder.path}: highestUid=${highestUid}, fetching...`);
         let messages;
         const firstSync = highestUid === 0;
@@ -997,21 +1022,14 @@ export class ImapManager extends EventEmitter {
             try {
                 for (let i = batchStart; i < batchEnd; i++) {
                     const msg = messages[i];
-                    // Skip if we already have this UID
-                    if (msg.uid <= highestUid) {
-                        // But update flags in case they changed
-                        const flags = [];
-                        if (msg.seen)
-                            flags.push("\\Seen");
-                        if (msg.flagged)
-                            flags.push("\\Flagged");
-                        if (msg.answered)
-                            flags.push("\\Answered");
-                        if (msg.draft)
-                            flags.push("\\Draft");
-                        this.db.updateMessageFlags(accountId, msg.uid, flags);
-                        continue;
-                    }
+                    // CRITICAL: was `if (msg.uid <= highestUid) { update flags; continue }`.
+                    // Same bug as the streamy storeMessages path on line ~861:
+                    // it dropped every gap-fill message because gap-fills
+                    // recover UIDs in the already-scanned range (all <=
+                    // highestUid by definition). Trust upsertMessage's
+                    // UNIQUE constraint to dedupe — if mailx truly has the
+                    // row, the upsert becomes an UPDATE that refreshes
+                    // flags too, all in one path.
                     // Tombstone check — same reason as the streamy onChunk path
                     // at storeMessages: a locally-deleted message that the server
                     // hasn't EXPUNGEd yet would otherwise reappear on next sync.
@@ -1102,17 +1120,25 @@ export class ImapManager extends EventEmitter {
                 else if (localUids.length > 0 && toDelete.length / localUids.length > 0.5) {
                     console.log(`  [sync] ${accountId}/${folder.path}: reconcile REFUSED — would delete ${toDelete.length}/${localUids.length} (${Math.round(toDelete.length / localUids.length * 100)}%) — probably a sync bug, skipping`);
                 }
-                else {
+                else if (toDelete.length > 0) {
+                    // DEFERRED DELETE — DO NOT delete immediately. Server-side
+                    // moves (Sieve filters, IMAP MOVE from another client)
+                    // make a UID disappear from the source folder before
+                    // showing up in the destination. mailx syncs folders
+                    // sequentially; if we delete on first detection, the
+                    // dest's `upsertMessage`-side move-detect can't rebind
+                    // (the row is gone), and we lose the UUID + body cache,
+                    // forcing a re-fetch from server.
+                    //
+                    // Instead: schedule the delete 60s out. If between now
+                    // and then move-detect rebinds the row (folder_id changes
+                    // to dest), the (acc, folder, uid) lookup at fire time
+                    // won't match — skip. If still at original (truly gone
+                    // from server, not just moved), commit the delete.
                     for (const uid of toDelete) {
-                        const env = this.db.getMessageByUid(accountId, uid);
-                        const tag = env ? `msgid=${env.messageId || "?"} subj="${(env.subject || "").slice(0, 60)}"` : "unknown";
-                        console.log(`  [reconcile-delete] ${accountId}/${folder.path} uid=${uid} ${tag}`);
-                        this.unlinkBodyFile(accountId, uid, folderId).catch(() => { });
-                        this.db.deleteMessage(accountId, uid);
-                        deletedCount++;
+                        this.scheduleDeferredReconcileDelete(accountId, folderId, uid, folder.path);
                     }
-                    if (deletedCount > 0)
-                        console.log(`    removed ${deletedCount} deleted messages`);
+                    console.log(`  [reconcile-defer] ${accountId}/${folder.path}: scheduled ${toDelete.length} deletes (60s grace for move-detect)`);
                 }
             }
             catch (e) {
@@ -1484,7 +1510,7 @@ export class ImapManager extends EventEmitter {
                         const tag = env ? `msgid=${env.messageId || "?"} subj="${(env.subject || "").slice(0, 60)}"` : "unknown";
                         console.log(`  [reconcile-delete] ${accountId}/${folder.path} uid=${uid} ${tag}`);
                         this.unlinkBodyFile(accountId, uid, folder.id).catch(() => { });
-                        this.db.deleteMessage(accountId, uid);
+                        this.db.deleteMessage(accountId, uid, "Gmail-API reconcile: server list missing this UID", `mailx-imap Gmail reconcile (${folder.path})`);
                     }
                     if (toDelete.length > 0)
                         console.log(`  [api] ${accountId}/${folder.path}: ${toDelete.length} deleted`);
@@ -1863,7 +1889,10 @@ export class ImapManager extends EventEmitter {
         const config = this.configs.get(accountId);
         return !!config?.tokenProvider;
     }
-    /** Start IMAP IDLE watchers for INBOX on each account */
+    /** Start an IMAP IDLE watcher per account on INBOX. Other folders are
+     *  not special-cased here — sync runs uniformly on every folder via the
+     *  periodic full-sync timer (STATUS-before-SELECT optimization is the
+     *  right way to make that cheap, not piling per-folder IDLE sockets). */
     async startWatching() {
         for (const [accountId] of this.configs) {
             if (this.watchers.has(accountId))
@@ -1873,70 +1902,16 @@ export class ImapManager extends EventEmitter {
                 // is parked in IDLE, it's unusable for any other command, so
                 // it can't share the ops queue. Counts against the per-host
                 // semaphore (one slot for the IDLE socket).
-                //
-                // We watch INBOX (incoming mail) and Sent + Drafts (outgoing
-                // changes from another client — Thunderbird, phone, web).
-                // Without IDLE on Sent, a message just sent from another
-                // device would only show in mailx after the next periodic
-                // full sync.
-                const stops = [];
-                const clients = [];
-                const watchOne = async (mailboxLabel, path) => {
-                    const client = await this.createClient(accountId, "idle");
-                    clients.push(client);
-                    const stop = await client.watchMailbox(path, (newCount) => {
-                        console.log(`  [idle] ${accountId} ${path}: ${newCount} new message(s)`);
-                        if (mailboxLabel === "inbox") {
-                            // Fast path: incremental fetch of NEW UIDs only.
-                            // Heavy reconcile runs on the 5-minute STATUS poll.
-                            this.syncInboxNewOnly(accountId).catch(e => console.error(`  [idle] inbox sync error: ${e.message}`));
-                        }
-                        else {
-                            // Sent / Drafts changed elsewhere. Use the
-                            // standard folder sync — picks up the new UID,
-                            // rebinds any optimistic local row by Message-ID.
-                            const folder = this.findFolder(accountId, mailboxLabel);
-                            if (folder) {
-                                this.syncFolder(accountId, folder.id).catch(e => console.error(`  [idle] ${path} sync error: ${e.message}`));
-                            }
-                        }
-                    });
-                    stops.push(stop);
-                };
-                await watchOne("inbox", "INBOX");
-                const sent = this.findFolder(accountId, "sent");
-                if (sent) {
-                    try {
-                        await watchOne("sent", sent.path);
-                    }
-                    catch (e) {
-                        console.error(`  [idle] Failed to watch ${sent.path}: ${e.message}`);
-                    }
-                }
-                const drafts = this.findFolder(accountId, "drafts");
-                if (drafts) {
-                    try {
-                        await watchOne("drafts", drafts.path);
-                    }
-                    catch (e) {
-                        console.error(`  [idle] Failed to watch ${drafts.path}: ${e.message}`);
-                    }
-                }
+                const watchClient = await this.createClient(accountId, "idle");
+                const stop = await watchClient.watchMailbox("INBOX", (newCount) => {
+                    console.log(`  [idle] ${accountId}: ${newCount} new message(s)`);
+                    this.syncInboxNewOnly(accountId).catch(e => console.error(`  [idle] sync error: ${e.message}`));
+                });
                 this.watchers.set(accountId, async () => {
-                    for (const stop of stops) {
-                        try {
-                            await stop();
-                        }
-                        catch { /* ignore */ }
-                    }
-                    for (const c of clients) {
-                        try {
-                            await c.logout();
-                        }
-                        catch { /* ignore */ }
-                    }
+                    await stop();
+                    await watchClient.logout();
                 });
-                console.log(`  [idle] Watching INBOX${sent ? "+Sent" : ""}${drafts ? "+Drafts" : ""} for ${accountId}`);
+                console.log(`  [idle] Watching INBOX for ${accountId}`);
             }
             catch (e) {
                 console.error(`  [idle] Failed to watch ${accountId}: ${e.message}`);
@@ -1956,6 +1931,47 @@ export class ImapManager extends EventEmitter {
     /** Unlink the on-disk body file for a message by reading its `body_path`
      *  from the DB. Safe to call either before or after `db.deleteMessage`
      *  — read body_path first, store it, then unlink whenever. */
+    /** Per-(account, folder, uid) deferred-delete timer. Reconcile populates
+     *  this; the timer fires 60s later and re-checks whether the row at that
+     *  exact (folder, uid) still exists. If yes → really delete (server
+     *  expunged it). If no → skip (move-detect rebound it elsewhere; the
+     *  row already moved to a new folder/uid in the same DB row, so the
+     *  original key no longer matches anything). Net effect: server-side
+     *  moves preserve UUID + body cache + flags. */
+    deferredDeletes = new Map();
+    /** Grace window for reconcile-delete. Set to 30 minutes because the
+     *  full-folder sync loop walks ~96 folders sequentially and each one
+     *  takes 1-30 seconds; a server-side move from INBOX to a folder that
+     *  syncs late (alphabetically far from INBOX) can take 10+ minutes
+     *  before move-detect fires in the destination. 60s was way too short
+     *  in production — the grace expired and rows were committed-deleted
+     *  before _Spam / Sent / archive folders had a chance to rebind. */
+    static RECONCILE_DELETE_GRACE_MS = 30 * 60_000;
+    scheduleDeferredReconcileDelete(accountId, folderId, uid, folderPath) {
+        const key = `${accountId}:${folderId}:${uid}`;
+        // If already pending, don't reset — the 60s clock should run from
+        // the FIRST detection so a flapping server can't keep deferring.
+        if (this.deferredDeletes.has(key))
+            return;
+        const t = setTimeout(() => {
+            this.deferredDeletes.delete(key);
+            // Verify the row still exists at the original (folder, uid).
+            // If move-detect rebound it during the grace window, this
+            // lookup returns null — skip the delete.
+            const env = this.db.getMessageByUid(accountId, uid, folderId);
+            if (!env || env.folderId !== folderId) {
+                console.log(`  [reconcile-skip] ${accountId}/${folderPath} uid=${uid}: row moved during grace window (rebound to folder ${env?.folderId ?? "deleted"})`);
+                return;
+            }
+            const tag = env.messageId ? `msgid=${env.messageId} subj="${(env.subject || "").slice(0, 60)}"` : "no-msgid";
+            console.log(`  [reconcile-delete] ${accountId}/${folderPath} uid=${uid} ${tag} (after 60s grace, no rebind)`);
+            this.unlinkBodyFile(accountId, uid, folderId).catch(() => { });
+            this.db.deleteMessage(accountId, uid, "reconcile: server missing this UID 60s after detection (no move-detect rebind)", `mailx-imap syncFolder deferred reconcile (${folderPath})`);
+            this.db.recalcFolderCounts(folderId);
+            this.emit("folderCountsChanged", accountId, {});
+        }, ImapManager.RECONCILE_DELETE_GRACE_MS);
+        this.deferredDeletes.set(key, t);
+    }
     async unlinkBodyFile(accountId, uid, folderId) {
         try {
             const row = this.db.getMessageByUid(accountId, uid, folderId);
@@ -2204,7 +2220,7 @@ export class ImapManager extends EventEmitter {
                                     continue;
                                 try {
                                     this.unlinkBodyFile(accountId, uid, folderId).catch(() => { });
-                                    this.db.deleteMessage(accountId, uid);
+                                    this.db.deleteMessage(accountId, uid, "prefetch batch: server didn't return body for queued UID — assumed deleted", "mailx-imap prefetchBodies (Gmail batch)");
                                     counters.deleted++;
                                     madeProgress = true;
                                 }
@@ -2304,7 +2320,7 @@ export class ImapManager extends EventEmitter {
                                 continue;
                             try {
                                 this.unlinkBodyFile(accountId, uid, folderId).catch(() => { });
-                                this.db.deleteMessage(accountId, uid);
+                                this.db.deleteMessage(accountId, uid, "prefetch batch: server didn't return body for queued UID — assumed deleted", "mailx-imap prefetchBodies (IMAP batch)");
                                 counters.deleted++;
                                 madeProgress = true;
                             }
@@ -2340,7 +2356,7 @@ export class ImapManager extends EventEmitter {
         // Local first — remove all from DB immediately
         for (const msg of messages) {
             this.unlinkBodyFile(accountId, msg.uid, msg.folderId).catch(() => { });
-            this.db.deleteMessage(accountId, msg.uid);
+            this.db.deleteMessage(accountId, msg.uid, "user-initiated delete (bulk)", "mailx-imap deleteMessages");
         }
         console.log(`  Deleted ${messages.length} messages locally`);
         // Queue IMAP actions
@@ -2393,7 +2409,7 @@ export class ImapManager extends EventEmitter {
         const trash = this.findFolder(accountId, "trash");
         // Local first — remove from DB immediately
         this.unlinkBodyFile(accountId, uid, folderId).catch(() => { });
-        this.db.deleteMessage(accountId, uid);
+        this.db.deleteMessage(accountId, uid, "user-initiated trash", "mailx-imap trashMessage");
         // Queue IMAP action + log the resolution so "I deleted a message and
         // now it's in neither trash nor deleted" is diagnosable from the log.
         if (trash && trash.id !== folderId) {
@@ -2435,7 +2451,7 @@ export class ImapManager extends EventEmitter {
                 if (!msg)
                     throw new Error(`Message UID ${uid} not found in ${fromFolder.path}`);
                 await sourceClient.moveMessageToServer(msg, fromFolder.path, targetClient, toFolder.path);
-                this.db.deleteMessage(fromAccountId, uid);
+                this.db.deleteMessage(fromAccountId, uid, `cross-account move to ${toAccountId}/${toFolder.path}`, "mailx-imap moveBetweenAccounts");
                 console.log(`  Cross-account move: ${fromAccountId}/${fromFolder.path} UID ${uid} → ${toAccountId}/${toFolder.path}`);
             });
         });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bobfrankston/mailx-imap",
-  "version": "0.1.25",
+  "version": "0.1.27",
   "type": "module",
   "main": "index.js",
   "types": "index.d.ts",
@@ -11,8 +11,8 @@
   "dependencies": {
     "@bobfrankston/mailx-types": "^0.1.10",
     "@bobfrankston/mailx-settings": "^0.1.13",
-    "@bobfrankston/mailx-store": "^0.1.10",
-    "@bobfrankston/iflow-direct": "^0.1.30",
+    "@bobfrankston/mailx-store": "^0.1.12",
+    "@bobfrankston/iflow-direct": "^0.1.35",
     "@bobfrankston/tcp-transport": "^0.1.5",
     "@bobfrankston/smtp-direct": "^0.1.5",
     "@bobfrankston/mailx-sync": "^0.1.15",
@@ -39,8 +39,8 @@
     "dependencies": {
       "@bobfrankston/mailx-types": "^0.1.10",
       "@bobfrankston/mailx-settings": "^0.1.13",
-      "@bobfrankston/mailx-store": "^0.1.10",
-      "@bobfrankston/iflow-direct": "^0.1.30",
+      "@bobfrankston/mailx-store": "^0.1.12",
+      "@bobfrankston/iflow-direct": "^0.1.35",
       "@bobfrankston/tcp-transport": "^0.1.5",
       "@bobfrankston/smtp-direct": "^0.1.5",
       "@bobfrankston/mailx-sync": "^0.1.15",