@bobfrankston/mailx-imap 0.1.26 → 0.1.28

package/index.d.ts

@@ -257,6 +257,23 @@ export declare class ImapManager extends EventEmitter {
     /** Unlink the on-disk body file for a message by reading its `body_path`
      *  from the DB. Safe to call either before or after `db.deleteMessage`
      *  — read body_path first, store it, then unlink whenever. */
+    /** Per-(account, folder, uid) deferred-delete timer. Reconcile populates
+     *  this; the timer fires 60s later and re-checks whether the row at that
+     *  exact (folder, uid) still exists. If yes → really delete (server
+     *  expunged it). If no → skip (move-detect rebound it elsewhere; the
+     *  row already moved to a new folder/uid in the same DB row, so the
+     *  original key no longer matches anything). Net effect: server-side
+     *  moves preserve UUID + body cache + flags. */
+    private deferredDeletes;
+    /** Grace window for reconcile-delete. Set to 30 minutes because the
+     *  full-folder sync loop walks ~96 folders sequentially and each one
+     *  takes 1-30 seconds; a server-side move from INBOX to a folder that
+     *  syncs late (alphabetically far from INBOX) can take 10+ minutes
+     *  before move-detect fires in the destination. 60s was way too short
+     *  in production — the grace expired and rows were committed-deleted
+     *  before _Spam / Sent / archive folders had a chance to rebind. */
+    private static readonly RECONCILE_DELETE_GRACE_MS;
+    private scheduleDeferredReconcileDelete;
     private unlinkBodyFile;
     /** Fetch a single message body on demand, caching in the store.
      *

package/index.js

@@ -199,6 +199,19 @@ export class ImapManager extends EventEmitter {
         this.transportFactory = transportFactory;
         const storePath = getStorePath();
         this.bodyStore = new FileMessageStore(storePath);
+        // Cancel pending deferred-delete when move-detect rebinds a row.
+        // Without this, the source folder's reconcile-delete grace timer
+        // would fire 30 minutes after detection for a row that's already
+        // moved elsewhere — the user would see the message vanish.
+        this.db.setOnMoveDetected((info) => {
+            const key = `${info.accountId}:${info.fromFolderId}:${info.fromUid}`;
+            const t = this.deferredDeletes.get(key);
+            if (t) {
+                clearTimeout(t);
+                this.deferredDeletes.delete(key);
+                console.log(`  [reconcile-cancel] ${info.accountId} ${info.fromFolderId}/${info.fromUid}: deferred delete cancelled (move-detect rebound to ${info.toFolderId}/${info.toUid})`);
+            }
+        });
     }
     /** Get OAuth access token for an account (for SMTP auth) */
     async getOAuthToken(accountId) {
@@ -813,8 +826,15 @@ export class ImapManager extends EventEmitter {
                     const hex = Buffer.from(msg.subject, "utf-8").subarray(0, 40).toString("hex");
                     console.log(`  [encoding] subject: "${msg.subject.substring(0, 60)}" hex: ${hex}`);
                 }
-                if (msg.uid <= highestUid)
-                    continue; // already have it
+                // CRITICAL: do NOT skip on `uid <= highestUid`. That check
+                // was a major bug — it silently dropped every gap-filled
+                // message (gap-fill specifically recovers UIDs in the
+                // already-scanned range, all of which are <= highestUid).
+                // upsertMessage's UNIQUE(account, folder, uid) constraint
+                // already deduplicates: if mailx truly has the row, the
+                // upsert turns into an UPDATE (cheap); if it doesn't, the
+                // insert proceeds. Trust the constraint — don't second-
+                // guess it on a stale highestUid snapshot.
                 // Tombstone check: if the user locally deleted this Message-ID,
                 // don't re-import it. Server-side EXPUNGE may lag, or reconcile
                 // may find the message in an old list snapshot. Without this,
@@ -878,36 +898,11 @@ export class ImapManager extends EventEmitter {
         // monotonically increasing within a UIDVALIDITY (RFC 3501); a
         // high-water mark is the right anchor for incremental fetch.
         const highestUid = this.db.getHighestUid(accountId, folderId);
-        // STATUS-before-SELECT: ask the server cheaply whether anything has
-        // changed (UIDNEXT moved or message count differs). STATUS doesn't
-        // load the mailbox index, doesn't lock the mailbox, doesn't risk the
-        // SELECT-wedge that's been hammering bobma. Only when STATUS says
-        // there's actual work to do do we fall through to SELECT+FETCH.
-        // Apply uniformly to every folder — no special-casing.
-        try {
-            if (typeof client.getStatus === "function") {
-                const status = await client.getStatus(folder.path);
-                const serverHighest = (status.uidNext || 1) - 1;
-                const serverCount = status.messages ?? -1;
-                const localCount = this.db.getMessageCount(accountId, folderId);
-                const isUpToDate = highestUid > 0
-                    && serverHighest <= highestUid
-                    && serverCount >= 0
-                    && serverCount === localCount;
-                if (isUpToDate) {
-                    console.log(`  [sync] ${accountId}/${folder.path}: STATUS up-to-date (uidNext=${status.uidNext}, server=${serverCount}, local=${localCount}) — skipping SELECT`);
-                    this.emit("syncProgress", accountId, `sync:${folder.path}`, 100);
-                    return 0;
-                }
-                console.log(`  [sync] ${accountId}/${folder.path}: STATUS uidNext=${status.uidNext} server=${serverCount} local=${localCount} highestUid=${highestUid} — needs SELECT`);
-            }
-        }
-        catch (e) {
-            // STATUS shouldn't fail, but don't make it load-bearing — fall
-            // through to the SELECT path on any error and let that path's
-            // existing handling deal with it.
-            console.log(`  [sync] ${accountId}/${folder.path}: STATUS failed (${e?.message || e}) — falling through to SELECT`);
-        }
+        // STATUS-before-SELECT was here. Removed — added a round-trip per
+        // folder with no measured benefit on Bob's link, and the speculative
+        // "skip SELECT when nothing changed" optimization didn't actually
+        // skip anything in practice (server count vs local count nearly
+        // always differs slightly because of in-flight deletes/moves).
         console.log(`  [sync] ${accountId}/${folder.path}: highestUid=${highestUid}, fetching...`);
         let messages;
         const firstSync = highestUid === 0;
@@ -1027,21 +1022,14 @@ export class ImapManager extends EventEmitter {
             try {
                 for (let i = batchStart; i < batchEnd; i++) {
                     const msg = messages[i];
-                    // Skip if we already have this UID
-                    if (msg.uid <= highestUid) {
-                        // But update flags in case they changed
-                        const flags = [];
-                        if (msg.seen)
-                            flags.push("\\Seen");
-                        if (msg.flagged)
-                            flags.push("\\Flagged");
-                        if (msg.answered)
-                            flags.push("\\Answered");
-                        if (msg.draft)
-                            flags.push("\\Draft");
-                        this.db.updateMessageFlags(accountId, msg.uid, flags);
-                        continue;
-                    }
+                    // CRITICAL: was `if (msg.uid <= highestUid) { update flags; continue }`.
+                    // Same bug as the streamy storeMessages path on line ~861:
+                    // it dropped every gap-fill message because gap-fills
+                    // recover UIDs in the already-scanned range (all <=
+                    // highestUid by definition). Trust upsertMessage's
+                    // UNIQUE constraint to dedupe — if mailx truly has the
+                    // row, the upsert becomes an UPDATE that refreshes
+                    // flags too, all in one path.
                     // Tombstone check — same reason as the streamy onChunk path
                     // at storeMessages: a locally-deleted message that the server
                     // hasn't EXPUNGEd yet would otherwise reappear on next sync.
@@ -1132,17 +1120,25 @@ export class ImapManager extends EventEmitter {
                 else if (localUids.length > 0 && toDelete.length / localUids.length > 0.5) {
                     console.log(`  [sync] ${accountId}/${folder.path}: reconcile REFUSED — would delete ${toDelete.length}/${localUids.length} (${Math.round(toDelete.length / localUids.length * 100)}%) — probably a sync bug, skipping`);
                 }
-                else {
+                else if (toDelete.length > 0) {
+                    // DEFERRED DELETE — DO NOT delete immediately. Server-side
+                    // moves (Sieve filters, IMAP MOVE from another client)
+                    // make a UID disappear from the source folder before
+                    // showing up in the destination. mailx syncs folders
+                    // sequentially; if we delete on first detection, the
+                    // dest's `upsertMessage`-side move-detect can't rebind
+                    // (the row is gone), and we lose the UUID + body cache,
+                    // forcing a re-fetch from server.
+                    //
+                    // Instead: schedule the delete 60s out. If between now
+                    // and then move-detect rebinds the row (folder_id changes
+                    // to dest), the (acc, folder, uid) lookup at fire time
+                    // won't match — skip. If still at original (truly gone
+                    // from server, not just moved), commit the delete.
                     for (const uid of toDelete) {
-                        const env = this.db.getMessageByUid(accountId, uid);
-                        const tag = env ? `msgid=${env.messageId || "?"} subj="${(env.subject || "").slice(0, 60)}"` : "unknown";
-                        console.log(`  [reconcile-delete] ${accountId}/${folder.path} uid=${uid} ${tag}`);
-                        this.unlinkBodyFile(accountId, uid, folderId).catch(() => { });
-                        this.db.deleteMessage(accountId, uid, "reconcile: server returned UID list without this row", `mailx-imap syncFolder reconcile (${folder.path})`);
-                        deletedCount++;
+                        this.scheduleDeferredReconcileDelete(accountId, folderId, uid, folder.path);
                     }
-                    if (deletedCount > 0)
-                        console.log(`    removed ${deletedCount} deleted messages`);
+                    console.log(`  [reconcile-defer] ${accountId}/${folder.path}: scheduled ${toDelete.length} deletes (60s grace for move-detect)`);
                 }
             }
             catch (e) {
@@ -1935,6 +1931,63 @@ export class ImapManager extends EventEmitter {
     /** Unlink the on-disk body file for a message by reading its `body_path`
      *  from the DB. Safe to call either before or after `db.deleteMessage`
      *  — read body_path first, store it, then unlink whenever. */
+    /** Per-(account, folder, uid) deferred-delete timer. Reconcile populates
+     *  this; the timer fires 60s later and re-checks whether the row at that
+     *  exact (folder, uid) still exists. If yes → really delete (server
+     *  expunged it). If no → skip (move-detect rebound it elsewhere; the
+     *  row already moved to a new folder/uid in the same DB row, so the
+     *  original key no longer matches anything). Net effect: server-side
+     *  moves preserve UUID + body cache + flags. */
+    deferredDeletes = new Map();
+    /** Grace window for reconcile-delete. Set to 30 minutes because the
+     *  full-folder sync loop walks ~96 folders sequentially and each one
+     *  takes 1-30 seconds; a server-side move from INBOX to a folder that
+     *  syncs late (alphabetically far from INBOX) can take 10+ minutes
+     *  before move-detect fires in the destination. 60s was way too short
+     *  in production — the grace expired and rows were committed-deleted
+     *  before _Spam / Sent / archive folders had a chance to rebind. */
+    static RECONCILE_DELETE_GRACE_MS = 30 * 60_000;
+    scheduleDeferredReconcileDelete(accountId, folderId, uid, folderPath) {
+        const key = `${accountId}:${folderId}:${uid}`;
+        // If already pending, don't reset — the grace clock runs from the
+        // FIRST detection so a flapping server can't keep deferring.
+        if (this.deferredDeletes.has(key))
+            return;
+        const t = setTimeout(() => {
+            this.deferredDeletes.delete(key);
+            // NEW SEMANTICS (post-message_folders refactor): we drop the
+            // FOLDER MEMBERSHIP, not the messages row. The message itself
+            // may exist in other folders (Gmail labels, server-side moves
+            // already reconciled by the dest folder's sync); deleting the
+            // messages row would clobber those.
+            //
+            // 1. Delete from message_folders for THIS (folder, uid).
+            // 2. Look up if the messages row has any other memberships.
+            //    If yes — message lives elsewhere; just drop our row's
+            //    body-cache reference if it was tied to this location.
+            //    If no — message is truly orphaned; delete the messages
+            //    row + unlink body file.
+            const ds = this.db.dropFolderMembership?.(accountId, folderId, uid, folderPath);
+            // Fall back to the legacy path if the new method isn't there
+            // for some reason (defensive — shouldn't happen, but the same
+            // call site is hit by Gmail-API reconcile too which I haven't
+            // converted yet).
+            if (!ds) {
+                const env = this.db.getMessageByUid(accountId, uid, folderId);
+                if (!env || env.folderId !== folderId) {
+                    console.log(`  [reconcile-skip] ${accountId}/${folderPath} uid=${uid}: row moved during grace window`);
+                    return;
+                }
+                const tag = env.messageId ? `msgid=${env.messageId} subj="${(env.subject || "").slice(0, 60)}"` : "no-msgid";
+                console.log(`  [reconcile-delete] ${accountId}/${folderPath} uid=${uid} ${tag} (legacy path)`);
+                this.unlinkBodyFile(accountId, uid, folderId).catch(() => { });
+                this.db.deleteMessage(accountId, uid, "reconcile: server missing this UID after grace (legacy)", `mailx-imap deferred reconcile (${folderPath})`);
+            }
+            this.db.recalcFolderCounts(folderId);
+            this.emit("folderCountsChanged", accountId, {});
+        }, ImapManager.RECONCILE_DELETE_GRACE_MS);
+        this.deferredDeletes.set(key, t);
+    }
     async unlinkBodyFile(accountId, uid, folderId) {
         try {
             const row = this.db.getMessageByUid(accountId, uid, folderId);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bobfrankston/mailx-imap",
-  "version": "0.1.26",
+  "version": "0.1.28",
   "type": "module",
   "main": "index.js",
   "types": "index.d.ts",
@@ -11,8 +11,8 @@
   "dependencies": {
     "@bobfrankston/mailx-types": "^0.1.10",
     "@bobfrankston/mailx-settings": "^0.1.13",
-    "@bobfrankston/mailx-store": "^0.1.11",
-    "@bobfrankston/iflow-direct": "^0.1.32",
+    "@bobfrankston/mailx-store": "^0.1.13",
+    "@bobfrankston/iflow-direct": "^0.1.35",
     "@bobfrankston/tcp-transport": "^0.1.5",
     "@bobfrankston/smtp-direct": "^0.1.5",
     "@bobfrankston/mailx-sync": "^0.1.15",
@@ -39,8 +39,8 @@
     "dependencies": {
       "@bobfrankston/mailx-types": "^0.1.10",
       "@bobfrankston/mailx-settings": "^0.1.13",
-      "@bobfrankston/mailx-store": "^0.1.11",
-      "@bobfrankston/iflow-direct": "^0.1.32",
+      "@bobfrankston/mailx-store": "^0.1.13",
+      "@bobfrankston/iflow-direct": "^0.1.35",
       "@bobfrankston/tcp-transport": "^0.1.5",
       "@bobfrankston/smtp-direct": "^0.1.5",
       "@bobfrankston/mailx-sync": "^0.1.15",