@bobfrankston/mailx-store 0.1.57 → 0.1.59

package/db.d.ts

@@ -301,6 +301,13 @@ export declare class MailxDB {
          *  message_folders. Omitted/false for Gmail (label = folder, many per
          *  message). Set true by the IMAP storeMessages path. */
         exclusive?: boolean;
+        /** UIDs the server currently lists for THIS folder, when the caller
+         *  just enumerated them (the set-diff backfill path). Lets move-detect
+         *  distinguish a real move (old uid gone) from a DUPLICATE DELIVERY
+         *  (same Message-ID at two live uids in one folder — e.g. a message
+         *  addressed to two aliases of the same mailbox). Without this the
+         *  single row ping-pongs between the two uids every sync cycle. */
+        liveServerUids?: Set<number>;
     }): number;
     /** Backfill the FTS5 `body_text` column for a message after its body
      *  has been parsed. Capped at ~64 KB of text per row — FTS5 stores the

package/db.js

@@ -540,6 +540,20 @@ export class MailxDB {
             this.db.exec("CREATE INDEX IF NOT EXISTS idx_messages_acct_msgid ON messages(account_id, message_id)");
         }
         catch { /* already exists */ }
+        // GLOBAL date-order indexes for the unified-inbox streaming walk.
+        // Every folder-scoped index leads with account_id or folder_id, so a
+        // cross-folder `ORDER BY sent_date DESC` had NO usable index and
+        // SQLite sorted all ~139k inbox rows into a temp B-tree on EVERY
+        // unified page fetch — a fixed ~750ms per call (profiled 2026-07-14;
+        // user-visible as "scroll pauses at the bottom then continues", the
+        // load-more gap outrunning its 1000px lead). With these, the walk
+        // streams straight off the index: ~1ms for a 100-row page. One per
+        // date basis (sent_date default, date for the Received toggle).
+        try {
+            this.db.exec("CREATE INDEX IF NOT EXISTS idx_messages_sentdate_global ON messages(sent_date DESC)");
+            this.db.exec("CREATE INDEX IF NOT EXISTS idx_messages_date_global ON messages(date DESC)");
+        }
+        catch { /* already exists */ }
         // is_replied: set when ANY other message in this account has in_reply_to
         // pointing at this row's message_id. Primary source of truth for the ↩
         // marker — \Answered is plan B (some servers strip it, Gmail labels
@@ -1828,7 +1842,48 @@ export class MailxDB {
         // surface transient move-dups. Keeping the collapse for now.
         if (msg.messageId) {
             const moved = this.db.prepare("SELECT id, folder_id, uid FROM messages WHERE account_id = ? AND message_id = ? LIMIT 1").get(msg.accountId, msg.messageId);
-            if (moved) {
+            // Duplicate delivery, NOT a move: same folder, different uid, and
+            // the caller's server UID list confirms the OLD uid is still in the
+            // mailbox. Rebinding here would ping-pong the row between the two
+            // uids on every sync (each cycle sees the unbound uid as
+            // "server-only", refetches it, and rebinds). Fall through to a
+            // fresh insert instead — both copies get their own row, matching
+            // what the server (and every other client) actually shows, and the
+            // set-diff converges.
+            const sameFolderDup = !!moved && moved.folder_id === msg.folderId && moved.uid !== msg.uid
+                && !!msg.liveServerUids?.has(moved.uid);
+            // Cross-folder copy, NOT a move (2026-07-16). Self-addressed mail
+            // legitimately carries ONE Message-ID in both INBOX and Sent — a
+            // Sent backfill "moving" the INBOX row to Sent stole thousands of
+            // INBOX rows, and the next INBOX sync saw 877 "server-only" UIDs,
+            // refetched them, rebound them back … an infinite cross-folder
+            // flap that hammered Dovecot, flip-flopped read/unread flags, and
+            // starved the interactive body-fetch lane ("Fetching body from
+            // server…" for minutes). Evidence available at this call site:
+            // the source folder's membership row — its last_seen_at refreshes
+            // every time that folder's sync confirms the UID is still on the
+            // server. A recently-confirmed source instance means the message
+            // exists in BOTH folders → insert a second row (matches server
+            // truth). Reconcile converges either way: a real move's source
+            // membership is dropped by the source folder's next sync and the
+            // orphaned row is deleted. The window covers hot folders (INBOX
+            // confirms every ~30 s); a stale or missing membership falls
+            // through to the rebind, preserving UUID continuity for genuine
+            // server-side moves.
+            let crossFolderCopy = false;
+            if (moved && !sameFolderDup && moved.folder_id !== msg.folderId) {
+                const SOURCE_CONFIRM_WINDOW_MS = 10 * 60_000;
+                const src = this.db.prepare("SELECT message_row_id, last_seen_at FROM message_folders WHERE folder_id = ? AND uid = ?").get(moved.folder_id, moved.uid);
+                crossFolderCopy = !!src && src.message_row_id === moved.id
+                    && (Date.now() - src.last_seen_at) < SOURCE_CONFIRM_WINDOW_MS;
+            }
+            if (sameFolderDup && moved) {
+                console.log(`  [move-detect] ${msg.accountId} ${msg.messageId}: uid ${moved.uid} still on server — duplicate copy at uid ${msg.uid}, inserting second row`);
+            }
+            else if (crossFolderCopy && moved) {
+                console.log(`  [move-detect] ${msg.accountId} ${msg.messageId}: source folder ${moved.folder_id}/uid ${moved.uid} recently confirmed on server — cross-folder copy at folder ${msg.folderId}/uid ${msg.uid}, inserting second row`);
+            }
+            else if (moved) {
                 console.log(`  [move-detect] ${msg.accountId} ${msg.messageId}: rebinding row ${moved.id} (folder ${moved.folder_id}/uid ${moved.uid} → folder ${msg.folderId}/uid ${msg.uid})`);
                 this.db.prepare("UPDATE messages SET folder_id = ?, uid = ?, cached_at = ? WHERE id = ?").run(msg.folderId, msg.uid, Date.now(), moved.id);
                 this.upsertMessageFolder(moved.id, msg.folderId, msg.uid, msg.exclusive);
@@ -2061,7 +2116,11 @@ export class MailxDB {
         const totalKey = `${flaggedOnly}:${folderIds.join(",")}`;
         const cachedTotal = this._unifiedTotalCache.get(totalKey);
         let total;
-        if (cachedTotal && (Date.now() - cachedTotal.at) < 4000) {
+        // 15s TTL (was 4s): the COUNT is a ~450ms GROUP BY scan of every inbox
+        // row, and background refreshes land often enough that a 4s TTL
+        // recomputed it nearly every fetch. Staleness only affects the
+        // page-count indicator / load-more end detection, never row data.
+        if (cachedTotal && (Date.now() - cachedTotal.at) < 15_000) {
             total = cachedTotal.total;
         }
         else {
@@ -2085,9 +2144,17 @@ export class MailxDB {
         // idx_messages_folder_date supplies the date order. Deep pages walk more
         // (bounded by offset+pageSize survivors) but remain far cheaper than a
         // full window sort, and are rare in the unified inbox.
+        // CROSS JOIN forces messages as the OUTER table so the plan drives off
+        // idx_messages_sentdate_global / idx_messages_date_global (global
+        // date-DESC order, no temp B-tree — the `, m.id` tiebreak only sorts
+        // within equal-date runs) and probes message_folders per row. The
+        // planner can't see our early `break`, so with a plain JOIN it
+        // "optimizes" for full output: drives from mf's folder index and
+        // sorts ALL ~139k inbox rows before yielding row 1 (~750ms/page,
+        // profiled 2026-07-14). Forced order: ~1ms.
         const baseStmt = this.db.prepare(`SELECT m.*, mf.uid AS uid, mf.folder_id AS folder_id
              FROM messages m
-             JOIN message_folders mf ON mf.message_row_id = m.id
+             CROSS JOIN message_folders mf ON mf.message_row_id = m.id
              WHERE mf.folder_id IN (${placeholders})${flagFilter}
              ORDER BY ${dateCol} DESC, m.id DESC`);
         // Per-survivor enrichment (runs only for the ≤pageSize kept rows, not the

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bobfrankston/mailx-store",
-  "version": "0.1.57",
+  "version": "0.1.59",
   "type": "module",
   "main": "index.js",
   "types": "index.d.ts",
@@ -9,8 +9,8 @@
   },
   "license": "ISC",
   "dependencies": {
-    "@bobfrankston/mailx-types": "^0.1.22",
-    "@bobfrankston/mailx-settings": "^0.1.33",
+    "@bobfrankston/mailx-types": "^0.1.24",
+    "@bobfrankston/mailx-settings": "^0.1.35",
     "@bobfrankston/mailx-bus": "^0.1.2",
     "mailparser": "^3.7.2"
   },
@@ -29,8 +29,8 @@
   },
   ".transformedSnapshot": {
     "dependencies": {
-      "@bobfrankston/mailx-types": "^0.1.22",
-      "@bobfrankston/mailx-settings": "^0.1.33",
+      "@bobfrankston/mailx-types": "^0.1.24",
+      "@bobfrankston/mailx-settings": "^0.1.35",
       "@bobfrankston/mailx-bus": "^0.1.2",
       "mailparser": "^3.7.2"
     }