@bobfrankston/mailx-imap 0.1.38 → 0.1.40

package/index.js

@@ -861,9 +861,30 @@ export class ImapManager extends EventEmitter {
     }
     /** Store a batch of messages to DB immediately — used by onChunk for incremental sync */
     async storeMessages(accountId, folderId, folder, msgs, highestUid) {
+        // Chunked transactions: better-sqlite3 is synchronous and an open
+        // transaction blocks every other query on the same DB. A 1881-row
+        // INBOX sync inside ONE transaction locked the connection for ~1.5 s
+        // of straight upserts; any concurrent `getMessage` IPC had to wait
+        // out the whole loop before its own DB read could run. Bob 2026-05-13:
+        // "long long loading again." Solution: commit every BATCH_SIZE rows,
+        // yield the event loop with `setImmediate`, then begin a new
+        // transaction. User clicks land in those gaps with a single-row
+        // worth of latency instead of the full loop's worth.
+        const BATCH_SIZE = 50;
         let stored = 0;
-        this.db.beginTransaction();
+        let inTxn = false;
+        const startTxn = () => { this.db.beginTransaction(); inTxn = true; };
+        const commitTxn = () => { if (inTxn) {
+            this.db.commitTransaction();
+            inTxn = false;
+        } };
+        const rollbackTxn = () => { if (inTxn) {
+            this.db.rollbackTransaction();
+            inTxn = false;
+        } };
         try {
+            startTxn();
+            let batchCount = 0;
             for (const msg of msgs) {
                 // Debug: log subjects with non-ASCII to trace encoding issues
                 if (msg.subject && /[^\x00-\x7F]/.test(msg.subject)) {
@@ -919,11 +940,23 @@ export class ImapManager extends EventEmitter {
                     flags, size: msg.size || 0, hasAttachments, preview, bodyPath
                 });
                 stored++;
+                batchCount++;
+                if (batchCount >= BATCH_SIZE) {
+                    commitTxn();
+                    // Hand the event loop to any waiting IPC (user clicks,
+                    // outbox drains, alarm polls). setImmediate runs AFTER
+                    // pending I/O callbacks, so a stack of getMessage IPCs
+                    // that arrived during the previous chunk gets serviced
+                    // before we start the next batch.
+                    await new Promise(r => setImmediate(r));
+                    batchCount = 0;
+                    startTxn();
+                }
             }
-            this.db.commitTransaction();
+            commitTxn();
         }
         catch (e) {
-            this.db.rollbackTransaction();
+            rollbackTxn();
             console.error(`  storeMessages error: ${e.message}`);
         }
         return stored;
@@ -1041,6 +1074,11 @@ export class ImapManager extends EventEmitter {
         // matches our local count, NOTHING changed — skip SELECT entirely.
         // For first sync (highestUid = 0) skip the STATUS check and let the
         // existing first-sync path (sequence-FETCH the latest N) run.
+        // Captured at function scope so the set-diff block below can decide
+        // when to skip the date-bound and fetch all UIDs (small folder
+        // where a full UID SEARCH is cheap and reveals server-side deletes
+        // of older messages).
+        let statusMessageCount = null;
         if (highestUid > 0) {
             try {
                 console.log(`  [sync-status] ${accountId}/${folder.path}: calling STATUS...`);
@@ -1048,6 +1086,8 @@ export class ImapManager extends EventEmitter {
                 const status = await client.native?.getStatus?.(folder.path)
                     ?? await client.getStatus?.(folder.path);
                 console.log(`  [sync-status] ${accountId}/${folder.path}: STATUS returned in ${Date.now() - __statusT0}ms`);
+                if (status && typeof status.messages === "number")
+                    statusMessageCount = status.messages;
                 if (status && typeof status.uidNext === "number") {
                     const serverHighest = status.uidNext - 1;
                     const noNewUids = serverHighest <= highestUid;
@@ -1132,19 +1172,38 @@ export class ImapManager extends EventEmitter {
                 // archive every cycle. First sync gets all UIDs (no anchor
                 // yet so we have to compare against the empty local set
                 // anyway).
+                // Small folders (Drafts, Sent, Outbox, Trash, …) get a FULL
+                // UID fetch instead of date-bounded. Without this, server-side
+                // deletions of OLDER messages (e.g., user emptied Trash from
+                // Thunderbird) are never reflected — date-bound filters them
+                // out of the comparison entirely. Bob 2026-05-12: "I deleted
+                // my drafts and Thunderbird shows that but rmfmail is not
+                // acknowledging the deletions." Threshold tuned conservatively
+                // — UID SEARCH ALL on a 500-message folder is sub-second; the
+                // pathology being avoided is the same call on 130k+ INBOX.
+                const SMALL_FOLDER_THRESHOLD = 500;
+                const isSmallFolder = statusMessageCount !== null && statusMessageCount <= SMALL_FOLDER_THRESHOLD;
                 const SINCE_DAYS = effectiveDays > 0 ? effectiveDays : 1825;
                 const sinceDate = new Date(Date.now() - SINCE_DAYS * 86400000);
-                console.log(`  [sync-uids] ${accountId}/${folder.path}: getUidsSince ${sinceDate.toISOString().slice(0, 10)}...`);
+                let allServerUids;
                 const __uidsT0 = Date.now();
-                const allServerUids = typeof client.getUidsSince === "function"
-                    ? await client.getUidsSince(folder.path, sinceDate)
-                    : await client.getUids(folder.path);
+                if (isSmallFolder) {
+                    console.log(`  [sync-uids] ${accountId}/${folder.path}: small folder (server=${statusMessageCount}) — getUids ALL`);
+                    allServerUids = await client.getUids(folder.path);
+                    serverUidsAreDateBounded = false;
+                }
+                else {
+                    console.log(`  [sync-uids] ${accountId}/${folder.path}: getUidsSince ${sinceDate.toISOString().slice(0, 10)}...`);
+                    allServerUids = typeof client.getUidsSince === "function"
+                        ? await client.getUidsSince(folder.path, sinceDate)
+                        : await client.getUids(folder.path);
+                    serverUidsAreDateBounded = true;
+                }
                 console.log(`  [sync-uids] ${accountId}/${folder.path}: ${allServerUids.length} UIDs in ${Date.now() - __uidsT0}ms`);
                 // Stash for the deletion-reconciliation block below — we
-                // already have the date-bounded server UID list, no point
-                // hitting the server a second time with UID SEARCH ALL.
+                // already have the server UID list, no point hitting the
+                // server a second time.
                 serverUidsCached = allServerUids;
-                serverUidsAreDateBounded = true;
                 const existingSet = new Set(existingUids);
                 const newSet = new Set(messages.map(m => m.uid));
                 const missingUids = allServerUids.filter((uid) => !existingSet.has(uid) && !newSet.has(uid));
@@ -1367,10 +1426,21 @@ export class ImapManager extends EventEmitter {
                     localUids = localUidsAll.filter(u => u >= minServerUid);
                 }
                 const toDelete = localUids.filter(uid => !serverUids.has(uid));
-                if (serverUidsArr.length === 0 && localUidsAll.length > 0) {
+                // Authoritative-list signal: when the server's UID list came
+                // back as the FULL set (not date-bounded) AND the size matches
+                // what STATUS reported, we know we have the server's truth.
+                // The 50% safety guard exists to protect against partial /
+                // transient server responses; it should NOT block reconcile
+                // when the response is authoritative. Without this carve-out,
+                // emptying a folder externally (Thunderbird, webmail, mobile)
+                // leaves the old rows in mailx forever (Bob 2026-05-12).
+                const authoritative = !serverUidsAreDateBounded
+                    && statusMessageCount !== null
+                    && serverUidsArr.length === statusMessageCount;
+                if (serverUidsArr.length === 0 && localUidsAll.length > 0 && !authoritative) {
                     console.log(`  [sync] ${accountId}/${folder.path}: reconcile skipped — server UID list empty but local has ${localUidsAll.length} (treating as transient)`);
                 }
-                else if (localUids.length > 0 && toDelete.length / localUids.length > 0.5) {
+                else if (!authoritative && localUids.length > 0 && toDelete.length / localUids.length > 0.5) {
                     console.log(`  [sync] ${accountId}/${folder.path}: reconcile REFUSED — would delete ${toDelete.length}/${localUids.length} (${Math.round(toDelete.length / localUids.length * 100)}%) — probably a sync bug, skipping`);
                 }
                 else if (toDelete.length > 0) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bobfrankston/mailx-imap",
-  "version": "0.1.38",
+  "version": "0.1.40",
   "type": "module",
   "main": "index.js",
   "types": "index.d.ts",
@@ -11,7 +11,7 @@
   "dependencies": {
     "@bobfrankston/mailx-types": "^0.1.11",
     "@bobfrankston/mailx-settings": "^0.1.16",
-    "@bobfrankston/mailx-store": "^0.1.19",
+    "@bobfrankston/mailx-store": "^0.1.21",
     "@bobfrankston/iflow-direct": "^0.1.41",
     "@bobfrankston/tcp-transport": "^0.1.6",
     "@bobfrankston/smtp-direct": "^0.1.8",
@@ -39,7 +39,7 @@
     "dependencies": {
       "@bobfrankston/mailx-types": "^0.1.11",
       "@bobfrankston/mailx-settings": "^0.1.16",
-      "@bobfrankston/mailx-store": "^0.1.19",
+      "@bobfrankston/mailx-store": "^0.1.21",
       "@bobfrankston/iflow-direct": "^0.1.41",
       "@bobfrankston/tcp-transport": "^0.1.6",
       "@bobfrankston/smtp-direct": "^0.1.8",