@bobfrankston/mailx-imap 0.1.102 → 0.1.104

package/index.js

@@ -1331,7 +1331,8 @@ export class ImapManager extends EventEmitter {
      * caller's deletion-reconcile can reuse it instead of paying for a second
      * UID SEARCH round-trip.
      */
-    async fetchServerOnlyUids(client, accountId, folderId, folderPath, statusMessageCount, excludeUids) {
+    async fetchServerOnlyUids(client, accountId, folderId, folder, statusMessageCount, excludeUids) {
+        const folderPath = folder.path;
         const existingUids = this.db.getUidsForFolder(accountId, folderId);
         // Small folders (Drafts, Sent, …) get a FULL UID fetch so server-side
         // deletions of OLDER messages are reflected; large folders (134k INBOX)
@@ -1359,29 +1360,53 @@ export class ImapManager extends EventEmitter {
         console.log(`  [sync-uids] ${accountId}/${folderPath}: ${serverUids.length} UIDs in ${Date.now() - __t0}ms`);
         const existingSet = new Set(existingUids);
         const missingUids = serverUids.filter((uid) => !existingSet.has(uid) && !excludeUids.has(uid));
-        const recovered = [];
         if (missingUids.length === 0)
-            return { recovered, serverUids, dateBounded };
+            return { recoveredCount: 0, serverUids, dateBounded };
         // Cap so a misbehaving response / brand-new account doesn't pull tens
         // of thousands at once; the rest resumes next cycle. Higher UID ≈ more
-        // recent under stable UIDVALIDITY, so prefer newest when capping.
+        // recent under stable UIDVALIDITY, so prefer newest when capping — the
+        // user cares most about recent mail and the deficit gate keeps calling
+        // us until the backlog is drained.
+        //
+        // Kept deliberately small (Bob 2026-06-20): each backfilled batch feeds
+        // the body-prefetcher, which issues large UID FETCHes on the single
+        // shared slow-lane connection. A 5000-message batch flooded that
+        // connection (cmd-chain waits of 70s+, 90s FETCH timeouts) and wedged
+        // the whole sync worker. A small batch per cycle keeps prefetch's queue
+        // shallow so the connection stays responsive; the deficit gate drains
+        // the backlog over more cycles instead of one connection-killing burst.
         const BACKFILL_CHUNK_SIZE = 100;
-        const RECONCILE_FETCH_CAP = 5000;
+        const RECONCILE_FETCH_CAP = 1000;
         let toFetch = missingUids;
         if (missingUids.length > RECONCILE_FETCH_CAP) {
-            console.log(`  ${folderPath}: ${missingUids.length} server-only UIDs — capped at ${RECONCILE_FETCH_CAP}; will resume next cycle`);
+            console.log(`  ${folderPath}: ${missingUids.length} server-only UIDs — capped at ${RECONCILE_FETCH_CAP}; resumes next cycle`);
             toFetch = missingUids.slice().sort((a, b) => b - a).slice(0, RECONCILE_FETCH_CAP);
         }
         else {
             console.log(`  ${folderPath}: ${missingUids.length} server-only UIDs — fetching`);
         }
+        // Store each chunk as it arrives so a mid-stream connection drop (the
+        // "Not connected" / desync class on this server) keeps the progress it
+        // already made instead of losing the whole batch. Stop on the first
+        // chunk error; the deficit gate calls us again next cycle to resume.
+        let recoveredCount = 0;
         for (let i = 0; i < toFetch.length; i += BACKFILL_CHUNK_SIZE) {
             const chunk = toFetch.slice(i, i + BACKFILL_CHUNK_SIZE);
-            const got = await client.fetchMessages(folderPath, chunk.join(","), { source: false });
-            recovered.push(...got);
-            console.log(`  ${folderPath}: backfill ${recovered.length}/${toFetch.length}`);
+            try {
+                const got = await client.fetchMessages(folderPath, chunk.join(","), { source: false });
+                recoveredCount += await this.storeMessages(accountId, folderId, folder, got, 0);
+            }
+            catch (e) {
+                console.error(`  ${folderPath}: backfill chunk ${i}-${i + chunk.length} failed (${e?.message || e}) — keeping ${recoveredCount} so far, resuming next cycle`);
+                break;
+            }
+            console.log(`  ${folderPath}: backfill ${Math.min(i + BACKFILL_CHUNK_SIZE, toFetch.length)}/${toFetch.length} (stored ${recoveredCount})`);
         }
-        return { recovered, serverUids, dateBounded };
+        if (recoveredCount > 0) {
+            this.db.recalcFolderCounts(folderId);
+            this.emit("folderCountsChanged", accountId, {});
+        }
+        return { recoveredCount, serverUids, dateBounded };
     }
     async syncFolder(accountId, folderId, client) {
         const lockKey = `${accountId}:${folderId}`;
@@ -1590,17 +1615,28 @@ export class ImapManager extends EventEmitter {
                     // set-diff on the first sync after boot, then at most once
                     // per RECONCILE_THROTTLE_MS so the fast path stays fast.
                     let backfilledCount = 0;
+                    // Run the set-diff when EITHER the folder has a known
+                    // deficit (local count < server count → messages are
+                    // missing, keep reconciling every cycle until caught up) OR
+                    // the throttle window elapsed (periodic safety sweep for
+                    // gaps that don't move the count, e.g. a desync-dropped
+                    // message replaced by a later arrival). On success advance
+                    // the throttle; on FAILURE leave it so the next cycle
+                    // retries instead of locking out for RECONCILE_THROTTLE_MS
+                    // (Bob 2026-06-20: boot-time "Not connected" failure locked
+                    // INBOX recovery out for 15 min).
+                    const localCount = this.db.getMessageCount(accountId, folderId);
+                    const hasDeficit = statusMessageCount !== null && localCount < statusMessageCount;
                     const lastRecon = this.lastReconcileMs.get(folderId) ?? 0;
-                    if (Date.now() - lastRecon > RECONCILE_THROTTLE_MS) {
-                        this.lastReconcileMs.set(folderId, Date.now());
+                    const throttleElapsed = Date.now() - lastRecon > RECONCILE_THROTTLE_MS;
+                    if (hasDeficit || throttleElapsed) {
                         try {
-                            const sd = await this.fetchServerOnlyUids(client, accountId, folderId, folder.path, statusMessageCount, new Set(newOnes.map((m) => m.uid)));
-                            if (sd.recovered.length > 0) {
-                                backfilledCount = await this.storeMessages(accountId, folderId, folder, sd.recovered, highestUid);
+                            const sd = await this.fetchServerOnlyUids(client, accountId, folderId, folder, statusMessageCount, new Set(newOnes.map((m) => m.uid)));
+                            backfilledCount = sd.recoveredCount;
+                            if (backfilledCount > 0) {
                                 console.log(`  [qresync] ${accountId}/${folder.path}: set-diff backfilled ${backfilledCount} server-only UID(s)`);
-                                this.db.recalcFolderCounts(folderId);
-                                this.emit("folderCountsChanged", accountId, {});
                             }
+                            this.lastReconcileMs.set(folderId, Date.now());
                         }
                         catch (e) {
                             console.error(`  [qresync] ${accountId}/${folder.path}: set-diff backfill failed: ${e?.message || e}`);
@@ -1677,21 +1713,22 @@ export class ImapManager extends EventEmitter {
             // historical messages in one cycle. Hit the cap and the next
             // sync picks up the rest.
             try {
-                // Shared with the QRESYNC fast path (fetchServerOnlyUids).
-                // Exclude the UIDs we just fetched via fetchMessagesSinceUid —
-                // they're in `messages` but not yet in the DB, so they'd
-                // otherwise look "missing" and get re-fetched.
-                const sd = await this.fetchServerOnlyUids(client, accountId, folderId, folder.path, statusMessageCount, new Set(messages.map(m => m.uid)));
-                messages.push(...sd.recovered);
+                // Shared with the QRESYNC fast path (fetchServerOnlyUids). It
+                // stores recovered messages itself (chunk-at-a-time, so a
+                // dropped connection keeps partial progress). Exclude the UIDs
+                // we just fetched via fetchMessagesSinceUid — they're in
+                // `messages`, about to be stored by the batch loop below, so
+                // they'd otherwise look "missing" and get re-fetched.
+                const sd = await this.fetchServerOnlyUids(client, accountId, folderId, folder, statusMessageCount, new Set(messages.map(m => m.uid)));
                 // Stash the server UID list for the deletion-reconciliation
                 // block below — no point hitting the server a second time.
                 serverUidsCached = sd.serverUids;
                 serverUidsAreDateBounded = sd.dateBounded;
+                this.lastReconcileMs.set(folderId, Date.now());
             }
             catch (e) {
                 console.error(`  ${folder.path}: reconciliation failed: ${e.message}`);
             }
-            this.lastReconcileMs.set(folderId, Date.now());
             // Date-based backfill via SEARCH SINCE was here. Removed —
             // set-diff above already covers the same case (any server UID
             // not in our local set gets fetched, regardless of whether it's

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bobfrankston/mailx-imap",
-  "version": "0.1.102",
+  "version": "0.1.104",
   "type": "module",
   "main": "index.js",
   "types": "index.d.ts",
@@ -10,7 +10,7 @@
   "license": "ISC",
   "dependencies": {
     "@bobfrankston/mailx-types": "^0.1.19",
-    "@bobfrankston/mailx-settings": "^0.1.28",
+    "@bobfrankston/mailx-settings": "^0.1.30",
     "@bobfrankston/mailx-store": "^0.1.54",
     "@bobfrankston/iflow-direct": "^0.1.55",
     "@bobfrankston/tcp-transport": "^0.1.7",
@@ -38,7 +38,7 @@
   ".transformedSnapshot": {
     "dependencies": {
       "@bobfrankston/mailx-types": "^0.1.19",
-      "@bobfrankston/mailx-settings": "^0.1.28",
+      "@bobfrankston/mailx-settings": "^0.1.30",
       "@bobfrankston/mailx-store": "^0.1.54",
       "@bobfrankston/iflow-direct": "^0.1.55",
       "@bobfrankston/tcp-transport": "^0.1.7",