@bobfrankston/rmfmail 1.2.46 → 1.2.48

package/packages/mailx-imap/index.ts

@@ -1346,10 +1346,11 @@ export class ImapManager extends EventEmitter {
         client: any,
         accountId: string,
         folderId: number,
-        folderPath: string,
+        folder: { id: number; path: string },
         statusMessageCount: number | null,
         excludeUids: Set<number>,
-    ): Promise<{ recovered: any[]; serverUids: number[] | null; dateBounded: boolean }> {
+    ): Promise<{ recoveredCount: number; serverUids: number[] | null; dateBounded: boolean }> {
+        const folderPath = folder.path;
         const existingUids = this.db.getUidsForFolder(accountId, folderId);
         // Small folders (Drafts, Sent, …) get a FULL UID fetch so server-side
         // deletions of OLDER messages are reflected; large folders (134k INBOX)
@@ -1376,27 +1377,50 @@ export class ImapManager extends EventEmitter {
         console.log(`  [sync-uids] ${accountId}/${folderPath}: ${serverUids.length} UIDs in ${Date.now() - __t0}ms`);
         const existingSet = new Set(existingUids);
         const missingUids = serverUids.filter((uid: number) => !existingSet.has(uid) && !excludeUids.has(uid));
-        const recovered: any[] = [];
-        if (missingUids.length === 0) return { recovered, serverUids, dateBounded };
+        if (missingUids.length === 0) return { recoveredCount: 0, serverUids, dateBounded };
         // Cap so a misbehaving response / brand-new account doesn't pull tens
         // of thousands at once; the rest resumes next cycle. Higher UID ≈ more
-        // recent under stable UIDVALIDITY, so prefer newest when capping.
+        // recent under stable UIDVALIDITY, so prefer newest when capping — the
+        // user cares most about recent mail and the deficit gate keeps calling
+        // us until the backlog is drained.
+        //
+        // Kept deliberately small (Bob 2026-06-20): each backfilled batch feeds
+        // the body-prefetcher, which issues large UID FETCHes on the single
+        // shared slow-lane connection. A 5000-message batch flooded that
+        // connection (cmd-chain waits of 70s+, 90s FETCH timeouts) and wedged
+        // the whole sync worker. A small batch per cycle keeps prefetch's queue
+        // shallow so the connection stays responsive; the deficit gate drains
+        // the backlog over more cycles instead of one connection-killing burst.
         const BACKFILL_CHUNK_SIZE = 100;
-        const RECONCILE_FETCH_CAP = 5000;
+        const RECONCILE_FETCH_CAP = 1000;
         let toFetch = missingUids;
         if (missingUids.length > RECONCILE_FETCH_CAP) {
-            console.log(`  ${folderPath}: ${missingUids.length} server-only UIDs — capped at ${RECONCILE_FETCH_CAP}; will resume next cycle`);
+            console.log(`  ${folderPath}: ${missingUids.length} server-only UIDs — capped at ${RECONCILE_FETCH_CAP}; resumes next cycle`);
             toFetch = missingUids.slice().sort((a: number, b: number) => b - a).slice(0, RECONCILE_FETCH_CAP);
         } else {
             console.log(`  ${folderPath}: ${missingUids.length} server-only UIDs — fetching`);
         }
+        // Store each chunk as it arrives so a mid-stream connection drop (the
+        // "Not connected" / desync class on this server) keeps the progress it
+        // already made instead of losing the whole batch. Stop on the first
+        // chunk error; the deficit gate calls us again next cycle to resume.
+        let recoveredCount = 0;
         for (let i = 0; i < toFetch.length; i += BACKFILL_CHUNK_SIZE) {
             const chunk = toFetch.slice(i, i + BACKFILL_CHUNK_SIZE);
-            const got = await client.fetchMessages(folderPath, chunk.join(","), { source: false });
-            recovered.push(...got);
-            console.log(`  ${folderPath}: backfill ${recovered.length}/${toFetch.length}`);
+            try {
+                const got = await client.fetchMessages(folderPath, chunk.join(","), { source: false });
+                recoveredCount += await this.storeMessages(accountId, folderId, folder, got, 0);
+            } catch (e: any) {
+                console.error(`  ${folderPath}: backfill chunk ${i}-${i + chunk.length} failed (${e?.message || e}) — keeping ${recoveredCount} so far, resuming next cycle`);
+                break;
+            }
+            console.log(`  ${folderPath}: backfill ${Math.min(i + BACKFILL_CHUNK_SIZE, toFetch.length)}/${toFetch.length} (stored ${recoveredCount})`);
         }
-        return { recovered, serverUids, dateBounded };
+        if (recoveredCount > 0) {
+            this.db.recalcFolderCounts(folderId);
+            this.emit("folderCountsChanged", accountId, {});
+        }
+        return { recoveredCount, serverUids, dateBounded };
     }
 
     async syncFolder(accountId: string, folderId: number, client?: any): Promise<number> {
@@ -1611,20 +1635,31 @@ export class ImapManager extends EventEmitter {
                     // set-diff on the first sync after boot, then at most once
                     // per RECONCILE_THROTTLE_MS so the fast path stays fast.
                     let backfilledCount = 0;
+                    // Run the set-diff when EITHER the folder has a known
+                    // deficit (local count < server count → messages are
+                    // missing, keep reconciling every cycle until caught up) OR
+                    // the throttle window elapsed (periodic safety sweep for
+                    // gaps that don't move the count, e.g. a desync-dropped
+                    // message replaced by a later arrival). On success advance
+                    // the throttle; on FAILURE leave it so the next cycle
+                    // retries instead of locking out for RECONCILE_THROTTLE_MS
+                    // (Bob 2026-06-20: boot-time "Not connected" failure locked
+                    // INBOX recovery out for 15 min).
+                    const localCount = this.db.getMessageCount(accountId, folderId);
+                    const hasDeficit = statusMessageCount !== null && localCount < statusMessageCount;
                     const lastRecon = this.lastReconcileMs.get(folderId) ?? 0;
-                    if (Date.now() - lastRecon > RECONCILE_THROTTLE_MS) {
-                        this.lastReconcileMs.set(folderId, Date.now());
+                    const throttleElapsed = Date.now() - lastRecon > RECONCILE_THROTTLE_MS;
+                    if (hasDeficit || throttleElapsed) {
                         try {
                             const sd = await this.fetchServerOnlyUids(
-                                client, accountId, folderId, folder.path, statusMessageCount,
+                                client, accountId, folderId, folder, statusMessageCount,
                                 new Set(newOnes.map((m: any) => m.uid)),
                             );
-                            if (sd.recovered.length > 0) {
-                                backfilledCount = await this.storeMessages(accountId, folderId, folder, sd.recovered, highestUid);
+                            backfilledCount = sd.recoveredCount;
+                            if (backfilledCount > 0) {
                                 console.log(`  [qresync] ${accountId}/${folder.path}: set-diff backfilled ${backfilledCount} server-only UID(s)`);
-                                this.db.recalcFolderCounts(folderId);
-                                this.emit("folderCountsChanged", accountId, {});
                             }
+                            this.lastReconcileMs.set(folderId, Date.now());
                         } catch (e: any) {
                             console.error(`  [qresync] ${accountId}/${folder.path}: set-diff backfill failed: ${e?.message || e}`);
                         }
@@ -1704,23 +1739,24 @@ export class ImapManager extends EventEmitter {
             // historical messages in one cycle. Hit the cap and the next
             // sync picks up the rest.
             try {
-                // Shared with the QRESYNC fast path (fetchServerOnlyUids).
-                // Exclude the UIDs we just fetched via fetchMessagesSinceUid —
-                // they're in `messages` but not yet in the DB, so they'd
-                // otherwise look "missing" and get re-fetched.
+                // Shared with the QRESYNC fast path (fetchServerOnlyUids). It
+                // stores recovered messages itself (chunk-at-a-time, so a
+                // dropped connection keeps partial progress). Exclude the UIDs
+                // we just fetched via fetchMessagesSinceUid — they're in
+                // `messages`, about to be stored by the batch loop below, so
+                // they'd otherwise look "missing" and get re-fetched.
                 const sd = await this.fetchServerOnlyUids(
-                    client, accountId, folderId, folder.path, statusMessageCount,
+                    client, accountId, folderId, folder, statusMessageCount,
                     new Set(messages.map(m => m.uid)),
                 );
-                messages.push(...sd.recovered);
                 // Stash the server UID list for the deletion-reconciliation
                 // block below — no point hitting the server a second time.
                 serverUidsCached = sd.serverUids;
                 serverUidsAreDateBounded = sd.dateBounded;
+                this.lastReconcileMs.set(folderId, Date.now());
             } catch (e: any) {
                 console.error(`  ${folder.path}: reconciliation failed: ${e.message}`);
             }
-            this.lastReconcileMs.set(folderId, Date.now());
 
             // Date-based backfill via SEARCH SINCE was here. Removed —
             // set-diff above already covers the same case (any server UID

package/packages/mailx-imap/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "@bobfrankston/mailx-imap",
-  "version": "0.1.102",
+  "version": "0.1.104",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@bobfrankston/mailx-imap",
-      "version": "0.1.102",
+      "version": "0.1.104",
       "license": "ISC",
       "dependencies": {
         "@bobfrankston/iflow-direct": "^0.1.27",

package/packages/mailx-imap/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bobfrankston/mailx-imap",
-  "version": "0.1.102",
+  "version": "0.1.104",
   "type": "module",
   "main": "index.js",
   "types": "index.d.ts",

package/packages/mailx-settings/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bobfrankston/mailx-settings",
-  "version": "0.1.29",
+  "version": "0.1.30",
   "type": "module",
   "main": "index.js",
   "types": "index.d.ts",