@bobfrankston/rmfmail 1.0.590 → 1.0.594

package/packages/mailx-imap/index.ts

@@ -1013,7 +1013,43 @@ export class ImapManager extends EventEmitter {
     }
 
     /** Sync messages for a specific folder */
+    /** Per-(accountId,folderId) sync lock. Multiple paths can call syncFolder
+     *  for the same folder concurrently — `syncOne` (full sync), `syncInbox`
+     *  (5-min fast poll), `quickInboxCheckAccount` (startup quick check),
+     *  `syncInboxNewOnly` (IDLE callback). Each path takes a different
+     *  connection, so withConnection can't serialize them. The DB layer
+     *  enforces "one transaction per connection" but doesn't notice the
+     *  concurrent UID set being mutated underneath. Symptom: two
+     *  `[sync-enter]` log lines for the same folder within ms (Bob 2026-05-08
+     *  21:47:52), `cannot start a transaction within a transaction` SQLite
+     *  errors, and prefetch SELECT/FETCH races when sync's SELECT runs
+     *  between prefetch's SELECT and FETCH.
+     *
+     *  This per-folder mutex means the second concurrent caller waits
+     *  rather than silently racing. The waiter still gets the lock when
+     *  the first caller finishes — important for outbox flushes that
+     *  expect their syncFolder for `Sent` to actually run. */
+    private syncFolderLocks = new Map<string, Promise<number>>();
+
     async syncFolder(accountId: string, folderId: number, client?: any): Promise<number> {
+        const lockKey = `${accountId}:${folderId}`;
+        const inflight = this.syncFolderLocks.get(lockKey);
+        if (inflight) {
+            // Coalesce: callers that fire while a sync is in flight get the
+            // result of the in-flight call rather than starting a duplicate.
+            // For "quick check that finds new mail and triggers sync" this
+            // means the quick check waits for the existing sync to complete
+            // — which is what the user wants anyway, no double work.
+            console.log(`  [sync-enter] ${accountId}/${folderId}: coalescing (sync already in flight)`);
+            return inflight;
+        }
+        const promise = this._syncFolderImpl(accountId, folderId, client)
+            .finally(() => { this.syncFolderLocks.delete(lockKey); });
+        this.syncFolderLocks.set(lockKey, promise);
+        return promise;
+    }
+
+    private async _syncFolderImpl(accountId: string, folderId: number, client?: any): Promise<number> {
         if (!client) client = await this.getOpsClient(accountId);
         const prefetch = getPrefetch();
 
@@ -1159,21 +1195,13 @@ export class ImapManager extends EventEmitter {
                 const missingUids = allServerUids.filter((uid: number) =>
                     !existingSet.has(uid) && !newSet.has(uid)
                 );
-                // Backfill chunk size: ops queue yields between chunks so
-                // a click-time body fetch can interleave. 100 UIDs per
-                // chunk balances throughput (one FETCH command per chunk)
-                // against latency (a chunk takes 1-3s on Dovecot, which
-                // is the worst-case wait an interactive click endures).
-                // The 500-chunk version held the ops queue for the
-                // entire backfill — Bob 2026-05-08 saw a click-to-render
-                // wait of 100+ minutes on a busy backfill of the IP
-                // folder.
+                // Backfill chunk size. Use the passed-in `client` directly
+                // (NOT a nested withConnection) — syncFolder is now wrapped
+                // in withConnection at the call site, so the slow lane is
+                // already locked for our duration. A nested withConnection
+                // would deadlock waiting for the slot we hold.
                 const BACKFILL_CHUNK_SIZE = 100;
                 if (missingUids.length > 0 && missingUids.length <= 5000) {
-                    // For the log line we report a count; computing
-                    // min/max via a spread (`Math.min(...arr)`) blows V8's
-                    // argument limit on folders with tens of thousands of
-                    // UIDs. Use a manual reduce.
                     let minU = existingUids[0] ?? 0;
                     for (let i = 1; i < existingUids.length; i++) if (existingUids[i] < minU) minU = existingUids[i];
                     console.log(`  ${folder.path}: ${missingUids.length} server-only UIDs (local lowest=${minU}, highest=${highestUid}) — fetching`);
@@ -1181,40 +1209,22 @@ export class ImapManager extends EventEmitter {
                     for (let i = 0; i < missingUids.length; i += BACKFILL_CHUNK_SIZE) {
                         const chunk = missingUids.slice(i, i + BACKFILL_CHUNK_SIZE);
                         const range = chunk.join(",");
-                        // Each chunk gets its own withConnection slow-lane
-                        // turn so any fast-lane click queued in the
-                        // meantime gets serviced between chunks. The
-                        // outer `client` param is bypassed here; the
-                        // queue-managed client is the same persistent
-                        // ops client (getOpsClient).
-                        const recovered = await this.withConnection(accountId, async (c) =>
-                            await (c as any).fetchMessages(folder.path, range, { source: false }),
-                            { slow: true },
-                        );
+                        const recovered = await (client as any).fetchMessages(folder.path, range, { source: false });
                         messages.push(...recovered);
                         recoveredTotal += recovered.length;
                         console.log(`  ${folder.path}: fetch ${recoveredTotal}/${missingUids.length}`);
                     }
                 } else if (missingUids.length > 5000) {
                     console.log(`  ${folder.path}: ${missingUids.length} server-only UIDs — capped; will resume next cycle`);
-                    // ASSUMPTION: vanilla IMAP under stable UIDVALIDITY,
-                    // higher UID = later assignment ≈ more recent message.
-                    // True for Dovecot / Cyrus / standard IMAP, which is the
-                    // only place this code path runs (Gmail-API mode goes
-                    // through syncAccountViaApi and doesn't reach here —
-                    // its synthesized hash-UIDs have no temporal meaning).
-                    // If we ever wire this for a non-monotonic UID source,
-                    // sort by date instead — but that means an extra
-                    // fetch-of-INTERNALDATE round-trip, which we avoid for
-                    // free here under the IMAP guarantee.
+                    // Vanilla IMAP under stable UIDVALIDITY: higher UID =
+                    // later assignment ≈ more recent message (Dovecot/
+                    // Cyrus). Gmail-API path is separate (no temporal
+                    // meaning to its hash-UIDs).
                     const cappedSlice = missingUids.sort((a: number, b: number) => b - a).slice(0, 5000);
                     let recoveredTotal = 0;
                     for (let i = 0; i < cappedSlice.length; i += BACKFILL_CHUNK_SIZE) {
                         const chunk = cappedSlice.slice(i, i + BACKFILL_CHUNK_SIZE);
-                        const recovered = await this.withConnection(accountId, async (c) =>
-                            await (c as any).fetchMessages(folder.path, chunk.join(","), { source: false }),
-                            { slow: true },
-                        );
+                        const recovered = await (client as any).fetchMessages(folder.path, chunk.join(","), { source: false });
                         messages.push(...recovered);
                         recoveredTotal += recovered.length;
                         console.log(`  ${folder.path}: fetch ${recoveredTotal}/5000 (capped)`);
@@ -1610,26 +1620,38 @@ export class ImapManager extends EventEmitter {
                 const isTrashChild = folder.path.includes("/") && folder.path.toLowerCase().startsWith("trash");
                 const highestUid = this.db.getHighestUid(accountId, folder.id);
                 if (isTrashChild && highestUid === 0) return;
-                let fresh: any = null;
+                let clientForDiag: any = null;
                 try {
-                    fresh = await this.getOpsClient(accountId);
-                    await Promise.race([
-                        this.syncFolder(accountId, folder.id, fresh),
-                        new Promise((_, reject) => setTimeout(() => {
-                            // C120: pull TCP transport diagnostics into the
-                            // per-folder timeout error so the [sync] log
-                            // distinguishes "server stopped responding"
-                            // (sinceLastRead high) from "we never finished
-                            // writing" (writes climbing without reads). Same
-                            // shape iflow-direct emits on its own timeouts.
-                            const d = (fresh as any)?.transport?.diagnostics;
-                            const diag = d
-                                ? ` [conn#${d.connId} r=${d.bytesRead}B w=${d.bytesWritten}B writes=${d.writeCount} sinceLastRead=${d.lastReadAt ? Date.now() - d.lastReadAt : -1}ms]`
-                                : "";
-                            reject(new Error(`per-folder timeout (${PER_FOLDER_TIMEOUT_MS / 1000}s): ${folder.path}${diag}`));
-                        }, PER_FOLDER_TIMEOUT_MS)),
-                    ]);
+                    // Route syncFolder through the slow-lane queue so prefetch
+                    // (also slow lane) and sync take strict turns on the slow
+                    // client. Previously syncOne grabbed `getOpsClient` and
+                    // ran syncFolder directly OUTSIDE the queue; prefetch
+                    // chunks via withConnection raced against it on the same
+                    // client. Symptom: prefetch sent `SELECT INBOX` then sync
+                    // sent `SELECT Sent/Drafts`, then prefetch's `UID FETCH
+                    // <inbox-uids>` ran against Drafts → 0 bodies returned →
+                    // prefetch logs "0/N — NOT pruning" but bodies never
+                    // download. With C123 the fast lane has its own
+                    // independent client, so wrapping sync in slow-lane
+                    // withConnection doesn't block click-time body fetches.
+                    await this.withConnection(accountId, async (client) => {
+                        clientForDiag = client;
+                        await this.syncFolder(accountId, folder.id, client);
+                    }, { slow: true, timeoutMs: PER_FOLDER_TIMEOUT_MS });
                 } catch (e: any) {
+                    // C120: per-folder timeout error appends transport
+                    // diagnostics so the [sync] log distinguishes "server
+                    // stopped responding" (sinceLastRead high) from "we
+                    // never finished writing" (writes climbing without
+                    // reads). The withConnection timeout already includes
+                    // its own message; we annotate further only for the
+                    // timeout path.
+                    if (/timeout/i.test(e?.message || "")) {
+                        const d = (clientForDiag as any)?.transport?.diagnostics;
+                        if (d) {
+                            e.message = `${e.message} [conn#${d.connId} r=${d.bytesRead}B w=${d.bytesWritten}B writes=${d.writeCount} sinceLastRead=${d.lastReadAt ? Date.now() - d.lastReadAt : -1}ms] folder=${folder.path}`;
+                        }
+                    }
                     if (e.responseText?.includes("doesn't exist")) {
                         this.db.deleteFolder(folder.id);
                     } else {
@@ -2444,6 +2466,15 @@ export class ImapManager extends EventEmitter {
      *  too and the per-account `prefetchingAccounts` set deduplicates. */
     async prefetchBodies(accountId: string): Promise<void> {
         if (this.prefetchingAccounts.has(accountId)) return;
+        // Skip if the account isn't registered yet — the reconciler tick
+        // can fire 2 s after daemon start, and addAccount may still be
+        // running its OAuth flow. Without this guard we hit
+        // ERROR_BUDGET (20) failures with "No config for account X" and
+        // exit prefetch silently for the rest of the session — bodies
+        // never download. Symptom: log shows 20 lines of `[prefetch] X
+        // folder Y chunk 0: batch fetch failed: No config for account X`
+        // followed by `stopping after 20 errors`.
+        if (!this.configs.has(accountId)) return;
         this.prefetchingAccounts.add(accountId);
         try { await this._prefetchBodies(accountId); }
         finally { this.prefetchingAccounts.delete(accountId); }

package/packages/mailx-imap/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "@bobfrankston/mailx-imap",
-  "version": "0.1.29",
+  "version": "0.1.31",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@bobfrankston/mailx-imap",
-      "version": "0.1.29",
+      "version": "0.1.31",
       "license": "ISC",
       "dependencies": {
         "@bobfrankston/iflow-direct": "^0.1.27",

package/packages/mailx-imap/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bobfrankston/mailx-imap",
-  "version": "0.1.29",
+  "version": "0.1.31",
   "type": "module",
   "main": "index.js",
   "types": "index.d.ts",