@bobfrankston/mailx-imap 0.1.29 → 0.1.31

package/index.d.ts

@@ -224,7 +224,25 @@ export declare class ImapManager extends EventEmitter {
         path: string;
     }, uid: number, source: string, flags: string[]): Promise<void>;
     /** Sync messages for a specific folder */
+    /** Per-(accountId,folderId) sync lock. Multiple paths can call syncFolder
+     *  for the same folder concurrently — `syncOne` (full sync), `syncInbox`
+     *  (5-min fast poll), `quickInboxCheckAccount` (startup quick check),
+     *  `syncInboxNewOnly` (IDLE callback). Each path takes a different
+     *  connection, so withConnection can't serialize them. The DB layer
+     *  enforces "one transaction per connection" but doesn't notice the
+     *  concurrent UID set being mutated underneath. Symptom: two
+     *  `[sync-enter]` log lines for the same folder within ms (Bob 2026-05-08
+     *  21:47:52), `cannot start a transaction within a transaction` SQLite
+     *  errors, and prefetch SELECT/FETCH races when sync's SELECT runs
+     *  between prefetch's SELECT and FETCH.
+     *
+     *  This per-folder mutex means the second concurrent caller waits
+     *  rather than silently racing. The waiter still gets the lock when
+     *  the first caller finishes — important for outbox flushes that
+     *  expect their syncFolder for `Sent` to actually run. */
+    private syncFolderLocks;
     syncFolder(accountId: string, folderId: number, client?: any): Promise<number>;
+    private _syncFolderImpl;
     /** Sync all folders for all accounts */
     syncAll(): Promise<void>;
     private _syncAll;

package/index.js

@@ -972,7 +972,41 @@ export class ImapManager extends EventEmitter {
         console.log(`  [local-insert] ${folder.path} UID ${uid}: ${parsed.subject || "(no subject)"} (no IMAP roundtrip)`);
     }
     /** Sync messages for a specific folder */
+    /** Per-(accountId,folderId) sync lock. Multiple paths can call syncFolder
+     *  for the same folder concurrently — `syncOne` (full sync), `syncInbox`
+     *  (5-min fast poll), `quickInboxCheckAccount` (startup quick check),
+     *  `syncInboxNewOnly` (IDLE callback). Each path takes a different
+     *  connection, so withConnection can't serialize them. The DB layer
+     *  enforces "one transaction per connection" but doesn't notice the
+     *  concurrent UID set being mutated underneath. Symptom: two
+     *  `[sync-enter]` log lines for the same folder within ms (Bob 2026-05-08
+     *  21:47:52), `cannot start a transaction within a transaction` SQLite
+     *  errors, and prefetch SELECT/FETCH races when sync's SELECT runs
+     *  between prefetch's SELECT and FETCH.
+     *
+     *  This per-folder mutex means the second concurrent caller waits
+     *  rather than silently racing. The waiter still gets the lock when
+     *  the first caller finishes — important for outbox flushes that
+     *  expect their syncFolder for `Sent` to actually run. */
+    syncFolderLocks = new Map();
     async syncFolder(accountId, folderId, client) {
+        const lockKey = `${accountId}:${folderId}`;
+        const inflight = this.syncFolderLocks.get(lockKey);
+        if (inflight) {
+            // Coalesce: callers that fire while a sync is in flight get the
+            // result of the in-flight call rather than starting a duplicate.
+            // For "quick check that finds new mail and triggers sync" this
+            // means the quick check waits for the existing sync to complete
+            // — which is what the user wants anyway, no double work.
+            console.log(`  [sync-enter] ${accountId}/${folderId}: coalescing (sync already in flight)`);
+            return inflight;
+        }
+        const promise = this._syncFolderImpl(accountId, folderId, client)
+            .finally(() => { this.syncFolderLocks.delete(lockKey); });
+        this.syncFolderLocks.set(lockKey, promise);
+        return promise;
+    }
+    async _syncFolderImpl(accountId, folderId, client) {
         if (!client)
             client = await this.getOpsClient(accountId);
         const prefetch = getPrefetch();
@@ -1111,21 +1145,13 @@ export class ImapManager extends EventEmitter {
                 const existingSet = new Set(existingUids);
                 const newSet = new Set(messages.map(m => m.uid));
                 const missingUids = allServerUids.filter((uid) => !existingSet.has(uid) && !newSet.has(uid));
-                // Backfill chunk size: ops queue yields between chunks so
-                // a click-time body fetch can interleave. 100 UIDs per
-                // chunk balances throughput (one FETCH command per chunk)
-                // against latency (a chunk takes 1-3s on Dovecot, which
-                // is the worst-case wait an interactive click endures).
-                // The 500-chunk version held the ops queue for the
-                // entire backfill — Bob 2026-05-08 saw a click-to-render
-                // wait of 100+ minutes on a busy backfill of the IP
-                // folder.
+                // Backfill chunk size. Use the passed-in `client` directly
+                // (NOT a nested withConnection) — syncFolder is now wrapped
+                // in withConnection at the call site, so the slow lane is
+                // already locked for our duration. A nested withConnection
+                // would deadlock waiting for the slot we hold.
                 const BACKFILL_CHUNK_SIZE = 100;
                 if (missingUids.length > 0 && missingUids.length <= 5000) {
-                    // For the log line we report a count; computing
-                    // min/max via a spread (`Math.min(...arr)`) blows V8's
-                    // argument limit on folders with tens of thousands of
-                    // UIDs. Use a manual reduce.
                     let minU = existingUids[0] ?? 0;
                     for (let i = 1; i < existingUids.length; i++)
                         if (existingUids[i] < minU)
@@ -1135,13 +1161,7 @@ export class ImapManager extends EventEmitter {
                     for (let i = 0; i < missingUids.length; i += BACKFILL_CHUNK_SIZE) {
                         const chunk = missingUids.slice(i, i + BACKFILL_CHUNK_SIZE);
                         const range = chunk.join(",");
-                        // Each chunk gets its own withConnection slow-lane
-                        // turn so any fast-lane click queued in the
-                        // meantime gets serviced between chunks. The
-                        // outer `client` param is bypassed here; the
-                        // queue-managed client is the same persistent
-                        // ops client (getOpsClient).
-                        const recovered = await this.withConnection(accountId, async (c) => await c.fetchMessages(folder.path, range, { source: false }), { slow: true });
+                        const recovered = await client.fetchMessages(folder.path, range, { source: false });
                         messages.push(...recovered);
                         recoveredTotal += recovered.length;
                         console.log(`  ${folder.path}: fetch ${recoveredTotal}/${missingUids.length}`);
@@ -1149,21 +1169,15 @@ export class ImapManager extends EventEmitter {
                 }
                 else if (missingUids.length > 5000) {
                     console.log(`  ${folder.path}: ${missingUids.length} server-only UIDs — capped; will resume next cycle`);
-                    // ASSUMPTION: vanilla IMAP under stable UIDVALIDITY,
-                    // higher UID = later assignment ≈ more recent message.
-                    // True for Dovecot / Cyrus / standard IMAP, which is the
-                    // only place this code path runs (Gmail-API mode goes
-                    // through syncAccountViaApi and doesn't reach here —
-                    // its synthesized hash-UIDs have no temporal meaning).
-                    // If we ever wire this for a non-monotonic UID source,
-                    // sort by date instead — but that means an extra
-                    // fetch-of-INTERNALDATE round-trip, which we avoid for
-                    // free here under the IMAP guarantee.
+                    // Vanilla IMAP under stable UIDVALIDITY: higher UID =
+                    // later assignment ≈ more recent message (Dovecot/
+                    // Cyrus). Gmail-API path is separate (no temporal
+                    // meaning to its hash-UIDs).
                     const cappedSlice = missingUids.sort((a, b) => b - a).slice(0, 5000);
                     let recoveredTotal = 0;
                     for (let i = 0; i < cappedSlice.length; i += BACKFILL_CHUNK_SIZE) {
                         const chunk = cappedSlice.slice(i, i + BACKFILL_CHUNK_SIZE);
-                        const recovered = await this.withConnection(accountId, async (c) => await c.fetchMessages(folder.path, chunk.join(","), { source: false }), { slow: true });
+                        const recovered = await client.fetchMessages(folder.path, chunk.join(","), { source: false });
                         messages.push(...recovered);
                         recoveredTotal += recovered.length;
                         console.log(`  ${folder.path}: fetch ${recoveredTotal}/5000 (capped)`);
@@ -1547,27 +1561,39 @@ export class ImapManager extends EventEmitter {
                 const highestUid = this.db.getHighestUid(accountId, folder.id);
                 if (isTrashChild && highestUid === 0)
                     return;
-                let fresh = null;
+                let clientForDiag = null;
                 try {
-                    fresh = await this.getOpsClient(accountId);
-                    await Promise.race([
-                        this.syncFolder(accountId, folder.id, fresh),
-                        new Promise((_, reject) => setTimeout(() => {
-                            // C120: pull TCP transport diagnostics into the
-                            // per-folder timeout error so the [sync] log
-                            // distinguishes "server stopped responding"
-                            // (sinceLastRead high) from "we never finished
-                            // writing" (writes climbing without reads). Same
-                            // shape iflow-direct emits on its own timeouts.
-                            const d = fresh?.transport?.diagnostics;
-                            const diag = d
-                                ? ` [conn#${d.connId} r=${d.bytesRead}B w=${d.bytesWritten}B writes=${d.writeCount} sinceLastRead=${d.lastReadAt ? Date.now() - d.lastReadAt : -1}ms]`
-                                : "";
-                            reject(new Error(`per-folder timeout (${PER_FOLDER_TIMEOUT_MS / 1000}s): ${folder.path}${diag}`));
-                        }, PER_FOLDER_TIMEOUT_MS)),
-                    ]);
+                    // Route syncFolder through the slow-lane queue so prefetch
+                    // (also slow lane) and sync take strict turns on the slow
+                    // client. Previously syncOne grabbed `getOpsClient` and
+                    // ran syncFolder directly OUTSIDE the queue; prefetch
+                    // chunks via withConnection raced against it on the same
+                    // client. Symptom: prefetch sent `SELECT INBOX` then sync
+                    // sent `SELECT Sent/Drafts`, then prefetch's `UID FETCH
+                    // <inbox-uids>` ran against Drafts → 0 bodies returned →
+                    // prefetch logs "0/N — NOT pruning" but bodies never
+                    // download. With C123 the fast lane has its own
+                    // independent client, so wrapping sync in slow-lane
+                    // withConnection doesn't block click-time body fetches.
+                    await this.withConnection(accountId, async (client) => {
+                        clientForDiag = client;
+                        await this.syncFolder(accountId, folder.id, client);
+                    }, { slow: true, timeoutMs: PER_FOLDER_TIMEOUT_MS });
                 }
                 catch (e) {
+                    // C120: per-folder timeout error appends transport
+                    // diagnostics so the [sync] log distinguishes "server
+                    // stopped responding" (sinceLastRead high) from "we
+                    // never finished writing" (writes climbing without
+                    // reads). The withConnection timeout already includes
+                    // its own message; we annotate further only for the
+                    // timeout path.
+                    if (/timeout/i.test(e?.message || "")) {
+                        const d = clientForDiag?.transport?.diagnostics;
+                        if (d) {
+                            e.message = `${e.message} [conn#${d.connId} r=${d.bytesRead}B w=${d.bytesWritten}B writes=${d.writeCount} sinceLastRead=${d.lastReadAt ? Date.now() - d.lastReadAt : -1}ms] folder=${folder.path}`;
+                        }
+                    }
                     if (e.responseText?.includes("doesn't exist")) {
                         this.db.deleteFolder(folder.id);
                     }
@@ -2389,6 +2415,16 @@ export class ImapManager extends EventEmitter {
     async prefetchBodies(accountId) {
         if (this.prefetchingAccounts.has(accountId))
             return;
+        // Skip if the account isn't registered yet — the reconciler tick
+        // can fire 2 s after daemon start, and addAccount may still be
+        // running its OAuth flow. Without this guard we hit
+        // ERROR_BUDGET (20) failures with "No config for account X" and
+        // exit prefetch silently for the rest of the session — bodies
+        // never download. Symptom: log shows 20 lines of `[prefetch] X
+        // folder Y chunk 0: batch fetch failed: No config for account X`
+        // followed by `stopping after 20 errors`.
+        if (!this.configs.has(accountId))
+            return;
         this.prefetchingAccounts.add(accountId);
         try {
             await this._prefetchBodies(accountId);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bobfrankston/mailx-imap",
-  "version": "0.1.29",
+  "version": "0.1.31",
   "type": "module",
   "main": "index.js",
   "types": "index.d.ts",
@@ -14,7 +14,7 @@
     "@bobfrankston/mailx-store": "^0.1.15",
     "@bobfrankston/iflow-direct": "^0.1.39",
     "@bobfrankston/tcp-transport": "^0.1.6",
-    "@bobfrankston/smtp-direct": "^0.1.6",
+    "@bobfrankston/smtp-direct": "^0.1.8",
     "@bobfrankston/mailx-sync": "^0.1.16",
     "@bobfrankston/oauthsupport": "^1.0.26"
   },
@@ -42,7 +42,7 @@
       "@bobfrankston/mailx-store": "^0.1.15",
       "@bobfrankston/iflow-direct": "^0.1.39",
       "@bobfrankston/tcp-transport": "^0.1.6",
-      "@bobfrankston/smtp-direct": "^0.1.6",
+      "@bobfrankston/smtp-direct": "^0.1.8",
       "@bobfrankston/mailx-sync": "^0.1.16",
       "@bobfrankston/oauthsupport": "^1.0.26"
     }