@bobfrankston/rmfmail 1.1.239 → 1.1.241

package/packages/mailx-imap/index.ts

@@ -1017,14 +1017,24 @@ export class ImapManager extends EventEmitter {
         // worth of latency instead of the full loop's worth.
         const BATCH_SIZE = 50;
         let stored = 0;
-        let inTxn = false;
-        const startTxn = () => { this.db.beginTransaction(); inTxn = true; };
-        const commitTxn = () => { if (inTxn) { this.db.commitTransaction(); inTxn = false; } };
-        const rollbackTxn = () => { if (inTxn) { this.db.rollbackTransaction(); inTxn = false; } };
-        try {
-            startTxn();
-            let batchCount = 0;
-            for (const msg of msgs) {
+        // Two-phase batching. The async I/O (write .eml to disk, parse the body
+        // for preview/FTS) runs OUTSIDE any transaction; only the synchronous
+        // upserts run inside one. The previous version held a raw BEGIN open
+        // across `await putMessage` / `await extractPreview`, so a concurrent
+        // storeMessages or the contacts walker could hit BEGIN on the same
+        // connection mid-await → "cannot start a transaction within a
+        // transaction". That threw out the whole batch: messages stored with
+        // empty From/Subject AND the local high-water never advanced, so the
+        // same UIDs re-fetched forever (Bob 2026-06-11: npm notice with blank
+        // From/Subject, its UID re-fetched 51×, sync wedged). Keeping every
+        // `await` out of the transaction makes the collision structurally
+        // impossible, and runInTxn() is nesting-safe besides.
+        type PreparedRow = { upsert: Parameters<MailxDB["upsertMessage"]>[0]; ftsBody: string };
+        for (let i = 0; i < msgs.length; i += BATCH_SIZE) {
+            const chunk = msgs.slice(i, i + BATCH_SIZE);
+            // Phase 1 — async prep, NO transaction held.
+            const prepared: PreparedRow[] = [];
+            for (const msg of chunk) {
                 // Debug: log subjects with non-ASCII to trace encoding issues
                 if (msg.subject && /[^\x00-\x7F]/.test(msg.subject)) {
                     const hex = Buffer.from(msg.subject, "utf-8").subarray(0, 40).toString("hex");
@@ -1070,41 +1080,43 @@ export class ImapManager extends EventEmitter {
                 if (msg.flagged) flags.push("\\Flagged");
                 if (msg.answered) flags.push("\\Answered");
                 if (msg.draft) flags.push("\\Draft");
-                const ftsRowId = this.db.upsertMessage({
-                    accountId, folderId, uid: msg.uid,
-                    messageId: msg.messageId || "",
-                    inReplyTo: (msg as any).inReplyTo || "",
-                    references: [],
-                    date: toFiniteDateMs(msg.date),
-                    subject: msg.subject || "",
-                    from: toEmailAddress(msg.from?.[0] || {}),
-                    to: toEmailAddresses(msg.to || []),
-                    cc: toEmailAddresses(msg.cc || []),
-                    flags, size: msg.size || 0, hasAttachments, preview, bodyPath
+                prepared.push({
+                    upsert: {
+                        accountId, folderId, uid: msg.uid,
+                        messageId: msg.messageId || "",
+                        inReplyTo: (msg as any).inReplyTo || "",
+                        references: [],
+                        date: toFiniteDateMs(msg.date),
+                        subject: msg.subject || "",
+                        from: toEmailAddress(msg.from?.[0] || {}),
+                        to: toEmailAddresses(msg.to || []),
+                        cc: toEmailAddresses(msg.cc || []),
+                        flags, size: msg.size || 0, hasAttachments, preview, bodyPath,
+                    },
+                    ftsBody,
                 });
-                // Index the full body for search. extractPreview already
-                // parsed `source`, so the body text is free here — and
-                // indexing it at sync time means a word buried in the body
-                // is searchable without the user first opening the message.
-                if (ftsRowId && ftsBody) this.db.updateFtsBody(ftsRowId, ftsBody);
-                stored++;
-                batchCount++;
-                if (batchCount >= BATCH_SIZE) {
-                    commitTxn();
-                    // Hand the event loop to any waiting IPC (user clicks,
-                    // outbox drains, alarm polls). setImmediate runs AFTER
-                    // pending I/O callbacks, so a stack of getMessage IPCs
-                    // that arrived during the previous chunk gets serviced
-                    // before we start the next batch.
-                    await new Promise<void>(r => setImmediate(r));
-                    batchCount = 0;
-                    startTxn();
-                }
             }
-            commitTxn();
-        } catch (e: any) {
-            rollbackTxn();
-            console.error(`  storeMessages error: ${e.message}`);
+            // Phase 2 — synchronous transaction, no awaits inside. runInTxn
+            // owns its own BEGIN/COMMIT (or joins an already-open txn) and
+            // ROLLBACKs on any throw, so a bad row can't leak an open txn.
+            try {
+                this.db.runInTxn(() => {
+                    for (const r of prepared) {
+                        const ftsRowId = this.db.upsertMessage(r.upsert);
+                        // Index the full body for search. extractPreview already
+                        // parsed `source`, so the body text is free here — and
+                        // indexing it at sync time means a word buried in the
+                        // body is searchable without first opening the message.
+                        if (ftsRowId && r.ftsBody) this.db.updateFtsBody(ftsRowId, r.ftsBody);
+                    }
+                });
+                stored += prepared.length;
+            } catch (e: any) {
+                console.error(`  storeMessages error: ${e.message}`);
+            }
+            // Hand the event loop to any waiting IPC (user clicks, outbox
+            // drains, alarm polls) between batches.
+            await new Promise<void>(r => setImmediate(r));
         }
         return stored;
     }
@@ -1594,50 +1606,42 @@ export class ImapManager extends EventEmitter {
 
         for (let batchStart = 0; batchStart < messages.length; batchStart += batchSize) {
             const batchEnd = Math.min(batchStart + batchSize, messages.length);
-            this.db.beginTransaction();
-            try {
-                for (let i = batchStart; i < batchEnd; i++) {
-                    const msg = messages[i];
-
-                    // CRITICAL: was `if (msg.uid <= highestUid) { update flags; continue }`.
-                    // Same bug as the streamy storeMessages path on line ~861:
-                    // it dropped every gap-fill message because gap-fills
-                    // recover UIDs in the already-scanned range (all <=
-                    // highestUid by definition). Trust upsertMessage's
-                    // UNIQUE constraint to dedupe — if mailx truly has the
-                    // row, the upsert becomes an UPDATE that refreshes
-                    // flags too, all in one path.
-
-                    // Tombstone-skip: in-flight local delete/move waiting
-                    // for server confirmation. Cleared on action success or
-                    // permanent failure (see clearTombstoneForUid).
-                    if (msg.messageId && this.db.hasTombstone(accountId, msg.messageId)) {
-                        console.log(`  [tombstone] ${accountId}: skipping ${msg.messageId} in syncFolder (locally deleted)`);
-                        continue;
-                    }
-
-                    // Store body
-                    const source = msg.source || "";
-                    let bodyPath = "";
-                    if (source) {
-                        bodyPath = await this.bodyStore.putMessage(
-                            accountId, folderId, msg.uid,
-                            Buffer.from(source, "utf-8")
-                        );
-                    }
-
-                    // Parse for preview and attachment info
-                    const parsed = await extractPreview(source);
-
-                    // Build flags array
-                    const flags: string[] = [];
-                    if (msg.seen) flags.push("\\Seen");
-                    if (msg.flagged) flags.push("\\Flagged");
-                    if (msg.answered) flags.push("\\Answered");
-                    if (msg.draft) flags.push("\\Draft");
-
-                    // Store metadata
-                    const ftsRowId = this.db.upsertMessage({
+            // Phase 1 — async prep (write .eml, parse preview), NO transaction
+            // held. The previous version kept a raw BEGIN open across
+            // `await putMessage` / `await extractPreview`, so a concurrent
+            // store on the same connection could hit BEGIN mid-await →
+            // "cannot start a transaction within a transaction", wedging the
+            // sync. Same fix + cause as storeMessages above (Bob 2026-06-11).
+            //
+            // NOTE: the old `if (msg.uid <= highestUid) continue` skip is
+            // intentionally gone — gap-fills recover UIDs in the scanned range,
+            // and upsertMessage's UNIQUE(account,folder,uid) constraint dedupes
+            // (truly-present rows become a cheap UPDATE).
+            const prepared: Array<{ upsert: Parameters<MailxDB["upsertMessage"]>[0]; ftsBody: string }> = [];
+            for (let i = batchStart; i < batchEnd; i++) {
+                const msg = messages[i];
+                // Tombstone-skip: in-flight local delete/move awaiting server
+                // confirmation (cleared on success or permanent failure).
+                if (msg.messageId && this.db.hasTombstone(accountId, msg.messageId)) {
+                    console.log(`  [tombstone] ${accountId}: skipping ${msg.messageId} in syncFolder (locally deleted)`);
+                    continue;
+                }
+                const source = msg.source || "";
+                let bodyPath = "";
+                if (source) {
+                    bodyPath = await this.bodyStore.putMessage(
+                        accountId, folderId, msg.uid,
+                        Buffer.from(source, "utf-8")
+                    );
+                }
+                const parsed = await extractPreview(source);
+                const flags: string[] = [];
+                if (msg.seen) flags.push("\\Seen");
+                if (msg.flagged) flags.push("\\Flagged");
+                if (msg.answered) flags.push("\\Answered");
+                if (msg.draft) flags.push("\\Draft");
+                prepared.push({
+                    upsert: {
                         accountId,
                         folderId,
                         uid: msg.uid,
@@ -1653,19 +1657,25 @@ export class ImapManager extends EventEmitter {
                         size: msg.size || 0,
                         hasAttachments: parsed.hasAttachments,
                         preview: parsed.preview,
-                        bodyPath
-                    });
-                    // Full body into the FTS index — the parse above already
-                    // produced the text, so search covers body content even
-                    // for messages the user hasn't opened.
-                    if (ftsRowId && parsed.bodyText) this.db.updateFtsBody(ftsRowId, parsed.bodyText);
-
-                    newCount++;
-                }
-                this.db.commitTransaction();
+                        bodyPath,
+                    },
+                    ftsBody: parsed.bodyText || "",
+                });
+            }
+            // Phase 2 — synchronous transaction, no awaits inside. runInTxn
+            // owns BEGIN/COMMIT (or joins an open txn) and ROLLBACKs on throw.
+            try {
+                this.db.runInTxn(() => {
+                    for (const r of prepared) {
+                        const ftsRowId = this.db.upsertMessage(r.upsert);
+                        // Full body into FTS — parse above already produced the
+                        // text, so search covers unopened messages too.
+                        if (ftsRowId && r.ftsBody) this.db.updateFtsBody(ftsRowId, r.ftsBody);
+                    }
+                });
+                newCount += prepared.length;
             } catch (e: any) {
                 console.error(`    transaction error: ${e.message}`);
-                this.db.rollbackTransaction();
                 throw e;
             }
 
@@ -2700,6 +2710,22 @@ export class ImapManager extends EventEmitter {
                     if (expungeTimer) clearTimeout(expungeTimer);
                     expungeTimer = setTimeout(() => {
                         expungeTimer = null;
+                        // Skip while WE have sync actions draining. Every
+                        // delete/move we push EXPUNGEs on the server and bounces
+                        // straight back here as an onExpunge — and a reconcile
+                        // of a large INBOX is a full server-vs-local set-diff.
+                        // During a bulk delete that meant one ~135k-row reconcile
+                        // per message (the 2s debounce can't coalesce them — our
+                        // moves arrive ~2.5s apart), hammering the DB so the list
+                        // sat on "Loading…" with a stale preview (Bob 2026-06-12).
+                        // The local store already reflects our own moves, so these
+                        // are redundant. The last move empties the queue, so its
+                        // trailing expunge runs exactly ONE reconcile to catch any
+                        // genuinely-external change.
+                        if (this.db.getPendingSyncActions(accountId).length > 0) {
+                            console.log(`  [idle] ${accountId}: INBOX expunge during own drain — deferring reconcile`);
+                            return;
+                        }
                         const inbox = this.db.getFolders(accountId).find(f => f.specialUse === "inbox");
                         if (!inbox) return;
                         console.log(`  [idle] ${accountId}: INBOX expunge pushed → reconcile`);

package/packages/mailx-imap/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "@bobfrankston/mailx-imap",
-  "version": "0.1.86",
+  "version": "0.1.88",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@bobfrankston/mailx-imap",
-      "version": "0.1.86",
+      "version": "0.1.88",
       "license": "ISC",
       "dependencies": {
         "@bobfrankston/iflow-direct": "^0.1.27",

package/packages/mailx-imap/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bobfrankston/mailx-imap",
-  "version": "0.1.86",
+  "version": "0.1.88",
   "type": "module",
   "main": "index.js",
   "types": "index.d.ts",