@bobfrankston/rmfmail 1.1.238 → 1.1.240

package/packages/mailx-imap/index.js

@@ -986,20 +986,11 @@ export class ImapManager extends EventEmitter {
         // worth of latency instead of the full loop's worth.
         const BATCH_SIZE = 50;
         let stored = 0;
-        let inTxn = false;
-        const startTxn = () => { this.db.beginTransaction(); inTxn = true; };
-        const commitTxn = () => { if (inTxn) {
-            this.db.commitTransaction();
-            inTxn = false;
-        } };
-        const rollbackTxn = () => { if (inTxn) {
-            this.db.rollbackTransaction();
-            inTxn = false;
-        } };
-        try {
-            startTxn();
-            let batchCount = 0;
-            for (const msg of msgs) {
+        for (let i = 0; i < msgs.length; i += BATCH_SIZE) {
+            const chunk = msgs.slice(i, i + BATCH_SIZE);
+            // Phase 1 — async prep, NO transaction held.
+            const prepared = [];
+            for (const msg of chunk) {
                 // Debug: log subjects with non-ASCII to trace encoding issues
                 if (msg.subject && /[^\x00-\x7F]/.test(msg.subject)) {
                     const hex = Buffer.from(msg.subject, "utf-8").subarray(0, 40).toString("hex");
@@ -1046,43 +1037,45 @@ export class ImapManager extends EventEmitter {
                     flags.push("\\Answered");
                 if (msg.draft)
                     flags.push("\\Draft");
-                const ftsRowId = this.db.upsertMessage({
-                    accountId, folderId, uid: msg.uid,
-                    messageId: msg.messageId || "",
-                    inReplyTo: msg.inReplyTo || "",
-                    references: [],
-                    date: toFiniteDateMs(msg.date),
-                    subject: msg.subject || "",
-                    from: toEmailAddress(msg.from?.[0] || {}),
-                    to: toEmailAddresses(msg.to || []),
-                    cc: toEmailAddresses(msg.cc || []),
-                    flags, size: msg.size || 0, hasAttachments, preview, bodyPath
+                prepared.push({
+                    upsert: {
+                        accountId, folderId, uid: msg.uid,
+                        messageId: msg.messageId || "",
+                        inReplyTo: msg.inReplyTo || "",
+                        references: [],
+                        date: toFiniteDateMs(msg.date),
+                        subject: msg.subject || "",
+                        from: toEmailAddress(msg.from?.[0] || {}),
+                        to: toEmailAddresses(msg.to || []),
+                        cc: toEmailAddresses(msg.cc || []),
+                        flags, size: msg.size || 0, hasAttachments, preview, bodyPath,
+                    },
+                    ftsBody,
                 });
-                // Index the full body for search. extractPreview already
-                // parsed `source`, so the body text is free here — and
-                // indexing it at sync time means a word buried in the body
-                // is searchable without the user first opening the message.
-                if (ftsRowId && ftsBody)
-                    this.db.updateFtsBody(ftsRowId, ftsBody);
-                stored++;
-                batchCount++;
-                if (batchCount >= BATCH_SIZE) {
-                    commitTxn();
-                    // Hand the event loop to any waiting IPC (user clicks,
-                    // outbox drains, alarm polls). setImmediate runs AFTER
-                    // pending I/O callbacks, so a stack of getMessage IPCs
-                    // that arrived during the previous chunk gets serviced
-                    // before we start the next batch.
-                    await new Promise(r => setImmediate(r));
-                    batchCount = 0;
-                    startTxn();
-                }
             }
-            commitTxn();
-        }
-        catch (e) {
-            rollbackTxn();
-            console.error(`  storeMessages error: ${e.message}`);
+            // Phase 2 — synchronous transaction, no awaits inside. runInTxn
+            // owns its own BEGIN/COMMIT (or joins an already-open txn) and
+            // ROLLBACKs on any throw, so a bad row can't leak an open txn.
+            try {
+                this.db.runInTxn(() => {
+                    for (const r of prepared) {
+                        const ftsRowId = this.db.upsertMessage(r.upsert);
+                        // Index the full body for search. extractPreview already
+                        // parsed `source`, so the body text is free here — and
+                        // indexing it at sync time means a word buried in the
+                        // body is searchable without first opening the message.
+                        if (ftsRowId && r.ftsBody)
+                            this.db.updateFtsBody(ftsRowId, r.ftsBody);
+                    }
+                });
+                stored += prepared.length;
+            }
+            catch (e) {
+                console.error(`  storeMessages error: ${e.message}`);
+            }
+            // Hand the event loop to any waiting IPC (user clicks, outbox
+            // drains, alarm polls) between batches.
+            await new Promise(r => setImmediate(r));
         }
         return stored;
     }
@@ -1561,45 +1554,43 @@ export class ImapManager extends EventEmitter {
         const batchSize = 50;
         for (let batchStart = 0; batchStart < messages.length; batchStart += batchSize) {
             const batchEnd = Math.min(batchStart + batchSize, messages.length);
-            this.db.beginTransaction();
-            try {
-                for (let i = batchStart; i < batchEnd; i++) {
-                    const msg = messages[i];
-                    // CRITICAL: was `if (msg.uid <= highestUid) { update flags; continue }`.
-                    // Same bug as the streamy storeMessages path on line ~861:
-                    // it dropped every gap-fill message because gap-fills
-                    // recover UIDs in the already-scanned range (all <=
-                    // highestUid by definition). Trust upsertMessage's
-                    // UNIQUE constraint to dedupe — if mailx truly has the
-                    // row, the upsert becomes an UPDATE that refreshes
-                    // flags too, all in one path.
-                    // Tombstone-skip: in-flight local delete/move waiting
-                    // for server confirmation. Cleared on action success or
-                    // permanent failure (see clearTombstoneForUid).
-                    if (msg.messageId && this.db.hasTombstone(accountId, msg.messageId)) {
-                        console.log(`  [tombstone] ${accountId}: skipping ${msg.messageId} in syncFolder (locally deleted)`);
-                        continue;
-                    }
-                    // Store body
-                    const source = msg.source || "";
-                    let bodyPath = "";
-                    if (source) {
-                        bodyPath = await this.bodyStore.putMessage(accountId, folderId, msg.uid, Buffer.from(source, "utf-8"));
-                    }
-                    // Parse for preview and attachment info
-                    const parsed = await extractPreview(source);
-                    // Build flags array
-                    const flags = [];
-                    if (msg.seen)
-                        flags.push("\\Seen");
-                    if (msg.flagged)
-                        flags.push("\\Flagged");
-                    if (msg.answered)
-                        flags.push("\\Answered");
-                    if (msg.draft)
-                        flags.push("\\Draft");
-                    // Store metadata
-                    const ftsRowId = this.db.upsertMessage({
+            // Phase 1 — async prep (write .eml, parse preview), NO transaction
+            // held. The previous version kept a raw BEGIN open across
+            // `await putMessage` / `await extractPreview`, so a concurrent
+            // store on the same connection could hit BEGIN mid-await →
+            // "cannot start a transaction within a transaction", wedging the
+            // sync. Same fix + cause as storeMessages above (Bob 2026-06-11).
+            //
+            // NOTE: the old `if (msg.uid <= highestUid) continue` skip is
+            // intentionally gone — gap-fills recover UIDs in the scanned range,
+            // and upsertMessage's UNIQUE(account,folder,uid) constraint dedupes
+            // (truly-present rows become a cheap UPDATE).
+            const prepared = [];
+            for (let i = batchStart; i < batchEnd; i++) {
+                const msg = messages[i];
+                // Tombstone-skip: in-flight local delete/move awaiting server
+                // confirmation (cleared on success or permanent failure).
+                if (msg.messageId && this.db.hasTombstone(accountId, msg.messageId)) {
+                    console.log(`  [tombstone] ${accountId}: skipping ${msg.messageId} in syncFolder (locally deleted)`);
+                    continue;
+                }
+                const source = msg.source || "";
+                let bodyPath = "";
+                if (source) {
+                    bodyPath = await this.bodyStore.putMessage(accountId, folderId, msg.uid, Buffer.from(source, "utf-8"));
+                }
+                const parsed = await extractPreview(source);
+                const flags = [];
+                if (msg.seen)
+                    flags.push("\\Seen");
+                if (msg.flagged)
+                    flags.push("\\Flagged");
+                if (msg.answered)
+                    flags.push("\\Answered");
+                if (msg.draft)
+                    flags.push("\\Draft");
+                prepared.push({
+                    upsert: {
                         accountId,
                         folderId,
                         uid: msg.uid,
@@ -1615,20 +1606,27 @@ export class ImapManager extends EventEmitter {
                         size: msg.size || 0,
                         hasAttachments: parsed.hasAttachments,
                         preview: parsed.preview,
-                        bodyPath
-                    });
-                    // Full body into the FTS index — the parse above already
-                    // produced the text, so search covers body content even
-                    // for messages the user hasn't opened.
-                    if (ftsRowId && parsed.bodyText)
-                        this.db.updateFtsBody(ftsRowId, parsed.bodyText);
-                    newCount++;
-                }
-                this.db.commitTransaction();
+                        bodyPath,
+                    },
+                    ftsBody: parsed.bodyText || "",
+                });
+            }
+            // Phase 2 — synchronous transaction, no awaits inside. runInTxn
+            // owns BEGIN/COMMIT (or joins an open txn) and ROLLBACKs on throw.
+            try {
+                this.db.runInTxn(() => {
+                    for (const r of prepared) {
+                        const ftsRowId = this.db.upsertMessage(r.upsert);
+                        // Full body into FTS — parse above already produced the
+                        // text, so search covers unopened messages too.
+                        if (ftsRowId && r.ftsBody)
+                            this.db.updateFtsBody(ftsRowId, r.ftsBody);
+                    }
+                });
+                newCount += prepared.length;
             }
             catch (e) {
                 console.error(`    transaction error: ${e.message}`);
-                this.db.rollbackTransaction();
                 throw e;
             }
             // Emit progress and notify client after each batch