@bobfrankston/mailx-imap 0.1.85 → 0.1.87

package/index.js

@@ -986,20 +986,11 @@ export class ImapManager extends EventEmitter {
         // worth of latency instead of the full loop's worth.
         const BATCH_SIZE = 50;
         let stored = 0;
-        let inTxn = false;
-        const startTxn = () => { this.db.beginTransaction(); inTxn = true; };
-        const commitTxn = () => { if (inTxn) {
-            this.db.commitTransaction();
-            inTxn = false;
-        } };
-        const rollbackTxn = () => { if (inTxn) {
-            this.db.rollbackTransaction();
-            inTxn = false;
-        } };
-        try {
-            startTxn();
-            let batchCount = 0;
-            for (const msg of msgs) {
+        for (let i = 0; i < msgs.length; i += BATCH_SIZE) {
+            const chunk = msgs.slice(i, i + BATCH_SIZE);
+            // Phase 1 — async prep, NO transaction held.
+            const prepared = [];
+            for (const msg of chunk) {
                 // Debug: log subjects with non-ASCII to trace encoding issues
                 if (msg.subject && /[^\x00-\x7F]/.test(msg.subject)) {
                     const hex = Buffer.from(msg.subject, "utf-8").subarray(0, 40).toString("hex");
@@ -1046,43 +1037,45 @@ export class ImapManager extends EventEmitter {
                     flags.push("\\Answered");
                 if (msg.draft)
                     flags.push("\\Draft");
-                const ftsRowId = this.db.upsertMessage({
-                    accountId, folderId, uid: msg.uid,
-                    messageId: msg.messageId || "",
-                    inReplyTo: msg.inReplyTo || "",
-                    references: [],
-                    date: toFiniteDateMs(msg.date),
-                    subject: msg.subject || "",
-                    from: toEmailAddress(msg.from?.[0] || {}),
-                    to: toEmailAddresses(msg.to || []),
-                    cc: toEmailAddresses(msg.cc || []),
-                    flags, size: msg.size || 0, hasAttachments, preview, bodyPath
+                prepared.push({
+                    upsert: {
+                        accountId, folderId, uid: msg.uid,
+                        messageId: msg.messageId || "",
+                        inReplyTo: msg.inReplyTo || "",
+                        references: [],
+                        date: toFiniteDateMs(msg.date),
+                        subject: msg.subject || "",
+                        from: toEmailAddress(msg.from?.[0] || {}),
+                        to: toEmailAddresses(msg.to || []),
+                        cc: toEmailAddresses(msg.cc || []),
+                        flags, size: msg.size || 0, hasAttachments, preview, bodyPath,
+                    },
+                    ftsBody,
                 });
-                // Index the full body for search. extractPreview already
-                // parsed `source`, so the body text is free here — and
-                // indexing it at sync time means a word buried in the body
-                // is searchable without the user first opening the message.
-                if (ftsRowId && ftsBody)
-                    this.db.updateFtsBody(ftsRowId, ftsBody);
-                stored++;
-                batchCount++;
-                if (batchCount >= BATCH_SIZE) {
-                    commitTxn();
-                    // Hand the event loop to any waiting IPC (user clicks,
-                    // outbox drains, alarm polls). setImmediate runs AFTER
-                    // pending I/O callbacks, so a stack of getMessage IPCs
-                    // that arrived during the previous chunk gets serviced
-                    // before we start the next batch.
-                    await new Promise(r => setImmediate(r));
-                    batchCount = 0;
-                    startTxn();
-                }
             }
-            commitTxn();
-        }
-        catch (e) {
-            rollbackTxn();
-            console.error(`  storeMessages error: ${e.message}`);
+            // Phase 2 — synchronous transaction, no awaits inside. runInTxn
+            // owns its own BEGIN/COMMIT (or joins an already-open txn) and
+            // ROLLBACKs on any throw, so a bad row can't leak an open txn.
+            try {
+                this.db.runInTxn(() => {
+                    for (const r of prepared) {
+                        const ftsRowId = this.db.upsertMessage(r.upsert);
+                        // Index the full body for search. extractPreview already
+                        // parsed `source`, so the body text is free here — and
+                        // indexing it at sync time means a word buried in the
+                        // body is searchable without first opening the message.
+                        if (ftsRowId && r.ftsBody)
+                            this.db.updateFtsBody(ftsRowId, r.ftsBody);
+                    }
+                });
+                stored += prepared.length;
+            }
+            catch (e) {
+                console.error(`  storeMessages error: ${e.message}`);
+            }
+            // Hand the event loop to any waiting IPC (user clicks, outbox
+            // drains, alarm polls) between batches.
+            await new Promise(r => setImmediate(r));
         }
         return stored;
     }
@@ -1561,45 +1554,43 @@ export class ImapManager extends EventEmitter {
         const batchSize = 50;
         for (let batchStart = 0; batchStart < messages.length; batchStart += batchSize) {
             const batchEnd = Math.min(batchStart + batchSize, messages.length);
-            this.db.beginTransaction();
-            try {
-                for (let i = batchStart; i < batchEnd; i++) {
-                    const msg = messages[i];
-                    // CRITICAL: was `if (msg.uid <= highestUid) { update flags; continue }`.
-                    // Same bug as the streamy storeMessages path on line ~861:
-                    // it dropped every gap-fill message because gap-fills
-                    // recover UIDs in the already-scanned range (all <=
-                    // highestUid by definition). Trust upsertMessage's
-                    // UNIQUE constraint to dedupe — if mailx truly has the
-                    // row, the upsert becomes an UPDATE that refreshes
-                    // flags too, all in one path.
-                    // Tombstone-skip: in-flight local delete/move waiting
-                    // for server confirmation. Cleared on action success or
-                    // permanent failure (see clearTombstoneForUid).
-                    if (msg.messageId && this.db.hasTombstone(accountId, msg.messageId)) {
-                        console.log(`  [tombstone] ${accountId}: skipping ${msg.messageId} in syncFolder (locally deleted)`);
-                        continue;
-                    }
-                    // Store body
-                    const source = msg.source || "";
-                    let bodyPath = "";
-                    if (source) {
-                        bodyPath = await this.bodyStore.putMessage(accountId, folderId, msg.uid, Buffer.from(source, "utf-8"));
-                    }
-                    // Parse for preview and attachment info
-                    const parsed = await extractPreview(source);
-                    // Build flags array
-                    const flags = [];
-                    if (msg.seen)
-                        flags.push("\\Seen");
-                    if (msg.flagged)
-                        flags.push("\\Flagged");
-                    if (msg.answered)
-                        flags.push("\\Answered");
-                    if (msg.draft)
-                        flags.push("\\Draft");
-                    // Store metadata
-                    const ftsRowId = this.db.upsertMessage({
+            // Phase 1 — async prep (write .eml, parse preview), NO transaction
+            // held. The previous version kept a raw BEGIN open across
+            // `await putMessage` / `await extractPreview`, so a concurrent
+            // store on the same connection could hit BEGIN mid-await →
+            // "cannot start a transaction within a transaction", wedging the
+            // sync. Same fix + cause as storeMessages above (Bob 2026-06-11).
+            //
+            // NOTE: the old `if (msg.uid <= highestUid) continue` skip is
+            // intentionally gone — gap-fills recover UIDs in the scanned range,
+            // and upsertMessage's UNIQUE(account,folder,uid) constraint dedupes
+            // (truly-present rows become a cheap UPDATE).
+            const prepared = [];
+            for (let i = batchStart; i < batchEnd; i++) {
+                const msg = messages[i];
+                // Tombstone-skip: in-flight local delete/move awaiting server
+                // confirmation (cleared on success or permanent failure).
+                if (msg.messageId && this.db.hasTombstone(accountId, msg.messageId)) {
+                    console.log(`  [tombstone] ${accountId}: skipping ${msg.messageId} in syncFolder (locally deleted)`);
+                    continue;
+                }
+                const source = msg.source || "";
+                let bodyPath = "";
+                if (source) {
+                    bodyPath = await this.bodyStore.putMessage(accountId, folderId, msg.uid, Buffer.from(source, "utf-8"));
+                }
+                const parsed = await extractPreview(source);
+                const flags = [];
+                if (msg.seen)
+                    flags.push("\\Seen");
+                if (msg.flagged)
+                    flags.push("\\Flagged");
+                if (msg.answered)
+                    flags.push("\\Answered");
+                if (msg.draft)
+                    flags.push("\\Draft");
+                prepared.push({
+                    upsert: {
                         accountId,
                         folderId,
                         uid: msg.uid,
@@ -1615,20 +1606,27 @@ export class ImapManager extends EventEmitter {
                         size: msg.size || 0,
                         hasAttachments: parsed.hasAttachments,
                         preview: parsed.preview,
-                        bodyPath
-                    });
-                    // Full body into the FTS index — the parse above already
-                    // produced the text, so search covers body content even
-                    // for messages the user hasn't opened.
-                    if (ftsRowId && parsed.bodyText)
-                        this.db.updateFtsBody(ftsRowId, parsed.bodyText);
-                    newCount++;
-                }
-                this.db.commitTransaction();
+                        bodyPath,
+                    },
+                    ftsBody: parsed.bodyText || "",
+                });
+            }
+            // Phase 2 — synchronous transaction, no awaits inside. runInTxn
+            // owns BEGIN/COMMIT (or joins an open txn) and ROLLBACKs on throw.
+            try {
+                this.db.runInTxn(() => {
+                    for (const r of prepared) {
+                        const ftsRowId = this.db.upsertMessage(r.upsert);
+                        // Full body into FTS — parse above already produced the
+                        // text, so search covers unopened messages too.
+                        if (ftsRowId && r.ftsBody)
+                            this.db.updateFtsBody(ftsRowId, r.ftsBody);
+                    }
+                });
+                newCount += prepared.length;
             }
             catch (e) {
                 console.error(`    transaction error: ${e.message}`);
-                this.db.rollbackTransaction();
                 throw e;
             }
             // Emit progress and notify client after each batch
@@ -4193,19 +4191,37 @@ export class ImapManager extends EventEmitter {
                 if (host !== this.hostname)
                     continue;
                 const pid = parseInt(pidStr);
-                if (pid === myPid)
-                    continue; // it's us
+                let ageMs = Infinity;
+                try {
+                    ageMs = Date.now() - fs.statSync(path.join(dir, f)).mtimeMs;
+                }
+                catch { /* */ }
+                if (pid === myPid) {
+                    // Our own claim. Normally we're actively sending it — leave
+                    // it. But the send is now bounded (60s APPEND timeout), so
+                    // an OWN claim older than STALE_CLAIM_MS means the send
+                    // hung past its timeout, or a prior tick orphaned it (the
+                    // release rename failed). Without reclaiming it the file
+                    // sits in `.sending-` forever — recovery used to skip every
+                    // own-PID claim unconditionally, so a transient connection
+                    // wedge pinned the message even after the link recovered
+                    // (Bob 2026-06-11: two messages stuck .sending-<ourpid>
+                    // while SELECT Outbox was already succeeding again).
+                    if (ageMs < STALE_CLAIM_MS)
+                        continue;
+                    try {
+                        fs.renameSync(path.join(dir, f), path.join(dir, original));
+                        console.log(`  [outbox] Recovered our own stale claim ${f} → ${original} (hung ${Math.round(ageMs / 60_000)}m)`);
+                    }
+                    catch { /* ignore */ }
+                    continue;
+                }
                 let alive = false;
                 try {
                     process.kill(pid, 0);
                     alive = true;
                 }
                 catch { /* dead */ }
-                let ageMs = Infinity;
-                try {
-                    ageMs = Date.now() - fs.statSync(path.join(dir, f)).mtimeMs;
-                }
-                catch { /* */ }
                 // Live PID + recent mtime → genuine sibling owner, leave it.
                 // Live PID + ancient mtime → recycled PID, sweep. Dead PID → sweep.
                 if (alive && ageMs < STALE_CLAIM_MS)
@@ -4338,7 +4354,15 @@ export class ImapManager extends EventEmitter {
                     }
                     try {
                         const raw = fs.readFileSync(claimedPath, "utf-8");
-                        await client.appendMessage(outboxPath, raw, ["\\Seen"]);
+                        // Bound the APPEND. On a wedged connection (Dovecot
+                        // ETIMEDOUT storm) the bare await can hang the full
+                        // 300s inactivity timeout, pinning the file in
+                        // `.sending-` state the whole time and reading to the
+                        // user as "stuck, not sending" (Bob 2026-06-11). A
+                        // 60s cap force-closes the socket and throws, so the
+                        // catch below releases the claim and the next tick
+                        // retries instead of hanging for 5 minutes.
+                        await withTimeout(client.appendMessage(outboxPath, raw, ["\\Seen"]), 60_000, client, `outbox APPEND ${file}`);
                         fs.renameSync(claimedPath, path.join(sentDir, file));
                         console.log(`  [outbox] Moved ${file} to IMAP Outbox → sent/`);
                     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bobfrankston/mailx-imap",
-  "version": "0.1.85",
+  "version": "0.1.87",
   "type": "module",
   "main": "index.js",
   "types": "index.d.ts",
@@ -9,14 +9,14 @@
   },
   "license": "ISC",
   "dependencies": {
-    "@bobfrankston/mailx-types": "^0.1.18",
+    "@bobfrankston/mailx-types": "^0.1.19",
     "@bobfrankston/mailx-settings": "^0.1.26",
-    "@bobfrankston/mailx-store": "^0.1.45",
-    "@bobfrankston/iflow-direct": "^0.1.52",
-    "@bobfrankston/tcp-transport": "^0.1.6",
-    "@bobfrankston/smtp-direct": "^0.1.8",
-    "@bobfrankston/mailx-sync": "^0.1.19",
-    "@bobfrankston/oauthsupport": "^1.0.31"
+    "@bobfrankston/mailx-store": "^0.1.46",
+    "@bobfrankston/iflow-direct": "^0.1.53",
+    "@bobfrankston/tcp-transport": "^0.1.7",
+    "@bobfrankston/smtp-direct": "^0.1.9",
+    "@bobfrankston/mailx-sync": "^0.1.20",
+    "@bobfrankston/oauthsupport": "^1.0.32"
   },
   "repository": {
     "type": "git",
@@ -37,14 +37,14 @@
   },
   ".transformedSnapshot": {
     "dependencies": {
-      "@bobfrankston/mailx-types": "^0.1.18",
+      "@bobfrankston/mailx-types": "^0.1.19",
       "@bobfrankston/mailx-settings": "^0.1.26",
-      "@bobfrankston/mailx-store": "^0.1.45",
-      "@bobfrankston/iflow-direct": "^0.1.52",
-      "@bobfrankston/tcp-transport": "^0.1.6",
-      "@bobfrankston/smtp-direct": "^0.1.8",
-      "@bobfrankston/mailx-sync": "^0.1.19",
-      "@bobfrankston/oauthsupport": "^1.0.31"
+      "@bobfrankston/mailx-store": "^0.1.46",
+      "@bobfrankston/iflow-direct": "^0.1.53",
+      "@bobfrankston/tcp-transport": "^0.1.7",
+      "@bobfrankston/smtp-direct": "^0.1.9",
+      "@bobfrankston/mailx-sync": "^0.1.20",
+      "@bobfrankston/oauthsupport": "^1.0.32"
     }
   }
 }