@bobfrankston/mailx-imap 0.1.110 → 0.1.112

package/index.d.ts

@@ -604,7 +604,15 @@ export declare class ImapManager extends EventEmitter {
      *  Each per-UID step is its own withConnection({slow}) call so the queue
      *  yields between messages: a click-to-view body in the middle of a
      *  10-message outbox drain doesn't wait for all 10 to finish. */
+    /** Per-account re-entrancy guard for processOutbox. The worker tick and
+     *  the chained post-APPEND call (processLocalQueue) can overlap; on
+     *  servers without custom-keyword support ($Sending interlock) two
+     *  concurrent drains could both fetch and SMTP the same Outbox UID —
+     *  a double-send. One drain per account at a time; latecomers no-op
+     *  (the 10s tick retries anything they would have handled). */
+    private outboxDrainInFlight;
     processOutbox(accountId: string): Promise<void>;
+    private _processOutbox;
     /** Start background Outbox worker — runs immediately then every 10 seconds */
     private outboxBackoff;
     private outboxBackoffDelay;
@@ -660,6 +668,15 @@ export declare class ImapManager extends EventEmitter {
     private readonly SENT_SWEEP_INTERVAL_MS;
     startSentSweep(): void;
     stopSentSweep(): void;
+    /** Message-IDs sweepSentOnce has already re-APPENDed this daemon run.
+     *  HARD GUARD against perpetual motion: a re-appended copy syncs back in,
+     *  refreshes the row's cached_at, and keeps it inside the sweep window —
+     *  so if the header search ever reports "missing" again (it failed
+     *  silently for ~13 days on a flaky lane connection, 2026-07-02, 3,704
+     *  copies of one message), the sweep would append forever. One re-append
+     *  per Message-ID per daemon lifetime; a genuinely-lost message still
+     *  gets its one repair, and a false "missing" can no longer loop. */
+    private sentSweepAppended;
     private sweepSentOnce;
     private configWatchers;
     private cloudPollTimers;

package/index.js

@@ -1222,6 +1222,7 @@ export class ImapManager extends EventEmitter {
             uid: msg.uid,
             messageId: msg.messageId,
             date: msg.date || new Date(),
+            sentDate: msg.sentDate,
             subject: msg.subject,
             from: msg.from,
             to: msg.to,
@@ -1306,7 +1307,7 @@ export class ImapManager extends EventEmitter {
         return dbFolders;
     }
     /** Store a batch of messages to DB immediately — used by onChunk for incremental sync */
-    async storeMessages(accountId, folderId, folder, msgs, highestUid) {
+    async storeMessages(accountId, folderId, folder, msgs, highestUid, liveServerUids) {
         // Chunked transactions: better-sqlite3 is synchronous and an open
         // transaction blocks every other query on the same DB. A 1881-row
         // INBOX sync inside ONE transaction locked the connection for ~1.5 s
@@ -1376,12 +1377,14 @@ export class ImapManager extends EventEmitter {
                         inReplyTo: msg.inReplyTo || "",
                         references: [],
                         date: toFiniteDateMs(msg.date),
+                        sentDate: msg.sentDate instanceof Date ? toFiniteDateMs(msg.sentDate) : undefined,
                         subject: msg.subject || "",
                         from: toEmailAddress(msg.from?.[0] || {}),
                         to: toEmailAddresses(msg.to || []),
                         cc: toEmailAddresses(msg.cc || []),
                         flags, size: msg.size || 0, hasAttachments, preview, bodyPath,
                         exclusive: true, // IMAP path → one folder per message
+                        liveServerUids,
                     },
                     ftsBody,
                 });
@@ -1579,7 +1582,10 @@ export class ImapManager extends EventEmitter {
             const chunk = toFetch.slice(i, i + BACKFILL_CHUNK_SIZE);
             try {
                 const got = await client.fetchMessages(folderPath, chunk.join(","), { source: false });
-                recoveredCount += await this.storeMessages(accountId, folderId, folder, got, 0);
+                // Pass the server's live UID list so move-detect can tell a
+                // duplicate delivery (both uids live in this folder) from a
+                // real move — see upsertMessage.liveServerUids.
+                recoveredCount += await this.storeMessages(accountId, folderId, folder, got, 0, new Set(serverUids));
             }
             catch (e) {
                 console.error(`  ${folderPath}: backfill chunk ${i}-${i + chunk.length} failed (${e?.message || e}) — keeping ${recoveredCount} so far, resuming next cycle`);
@@ -2025,6 +2031,7 @@ export class ImapManager extends EventEmitter {
                         inReplyTo: msg.inReplyTo || "",
                         references: [],
                         date: toFiniteDateMs(msg.date),
+                        sentDate: msg.sentDate instanceof Date ? toFiniteDateMs(msg.sentDate) : undefined,
                         subject: msg.subject || "",
                         from: toEmailAddress(msg.from?.[0] || {}),
                         to: toEmailAddresses(msg.to || []),
@@ -2690,12 +2697,22 @@ export class ImapManager extends EventEmitter {
                     if (Number.isFinite(t))
                         dateMs = t;
                 }
+                // Sent time from the Date: header (iflow exposes it as sentDate,
+                // separate from msg.date = INTERNALDATE). Undefined → store
+                // defaults sent_date to dateMs.
+                let sentMs;
+                if (msg.sentDate instanceof Date) {
+                    const st = msg.sentDate.getTime();
+                    if (Number.isFinite(st))
+                        sentMs = st;
+                }
                 this.db.upsertMessage({
                     accountId, folderId, uid: msg.uid,
                     messageId: msg.messageId || "",
                     inReplyTo: msg.inReplyTo || "",
                     references: msg.references || [],
                     date: dateMs,
+                    sentDate: sentMs,
                     subject: msg.subject || "",
                     from: toEmailAddress(msg.from?.[0] || {}),
                     to: toEmailAddresses(msg.to || []),
@@ -5041,6 +5058,16 @@ export class ImapManager extends EventEmitter {
                         await withTimeout(client.appendMessage(outboxPath, raw, ["\\Seen"]), 60_000, client, `outbox APPEND ${file}`);
                         fs.renameSync(claimedPath, path.join(sentDir, file));
                         console.log(`  [outbox] Moved ${file} to IMAP Outbox → sent/`);
+                        // Chain the SMTP delivery immediately instead of waiting
+                        // for the next worker tick. The APPEND→tick→SMTP→Sent
+                        // chain added up to ~30s before the message appeared in
+                        // Sent, which read as "my message is gone" (Bob
+                        // 2026-07-01, repeatedly). Fire-and-forget; the tick
+                        // remains the retry backstop.
+                        setImmediate(() => {
+                            this.processOutbox(accountId)
+                                .catch((e) => console.error(`  [outbox] chained processOutbox: ${e?.message || e}`));
+                        });
                     }
                     catch (e) {
                         // APPEND failed (connection dropped mid-send, server
@@ -5127,8 +5154,17 @@ export class ImapManager extends EventEmitter {
             throw new Error("No recipients");
         // Dedup: skip if this Message-ID has already been sent. Prevents the
         // outbox from re-sending the same file across crash/restart cycles.
-        const messageId = messageIdMatch ? messageIdMatch[1] : "";
-        if (messageId && this.db.hasSentMessage(messageId)) {
+        // A message WITHOUT a Message-ID gets one synthesized here — the old
+        // `if (messageId && …)` guard silently skipped BOTH the dedup check
+        // AND recordSent for ID-less mail, so such a message could re-send
+        // forever with no brake (2026-07-02 resend-loop audit).
+        let messageId = messageIdMatch ? messageIdMatch[1] : "";
+        if (!messageId) {
+            messageId = `<mailx-${Date.now()}-${Math.random().toString(36).slice(2, 10)}@${account.email.split("@")[1] || "mailx.local"}>`;
+            raw = raw.replace(/\r?\n\r?\n/, `\r\nMessage-ID: ${messageId}\r\n\r\n`);
+            console.log(`  [smtp] ${accountId}: synthesized Message-ID ${messageId} for ID-less outgoing message`);
+        }
+        if (this.db.hasSentMessage(messageId)) {
             console.log(`  [smtp] ${accountId}: SKIP ${messageId} — already in sent_log`);
             return;
         }
@@ -5163,7 +5199,25 @@ export class ImapManager extends EventEmitter {
      *  Each per-UID step is its own withConnection({slow}) call so the queue
      *  yields between messages: a click-to-view body in the middle of a
      *  10-message outbox drain doesn't wait for all 10 to finish. */
+    /** Per-account re-entrancy guard for processOutbox. The worker tick and
+     *  the chained post-APPEND call (processLocalQueue) can overlap; on
+     *  servers without custom-keyword support ($Sending interlock) two
+     *  concurrent drains could both fetch and SMTP the same Outbox UID —
+     *  a double-send. One drain per account at a time; latecomers no-op
+     *  (the 10s tick retries anything they would have handled). */
+    outboxDrainInFlight = new Set();
     async processOutbox(accountId) {
+        if (this.outboxDrainInFlight.has(accountId))
+            return;
+        this.outboxDrainInFlight.add(accountId);
+        try {
+            await this._processOutbox(accountId);
+        }
+        finally {
+            this.outboxDrainInFlight.delete(accountId);
+        }
+    }
+    async _processOutbox(accountId) {
         const outboxFolder = this.findFolder(accountId, "outbox");
         if (!outboxFolder)
             return;
@@ -5553,6 +5607,15 @@ export class ImapManager extends EventEmitter {
             this.sentSweepInterval = null;
         }
     }
+    /** Message-IDs sweepSentOnce has already re-APPENDed this daemon run.
+     *  HARD GUARD against perpetual motion: a re-appended copy syncs back in,
+     *  refreshes the row's cached_at, and keeps it inside the sweep window —
+     *  so if the header search ever reports "missing" again (it failed
+     *  silently for ~13 days on a flaky lane connection, 2026-07-02, 3,704
+     *  copies of one message), the sweep would append forever. One re-append
+     *  per Message-ID per daemon lifetime; a genuinely-lost message still
+     *  gets its one repair, and a false "missing" can no longer loop. */
+    sentSweepAppended = new Set();
     async sweepSentOnce(accountId) {
         const sent = this.findFolder(accountId, "sent");
         if (!sent)
@@ -5578,7 +5641,11 @@ export class ImapManager extends EventEmitter {
         //     a finally, the opposite of "dedicated".
         await this.withConnection(accountId, async (client) => {
             for (const row of rows) {
-                const msgId = row.message_id;
+                // Trim defensively: rows written before the iflow-direct
+                // envelope-parser trim carry the header's fold whitespace
+                // ("\t<...>"), which made HEADER searches return 0 and drove
+                // the re-APPEND loop (2026-07-02, 3,707 duplicates).
+                const msgId = (row.message_id || "").trim();
                 if (!msgId)
                     continue;
                 let serverUids = [];
@@ -5590,13 +5657,32 @@ export class ImapManager extends EventEmitter {
                     continue;
                 }
                 if (serverUids.length === 0) {
-                    // Not on server. Re-APPEND from .eml if we have one.
+                    // Not on server. Re-APPEND from .eml if we have one —
+                    // but at most ONCE per Message-ID per daemon run (see
+                    // sentSweepAppended above; the 3,704-duplicate loop).
                     if (!row.body_path)
                         continue;
+                    if (this.sentSweepAppended.has(msgId)) {
+                        console.error(`  [sent-sweep] ${accountId}: SKIP re-append of ${msgId} — already re-appended once this run; header search reporting it missing AGAIN means the search is unreliable, not the message lost`);
+                        continue;
+                    }
+                    // Cross-check the local DB before trusting the server
+                    // search: multiple local Sent rows with this Message-ID
+                    // mean prior copies synced IN from the server — it exists
+                    // there regardless of what the search just said.
+                    try {
+                        const localCopies = this.db.countMessagesByMessageId?.(accountId, sent.id, msgId) ?? 0;
+                        if (localCopies > 1) {
+                            console.error(`  [sent-sweep] ${accountId}: SKIP re-append of ${msgId} — ${localCopies} local Sent rows already exist (server search returned 0 falsely)`);
+                            continue;
+                        }
+                    }
+                    catch { /* count helper missing — proceed to the once-guard */ }
                     try {
                         const buf = await this.bodyStore.readByPath(row.body_path);
                         if (!buf)
                             continue;
+                        this.sentSweepAppended.add(msgId);
                         await client.appendMessage(sent.path, buf, ["\\Seen"]);
                         appended++;
                         console.log(`  [sent-sweep] ${accountId}: re-APPENDed ${msgId} (was missing on server)`);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bobfrankston/mailx-imap",
-  "version": "0.1.110",
+  "version": "0.1.112",
   "type": "module",
   "main": "index.js",
   "types": "index.d.ts",
@@ -9,13 +9,13 @@
   },
   "license": "ISC",
   "dependencies": {
-    "@bobfrankston/mailx-types": "^0.1.20",
-    "@bobfrankston/mailx-settings": "^0.1.32",
-    "@bobfrankston/mailx-store": "^0.1.56",
-    "@bobfrankston/iflow-direct": "^0.1.55",
+    "@bobfrankston/mailx-types": "^0.1.22",
+    "@bobfrankston/mailx-settings": "^0.1.33",
+    "@bobfrankston/mailx-store": "^0.1.57",
+    "@bobfrankston/iflow-direct": "^0.1.56",
     "@bobfrankston/tcp-transport": "^0.1.7",
     "@bobfrankston/smtp-direct": "^0.1.9",
-    "@bobfrankston/mailx-sync": "^0.1.23",
+    "@bobfrankston/mailx-sync": "^0.1.27",
     "@bobfrankston/oauthsupport": "^1.0.33"
   },
   "repository": {
@@ -37,13 +37,13 @@
   },
   ".transformedSnapshot": {
     "dependencies": {
-      "@bobfrankston/mailx-types": "^0.1.20",
-      "@bobfrankston/mailx-settings": "^0.1.32",
-      "@bobfrankston/mailx-store": "^0.1.56",
-      "@bobfrankston/iflow-direct": "^0.1.55",
+      "@bobfrankston/mailx-types": "^0.1.22",
+      "@bobfrankston/mailx-settings": "^0.1.33",
+      "@bobfrankston/mailx-store": "^0.1.57",
+      "@bobfrankston/iflow-direct": "^0.1.56",
       "@bobfrankston/tcp-transport": "^0.1.7",
       "@bobfrankston/smtp-direct": "^0.1.9",
-      "@bobfrankston/mailx-sync": "^0.1.23",
+      "@bobfrankston/mailx-sync": "^0.1.27",
       "@bobfrankston/oauthsupport": "^1.0.33"
     }
   }