@bobfrankston/mailx-imap 0.1.109 → 0.1.111

package/index.d.ts

@@ -604,7 +604,15 @@ export declare class ImapManager extends EventEmitter {
      *  Each per-UID step is its own withConnection({slow}) call so the queue
      *  yields between messages: a click-to-view body in the middle of a
      *  10-message outbox drain doesn't wait for all 10 to finish. */
+    /** Per-account re-entrancy guard for processOutbox. The worker tick and
+     *  the chained post-APPEND call (processLocalQueue) can overlap; on
+     *  servers without custom-keyword support ($Sending interlock) two
+     *  concurrent drains could both fetch and SMTP the same Outbox UID —
+     *  a double-send. One drain per account at a time; latecomers no-op
+     *  (the 10s tick retries anything they would have handled). */
+    private outboxDrainInFlight;
     processOutbox(accountId: string): Promise<void>;
+    private _processOutbox;
     /** Start background Outbox worker — runs immediately then every 10 seconds */
     private outboxBackoff;
     private outboxBackoffDelay;
@@ -660,6 +668,15 @@ export declare class ImapManager extends EventEmitter {
     private readonly SENT_SWEEP_INTERVAL_MS;
     startSentSweep(): void;
     stopSentSweep(): void;
+    /** Message-IDs sweepSentOnce has already re-APPENDed this daemon run.
+     *  HARD GUARD against perpetual motion: a re-appended copy syncs back in,
+     *  refreshes the row's cached_at, and keeps it inside the sweep window —
+     *  so if the header search ever reports "missing" again (it failed
+     *  silently for ~13 days on a flaky lane connection, 2026-07-02, 3,704
+     *  copies of one message), the sweep would append forever. One re-append
+     *  per Message-ID per daemon lifetime; a genuinely-lost message still
+     *  gets its one repair, and a false "missing" can no longer loop. */
+    private sentSweepAppended;
     private sweepSentOnce;
     private configWatchers;
     private cloudPollTimers;

package/index.js

@@ -1222,6 +1222,7 @@ export class ImapManager extends EventEmitter {
             uid: msg.uid,
             messageId: msg.messageId,
             date: msg.date || new Date(),
+            sentDate: msg.sentDate,
             subject: msg.subject,
             from: msg.from,
             to: msg.to,
@@ -1376,6 +1377,7 @@ export class ImapManager extends EventEmitter {
                         inReplyTo: msg.inReplyTo || "",
                         references: [],
                         date: toFiniteDateMs(msg.date),
+                        sentDate: msg.sentDate instanceof Date ? toFiniteDateMs(msg.sentDate) : undefined,
                         subject: msg.subject || "",
                         from: toEmailAddress(msg.from?.[0] || {}),
                         to: toEmailAddresses(msg.to || []),
@@ -1764,7 +1766,18 @@ export class ImapManager extends EventEmitter {
                     // Apply changed-state FETCH — flag updates since prior modSeq.
                     for (const m of qr.changedMessages) {
                         try {
+                            // LOCAL-FIRST: never let a server reconcile clobber a flag
+                            // the user just changed locally but whose push hasn't drained
+                            // yet — that's the "I unflag it and it quickly comes back"
+                            // bug (Bob 2026-06-27). Skip any message with a pending flag
+                            // action; the drain will push it and a later reconcile syncs.
+                            const pendingFlag = this.db.findPendingSyncAction(accountId, "flags", m.uid, folderId);
+                            if (pendingFlag) {
+                                console.error(`  [flag-reconcile] SKIP uid=${m.uid} folder=${folderId} — pending local flag action wins`);
+                                continue;
+                            }
                             const flagsArr = Array.from(m.flags || []).map(f => String(f));
+                            console.error(`  [flag-reconcile] OVERWRITE uid=${m.uid} folder=${folderId} server=[${flagsArr.join(",")}]`);
                             this.db.updateMessageFlags(accountId, folderId, m.uid, flagsArr);
                         }
                         catch { /* row may have just been VANISHED */ }
@@ -2014,6 +2027,7 @@ export class ImapManager extends EventEmitter {
                         inReplyTo: msg.inReplyTo || "",
                         references: [],
                         date: toFiniteDateMs(msg.date),
+                        sentDate: msg.sentDate instanceof Date ? toFiniteDateMs(msg.sentDate) : undefined,
                         subject: msg.subject || "",
                         from: toEmailAddress(msg.from?.[0] || {}),
                         to: toEmailAddresses(msg.to || []),
@@ -2679,12 +2693,22 @@ export class ImapManager extends EventEmitter {
                     if (Number.isFinite(t))
                         dateMs = t;
                 }
+                // Sent time from the Date: header (iflow exposes it as sentDate,
+                // separate from msg.date = INTERNALDATE). Undefined → store
+                // defaults sent_date to dateMs.
+                let sentMs;
+                if (msg.sentDate instanceof Date) {
+                    const st = msg.sentDate.getTime();
+                    if (Number.isFinite(st))
+                        sentMs = st;
+                }
                 this.db.upsertMessage({
                     accountId, folderId, uid: msg.uid,
                     messageId: msg.messageId || "",
                     inReplyTo: msg.inReplyTo || "",
                     references: msg.references || [],
                     date: dateMs,
+                    sentDate: sentMs,
                     subject: msg.subject || "",
                     from: toEmailAddress(msg.from?.[0] || {}),
                     to: toEmailAddresses(msg.to || []),
@@ -4168,13 +4192,17 @@ export class ImapManager extends EventEmitter {
                             const MANAGED = ["\\Seen", "\\Flagged"];
                             const impliedRemove = MANAGED.filter(f => !desired.includes(f));
                             const toRemove = [...new Set([...explicitRemove, ...impliedRemove])];
+                            // console.error so it lands in the captured daemon log
+                            // (worker stdout is NOT piped; stderr is). Temporary
+                            // diagnostic for the "flag won't persist" hunt.
+                            console.error(`  [flag-drain] ${accountId} uid=${action.uid} folder="${folder.path}" +[${desired.join(",")}] -[${toRemove.join(",")}]`);
                             if (desired.length > 0) {
                                 await client.addFlags(folder.path, action.uid, desired);
                             }
                             if (toRemove.length > 0) {
                                 await client.removeFlags(folder.path, action.uid, toRemove);
                             }
-                            console.log(`  [sync] Updated flags UID ${action.uid} (+[${desired.join(",")}] -[${toRemove.join(",")}])`);
+                            console.error(`  [flag-drain] ${accountId} uid=${action.uid} OK (server STORE done)`);
                             break;
                         }
                         case "append": {
@@ -5026,6 +5054,16 @@ export class ImapManager extends EventEmitter {
                         await withTimeout(client.appendMessage(outboxPath, raw, ["\\Seen"]), 60_000, client, `outbox APPEND ${file}`);
                         fs.renameSync(claimedPath, path.join(sentDir, file));
                         console.log(`  [outbox] Moved ${file} to IMAP Outbox → sent/`);
+                        // Chain the SMTP delivery immediately instead of waiting
+                        // for the next worker tick. The APPEND→tick→SMTP→Sent
+                        // chain added up to ~30s before the message appeared in
+                        // Sent, which read as "my message is gone" (Bob
+                        // 2026-07-01, repeatedly). Fire-and-forget; the tick
+                        // remains the retry backstop.
+                        setImmediate(() => {
+                            this.processOutbox(accountId)
+                                .catch((e) => console.error(`  [outbox] chained processOutbox: ${e?.message || e}`));
+                        });
                     }
                     catch (e) {
                         // APPEND failed (connection dropped mid-send, server
@@ -5112,8 +5150,17 @@ export class ImapManager extends EventEmitter {
             throw new Error("No recipients");
         // Dedup: skip if this Message-ID has already been sent. Prevents the
         // outbox from re-sending the same file across crash/restart cycles.
-        const messageId = messageIdMatch ? messageIdMatch[1] : "";
-        if (messageId && this.db.hasSentMessage(messageId)) {
+        // A message WITHOUT a Message-ID gets one synthesized here — the old
+        // `if (messageId && …)` guard silently skipped BOTH the dedup check
+        // AND recordSent for ID-less mail, so such a message could re-send
+        // forever with no brake (2026-07-02 resend-loop audit).
+        let messageId = messageIdMatch ? messageIdMatch[1] : "";
+        if (!messageId) {
+            messageId = `<mailx-${Date.now()}-${Math.random().toString(36).slice(2, 10)}@${account.email.split("@")[1] || "mailx.local"}>`;
+            raw = raw.replace(/\r?\n\r?\n/, `\r\nMessage-ID: ${messageId}\r\n\r\n`);
+            console.log(`  [smtp] ${accountId}: synthesized Message-ID ${messageId} for ID-less outgoing message`);
+        }
+        if (this.db.hasSentMessage(messageId)) {
             console.log(`  [smtp] ${accountId}: SKIP ${messageId} — already in sent_log`);
             return;
         }
@@ -5148,7 +5195,25 @@ export class ImapManager extends EventEmitter {
      *  Each per-UID step is its own withConnection({slow}) call so the queue
      *  yields between messages: a click-to-view body in the middle of a
      *  10-message outbox drain doesn't wait for all 10 to finish. */
+    /** Per-account re-entrancy guard for processOutbox. The worker tick and
+     *  the chained post-APPEND call (processLocalQueue) can overlap; on
+     *  servers without custom-keyword support ($Sending interlock) two
+     *  concurrent drains could both fetch and SMTP the same Outbox UID —
+     *  a double-send. One drain per account at a time; latecomers no-op
+     *  (the 10s tick retries anything they would have handled). */
+    outboxDrainInFlight = new Set();
     async processOutbox(accountId) {
+        if (this.outboxDrainInFlight.has(accountId))
+            return;
+        this.outboxDrainInFlight.add(accountId);
+        try {
+            await this._processOutbox(accountId);
+        }
+        finally {
+            this.outboxDrainInFlight.delete(accountId);
+        }
+    }
+    async _processOutbox(accountId) {
         const outboxFolder = this.findFolder(accountId, "outbox");
         if (!outboxFolder)
             return;
@@ -5538,6 +5603,15 @@ export class ImapManager extends EventEmitter {
             this.sentSweepInterval = null;
         }
     }
+    /** Message-IDs sweepSentOnce has already re-APPENDed this daemon run.
+     *  HARD GUARD against perpetual motion: a re-appended copy syncs back in,
+     *  refreshes the row's cached_at, and keeps it inside the sweep window —
+     *  so if the header search ever reports "missing" again (it failed
+     *  silently for ~13 days on a flaky lane connection, 2026-07-02, 3,704
+     *  copies of one message), the sweep would append forever. One re-append
+     *  per Message-ID per daemon lifetime; a genuinely-lost message still
+     *  gets its one repair, and a false "missing" can no longer loop. */
+    sentSweepAppended = new Set();
     async sweepSentOnce(accountId) {
         const sent = this.findFolder(accountId, "sent");
         if (!sent)
@@ -5563,7 +5637,11 @@ export class ImapManager extends EventEmitter {
         //     a finally, the opposite of "dedicated".
         await this.withConnection(accountId, async (client) => {
             for (const row of rows) {
-                const msgId = row.message_id;
+                // Trim defensively: rows written before the iflow-direct
+                // envelope-parser trim carry the header's fold whitespace
+                // ("\t<...>"), which made HEADER searches return 0 and drove
+                // the re-APPEND loop (2026-07-02, 3,707 duplicates).
+                const msgId = (row.message_id || "").trim();
                 if (!msgId)
                     continue;
                 let serverUids = [];
@@ -5575,13 +5653,32 @@ export class ImapManager extends EventEmitter {
                     continue;
                 }
                 if (serverUids.length === 0) {
-                    // Not on server. Re-APPEND from .eml if we have one.
+                    // Not on server. Re-APPEND from .eml if we have one —
+                    // but at most ONCE per Message-ID per daemon run (see
+                    // sentSweepAppended above; the 3,704-duplicate loop).
                     if (!row.body_path)
                         continue;
+                    if (this.sentSweepAppended.has(msgId)) {
+                        console.error(`  [sent-sweep] ${accountId}: SKIP re-append of ${msgId} — already re-appended once this run; header search reporting it missing AGAIN means the search is unreliable, not the message lost`);
+                        continue;
+                    }
+                    // Cross-check the local DB before trusting the server
+                    // search: multiple local Sent rows with this Message-ID
+                    // mean prior copies synced IN from the server — it exists
+                    // there regardless of what the search just said.
+                    try {
+                        const localCopies = this.db.countMessagesByMessageId?.(accountId, sent.id, msgId) ?? 0;
+                        if (localCopies > 1) {
+                            console.error(`  [sent-sweep] ${accountId}: SKIP re-append of ${msgId} — ${localCopies} local Sent rows already exist (server search returned 0 falsely)`);
+                            continue;
+                        }
+                    }
+                    catch { /* count helper missing — proceed to the once-guard */ }
                     try {
                         const buf = await this.bodyStore.readByPath(row.body_path);
                         if (!buf)
                             continue;
+                        this.sentSweepAppended.add(msgId);
                         await client.appendMessage(sent.path, buf, ["\\Seen"]);
                         appended++;
                         console.log(`  [sent-sweep] ${accountId}: re-APPENDed ${msgId} (was missing on server)`);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bobfrankston/mailx-imap",
-  "version": "0.1.109",
+  "version": "0.1.111",
   "type": "module",
   "main": "index.js",
   "types": "index.d.ts",
@@ -9,13 +9,13 @@
   },
   "license": "ISC",
   "dependencies": {
-    "@bobfrankston/mailx-types": "^0.1.20",
+    "@bobfrankston/mailx-types": "^0.1.21",
     "@bobfrankston/mailx-settings": "^0.1.32",
     "@bobfrankston/mailx-store": "^0.1.56",
-    "@bobfrankston/iflow-direct": "^0.1.55",
+    "@bobfrankston/iflow-direct": "^0.1.56",
     "@bobfrankston/tcp-transport": "^0.1.7",
     "@bobfrankston/smtp-direct": "^0.1.9",
-    "@bobfrankston/mailx-sync": "^0.1.23",
+    "@bobfrankston/mailx-sync": "^0.1.25",
     "@bobfrankston/oauthsupport": "^1.0.33"
   },
   "repository": {
@@ -37,13 +37,13 @@
   },
   ".transformedSnapshot": {
     "dependencies": {
-      "@bobfrankston/mailx-types": "^0.1.20",
+      "@bobfrankston/mailx-types": "^0.1.21",
       "@bobfrankston/mailx-settings": "^0.1.32",
       "@bobfrankston/mailx-store": "^0.1.56",
-      "@bobfrankston/iflow-direct": "^0.1.55",
+      "@bobfrankston/iflow-direct": "^0.1.56",
       "@bobfrankston/tcp-transport": "^0.1.7",
       "@bobfrankston/smtp-direct": "^0.1.9",
-      "@bobfrankston/mailx-sync": "^0.1.23",
+      "@bobfrankston/mailx-sync": "^0.1.25",
       "@bobfrankston/oauthsupport": "^1.0.33"
     }
   }