@bobfrankston/rmfmail 1.2.87 → 1.2.95

package/packages/mailx-store/db.js

@@ -556,6 +556,44 @@ export class MailxDB {
         // parsed (existing IMAP-synced rows that predate the prefetch-reparse
         // fix, where has_attachments stays 0 forever). NULL = never parsed.
         this.addColumnIfMissing("messages", "body_parsed_at", "INTEGER");
+        // sent_date: the message's own Date: header (when it was SENT), distinct
+        // from `date` which is the per-account arrival time (IMAP INTERNALDATE /
+        // Gmail internalDate). Kept as its own column so the list can offer a
+        // Sent/Received date basis (Bob 2026-07-01) AND so identical messages
+        // delivered to multiple accounts cluster on send time. Backfilled to
+        // `date` for existing rows so it is ALWAYS non-null → a plain index sorts
+        // it as fast as the received-date path (no COALESCE, no hot-path
+        // regression of the getUnifiedInbox streaming walk). Real send times fill
+        // in as folders re-sync (or via `-repair`).
+        this.addColumnIfMissing("messages", "sent_date", "INTEGER");
+        // One-shot heal: strip header-fold whitespace from stored Message-IDs /
+        // In-Reply-To. A folded "Message-ID:\r\n\t<...>" header reached the DB
+        // as "\t<...>" (pre-trim iflow-direct envelope parser); the tab made
+        // every HEADER search for that id return 0 → the sent-sweep re-APPENDed
+        // "missing" messages forever (3,707 duplicates of one reply,
+        // 2026-07-02). Idempotent — after the first pass the WHERE matches
+        // nothing. char(9/10/13)+space covers all RFC 5322 fold whitespace.
+        try {
+            const ws = "char(9)||char(10)||char(13)||' '";
+            this.db.exec(`UPDATE messages SET message_id = TRIM(message_id, ${ws}) WHERE message_id <> TRIM(message_id, ${ws})`);
+            this.db.exec(`UPDATE messages SET in_reply_to = TRIM(in_reply_to, ${ws}) WHERE in_reply_to IS NOT NULL AND in_reply_to <> TRIM(in_reply_to, ${ws})`);
+        }
+        catch { /* non-fatal */ }
+        try {
+            this.db.exec("CREATE INDEX IF NOT EXISTS idx_messages_folder_sentdate ON messages(account_id, folder_id, sent_date DESC)");
+        }
+        catch { /* already exists */ }
+        try {
+            this.db.exec("CREATE INDEX IF NOT EXISTS idx_messages_sentdate ON messages(folder_id, sent_date DESC)");
+        }
+        catch { /* already exists */ }
+        try {
+            // One-shot backfill: seed sent_date from date wherever it's still
+            // null. Indexed IS NULL check keeps this near-instant after the first
+            // pass; new writes set sent_date directly so it converges to a no-op.
+            this.db.exec("UPDATE messages SET sent_date = date WHERE sent_date IS NULL");
+        }
+        catch { /* non-fatal */ }
         // One-shot is_replied backfill on every boot. SQL-only, indexed
         // both sides via idx_messages_message_id and idx_messages_in_reply_to,
         // so a 100k-row mailbox finishes in milliseconds. Idempotent — after
@@ -1757,7 +1795,7 @@ export class MailxDB {
             // subject HERE, but only when the existing mid is empty AND the
             // incoming msg actually has one — never clobber a good value.
             if (msg.messageId && (!existing.message_id || existing.message_id === "")) {
-                this.db.prepare("UPDATE messages SET message_id = ?, date = ?, subject = ? WHERE id = ?").run(msg.messageId, msg.date || Date.now(), msg.subject || "", existing.id);
+                this.db.prepare("UPDATE messages SET message_id = ?, date = ?, sent_date = ?, subject = ? WHERE id = ?").run(msg.messageId, msg.date || Date.now(), msg.sentDate ?? msg.date ?? Date.now(), msg.subject || "", existing.id);
                 // The row was indexed with an empty subject (it had no envelope),
                 // so search missed it. Re-index now that we have the real subject
                 // + from/to. Standalone FTS5 → delete + insert. body_text
@@ -1832,10 +1870,10 @@ export class MailxDB {
         const result = this.db.prepare(`
             INSERT INTO messages (
                 account_id, folder_id, uid, uuid, message_id, in_reply_to, refs, thread_id,
-                date, subject, from_address, from_name, to_json, cc_json,
+                date, sent_date, subject, from_address, from_name, to_json, cc_json,
                 flags_json, size, has_attachments, preview, body_path, body_parsed_at, cached_at, provider_id, is_replied
-            ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
-        `).run(msg.accountId, msg.folderId, msg.uid, uuid, msg.messageId, msg.inReplyTo, JSON.stringify(msg.references), threadId, msg.date, msg.subject, msg.from.address, msg.from.name, JSON.stringify(msg.to), JSON.stringify(msg.cc), JSON.stringify(msg.flags), msg.size, msg.hasAttachments ? 1 : 0, msg.preview, msg.bodyPath, msg.bodyPath ? Date.now() : null, Date.now(), msg.providerId || null, alreadyReplied ? 1 : 0);
+            ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+        `).run(msg.accountId, msg.folderId, msg.uid, uuid, msg.messageId, msg.inReplyTo, JSON.stringify(msg.references), threadId, msg.date, msg.sentDate ?? msg.date, msg.subject, msg.from.address, msg.from.name, JSON.stringify(msg.to), JSON.stringify(msg.cc), JSON.stringify(msg.flags), msg.size, msg.hasAttachments ? 1 : 0, msg.preview, msg.bodyPath, msg.bodyPath ? Date.now() : null, Date.now(), msg.providerId || null, alreadyReplied ? 1 : 0);
         const rowId = Number(result.lastInsertRowid);
         // Forward direction for ↩: if this message is a reply, mark the parent
         // as replied-to. Cheap indexed UPDATE; no-op when the parent isn't in
@@ -1893,7 +1931,10 @@ export class MailxDB {
         const offset = (page - 1) * pageSize;
         const sort = query.sort || "date";
         const sortDir = query.sortDir || "desc";
-        const sortCol = sort === "from" ? "m.from_name" : sort === "subject" ? "m.subject" : "m.date";
+        // Date basis: "sent" (Date: header, default — clusters identical mail
+        // and matches other clients) or "received" (per-account arrival time).
+        const dateCol = query.dateBasis === "received" ? "m.date" : "m.sent_date";
+        const sortCol = sort === "from" ? "m.from_name" : sort === "subject" ? "m.subject" : dateCol;
         let where = "m.account_id = ? AND mf.folder_id = ?";
         const params = [query.accountId, query.folderId];
         if (query.search) {
@@ -1958,6 +1999,7 @@ export class MailxDB {
             references: JSON.parse(r.refs || "[]"),
             threadId: r.thread_id || undefined,
             date: r.date,
+            sentDate: r.sent_date ?? r.date,
             subject: r.subject,
             from: { name: r.from_name, address: r.from_address },
             to: JSON.parse(r.to_json),
@@ -1976,8 +2018,13 @@ export class MailxDB {
     /** Short-TTL cache for the unified-inbox survivor COUNT (the GROUP BY that
      *  scans the whole inbox). Keyed by flaggedOnly+folder-set. */
     _unifiedTotalCache = new Map();
-    getUnifiedInbox(page = 1, pageSize = 50, flaggedOnly = false) {
+    getUnifiedInbox(page = 1, pageSize = 50, flaggedOnly = false, dateBasis = "sent") {
         const offset = (page - 1) * pageSize;
+        // Date basis: sort/stream by the Date: header (sent, default) or the
+        // per-account arrival time (received). sent_date is always non-null
+        // (backfilled to date), and idx_messages_sentdate mirrors
+        // idx_messages_folder_date, so the streaming walk stays O(page).
+        const dateCol = dateBasis === "received" ? "m.date" : "m.sent_date";
         // Find all inbox folder IDs
         const inboxRows = this.db.prepare("SELECT id FROM folders WHERE special_use = 'inbox'").all();
         if (inboxRows.length === 0)
@@ -2042,7 +2089,7 @@ export class MailxDB {
              FROM messages m
              JOIN message_folders mf ON mf.message_row_id = m.id
              WHERE mf.folder_id IN (${placeholders})${flagFilter}
-             ORDER BY m.date DESC, m.id DESC`);
+             ORDER BY ${dateCol} DESC, m.id DESC`);
         // Per-survivor enrichment (runs only for the ≤pageSize kept rows, not the
         // whole inbox). Same semantics as the old inline correlated subqueries.
         const pendingStmt = this.db.prepare("SELECT 1 FROM sync_actions WHERE account_id = ? AND uid = ? LIMIT 1");
@@ -2083,6 +2130,7 @@ export class MailxDB {
             references: JSON.parse(r.refs || "[]"),
             threadId: r.thread_id || undefined,
             date: r.date,
+            sentDate: r.sent_date ?? r.date,
             subject: r.subject,
             from: { name: r.from_name, address: r.from_address },
             to: JSON.parse(r.to_json),
@@ -2516,6 +2564,19 @@ export class MailxDB {
              WHERE account_id = ? AND folder_id = ? AND cached_at >= ?
                AND message_id IS NOT NULL AND message_id <> ''`).all(accountId, folderId, sinceMs);
     }
+    /** Count local rows in a folder sharing one Message-ID. The Sent-sweep
+     *  uses this as a sanity cross-check before re-APPENDing a "missing"
+     *  message: >1 local copies mean earlier duplicates synced IN from the
+     *  server, so the server-side header search that just said "missing"
+     *  was lying (the 3,704-duplicate loop, 2026-07-02). */
+    countMessagesByMessageId(accountId, folderId, messageId) {
+        if (!messageId)
+            return 0;
+        const r = this.db.prepare(`SELECT COUNT(*) AS c FROM messages m
+             JOIN message_folders mf ON mf.message_row_id = m.id
+             WHERE m.account_id = ? AND mf.folder_id = ? AND m.message_id = ?`).get(accountId, folderId, messageId);
+        return r?.c | 0;
+    }
     /** Rebind a local row to a different server UID without re-upserting.
      *  Used by the Sent-sweep to repair optimistic-insert mispredictions
      *  (local row was inserted at the predicted UIDNEXT but the server