@bobfrankston/mailx-store 0.1.56 → 0.1.58

package/db.d.ts

@@ -284,6 +284,9 @@ export declare class MailxDB {
         inReplyTo: string;
         references: string[];
         date: number;
+        /** Date: header (send time). Defaults to `date` when the caller has no
+         *  header date, so the column is never null. */
+        sentDate?: number;
         subject: string;
         from: EmailAddress;
         to: EmailAddress[];
@@ -298,6 +301,13 @@ export declare class MailxDB {
          *  message_folders. Omitted/false for Gmail (label = folder, many per
          *  message). Set true by the IMAP storeMessages path. */
         exclusive?: boolean;
+        /** UIDs the server currently lists for THIS folder, when the caller
+         *  just enumerated them (the set-diff backfill path). Lets move-detect
+         *  distinguish a real move (old uid gone) from a DUPLICATE DELIVERY
+         *  (same Message-ID at two live uids in one folder — e.g. a message
+         *  addressed to two aliases of the same mailbox). Without this the
+         *  single row ping-pongs between the two uids every sync cycle. */
+        liveServerUids?: Set<number>;
     }): number;
     /** Backfill the FTS5 `body_text` column for a message after its body
      *  has been parsed. Capped at ~64 KB of text per row — FTS5 stores the
@@ -322,7 +332,7 @@ export declare class MailxDB {
     /** Short-TTL cache for the unified-inbox survivor COUNT (the GROUP BY that
      *  scans the whole inbox). Keyed by flaggedOnly+folder-set. */
     private _unifiedTotalCache;
-    getUnifiedInbox(page?: number, pageSize?: number, flaggedOnly?: boolean): PagedResult<MessageEnvelope>;
+    getUnifiedInbox(page?: number, pageSize?: number, flaggedOnly?: boolean, dateBasis?: "sent" | "received"): PagedResult<MessageEnvelope>;
     /** Map a `messages` row to a MessageEnvelope. Exposes `uuid` (stable local
      *  identity) and `bodyPath` (authoritative on-disk location) in addition
      *  to the server-binding metadata. */
@@ -496,6 +506,12 @@ export declare class MailxDB {
         preview: string;
         body_path: string;
     }[];
+    /** Count local rows in a folder sharing one Message-ID. The Sent-sweep
+     *  uses this as a sanity cross-check before re-APPENDing a "missing"
+     *  message: >1 local copies mean earlier duplicates synced IN from the
+     *  server, so the server-side header search that just said "missing"
+     *  was lying (the 3,704-duplicate loop, 2026-07-02). */
+    countMessagesByMessageId(accountId: string, folderId: number, messageId: string): number;
     /** Rebind a local row to a different server UID without re-upserting.
      *  Used by the Sent-sweep to repair optimistic-insert mispredictions
      *  (local row was inserted at the predicted UIDNEXT but the server

package/db.js

@@ -556,6 +556,44 @@ export class MailxDB {
         // parsed (existing IMAP-synced rows that predate the prefetch-reparse
         // fix, where has_attachments stays 0 forever). NULL = never parsed.
         this.addColumnIfMissing("messages", "body_parsed_at", "INTEGER");
+        // sent_date: the message's own Date: header (when it was SENT), distinct
+        // from `date` which is the per-account arrival time (IMAP INTERNALDATE /
+        // Gmail internalDate). Kept as its own column so the list can offer a
+        // Sent/Received date basis (Bob 2026-07-01) AND so identical messages
+        // delivered to multiple accounts cluster on send time. Backfilled to
+        // `date` for existing rows so it is ALWAYS non-null → a plain index sorts
+        // it as fast as the received-date path (no COALESCE, no hot-path
+        // regression of the getUnifiedInbox streaming walk). Real send times fill
+        // in as folders re-sync (or via `-repair`).
+        this.addColumnIfMissing("messages", "sent_date", "INTEGER");
+        // One-shot heal: strip header-fold whitespace from stored Message-IDs /
+        // In-Reply-To. A folded "Message-ID:\r\n\t<...>" header reached the DB
+        // as "\t<...>" (pre-trim iflow-direct envelope parser); the tab made
+        // every HEADER search for that id return 0 → the sent-sweep re-APPENDed
+        // "missing" messages forever (3,707 duplicates of one reply,
+        // 2026-07-02). Idempotent — after the first pass the WHERE matches
+        // nothing. char(9/10/13)+space covers all RFC 5322 fold whitespace.
+        try {
+            const ws = "char(9)||char(10)||char(13)||' '";
+            this.db.exec(`UPDATE messages SET message_id = TRIM(message_id, ${ws}) WHERE message_id <> TRIM(message_id, ${ws})`);
+            this.db.exec(`UPDATE messages SET in_reply_to = TRIM(in_reply_to, ${ws}) WHERE in_reply_to IS NOT NULL AND in_reply_to <> TRIM(in_reply_to, ${ws})`);
+        }
+        catch { /* non-fatal */ }
+        try {
+            this.db.exec("CREATE INDEX IF NOT EXISTS idx_messages_folder_sentdate ON messages(account_id, folder_id, sent_date DESC)");
+        }
+        catch { /* already exists */ }
+        try {
+            this.db.exec("CREATE INDEX IF NOT EXISTS idx_messages_sentdate ON messages(folder_id, sent_date DESC)");
+        }
+        catch { /* already exists */ }
+        try {
+            // One-shot backfill: seed sent_date from date wherever it's still
+            // null. Indexed IS NULL check keeps this near-instant after the first
+            // pass; new writes set sent_date directly so it converges to a no-op.
+            this.db.exec("UPDATE messages SET sent_date = date WHERE sent_date IS NULL");
+        }
+        catch { /* non-fatal */ }
         // One-shot is_replied backfill on every boot. SQL-only, indexed
         // both sides via idx_messages_message_id and idx_messages_in_reply_to,
         // so a 100k-row mailbox finishes in milliseconds. Idempotent — after
@@ -1757,7 +1795,7 @@ export class MailxDB {
             // subject HERE, but only when the existing mid is empty AND the
             // incoming msg actually has one — never clobber a good value.
             if (msg.messageId && (!existing.message_id || existing.message_id === "")) {
-                this.db.prepare("UPDATE messages SET message_id = ?, date = ?, subject = ? WHERE id = ?").run(msg.messageId, msg.date || Date.now(), msg.subject || "", existing.id);
+                this.db.prepare("UPDATE messages SET message_id = ?, date = ?, sent_date = ?, subject = ? WHERE id = ?").run(msg.messageId, msg.date || Date.now(), msg.sentDate ?? msg.date ?? Date.now(), msg.subject || "", existing.id);
                 // The row was indexed with an empty subject (it had no envelope),
                 // so search missed it. Re-index now that we have the real subject
                 // + from/to. Standalone FTS5 → delete + insert. body_text
@@ -1790,7 +1828,19 @@ export class MailxDB {
         // surface transient move-dups. Keeping the collapse for now.
         if (msg.messageId) {
             const moved = this.db.prepare("SELECT id, folder_id, uid FROM messages WHERE account_id = ? AND message_id = ? LIMIT 1").get(msg.accountId, msg.messageId);
-            if (moved) {
+            // Duplicate delivery, NOT a move: same folder, different uid, and
+            // the caller's server UID list confirms the OLD uid is still in the
+            // mailbox. Rebinding here would ping-pong the row between the two
+            // uids on every sync (each cycle sees the unbound uid as
+            // "server-only", refetches it, and rebinds). Fall through to a
+            // fresh insert instead — both copies get their own row, matching
+            // what the server (and every other client) actually shows, and the
+            // set-diff converges.
+            if (moved && moved.folder_id === msg.folderId && moved.uid !== msg.uid
+                && msg.liveServerUids?.has(moved.uid)) {
+                console.log(`  [move-detect] ${msg.accountId} ${msg.messageId}: uid ${moved.uid} still on server — duplicate copy at uid ${msg.uid}, inserting second row`);
+            }
+            else if (moved) {
                 console.log(`  [move-detect] ${msg.accountId} ${msg.messageId}: rebinding row ${moved.id} (folder ${moved.folder_id}/uid ${moved.uid} → folder ${msg.folderId}/uid ${msg.uid})`);
                 this.db.prepare("UPDATE messages SET folder_id = ?, uid = ?, cached_at = ? WHERE id = ?").run(msg.folderId, msg.uid, Date.now(), moved.id);
                 this.upsertMessageFolder(moved.id, msg.folderId, msg.uid, msg.exclusive);
@@ -1832,10 +1882,10 @@ export class MailxDB {
         const result = this.db.prepare(`
             INSERT INTO messages (
                 account_id, folder_id, uid, uuid, message_id, in_reply_to, refs, thread_id,
-                date, subject, from_address, from_name, to_json, cc_json,
+                date, sent_date, subject, from_address, from_name, to_json, cc_json,
                 flags_json, size, has_attachments, preview, body_path, body_parsed_at, cached_at, provider_id, is_replied
-            ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
-        `).run(msg.accountId, msg.folderId, msg.uid, uuid, msg.messageId, msg.inReplyTo, JSON.stringify(msg.references), threadId, msg.date, msg.subject, msg.from.address, msg.from.name, JSON.stringify(msg.to), JSON.stringify(msg.cc), JSON.stringify(msg.flags), msg.size, msg.hasAttachments ? 1 : 0, msg.preview, msg.bodyPath, msg.bodyPath ? Date.now() : null, Date.now(), msg.providerId || null, alreadyReplied ? 1 : 0);
+            ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+        `).run(msg.accountId, msg.folderId, msg.uid, uuid, msg.messageId, msg.inReplyTo, JSON.stringify(msg.references), threadId, msg.date, msg.sentDate ?? msg.date, msg.subject, msg.from.address, msg.from.name, JSON.stringify(msg.to), JSON.stringify(msg.cc), JSON.stringify(msg.flags), msg.size, msg.hasAttachments ? 1 : 0, msg.preview, msg.bodyPath, msg.bodyPath ? Date.now() : null, Date.now(), msg.providerId || null, alreadyReplied ? 1 : 0);
         const rowId = Number(result.lastInsertRowid);
         // Forward direction for ↩: if this message is a reply, mark the parent
         // as replied-to. Cheap indexed UPDATE; no-op when the parent isn't in
@@ -1893,7 +1943,10 @@ export class MailxDB {
         const offset = (page - 1) * pageSize;
         const sort = query.sort || "date";
         const sortDir = query.sortDir || "desc";
-        const sortCol = sort === "from" ? "m.from_name" : sort === "subject" ? "m.subject" : "m.date";
+        // Date basis: "sent" (Date: header, default — clusters identical mail
+        // and matches other clients) or "received" (per-account arrival time).
+        const dateCol = query.dateBasis === "received" ? "m.date" : "m.sent_date";
+        const sortCol = sort === "from" ? "m.from_name" : sort === "subject" ? "m.subject" : dateCol;
         let where = "m.account_id = ? AND mf.folder_id = ?";
         const params = [query.accountId, query.folderId];
         if (query.search) {
@@ -1958,6 +2011,7 @@ export class MailxDB {
             references: JSON.parse(r.refs || "[]"),
             threadId: r.thread_id || undefined,
             date: r.date,
+            sentDate: r.sent_date ?? r.date,
             subject: r.subject,
             from: { name: r.from_name, address: r.from_address },
             to: JSON.parse(r.to_json),
@@ -1976,8 +2030,13 @@ export class MailxDB {
     /** Short-TTL cache for the unified-inbox survivor COUNT (the GROUP BY that
      *  scans the whole inbox). Keyed by flaggedOnly+folder-set. */
     _unifiedTotalCache = new Map();
-    getUnifiedInbox(page = 1, pageSize = 50, flaggedOnly = false) {
+    getUnifiedInbox(page = 1, pageSize = 50, flaggedOnly = false, dateBasis = "sent") {
         const offset = (page - 1) * pageSize;
+        // Date basis: sort/stream by the Date: header (sent, default) or the
+        // per-account arrival time (received). sent_date is always non-null
+        // (backfilled to date), and idx_messages_sentdate mirrors
+        // idx_messages_folder_date, so the streaming walk stays O(page).
+        const dateCol = dateBasis === "received" ? "m.date" : "m.sent_date";
         // Find all inbox folder IDs
         const inboxRows = this.db.prepare("SELECT id FROM folders WHERE special_use = 'inbox'").all();
         if (inboxRows.length === 0)
@@ -2042,7 +2101,7 @@ export class MailxDB {
              FROM messages m
              JOIN message_folders mf ON mf.message_row_id = m.id
              WHERE mf.folder_id IN (${placeholders})${flagFilter}
-             ORDER BY m.date DESC, m.id DESC`);
+             ORDER BY ${dateCol} DESC, m.id DESC`);
         // Per-survivor enrichment (runs only for the ≤pageSize kept rows, not the
         // whole inbox). Same semantics as the old inline correlated subqueries.
         const pendingStmt = this.db.prepare("SELECT 1 FROM sync_actions WHERE account_id = ? AND uid = ? LIMIT 1");
@@ -2083,6 +2142,7 @@ export class MailxDB {
             references: JSON.parse(r.refs || "[]"),
             threadId: r.thread_id || undefined,
             date: r.date,
+            sentDate: r.sent_date ?? r.date,
             subject: r.subject,
             from: { name: r.from_name, address: r.from_address },
             to: JSON.parse(r.to_json),
@@ -2197,7 +2257,26 @@ export class MailxDB {
      *  for code paths that mean it; everyone else must pass folderId. */
     updateMessageFlags(accountId, folderId, uid, flags) {
         if (folderId != null) {
-            this.db.prepare("UPDATE messages SET flags_json = ? WHERE account_id = ? AND folder_id = ? AND uid = ?").run(JSON.stringify(flags), accountId, folderId, uid);
+            // Resolve the row via its folder MEMBERSHIP, not messages.folder_id.
+            // The message-id collapse (upsertMessage move-detect) keeps ONE
+            // `messages` row per message_id whose `folder_id` is an arbitrary
+            // "primary" folder, while the message appears in OTHER folders via
+            // `message_folders`. A `WHERE folder_id = ?` update therefore MISSED
+            // any message viewed in a folder other than its stored primary —
+            // 0 rows changed, so star/mark-read silently did nothing locally and
+            // the change "came right back" on the next render (Bob 2026-06-27).
+            // Match the membership (the same join getMessageByUid uses).
+            const r = this.db.prepare(`
+                UPDATE messages SET flags_json = ? WHERE id = (
+                    SELECT m.id FROM messages m
+                    JOIN message_folders mf ON mf.message_row_id = m.id
+                    WHERE m.account_id = ? AND mf.folder_id = ? AND mf.uid = ? LIMIT 1
+                )
+            `).run(JSON.stringify(flags), accountId, folderId, uid);
+            // Fallback for any legacy row lacking a membership entry.
+            if (r.changes === 0) {
+                this.db.prepare("UPDATE messages SET flags_json = ? WHERE account_id = ? AND folder_id = ? AND uid = ?").run(JSON.stringify(flags), accountId, folderId, uid);
+            }
         }
         else {
             this.db.prepare("UPDATE messages SET flags_json = ? WHERE account_id = ? AND uid = ?").run(JSON.stringify(flags), accountId, uid);
@@ -2497,6 +2576,19 @@ export class MailxDB {
              WHERE account_id = ? AND folder_id = ? AND cached_at >= ?
                AND message_id IS NOT NULL AND message_id <> ''`).all(accountId, folderId, sinceMs);
     }
+    /** Count local rows in a folder sharing one Message-ID. The Sent-sweep
+     *  uses this as a sanity cross-check before re-APPENDing a "missing"
+     *  message: >1 local copies mean earlier duplicates synced IN from the
+     *  server, so the server-side header search that just said "missing"
+     *  was lying (the 3,704-duplicate loop, 2026-07-02). */
+    countMessagesByMessageId(accountId, folderId, messageId) {
+        if (!messageId)
+            return 0;
+        const r = this.db.prepare(`SELECT COUNT(*) AS c FROM messages m
+             JOIN message_folders mf ON mf.message_row_id = m.id
+             WHERE m.account_id = ? AND mf.folder_id = ? AND m.message_id = ?`).get(accountId, folderId, messageId);
+        return r?.c | 0;
+    }
     /** Rebind a local row to a different server UID without re-upserting.
      *  Used by the Sent-sweep to repair optimistic-insert mispredictions
      *  (local row was inserted at the predicted UIDNEXT but the server

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bobfrankston/mailx-store",
-  "version": "0.1.56",
+  "version": "0.1.58",
   "type": "module",
   "main": "index.js",
   "types": "index.d.ts",
@@ -9,8 +9,8 @@
   },
   "license": "ISC",
   "dependencies": {
-    "@bobfrankston/mailx-types": "^0.1.20",
-    "@bobfrankston/mailx-settings": "^0.1.31",
+    "@bobfrankston/mailx-types": "^0.1.22",
+    "@bobfrankston/mailx-settings": "^0.1.33",
     "@bobfrankston/mailx-bus": "^0.1.2",
     "mailparser": "^3.7.2"
   },
@@ -29,8 +29,8 @@
   },
   ".transformedSnapshot": {
     "dependencies": {
-      "@bobfrankston/mailx-types": "^0.1.20",
-      "@bobfrankston/mailx-settings": "^0.1.31",
+      "@bobfrankston/mailx-types": "^0.1.22",
+      "@bobfrankston/mailx-settings": "^0.1.33",
       "@bobfrankston/mailx-bus": "^0.1.2",
       "mailparser": "^3.7.2"
     }

package/store.d.ts

@@ -130,7 +130,7 @@ export declare class Store {
     /** Paginated message list for a (account, folder, ...) query. */
     getMessages(query: MessageQuery): PagedResult<MessageEnvelope>;
     /** All-Inboxes view: union of every account's INBOX, paginated. */
-    getUnifiedInbox(page?: number, pageSize?: number, flaggedOnly?: boolean): PagedResult<MessageEnvelope>;
+    getUnifiedInbox(page?: number, pageSize?: number, flaggedOnly?: boolean, dateBasis?: "sent" | "received"): PagedResult<MessageEnvelope>;
     /** Local FTS5 search. Server-scope search is the reconciler's job. */
     searchMessages(query: string, page?: number, pageSize?: number, accountId?: string, folderId?: number, includeTrashSpam?: boolean): PagedResult<MessageEnvelope>;
     /** Read a fully-parsed message (envelope + body + attachments) entirely

package/store.js

@@ -182,8 +182,8 @@ export class Store {
         return this.db.getMessages(query);
     }
     /** All-Inboxes view: union of every account's INBOX, paginated. */
-    getUnifiedInbox(page = 1, pageSize = 50, flaggedOnly = false) {
-        return this.db.getUnifiedInbox(page, pageSize, flaggedOnly);
+    getUnifiedInbox(page = 1, pageSize = 50, flaggedOnly = false, dateBasis = "sent") {
+        return this.db.getUnifiedInbox(page, pageSize, flaggedOnly, dateBasis);
     }
     /** Local FTS5 search. Server-scope search is the reconciler's job. */
     searchMessages(query, page = 1, pageSize = 50, accountId, folderId, includeTrashSpam = false) {