@bobfrankston/mailx-store 0.1.52 → 0.1.53

package/db.d.ts

@@ -226,7 +226,7 @@ export declare class MailxDB {
      *  insert / move-detect / existing-row upsert during the additive
      *  migration. Reconcile is the only path that DELETES from this table
      *  (when the server stops listing a UID in a folder). */
-    upsertMessageFolder(messageRowId: number, folderId: number, uid: number): void;
+    upsertMessageFolder(messageRowId: number, folderId: number, uid: number, exclusive?: boolean): void;
     /** Drop a membership row when reconcile finds the UID is no longer
      *  on the server in this folder. The messages row itself is NOT
      *  touched — it may still exist in other folders, or the deferred-
@@ -262,6 +262,10 @@ export declare class MailxDB {
         preview: string;
         bodyPath: string;
         providerId?: string;
+        /** Exclusive-folder account (IMAP): enforce one-folder-per-message in
+         *  message_folders. Omitted/false for Gmail (label = folder, many per
+         *  message). Set true by the IMAP storeMessages path. */
+        exclusive?: boolean;
     }): number;
     /** Backfill the FTS5 `body_text` column for a message after its body
      *  has been parsed. Capped at ~64 KB of text per row — FTS5 stores the
@@ -283,6 +287,9 @@ export declare class MailxDB {
      *  legacy columns are frozen at first-sight values. */
     getMessages(query: MessageQuery): PagedResult<MessageEnvelope>;
     /** Unified inbox: all inbox folders across accounts, sorted by date, paginated in SQL */
+    /** Short-TTL cache for the unified-inbox survivor COUNT (the GROUP BY that
+     *  scans the whole inbox). Keyed by flaggedOnly+folder-set. */
+    private _unifiedTotalCache;
     getUnifiedInbox(page?: number, pageSize?: number, flaggedOnly?: boolean): PagedResult<MessageEnvelope>;
     /** Map a `messages` row to a MessageEnvelope. Exposes `uuid` (stable local
      *  identity) and `bodyPath` (authoritative on-disk location) in addition

package/db.js

@@ -508,6 +508,19 @@ export class MailxDB {
             this.db.exec("CREATE INDEX IF NOT EXISTS idx_messages_thread_id ON messages(account_id, thread_id)");
         }
         catch { /* already exists */ }
+        // (account_id, message_id) composite — THE hot-path lookup. upsertMessage's
+        // move-detect and computeThreadId both query `WHERE account_id=? AND
+        // message_id=?`. The old idx_messages_message_id is on message_id ALONE, so
+        // with an account_id predicate the planner instead picks an account_id-only
+        // index and SCANS every row for the account (Bob's bobma INBOX = 136k rows).
+        // One new-message upsert ran several such scans (one per reference); a
+        // 50-message batch = the 62-SECOND write txn the slow-txn logger caught
+        // (2026-06-16) — which held the write lock and starved every other writer.
+        // A leading composite turns each into an O(log n) seek.
+        try {
+            this.db.exec("CREATE INDEX IF NOT EXISTS idx_messages_acct_msgid ON messages(account_id, message_id)");
+        }
+        catch { /* already exists */ }
         // is_replied: set when ANY other message in this account has in_reply_to
         // pointing at this row's message_id. Primary source of truth for the ↩
         // marker — \Answered is plan B (some servers strip it, Gmail labels
@@ -601,6 +614,25 @@ export class MailxDB {
                 console.log(`  [db] cleared ${cleared} prefetch-failure backoff(s) for a fresh session`);
         }
         catch { /* non-fatal */ }
+        // Per-boot: collapse duplicate memberships — rows where the SAME message
+        // holds MULTIPLE uids in the SAME folder (the move-detect accretion bug,
+        // Bob 2026-06-16: msg "Re: FYI" had 458 Sent membership rows). Keep the
+        // newest (MAX(id)) per (message_row_id, folder_id); the new
+        // upsertMessageFolder invariant prevents re-accumulation, this clears the
+        // backlog without waiting for each folder to re-sync. Same-folder only —
+        // cross-folder memberships (a real move's stale source) are NOT touched
+        // here; those need server reconcile to resolve safely.
+        try {
+            const r = this.db.prepare(`DELETE FROM message_folders WHERE id NOT IN (
+                    SELECT MAX(id) FROM message_folders GROUP BY message_row_id, folder_id
+                 )`).run();
+            const n = Number(r.changes || 0);
+            if (n > 0)
+                console.log(`  [db] collapsed ${n} duplicate same-folder membership row(s)`);
+        }
+        catch (e) {
+            console.error(`  [db] membership collapse failed: ${e?.message || e}`);
+        }
         // One-shot cleanup: the retired insertOptimisticSentRow path wrote
         // synthetic-negative-UID rows into Sent. Those rows are stale (the
         // real server-synced row eventually appears with a positive UID),
@@ -1461,8 +1493,29 @@ export class MailxDB {
      *  insert / move-detect / existing-row upsert during the additive
      *  migration. Reconcile is the only path that DELETES from this table
      *  (when the server stops listing a UID in a folder). */
-    upsertMessageFolder(messageRowId, folderId, uid) {
+    upsertMessageFolder(messageRowId, folderId, uid, exclusive = false) {
         const now = Date.now();
+        if (exclusive) {
+            // EXCLUSIVE-folder account (IMAP — Bob's iecc): a message lives in
+            // exactly ONE folder. Drop EVERY other membership for this message
+            // before recording the current one; the folder that most recently
+            // saw it (this sync) is its real location. This both prevents the
+            // cross-folder "moved but still in source" accretion (Bob 2026-06-16
+            // "colleagues still in main folder") and self-heals the backlog as
+            // each folder syncs — only the folder that genuinely holds the
+            // message on the server will ever see+set it. NOT used for Gmail,
+            // where one message legitimately carries several label-folders.
+            this.db.prepare("DELETE FROM message_folders WHERE message_row_id = ? AND NOT (folder_id = ? AND uid = ?)").run(messageRowId, folderId, uid);
+        }
+        else {
+            // INVARIANT (all accounts): a message lives at exactly ONE uid per
+            // folder. UNIQUE(folder_id, uid) only stops the SAME (folder,uid)
+            // dup'ing — not the SAME message gaining N rows in ONE folder under N
+            // different uids (move-detect re-binding to fresh uids each sync →
+            // msg 7 "Re: FYI" had 458 Sent rows). Drop any OTHER uid this message
+            // held in THIS folder before inserting the new one.
+            this.db.prepare("DELETE FROM message_folders WHERE message_row_id = ? AND folder_id = ? AND uid != ?").run(messageRowId, folderId, uid);
+        }
         // INSERT OR REPLACE on (folder_id, uid) — if some other message_row_id
         // somehow had this slot, replace it. In practice this happens after
         // an EXPUNGE+reinsert: a UID gets reused by the server for a different
@@ -1646,7 +1699,7 @@ export class MailxDB {
             // is still in this folder. No-op if migration already populated
             // it; defensive INSERT-OR-UPDATE handles the race where the row
             // was created without a corresponding membership.
-            this.upsertMessageFolder(existing.id, msg.folderId, msg.uid);
+            this.upsertMessageFolder(existing.id, msg.folderId, msg.uid, msg.exclusive);
             return existing.id;
         }
         // Move-detection: if this Message-ID already exists for this account
@@ -1671,7 +1724,7 @@ export class MailxDB {
                 // is what eventually drops it (when the server stops
                 // listing the UID there). For Gmail multi-label messages,
                 // both memberships persist legitimately.
-                this.upsertMessageFolder(moved.id, msg.folderId, msg.uid);
+                this.upsertMessageFolder(moved.id, msg.folderId, msg.uid, msg.exclusive);
                 // Notify subscribers so e.g. the reconciler can cancel any
                 // pending deferred-delete for the original (folder, uid) —
                 // otherwise the reconcile-delete grace timer fires for a
@@ -1729,7 +1782,7 @@ export class MailxDB {
         // source of truth — old folder_id/uid columns above are kept in
         // sync during the additive migration but reads will move to JOIN
         // with message_folders.
-        this.upsertMessageFolder(rowId, msg.folderId, msg.uid);
+        this.upsertMessageFolder(rowId, msg.folderId, msg.uid, msg.exclusive);
         // Index for full-text search. body_text seeded from `msg.preview`
         // here — the full parsed body isn't available at upsert time (we
         // store .eml on disk; parsing is on-demand). LocalStore.getMessage
@@ -1856,6 +1909,9 @@ export class MailxDB {
         return { items, total, page, pageSize };
     }
     /** Unified inbox: all inbox folders across accounts, sorted by date, paginated in SQL */
+    /** Short-TTL cache for the unified-inbox survivor COUNT (the GROUP BY that
+     *  scans the whole inbox). Keyed by flaggedOnly+folder-set. */
+    _unifiedTotalCache = new Map();
     getUnifiedInbox(page = 1, pageSize = 50, flaggedOnly = false) {
         const offset = (page - 1) * pageSize;
         // Find all inbox folder IDs
@@ -1887,43 +1943,72 @@ export class MailxDB {
         // copies the user expects to see in All Inboxes — collapsing across
         // accounts hid the second one (Bob 2026-06-12). Within a single inbox,
         // the to-self N-copies case still collapses as before.
-        const total = this.db.prepare(`SELECT COUNT(*) as cnt FROM (
-                SELECT 1
-                FROM messages m
-                JOIN message_folders mf ON mf.message_row_id = m.id
-                WHERE mf.folder_id IN (${placeholders})${flagFilter}
-                GROUP BY m.account_id, CASE WHEN COALESCE(m.message_id, '') = '' THEN 'mid-empty:' || m.id ELSE m.message_id END
-            )`).get(...folderIds).cnt;
-        const rows = this.db.prepare(`WITH ranked AS (
-                SELECT m.id AS m_id, mf.uid AS mf_uid, mf.folder_id AS mf_folder_id,
-                    ROW_NUMBER() OVER (
-                        PARTITION BY m.account_id, CASE WHEN COALESCE(m.message_id, '') = '' THEN 'mid-empty:' || m.id ELSE m.message_id END
-                        ORDER BY m.date DESC, m.id DESC
-                    ) AS rn
-                FROM messages m
-                JOIN message_folders mf ON mf.message_row_id = m.id
-                WHERE mf.folder_id IN (${placeholders})${flagFilter}
-            )
-            SELECT m.*, r.mf_uid AS uid, r.mf_folder_id AS folder_id,
-                EXISTS(
-                    SELECT 1 FROM sync_actions sa
-                    WHERE sa.account_id = m.account_id AND sa.uid = r.mf_uid
-                ) AS pending,
-                (SELECT COUNT(DISTINCT account_id) FROM messages m2
-                 WHERE m2.message_id = m.message_id AND COALESCE(m.message_id, '') != '') AS dupeCount,
-                -- Replied state aggregated across the whole Message-ID group, not
-                -- just the dedup survivor. When duplicates collapse (same message
-                -- delivered to multiple self-addresses), the \Answered flag /
-                -- is_replied can live on a NON-surviving copy — so the survivor
-                -- showed no reply arrow even though TB did. OR it across the group
-                -- so the marker is correct regardless of which row wins dedup
-                -- (Bob 2026-06-01). Empty message_id → falls back to the row's own.
-                COALESCE((SELECT MAX(CASE WHEN m3.is_replied = 1 OR m3.flags_json LIKE '%Answered%' THEN 1 ELSE 0 END)
-                          FROM messages m3 WHERE m3.message_id = m.message_id AND COALESCE(m.message_id, '') != ''), m.is_replied) AS groupReplied
-            FROM ranked r
-            JOIN messages m ON m.id = r.m_id
-            WHERE r.rn = 1
-            ORDER BY m.date DESC LIMIT ? OFFSET ?`).all(...folderIds, pageSize, offset);
+        // total = number of deduped survivors. The GROUP BY scans every inbox
+        // row (~230ms on a 90k inbox) so cache it briefly — it only changes on
+        // sync/delete, and a few seconds of staleness affects only the page-count
+        // indicator, never the (always-live) row data (Bob 2026-06-16).
+        const totalKey = `${flaggedOnly}:${folderIds.join(",")}`;
+        const cachedTotal = this._unifiedTotalCache.get(totalKey);
+        let total;
+        if (cachedTotal && (Date.now() - cachedTotal.at) < 4000) {
+            total = cachedTotal.total;
+        }
+        else {
+            total = this.db.prepare(`SELECT COUNT(*) as cnt FROM (
+                    SELECT 1
+                    FROM messages m
+                    JOIN message_folders mf ON mf.message_row_id = m.id
+                    WHERE mf.folder_id IN (${placeholders})${flagFilter}
+                    GROUP BY m.account_id, CASE WHEN COALESCE(m.message_id, '') = '' THEN 'mid-empty:' || m.id ELSE m.message_id END
+                )`).get(...folderIds).cnt;
+            this._unifiedTotalCache.set(totalKey, { total, at: Date.now() });
+        }
+        // STREAMING DEDUP. The old query ran ROW_NUMBER() OVER (PARTITION BY
+        // message_id ORDER BY date) across EVERY inbox row, then took rn=1 — a
+        // full sort of ~90k rows for a 50-row page (~780ms, Bob 2026-06-16
+        // "getUnifiedInbox 4s/2s"). But page 1 only needs the 50 newest deduped
+        // survivors: walk rows in date-DESC order, keep the first row seen per
+        // (account_id, message_id) group (= the rn=1 survivor — identical rule),
+        // and STOP once the page is filled. Typical page-1 read: ~50-100 physical
+        // rows instead of 90k sorted. .iterate() lets us break early; the index
+        // idx_messages_folder_date supplies the date order. Deep pages walk more
+        // (bounded by offset+pageSize survivors) but remain far cheaper than a
+        // full window sort, and are rare in the unified inbox.
+        const baseStmt = this.db.prepare(`SELECT m.*, mf.uid AS uid, mf.folder_id AS folder_id
+             FROM messages m
+             JOIN message_folders mf ON mf.message_row_id = m.id
+             WHERE mf.folder_id IN (${placeholders})${flagFilter}
+             ORDER BY m.date DESC, m.id DESC`);
+        // Per-survivor enrichment (runs only for the ≤pageSize kept rows, not the
+        // whole inbox). Same semantics as the old inline correlated subqueries.
+        const pendingStmt = this.db.prepare("SELECT 1 FROM sync_actions WHERE account_id = ? AND uid = ? LIMIT 1");
+        const dupeStmt = this.db.prepare("SELECT COUNT(DISTINCT account_id) AS c FROM messages WHERE message_id = ?");
+        const groupRepliedStmt = this.db.prepare("SELECT MAX(CASE WHEN is_replied = 1 OR flags_json LIKE '%Answered%' THEN 1 ELSE 0 END) AS r FROM messages WHERE message_id = ?");
+        const seen = new Set();
+        const kept = [];
+        let survivorIdx = 0;
+        const lastWanted = offset + pageSize;
+        for (const r of baseStmt.iterate(...folderIds)) {
+            const key = (r.message_id && r.message_id !== "")
+                ? `${r.account_id}${r.message_id}`
+                : `${r.account_id}mid-empty:${r.id}`;
+            if (seen.has(key))
+                continue; // a newer copy of this group already won
+            seen.add(key);
+            if (survivorIdx >= offset && survivorIdx < lastWanted)
+                kept.push(r);
+            survivorIdx++;
+            if (survivorIdx >= lastWanted)
+                break; // page filled — stop walking
+        }
+        const rows = kept.map(r => {
+            const hasMid = !!(r.message_id && r.message_id !== "");
+            r.pending = pendingStmt.get(r.account_id, r.uid) ? 1 : 0;
+            r.dupeCount = hasMid ? (dupeStmt.get(r.message_id)?.c | 0) : 0;
+            const gr = hasMid ? groupRepliedStmt.get(r.message_id)?.r : null;
+            r.groupReplied = (gr == null) ? r.is_replied : gr;
+            return r;
+        });
         const items = rows.map(r => ({
             id: r.id,
             accountId: r.account_id,
@@ -2418,12 +2503,31 @@ export class MailxDB {
      *  Returns {busy, log, checkpointed} (log = frames in WAL, checkpointed =
      *  frames moved into the db) or null on error. */
     checkpoint(mode = "PASSIVE") {
+        // TRUNCATE/RESTART need the WAL write lock and otherwise WAIT up to
+        // busy_timeout (5s on main) if a sync write is in flight — blocking the
+        // main thread, the exact stall we are trying to avoid. Drop busy_timeout
+        // to 0 around those modes so the checkpoint returns busy=1 INSTANTLY
+        // instead of waiting; the next 8s tick retries. PASSIVE never waits, so
+        // it keeps the default timeout. (PASSIVE keeps frames checkpointed but
+        // never shrinks the WAL FILE; TRUNCATE is what reclaims the 26 MB the
+        // file grows to during a heavy sync — Bob 2026-06-16.)
+        const needsLock = mode === "TRUNCATE" || mode === "RESTART";
         try {
+            if (needsLock)
+                this.db.exec("PRAGMA busy_timeout=0");
             return this.db.prepare(`PRAGMA wal_checkpoint(${mode})`).get();
         }
         catch {
             return null;
         }
+        finally {
+            if (needsLock) {
+                try {
+                    this.db.exec("PRAGMA busy_timeout=5000");
+                }
+                catch { /* ignore */ }
+            }
+        }
     }
     runInTxn(fn) {
         if (this.db.isTransaction)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bobfrankston/mailx-store",
-  "version": "0.1.52",
+  "version": "0.1.53",
   "type": "module",
   "main": "index.js",
   "types": "index.d.ts",