@bobfrankston/rmfmail 1.2.17 → 1.2.19

package/packages/mailx-store/db.ts

@@ -488,6 +488,18 @@ export class MailxDB {
         try {
             this.db.exec("CREATE INDEX IF NOT EXISTS idx_messages_thread_id ON messages(account_id, thread_id)");
         } catch { /* already exists */ }
+        // (account_id, message_id) composite — THE hot-path lookup. upsertMessage's
+        // move-detect and computeThreadId both query `WHERE account_id=? AND
+        // message_id=?`. The old idx_messages_message_id is on message_id ALONE, so
+        // with an account_id predicate the planner instead picks an account_id-only
+        // index and SCANS every row for the account (Bob's bobma INBOX = 136k rows).
+        // One new-message upsert ran several such scans (one per reference); a
+        // 50-message batch = the 62-SECOND write txn the slow-txn logger caught
+        // (2026-06-16) — which held the write lock and starved every other writer.
+        // A leading composite turns each into an O(log n) seek.
+        try {
+            this.db.exec("CREATE INDEX IF NOT EXISTS idx_messages_acct_msgid ON messages(account_id, message_id)");
+        } catch { /* already exists */ }
         // is_replied: set when ANY other message in this account has in_reply_to
         // pointing at this row's message_id. Primary source of truth for the ↩
         // marker — \Answered is plan B (some servers strip it, Gmail labels
@@ -2027,6 +2039,10 @@ export class MailxDB {
     }
 
     /** Unified inbox: all inbox folders across accounts, sorted by date, paginated in SQL */
+    /** Short-TTL cache for the unified-inbox survivor COUNT (the GROUP BY that
+     *  scans the whole inbox). Keyed by flaggedOnly+folder-set. */
+    private _unifiedTotalCache = new Map<string, { total: number; at: number }>();
+
     getUnifiedInbox(page = 1, pageSize = 50, flaggedOnly = false): PagedResult<MessageEnvelope> {
         const offset = (page - 1) * pageSize;
         // Find all inbox folder IDs
@@ -2061,48 +2077,78 @@ export class MailxDB {
         // copies the user expects to see in All Inboxes — collapsing across
         // accounts hid the second one (Bob 2026-06-12). Within a single inbox,
         // the to-self N-copies case still collapses as before.
-        const total = (this.db.prepare(
-            `SELECT COUNT(*) as cnt FROM (
-                SELECT 1
-                FROM messages m
-                JOIN message_folders mf ON mf.message_row_id = m.id
-                WHERE mf.folder_id IN (${placeholders})${flagFilter}
-                GROUP BY m.account_id, CASE WHEN COALESCE(m.message_id, '') = '' THEN 'mid-empty:' || m.id ELSE m.message_id END
-            )`
-        ).get(...folderIds) as any).cnt;
+        // total = number of deduped survivors. The GROUP BY scans every inbox
+        // row (~230ms on a 90k inbox) so cache it briefly — it only changes on
+        // sync/delete, and a few seconds of staleness affects only the page-count
+        // indicator, never the (always-live) row data (Bob 2026-06-16).
+        const totalKey = `${flaggedOnly}:${folderIds.join(",")}`;
+        const cachedTotal = this._unifiedTotalCache.get(totalKey);
+        let total: number;
+        if (cachedTotal && (Date.now() - cachedTotal.at) < 4000) {
+            total = cachedTotal.total;
+        } else {
+            total = (this.db.prepare(
+                `SELECT COUNT(*) as cnt FROM (
+                    SELECT 1
+                    FROM messages m
+                    JOIN message_folders mf ON mf.message_row_id = m.id
+                    WHERE mf.folder_id IN (${placeholders})${flagFilter}
+                    GROUP BY m.account_id, CASE WHEN COALESCE(m.message_id, '') = '' THEN 'mid-empty:' || m.id ELSE m.message_id END
+                )`
+            ).get(...folderIds) as any).cnt;
+            this._unifiedTotalCache.set(totalKey, { total, at: Date.now() });
+        }
 
-        const rows = this.db.prepare(
-            `WITH ranked AS (
-                SELECT m.id AS m_id, mf.uid AS mf_uid, mf.folder_id AS mf_folder_id,
-                    ROW_NUMBER() OVER (
-                        PARTITION BY m.account_id, CASE WHEN COALESCE(m.message_id, '') = '' THEN 'mid-empty:' || m.id ELSE m.message_id END
-                        ORDER BY m.date DESC, m.id DESC
-                    ) AS rn
-                FROM messages m
-                JOIN message_folders mf ON mf.message_row_id = m.id
-                WHERE mf.folder_id IN (${placeholders})${flagFilter}
-            )
-            SELECT m.*, r.mf_uid AS uid, r.mf_folder_id AS folder_id,
-                EXISTS(
-                    SELECT 1 FROM sync_actions sa
-                    WHERE sa.account_id = m.account_id AND sa.uid = r.mf_uid
-                ) AS pending,
-                (SELECT COUNT(DISTINCT account_id) FROM messages m2
-                 WHERE m2.message_id = m.message_id AND COALESCE(m.message_id, '') != '') AS dupeCount,
-                -- Replied state aggregated across the whole Message-ID group, not
-                -- just the dedup survivor. When duplicates collapse (same message
-                -- delivered to multiple self-addresses), the \Answered flag /
-                -- is_replied can live on a NON-surviving copy — so the survivor
-                -- showed no reply arrow even though TB did. OR it across the group
-                -- so the marker is correct regardless of which row wins dedup
-                -- (Bob 2026-06-01). Empty message_id → falls back to the row's own.
-                COALESCE((SELECT MAX(CASE WHEN m3.is_replied = 1 OR m3.flags_json LIKE '%Answered%' THEN 1 ELSE 0 END)
-                          FROM messages m3 WHERE m3.message_id = m.message_id AND COALESCE(m.message_id, '') != ''), m.is_replied) AS groupReplied
-            FROM ranked r
-            JOIN messages m ON m.id = r.m_id
-            WHERE r.rn = 1
-            ORDER BY m.date DESC LIMIT ? OFFSET ?`
-        ).all(...folderIds, pageSize, offset) as any[];
+        // STREAMING DEDUP. The old query ran ROW_NUMBER() OVER (PARTITION BY
+        // message_id ORDER BY date) across EVERY inbox row, then took rn=1 — a
+        // full sort of ~90k rows for a 50-row page (~780ms, Bob 2026-06-16
+        // "getUnifiedInbox 4s/2s"). But page 1 only needs the 50 newest deduped
+        // survivors: walk rows in date-DESC order, keep the first row seen per
+        // (account_id, message_id) group (= the rn=1 survivor — identical rule),
+        // and STOP once the page is filled. Typical page-1 read: ~50-100 physical
+        // rows instead of 90k sorted. .iterate() lets us break early; the index
+        // idx_messages_folder_date supplies the date order. Deep pages walk more
+        // (bounded by offset+pageSize survivors) but remain far cheaper than a
+        // full window sort, and are rare in the unified inbox.
+        const baseStmt = this.db.prepare(
+            `SELECT m.*, mf.uid AS uid, mf.folder_id AS folder_id
+             FROM messages m
+             JOIN message_folders mf ON mf.message_row_id = m.id
+             WHERE mf.folder_id IN (${placeholders})${flagFilter}
+             ORDER BY m.date DESC, m.id DESC`
+        );
+        // Per-survivor enrichment (runs only for the ≤pageSize kept rows, not the
+        // whole inbox). Same semantics as the old inline correlated subqueries.
+        const pendingStmt = this.db.prepare(
+            "SELECT 1 FROM sync_actions WHERE account_id = ? AND uid = ? LIMIT 1");
+        const dupeStmt = this.db.prepare(
+            "SELECT COUNT(DISTINCT account_id) AS c FROM messages WHERE message_id = ?");
+        const groupRepliedStmt = this.db.prepare(
+            "SELECT MAX(CASE WHEN is_replied = 1 OR flags_json LIKE '%Answered%' THEN 1 ELSE 0 END) AS r FROM messages WHERE message_id = ?");
+
+        const seen = new Set<string>();
+        const kept: any[] = [];
+        let survivorIdx = 0;
+        const lastWanted = offset + pageSize;
+        for (const r of baseStmt.iterate(...folderIds) as Iterable<any>) {
+            const key = (r.message_id && r.message_id !== "")
+                ? `${r.account_id}${r.message_id}`
+                : `${r.account_id}mid-empty:${r.id}`;
+            if (seen.has(key)) continue;       // a newer copy of this group already won
+            seen.add(key);
+            if (survivorIdx >= offset && survivorIdx < lastWanted) kept.push(r);
+            survivorIdx++;
+            if (survivorIdx >= lastWanted) break; // page filled — stop walking
+        }
+
+        const rows = kept.map(r => {
+            const hasMid = !!(r.message_id && r.message_id !== "");
+            r.pending = pendingStmt.get(r.account_id, r.uid) ? 1 : 0;
+            r.dupeCount = hasMid ? ((dupeStmt.get(r.message_id) as any)?.c | 0) : 0;
+            const gr = hasMid ? (groupRepliedStmt.get(r.message_id) as any)?.r : null;
+            r.groupReplied = (gr == null) ? r.is_replied : gr;
+            return r;
+        });
 
         const items: MessageEnvelope[] = rows.map(r => ({
             id: r.id,
@@ -2665,8 +2711,20 @@ export class MailxDB {
      *  Returns {busy, log, checkpointed} (log = frames in WAL, checkpointed =
      *  frames moved into the db) or null on error. */
     checkpoint(mode: "PASSIVE" | "FULL" | "RESTART" | "TRUNCATE" = "PASSIVE"): { busy: number; log: number; checkpointed: number } | null {
-        try { return this.db.prepare(`PRAGMA wal_checkpoint(${mode})`).get() as any; }
-        catch { return null; }
+        // TRUNCATE/RESTART need the WAL write lock and otherwise WAIT up to
+        // busy_timeout (5s on main) if a sync write is in flight — blocking the
+        // main thread, the exact stall we are trying to avoid. Drop busy_timeout
+        // to 0 around those modes so the checkpoint returns busy=1 INSTANTLY
+        // instead of waiting; the next 8s tick retries. PASSIVE never waits, so
+        // it keeps the default timeout. (PASSIVE keeps frames checkpointed but
+        // never shrinks the WAL FILE; TRUNCATE is what reclaims the 26 MB the
+        // file grows to during a heavy sync — Bob 2026-06-16.)
+        const needsLock = mode === "TRUNCATE" || mode === "RESTART";
+        try {
+            if (needsLock) this.db.exec("PRAGMA busy_timeout=0");
+            return this.db.prepare(`PRAGMA wal_checkpoint(${mode})`).get() as any;
+        } catch { return null; }
+        finally { if (needsLock) { try { this.db.exec("PRAGMA busy_timeout=5000"); } catch { /* ignore */ } } }
     }
 
     runInTxn<T>(fn: () => T): T {