@bobfrankston/rmfmail 1.2.18 → 1.2.20

package/packages/mailx-store/db.ts

@@ -589,6 +589,24 @@ export class MailxDB {
             if (cleared > 0) console.log(`  [db] cleared ${cleared} prefetch-failure backoff(s) for a fresh session`);
         } catch { /* non-fatal */ }
 
+        // Per-boot: collapse duplicate memberships — rows where the SAME message
+        // holds MULTIPLE uids in the SAME folder (the move-detect accretion bug,
+        // Bob 2026-06-16: msg "Re: FYI" had 458 Sent membership rows). Keep the
+        // newest (MAX(id)) per (message_row_id, folder_id); the new
+        // upsertMessageFolder invariant prevents re-accumulation, this clears the
+        // backlog without waiting for each folder to re-sync. Same-folder only —
+        // cross-folder memberships (a real move's stale source) are NOT touched
+        // here; those need server reconcile to resolve safely.
+        try {
+            const r = this.db.prepare(
+                `DELETE FROM message_folders WHERE id NOT IN (
+                    SELECT MAX(id) FROM message_folders GROUP BY message_row_id, folder_id
+                 )`
+            ).run();
+            const n = Number((r as any).changes || 0);
+            if (n > 0) console.log(`  [db] collapsed ${n} duplicate same-folder membership row(s)`);
+        } catch (e: any) { console.error(`  [db] membership collapse failed: ${e?.message || e}`); }
+
         // One-shot cleanup: the retired insertOptimisticSentRow path wrote
         // synthetic-negative-UID rows into Sent. Those rows are stale (the
         // real server-synced row eventually appears with a positive UID),
@@ -1555,6 +1573,20 @@ export class MailxDB {
      *  (when the server stops listing a UID in a folder). */
     upsertMessageFolder(messageRowId: number, folderId: number, uid: number): void {
         const now = Date.now();
+        // INVARIANT: a message lives at exactly ONE uid per folder. The schema's
+        // UNIQUE(folder_id, uid) only stops the SAME (folder,uid) from dup'ing —
+        // it does NOT stop the SAME message from gaining N rows in ONE folder
+        // under N different uids. That's exactly what happened: move-detect kept
+        // re-binding a message to fresh uids on every Sent sync, so msg 7 ("Re:
+        // FYI") accreted 458 membership rows in Sent alone (Bob 2026-06-16,
+        // "amazon"/"colleagues still in folder"). Before inserting the new
+        // location, drop any OTHER uid this message held in THIS folder — the
+        // newest sighting is the truth; a message can't be at two uids in one
+        // mailbox. This caps membership at one-row-per-(message,folder) and
+        // stops the unbounded accretion.
+        this.db.prepare(
+            "DELETE FROM message_folders WHERE message_row_id = ? AND folder_id = ? AND uid != ?"
+        ).run(messageRowId, folderId, uid);
         // INSERT OR REPLACE on (folder_id, uid) — if some other message_row_id
         // somehow had this slot, replace it. In practice this happens after
         // an EXPUNGE+reinsert: a UID gets reused by the server for a different
@@ -2039,6 +2071,10 @@ export class MailxDB {
     }
 
     /** Unified inbox: all inbox folders across accounts, sorted by date, paginated in SQL */
+    /** Short-TTL cache for the unified-inbox survivor COUNT (the GROUP BY that
+     *  scans the whole inbox). Keyed by flaggedOnly+folder-set. */
+    private _unifiedTotalCache = new Map<string, { total: number; at: number }>();
+
     getUnifiedInbox(page = 1, pageSize = 50, flaggedOnly = false): PagedResult<MessageEnvelope> {
         const offset = (page - 1) * pageSize;
         // Find all inbox folder IDs
@@ -2073,48 +2109,78 @@ export class MailxDB {
         // copies the user expects to see in All Inboxes — collapsing across
         // accounts hid the second one (Bob 2026-06-12). Within a single inbox,
         // the to-self N-copies case still collapses as before.
-        const total = (this.db.prepare(
-            `SELECT COUNT(*) as cnt FROM (
-                SELECT 1
-                FROM messages m
-                JOIN message_folders mf ON mf.message_row_id = m.id
-                WHERE mf.folder_id IN (${placeholders})${flagFilter}
-                GROUP BY m.account_id, CASE WHEN COALESCE(m.message_id, '') = '' THEN 'mid-empty:' || m.id ELSE m.message_id END
-            )`
-        ).get(...folderIds) as any).cnt;
+        // total = number of deduped survivors. The GROUP BY scans every inbox
+        // row (~230ms on a 90k inbox) so cache it briefly — it only changes on
+        // sync/delete, and a few seconds of staleness affects only the page-count
+        // indicator, never the (always-live) row data (Bob 2026-06-16).
+        const totalKey = `${flaggedOnly}:${folderIds.join(",")}`;
+        const cachedTotal = this._unifiedTotalCache.get(totalKey);
+        let total: number;
+        if (cachedTotal && (Date.now() - cachedTotal.at) < 4000) {
+            total = cachedTotal.total;
+        } else {
+            total = (this.db.prepare(
+                `SELECT COUNT(*) as cnt FROM (
+                    SELECT 1
+                    FROM messages m
+                    JOIN message_folders mf ON mf.message_row_id = m.id
+                    WHERE mf.folder_id IN (${placeholders})${flagFilter}
+                    GROUP BY m.account_id, CASE WHEN COALESCE(m.message_id, '') = '' THEN 'mid-empty:' || m.id ELSE m.message_id END
+                )`
+            ).get(...folderIds) as any).cnt;
+            this._unifiedTotalCache.set(totalKey, { total, at: Date.now() });
+        }
 
-        const rows = this.db.prepare(
-            `WITH ranked AS (
-                SELECT m.id AS m_id, mf.uid AS mf_uid, mf.folder_id AS mf_folder_id,
-                    ROW_NUMBER() OVER (
-                        PARTITION BY m.account_id, CASE WHEN COALESCE(m.message_id, '') = '' THEN 'mid-empty:' || m.id ELSE m.message_id END
-                        ORDER BY m.date DESC, m.id DESC
-                    ) AS rn
-                FROM messages m
-                JOIN message_folders mf ON mf.message_row_id = m.id
-                WHERE mf.folder_id IN (${placeholders})${flagFilter}
-            )
-            SELECT m.*, r.mf_uid AS uid, r.mf_folder_id AS folder_id,
-                EXISTS(
-                    SELECT 1 FROM sync_actions sa
-                    WHERE sa.account_id = m.account_id AND sa.uid = r.mf_uid
-                ) AS pending,
-                (SELECT COUNT(DISTINCT account_id) FROM messages m2
-                 WHERE m2.message_id = m.message_id AND COALESCE(m.message_id, '') != '') AS dupeCount,
-                -- Replied state aggregated across the whole Message-ID group, not
-                -- just the dedup survivor. When duplicates collapse (same message
-                -- delivered to multiple self-addresses), the \Answered flag /
-                -- is_replied can live on a NON-surviving copy — so the survivor
-                -- showed no reply arrow even though TB did. OR it across the group
-                -- so the marker is correct regardless of which row wins dedup
-                -- (Bob 2026-06-01). Empty message_id → falls back to the row's own.
-                COALESCE((SELECT MAX(CASE WHEN m3.is_replied = 1 OR m3.flags_json LIKE '%Answered%' THEN 1 ELSE 0 END)
-                          FROM messages m3 WHERE m3.message_id = m.message_id AND COALESCE(m.message_id, '') != ''), m.is_replied) AS groupReplied
-            FROM ranked r
-            JOIN messages m ON m.id = r.m_id
-            WHERE r.rn = 1
-            ORDER BY m.date DESC LIMIT ? OFFSET ?`
-        ).all(...folderIds, pageSize, offset) as any[];
+        // STREAMING DEDUP. The old query ran ROW_NUMBER() OVER (PARTITION BY
+        // message_id ORDER BY date) across EVERY inbox row, then took rn=1 — a
+        // full sort of ~90k rows for a 50-row page (~780ms, Bob 2026-06-16
+        // "getUnifiedInbox 4s/2s"). But page 1 only needs the 50 newest deduped
+        // survivors: walk rows in date-DESC order, keep the first row seen per
+        // (account_id, message_id) group (= the rn=1 survivor — identical rule),
+        // and STOP once the page is filled. Typical page-1 read: ~50-100 physical
+        // rows instead of 90k sorted. .iterate() lets us break early; the index
+        // idx_messages_folder_date supplies the date order. Deep pages walk more
+        // (bounded by offset+pageSize survivors) but remain far cheaper than a
+        // full window sort, and are rare in the unified inbox.
+        const baseStmt = this.db.prepare(
+            `SELECT m.*, mf.uid AS uid, mf.folder_id AS folder_id
+             FROM messages m
+             JOIN message_folders mf ON mf.message_row_id = m.id
+             WHERE mf.folder_id IN (${placeholders})${flagFilter}
+             ORDER BY m.date DESC, m.id DESC`
+        );
+        // Per-survivor enrichment (runs only for the ≤pageSize kept rows, not the
+        // whole inbox). Same semantics as the old inline correlated subqueries.
+        const pendingStmt = this.db.prepare(
+            "SELECT 1 FROM sync_actions WHERE account_id = ? AND uid = ? LIMIT 1");
+        const dupeStmt = this.db.prepare(
+            "SELECT COUNT(DISTINCT account_id) AS c FROM messages WHERE message_id = ?");
+        const groupRepliedStmt = this.db.prepare(
+            "SELECT MAX(CASE WHEN is_replied = 1 OR flags_json LIKE '%Answered%' THEN 1 ELSE 0 END) AS r FROM messages WHERE message_id = ?");
+
+        const seen = new Set<string>();
+        const kept: any[] = [];
+        let survivorIdx = 0;
+        const lastWanted = offset + pageSize;
+        for (const r of baseStmt.iterate(...folderIds) as Iterable<any>) {
+            const key = (r.message_id && r.message_id !== "")
+                ? `${r.account_id}${r.message_id}`
+                : `${r.account_id}mid-empty:${r.id}`;
+            if (seen.has(key)) continue;       // a newer copy of this group already won
+            seen.add(key);
+            if (survivorIdx >= offset && survivorIdx < lastWanted) kept.push(r);
+            survivorIdx++;
+            if (survivorIdx >= lastWanted) break; // page filled — stop walking
+        }
+
+        const rows = kept.map(r => {
+            const hasMid = !!(r.message_id && r.message_id !== "");
+            r.pending = pendingStmt.get(r.account_id, r.uid) ? 1 : 0;
+            r.dupeCount = hasMid ? ((dupeStmt.get(r.message_id) as any)?.c | 0) : 0;
+            const gr = hasMid ? (groupRepliedStmt.get(r.message_id) as any)?.r : null;
+            r.groupReplied = (gr == null) ? r.is_replied : gr;
+            return r;
+        });
 
         const items: MessageEnvelope[] = rows.map(r => ({
             id: r.id,
@@ -2677,8 +2743,20 @@ export class MailxDB {
      *  Returns {busy, log, checkpointed} (log = frames in WAL, checkpointed =
      *  frames moved into the db) or null on error. */
     checkpoint(mode: "PASSIVE" | "FULL" | "RESTART" | "TRUNCATE" = "PASSIVE"): { busy: number; log: number; checkpointed: number } | null {
-        try { return this.db.prepare(`PRAGMA wal_checkpoint(${mode})`).get() as any; }
-        catch { return null; }
+        // TRUNCATE/RESTART need the WAL write lock and otherwise WAIT up to
+        // busy_timeout (5s on main) if a sync write is in flight — blocking the
+        // main thread, the exact stall we are trying to avoid. Drop busy_timeout
+        // to 0 around those modes so the checkpoint returns busy=1 INSTANTLY
+        // instead of waiting; the next 8s tick retries. PASSIVE never waits, so
+        // it keeps the default timeout. (PASSIVE keeps frames checkpointed but
+        // never shrinks the WAL FILE; TRUNCATE is what reclaims the 26 MB the
+        // file grows to during a heavy sync — Bob 2026-06-16.)
+        const needsLock = mode === "TRUNCATE" || mode === "RESTART";
+        try {
+            if (needsLock) this.db.exec("PRAGMA busy_timeout=0");
+            return this.db.prepare(`PRAGMA wal_checkpoint(${mode})`).get() as any;
+        } catch { return null; }
+        finally { if (needsLock) { try { this.db.exec("PRAGMA busy_timeout=5000"); } catch { /* ignore */ } } }
     }
 
     runInTxn<T>(fn: () => T): T {