@bobfrankston/mailx-store 0.1.52 → 0.1.54

package/db.d.ts

@@ -226,7 +226,7 @@ export declare class MailxDB {
      *  insert / move-detect / existing-row upsert during the additive
      *  migration. Reconcile is the only path that DELETES from this table
      *  (when the server stops listing a UID in a folder). */
-    upsertMessageFolder(messageRowId: number, folderId: number, uid: number): void;
+    upsertMessageFolder(messageRowId: number, folderId: number, uid: number, exclusive?: boolean): void;
     /** Drop a membership row when reconcile finds the UID is no longer
      *  on the server in this folder. The messages row itself is NOT
      *  touched — it may still exist in other folders, or the deferred-
@@ -262,6 +262,10 @@ export declare class MailxDB {
         preview: string;
         bodyPath: string;
         providerId?: string;
+        /** Exclusive-folder account (IMAP): enforce one-folder-per-message in
+         *  message_folders. Omitted/false for Gmail (label = folder, many per
+         *  message). Set true by the IMAP storeMessages path. */
+        exclusive?: boolean;
     }): number;
     /** Backfill the FTS5 `body_text` column for a message after its body
      *  has been parsed. Capped at ~64 KB of text per row — FTS5 stores the
@@ -283,6 +287,9 @@ export declare class MailxDB {
      *  legacy columns are frozen at first-sight values. */
     getMessages(query: MessageQuery): PagedResult<MessageEnvelope>;
     /** Unified inbox: all inbox folders across accounts, sorted by date, paginated in SQL */
+    /** Short-TTL cache for the unified-inbox survivor COUNT (the GROUP BY that
+     *  scans the whole inbox). Keyed by flaggedOnly+folder-set. */
+    private _unifiedTotalCache;
     getUnifiedInbox(page?: number, pageSize?: number, flaggedOnly?: boolean): PagedResult<MessageEnvelope>;
     /** Map a `messages` row to a MessageEnvelope. Exposes `uuid` (stable local
      *  identity) and `bodyPath` (authoritative on-disk location) in addition
@@ -552,7 +559,7 @@ export declare class MailxDB {
      *  click, queued behind it; that IS the "loading body takes forever"
      *  bug). Now keyset-paginated by `m.id` with a `setImmediate` yield
      *  between pages, so user IPC lands in the gaps. */
-    seedContactsFromMessages(): Promise<number>;
+    seedContactsFromMessages(force?: boolean): Promise<number>;
     /** Apply the contents of contacts.jsonc — replaces all preferred-tier rows
      *  with the entries in `preferred[]`, merges `discovered[]` into the local
      *  cache, sets the in-memory denylist, and purges any discovered rows

package/db.js

@@ -508,6 +508,19 @@ export class MailxDB {
             this.db.exec("CREATE INDEX IF NOT EXISTS idx_messages_thread_id ON messages(account_id, thread_id)");
         }
         catch { /* already exists */ }
+        // (account_id, message_id) composite — THE hot-path lookup. upsertMessage's
+        // move-detect and computeThreadId both query `WHERE account_id=? AND
+        // message_id=?`. The old idx_messages_message_id is on message_id ALONE, so
+        // with an account_id predicate the planner instead picks an account_id-only
+        // index and SCANS every row for the account (Bob's bobma INBOX = 136k rows).
+        // One new-message upsert ran several such scans (one per reference); a
+        // 50-message batch = the 62-SECOND write txn the slow-txn logger caught
+        // (2026-06-16) — which held the write lock and starved every other writer.
+        // A leading composite turns each into an O(log n) seek.
+        try {
+            this.db.exec("CREATE INDEX IF NOT EXISTS idx_messages_acct_msgid ON messages(account_id, message_id)");
+        }
+        catch { /* already exists */ }
         // is_replied: set when ANY other message in this account has in_reply_to
         // pointing at this row's message_id. Primary source of truth for the ↩
         // marker — \Answered is plan B (some servers strip it, Gmail labels
@@ -601,6 +614,29 @@ export class MailxDB {
                 console.log(`  [db] cleared ${cleared} prefetch-failure backoff(s) for a fresh session`);
         }
         catch { /* non-fatal */ }
+        // Per-boot: collapse duplicate memberships — rows where the SAME message
+        // holds MULTIPLE uids in the SAME folder (the move-detect accretion bug,
+        // Bob 2026-06-16: msg "Re: FYI" had 458 Sent membership rows). Keep the
+        // newest (MAX(id)) per (message_row_id, folder_id); the new
+        // upsertMessageFolder invariant prevents re-accumulation, this clears the
+        // backlog without waiting for each folder to re-sync. Same-folder only —
+        // cross-folder memberships (a real move's stale source) are NOT touched
+        // here; those need server reconcile to resolve safely.
+        try {
+            const r = this.db.prepare(`DELETE FROM message_folders WHERE id NOT IN (
+                    SELECT MAX(id) FROM message_folders GROUP BY message_row_id, folder_id
+                 )`).run();
+            const n = Number(r.changes || 0);
+            if (n > 0)
+                console.log(`  [db] collapsed ${n} duplicate same-folder membership row(s)`);
+        }
+        catch (e) {
+            console.error(`  [db] membership collapse failed: ${e?.message || e}`);
+        }
+        // (The per-boot cross-folder collapse for "exclusive" accounts was
+        // REMOVED 2026-06-16 — it deleted intentional copies. Cross-folder stale
+        // memberships are now cleaned by server-truth reconcile, which keeps real
+        // copies. See upsertMessageFolder's note.)
         // One-shot cleanup: the retired insertOptimisticSentRow path wrote
         // synthetic-negative-UID rows into Sent. Those rows are stale (the
         // real server-synced row eventually appears with a positive UID),
@@ -1461,8 +1497,26 @@ export class MailxDB {
      *  insert / move-detect / existing-row upsert during the additive
      *  migration. Reconcile is the only path that DELETES from this table
      *  (when the server stops listing a UID in a folder). */
-    upsertMessageFolder(messageRowId, folderId, uid) {
+    upsertMessageFolder(messageRowId, folderId, uid, exclusive = false) {
         const now = Date.now();
+        // NOTE: `exclusive` (drop-all-other-folder-memberships) was REVERTED
+        // 2026-06-16. It deletes INTENTIONAL copies — Bob copies messages into
+        // multiple folders, so the same Message-ID legitimately lives in several
+        // places, and exclusivity would flip-flop a real copy between its folders
+        // every sync. Cross-folder stale-move-source memberships are cleaned by
+        // server-truth RECONCILE (the folder's sync drops uids the server no
+        // longer lists) — which preserves copies (server confirms both) while
+        // dropping accidents (server confirms one). The param is kept for the
+        // signature but no longer drops cross-folder rows.
+        void exclusive;
+        // INVARIANT (all accounts): a message lives at exactly ONE uid per
+        // folder. UNIQUE(folder_id, uid) only stops the SAME (folder,uid)
+        // dup'ing — not the SAME message gaining N rows in ONE folder under N
+        // different uids (move-detect re-binding to fresh uids each sync →
+        // msg 7 "Re: FYI" had 458 Sent rows). Drop any OTHER uid this message
+        // held in THIS folder before inserting the new one. (Same-folder only —
+        // never touches other folders, so copies survive.)
+        this.db.prepare("DELETE FROM message_folders WHERE message_row_id = ? AND folder_id = ? AND uid != ?").run(messageRowId, folderId, uid);
         // INSERT OR REPLACE on (folder_id, uid) — if some other message_row_id
         // somehow had this slot, replace it. In practice this happens after
         // an EXPUNGE+reinsert: a UID gets reused by the server for a different
@@ -1646,7 +1700,7 @@ export class MailxDB {
             // is still in this folder. No-op if migration already populated
             // it; defensive INSERT-OR-UPDATE handles the race where the row
             // was created without a corresponding membership.
-            this.upsertMessageFolder(existing.id, msg.folderId, msg.uid);
+            this.upsertMessageFolder(existing.id, msg.folderId, msg.uid, msg.exclusive);
             return existing.id;
         }
         // Move-detection: if this Message-ID already exists for this account
@@ -1657,26 +1711,17 @@ export class MailxDB {
         // stacks) that point at the UUID. Only kicks in when messageId is
         // present (servers usually include it; if not we fall through to a
         // fresh insert which mints a new UUID).
+        //
+        // NOTE (2026-06-16): the instance-model refactor (docs/instance-model-plan.md)
+        // will REPLACE this collapse with per-(folder,uid) instances. That change
+        // is held until the full model + a rebuild land — shipping it alone would
+        // surface transient move-dups. Keeping the collapse for now.
         if (msg.messageId) {
             const moved = this.db.prepare("SELECT id, folder_id, uid FROM messages WHERE account_id = ? AND message_id = ? LIMIT 1").get(msg.accountId, msg.messageId);
             if (moved) {
                 console.log(`  [move-detect] ${msg.accountId} ${msg.messageId}: rebinding row ${moved.id} (folder ${moved.folder_id}/uid ${moved.uid} → folder ${msg.folderId}/uid ${msg.uid})`);
-                // Update folder_id + uid on the messages row (additive
-                // migration: keeps the "primary location" columns valid for
-                // existing reads). The new schema's source of truth is
-                // message_folders — see below.
                 this.db.prepare("UPDATE messages SET folder_id = ?, uid = ?, cached_at = ? WHERE id = ?").run(msg.folderId, msg.uid, Date.now(), moved.id);
-                // Add the new membership row. The OLD membership (source
-                // folder + old uid) stays — reconcile in the source folder
-                // is what eventually drops it (when the server stops
-                // listing the UID there). For Gmail multi-label messages,
-                // both memberships persist legitimately.
-                this.upsertMessageFolder(moved.id, msg.folderId, msg.uid);
-                // Notify subscribers so e.g. the reconciler can cancel any
-                // pending deferred-delete for the original (folder, uid) —
-                // otherwise the reconcile-delete grace timer fires for a
-                // row that's been rebound elsewhere, and the user sees the
-                // moved message vanish 30 minutes after the move.
+                this.upsertMessageFolder(moved.id, msg.folderId, msg.uid, msg.exclusive);
                 if (this._onMoveDetected) {
                     try {
                         this._onMoveDetected({
@@ -1729,7 +1774,7 @@ export class MailxDB {
         // source of truth — old folder_id/uid columns above are kept in
         // sync during the additive migration but reads will move to JOIN
         // with message_folders.
-        this.upsertMessageFolder(rowId, msg.folderId, msg.uid);
+        this.upsertMessageFolder(rowId, msg.folderId, msg.uid, msg.exclusive);
         // Index for full-text search. body_text seeded from `msg.preview`
         // here — the full parsed body isn't available at upsert time (we
         // store .eml on disk; parsing is on-demand). LocalStore.getMessage
@@ -1856,6 +1901,9 @@ export class MailxDB {
         return { items, total, page, pageSize };
     }
     /** Unified inbox: all inbox folders across accounts, sorted by date, paginated in SQL */
+    /** Short-TTL cache for the unified-inbox survivor COUNT (the GROUP BY that
+     *  scans the whole inbox). Keyed by flaggedOnly+folder-set. */
+    _unifiedTotalCache = new Map();
     getUnifiedInbox(page = 1, pageSize = 50, flaggedOnly = false) {
         const offset = (page - 1) * pageSize;
         // Find all inbox folder IDs
@@ -1887,43 +1935,72 @@ export class MailxDB {
         // copies the user expects to see in All Inboxes — collapsing across
         // accounts hid the second one (Bob 2026-06-12). Within a single inbox,
         // the to-self N-copies case still collapses as before.
-        const total = this.db.prepare(`SELECT COUNT(*) as cnt FROM (
-                SELECT 1
-                FROM messages m
-                JOIN message_folders mf ON mf.message_row_id = m.id
-                WHERE mf.folder_id IN (${placeholders})${flagFilter}
-                GROUP BY m.account_id, CASE WHEN COALESCE(m.message_id, '') = '' THEN 'mid-empty:' || m.id ELSE m.message_id END
-            )`).get(...folderIds).cnt;
-        const rows = this.db.prepare(`WITH ranked AS (
-                SELECT m.id AS m_id, mf.uid AS mf_uid, mf.folder_id AS mf_folder_id,
-                    ROW_NUMBER() OVER (
-                        PARTITION BY m.account_id, CASE WHEN COALESCE(m.message_id, '') = '' THEN 'mid-empty:' || m.id ELSE m.message_id END
-                        ORDER BY m.date DESC, m.id DESC
-                    ) AS rn
-                FROM messages m
-                JOIN message_folders mf ON mf.message_row_id = m.id
-                WHERE mf.folder_id IN (${placeholders})${flagFilter}
-            )
-            SELECT m.*, r.mf_uid AS uid, r.mf_folder_id AS folder_id,
-                EXISTS(
-                    SELECT 1 FROM sync_actions sa
-                    WHERE sa.account_id = m.account_id AND sa.uid = r.mf_uid
-                ) AS pending,
-                (SELECT COUNT(DISTINCT account_id) FROM messages m2
-                 WHERE m2.message_id = m.message_id AND COALESCE(m.message_id, '') != '') AS dupeCount,
-                -- Replied state aggregated across the whole Message-ID group, not
-                -- just the dedup survivor. When duplicates collapse (same message
-                -- delivered to multiple self-addresses), the \Answered flag /
-                -- is_replied can live on a NON-surviving copy — so the survivor
-                -- showed no reply arrow even though TB did. OR it across the group
-                -- so the marker is correct regardless of which row wins dedup
-                -- (Bob 2026-06-01). Empty message_id → falls back to the row's own.
-                COALESCE((SELECT MAX(CASE WHEN m3.is_replied = 1 OR m3.flags_json LIKE '%Answered%' THEN 1 ELSE 0 END)
-                          FROM messages m3 WHERE m3.message_id = m.message_id AND COALESCE(m.message_id, '') != ''), m.is_replied) AS groupReplied
-            FROM ranked r
-            JOIN messages m ON m.id = r.m_id
-            WHERE r.rn = 1
-            ORDER BY m.date DESC LIMIT ? OFFSET ?`).all(...folderIds, pageSize, offset);
+        // total = number of deduped survivors. The GROUP BY scans every inbox
+        // row (~230ms on a 90k inbox) so cache it briefly — it only changes on
+        // sync/delete, and a few seconds of staleness affects only the page-count
+        // indicator, never the (always-live) row data (Bob 2026-06-16).
+        const totalKey = `${flaggedOnly}:${folderIds.join(",")}`;
+        const cachedTotal = this._unifiedTotalCache.get(totalKey);
+        let total;
+        if (cachedTotal && (Date.now() - cachedTotal.at) < 4000) {
+            total = cachedTotal.total;
+        }
+        else {
+            total = this.db.prepare(`SELECT COUNT(*) as cnt FROM (
+                    SELECT 1
+                    FROM messages m
+                    JOIN message_folders mf ON mf.message_row_id = m.id
+                    WHERE mf.folder_id IN (${placeholders})${flagFilter}
+                    GROUP BY m.account_id, CASE WHEN COALESCE(m.message_id, '') = '' THEN 'mid-empty:' || m.id ELSE m.message_id END
+                )`).get(...folderIds).cnt;
+            this._unifiedTotalCache.set(totalKey, { total, at: Date.now() });
+        }
+        // STREAMING DEDUP. The old query ran ROW_NUMBER() OVER (PARTITION BY
+        // message_id ORDER BY date) across EVERY inbox row, then took rn=1 — a
+        // full sort of ~90k rows for a 50-row page (~780ms, Bob 2026-06-16
+        // "getUnifiedInbox 4s/2s"). But page 1 only needs the 50 newest deduped
+        // survivors: walk rows in date-DESC order, keep the first row seen per
+        // (account_id, message_id) group (= the rn=1 survivor — identical rule),
+        // and STOP once the page is filled. Typical page-1 read: ~50-100 physical
+        // rows instead of 90k sorted. .iterate() lets us break early; the index
+        // idx_messages_folder_date supplies the date order. Deep pages walk more
+        // (bounded by offset+pageSize survivors) but remain far cheaper than a
+        // full window sort, and are rare in the unified inbox.
+        const baseStmt = this.db.prepare(`SELECT m.*, mf.uid AS uid, mf.folder_id AS folder_id
+             FROM messages m
+             JOIN message_folders mf ON mf.message_row_id = m.id
+             WHERE mf.folder_id IN (${placeholders})${flagFilter}
+             ORDER BY m.date DESC, m.id DESC`);
+        // Per-survivor enrichment (runs only for the ≤pageSize kept rows, not the
+        // whole inbox). Same semantics as the old inline correlated subqueries.
+        const pendingStmt = this.db.prepare("SELECT 1 FROM sync_actions WHERE account_id = ? AND uid = ? LIMIT 1");
+        const dupeStmt = this.db.prepare("SELECT COUNT(DISTINCT account_id) AS c FROM messages WHERE message_id = ?");
+        const groupRepliedStmt = this.db.prepare("SELECT MAX(CASE WHEN is_replied = 1 OR flags_json LIKE '%Answered%' THEN 1 ELSE 0 END) AS r FROM messages WHERE message_id = ?");
+        const seen = new Set();
+        const kept = [];
+        let survivorIdx = 0;
+        const lastWanted = offset + pageSize;
+        for (const r of baseStmt.iterate(...folderIds)) {
+            const key = (r.message_id && r.message_id !== "")
+                ? `${r.account_id}${r.message_id}`
+                : `${r.account_id}mid-empty:${r.id}`;
+            if (seen.has(key))
+                continue; // a newer copy of this group already won
+            seen.add(key);
+            if (survivorIdx >= offset && survivorIdx < lastWanted)
+                kept.push(r);
+            survivorIdx++;
+            if (survivorIdx >= lastWanted)
+                break; // page filled — stop walking
+        }
+        const rows = kept.map(r => {
+            const hasMid = !!(r.message_id && r.message_id !== "");
+            r.pending = pendingStmt.get(r.account_id, r.uid) ? 1 : 0;
+            r.dupeCount = hasMid ? (dupeStmt.get(r.message_id)?.c | 0) : 0;
+            const gr = hasMid ? groupRepliedStmt.get(r.message_id)?.r : null;
+            r.groupReplied = (gr == null) ? r.is_replied : gr;
+            return r;
+        });
         const items = rows.map(r => ({
             id: r.id,
             accountId: r.account_id,
@@ -2418,12 +2495,31 @@ export class MailxDB {
      *  Returns {busy, log, checkpointed} (log = frames in WAL, checkpointed =
      *  frames moved into the db) or null on error. */
     checkpoint(mode = "PASSIVE") {
+        // TRUNCATE/RESTART need the WAL write lock and otherwise WAIT up to
+        // busy_timeout (5s on main) if a sync write is in flight — blocking the
+        // main thread, the exact stall we are trying to avoid. Drop busy_timeout
+        // to 0 around those modes so the checkpoint returns busy=1 INSTANTLY
+        // instead of waiting; the next 8s tick retries. PASSIVE never waits, so
+        // it keeps the default timeout. (PASSIVE keeps frames checkpointed but
+        // never shrinks the WAL FILE; TRUNCATE is what reclaims the 26 MB the
+        // file grows to during a heavy sync — Bob 2026-06-16.)
+        const needsLock = mode === "TRUNCATE" || mode === "RESTART";
         try {
+            if (needsLock)
+                this.db.exec("PRAGMA busy_timeout=0");
             return this.db.prepare(`PRAGMA wal_checkpoint(${mode})`).get();
         }
         catch {
             return null;
         }
+        finally {
+            if (needsLock) {
+                try {
+                    this.db.exec("PRAGMA busy_timeout=5000");
+                }
+                catch { /* ignore */ }
+            }
+        }
     }
     runInTxn(fn) {
         if (this.db.isTransaction)
@@ -2552,9 +2648,19 @@ export class MailxDB {
      *  click, queued behind it; that IS the "loading body takes forever"
      *  bug). Now keyset-paginated by `m.id` with a `setImmediate` yield
      *  between pages, so user IPC lands in the gaps. */
-    async seedContactsFromMessages() {
+    async seedContactsFromMessages(force = false) {
         const VALID = /^[^\s<>@]+@[^\s<>@]+\.[^\s<>@]+$/;
         const now = Date.now();
+        // INCREMENTAL via a persisted high-water-mark on messages.id. The old
+        // behaviour re-aggregated ALL ~180k messages and re-wrote the ENTIRE
+        // contacts table every 30 min — 1–2s write txns that locked out the sync
+        // worker ("database is locked", Bob 2026-06-16 "locked"). Now each run
+        // only scans rows with id > the last-seeded id, so a periodic run touches
+        // just the handful of messages that arrived since — a tiny, fast write.
+        // First run (no watermark) and `force` do the full scan. The watermark
+        // resets to 0 on rebuild (cache wiped), so a rebuild re-seeds fully.
+        const startId = force ? 0 : Number(this.getKv("contacts", "seed_high_id") || "0");
+        let maxSeenId = startId;
         const agg = new Map();
         const bump = (name, address, date) => {
             const email = (address || "").trim().toLowerCase();
@@ -2605,7 +2711,7 @@ export class MailxDB {
                  JOIN folders f ON m.folder_id = f.id
                  WHERE f.special_use = 'sent' AND m.id > ?
                  ORDER BY m.id LIMIT ?`);
-            let lastId = 0;
+            let lastId = startId;
             for (;;) {
                 const rows = stmt.all(lastId, PAGE);
                 if (rows.length === 0)
@@ -2617,6 +2723,8 @@ export class MailxDB {
                     eatRecipients(r.bcc_json, date);
                 }
                 lastId = rows[rows.length - 1].id;
+                if (lastId > maxSeenId)
+                    maxSeenId = lastId;
                 if (rows.length < PAGE)
                     break;
                 await yieldLoop();
@@ -2629,7 +2737,7 @@ export class MailxDB {
                  LEFT JOIN folders f ON m.folder_id = f.id
                  WHERE (f.special_use IS NULL OR f.special_use != 'sent') AND m.id > ?
                  ORDER BY m.id LIMIT ?`);
-            let lastId = 0;
+            let lastId = startId;
             for (;;) {
                 const rows = stmt.all(lastId, PAGE);
                 if (rows.length === 0)
@@ -2642,6 +2750,8 @@ export class MailxDB {
                     eatRecipients(r.bcc_json, date);
                 }
                 lastId = rows[rows.length - 1].id;
+                if (lastId > maxSeenId)
+                    maxSeenId = lastId;
                 if (rows.length < PAGE)
                     break;
                 await yieldLoop();
@@ -2690,6 +2800,11 @@ export class MailxDB {
             console.log(`  [contacts] seed: ${added} new + ${bumped} refreshed (discovered)`);
             this.notifyContactsChanged();
         }
+        // Advance the high-water-mark so the next run only scans messages newer
+        // than the highest id we just processed. Monotonic — never regress (a
+        // concurrent insert mid-run keeps a higher max for the next pass).
+        if (maxSeenId > startId)
+            this.setKv("contacts", "seed_high_id", String(maxSeenId));
         return added;
     }
     /** Apply the contents of contacts.jsonc — replaces all preferred-tier rows

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bobfrankston/mailx-store",
-  "version": "0.1.52",
+  "version": "0.1.54",
   "type": "module",
   "main": "index.js",
   "types": "index.d.ts",