@bobfrankston/rmfmail 1.1.24 → 1.1.26

package/packages/mailx-store/db.ts

@@ -2129,6 +2129,26 @@ export class MailxDB {
     commitTransaction(): void { this.db.exec("COMMIT"); }
     rollbackTransaction(): void { this.db.exec("ROLLBACK"); }
 
+    /** Run `fn` inside a transaction — nesting-safe. `node:sqlite` throws on
+     *  a nested BEGIN, and the async chunked walkers (seedContactsFromMessages,
+     *  applyContactsConfig) can interleave with the sync backfill's own
+     *  transactions across their `setImmediate` yields. If a transaction is
+     *  already open this just runs `fn` (its writes join the open txn);
+     *  otherwise it owns a fresh BEGIN/COMMIT. `fn` MUST be synchronous so it
+     *  can't span an await and leave a transaction open across a yield. */
+    runInTxn<T>(fn: () => T): T {
+        if (this.db.isTransaction) return fn();
+        this.db.exec("BEGIN");
+        try {
+            const r = fn();
+            this.db.exec("COMMIT");
+            return r;
+        } catch (e) {
+            try { this.db.exec("ROLLBACK"); } catch { /* already rolled back */ }
+            throw e;
+        }
+    }
+
     // ── Contacts ──
 
     /** Record an address used in sent mail */
@@ -2224,7 +2244,16 @@ export class MailxDB {
      *  Discovered is a single tier; sub-distinctions like sent-vs-received
      *  collapse here because the user-facing UI shows them as one "discovered"
      *  source. Recency-weighted use_count differentiates within the tier. */
-    seedContactsFromMessages(): number {
+    /** ASYNC + chunked. This walks every cached message's address fields —
+     *  on a large account that is hundreds of thousands of rows. The old
+     *  synchronous version did `.all()` on the whole messages table in one
+     *  call, materialising every row and blocking the event loop for tens
+     *  of seconds to minutes (profiled 2026-05-15: 99% of daemon ticks were
+     *  in native SQLite while this ran — every getMessage IPC, every preview
+     *  click, queued behind it; that IS the "loading body takes forever"
+     *  bug). Now keyset-paginated by `m.id` with a `setImmediate` yield
+     *  between pages, so user IPC lands in the gaps. */
+    async seedContactsFromMessages(): Promise<number> {
         const VALID = /^[^\s<>@]+@[^\s<>@]+\.[^\s<>@]+$/;
         const now = Date.now();
         const agg = new Map<string, { name: string; cnt: number; last: number }>();
@@ -2242,47 +2271,69 @@ export class MailxDB {
                 agg.set(email, { name: name || "", cnt: 1, last: date || 0 });
             }
         };
+        const eatRecipients = (field: string, date: number): void => {
+            if (!field) return;
+            let parsed: any[];
+            try { parsed = JSON.parse(field); } catch { return; }
+            if (!Array.isArray(parsed)) return;
+            for (const a of parsed) {
+                if (!a) continue;
+                bump(a.name || "", a.address || a.email || "", date);
+            }
+        };
+        const yieldLoop = (): Promise<void> => new Promise<void>(r => setImmediate(r));
+        const PAGE = 2000;
 
         // Sent folder: recipients only (skip the user's own From address).
-        const sentRows = this.db.prepare(
-            `SELECT m.to_json, m.cc_json, m.bcc_json, m.date
-             FROM messages m
-             JOIN folders f ON m.folder_id = f.id
-             WHERE f.special_use = 'sent'`
-        ).all() as { to_json: string; cc_json: string; bcc_json: string; date: number }[];
-        for (const r of sentRows) {
-            const date = r.date || 0;
-            for (const field of [r.to_json, r.cc_json, r.bcc_json]) {
-                if (!field) continue;
-                let parsed: any[];
-                try { parsed = JSON.parse(field); } catch { continue; }
-                if (!Array.isArray(parsed)) continue;
-                for (const a of parsed) {
-                    if (!a) continue;
-                    bump(a.name || "", a.address || a.email || "", date);
+        // Keyset pagination by m.id — each page is a bounded `.all()`, and
+        // the event loop is handed back between pages.
+        {
+            const stmt = this.db.prepare(
+                `SELECT m.id AS id, m.to_json, m.cc_json, m.bcc_json, m.date
+                 FROM messages m
+                 JOIN folders f ON m.folder_id = f.id
+                 WHERE f.special_use = 'sent' AND m.id > ?
+                 ORDER BY m.id LIMIT ?`
+            );
+            let lastId = 0;
+            for (;;) {
+                const rows = stmt.all(lastId, PAGE) as { id: number; to_json: string; cc_json: string; bcc_json: string; date: number }[];
+                if (rows.length === 0) break;
+                for (const r of rows) {
+                    const date = r.date || 0;
+                    eatRecipients(r.to_json, date);
+                    eatRecipients(r.cc_json, date);
+                    eatRecipients(r.bcc_json, date);
                 }
+                lastId = rows[rows.length - 1].id;
+                if (rows.length < PAGE) break;
+                await yieldLoop();
             }
         }
 
         // Other folders: From + recipients.
-        const recvRows = this.db.prepare(
-            `SELECT m.from_name, m.from_address, m.to_json, m.cc_json, m.bcc_json, m.date
-             FROM messages m
-             LEFT JOIN folders f ON m.folder_id = f.id
-             WHERE f.special_use IS NULL OR f.special_use != 'sent'`
-        ).all() as { from_name: string; from_address: string; to_json: string; cc_json: string; bcc_json: string; date: number }[];
-        for (const r of recvRows) {
-            const date = r.date || 0;
-            bump(r.from_name, r.from_address, date);
-            for (const field of [r.to_json, r.cc_json, r.bcc_json]) {
-                if (!field) continue;
-                let parsed: any[];
-                try { parsed = JSON.parse(field); } catch { continue; }
-                if (!Array.isArray(parsed)) continue;
-                for (const a of parsed) {
-                    if (!a) continue;
-                    bump(a.name || "", a.address || a.email || "", date);
+        {
+            const stmt = this.db.prepare(
+                `SELECT m.id AS id, m.from_name, m.from_address, m.to_json, m.cc_json, m.bcc_json, m.date
+                 FROM messages m
+                 LEFT JOIN folders f ON m.folder_id = f.id
+                 WHERE (f.special_use IS NULL OR f.special_use != 'sent') AND m.id > ?
+                 ORDER BY m.id LIMIT ?`
+            );
+            let lastId = 0;
+            for (;;) {
+                const rows = stmt.all(lastId, PAGE) as { id: number; from_name: string; from_address: string; to_json: string; cc_json: string; bcc_json: string; date: number }[];
+                if (rows.length === 0) break;
+                for (const r of rows) {
+                    const date = r.date || 0;
+                    bump(r.from_name, r.from_address, date);
+                    eatRecipients(r.to_json, date);
+                    eatRecipients(r.cc_json, date);
+                    eatRecipients(r.bcc_json, date);
                 }
+                lastId = rows[rows.length - 1].id;
+                if (rows.length < PAGE) break;
+                await yieldLoop();
             }
         }
 
@@ -2298,17 +2349,31 @@ export class MailxDB {
                                  updated_at = ?
              WHERE id = ?`
         );
-        for (const [email, info] of agg) {
-            const existing = this.db.prepare(
-                "SELECT id FROM contacts WHERE source = 'discovered' AND lower(email) = ?"
-            ).get(email) as { id: number } | undefined;
-            if (!existing) {
-                insStmt.run(info.name, email, info.last, info.cnt, now);
-                added++;
-            } else {
-                updStmt.run(info.cnt, info.last, info.name, info.name, now, existing.id);
-                bumped++;
-            }
+        const findStmt = this.db.prepare(
+            "SELECT id FROM contacts WHERE source = 'discovered' AND lower(email) = ?"
+        );
+        // Write phase — chunked. Each chunk runs via `db.transaction()`
+        // (one synchronous turn, one commit) rather than raw BEGIN/COMMIT
+        // straddling an `await` — the wrapper is nesting-safe (savepoints)
+        // so it can't collide with the sync backfill's own transactions
+        // when the two interleave across the yield below.
+        const WRITE_CHUNK = 500;
+        const entries = [...agg.entries()];
+        for (let i = 0; i < entries.length; i += WRITE_CHUNK) {
+            const slice = entries.slice(i, i + WRITE_CHUNK);
+            this.runInTxn(() => {
+                for (const [email, info] of slice) {
+                    const existing = findStmt.get(email) as { id: number } | undefined;
+                    if (!existing) {
+                        insStmt.run(info.name, email, info.last, info.cnt, now);
+                        added++;
+                    } else {
+                        updStmt.run(info.cnt, info.last, info.name, info.name, now, existing.id);
+                        bumped++;
+                    }
+                }
+            });
+            await yieldLoop();
         }
         if (added > 0 || bumped > 0) {
             console.log(`  [contacts] seed: ${added} new + ${bumped} refreshed (discovered)`);
@@ -2327,13 +2392,13 @@ export class MailxDB {
      *  Discovered rows from the file are MERGED with whatever the local
      *  message-corpus seeder has produced. Each device contributes its
      *  observed addresses; over time GDrive accumulates the union. */
-    applyContactsConfig(cfg: {
+    async applyContactsConfig(cfg: {
         preferred?: { name?: string; email: string; source?: string; organization?: string; org?: string; priority?: boolean }[];
         denylist?: string[];
         denylistPatterns?: string[];
         priorityDomains?: string[];
         discovered?: { name?: string; email: string; useCount?: number; lastUsed?: number }[];
-    }): { preferred: number; discovered: number; purged: number; conflicts: string[] } {
+    }): Promise<{ preferred: number; discovered: number; purged: number; conflicts: string[] }> {
         const preferred = Array.isArray(cfg.preferred) ? cfg.preferred : [];
         const denylist = Array.isArray(cfg.denylist) ? cfg.denylist : [];
         const denylistPatterns = Array.isArray(cfg.denylistPatterns) ? cfg.denylistPatterns : [];
@@ -2349,41 +2414,67 @@ export class MailxDB {
             .map(e => e.email);
         this.setPriorityIndex(prioritySenders, priorityDomains);
 
+        const VALID = /^[^\s<>@]+@[^\s<>@]+\.[^\s<>@]+$/;
+        const now = Date.now();
+        const denySet = new Set(denylist.map(e => (e || "").trim().toLowerCase()).filter(Boolean));
+        const conflicts: string[] = [];
+        const yieldLoop = (): Promise<void> => new Promise<void>(r => setImmediate(r));
+        const CHUNK = 500;
+
         // Wipe and rewrite preferred-tier rows owned by contacts.jsonc.
         // The address-book UI's legacy `upsertContact` still writes
         // source='manual' rows; those are owned by the address-book code
         // path, not contacts.jsonc, so we leave them alone here.
-        this.db.exec("DELETE FROM contacts WHERE source NOT IN ('google', 'discovered', 'manual')");
-
-        const VALID = /^[^\s<>@]+@[^\s<>@]+\.[^\s<>@]+$/;
-        const now = Date.now();
+        // ALL writes below run inside transactions and yield the event
+        // loop between chunks. The prior version did one autocommitting
+        // INSERT/UPDATE per row with NO transaction — on a contacts.jsonc
+        // that has accumulated thousands of `discovered` entries across
+        // devices, that was thousands of fsyncs back-to-back, blocking the
+        // daemon for 25-30 s (profiled 2026-05-15: applyContactsConfig was
+        // ~97% of the wedge). It also re-`prepare()`d a `lower(email)`
+        // full-scan SELECT every iteration. Now: one statement prep, one
+        // up-front map of existing discovered rows (kills the N×N scan),
+        // chunked transactions, yields between chunks.
         const ins = this.db.prepare(
             `INSERT OR IGNORE INTO contacts (source, name, email, organization, last_used, use_count, updated_at)
              VALUES (?, ?, ?, ?, 0, 0, ?)`
         );
-        const denySet = new Set(denylist.map(e => (e || "").trim().toLowerCase()).filter(Boolean));
-        const conflicts: string[] = [];
         let inserted = 0;
-        for (const entry of preferred) {
-            if (!entry) continue;
-            const email = (entry.email || "").trim();
-            if (!email || !VALID.test(email)) continue;
-            if (denySet.has(email.toLowerCase())) {
-                conflicts.push(email);
-                continue;
+        // runInTxn: nesting-safe and atomic within one synchronous turn, so
+        // these writes can't collide with the sync backfill's transactions
+        // when the two interleave across the yields below.
+        this.runInTxn(() => {
+            this.db.exec("DELETE FROM contacts WHERE source NOT IN ('google', 'discovered', 'manual')");
+            for (const entry of preferred) {
+                if (!entry) continue;
+                const email = (entry.email || "").trim();
+                if (!email || !VALID.test(email)) continue;
+                if (denySet.has(email.toLowerCase())) {
+                    conflicts.push(email);
+                    continue;
+                }
+                const source = (entry.source || "preferred").trim() || "preferred";
+                const name = (entry.name || "").trim();
+                const org = (entry.organization || entry.org || "").trim();
+                try {
+                    const r = ins.run(source, name, email, org, now);
+                    if ((r as any).changes) inserted++;
+                } catch { /* dup row, skip */ }
             }
-            const source = (entry.source || "preferred").trim() || "preferred";
-            const name = (entry.name || "").trim();
-            const org = (entry.organization || entry.org || "").trim();
-            try {
-                const r = ins.run(source, name, email, org, now);
-                if ((r as any).changes) inserted++;
-            } catch { /* dup row, skip */ }
-        }
+        });
+        await yieldLoop();
 
         // Merge discovered[] from cloud into local cache. For each entry:
         // existing row wins on use_count (max), name fills if empty, lastUsed
         // is max. Missing rows are inserted. Denylisted entries skipped.
+        // Existing discovered rows are mapped ONCE up front so the merge is
+        // O(N) hash lookups, not O(N) `lower(email)` table scans.
+        const existingDiscovered = new Map<string, number>();
+        for (const row of this.db.prepare(
+            "SELECT id, lower(email) AS le FROM contacts WHERE source = 'discovered'"
+        ).all() as { id: number; le: string }[]) {
+            existingDiscovered.set(row.le, row.id);
+        }
         const insDiscovered = this.db.prepare(
             "INSERT INTO contacts (source, name, email, last_used, use_count, updated_at) VALUES ('discovered', ?, ?, ?, ?, ?)"
         );
@@ -2395,34 +2486,40 @@ export class MailxDB {
              WHERE id = ?`
         );
         let discoveredAdded = 0;
-        for (const entry of discovered) {
-            if (!entry) continue;
-            const email = (entry.email || "").trim();
-            if (!email || !VALID.test(email)) continue;
-            const lower = email.toLowerCase();
-            if (denySet.has(lower)) continue;
-            if (isJunkContact(lower, entry.name || "")) continue;
-            const name = (entry.name || "").trim();
-            const useCount = Math.max(0, entry.useCount || 0);
-            const lastUsed = Math.max(0, entry.lastUsed || 0);
-            const existing = this.db.prepare(
-                "SELECT id FROM contacts WHERE source = 'discovered' AND lower(email) = ?"
-            ).get(lower) as { id: number } | undefined;
-            if (!existing) {
-                insDiscovered.run(name, email, lastUsed, useCount, now);
-                discoveredAdded++;
-            } else {
-                updDiscovered.run(useCount, lastUsed, name, name, now, existing.id);
-            }
+        for (let i = 0; i < discovered.length; i += CHUNK) {
+            const slice = discovered.slice(i, i + CHUNK);
+            this.runInTxn(() => {
+                for (const entry of slice) {
+                    if (!entry) continue;
+                    const email = (entry.email || "").trim();
+                    if (!email || !VALID.test(email)) continue;
+                    const lower = email.toLowerCase();
+                    if (denySet.has(lower)) continue;
+                    if (isJunkContact(lower, entry.name || "")) continue;
+                    const name = (entry.name || "").trim();
+                    const useCount = Math.max(0, entry.useCount || 0);
+                    const lastUsed = Math.max(0, entry.lastUsed || 0);
+                    const existingId = existingDiscovered.get(lower);
+                    if (existingId === undefined) {
+                        insDiscovered.run(name, email, lastUsed, useCount, now);
+                        discoveredAdded++;
+                    } else {
+                        updDiscovered.run(useCount, lastUsed, name, name, now, existingId);
+                    }
+                }
+            });
+            await yieldLoop();
         }
 
         // Purge discovered rows for any denylisted email.
         const purge = this.db.prepare("DELETE FROM contacts WHERE source = 'discovered' AND lower(email) = ?");
         let purged = 0;
-        for (const e of denySet) {
-            const r = purge.run(e);
-            purged += Number((r as any).changes || 0);
-        }
+        this.runInTxn(() => {
+            for (const e of denySet) {
+                const r = purge.run(e);
+                purged += Number((r as any).changes || 0);
+            }
+        });
 
         if (conflicts.length > 0) {
             console.warn(`  [contacts] config: ${conflicts.length} preferred entries also appear in denylist — denylist wins, entries skipped: ${conflicts.join(", ")}`);