@bobfrankston/rmfmail 1.2.155 → 1.2.158

package/packages/mailx-store/db.ts

@@ -205,8 +205,14 @@ const SCHEMA = `
     -- the index stores its own text (we INSERT it explicitly on upsert), so
     -- there's nothing to dereference and DELETE/INSERT/reindex all work. The
     -- migrateFtsSchema() pass below rebuilds an existing external-content table.
+    -- Tokenizer is TRIGRAM (not unicode61): unicode61 indexes whole words, so
+    -- "opens" prefix-matched "opensprinkler" but "sprinkler" found nothing
+    -- (Bob 2026-07-19). Trigram matches any >=3-char substring; searchMessages
+    -- falls back to LIKE for 1-2 char terms. Existing DBs with the old
+    -- tokenizer are converted in the background by startFtsTrigramMigration().
     CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts USING fts5(
-        subject, from_name, from_address, to_text, cc_text, body_text
+        subject, from_name, from_address, to_text, cc_text, body_text,
+        tokenize='trigram remove_diacritics 1'
     );
 
     CREATE TABLE IF NOT EXISTS sync_actions (
@@ -1953,14 +1959,24 @@ export class MailxDB {
                 // + from/to. Standalone FTS5 → delete + insert. body_text
                 // backfills when the body parses.
                 try {
-                    this.db.prepare("DELETE FROM messages_fts WHERE rowid = ?").run(existing.id);
-                    this.db.prepare(
-                        "INSERT INTO messages_fts (rowid, subject, from_name, from_address, to_text, cc_text, body_text) VALUES (?, ?, ?, ?, ?, ?, '')"
-                    ).run(
+                    const vals = [
                         existing.id, msg.subject || "", msg.from?.name || "", msg.from?.address || "",
                         (msg.to || []).map(a => `${a.name || ""} ${a.address || ""}`).join(" "),
                         (msg.cc || []).map(a => `${a.name || ""} ${a.address || ""}`).join(" "),
-                    );
+                    ] as const;
+                    this.db.prepare("DELETE FROM messages_fts WHERE rowid = ?").run(existing.id);
+                    this.db.prepare(
+                        "INSERT INTO messages_fts (rowid, subject, from_name, from_address, to_text, cc_text, body_text) VALUES (?, ?, ?, ?, ?, ?, '')"
+                    ).run(...vals);
+                    // Mirror into the trigram-migration shadow so the swap
+                    // doesn't lose this re-index (see startFtsTrigramMigration).
+                    if (this.ftsTriShadowExists()) {
+                        try {
+                            this.db.prepare(
+                                "INSERT OR REPLACE INTO messages_fts_tri (rowid, subject, from_name, from_address, to_text, cc_text, body_text) VALUES (?, ?, ?, ?, ?, ?, '')"
+                            ).run(...vals);
+                        } catch { /* shadow may have just been swapped away */ }
+                    }
                 } catch { /* best-effort */ }
             }
             // Refresh membership last_seen_at — server confirmed this UID
@@ -2118,6 +2134,14 @@ export class MailxDB {
             this.db.prepare(
                 "INSERT INTO messages_fts (rowid, subject, from_name, from_address, to_text, cc_text, body_text) VALUES (?, ?, ?, ?, ?, ?, ?)"
             ).run(rowId, msg.subject, msg.from.name, msg.from.address, toText, ccText, msg.preview);
+            // Mirror into the trigram-migration shadow (see startFtsTrigramMigration).
+            if (this.ftsTriShadowExists()) {
+                try {
+                    this.db.prepare(
+                        "INSERT OR REPLACE INTO messages_fts_tri (rowid, subject, from_name, from_address, to_text, cc_text, body_text) VALUES (?, ?, ?, ?, ?, ?, ?)"
+                    ).run(rowId, msg.subject, msg.from.name, msg.from.address, toText, ccText, msg.preview);
+                } catch { /* shadow may have just been swapped away */ }
+            }
         } catch { /* FTS insert may fail on rebuild, non-fatal */ }
 
         return rowId;
@@ -2141,6 +2165,14 @@ export class MailxDB {
             this.db.prepare(
                 "UPDATE messages_fts SET body_text = ? WHERE rowid = ?",
             ).run(capped, rowId);
+            // Mirror into the trigram-migration shadow (see startFtsTrigramMigration).
+            // No-op if the copy loop hasn't reached this rowid yet — the chunk
+            // will carry the (also-updated) old-table value across later.
+            if (this.ftsTriShadowExists()) {
+                try {
+                    this.db.prepare("UPDATE messages_fts_tri SET body_text = ? WHERE rowid = ?").run(capped, rowId);
+                } catch { /* shadow may have just been swapped away */ }
+            }
         } catch { /* FTS update is best-effort */ }
     }
 
@@ -2742,7 +2774,7 @@ export class MailxDB {
             if (!row || !/content\s*=\s*messages/i.test(row.sql)) return; // already standalone
             console.log("  [db] messages_fts had broken external-content schema — rebuilding standalone + reindexing");
             this.db.exec("DROP TABLE IF EXISTS messages_fts");
-            this.db.exec("CREATE VIRTUAL TABLE messages_fts USING fts5(subject, from_name, from_address, to_text, cc_text, body_text)");
+            this.db.exec("CREATE VIRTUAL TABLE messages_fts USING fts5(subject, from_name, from_address, to_text, cc_text, body_text, tokenize='trigram remove_diacritics 1')");
             const addrText = (j: string): string => {
                 try { return (JSON.parse(j || "[]") as any[]).map(a => `${a.name || ""} ${a.address || ""}`).join(" "); }
                 catch { return ""; }
@@ -2760,6 +2792,121 @@ export class MailxDB {
         }
     }
 
+    /** True while the trigram migration's shadow table exists — the FTS write
+     *  paths (upsert / updateFtsBody) mirror into it so the copy loop never
+     *  chases a moving target. TTL-cached: upserts run hundreds/sec during
+     *  sync and the answer only changes twice in the DB's lifetime. Checked
+     *  via sqlite_master (not an in-memory flag) because the migration runs on
+     *  the main writer while upserts run on the sync worker's connection. */
+    private triShadowCache: { at: number; exists: boolean } | null = null;
+    private ftsTriShadowExists(): boolean {
+        const now = Date.now();
+        if (this.triShadowCache && now - this.triShadowCache.at < 2000) return this.triShadowCache.exists;
+        let exists = false;
+        try {
+            exists = !!this.db.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name='messages_fts_tri'").get();
+        } catch { /* treat as absent */ }
+        this.triShadowCache = { at: now, exists };
+        return exists;
+    }
+
+    /** Does messages_fts still use the old unicode61 word tokenizer? Checked
+     *  per-call (not cached): the answer flips mid-session when the background
+     *  migration swaps tables, and searchMessages must switch query dialects
+     *  the moment it does — including on the read-worker's own connection. */
+    private ftsIsTrigram(): boolean {
+        try {
+            const row = this.db.prepare("SELECT sql FROM sqlite_master WHERE type='table' AND name='messages_fts'").get() as { sql: string } | undefined;
+            return !!row && /trigram/i.test(row.sql || "");
+        } catch { return false; }
+    }
+
+    ftsNeedsTrigramMigration(): boolean {
+        if (this.readOnly) return false;
+        try {
+            const row = this.db.prepare("SELECT sql FROM sqlite_master WHERE type='table' AND name='messages_fts'").get() as { sql: string } | undefined;
+            return !!row && !/trigram/i.test(row.sql || "");
+        } catch { return false; }
+    }
+
+    /** One-time background conversion of messages_fts from the unicode61 word
+     *  tokenizer to trigram (substring search — "sprinkler" now finds
+     *  "OpenSprinkler", Bob 2026-07-19). Word tokens only match by prefix, so
+     *  any mid-word search silently found nothing.
+     *
+     *  Runs chunked on the main writer (~3-4 min for 190k rows, measured) so
+     *  boot isn't blocked and sync writes stay responsive between chunks. The
+     *  existing index keeps serving searches the whole time; concurrent FTS
+     *  writes land in BOTH tables via ftsTriShadowExists() mirroring. The
+     *  final swap (DROP + RENAME) is a single transaction, so a crash at any
+     *  point leaves either the old table (migration restarts next boot) or
+     *  the finished trigram table — never neither. Stored body_text survives
+     *  because the copy reads it straight out of the old standalone table.
+     *  Call ONLY from the long-lived daemon: a one-shot CLI command would sit
+     *  alive for minutes finishing the copy. */
+    startFtsTrigramMigration(): void {
+        if (!this.ftsNeedsTrigramMigration()) return;
+        const COLS = "rowid, subject, from_name, from_address, to_text, cc_text, body_text";
+        try {
+            // A leftover shadow from an interrupted run may be stale (writes
+            // that happened while no daemon mirrored them) — start clean.
+            this.db.exec("DROP TABLE IF EXISTS messages_fts_tri");
+            this.db.exec("CREATE VIRTUAL TABLE messages_fts_tri USING fts5(subject, from_name, from_address, to_text, cc_text, body_text, tokenize='trigram remove_diacritics 1')");
+        } catch (e: any) {
+            console.error(`  [db] FTS trigram migration setup failed: ${e?.message || e}`);
+            return;
+        }
+        let total = 0;
+        try { total = (this.db.prepare("SELECT COUNT(*) AS c FROM messages_fts").get() as any)?.c || 0; } catch { /* */ }
+        console.log(`  [db] FTS trigram migration started: ${total} rows in background (substring search when done)`);
+        const CHUNK = 500;          // ~0.35s of tokenization per chunk (measured on Bob's DB)
+        let last = 0, copied = 0, chunkN = 0;
+        const step = (): void => {
+            try {
+                const rows = this.db.prepare("SELECT rowid FROM messages_fts WHERE rowid > ? ORDER BY rowid LIMIT ?").all(last, CHUNK) as { rowid: number }[];
+                if (rows.length === 0) { finish(); return; }
+                const hi = rows[rows.length - 1].rowid;
+                // Single INSERT..SELECT — atomic against the sync worker's
+                // mirrored writes (WAL serializes writers), so old and shadow
+                // can't diverge inside a chunk. OR REPLACE because mirrored
+                // writes may have already landed rows past the frontier.
+                this.db.prepare(
+                    `INSERT OR REPLACE INTO messages_fts_tri (${COLS}) SELECT ${COLS} FROM messages_fts WHERE rowid > ? AND rowid <= ?`
+                ).run(last, hi);
+                last = hi; copied += rows.length; chunkN++;
+                if (chunkN % 40 === 0) {
+                    console.log(`  [db] FTS trigram migration: ${copied}/${total}`);
+                    // The copy writes ~1 GB through the WAL in total — keep it drained.
+                    try { this.db.exec("PRAGMA wal_checkpoint(PASSIVE)"); } catch { /* */ }
+                }
+                setTimeout(step, 100);   // yield the writer between chunks
+            } catch (e: any) {
+                console.error(`  [db] FTS trigram migration chunk failed (${e?.message || e}) — retrying in 5s`);
+                setTimeout(step, 5000);
+            }
+        };
+        const finish = (): void => {
+            try {
+                this.db.exec("BEGIN IMMEDIATE");
+                // Rows past the frontier that landed after the last chunk are
+                // already mirrored into the shadow, but belt-and-braces: if any
+                // exist, loop once more instead of trusting the mirror.
+                const strag = (this.db.prepare("SELECT COUNT(*) AS c FROM messages_fts WHERE rowid > ?").get(last) as any)?.c || 0;
+                if (strag > 0) { this.db.exec("ROLLBACK"); setTimeout(step, 100); return; }
+                this.db.exec("DROP TABLE messages_fts");
+                this.db.exec("ALTER TABLE messages_fts_tri RENAME TO messages_fts");
+                this.db.exec("COMMIT");
+                console.log(`  [db] FTS trigram migration complete: ${copied} rows — substring search active`);
+                try { this.db.exec("PRAGMA wal_checkpoint(PASSIVE)"); } catch { /* */ }
+            } catch (e: any) {
+                try { this.db.exec("ROLLBACK"); } catch { /* */ }
+                console.error(`  [db] FTS trigram migration swap failed (${e?.message || e}) — retrying in 5s`);
+                setTimeout(finish, 5000);
+            }
+        };
+        setTimeout(step, 3000);      // let boot (folder loads, first sync) settle first
+    }
+
     /** Record a prefetch failure (0-body fetch / store-write fail) for a UID,
      *  incrementing its backoff count. Persisted so it survives restarts. */
     recordPrefetchFailure(accountId: string, folderId: number, uid: number): void {
@@ -3701,10 +3848,57 @@ export class MailxDB {
         // Strip FTS5 metacharacters from a user term. A stray `;` `(` `:` `"`
         // etc. is a hard `fts5: syntax error` (Bob 2026-05-21 log). FTS5's
         // tokenizer splits on non-word chars anyway, so removing them loses no
-        // real matching ability.
+        // real matching ability. (unicode61 dialect only — see isTri below.)
         const ftsClean = (t: string): string => t.replace(/["';:(){}\[\]^~\\/]/g, "").trim();
 
-        for (const part of parts) {
+        // ── Tokenizer dialects ──
+        // The index is migrating from unicode61 (word tokens, prefix-only
+        // matching — "sprinkler" could never find "opensprinkler") to trigram
+        // (substring matching, Bob 2026-07-19). Which dialect to emit depends
+        // on which table is live RIGHT NOW: the background migration swaps it
+        // mid-session, and this runs on the read-worker's own connection, so
+        // ask sqlite_master rather than trusting boot-time state.
+        //
+        // Trigram dialect rules:
+        //  • every term is emitted as a QUOTED string — FTS5 then matches it
+        //    as a case-folded substring anywhere in the text, punctuation and
+        //    all ("192.55.226" matches literally; no dot-splitting, no `*`).
+        //    Only embedded double-quotes need stripping.
+        //  • terms under 3 chars can't form a trigram and MATCH nothing, so
+        //    they fall back to a LIKE over the header columns. Headers only:
+        //    scanning preview/to/cc for a 1-2 char pattern measured ~5s on
+        //    190k rows vs ~0.3s worst-case for headers, and short terms are
+        //    transient search-as-you-type states anyway.
+        const isTri = this.ftsIsTrigram();
+        const triClean = (t: string): string => t.replace(/"/g, "").trim();
+        const triQuote = (t: string): string => `"${t}"`;
+        const likeEsc = (t: string): string => t.replace(/[~%_]/g, m => `~${m}`);
+        // One OR-group per call: multiple terms land in the SAME group so an
+        // all-short alternation ("ab|cd") stays an OR, not an AND of groups.
+        const pushLike = (cols: string[], terms: string | string[]): void => {
+            const list = Array.isArray(terms) ? terms : [terms];
+            const clauses: string[] = [];
+            for (const t of list) {
+                for (const c of cols) {
+                    clauses.push(`${c} LIKE ? ESCAPE '~'`);
+                    extraParams.push(`%${likeEsc(t)}%`);
+                }
+            }
+            if (clauses.length) extraWhere.push("(" + clauses.join(" OR ") + ")");
+        };
+        const LIKE_FROM = ["m.from_name", "m.from_address"];
+        const LIKE_TO = ["m.to_json", "m.cc_json"];
+        const LIKE_CC = ["m.cc_json"];
+        const LIKE_SUBJ = ["m.subject"];
+        const LIKE_ANY = ["m.subject", "m.from_name", "m.from_address"];
+
+        for (let pi = 0; pi < parts.length; pi++) {
+            const part = parts[pi];
+            // A 1-2 char term next to a user-typed OR must be DROPPED, not
+            // LIKE'd: LIKE clauses AND against the MATCH, so "ab OR sprinkler"
+            // would intersect down to zero. Over-matching (just "sprinkler")
+            // beats returning nothing. (Trigram dialect only.)
+            const orAdjacent = parts[pi - 1] === "OR" || parts[pi + 1] === "OR";
             const fromMatch = part.match(/^from:(.+)$/i);
             const toMatch = part.match(/^to:(.+)$/i);
             const ccMatch = part.match(/^cc:(.+)$/i);
@@ -3719,21 +3913,45 @@ export class MailxDB {
             if (fromMatch) {
                 // FTS5 column-group `{c1 c2}:term` matches term in either
                 // column without the parens that break a trailing implicit-AND.
-                const term = ftsClean(fromMatch[1]);
-                if (term) frags.push({ s: `{from_name from_address}:${term}*`, op: false });
+                if (isTri) {
+                    const term = triClean(fromMatch[1]);
+                    if (term.length >= 3) frags.push({ s: `{from_name from_address}:${triQuote(term)}`, op: false });
+                    else if (term && !orAdjacent) pushLike(LIKE_FROM, term);
+                } else {
+                    const term = ftsClean(fromMatch[1]);
+                    if (term) frags.push({ s: `{from_name from_address}:${term}*`, op: false });
+                }
             } else if (toMatch) {
                 // `to:` deliberately spans To AND Cc — when someone searches
                 // "to:bob" they mean "addressed to bob", and being Cc'd counts.
                 // Use `cc:` for a Cc-only match. (Bcc isn't indexed: it's absent
                 // on received mail by design, and only meaningful in Sent.)
-                const term = ftsClean(toMatch[1]);
-                if (term) frags.push({ s: `{to_text cc_text}:${term}*`, op: false });
+                if (isTri) {
+                    const term = triClean(toMatch[1]);
+                    if (term.length >= 3) frags.push({ s: `{to_text cc_text}:${triQuote(term)}`, op: false });
+                    else if (term && !orAdjacent) pushLike(LIKE_TO, term);
+                } else {
+                    const term = ftsClean(toMatch[1]);
+                    if (term) frags.push({ s: `{to_text cc_text}:${term}*`, op: false });
+                }
             } else if (ccMatch) {
-                const term = ftsClean(ccMatch[1]);
-                if (term) frags.push({ s: `cc_text:${term}*`, op: false });
+                if (isTri) {
+                    const term = triClean(ccMatch[1]);
+                    if (term.length >= 3) frags.push({ s: `cc_text:${triQuote(term)}`, op: false });
+                    else if (term && !orAdjacent) pushLike(LIKE_CC, term);
+                } else {
+                    const term = ftsClean(ccMatch[1]);
+                    if (term) frags.push({ s: `cc_text:${term}*`, op: false });
+                }
             } else if (subjectMatch) {
-                const term = ftsClean(subjectMatch[1]);
-                if (term) frags.push({ s: `subject:${term}*`, op: false });
+                if (isTri) {
+                    const term = triClean(subjectMatch[1]);
+                    if (term.length >= 3) frags.push({ s: `subject:${triQuote(term)}`, op: false });
+                    else if (term && !orAdjacent) pushLike(LIKE_SUBJ, term);
+                } else {
+                    const term = ftsClean(subjectMatch[1]);
+                    if (term) frags.push({ s: `subject:${term}*`, op: false });
+                }
             } else if (dateMatch || afterMatch || beforeMatch) {
                 const op = dateMatch ? (dateMatch[1] || "=") : (afterMatch ? ">" : "<");
                 const valStr = dateMatch ? dateMatch[2] : (afterMatch ? afterMatch[1] : beforeMatch![1]);
@@ -3764,6 +3982,31 @@ export class MailxDB {
                 // (one of them being any word starting with "AND") — so a real
                 // match like "Peter Hoddie" + "github" returned zero hits.
                 frags.push({ s: part, op: true });
+            } else if (isTri) {
+                // Unqualified, trigram dialect — quoted substring match. A
+                // user-quoted "foo bar" phrase survives as one literal
+                // substring including the space (real phrase search, which
+                // the unicode61 dialect approximated with AND'd words).
+                const term = part.replace(/^\/|\/$/g, "");
+                if (term.includes("|")) {
+                    const alts = term.split("|").map(t => triClean(t)).filter(Boolean);
+                    const long = alts.filter(t => t.length >= 3);
+                    const short = alts.filter(t => t.length < 3);
+                    if (long.length) {
+                        // Mixed `abc|xy`: drop the short alternates. An OR
+                        // can't span MATCH and LIKE (LIKE would AND against
+                        // the MATCH and turn the union into an intersection
+                        // that returns nothing).
+                        frags.push({ s: `(${long.map(triQuote).join(" OR ")})`, op: false });
+                    } else if (short.length && !orAdjacent) {
+                        // All-short: one OR'd LIKE group keeps `ab|cd` a union.
+                        pushLike(LIKE_ANY, short);
+                    }
+                } else {
+                    const t = triClean(term);
+                    if (t.length >= 3) frags.push({ s: triQuote(t), op: false });
+                    else if (t && !orAdjacent) pushLike(LIKE_ANY, t);
+                }
             } else {
                 // Unqualified — search everything. Strip /regex-literal/
                 // slashes and FTS5 metacharacters before wildcarding.
@@ -3792,6 +4035,13 @@ export class MailxDB {
             }
         }
 
+        // A term diverted to the LIKE fallback can strand a user-typed
+        // operator at the edge of the MATCH string ("ab OR sprinkler" → the
+        // "ab" became a LIKE, leaving "OR sprinkler" — an FTS5 syntax error
+        // that would silently empty the whole search). Trim dangling operators.
+        while (frags.length && frags[0].op) frags.shift();
+        while (frags.length && frags[frags.length - 1].op) frags.pop();
+
         // Join fragments. Two adjacent value fragments need an explicit `AND`
         // (implicit-AND after a `(...)` / `{...}:` group is an FTS5 syntax
         // error); a user operator fragment glues itself, so no insert around it.
@@ -3912,8 +4162,12 @@ export class MailxDB {
         // Recreating it here used to quietly reintroduce that bug on every
         // -reindex (Bob 2026-06-05 "search missed to/cc/body").
         try { this.db.exec("DROP TABLE IF EXISTS messages_fts"); } catch { /* ignore */ }
+        // Trigram tokenization is ~10x slower to build than unicode61, so a
+        // full -reindex on a 190k-row DB takes minutes. Explicit user command,
+        // runs once — substring search is worth it.
         this.db.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts USING fts5(
-            subject, from_name, from_address, to_text, cc_text, body_text
+            subject, from_name, from_address, to_text, cc_text, body_text,
+            tokenize='trigram remove_diacritics 1'
         )`);
 
         // Use a single transaction + prepared statement for speed (~50x faster than individual inserts)