@bobfrankston/mailx-store 0.1.59 → 0.1.63

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (3) hide show
  1. package/db.d.ts +30 -0
  2. package/db.js +321 -18
  3. package/package.json +1 -1
package/db.d.ts CHANGED
@@ -439,6 +439,36 @@ export declare class MailxDB {
439
439
  * per DB: drop, recreate standalone, reindex subject/from + derived to/cc.
440
440
  * body_text backfills as bodies parse. */
441
441
  private migrateFtsSchema;
442
+ /** True while the trigram migration's shadow table exists — the FTS write
443
+ * paths (upsert / updateFtsBody) mirror into it so the copy loop never
444
+ * chases a moving target. TTL-cached: upserts run hundreds/sec during
445
+ * sync and the answer only changes twice in the DB's lifetime. Checked
446
+ * via sqlite_master (not an in-memory flag) because the migration runs on
447
+ * the main writer while upserts run on the sync worker's connection. */
448
+ private triShadowCache;
449
+ private ftsTriShadowExists;
450
+ /** Does messages_fts still use the old unicode61 word tokenizer? Checked
451
+ * per-call (not cached): the answer flips mid-session when the background
452
+ * migration swaps tables, and searchMessages must switch query dialects
453
+ * the moment it does — including on the read-worker's own connection. */
454
+ private ftsIsTrigram;
455
+ ftsNeedsTrigramMigration(): boolean;
456
+ /** One-time background conversion of messages_fts from the unicode61 word
457
+ * tokenizer to trigram (substring search — "sprinkler" now finds
458
+ * "OpenSprinkler", Bob 2026-07-19). Word tokens only match by prefix, so
459
+ * any mid-word search silently found nothing.
460
+ *
461
+ * Runs chunked on the main writer (~3-4 min for 190k rows, measured) so
462
+ * boot isn't blocked and sync writes stay responsive between chunks. The
463
+ * existing index keeps serving searches the whole time; concurrent FTS
464
+ * writes land in BOTH tables via ftsTriShadowExists() mirroring. The
465
+ * final swap (DROP + RENAME) is a single transaction, so a crash at any
466
+ * point leaves either the old table (migration restarts next boot) or
467
+ * the finished trigram table — never neither. Stored body_text survives
468
+ * because the copy reads it straight out of the old standalone table.
469
+ * Call ONLY from the long-lived daemon: a one-shot CLI command would sit
470
+ * alive for minutes finishing the copy. */
471
+ startFtsTrigramMigration(): void;
442
472
  /** Record a prefetch failure (0-body fetch / store-write fail) for a UID,
443
473
  * incrementing its backoff count. Persisted so it survives restarts. */
444
474
  recordPrefetchFailure(accountId: string, folderId: number, uid: number): void;
package/db.js CHANGED
@@ -219,8 +219,14 @@ const SCHEMA = `
219
219
  -- the index stores its own text (we INSERT it explicitly on upsert), so
220
220
  -- there's nothing to dereference and DELETE/INSERT/reindex all work. The
221
221
  -- migrateFtsSchema() pass below rebuilds an existing external-content table.
222
+ -- Tokenizer is TRIGRAM (not unicode61): unicode61 indexes whole words, so
223
+ -- "opens" prefix-matched "opensprinkler" but "sprinkler" found nothing
224
+ -- (Bob 2026-07-19). Trigram matches any >=3-char substring; searchMessages
225
+ -- falls back to LIKE for 1-2 char terms. Existing DBs with the old
226
+ -- tokenizer are converted in the background by startFtsTrigramMigration().
222
227
  CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts USING fts5(
223
- subject, from_name, from_address, to_text, cc_text, body_text
228
+ subject, from_name, from_address, to_text, cc_text, body_text,
229
+ tokenize='trigram remove_diacritics 1'
224
230
  );
225
231
 
226
232
  CREATE TABLE IF NOT EXISTS sync_actions (
@@ -1815,8 +1821,21 @@ export class MailxDB {
1815
1821
  // + from/to. Standalone FTS5 → delete + insert. body_text
1816
1822
  // backfills when the body parses.
1817
1823
  try {
1824
+ const vals = [
1825
+ existing.id, msg.subject || "", msg.from?.name || "", msg.from?.address || "",
1826
+ (msg.to || []).map(a => `${a.name || ""} ${a.address || ""}`).join(" "),
1827
+ (msg.cc || []).map(a => `${a.name || ""} ${a.address || ""}`).join(" "),
1828
+ ];
1818
1829
  this.db.prepare("DELETE FROM messages_fts WHERE rowid = ?").run(existing.id);
1819
- this.db.prepare("INSERT INTO messages_fts (rowid, subject, from_name, from_address, to_text, cc_text, body_text) VALUES (?, ?, ?, ?, ?, ?, '')").run(existing.id, msg.subject || "", msg.from?.name || "", msg.from?.address || "", (msg.to || []).map(a => `${a.name || ""} ${a.address || ""}`).join(" "), (msg.cc || []).map(a => `${a.name || ""} ${a.address || ""}`).join(" "));
1830
+ this.db.prepare("INSERT INTO messages_fts (rowid, subject, from_name, from_address, to_text, cc_text, body_text) VALUES (?, ?, ?, ?, ?, ?, '')").run(...vals);
1831
+ // Mirror into the trigram-migration shadow so the swap
1832
+ // doesn't lose this re-index (see startFtsTrigramMigration).
1833
+ if (this.ftsTriShadowExists()) {
1834
+ try {
1835
+ this.db.prepare("INSERT OR REPLACE INTO messages_fts_tri (rowid, subject, from_name, from_address, to_text, cc_text, body_text) VALUES (?, ?, ?, ?, ?, ?, '')").run(...vals);
1836
+ }
1837
+ catch { /* shadow may have just been swapped away */ }
1838
+ }
1820
1839
  }
1821
1840
  catch { /* best-effort */ }
1822
1841
  }
@@ -1950,6 +1969,13 @@ export class MailxDB {
1950
1969
  // which is in <...>.eml but it was not found") never match.
1951
1970
  try {
1952
1971
  this.db.prepare("INSERT INTO messages_fts (rowid, subject, from_name, from_address, to_text, cc_text, body_text) VALUES (?, ?, ?, ?, ?, ?, ?)").run(rowId, msg.subject, msg.from.name, msg.from.address, toText, ccText, msg.preview);
1972
+ // Mirror into the trigram-migration shadow (see startFtsTrigramMigration).
1973
+ if (this.ftsTriShadowExists()) {
1974
+ try {
1975
+ this.db.prepare("INSERT OR REPLACE INTO messages_fts_tri (rowid, subject, from_name, from_address, to_text, cc_text, body_text) VALUES (?, ?, ?, ?, ?, ?, ?)").run(rowId, msg.subject, msg.from.name, msg.from.address, toText, ccText, msg.preview);
1976
+ }
1977
+ catch { /* shadow may have just been swapped away */ }
1978
+ }
1953
1979
  }
1954
1980
  catch { /* FTS insert may fail on rebuild, non-fatal */ }
1955
1981
  return rowId;
@@ -1971,6 +1997,15 @@ export class MailxDB {
1971
1997
  try {
1972
1998
  const capped = bodyText.length > 64_000 ? bodyText.slice(0, 64_000) : bodyText;
1973
1999
  this.db.prepare("UPDATE messages_fts SET body_text = ? WHERE rowid = ?").run(capped, rowId);
2000
+ // Mirror into the trigram-migration shadow (see startFtsTrigramMigration).
2001
+ // No-op if the copy loop hasn't reached this rowid yet — the chunk
2002
+ // will carry the (also-updated) old-table value across later.
2003
+ if (this.ftsTriShadowExists()) {
2004
+ try {
2005
+ this.db.prepare("UPDATE messages_fts_tri SET body_text = ? WHERE rowid = ?").run(capped, rowId);
2006
+ }
2007
+ catch { /* shadow may have just been swapped away */ }
2008
+ }
1974
2009
  }
1975
2010
  catch { /* FTS update is best-effort */ }
1976
2011
  }
@@ -2503,7 +2538,7 @@ export class MailxDB {
2503
2538
  return; // already standalone
2504
2539
  console.log(" [db] messages_fts had broken external-content schema — rebuilding standalone + reindexing");
2505
2540
  this.db.exec("DROP TABLE IF EXISTS messages_fts");
2506
- this.db.exec("CREATE VIRTUAL TABLE messages_fts USING fts5(subject, from_name, from_address, to_text, cc_text, body_text)");
2541
+ this.db.exec("CREATE VIRTUAL TABLE messages_fts USING fts5(subject, from_name, from_address, to_text, cc_text, body_text, tokenize='trigram remove_diacritics 1')");
2507
2542
  const addrText = (j) => {
2508
2543
  try {
2509
2544
  return JSON.parse(j || "[]").map(a => `${a.name || ""} ${a.address || ""}`).join(" ");
@@ -2533,6 +2568,149 @@ export class MailxDB {
2533
2568
  console.error(` [db] migrateFtsSchema failed: ${e?.message || e}`);
2534
2569
  }
2535
2570
  }
2571
+ /** True while the trigram migration's shadow table exists — the FTS write
2572
+ * paths (upsert / updateFtsBody) mirror into it so the copy loop never
2573
+ * chases a moving target. TTL-cached: upserts run hundreds/sec during
2574
+ * sync and the answer only changes twice in the DB's lifetime. Checked
2575
+ * via sqlite_master (not an in-memory flag) because the migration runs on
2576
+ * the main writer while upserts run on the sync worker's connection. */
2577
+ triShadowCache = null;
2578
+ ftsTriShadowExists() {
2579
+ const now = Date.now();
2580
+ if (this.triShadowCache && now - this.triShadowCache.at < 2000)
2581
+ return this.triShadowCache.exists;
2582
+ let exists = false;
2583
+ try {
2584
+ exists = !!this.db.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name='messages_fts_tri'").get();
2585
+ }
2586
+ catch { /* treat as absent */ }
2587
+ this.triShadowCache = { at: now, exists };
2588
+ return exists;
2589
+ }
2590
+ /** Does messages_fts still use the old unicode61 word tokenizer? Checked
2591
+ * per-call (not cached): the answer flips mid-session when the background
2592
+ * migration swaps tables, and searchMessages must switch query dialects
2593
+ * the moment it does — including on the read-worker's own connection. */
2594
+ ftsIsTrigram() {
2595
+ try {
2596
+ const row = this.db.prepare("SELECT sql FROM sqlite_master WHERE type='table' AND name='messages_fts'").get();
2597
+ return !!row && /trigram/i.test(row.sql || "");
2598
+ }
2599
+ catch {
2600
+ return false;
2601
+ }
2602
+ }
2603
+ ftsNeedsTrigramMigration() {
2604
+ if (this.readOnly)
2605
+ return false;
2606
+ try {
2607
+ const row = this.db.prepare("SELECT sql FROM sqlite_master WHERE type='table' AND name='messages_fts'").get();
2608
+ return !!row && !/trigram/i.test(row.sql || "");
2609
+ }
2610
+ catch {
2611
+ return false;
2612
+ }
2613
+ }
2614
+ /** One-time background conversion of messages_fts from the unicode61 word
2615
+ * tokenizer to trigram (substring search — "sprinkler" now finds
2616
+ * "OpenSprinkler", Bob 2026-07-19). Word tokens only match by prefix, so
2617
+ * any mid-word search silently found nothing.
2618
+ *
2619
+ * Runs chunked on the main writer (~3-4 min for 190k rows, measured) so
2620
+ * boot isn't blocked and sync writes stay responsive between chunks. The
2621
+ * existing index keeps serving searches the whole time; concurrent FTS
2622
+ * writes land in BOTH tables via ftsTriShadowExists() mirroring. The
2623
+ * final swap (DROP + RENAME) is a single transaction, so a crash at any
2624
+ * point leaves either the old table (migration restarts next boot) or
2625
+ * the finished trigram table — never neither. Stored body_text survives
2626
+ * because the copy reads it straight out of the old standalone table.
2627
+ * Call ONLY from the long-lived daemon: a one-shot CLI command would sit
2628
+ * alive for minutes finishing the copy. */
2629
+ startFtsTrigramMigration() {
2630
+ if (!this.ftsNeedsTrigramMigration())
2631
+ return;
2632
+ const COLS = "rowid, subject, from_name, from_address, to_text, cc_text, body_text";
2633
+ try {
2634
+ // A leftover shadow from an interrupted run may be stale (writes
2635
+ // that happened while no daemon mirrored them) — start clean.
2636
+ this.db.exec("DROP TABLE IF EXISTS messages_fts_tri");
2637
+ this.db.exec("CREATE VIRTUAL TABLE messages_fts_tri USING fts5(subject, from_name, from_address, to_text, cc_text, body_text, tokenize='trigram remove_diacritics 1')");
2638
+ }
2639
+ catch (e) {
2640
+ console.error(` [db] FTS trigram migration setup failed: ${e?.message || e}`);
2641
+ return;
2642
+ }
2643
+ let total = 0;
2644
+ try {
2645
+ total = this.db.prepare("SELECT COUNT(*) AS c FROM messages_fts").get()?.c || 0;
2646
+ }
2647
+ catch { /* */ }
2648
+ console.log(` [db] Reindexing search index: ${total} messages in background (~5 min; substring search when done)`);
2649
+ const CHUNK = 500; // ~0.35s of tokenization per chunk (measured on Bob's DB)
2650
+ let last = 0, copied = 0, chunkN = 0;
2651
+ const step = () => {
2652
+ try {
2653
+ const rows = this.db.prepare("SELECT rowid FROM messages_fts WHERE rowid > ? ORDER BY rowid LIMIT ?").all(last, CHUNK);
2654
+ if (rows.length === 0) {
2655
+ finish();
2656
+ return;
2657
+ }
2658
+ const hi = rows[rows.length - 1].rowid;
2659
+ // Single INSERT..SELECT — atomic against the sync worker's
2660
+ // mirrored writes (WAL serializes writers), so old and shadow
2661
+ // can't diverge inside a chunk. OR REPLACE because mirrored
2662
+ // writes may have already landed rows past the frontier.
2663
+ this.db.prepare(`INSERT OR REPLACE INTO messages_fts_tri (${COLS}) SELECT ${COLS} FROM messages_fts WHERE rowid > ? AND rowid <= ?`).run(last, hi);
2664
+ last = hi;
2665
+ copied += rows.length;
2666
+ chunkN++;
2667
+ if (chunkN % 40 === 0) {
2668
+ console.log(` [db] Reindexing search index: ${copied}/${total}`);
2669
+ // The copy writes ~1 GB through the WAL in total — keep it drained.
2670
+ try {
2671
+ this.db.exec("PRAGMA wal_checkpoint(PASSIVE)");
2672
+ }
2673
+ catch { /* */ }
2674
+ }
2675
+ setTimeout(step, 100); // yield the writer between chunks
2676
+ }
2677
+ catch (e) {
2678
+ console.error(` [db] FTS trigram migration chunk failed (${e?.message || e}) — retrying in 5s`);
2679
+ setTimeout(step, 5000);
2680
+ }
2681
+ };
2682
+ const finish = () => {
2683
+ try {
2684
+ this.db.exec("BEGIN IMMEDIATE");
2685
+ // Rows past the frontier that landed after the last chunk are
2686
+ // already mirrored into the shadow, but belt-and-braces: if any
2687
+ // exist, loop once more instead of trusting the mirror.
2688
+ const strag = this.db.prepare("SELECT COUNT(*) AS c FROM messages_fts WHERE rowid > ?").get(last)?.c || 0;
2689
+ if (strag > 0) {
2690
+ this.db.exec("ROLLBACK");
2691
+ setTimeout(step, 100);
2692
+ return;
2693
+ }
2694
+ this.db.exec("DROP TABLE messages_fts");
2695
+ this.db.exec("ALTER TABLE messages_fts_tri RENAME TO messages_fts");
2696
+ this.db.exec("COMMIT");
2697
+ console.log(` [db] Search reindex complete: ${copied} messages — substring search active`);
2698
+ try {
2699
+ this.db.exec("PRAGMA wal_checkpoint(PASSIVE)");
2700
+ }
2701
+ catch { /* */ }
2702
+ }
2703
+ catch (e) {
2704
+ try {
2705
+ this.db.exec("ROLLBACK");
2706
+ }
2707
+ catch { /* */ }
2708
+ console.error(` [db] FTS trigram migration swap failed (${e?.message || e}) — retrying in 5s`);
2709
+ setTimeout(finish, 5000);
2710
+ }
2711
+ };
2712
+ setTimeout(step, 3000); // let boot (folder loads, first sync) settle first
2713
+ }
2536
2714
  /** Record a prefetch failure (0-body fetch / store-write fail) for a UID,
2537
2715
  * incrementing its backoff count. Persisted so it survives restarts. */
2538
2716
  recordPrefetchFailure(accountId, folderId, uid) {
@@ -3432,9 +3610,56 @@ export class MailxDB {
3432
3610
  // Strip FTS5 metacharacters from a user term. A stray `;` `(` `:` `"`
3433
3611
  // etc. is a hard `fts5: syntax error` (Bob 2026-05-21 log). FTS5's
3434
3612
  // tokenizer splits on non-word chars anyway, so removing them loses no
3435
- // real matching ability.
3613
+ // real matching ability. (unicode61 dialect only — see isTri below.)
3436
3614
  const ftsClean = (t) => t.replace(/["';:(){}\[\]^~\\/]/g, "").trim();
3437
- for (const part of parts) {
3615
+ // ── Tokenizer dialects ──
3616
+ // The index is migrating from unicode61 (word tokens, prefix-only
3617
+ // matching — "sprinkler" could never find "opensprinkler") to trigram
3618
+ // (substring matching, Bob 2026-07-19). Which dialect to emit depends
3619
+ // on which table is live RIGHT NOW: the background migration swaps it
3620
+ // mid-session, and this runs on the read-worker's own connection, so
3621
+ // ask sqlite_master rather than trusting boot-time state.
3622
+ //
3623
+ // Trigram dialect rules:
3624
+ // • every term is emitted as a QUOTED string — FTS5 then matches it
3625
+ // as a case-folded substring anywhere in the text, punctuation and
3626
+ // all ("192.55.226" matches literally; no dot-splitting, no `*`).
3627
+ // Only embedded double-quotes need stripping.
3628
+ // • terms under 3 chars can't form a trigram and MATCH nothing, so
3629
+ // they fall back to a LIKE over the header columns. Headers only:
3630
+ // scanning preview/to/cc for a 1-2 char pattern measured ~5s on
3631
+ // 190k rows vs ~0.3s worst-case for headers, and short terms are
3632
+ // transient search-as-you-type states anyway.
3633
+ const isTri = this.ftsIsTrigram();
3634
+ const triClean = (t) => t.replace(/"/g, "").trim();
3635
+ const triQuote = (t) => `"${t}"`;
3636
+ const likeEsc = (t) => t.replace(/[~%_]/g, m => `~${m}`);
3637
+ // One OR-group per call: multiple terms land in the SAME group so an
3638
+ // all-short alternation ("ab|cd") stays an OR, not an AND of groups.
3639
+ const pushLike = (cols, terms) => {
3640
+ const list = Array.isArray(terms) ? terms : [terms];
3641
+ const clauses = [];
3642
+ for (const t of list) {
3643
+ for (const c of cols) {
3644
+ clauses.push(`${c} LIKE ? ESCAPE '~'`);
3645
+ extraParams.push(`%${likeEsc(t)}%`);
3646
+ }
3647
+ }
3648
+ if (clauses.length)
3649
+ extraWhere.push("(" + clauses.join(" OR ") + ")");
3650
+ };
3651
+ const LIKE_FROM = ["m.from_name", "m.from_address"];
3652
+ const LIKE_TO = ["m.to_json", "m.cc_json"];
3653
+ const LIKE_CC = ["m.cc_json"];
3654
+ const LIKE_SUBJ = ["m.subject"];
3655
+ const LIKE_ANY = ["m.subject", "m.from_name", "m.from_address"];
3656
+ for (let pi = 0; pi < parts.length; pi++) {
3657
+ const part = parts[pi];
3658
+ // A 1-2 char term next to a user-typed OR must be DROPPED, not
3659
+ // LIKE'd: LIKE clauses AND against the MATCH, so "ab OR sprinkler"
3660
+ // would intersect down to zero. Over-matching (just "sprinkler")
3661
+ // beats returning nothing. (Trigram dialect only.)
3662
+ const orAdjacent = parts[pi - 1] === "OR" || parts[pi + 1] === "OR";
3438
3663
  const fromMatch = part.match(/^from:(.+)$/i);
3439
3664
  const toMatch = part.match(/^to:(.+)$/i);
3440
3665
  const ccMatch = part.match(/^cc:(.+)$/i);
@@ -3448,28 +3673,64 @@ export class MailxDB {
3448
3673
  if (fromMatch) {
3449
3674
  // FTS5 column-group `{c1 c2}:term` matches term in either
3450
3675
  // column without the parens that break a trailing implicit-AND.
3451
- const term = ftsClean(fromMatch[1]);
3452
- if (term)
3453
- frags.push({ s: `{from_name from_address}:${term}*`, op: false });
3676
+ if (isTri) {
3677
+ const term = triClean(fromMatch[1]);
3678
+ if (term.length >= 3)
3679
+ frags.push({ s: `{from_name from_address}:${triQuote(term)}`, op: false });
3680
+ else if (term && !orAdjacent)
3681
+ pushLike(LIKE_FROM, term);
3682
+ }
3683
+ else {
3684
+ const term = ftsClean(fromMatch[1]);
3685
+ if (term)
3686
+ frags.push({ s: `{from_name from_address}:${term}*`, op: false });
3687
+ }
3454
3688
  }
3455
3689
  else if (toMatch) {
3456
3690
  // `to:` deliberately spans To AND Cc — when someone searches
3457
3691
  // "to:bob" they mean "addressed to bob", and being Cc'd counts.
3458
3692
  // Use `cc:` for a Cc-only match. (Bcc isn't indexed: it's absent
3459
3693
  // on received mail by design, and only meaningful in Sent.)
3460
- const term = ftsClean(toMatch[1]);
3461
- if (term)
3462
- frags.push({ s: `{to_text cc_text}:${term}*`, op: false });
3694
+ if (isTri) {
3695
+ const term = triClean(toMatch[1]);
3696
+ if (term.length >= 3)
3697
+ frags.push({ s: `{to_text cc_text}:${triQuote(term)}`, op: false });
3698
+ else if (term && !orAdjacent)
3699
+ pushLike(LIKE_TO, term);
3700
+ }
3701
+ else {
3702
+ const term = ftsClean(toMatch[1]);
3703
+ if (term)
3704
+ frags.push({ s: `{to_text cc_text}:${term}*`, op: false });
3705
+ }
3463
3706
  }
3464
3707
  else if (ccMatch) {
3465
- const term = ftsClean(ccMatch[1]);
3466
- if (term)
3467
- frags.push({ s: `cc_text:${term}*`, op: false });
3708
+ if (isTri) {
3709
+ const term = triClean(ccMatch[1]);
3710
+ if (term.length >= 3)
3711
+ frags.push({ s: `cc_text:${triQuote(term)}`, op: false });
3712
+ else if (term && !orAdjacent)
3713
+ pushLike(LIKE_CC, term);
3714
+ }
3715
+ else {
3716
+ const term = ftsClean(ccMatch[1]);
3717
+ if (term)
3718
+ frags.push({ s: `cc_text:${term}*`, op: false });
3719
+ }
3468
3720
  }
3469
3721
  else if (subjectMatch) {
3470
- const term = ftsClean(subjectMatch[1]);
3471
- if (term)
3472
- frags.push({ s: `subject:${term}*`, op: false });
3722
+ if (isTri) {
3723
+ const term = triClean(subjectMatch[1]);
3724
+ if (term.length >= 3)
3725
+ frags.push({ s: `subject:${triQuote(term)}`, op: false });
3726
+ else if (term && !orAdjacent)
3727
+ pushLike(LIKE_SUBJ, term);
3728
+ }
3729
+ else {
3730
+ const term = ftsClean(subjectMatch[1]);
3731
+ if (term)
3732
+ frags.push({ s: `subject:${term}*`, op: false });
3733
+ }
3473
3734
  }
3474
3735
  else if (dateMatch || afterMatch || beforeMatch) {
3475
3736
  const op = dateMatch ? (dateMatch[1] || "=") : (afterMatch ? ">" : "<");
@@ -3522,6 +3783,36 @@ export class MailxDB {
3522
3783
  // match like "Peter Hoddie" + "github" returned zero hits.
3523
3784
  frags.push({ s: part, op: true });
3524
3785
  }
3786
+ else if (isTri) {
3787
+ // Unqualified, trigram dialect — quoted substring match. A
3788
+ // user-quoted "foo bar" phrase survives as one literal
3789
+ // substring including the space (real phrase search, which
3790
+ // the unicode61 dialect approximated with AND'd words).
3791
+ const term = part.replace(/^\/|\/$/g, "");
3792
+ if (term.includes("|")) {
3793
+ const alts = term.split("|").map(t => triClean(t)).filter(Boolean);
3794
+ const long = alts.filter(t => t.length >= 3);
3795
+ const short = alts.filter(t => t.length < 3);
3796
+ if (long.length) {
3797
+ // Mixed `abc|xy`: drop the short alternates. An OR
3798
+ // can't span MATCH and LIKE (LIKE would AND against
3799
+ // the MATCH and turn the union into an intersection
3800
+ // that returns nothing).
3801
+ frags.push({ s: `(${long.map(triQuote).join(" OR ")})`, op: false });
3802
+ }
3803
+ else if (short.length && !orAdjacent) {
3804
+ // All-short: one OR'd LIKE group keeps `ab|cd` a union.
3805
+ pushLike(LIKE_ANY, short);
3806
+ }
3807
+ }
3808
+ else {
3809
+ const t = triClean(term);
3810
+ if (t.length >= 3)
3811
+ frags.push({ s: triQuote(t), op: false });
3812
+ else if (t && !orAdjacent)
3813
+ pushLike(LIKE_ANY, t);
3814
+ }
3815
+ }
3525
3816
  else {
3526
3817
  // Unqualified — search everything. Strip /regex-literal/
3527
3818
  // slashes and FTS5 metacharacters before wildcarding.
@@ -3552,6 +3843,14 @@ export class MailxDB {
3552
3843
  }
3553
3844
  }
3554
3845
  }
3846
+ // A term diverted to the LIKE fallback can strand a user-typed
3847
+ // operator at the edge of the MATCH string ("ab OR sprinkler" → the
3848
+ // "ab" became a LIKE, leaving "OR sprinkler" — an FTS5 syntax error
3849
+ // that would silently empty the whole search). Trim dangling operators.
3850
+ while (frags.length && frags[0].op)
3851
+ frags.shift();
3852
+ while (frags.length && frags[frags.length - 1].op)
3853
+ frags.pop();
3555
3854
  // Join fragments. Two adjacent value fragments need an explicit `AND`
3556
3855
  // (implicit-AND after a `(...)` / `{...}:` group is an FTS5 syntax
3557
3856
  // error); a user operator fragment glues itself, so no insert around it.
@@ -3668,8 +3967,12 @@ export class MailxDB {
3668
3967
  this.db.exec("DROP TABLE IF EXISTS messages_fts");
3669
3968
  }
3670
3969
  catch { /* ignore */ }
3970
+ // Trigram tokenization is ~10x slower to build than unicode61, so a
3971
+ // full -reindex on a 190k-row DB takes minutes. Explicit user command,
3972
+ // runs once — substring search is worth it.
3671
3973
  this.db.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts USING fts5(
3672
- subject, from_name, from_address, to_text, cc_text, body_text
3974
+ subject, from_name, from_address, to_text, cc_text, body_text,
3975
+ tokenize='trigram remove_diacritics 1'
3673
3976
  )`);
3674
3977
  // Use a single transaction + prepared statement for speed (~50x faster than individual inserts)
3675
3978
  const insert = this.db.prepare("INSERT INTO messages_fts (rowid, subject, from_name, from_address, to_text, cc_text, body_text) VALUES (?, ?, ?, ?, ?, ?, ?)");
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@bobfrankston/mailx-store",
3
- "version": "0.1.59",
3
+ "version": "0.1.63",
4
4
  "type": "module",
5
5
  "main": "index.js",
6
6
  "types": "index.d.ts",