@bobfrankston/mailx-store 0.1.59 → 0.1.63
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/db.d.ts +30 -0
- package/db.js +321 -18
- package/package.json +1 -1
package/db.d.ts
CHANGED
|
@@ -439,6 +439,36 @@ export declare class MailxDB {
|
|
|
439
439
|
* per DB: drop, recreate standalone, reindex subject/from + derived to/cc.
|
|
440
440
|
* body_text backfills as bodies parse. */
|
|
441
441
|
private migrateFtsSchema;
|
|
442
|
+
/** True while the trigram migration's shadow table exists — the FTS write
|
|
443
|
+
* paths (upsert / updateFtsBody) mirror into it so the copy loop never
|
|
444
|
+
* chases a moving target. TTL-cached: upserts run hundreds/sec during
|
|
445
|
+
* sync and the answer only changes twice in the DB's lifetime. Checked
|
|
446
|
+
* via sqlite_master (not an in-memory flag) because the migration runs on
|
|
447
|
+
* the main writer while upserts run on the sync worker's connection. */
|
|
448
|
+
private triShadowCache;
|
|
449
|
+
private ftsTriShadowExists;
|
|
450
|
+
/** Does messages_fts still use the old unicode61 word tokenizer? Checked
|
|
451
|
+
* per-call (not cached): the answer flips mid-session when the background
|
|
452
|
+
* migration swaps tables, and searchMessages must switch query dialects
|
|
453
|
+
* the moment it does — including on the read-worker's own connection. */
|
|
454
|
+
private ftsIsTrigram;
|
|
455
|
+
ftsNeedsTrigramMigration(): boolean;
|
|
456
|
+
/** One-time background conversion of messages_fts from the unicode61 word
|
|
457
|
+
* tokenizer to trigram (substring search — "sprinkler" now finds
|
|
458
|
+
* "OpenSprinkler", Bob 2026-07-19). Word tokens only match by prefix, so
|
|
459
|
+
* any mid-word search silently found nothing.
|
|
460
|
+
*
|
|
461
|
+
* Runs chunked on the main writer (~3-4 min for 190k rows, measured) so
|
|
462
|
+
* boot isn't blocked and sync writes stay responsive between chunks. The
|
|
463
|
+
* existing index keeps serving searches the whole time; concurrent FTS
|
|
464
|
+
* writes land in BOTH tables via ftsTriShadowExists() mirroring. The
|
|
465
|
+
* final swap (DROP + RENAME) is a single transaction, so a crash at any
|
|
466
|
+
* point leaves either the old table (migration restarts next boot) or
|
|
467
|
+
* the finished trigram table — never neither. Stored body_text survives
|
|
468
|
+
* because the copy reads it straight out of the old standalone table.
|
|
469
|
+
* Call ONLY from the long-lived daemon: a one-shot CLI command would sit
|
|
470
|
+
* alive for minutes finishing the copy. */
|
|
471
|
+
startFtsTrigramMigration(): void;
|
|
442
472
|
/** Record a prefetch failure (0-body fetch / store-write fail) for a UID,
|
|
443
473
|
* incrementing its backoff count. Persisted so it survives restarts. */
|
|
444
474
|
recordPrefetchFailure(accountId: string, folderId: number, uid: number): void;
|
package/db.js
CHANGED
|
@@ -219,8 +219,14 @@ const SCHEMA = `
|
|
|
219
219
|
-- the index stores its own text (we INSERT it explicitly on upsert), so
|
|
220
220
|
-- there's nothing to dereference and DELETE/INSERT/reindex all work. The
|
|
221
221
|
-- migrateFtsSchema() pass below rebuilds an existing external-content table.
|
|
222
|
+
-- Tokenizer is TRIGRAM (not unicode61): unicode61 indexes whole words, so
|
|
223
|
+
-- "opens" prefix-matched "opensprinkler" but "sprinkler" found nothing
|
|
224
|
+
-- (Bob 2026-07-19). Trigram matches any >=3-char substring; searchMessages
|
|
225
|
+
-- falls back to LIKE for 1-2 char terms. Existing DBs with the old
|
|
226
|
+
-- tokenizer are converted in the background by startFtsTrigramMigration().
|
|
222
227
|
CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts USING fts5(
|
|
223
|
-
subject, from_name, from_address, to_text, cc_text, body_text
|
|
228
|
+
subject, from_name, from_address, to_text, cc_text, body_text,
|
|
229
|
+
tokenize='trigram remove_diacritics 1'
|
|
224
230
|
);
|
|
225
231
|
|
|
226
232
|
CREATE TABLE IF NOT EXISTS sync_actions (
|
|
@@ -1815,8 +1821,21 @@ export class MailxDB {
|
|
|
1815
1821
|
// + from/to. Standalone FTS5 → delete + insert. body_text
|
|
1816
1822
|
// backfills when the body parses.
|
|
1817
1823
|
try {
|
|
1824
|
+
const vals = [
|
|
1825
|
+
existing.id, msg.subject || "", msg.from?.name || "", msg.from?.address || "",
|
|
1826
|
+
(msg.to || []).map(a => `${a.name || ""} ${a.address || ""}`).join(" "),
|
|
1827
|
+
(msg.cc || []).map(a => `${a.name || ""} ${a.address || ""}`).join(" "),
|
|
1828
|
+
];
|
|
1818
1829
|
this.db.prepare("DELETE FROM messages_fts WHERE rowid = ?").run(existing.id);
|
|
1819
|
-
this.db.prepare("INSERT INTO messages_fts (rowid, subject, from_name, from_address, to_text, cc_text, body_text) VALUES (?, ?, ?, ?, ?, ?, '')").run(
|
|
1830
|
+
this.db.prepare("INSERT INTO messages_fts (rowid, subject, from_name, from_address, to_text, cc_text, body_text) VALUES (?, ?, ?, ?, ?, ?, '')").run(...vals);
|
|
1831
|
+
// Mirror into the trigram-migration shadow so the swap
|
|
1832
|
+
// doesn't lose this re-index (see startFtsTrigramMigration).
|
|
1833
|
+
if (this.ftsTriShadowExists()) {
|
|
1834
|
+
try {
|
|
1835
|
+
this.db.prepare("INSERT OR REPLACE INTO messages_fts_tri (rowid, subject, from_name, from_address, to_text, cc_text, body_text) VALUES (?, ?, ?, ?, ?, ?, '')").run(...vals);
|
|
1836
|
+
}
|
|
1837
|
+
catch { /* shadow may have just been swapped away */ }
|
|
1838
|
+
}
|
|
1820
1839
|
}
|
|
1821
1840
|
catch { /* best-effort */ }
|
|
1822
1841
|
}
|
|
@@ -1950,6 +1969,13 @@ export class MailxDB {
|
|
|
1950
1969
|
// which is in <...>.eml but it was not found") never match.
|
|
1951
1970
|
try {
|
|
1952
1971
|
this.db.prepare("INSERT INTO messages_fts (rowid, subject, from_name, from_address, to_text, cc_text, body_text) VALUES (?, ?, ?, ?, ?, ?, ?)").run(rowId, msg.subject, msg.from.name, msg.from.address, toText, ccText, msg.preview);
|
|
1972
|
+
// Mirror into the trigram-migration shadow (see startFtsTrigramMigration).
|
|
1973
|
+
if (this.ftsTriShadowExists()) {
|
|
1974
|
+
try {
|
|
1975
|
+
this.db.prepare("INSERT OR REPLACE INTO messages_fts_tri (rowid, subject, from_name, from_address, to_text, cc_text, body_text) VALUES (?, ?, ?, ?, ?, ?, ?)").run(rowId, msg.subject, msg.from.name, msg.from.address, toText, ccText, msg.preview);
|
|
1976
|
+
}
|
|
1977
|
+
catch { /* shadow may have just been swapped away */ }
|
|
1978
|
+
}
|
|
1953
1979
|
}
|
|
1954
1980
|
catch { /* FTS insert may fail on rebuild, non-fatal */ }
|
|
1955
1981
|
return rowId;
|
|
@@ -1971,6 +1997,15 @@ export class MailxDB {
|
|
|
1971
1997
|
try {
|
|
1972
1998
|
const capped = bodyText.length > 64_000 ? bodyText.slice(0, 64_000) : bodyText;
|
|
1973
1999
|
this.db.prepare("UPDATE messages_fts SET body_text = ? WHERE rowid = ?").run(capped, rowId);
|
|
2000
|
+
// Mirror into the trigram-migration shadow (see startFtsTrigramMigration).
|
|
2001
|
+
// No-op if the copy loop hasn't reached this rowid yet — the chunk
|
|
2002
|
+
// will carry the (also-updated) old-table value across later.
|
|
2003
|
+
if (this.ftsTriShadowExists()) {
|
|
2004
|
+
try {
|
|
2005
|
+
this.db.prepare("UPDATE messages_fts_tri SET body_text = ? WHERE rowid = ?").run(capped, rowId);
|
|
2006
|
+
}
|
|
2007
|
+
catch { /* shadow may have just been swapped away */ }
|
|
2008
|
+
}
|
|
1974
2009
|
}
|
|
1975
2010
|
catch { /* FTS update is best-effort */ }
|
|
1976
2011
|
}
|
|
@@ -2503,7 +2538,7 @@ export class MailxDB {
|
|
|
2503
2538
|
return; // already standalone
|
|
2504
2539
|
console.log(" [db] messages_fts had broken external-content schema — rebuilding standalone + reindexing");
|
|
2505
2540
|
this.db.exec("DROP TABLE IF EXISTS messages_fts");
|
|
2506
|
-
this.db.exec("CREATE VIRTUAL TABLE messages_fts USING fts5(subject, from_name, from_address, to_text, cc_text, body_text)");
|
|
2541
|
+
this.db.exec("CREATE VIRTUAL TABLE messages_fts USING fts5(subject, from_name, from_address, to_text, cc_text, body_text, tokenize='trigram remove_diacritics 1')");
|
|
2507
2542
|
const addrText = (j) => {
|
|
2508
2543
|
try {
|
|
2509
2544
|
return JSON.parse(j || "[]").map(a => `${a.name || ""} ${a.address || ""}`).join(" ");
|
|
@@ -2533,6 +2568,149 @@ export class MailxDB {
|
|
|
2533
2568
|
console.error(` [db] migrateFtsSchema failed: ${e?.message || e}`);
|
|
2534
2569
|
}
|
|
2535
2570
|
}
|
|
2571
|
+
/** True while the trigram migration's shadow table exists — the FTS write
|
|
2572
|
+
* paths (upsert / updateFtsBody) mirror into it so the copy loop never
|
|
2573
|
+
* chases a moving target. TTL-cached: upserts run hundreds/sec during
|
|
2574
|
+
* sync and the answer only changes twice in the DB's lifetime. Checked
|
|
2575
|
+
* via sqlite_master (not an in-memory flag) because the migration runs on
|
|
2576
|
+
* the main writer while upserts run on the sync worker's connection. */
|
|
2577
|
+
triShadowCache = null;
|
|
2578
|
+
ftsTriShadowExists() {
|
|
2579
|
+
const now = Date.now();
|
|
2580
|
+
if (this.triShadowCache && now - this.triShadowCache.at < 2000)
|
|
2581
|
+
return this.triShadowCache.exists;
|
|
2582
|
+
let exists = false;
|
|
2583
|
+
try {
|
|
2584
|
+
exists = !!this.db.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name='messages_fts_tri'").get();
|
|
2585
|
+
}
|
|
2586
|
+
catch { /* treat as absent */ }
|
|
2587
|
+
this.triShadowCache = { at: now, exists };
|
|
2588
|
+
return exists;
|
|
2589
|
+
}
|
|
2590
|
+
/** Does messages_fts still use the old unicode61 word tokenizer? Checked
|
|
2591
|
+
* per-call (not cached): the answer flips mid-session when the background
|
|
2592
|
+
* migration swaps tables, and searchMessages must switch query dialects
|
|
2593
|
+
* the moment it does — including on the read-worker's own connection. */
|
|
2594
|
+
ftsIsTrigram() {
|
|
2595
|
+
try {
|
|
2596
|
+
const row = this.db.prepare("SELECT sql FROM sqlite_master WHERE type='table' AND name='messages_fts'").get();
|
|
2597
|
+
return !!row && /trigram/i.test(row.sql || "");
|
|
2598
|
+
}
|
|
2599
|
+
catch {
|
|
2600
|
+
return false;
|
|
2601
|
+
}
|
|
2602
|
+
}
|
|
2603
|
+
ftsNeedsTrigramMigration() {
|
|
2604
|
+
if (this.readOnly)
|
|
2605
|
+
return false;
|
|
2606
|
+
try {
|
|
2607
|
+
const row = this.db.prepare("SELECT sql FROM sqlite_master WHERE type='table' AND name='messages_fts'").get();
|
|
2608
|
+
return !!row && !/trigram/i.test(row.sql || "");
|
|
2609
|
+
}
|
|
2610
|
+
catch {
|
|
2611
|
+
return false;
|
|
2612
|
+
}
|
|
2613
|
+
}
|
|
2614
|
+
/** One-time background conversion of messages_fts from the unicode61 word
|
|
2615
|
+
* tokenizer to trigram (substring search — "sprinkler" now finds
|
|
2616
|
+
* "OpenSprinkler", Bob 2026-07-19). Word tokens only match by prefix, so
|
|
2617
|
+
* any mid-word search silently found nothing.
|
|
2618
|
+
*
|
|
2619
|
+
* Runs chunked on the main writer (~3-4 min for 190k rows, measured) so
|
|
2620
|
+
* boot isn't blocked and sync writes stay responsive between chunks. The
|
|
2621
|
+
* existing index keeps serving searches the whole time; concurrent FTS
|
|
2622
|
+
* writes land in BOTH tables via ftsTriShadowExists() mirroring. The
|
|
2623
|
+
* final swap (DROP + RENAME) is a single transaction, so a crash at any
|
|
2624
|
+
* point leaves either the old table (migration restarts next boot) or
|
|
2625
|
+
* the finished trigram table — never neither. Stored body_text survives
|
|
2626
|
+
* because the copy reads it straight out of the old standalone table.
|
|
2627
|
+
* Call ONLY from the long-lived daemon: a one-shot CLI command would sit
|
|
2628
|
+
* alive for minutes finishing the copy. */
|
|
2629
|
+
startFtsTrigramMigration() {
|
|
2630
|
+
if (!this.ftsNeedsTrigramMigration())
|
|
2631
|
+
return;
|
|
2632
|
+
const COLS = "rowid, subject, from_name, from_address, to_text, cc_text, body_text";
|
|
2633
|
+
try {
|
|
2634
|
+
// A leftover shadow from an interrupted run may be stale (writes
|
|
2635
|
+
// that happened while no daemon mirrored them) — start clean.
|
|
2636
|
+
this.db.exec("DROP TABLE IF EXISTS messages_fts_tri");
|
|
2637
|
+
this.db.exec("CREATE VIRTUAL TABLE messages_fts_tri USING fts5(subject, from_name, from_address, to_text, cc_text, body_text, tokenize='trigram remove_diacritics 1')");
|
|
2638
|
+
}
|
|
2639
|
+
catch (e) {
|
|
2640
|
+
console.error(` [db] FTS trigram migration setup failed: ${e?.message || e}`);
|
|
2641
|
+
return;
|
|
2642
|
+
}
|
|
2643
|
+
let total = 0;
|
|
2644
|
+
try {
|
|
2645
|
+
total = this.db.prepare("SELECT COUNT(*) AS c FROM messages_fts").get()?.c || 0;
|
|
2646
|
+
}
|
|
2647
|
+
catch { /* */ }
|
|
2648
|
+
console.log(` [db] Reindexing search index: ${total} messages in background (~5 min; substring search when done)`);
|
|
2649
|
+
const CHUNK = 500; // ~0.35s of tokenization per chunk (measured on Bob's DB)
|
|
2650
|
+
let last = 0, copied = 0, chunkN = 0;
|
|
2651
|
+
const step = () => {
|
|
2652
|
+
try {
|
|
2653
|
+
const rows = this.db.prepare("SELECT rowid FROM messages_fts WHERE rowid > ? ORDER BY rowid LIMIT ?").all(last, CHUNK);
|
|
2654
|
+
if (rows.length === 0) {
|
|
2655
|
+
finish();
|
|
2656
|
+
return;
|
|
2657
|
+
}
|
|
2658
|
+
const hi = rows[rows.length - 1].rowid;
|
|
2659
|
+
// Single INSERT..SELECT — atomic against the sync worker's
|
|
2660
|
+
// mirrored writes (WAL serializes writers), so old and shadow
|
|
2661
|
+
// can't diverge inside a chunk. OR REPLACE because mirrored
|
|
2662
|
+
// writes may have already landed rows past the frontier.
|
|
2663
|
+
this.db.prepare(`INSERT OR REPLACE INTO messages_fts_tri (${COLS}) SELECT ${COLS} FROM messages_fts WHERE rowid > ? AND rowid <= ?`).run(last, hi);
|
|
2664
|
+
last = hi;
|
|
2665
|
+
copied += rows.length;
|
|
2666
|
+
chunkN++;
|
|
2667
|
+
if (chunkN % 40 === 0) {
|
|
2668
|
+
console.log(` [db] Reindexing search index: ${copied}/${total}`);
|
|
2669
|
+
// The copy writes ~1 GB through the WAL in total — keep it drained.
|
|
2670
|
+
try {
|
|
2671
|
+
this.db.exec("PRAGMA wal_checkpoint(PASSIVE)");
|
|
2672
|
+
}
|
|
2673
|
+
catch { /* */ }
|
|
2674
|
+
}
|
|
2675
|
+
setTimeout(step, 100); // yield the writer between chunks
|
|
2676
|
+
}
|
|
2677
|
+
catch (e) {
|
|
2678
|
+
console.error(` [db] FTS trigram migration chunk failed (${e?.message || e}) — retrying in 5s`);
|
|
2679
|
+
setTimeout(step, 5000);
|
|
2680
|
+
}
|
|
2681
|
+
};
|
|
2682
|
+
const finish = () => {
|
|
2683
|
+
try {
|
|
2684
|
+
this.db.exec("BEGIN IMMEDIATE");
|
|
2685
|
+
// Rows past the frontier that landed after the last chunk are
|
|
2686
|
+
// already mirrored into the shadow, but belt-and-braces: if any
|
|
2687
|
+
// exist, loop once more instead of trusting the mirror.
|
|
2688
|
+
const strag = this.db.prepare("SELECT COUNT(*) AS c FROM messages_fts WHERE rowid > ?").get(last)?.c || 0;
|
|
2689
|
+
if (strag > 0) {
|
|
2690
|
+
this.db.exec("ROLLBACK");
|
|
2691
|
+
setTimeout(step, 100);
|
|
2692
|
+
return;
|
|
2693
|
+
}
|
|
2694
|
+
this.db.exec("DROP TABLE messages_fts");
|
|
2695
|
+
this.db.exec("ALTER TABLE messages_fts_tri RENAME TO messages_fts");
|
|
2696
|
+
this.db.exec("COMMIT");
|
|
2697
|
+
console.log(` [db] Search reindex complete: ${copied} messages — substring search active`);
|
|
2698
|
+
try {
|
|
2699
|
+
this.db.exec("PRAGMA wal_checkpoint(PASSIVE)");
|
|
2700
|
+
}
|
|
2701
|
+
catch { /* */ }
|
|
2702
|
+
}
|
|
2703
|
+
catch (e) {
|
|
2704
|
+
try {
|
|
2705
|
+
this.db.exec("ROLLBACK");
|
|
2706
|
+
}
|
|
2707
|
+
catch { /* */ }
|
|
2708
|
+
console.error(` [db] FTS trigram migration swap failed (${e?.message || e}) — retrying in 5s`);
|
|
2709
|
+
setTimeout(finish, 5000);
|
|
2710
|
+
}
|
|
2711
|
+
};
|
|
2712
|
+
setTimeout(step, 3000); // let boot (folder loads, first sync) settle first
|
|
2713
|
+
}
|
|
2536
2714
|
/** Record a prefetch failure (0-body fetch / store-write fail) for a UID,
|
|
2537
2715
|
* incrementing its backoff count. Persisted so it survives restarts. */
|
|
2538
2716
|
recordPrefetchFailure(accountId, folderId, uid) {
|
|
@@ -3432,9 +3610,56 @@ export class MailxDB {
|
|
|
3432
3610
|
// Strip FTS5 metacharacters from a user term. A stray `;` `(` `:` `"`
|
|
3433
3611
|
// etc. is a hard `fts5: syntax error` (Bob 2026-05-21 log). FTS5's
|
|
3434
3612
|
// tokenizer splits on non-word chars anyway, so removing them loses no
|
|
3435
|
-
// real matching ability.
|
|
3613
|
+
// real matching ability. (unicode61 dialect only — see isTri below.)
|
|
3436
3614
|
const ftsClean = (t) => t.replace(/["';:(){}\[\]^~\\/]/g, "").trim();
|
|
3437
|
-
|
|
3615
|
+
// ── Tokenizer dialects ──
|
|
3616
|
+
// The index is migrating from unicode61 (word tokens, prefix-only
|
|
3617
|
+
// matching — "sprinkler" could never find "opensprinkler") to trigram
|
|
3618
|
+
// (substring matching, Bob 2026-07-19). Which dialect to emit depends
|
|
3619
|
+
// on which table is live RIGHT NOW: the background migration swaps it
|
|
3620
|
+
// mid-session, and this runs on the read-worker's own connection, so
|
|
3621
|
+
// ask sqlite_master rather than trusting boot-time state.
|
|
3622
|
+
//
|
|
3623
|
+
// Trigram dialect rules:
|
|
3624
|
+
// • every term is emitted as a QUOTED string — FTS5 then matches it
|
|
3625
|
+
// as a case-folded substring anywhere in the text, punctuation and
|
|
3626
|
+
// all ("192.55.226" matches literally; no dot-splitting, no `*`).
|
|
3627
|
+
// Only embedded double-quotes need stripping.
|
|
3628
|
+
// • terms under 3 chars can't form a trigram and MATCH nothing, so
|
|
3629
|
+
// they fall back to a LIKE over the header columns. Headers only:
|
|
3630
|
+
// scanning preview/to/cc for a 1-2 char pattern measured ~5s on
|
|
3631
|
+
// 190k rows vs ~0.3s worst-case for headers, and short terms are
|
|
3632
|
+
// transient search-as-you-type states anyway.
|
|
3633
|
+
const isTri = this.ftsIsTrigram();
|
|
3634
|
+
const triClean = (t) => t.replace(/"/g, "").trim();
|
|
3635
|
+
const triQuote = (t) => `"${t}"`;
|
|
3636
|
+
const likeEsc = (t) => t.replace(/[~%_]/g, m => `~${m}`);
|
|
3637
|
+
// One OR-group per call: multiple terms land in the SAME group so an
|
|
3638
|
+
// all-short alternation ("ab|cd") stays an OR, not an AND of groups.
|
|
3639
|
+
const pushLike = (cols, terms) => {
|
|
3640
|
+
const list = Array.isArray(terms) ? terms : [terms];
|
|
3641
|
+
const clauses = [];
|
|
3642
|
+
for (const t of list) {
|
|
3643
|
+
for (const c of cols) {
|
|
3644
|
+
clauses.push(`${c} LIKE ? ESCAPE '~'`);
|
|
3645
|
+
extraParams.push(`%${likeEsc(t)}%`);
|
|
3646
|
+
}
|
|
3647
|
+
}
|
|
3648
|
+
if (clauses.length)
|
|
3649
|
+
extraWhere.push("(" + clauses.join(" OR ") + ")");
|
|
3650
|
+
};
|
|
3651
|
+
const LIKE_FROM = ["m.from_name", "m.from_address"];
|
|
3652
|
+
const LIKE_TO = ["m.to_json", "m.cc_json"];
|
|
3653
|
+
const LIKE_CC = ["m.cc_json"];
|
|
3654
|
+
const LIKE_SUBJ = ["m.subject"];
|
|
3655
|
+
const LIKE_ANY = ["m.subject", "m.from_name", "m.from_address"];
|
|
3656
|
+
for (let pi = 0; pi < parts.length; pi++) {
|
|
3657
|
+
const part = parts[pi];
|
|
3658
|
+
// A 1-2 char term next to a user-typed OR must be DROPPED, not
|
|
3659
|
+
// LIKE'd: LIKE clauses AND against the MATCH, so "ab OR sprinkler"
|
|
3660
|
+
// would intersect down to zero. Over-matching (just "sprinkler")
|
|
3661
|
+
// beats returning nothing. (Trigram dialect only.)
|
|
3662
|
+
const orAdjacent = parts[pi - 1] === "OR" || parts[pi + 1] === "OR";
|
|
3438
3663
|
const fromMatch = part.match(/^from:(.+)$/i);
|
|
3439
3664
|
const toMatch = part.match(/^to:(.+)$/i);
|
|
3440
3665
|
const ccMatch = part.match(/^cc:(.+)$/i);
|
|
@@ -3448,28 +3673,64 @@ export class MailxDB {
|
|
|
3448
3673
|
if (fromMatch) {
|
|
3449
3674
|
// FTS5 column-group `{c1 c2}:term` matches term in either
|
|
3450
3675
|
// column without the parens that break a trailing implicit-AND.
|
|
3451
|
-
|
|
3452
|
-
|
|
3453
|
-
|
|
3676
|
+
if (isTri) {
|
|
3677
|
+
const term = triClean(fromMatch[1]);
|
|
3678
|
+
if (term.length >= 3)
|
|
3679
|
+
frags.push({ s: `{from_name from_address}:${triQuote(term)}`, op: false });
|
|
3680
|
+
else if (term && !orAdjacent)
|
|
3681
|
+
pushLike(LIKE_FROM, term);
|
|
3682
|
+
}
|
|
3683
|
+
else {
|
|
3684
|
+
const term = ftsClean(fromMatch[1]);
|
|
3685
|
+
if (term)
|
|
3686
|
+
frags.push({ s: `{from_name from_address}:${term}*`, op: false });
|
|
3687
|
+
}
|
|
3454
3688
|
}
|
|
3455
3689
|
else if (toMatch) {
|
|
3456
3690
|
// `to:` deliberately spans To AND Cc — when someone searches
|
|
3457
3691
|
// "to:bob" they mean "addressed to bob", and being Cc'd counts.
|
|
3458
3692
|
// Use `cc:` for a Cc-only match. (Bcc isn't indexed: it's absent
|
|
3459
3693
|
// on received mail by design, and only meaningful in Sent.)
|
|
3460
|
-
|
|
3461
|
-
|
|
3462
|
-
|
|
3694
|
+
if (isTri) {
|
|
3695
|
+
const term = triClean(toMatch[1]);
|
|
3696
|
+
if (term.length >= 3)
|
|
3697
|
+
frags.push({ s: `{to_text cc_text}:${triQuote(term)}`, op: false });
|
|
3698
|
+
else if (term && !orAdjacent)
|
|
3699
|
+
pushLike(LIKE_TO, term);
|
|
3700
|
+
}
|
|
3701
|
+
else {
|
|
3702
|
+
const term = ftsClean(toMatch[1]);
|
|
3703
|
+
if (term)
|
|
3704
|
+
frags.push({ s: `{to_text cc_text}:${term}*`, op: false });
|
|
3705
|
+
}
|
|
3463
3706
|
}
|
|
3464
3707
|
else if (ccMatch) {
|
|
3465
|
-
|
|
3466
|
-
|
|
3467
|
-
|
|
3708
|
+
if (isTri) {
|
|
3709
|
+
const term = triClean(ccMatch[1]);
|
|
3710
|
+
if (term.length >= 3)
|
|
3711
|
+
frags.push({ s: `cc_text:${triQuote(term)}`, op: false });
|
|
3712
|
+
else if (term && !orAdjacent)
|
|
3713
|
+
pushLike(LIKE_CC, term);
|
|
3714
|
+
}
|
|
3715
|
+
else {
|
|
3716
|
+
const term = ftsClean(ccMatch[1]);
|
|
3717
|
+
if (term)
|
|
3718
|
+
frags.push({ s: `cc_text:${term}*`, op: false });
|
|
3719
|
+
}
|
|
3468
3720
|
}
|
|
3469
3721
|
else if (subjectMatch) {
|
|
3470
|
-
|
|
3471
|
-
|
|
3472
|
-
|
|
3722
|
+
if (isTri) {
|
|
3723
|
+
const term = triClean(subjectMatch[1]);
|
|
3724
|
+
if (term.length >= 3)
|
|
3725
|
+
frags.push({ s: `subject:${triQuote(term)}`, op: false });
|
|
3726
|
+
else if (term && !orAdjacent)
|
|
3727
|
+
pushLike(LIKE_SUBJ, term);
|
|
3728
|
+
}
|
|
3729
|
+
else {
|
|
3730
|
+
const term = ftsClean(subjectMatch[1]);
|
|
3731
|
+
if (term)
|
|
3732
|
+
frags.push({ s: `subject:${term}*`, op: false });
|
|
3733
|
+
}
|
|
3473
3734
|
}
|
|
3474
3735
|
else if (dateMatch || afterMatch || beforeMatch) {
|
|
3475
3736
|
const op = dateMatch ? (dateMatch[1] || "=") : (afterMatch ? ">" : "<");
|
|
@@ -3522,6 +3783,36 @@ export class MailxDB {
|
|
|
3522
3783
|
// match like "Peter Hoddie" + "github" returned zero hits.
|
|
3523
3784
|
frags.push({ s: part, op: true });
|
|
3524
3785
|
}
|
|
3786
|
+
else if (isTri) {
|
|
3787
|
+
// Unqualified, trigram dialect — quoted substring match. A
|
|
3788
|
+
// user-quoted "foo bar" phrase survives as one literal
|
|
3789
|
+
// substring including the space (real phrase search, which
|
|
3790
|
+
// the unicode61 dialect approximated with AND'd words).
|
|
3791
|
+
const term = part.replace(/^\/|\/$/g, "");
|
|
3792
|
+
if (term.includes("|")) {
|
|
3793
|
+
const alts = term.split("|").map(t => triClean(t)).filter(Boolean);
|
|
3794
|
+
const long = alts.filter(t => t.length >= 3);
|
|
3795
|
+
const short = alts.filter(t => t.length < 3);
|
|
3796
|
+
if (long.length) {
|
|
3797
|
+
// Mixed `abc|xy`: drop the short alternates. An OR
|
|
3798
|
+
// can't span MATCH and LIKE (LIKE would AND against
|
|
3799
|
+
// the MATCH and turn the union into an intersection
|
|
3800
|
+
// that returns nothing).
|
|
3801
|
+
frags.push({ s: `(${long.map(triQuote).join(" OR ")})`, op: false });
|
|
3802
|
+
}
|
|
3803
|
+
else if (short.length && !orAdjacent) {
|
|
3804
|
+
// All-short: one OR'd LIKE group keeps `ab|cd` a union.
|
|
3805
|
+
pushLike(LIKE_ANY, short);
|
|
3806
|
+
}
|
|
3807
|
+
}
|
|
3808
|
+
else {
|
|
3809
|
+
const t = triClean(term);
|
|
3810
|
+
if (t.length >= 3)
|
|
3811
|
+
frags.push({ s: triQuote(t), op: false });
|
|
3812
|
+
else if (t && !orAdjacent)
|
|
3813
|
+
pushLike(LIKE_ANY, t);
|
|
3814
|
+
}
|
|
3815
|
+
}
|
|
3525
3816
|
else {
|
|
3526
3817
|
// Unqualified — search everything. Strip /regex-literal/
|
|
3527
3818
|
// slashes and FTS5 metacharacters before wildcarding.
|
|
@@ -3552,6 +3843,14 @@ export class MailxDB {
|
|
|
3552
3843
|
}
|
|
3553
3844
|
}
|
|
3554
3845
|
}
|
|
3846
|
+
// A term diverted to the LIKE fallback can strand a user-typed
|
|
3847
|
+
// operator at the edge of the MATCH string ("ab OR sprinkler" → the
|
|
3848
|
+
// "ab" became a LIKE, leaving "OR sprinkler" — an FTS5 syntax error
|
|
3849
|
+
// that would silently empty the whole search). Trim dangling operators.
|
|
3850
|
+
while (frags.length && frags[0].op)
|
|
3851
|
+
frags.shift();
|
|
3852
|
+
while (frags.length && frags[frags.length - 1].op)
|
|
3853
|
+
frags.pop();
|
|
3555
3854
|
// Join fragments. Two adjacent value fragments need an explicit `AND`
|
|
3556
3855
|
// (implicit-AND after a `(...)` / `{...}:` group is an FTS5 syntax
|
|
3557
3856
|
// error); a user operator fragment glues itself, so no insert around it.
|
|
@@ -3668,8 +3967,12 @@ export class MailxDB {
|
|
|
3668
3967
|
this.db.exec("DROP TABLE IF EXISTS messages_fts");
|
|
3669
3968
|
}
|
|
3670
3969
|
catch { /* ignore */ }
|
|
3970
|
+
// Trigram tokenization is ~10x slower to build than unicode61, so a
|
|
3971
|
+
// full -reindex on a 190k-row DB takes minutes. Explicit user command,
|
|
3972
|
+
// runs once — substring search is worth it.
|
|
3671
3973
|
this.db.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts USING fts5(
|
|
3672
|
-
subject, from_name, from_address, to_text, cc_text, body_text
|
|
3974
|
+
subject, from_name, from_address, to_text, cc_text, body_text,
|
|
3975
|
+
tokenize='trigram remove_diacritics 1'
|
|
3673
3976
|
)`);
|
|
3674
3977
|
// Use a single transaction + prepared statement for speed (~50x faster than individual inserts)
|
|
3675
3978
|
const insert = this.db.prepare("INSERT INTO messages_fts (rowid, subject, from_name, from_address, to_text, cc_text, body_text) VALUES (?, ?, ?, ?, ?, ?, ?)");
|