@bobfrankston/mailx-store 0.1.44 → 0.1.46
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/db.d.ts +9 -1
- package/db.js +165 -18
- package/package.json +3 -3
- package/store.d.ts +1 -1
- package/store.js +33 -12
package/db.d.ts
CHANGED
|
@@ -276,7 +276,7 @@ export declare class MailxDB {
|
|
|
276
276
|
* legacy columns are frozen at first-sight values. */
|
|
277
277
|
getMessages(query: MessageQuery): PagedResult<MessageEnvelope>;
|
|
278
278
|
/** Unified inbox: all inbox folders across accounts, sorted by date, paginated in SQL */
|
|
279
|
-
getUnifiedInbox(page?: number, pageSize?: number): PagedResult<MessageEnvelope>;
|
|
279
|
+
getUnifiedInbox(page?: number, pageSize?: number, flaggedOnly?: boolean): PagedResult<MessageEnvelope>;
|
|
280
280
|
/** Map a `messages` row to a MessageEnvelope. Exposes `uuid` (stable local
|
|
281
281
|
* identity) and `bodyPath` (authoritative on-disk location) in addition
|
|
282
282
|
* to the server-binding metadata. */
|
|
@@ -375,6 +375,14 @@ export declare class MailxDB {
|
|
|
375
375
|
uid: number;
|
|
376
376
|
folderId: number;
|
|
377
377
|
}[];
|
|
378
|
+
/** Rebuild messages_fts if it still has the OLD external-content schema
|
|
379
|
+
* (`content=messages`). That schema referenced columns (to_text/cc_text/
|
|
380
|
+
* body_text) that don't exist in `messages`, so content-dereferencing
|
|
381
|
+
* failed ("no such column: T.to_text") and corrupt/empty-subject rows
|
|
382
|
+
* silently never indexed → search missed them (Bob 2026-06-05). One-time
|
|
383
|
+
* per DB: drop, recreate standalone, reindex subject/from + derived to/cc.
|
|
384
|
+
* body_text backfills as bodies parse. */
|
|
385
|
+
private migrateFtsSchema;
|
|
378
386
|
/** Record a prefetch failure (0-body fetch / store-write fail) for a UID,
|
|
379
387
|
* incrementing its backoff count. Persisted so it survives restarts. */
|
|
380
388
|
recordPrefetchFailure(accountId: string, folderId: number, uid: number): void;
|
package/db.js
CHANGED
|
@@ -192,9 +192,16 @@ const SCHEMA = `
|
|
|
192
192
|
CREATE INDEX IF NOT EXISTS idx_contacts_email ON contacts(email);
|
|
193
193
|
CREATE INDEX IF NOT EXISTS idx_contacts_name ON contacts(name);
|
|
194
194
|
|
|
195
|
+
-- Standalone FTS5 (NOT external-content). It was declared
|
|
196
|
+
-- content=messages, but messages has no to_text/cc_text/body_text columns,
|
|
197
|
+
-- so any operation that dereferenced the "content" failed with
|
|
198
|
+
-- "no such column: T.to_text" — and corrupt/empty-subject rows never got
|
|
199
|
+
-- indexed, so search silently missed them (Bob 2026-06-05). Standalone =
|
|
200
|
+
-- the index stores its own text (we INSERT it explicitly on upsert), so
|
|
201
|
+
-- there's nothing to dereference and DELETE/INSERT/reindex all work. The
|
|
202
|
+
-- migrateFtsSchema() pass below rebuilds an existing external-content table.
|
|
195
203
|
CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts USING fts5(
|
|
196
|
-
subject, from_name, from_address, to_text, cc_text, body_text
|
|
197
|
-
content=messages, content_rowid=id
|
|
204
|
+
subject, from_name, from_address, to_text, cc_text, body_text
|
|
198
205
|
);
|
|
199
206
|
|
|
200
207
|
CREATE TABLE IF NOT EXISTS sync_actions (
|
|
@@ -403,6 +410,27 @@ export class MailxDB {
|
|
|
403
410
|
this.db.exec("PRAGMA journal_mode = WAL");
|
|
404
411
|
this.db.exec("PRAGMA foreign_keys = ON");
|
|
405
412
|
this.db.exec(SCHEMA);
|
|
413
|
+
this.migrateFtsSchema();
|
|
414
|
+
// Purge phantom uid=0 rows (invalid IMAP UID — empty stub letters, the
|
|
415
|
+
// upsertMessage guard now prevents new ones; this clears any already on
|
|
416
|
+
// disk, e.g. Bob's 00:03 INBOX phantom). Cascade to FTS + membership.
|
|
417
|
+
try {
|
|
418
|
+
const ph = this.db.prepare("SELECT id FROM messages WHERE uid = 0").all();
|
|
419
|
+
if (ph.length) {
|
|
420
|
+
for (const r of ph) {
|
|
421
|
+
try {
|
|
422
|
+
this.db.prepare("DELETE FROM messages_fts WHERE rowid = ?").run(r.id);
|
|
423
|
+
}
|
|
424
|
+
catch { /* */ }
|
|
425
|
+
this.db.prepare("DELETE FROM message_folders WHERE message_row_id = ?").run(r.id);
|
|
426
|
+
}
|
|
427
|
+
this.db.prepare("DELETE FROM messages WHERE uid = 0").run();
|
|
428
|
+
console.log(` [db] purged ${ph.length} phantom uid=0 row(s)`);
|
|
429
|
+
}
|
|
430
|
+
}
|
|
431
|
+
catch (e) {
|
|
432
|
+
console.error(` [db] phantom purge failed: ${e?.message || e}`);
|
|
433
|
+
}
|
|
406
434
|
// Idempotent migrations for older databases that predate new columns.
|
|
407
435
|
// SQLite doesn't support "ADD COLUMN IF NOT EXISTS", so we just try the
|
|
408
436
|
// ALTER and catch the "duplicate column" error. Simpler and more robust
|
|
@@ -1457,6 +1485,17 @@ export class MailxDB {
|
|
|
1457
1485
|
// whole `name <addr>` into the mailbox local-part as one encoded
|
|
1458
1486
|
// word (`=?utf-8?q?...=3C...=40...=3E?=@host`). Decoding it at least
|
|
1459
1487
|
// renders readable text instead of raw `=3C`/`=C3=A9` gibberish.
|
|
1488
|
+
// Reject invalid UIDs CENTRALLY. Valid IMAP UIDs are >= 1; Gmail uses
|
|
1489
|
+
// large positive hash UIDs; -recover uses NEGATIVE uids deliberately.
|
|
1490
|
+
// A uid of exactly 0 means a FETCH/APPENDUID response arrived with no
|
|
1491
|
+
// parseable UID — inserting it creates a phantom empty INBOX row (no
|
|
1492
|
+
// message_id/subject/body — Bob's 00:03 "blank line"). A uid<=0 guard
|
|
1493
|
+
// already existed on ONE insert path (mailx-imap), but the phantom kept
|
|
1494
|
+
// recurring because OTHER paths (qresync new-message fetch, etc.)
|
|
1495
|
+
// bypassed it. Guarding at the storage seam covers all of them.
|
|
1496
|
+
if (msg.uid === 0 || !Number.isFinite(msg.uid)) {
|
|
1497
|
+
return -1;
|
|
1498
|
+
}
|
|
1460
1499
|
msg.subject = decodeHeaderWords(msg.subject);
|
|
1461
1500
|
if (msg.from)
|
|
1462
1501
|
msg.from = { name: decodeHeaderWords(msg.from.name || ""), address: decodeHeaderWords(msg.from.address || "") };
|
|
@@ -1464,7 +1503,7 @@ export class MailxDB {
|
|
|
1464
1503
|
msg.to = msg.to.map(a => ({ name: decodeHeaderWords(a.name || ""), address: decodeHeaderWords(a.address || "") }));
|
|
1465
1504
|
if (msg.cc)
|
|
1466
1505
|
msg.cc = msg.cc.map(a => ({ name: decodeHeaderWords(a.name || ""), address: decodeHeaderWords(a.address || "") }));
|
|
1467
|
-
const existing = this.db.prepare("SELECT id, provider_id FROM messages WHERE account_id = ? AND folder_id = ? AND uid = ?").get(msg.accountId, msg.folderId, msg.uid);
|
|
1506
|
+
const existing = this.db.prepare("SELECT id, provider_id, message_id FROM messages WHERE account_id = ? AND folder_id = ? AND uid = ?").get(msg.accountId, msg.folderId, msg.uid);
|
|
1468
1507
|
if (existing) {
|
|
1469
1508
|
// Backfill provider_id on existing rows that predate this column —
|
|
1470
1509
|
// critical for body fetch to bypass listMessageIds pagination.
|
|
@@ -1505,6 +1544,27 @@ export class MailxDB {
|
|
|
1505
1544
|
WHERE id = ?
|
|
1506
1545
|
`).run(flagsToWrite, Date.now(), existing.id);
|
|
1507
1546
|
}
|
|
1547
|
+
// SELF-HEAL corrupt rows. An empty message_id is the tell: a
|
|
1548
|
+
// partial fetch (no ENVELOPE) created the row with no Message-ID and
|
|
1549
|
+
// a fallback (today) date — which breaks dedup (empty mid = its own
|
|
1550
|
+
// bucket → zombie duplicate) and shows the wrong date (Bob 2026-06-05,
|
|
1551
|
+
// the Oct-2025 "Greeking Out" letter resurfacing). The old update
|
|
1552
|
+
// path only touched flags/body, so such a row never healed even when
|
|
1553
|
+
// a later fetch carried the real envelope. Backfill mid + date +
|
|
1554
|
+
// subject HERE, but only when the existing mid is empty AND the
|
|
1555
|
+
// incoming msg actually has one — never clobber a good value.
|
|
1556
|
+
if (msg.messageId && (!existing.message_id || existing.message_id === "")) {
|
|
1557
|
+
this.db.prepare("UPDATE messages SET message_id = ?, date = ?, subject = ? WHERE id = ?").run(msg.messageId, msg.date || Date.now(), msg.subject || "", existing.id);
|
|
1558
|
+
// The row was indexed with an empty subject (it had no envelope),
|
|
1559
|
+
// so search missed it. Re-index now that we have the real subject
|
|
1560
|
+
// + from/to. Standalone FTS5 → delete + insert. body_text
|
|
1561
|
+
// backfills when the body parses.
|
|
1562
|
+
try {
|
|
1563
|
+
this.db.prepare("DELETE FROM messages_fts WHERE rowid = ?").run(existing.id);
|
|
1564
|
+
this.db.prepare("INSERT INTO messages_fts (rowid, subject, from_name, from_address, to_text, cc_text, body_text) VALUES (?, ?, ?, ?, ?, ?, '')").run(existing.id, msg.subject || "", msg.from?.name || "", msg.from?.address || "", (msg.to || []).map(a => `${a.name || ""} ${a.address || ""}`).join(" "), (msg.cc || []).map(a => `${a.name || ""} ${a.address || ""}`).join(" "));
|
|
1565
|
+
}
|
|
1566
|
+
catch { /* best-effort */ }
|
|
1567
|
+
}
|
|
1508
1568
|
// Refresh membership last_seen_at — server confirmed this UID
|
|
1509
1569
|
// is still in this folder. No-op if migration already populated
|
|
1510
1570
|
// it; defensive INSERT-OR-UPDATE handles the race where the row
|
|
@@ -1683,7 +1743,12 @@ export class MailxDB {
|
|
|
1683
1743
|
EXISTS(
|
|
1684
1744
|
SELECT 1 FROM sync_actions sa
|
|
1685
1745
|
WHERE sa.account_id = m.account_id AND sa.uid = r.mf_uid
|
|
1686
|
-
) AS pending
|
|
1746
|
+
) AS pending,
|
|
1747
|
+
-- Replied state OR'd across the Message-ID group (see
|
|
1748
|
+
-- getUnifiedInbox) so the reply arrow is correct regardless of
|
|
1749
|
+
-- which duplicate survives dedup. Bob 2026-06-01.
|
|
1750
|
+
COALESCE((SELECT MAX(CASE WHEN m3.is_replied = 1 OR m3.flags_json LIKE '%Answered%' THEN 1 ELSE 0 END)
|
|
1751
|
+
FROM messages m3 WHERE m3.message_id = m.message_id AND COALESCE(m.message_id, '') != ''), m.is_replied) AS groupReplied
|
|
1687
1752
|
FROM ranked r
|
|
1688
1753
|
JOIN messages m ON m.id = r.m_id
|
|
1689
1754
|
WHERE r.rn = 1
|
|
@@ -1706,7 +1771,7 @@ export class MailxDB {
|
|
|
1706
1771
|
flags: JSON.parse(r.flags_json),
|
|
1707
1772
|
size: r.size,
|
|
1708
1773
|
hasAttachments: !!r.has_attachments,
|
|
1709
|
-
isReplied: !!r.
|
|
1774
|
+
isReplied: !!r.groupReplied,
|
|
1710
1775
|
preview: r.preview,
|
|
1711
1776
|
bodyPath: r.body_path || "",
|
|
1712
1777
|
pending: !!r.pending,
|
|
@@ -1714,7 +1779,7 @@ export class MailxDB {
|
|
|
1714
1779
|
return { items, total, page, pageSize };
|
|
1715
1780
|
}
|
|
1716
1781
|
/** Unified inbox: all inbox folders across accounts, sorted by date, paginated in SQL */
|
|
1717
|
-
getUnifiedInbox(page = 1, pageSize = 50) {
|
|
1782
|
+
getUnifiedInbox(page = 1, pageSize = 50, flaggedOnly = false) {
|
|
1718
1783
|
const offset = (page - 1) * pageSize;
|
|
1719
1784
|
// Find all inbox folder IDs
|
|
1720
1785
|
const inboxRows = this.db.prepare("SELECT id FROM folders WHERE special_use = 'inbox'").all();
|
|
@@ -1722,6 +1787,13 @@ export class MailxDB {
|
|
|
1722
1787
|
return { items: [], total: 0, page, pageSize };
|
|
1723
1788
|
const placeholders = inboxRows.map(() => "?").join(",");
|
|
1724
1789
|
const folderIds = inboxRows.map((r) => r.id);
|
|
1790
|
+
// Flagged-only filter must apply at the DB level, not just via the
|
|
1791
|
+
// client's CSS `.flagged-only` rule. The unified list is paginated
|
|
1792
|
+
// (50/page); a CSS-only filter would only hide non-flagged rows in
|
|
1793
|
+
// the loaded page, so flagged messages deeper in the mailbox never
|
|
1794
|
+
// appear and the ★ filter looks broken (Bob 2026-06-05). Same
|
|
1795
|
+
// predicate as getMessages (line ~1876).
|
|
1796
|
+
const flagFilter = flaggedOnly ? " AND m.flags_json LIKE '%\\\\Flagged%'" : "";
|
|
1725
1797
|
// Dedup by Message-ID. When a sender includes multiple of the
|
|
1726
1798
|
// user's own addresses on the recipient line (mailx@bob.ma +
|
|
1727
1799
|
// bobf2@bobf.frankston.com), the local mail server delivers ONE
|
|
@@ -1735,7 +1807,7 @@ export class MailxDB {
|
|
|
1735
1807
|
SELECT 1
|
|
1736
1808
|
FROM messages m
|
|
1737
1809
|
JOIN message_folders mf ON mf.message_row_id = m.id
|
|
1738
|
-
WHERE mf.folder_id IN (${placeholders})
|
|
1810
|
+
WHERE mf.folder_id IN (${placeholders})${flagFilter}
|
|
1739
1811
|
GROUP BY CASE WHEN COALESCE(m.message_id, '') = '' THEN 'mid-empty:' || m.id ELSE m.message_id END
|
|
1740
1812
|
)`).get(...folderIds).cnt;
|
|
1741
1813
|
const rows = this.db.prepare(`WITH ranked AS (
|
|
@@ -1746,7 +1818,7 @@ export class MailxDB {
|
|
|
1746
1818
|
) AS rn
|
|
1747
1819
|
FROM messages m
|
|
1748
1820
|
JOIN message_folders mf ON mf.message_row_id = m.id
|
|
1749
|
-
WHERE mf.folder_id IN (${placeholders})
|
|
1821
|
+
WHERE mf.folder_id IN (${placeholders})${flagFilter}
|
|
1750
1822
|
)
|
|
1751
1823
|
SELECT m.*, r.mf_uid AS uid, r.mf_folder_id AS folder_id,
|
|
1752
1824
|
EXISTS(
|
|
@@ -1754,7 +1826,16 @@ export class MailxDB {
|
|
|
1754
1826
|
WHERE sa.account_id = m.account_id AND sa.uid = r.mf_uid
|
|
1755
1827
|
) AS pending,
|
|
1756
1828
|
(SELECT COUNT(DISTINCT account_id) FROM messages m2
|
|
1757
|
-
WHERE m2.message_id = m.message_id AND COALESCE(m.message_id, '') != '') AS dupeCount
|
|
1829
|
+
WHERE m2.message_id = m.message_id AND COALESCE(m.message_id, '') != '') AS dupeCount,
|
|
1830
|
+
-- Replied state aggregated across the whole Message-ID group, not
|
|
1831
|
+
-- just the dedup survivor. When duplicates collapse (same message
|
|
1832
|
+
-- delivered to multiple self-addresses), the \Answered flag /
|
|
1833
|
+
-- is_replied can live on a NON-surviving copy — so the survivor
|
|
1834
|
+
-- showed no reply arrow even though TB did. OR it across the group
|
|
1835
|
+
-- so the marker is correct regardless of which row wins dedup
|
|
1836
|
+
-- (Bob 2026-06-01). Empty message_id → falls back to the row's own.
|
|
1837
|
+
COALESCE((SELECT MAX(CASE WHEN m3.is_replied = 1 OR m3.flags_json LIKE '%Answered%' THEN 1 ELSE 0 END)
|
|
1838
|
+
FROM messages m3 WHERE m3.message_id = m.message_id AND COALESCE(m.message_id, '') != ''), m.is_replied) AS groupReplied
|
|
1758
1839
|
FROM ranked r
|
|
1759
1840
|
JOIN messages m ON m.id = r.m_id
|
|
1760
1841
|
WHERE r.rn = 1
|
|
@@ -1776,7 +1857,7 @@ export class MailxDB {
|
|
|
1776
1857
|
flags: JSON.parse(r.flags_json),
|
|
1777
1858
|
size: r.size,
|
|
1778
1859
|
hasAttachments: !!r.has_attachments,
|
|
1779
|
-
isReplied: !!r.
|
|
1860
|
+
isReplied: !!r.groupReplied,
|
|
1780
1861
|
preview: r.preview,
|
|
1781
1862
|
bodyPath: r.body_path || "",
|
|
1782
1863
|
pending: !!r.pending,
|
|
@@ -2034,6 +2115,50 @@ export class MailxDB {
|
|
|
2034
2115
|
: "";
|
|
2035
2116
|
return this.db.prepare(`SELECT uid, folder_id as folderId FROM messages WHERE account_id = ? AND (body_path IS NULL OR body_path = '')${exclusion} ORDER BY (size IS NULL OR size = 0), size ASC, date DESC LIMIT ?`).all(accountId, ...excludeFolderIds, limit);
|
|
2036
2117
|
}
|
|
2118
|
+
/** Rebuild messages_fts if it still has the OLD external-content schema
|
|
2119
|
+
* (`content=messages`). That schema referenced columns (to_text/cc_text/
|
|
2120
|
+
* body_text) that don't exist in `messages`, so content-dereferencing
|
|
2121
|
+
* failed ("no such column: T.to_text") and corrupt/empty-subject rows
|
|
2122
|
+
* silently never indexed → search missed them (Bob 2026-06-05). One-time
|
|
2123
|
+
* per DB: drop, recreate standalone, reindex subject/from + derived to/cc.
|
|
2124
|
+
* body_text backfills as bodies parse. */
|
|
2125
|
+
migrateFtsSchema() {
|
|
2126
|
+
try {
|
|
2127
|
+
const row = this.db.prepare("SELECT sql FROM sqlite_master WHERE type='table' AND name='messages_fts'").get();
|
|
2128
|
+
if (!row || !/content\s*=\s*messages/i.test(row.sql))
|
|
2129
|
+
return; // already standalone
|
|
2130
|
+
console.log(" [db] messages_fts had broken external-content schema — rebuilding standalone + reindexing");
|
|
2131
|
+
this.db.exec("DROP TABLE IF EXISTS messages_fts");
|
|
2132
|
+
this.db.exec("CREATE VIRTUAL TABLE messages_fts USING fts5(subject, from_name, from_address, to_text, cc_text, body_text)");
|
|
2133
|
+
const addrText = (j) => {
|
|
2134
|
+
try {
|
|
2135
|
+
return JSON.parse(j || "[]").map(a => `${a.name || ""} ${a.address || ""}`).join(" ");
|
|
2136
|
+
}
|
|
2137
|
+
catch {
|
|
2138
|
+
return "";
|
|
2139
|
+
}
|
|
2140
|
+
};
|
|
2141
|
+
const rows = this.db.prepare("SELECT id, subject, from_name, from_address, to_json, cc_json FROM messages").all();
|
|
2142
|
+
const ins = this.db.prepare("INSERT INTO messages_fts (rowid, subject, from_name, from_address, to_text, cc_text, body_text) VALUES (?, ?, ?, ?, ?, ?, '')");
|
|
2143
|
+
this.db.exec("BEGIN");
|
|
2144
|
+
try {
|
|
2145
|
+
for (const r of rows)
|
|
2146
|
+
ins.run(r.id, r.subject || "", r.from_name || "", r.from_address || "", addrText(r.to_json), addrText(r.cc_json));
|
|
2147
|
+
this.db.exec("COMMIT");
|
|
2148
|
+
}
|
|
2149
|
+
catch (e) {
|
|
2150
|
+
try {
|
|
2151
|
+
this.db.exec("ROLLBACK");
|
|
2152
|
+
}
|
|
2153
|
+
catch { /* */ }
|
|
2154
|
+
throw e;
|
|
2155
|
+
}
|
|
2156
|
+
console.log(` [db] messages_fts rebuilt + reindexed ${rows.length} messages`);
|
|
2157
|
+
}
|
|
2158
|
+
catch (e) {
|
|
2159
|
+
console.error(` [db] migrateFtsSchema failed: ${e?.message || e}`);
|
|
2160
|
+
}
|
|
2161
|
+
}
|
|
2037
2162
|
/** Record a prefetch failure (0-body fetch / store-write fail) for a UID,
|
|
2038
2163
|
* incrementing its backoff count. Persisted so it survives restarts. */
|
|
2039
2164
|
recordPrefetchFailure(accountId, folderId, uid) {
|
|
@@ -2772,7 +2897,15 @@ export class MailxDB {
|
|
|
2772
2897
|
searchMessages(query, page = 1, pageSize = 50, accountId, folderId, includeTrashSpam = false) {
|
|
2773
2898
|
query = (query || "").trim();
|
|
2774
2899
|
// Parse qualifiers (C45: extended set — date:, has:, is:, folder:).
|
|
2775
|
-
|
|
2900
|
+
// Fragments accumulate here, then join with EXPLICIT `AND` between two
|
|
2901
|
+
// value fragments. FTS5 treats a parenthesised group followed by an
|
|
2902
|
+
// implicit-AND term as a syntax error — `(a OR b) c` throws "syntax
|
|
2903
|
+
// error near c" and the whole search silently returns nothing. That
|
|
2904
|
+
// killed every `from:X Y` / `to:X Y` / `a|b c` query (Bob 2026-06-08:
|
|
2905
|
+
// `from:Google Voice` found nothing). Explicit `AND` is accepted; an
|
|
2906
|
+
// `op:true` fragment is a user-typed AND/OR/NOT operator, which already
|
|
2907
|
+
// supplies its own glue so we don't insert another `AND` around it.
|
|
2908
|
+
const frags = [];
|
|
2776
2909
|
const parts = query.match(/(?:[^\s"]+|"[^"]*")+/g) || [];
|
|
2777
2910
|
// Extra SQL where-clauses for qualifiers that don't map to FTS columns.
|
|
2778
2911
|
const extraWhere = [];
|
|
@@ -2819,19 +2952,21 @@ export class MailxDB {
|
|
|
2819
2952
|
const isMatch = part.match(/^is:(.+)$/i);
|
|
2820
2953
|
const folderMatch = part.match(/^folder:(.+)$/i);
|
|
2821
2954
|
if (fromMatch) {
|
|
2955
|
+
// FTS5 column-group `{c1 c2}:term` matches term in either
|
|
2956
|
+
// column without the parens that break a trailing implicit-AND.
|
|
2822
2957
|
const term = ftsClean(fromMatch[1]);
|
|
2823
2958
|
if (term)
|
|
2824
|
-
|
|
2959
|
+
frags.push({ s: `{from_name from_address}:${term}*`, op: false });
|
|
2825
2960
|
}
|
|
2826
2961
|
else if (toMatch) {
|
|
2827
2962
|
const term = ftsClean(toMatch[1]);
|
|
2828
2963
|
if (term)
|
|
2829
|
-
|
|
2964
|
+
frags.push({ s: `{to_text cc_text}:${term}*`, op: false });
|
|
2830
2965
|
}
|
|
2831
2966
|
else if (subjectMatch) {
|
|
2832
2967
|
const term = ftsClean(subjectMatch[1]);
|
|
2833
2968
|
if (term)
|
|
2834
|
-
|
|
2969
|
+
frags.push({ s: `subject:${term}*`, op: false });
|
|
2835
2970
|
}
|
|
2836
2971
|
else if (dateMatch || afterMatch || beforeMatch) {
|
|
2837
2972
|
const op = dateMatch ? (dateMatch[1] || "=") : (afterMatch ? ">" : "<");
|
|
@@ -2882,7 +3017,7 @@ export class MailxDB {
|
|
|
2882
3017
|
// `hoddie* AND* git*`, which FTS5 reads as three required terms
|
|
2883
3018
|
// (one of them being any word starting with "AND") — so a real
|
|
2884
3019
|
// match like "Peter Hoddie" + "github" returned zero hits.
|
|
2885
|
-
|
|
3020
|
+
frags.push({ s: part, op: true });
|
|
2886
3021
|
}
|
|
2887
3022
|
else {
|
|
2888
3023
|
// Unqualified — search everything. Strip /regex-literal/
|
|
@@ -2891,7 +3026,7 @@ export class MailxDB {
|
|
|
2891
3026
|
if (term.includes("|")) {
|
|
2892
3027
|
const alts = term.split("|").map(t => ftsClean(t)).filter(Boolean).map(t => `${t}*`).join(" OR ");
|
|
2893
3028
|
if (alts)
|
|
2894
|
-
|
|
3029
|
+
frags.push({ s: `(${alts})`, op: false });
|
|
2895
3030
|
}
|
|
2896
3031
|
else {
|
|
2897
3032
|
// Match the FTS5 unicode61 tokenizer: split on any
|
|
@@ -2905,15 +3040,27 @@ export class MailxDB {
|
|
|
2905
3040
|
// a `*` so incremental typing keeps narrowing.
|
|
2906
3041
|
const sub = ftsClean(term).split(/[^\p{L}\p{N}_]+/u).filter(Boolean);
|
|
2907
3042
|
if (sub.length === 1) {
|
|
2908
|
-
|
|
3043
|
+
frags.push({ s: `${sub[0]}*`, op: false });
|
|
2909
3044
|
}
|
|
2910
3045
|
else if (sub.length > 1) {
|
|
2911
3046
|
const last = sub.pop();
|
|
2912
|
-
|
|
3047
|
+
frags.push({ s: `${sub.join(" ")} ${last}*`, op: false });
|
|
2913
3048
|
}
|
|
2914
3049
|
}
|
|
2915
3050
|
}
|
|
2916
3051
|
}
|
|
3052
|
+
// Join fragments. Two adjacent value fragments need an explicit `AND`
|
|
3053
|
+
// (implicit-AND after a `(...)` / `{...}:` group is an FTS5 syntax
|
|
3054
|
+
// error); a user operator fragment glues itself, so no insert around it.
|
|
3055
|
+
let ftsQuery = "";
|
|
3056
|
+
for (let i = 0; i < frags.length; i++) {
|
|
3057
|
+
if (i === 0) {
|
|
3058
|
+
ftsQuery = frags[i].s;
|
|
3059
|
+
continue;
|
|
3060
|
+
}
|
|
3061
|
+
const sep = (frags[i].op || frags[i - 1].op) ? " " : " AND ";
|
|
3062
|
+
ftsQuery += sep + frags[i].s;
|
|
3063
|
+
}
|
|
2917
3064
|
ftsQuery = ftsQuery.trim();
|
|
2918
3065
|
// No real FTS term — either qualifier-only ("is:flagged after:1w") or
|
|
2919
3066
|
// every term sanitized away (user typed only punctuation). There is NO
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@bobfrankston/mailx-store",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.46",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"main": "index.js",
|
|
6
6
|
"types": "index.d.ts",
|
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
},
|
|
10
10
|
"license": "ISC",
|
|
11
11
|
"dependencies": {
|
|
12
|
-
"@bobfrankston/mailx-types": "^0.1.
|
|
12
|
+
"@bobfrankston/mailx-types": "^0.1.19",
|
|
13
13
|
"@bobfrankston/mailx-settings": "^0.1.26",
|
|
14
14
|
"@bobfrankston/mailx-bus": "^0.1.2",
|
|
15
15
|
"mailparser": "^3.7.2"
|
|
@@ -29,7 +29,7 @@
|
|
|
29
29
|
},
|
|
30
30
|
".transformedSnapshot": {
|
|
31
31
|
"dependencies": {
|
|
32
|
-
"@bobfrankston/mailx-types": "^0.1.
|
|
32
|
+
"@bobfrankston/mailx-types": "^0.1.19",
|
|
33
33
|
"@bobfrankston/mailx-settings": "^0.1.26",
|
|
34
34
|
"@bobfrankston/mailx-bus": "^0.1.2",
|
|
35
35
|
"mailparser": "^3.7.2"
|
package/store.d.ts
CHANGED
|
@@ -130,7 +130,7 @@ export declare class Store {
|
|
|
130
130
|
/** Paginated message list for a (account, folder, ...) query. */
|
|
131
131
|
getMessages(query: MessageQuery): PagedResult<MessageEnvelope>;
|
|
132
132
|
/** All-Inboxes view: union of every account's INBOX, paginated. */
|
|
133
|
-
getUnifiedInbox(page?: number, pageSize?: number): PagedResult<MessageEnvelope>;
|
|
133
|
+
getUnifiedInbox(page?: number, pageSize?: number, flaggedOnly?: boolean): PagedResult<MessageEnvelope>;
|
|
134
134
|
/** Local FTS5 search. Server-scope search is the reconciler's job. */
|
|
135
135
|
searchMessages(query: string, page?: number, pageSize?: number, accountId?: string, folderId?: number, includeTrashSpam?: boolean): PagedResult<MessageEnvelope>;
|
|
136
136
|
/** Read a fully-parsed message (envelope + body + attachments) entirely
|
package/store.js
CHANGED
|
@@ -182,8 +182,8 @@ export class Store {
|
|
|
182
182
|
return this.db.getMessages(query);
|
|
183
183
|
}
|
|
184
184
|
/** All-Inboxes view: union of every account's INBOX, paginated. */
|
|
185
|
-
getUnifiedInbox(page = 1, pageSize = 50) {
|
|
186
|
-
return this.db.getUnifiedInbox(page, pageSize);
|
|
185
|
+
getUnifiedInbox(page = 1, pageSize = 50, flaggedOnly = false) {
|
|
186
|
+
return this.db.getUnifiedInbox(page, pageSize, flaggedOnly);
|
|
187
187
|
}
|
|
188
188
|
/** Local FTS5 search. Server-scope search is the reconciler's job. */
|
|
189
189
|
searchMessages(query, page = 1, pageSize = 50, accountId, folderId, includeTrashSpam = false) {
|
|
@@ -205,16 +205,21 @@ export class Store {
|
|
|
205
205
|
const allowList = this.getCachedAllowlist();
|
|
206
206
|
const senderAddr = (envelope.from?.address || "").toLowerCase();
|
|
207
207
|
const senderDomain = senderAddr.split("@")[1] || "";
|
|
208
|
-
|
|
208
|
+
// Recipient allowlist matches against every recipient-ish address, not
|
|
209
|
+
// just To: a mailing list often carries other subscribers in To/Cc and
|
|
210
|
+
// delivers to the user's allowlisted alias only via Delivered-To (added
|
|
211
|
+
// post-parse below).
|
|
212
|
+
const recipients = (allowList.recipients || []).map((r) => (r || "").toLowerCase());
|
|
213
|
+
const rcptAddrs = [...(envelope.to || []), ...(envelope.cc || [])]
|
|
214
|
+
.map((a) => (a.address || "").toLowerCase());
|
|
209
215
|
// Allowlist auto-allow: trusted sender / domain / recipient skips
|
|
210
216
|
// sanitization. Same rule as the legacy service implementation.
|
|
211
217
|
if (!allowRemote) {
|
|
212
218
|
const senders = (allowList.senders || []).map((s) => (s || "").toLowerCase());
|
|
213
219
|
const domains = (allowList.domains || []).map((d) => (d || "").toLowerCase());
|
|
214
|
-
const recipients = (allowList.recipients || []).map((r) => (r || "").toLowerCase());
|
|
215
220
|
if (senders.includes(senderAddr) ||
|
|
216
221
|
domains.includes(senderDomain) ||
|
|
217
|
-
|
|
222
|
+
rcptAddrs.some((a) => recipients.includes(a))) {
|
|
218
223
|
allowRemote = true;
|
|
219
224
|
}
|
|
220
225
|
}
|
|
@@ -254,8 +259,12 @@ export class Store {
|
|
|
254
259
|
const cached = this.parsedLruGet(cacheKey);
|
|
255
260
|
if (cached) {
|
|
256
261
|
// Allowlist state can change between views even with the same
|
|
257
|
-
// body cached; recompute the volatile fields and overlay.
|
|
258
|
-
|
|
262
|
+
// body cached; recompute the volatile fields and overlay. OR with
|
|
263
|
+
// the cached flag: if the body was already parsed-and-allowed (e.g.
|
|
264
|
+
// via a Delivered-To recipient match, which the early envelope-only
|
|
265
|
+
// check can't see), keep it allowed — the cached HTML is unsanitized
|
|
266
|
+
// and the flag must agree.
|
|
267
|
+
return { ...cached, remoteAllowed: allowRemote || cached.remoteAllowed, isFlagged };
|
|
259
268
|
}
|
|
260
269
|
// Synchronous parse. The .eml is on disk; read + parse it inline
|
|
261
270
|
// and return the fully-rendered message. The parse runs on the
|
|
@@ -362,11 +371,6 @@ export class Store {
|
|
|
362
371
|
size: x.a.size || 0,
|
|
363
372
|
contentId: x.a.contentId || "",
|
|
364
373
|
}));
|
|
365
|
-
if (bodyHtml && !allowRemote) {
|
|
366
|
-
const result = sanitizeHtml(bodyHtml);
|
|
367
|
-
bodyHtml = result.html;
|
|
368
|
-
hasRemoteContent = result.hasRemoteContent;
|
|
369
|
-
}
|
|
370
374
|
// Header extraction — Delivered-To, Return-Path, List-Unsubscribe.
|
|
371
375
|
// Each delivery agent PREPENDS its Delivered-To, so the FIRST
|
|
372
376
|
// (topmost) header is the final delivery to the user's actual
|
|
@@ -374,6 +378,8 @@ export class Store {
|
|
|
374
378
|
// routing artifacts. Take [0]. (The old code took the LAST entry —
|
|
375
379
|
// on a forwarded message that grabbed a stale hop, e.g. a malformed
|
|
376
380
|
// `…@elkin.ws@trap-prot` address — Bob 2026-05-21.)
|
|
381
|
+
// This runs BEFORE sanitization because Delivered-To feeds the
|
|
382
|
+
// recipient-allowlist re-check just below.
|
|
377
383
|
let deliveredTo = "";
|
|
378
384
|
const rawDelivered = parsed.headers.get("delivered-to");
|
|
379
385
|
if (rawDelivered) {
|
|
@@ -403,6 +409,21 @@ export class Store {
|
|
|
403
409
|
const mailxDraftId = hdr("x-mailx-draft-id").trim();
|
|
404
410
|
const { listUnsubscribeMail, listUnsubscribeHttp, listUnsubscribeOneClick } = parseListUnsubscribe(parsed.headers);
|
|
405
411
|
const listUnsubscribe = listUnsubscribeHttp || listUnsubscribeMail;
|
|
412
|
+
// Recipient-allowlist re-check with Delivered-To in hand. The early
|
|
413
|
+
// pass (envelope only) can't see Delivered-To, so a list message whose
|
|
414
|
+
// To/Cc are other subscribers and whose only allowlisted address is the
|
|
415
|
+
// Delivered-To alias (e.g. shsaa@bob.ma) would still be sanitized.
|
|
416
|
+
// Match the bare address out of "Name <addr>" / "<addr>" / "addr".
|
|
417
|
+
if (!allowRemote && deliveredTo) {
|
|
418
|
+
const dAddr = (deliveredTo.match(/[^\s<>]+@[^\s<>]+/)?.[0] || "").toLowerCase();
|
|
419
|
+
if (dAddr && recipients.includes(dAddr))
|
|
420
|
+
allowRemote = true;
|
|
421
|
+
}
|
|
422
|
+
if (bodyHtml && !allowRemote) {
|
|
423
|
+
const result = sanitizeHtml(bodyHtml);
|
|
424
|
+
bodyHtml = result.html;
|
|
425
|
+
hasRemoteContent = result.hasRemoteContent;
|
|
426
|
+
}
|
|
406
427
|
const result = {
|
|
407
428
|
...envelope,
|
|
408
429
|
bodyHtml, bodyText,
|