@bobfrankston/mailx-store 0.1.32 → 0.1.34

package/charset.d.ts

@@ -1,15 +1,29 @@
 /**
- * Charset normalization for incoming email bodies.
+ * Charset normalization for incoming email bodies. Two corrections, both
+ * applied by rewriting the part's `charset=` declaration before the parser
+ * decodes it:
  *
- * Many senders (esp. PHPMailer-driven marketing) declare
- * `charset=iso-8859-1` but emit UTF-8 bytes. simpleParser honors the
- * declared charset and produces "â??" garbage for every non-ASCII
- * codepoint (em-dash, smart quotes, …). When the raw body bytes are
- * valid UTF-8, rewrite the charset header before parsing. We only
- * override the obviously-wrong legacy declarations; explicit utf-8 /
- * koi8 / etc. pass through.
+ *  1. Mis-declared UTF-8: PHPMailer-style senders declare `iso-8859-1`
+ *     (the PHP default) but emit UTF-8 bytes. When the raw body really is
+ *     valid UTF-8, rewrite the declaration to utf-8.
+ *
+ *  2. iso-8859-1 → windows-1252: senders that genuinely mean a legacy
+ *     8-bit charset overwhelmingly emit Windows-1252, not strict
+ *     ISO-8859-1 — smart quotes / em-dash / euro live in 0x80-0x9F, which
+ *     ISO-8859-1 leaves as unusable C1 control codes. The WHATWG / browser
+ *     standard is to decode `iso-8859-1` AS `windows-1252`; iconv-lite
+ *     does not, so we rewrite the declaration. windows-1252 is a strict
+ *     superset of printable ISO-8859-1, so a genuine ISO-8859-1 char is
+ *     never lost.
  */
 /** Returns either the original buffer (no change needed) or a copy with
- *  the leading charset declaration rewritten to utf-8. */
+ *  the leading charset declaration corrected. */
 export declare function sniffAndFixCharset(raw: Buffer): Buffer;
+/** String form of sniffAndFixCharset — for the preview path, which parses
+ *  the message as a string (extractPreview). Same two corrections, applied
+ *  to the charset declaration in the text. The UTF-8-misdeclare correction
+ *  can't be done on a string (the bytes are already decoded), so this only
+ *  does correction 2 (iso-8859-1 → windows-1252), which is a pure
+ *  declaration rewrite and the common case for previews. */
+export declare function fixCharsetDeclString(s: string): string;
 //# sourceMappingURL=charset.d.ts.map

package/charset.js

@@ -1,26 +1,68 @@
 /**
- * Charset normalization for incoming email bodies.
+ * Charset normalization for incoming email bodies. Two corrections, both
+ * applied by rewriting the part's `charset=` declaration before the parser
+ * decodes it:
  *
- * Many senders (esp. PHPMailer-driven marketing) declare
- * `charset=iso-8859-1` but emit UTF-8 bytes. simpleParser honors the
- * declared charset and produces "â??" garbage for every non-ASCII
- * codepoint (em-dash, smart quotes, …). When the raw body bytes are
- * valid UTF-8, rewrite the charset header before parsing. We only
- * override the obviously-wrong legacy declarations; explicit utf-8 /
- * koi8 / etc. pass through.
+ *  1. Mis-declared UTF-8: PHPMailer-style senders declare `iso-8859-1`
+ *     (the PHP default) but emit UTF-8 bytes. When the raw body really is
+ *     valid UTF-8, rewrite the declaration to utf-8.
+ *
+ *  2. iso-8859-1 → windows-1252: senders that genuinely mean a legacy
+ *     8-bit charset overwhelmingly emit Windows-1252, not strict
+ *     ISO-8859-1 — smart quotes / em-dash / euro live in 0x80-0x9F, which
+ *     ISO-8859-1 leaves as unusable C1 control codes. The WHATWG / browser
+ *     standard is to decode `iso-8859-1` AS `windows-1252`; iconv-lite
+ *     does not, so we rewrite the declaration. windows-1252 is a strict
+ *     superset of printable ISO-8859-1, so a genuine ISO-8859-1 char is
+ *     never lost.
  */
 /** Returns either the original buffer (no change needed) or a copy with
- *  the leading charset declaration rewritten to utf-8. */
+ *  the leading charset declaration corrected. */
 export function sniffAndFixCharset(raw) {
     const HEAD_LIMIT = 16384;
     const head = raw.subarray(0, Math.min(HEAD_LIMIT, raw.length)).toString("latin1");
-    const re = /charset\s*=\s*"?(iso-8859-1|us-ascii|windows-1252|latin1)"?/gi;
-    if (!re.test(head))
-        return raw;
-    if (!isValidUtf8(raw))
+    const legacyRe = /charset\s*=\s*"?(iso-8859-1|us-ascii|windows-1252|latin1)"?/gi;
+    if (!legacyRe.test(head))
         return raw;
-    const fixed = head.replace(/charset\s*=\s*"?(iso-8859-1|us-ascii|windows-1252|latin1)"?/gi, "charset=utf-8");
-    return Buffer.concat([Buffer.from(fixed, "latin1"), raw.subarray(head.length)]);
+    // Correction 1 — mis-declared UTF-8. Only sound when the raw bytes ARE
+    // the body bytes (an 8bit part): for a quoted-printable / base64 part
+    // the raw .eml is pure ASCII, so isValidUtf8 passes vacuously and we'd
+    // wrongly relabel a real Windows-1252 part as utf-8 (Bob 2026-05-21).
+    // Requiring a real non-ASCII byte gates the UTF-8 sniff to where it
+    // means something.
+    if (hasNonAscii(raw) && isValidUtf8(raw)) {
+        const fixed = head.replace(legacyRe, "charset=utf-8");
+        return Buffer.concat([Buffer.from(fixed, "latin1"), raw.subarray(head.length)]);
+    }
+    // Correction 2 — iso-8859-1 / latin1 → windows-1252. Applies whether
+    // the part is QP or 8bit; the rewrite is purely on the declaration.
+    // (Bob 2026-05-22: an Intuit mail declared ISO-8859-1, QP-encoded a
+    // 0x92 apostrophe → latin1 decode produced a U+0092 control char.)
+    const isoRe = /charset\s*=\s*"?(iso-8859-1|latin1)"?/gi;
+    if (isoRe.test(head)) {
+        const fixed = head.replace(isoRe, "charset=windows-1252");
+        return Buffer.concat([Buffer.from(fixed, "latin1"), raw.subarray(head.length)]);
+    }
+    return raw;
+}
+/** String form of sniffAndFixCharset — for the preview path, which parses
+ *  the message as a string (extractPreview). Same two corrections, applied
+ *  to the charset declaration in the text. The UTF-8-misdeclare correction
+ *  can't be done on a string (the bytes are already decoded), so this only
+ *  does correction 2 (iso-8859-1 → windows-1252), which is a pure
+ *  declaration rewrite and the common case for previews. */
+export function fixCharsetDeclString(s) {
+    return s.replace(/charset\s*=\s*"?(iso-8859-1|latin1)"?/gi, "charset=windows-1252");
+}
+/** True if the buffer contains at least one byte >= 0x80. A pure-ASCII
+ *  buffer is trivially valid UTF-8, so the isValidUtf8 sniff tells us
+ *  nothing about a quoted-printable / base64 body — gate on this first. */
+function hasNonAscii(buf) {
+    for (let i = 0; i < buf.length; i++) {
+        if (buf[i] >= 0x80)
+            return true;
+    }
+    return false;
 }
 /** Strict UTF-8 validity check: rejects overlong forms, invalid start
  *  bytes, and dangling continuations. Used to confirm the body is really

package/db.d.ts

@@ -520,6 +520,7 @@ export declare class MailxDB {
         name: string;
         email: string;
         source: string;
+        sources: string[];
         useCount: number;
     }[];
     /** List all contacts (address-book view) with pagination + optional filter. */

package/db.js

@@ -23,11 +23,18 @@ const _libmime = _libmimeMod.default || _libmimeMod;
 function decodeHeaderWords(s) {
     if (!s || s.indexOf("=?") < 0)
         return s || "";
+    // Normalize a POSIX-locale charset to the bare charset before decoding.
+    // Some mailers emit `=?en_US.UTF-8?Q?...?=` — `en_US.UTF-8` is a locale
+    // string (language_TERRITORY.CODESET), not an RFC 2047 charset; libmime /
+    // iconv-lite don't recognize it and the encoded-word stays raw in the
+    // subject (Bob 2026-05-22, NNSquad mail). The real charset is the CODESET
+    // after the dot. Rewrite `=?lang_REGION.CHARSET?` → `=?CHARSET?`.
+    const fixed = s.replace(/=\?[A-Za-z]{1,8}_[A-Za-z]{1,8}\.([A-Za-z0-9][A-Za-z0-9._-]*)((?:\*[^?]*)?\?)/g, "=?$1$2");
     try {
-        return _libmime.decodeWords(s);
+        return _libmime.decodeWords(fixed);
     }
     catch {
-        return s;
+        return fixed;
     }
 }
 /** Addresses that have no business in autocomplete. Patterns load from
@@ -2451,6 +2458,13 @@ export class MailxDB {
         const tokens = query.split(/\s+/).filter(Boolean);
         const firstSubstr = `%${tokens[0]}%`;
         const firstPrefix = `${tokens[0]}%`;
+        // Word-prefix patterns: the token at the start of ANY name word, not
+        // just the whole name. Catches "Frankston, Bob" / "Frankston,Bob"
+        // when the user types "bob" — a first-name match on a Last,First
+        // contact (Bob 2026-05-21). The space variant also covers the normal
+        // "Bob Frankston" middle/last-word case.
+        const firstWordSpace = `% ${tokens[0]}%`;
+        const firstWordComma = `%,${tokens[0]}%`;
         const tokenWhere = tokens.map(() => "(name LIKE ? OR email LIKE ?)").join(" AND ");
         const tokenParams = [];
         for (const t of tokens) {
@@ -2463,11 +2477,19 @@ export class MailxDB {
             // contacts.jsonc#preferred[]. The user's `source: "work"` /
             // `source: "family"` tags all rank +40 alongside the default
             // `preferred` label.
+            // Ranking: match POSITION dominates, source is only a tiebreaker
+            // within the same position tier. Position tiers are spaced 100
+            // apart and source tops out at 40, so a prefix match (300+) can
+            // never be outranked by a mid-string match (≤140) — Bob 2026-05-21:
+            // "the match should favor starting letters rather than random
+            // letters in the middle." The old 3/2/1 + 0/30/40 weighting let a
+            // google contact matched mid-string (1+30=31) beat a discovered
+            // contact matched on prefix (3+0=3).
             rows = this.db.prepare(`SELECT name, email, source, use_count, last_used,
                         (CASE
-                            WHEN lower(name) LIKE lower(?) THEN 3
-                            WHEN substr(email, 1, instr(email, '@') - 1) LIKE lower(?) THEN 2
-                            WHEN email LIKE ? OR name LIKE ? THEN 1
+                            WHEN lower(name) LIKE lower(?) OR lower(name) LIKE lower(?) OR lower(name) LIKE lower(?) THEN 300
+                            WHEN substr(email, 1, instr(email, '@') - 1) LIKE lower(?) THEN 200
+                            WHEN email LIKE ? OR name LIKE ? THEN 100
                             ELSE 0
                          END) +
                         (CASE
@@ -2478,7 +2500,7 @@ export class MailxDB {
                  FROM contacts
                  WHERE ${tokenWhere}
                  ORDER BY match_rank DESC, use_count DESC, last_used DESC
-                 LIMIT ?`).all(firstPrefix, firstPrefix, firstSubstr, firstSubstr, ...tokenParams, limit * 2);
+                 LIMIT ?`).all(firstPrefix, firstWordSpace, firstWordComma, firstPrefix, firstSubstr, firstSubstr, ...tokenParams, limit * 2);
         }
         catch (e) {
             console.error(`  [searchContacts] ranked query failed (${e?.message}) — falling back to simple LIKE`);
@@ -2499,23 +2521,38 @@ export class MailxDB {
         rows.sort((a, b) => score(b) - score(a));
         // Dedup by lowercased email — same address often appears as both
         // source='google' (synced from Google Contacts) and source='discovered'
-        // (auto-collected from sent mail). The user only wants to see the
-        // best entry; we keep the higher-ranked source (which the sort above
-        // has already put first), and silently drop the duplicate. Without
-        // this, the autocomplete dropdown showed two identical Kevin Healy
-        // rows just labeled GOOGLE and DISCOVERED.
-        const seenEmails = new Set();
-        rows = rows.filter(r => {
+        // (auto-collected from sent mail). Keep the higher-ranked row (already
+        // sorted first) and FOLD subsequent rows' sources into a `sources` array
+        // on the survivor so the dropdown can display "google, discovered"
+        // instead of showing two duplicate lines.
+        const merged = new Map();
+        for (const r of rows) {
             const k = (r.email || "").toLowerCase();
-            if (!k)
-                return true;
-            if (seenEmails.has(k))
-                return false;
-            seenEmails.add(k);
-            return true;
-        });
-        rows = rows.slice(0, limit);
-        return rows.map(r => ({ name: r.name, email: r.email, source: r.source, useCount: r.use_count }));
+            if (!k) {
+                merged.set(`__noemail_${merged.size}`, { ...r, _sources: new Set([r.source]) });
+                continue;
+            }
+            const existing = merged.get(k);
+            if (!existing) {
+                merged.set(k, { ...r, _sources: new Set([r.source]) });
+            }
+            else {
+                existing._sources.add(r.source);
+                // If this row has a non-empty name and the kept one didn't,
+                // promote the name — Google rows often have proper names while
+                // discovered rows are bare-email.
+                if (!existing.name && r.name)
+                    existing.name = r.name;
+            }
+        }
+        const out = Array.from(merged.values()).slice(0, limit);
+        return out.map(r => ({
+            name: r.name,
+            email: r.email,
+            source: r.source,
+            sources: Array.from(r._sources).filter(Boolean),
+            useCount: r.use_count,
+        }));
     }
     /** List all contacts (address-book view) with pagination + optional filter. */
     listContacts(query, page = 1, pageSize = 100) {
@@ -2605,6 +2642,11 @@ export class MailxDB {
             const ts = Date.parse(s);
             return isNaN(ts) ? null : ts;
         };
+        // Strip FTS5 metacharacters from a user term. A stray `;` `(` `:` `"`
+        // etc. is a hard `fts5: syntax error` (Bob 2026-05-21 log). FTS5's
+        // tokenizer splits on non-word chars anyway, so removing them loses no
+        // real matching ability.
+        const ftsClean = (t) => t.replace(/["';:(){}\[\]^~\\/]/g, "").trim();
         for (const part of parts) {
             const fromMatch = part.match(/^from:(.+)$/i);
             const toMatch = part.match(/^to:(.+)$/i);
@@ -2616,16 +2658,19 @@ export class MailxDB {
             const isMatch = part.match(/^is:(.+)$/i);
             const folderMatch = part.match(/^folder:(.+)$/i);
             if (fromMatch) {
-                const term = fromMatch[1].replace(/"/g, "");
-                ftsQuery += `(from_name:${term} OR from_address:${term}) `;
+                const term = ftsClean(fromMatch[1]);
+                if (term)
+                    ftsQuery += `(from_name:${term}* OR from_address:${term}*) `;
             }
             else if (toMatch) {
-                const term = toMatch[1].replace(/"/g, "");
-                ftsQuery += `(to_text:${term} OR cc_text:${term}) `;
+                const term = ftsClean(toMatch[1]);
+                if (term)
+                    ftsQuery += `(to_text:${term}* OR cc_text:${term}*) `;
             }
             else if (subjectMatch) {
-                const term = subjectMatch[1].replace(/"/g, "");
-                ftsQuery += `subject:${term} `;
+                const term = ftsClean(subjectMatch[1]);
+                if (term)
+                    ftsQuery += `subject:${term}* `;
             }
             else if (dateMatch || afterMatch || beforeMatch) {
                 const op = dateMatch ? (dateMatch[1] || "=") : (afterMatch ? ">" : "<");
@@ -2679,22 +2724,32 @@ export class MailxDB {
                 ftsQuery += `${part} `;
             }
             else {
-                // Unqualified — search everything.
+                // Unqualified — search everything. Strip /regex-literal/
+                // slashes and FTS5 metacharacters before wildcarding.
                 let term = part.replace(/^\/|\/$/g, "");
                 if (term.includes("|")) {
-                    const alts = term.split("|").filter(Boolean).map(t => `${t}*`).join(" OR ");
-                    ftsQuery += `(${alts}) `;
+                    const alts = term.split("|").map(t => ftsClean(t)).filter(Boolean).map(t => `${t}*`).join(" OR ");
+                    if (alts)
+                        ftsQuery += `(${alts}) `;
                 }
                 else {
-                    ftsQuery += `${term}* `;
+                    term = ftsClean(term);
+                    if (term)
+                        ftsQuery += `${term}* `;
                 }
             }
         }
         ftsQuery = ftsQuery.trim();
-        // If the user typed only qualifier-only terms (e.g. "is:flagged after:1w"),
-        // FTS query is empty — match-all surrogate.
-        if (!ftsQuery)
-            ftsQuery = "*";
+        // No real FTS term — either qualifier-only ("is:flagged after:1w") or
+        // every term sanitized away (user typed only punctuation). There is NO
+        // valid FTS5 "match everything" string — `MATCH '*'` is itself an FTS5
+        // error ("unknown special query"). So: with qualifiers, query the
+        // messages table directly (no FTS join); with neither, the query is
+        // garbage — return empty rather than dumping the whole mailbox.
+        const hasFts = ftsQuery.length > 0;
+        if (!hasFts && extraWhere.length === 0) {
+            return { items: [], total: 0, page, pageSize };
+        }
         const offset = (page - 1) * pageSize;
         try {
             let scopeWhere = "";
@@ -2719,6 +2774,13 @@ export class MailxDB {
                 scopeWhere += " AND " + extraWhere.join(" AND ");
                 scopeParams.push(...extraParams);
             }
+            // FTS path joins messages_fts + MATCH; qualifier-only path skips
+            // the FTS table entirely and filters `messages` directly.
+            const fromJoin = hasFts
+                ? "FROM messages m JOIN messages_fts fts ON m.id = fts.rowid LEFT JOIN folders f ON f.id = m.folder_id AND f.account_id = m.account_id"
+                : "FROM messages m LEFT JOIN folders f ON f.id = m.folder_id AND f.account_id = m.account_id";
+            const matchClause = hasFts ? "messages_fts MATCH ?" : "1=1";
+            const matchParams = hasFts ? [ftsQuery] : [];
             // Cap COUNT at 1001 — a bare `SELECT COUNT(*)` on FTS5 has to
             // enumerate every match, which for a common term ("ieee", "the")
             // can be tens of thousands of rows and several seconds. We never
@@ -2726,19 +2788,15 @@ export class MailxDB {
             // count is invisible to the user and turns search from
             // multi-second to sub-second. UI shows "1000+" when cnt === 1001.
             const countRow = this.db.prepare(`SELECT COUNT(*) as cnt FROM (
-                    SELECT 1 FROM messages m
-                    JOIN messages_fts fts ON m.id = fts.rowid
-                    LEFT JOIN folders f ON f.id = m.folder_id AND f.account_id = m.account_id
-                    WHERE messages_fts MATCH ?${scopeWhere}
+                    SELECT 1 ${fromJoin}
+                    WHERE ${matchClause}${scopeWhere}
                     LIMIT 1001
-                 )`).get(ftsQuery, ...scopeParams);
+                 )`).get(...matchParams, ...scopeParams);
             const total = countRow?.cnt || 0;
-            const rows = this.db.prepare(`SELECT m.*, f.name AS folder_name FROM messages m
-                 JOIN messages_fts fts ON m.id = fts.rowid
-                 LEFT JOIN folders f ON f.id = m.folder_id AND f.account_id = m.account_id
-                 WHERE messages_fts MATCH ?${scopeWhere}
+            const rows = this.db.prepare(`SELECT m.*, f.name AS folder_name ${fromJoin}
+                 WHERE ${matchClause}${scopeWhere}
                  ORDER BY m.date DESC
-                 LIMIT ? OFFSET ?`).all(ftsQuery, ...scopeParams, pageSize, offset);
+                 LIMIT ? OFFSET ?`).all(...matchParams, ...scopeParams, pageSize, offset);
             const items = rows.map(r => ({
                 id: r.id,
                 accountId: r.account_id,

package/index.d.ts

@@ -5,6 +5,7 @@
 export { MailxDB } from "./db.js";
 export { FileMessageStore } from "./file-store.js";
 export { parseSerial, prewarmParseWorker } from "./parse-serial.js";
+export { sniffAndFixCharset, fixCharsetDeclString } from "./charset.js";
 export { Store } from "./store.js";
 export type { StoreMessage } from "./store.js";
 export { StoreBus, storeBus } from "@bobfrankston/mailx-bus";

package/index.js

@@ -5,6 +5,7 @@
 export { MailxDB } from "./db.js";
 export { FileMessageStore } from "./file-store.js";
 export { parseSerial, prewarmParseWorker } from "./parse-serial.js";
+export { sniffAndFixCharset, fixCharsetDeclString } from "./charset.js";
 // Store — the nexus. Owns DB + .eml files + operations + bus.
 export { Store } from "./store.js";
 // Store-event bus lives in `@bobfrankston/mailx-bus` so the browser-side

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bobfrankston/mailx-store",
-  "version": "0.1.32",
+  "version": "0.1.34",
   "type": "module",
   "main": "index.js",
   "types": "index.d.ts",

package/store.js

@@ -300,6 +300,29 @@ export class Store {
         }
         let bodyHtml = parsed.html || "";
         const bodyText = parsed.text || "";
+        // Charset diagnostic. If the parsed body has U+FFFD, distinguish the
+        // two very different causes:
+        //  - SOURCE already contains U+FFFD (raw EF BF BD bytes, or QP
+        //    =EF=BF=BD) → the message was composed/replied during the pre-fix
+        //    charset window; the original bytes are gone — UNRECOVERABLE, and
+        //    NOT a current bug.
+        //  - source is clean but the parse produced U+FFFD → a live
+        //    charset-decode bug worth chasing.
+        if (bodyHtml.includes("�") || bodyText.includes("�")) {
+            let sourceCorrupt = false;
+            try {
+                const src = await this.bodyStore.readByPath(storedPath);
+                sourceCorrupt = src.includes(Buffer.from([0xEF, 0xBF, 0xBD]))
+                    || src.toString("latin1").toUpperCase().includes("=EF=BF=BD");
+            }
+            catch { /* */ }
+            if (sourceCorrupt) {
+                console.error(`  [charset] U+FFFD baked into SOURCE — acct=${accountId} uid=${uid} path=${storedPath} — corrupt at compose time (pre-fix), unrecoverable`);
+            }
+            else {
+                console.error(`  [charset] U+FFFD from PARSE — acct=${accountId} uid=${uid} path=${storedPath} — LIVE charset-decode bug, source is clean`);
+            }
+        }
         // Backfill FTS body_text now that we've parsed the body. upsertMessage
         // only had the short `preview` snippet at index time; without this
         // backfill, searches miss any word that only appears deeper in the
@@ -345,17 +368,17 @@ export class Store {
             hasRemoteContent = result.hasRemoteContent;
         }
         // Header extraction — Delivered-To, Return-Path, List-Unsubscribe.
-        // mlproc preprocesses Delivered-To server-side, so the inner
-        // address is already clean by the time mailx sees it. We take the
-        // last entry of the chain (the final-delivery hop). The
-        // `relayDomains` filtering layer was retired 2026-05-13: nothing
-        // configures it in practice and the conditional made the code
-        // harder to reason about than the one-liner that replaces it.
+        // Each delivery agent PREPENDS its Delivered-To, so the FIRST
+        // (topmost) header is the final delivery to the user's actual
+        // mailbox; later ones are earlier forwarding hops / internal
+        // routing artifacts. Take [0]. (The old code took the LAST entry —
+        // on a forwarded message that grabbed a stale hop, e.g. a malformed
+        // `…@elkin.ws@trap-prot` address — Bob 2026-05-21.)
         let deliveredTo = "";
         const rawDelivered = parsed.headers.get("delivered-to");
         if (rawDelivered) {
             const deliveredList = Array.isArray(rawDelivered) ? rawDelivered : [rawDelivered];
-            const d = deliveredList[deliveredList.length - 1];
+            const d = deliveredList[0];
             deliveredTo = typeof d === "string" ? d : d?.text || d?.address || String(d);
         }
         const hdr = (key) => {