@bobfrankston/mailx-store 0.1.33 → 0.1.35

package/charset.d.ts

@@ -1,15 +1,29 @@
 /**
- * Charset normalization for incoming email bodies.
+ * Charset normalization for incoming email bodies. Two corrections, both
+ * applied by rewriting the part's `charset=` declaration before the parser
+ * decodes it:
  *
- * Many senders (esp. PHPMailer-driven marketing) declare
- * `charset=iso-8859-1` but emit UTF-8 bytes. simpleParser honors the
- * declared charset and produces "â??" garbage for every non-ASCII
- * codepoint (em-dash, smart quotes, …). When the raw body bytes are
- * valid UTF-8, rewrite the charset header before parsing. We only
- * override the obviously-wrong legacy declarations; explicit utf-8 /
- * koi8 / etc. pass through.
+ *  1. Mis-declared UTF-8: PHPMailer-style senders declare `iso-8859-1`
+ *     (the PHP default) but emit UTF-8 bytes. When the raw body really is
+ *     valid UTF-8, rewrite the declaration to utf-8.
+ *
+ *  2. iso-8859-1 → windows-1252: senders that genuinely mean a legacy
+ *     8-bit charset overwhelmingly emit Windows-1252, not strict
+ *     ISO-8859-1 — smart quotes / em-dash / euro live in 0x80-0x9F, which
+ *     ISO-8859-1 leaves as unusable C1 control codes. The WHATWG / browser
+ *     standard is to decode `iso-8859-1` AS `windows-1252`; iconv-lite
+ *     does not, so we rewrite the declaration. windows-1252 is a strict
+ *     superset of printable ISO-8859-1, so a genuine ISO-8859-1 char is
+ *     never lost.
  */
 /** Returns either the original buffer (no change needed) or a copy with
- *  the leading charset declaration rewritten to utf-8. */
+ *  the leading charset declaration corrected. */
 export declare function sniffAndFixCharset(raw: Buffer): Buffer;
+/** String form of sniffAndFixCharset — for the preview path, which parses
+ *  the message as a string (extractPreview). Same two corrections, applied
+ *  to the charset declaration in the text. The UTF-8-misdeclare correction
+ *  can't be done on a string (the bytes are already decoded), so this only
+ *  does correction 2 (iso-8859-1 → windows-1252), which is a pure
+ *  declaration rewrite and the common case for previews. */
+export declare function fixCharsetDeclString(s: string): string;
 //# sourceMappingURL=charset.d.ts.map

package/charset.js

@@ -1,36 +1,58 @@
 /**
- * Charset normalization for incoming email bodies.
+ * Charset normalization for incoming email bodies. Two corrections, both
+ * applied by rewriting the part's `charset=` declaration before the parser
+ * decodes it:
  *
- * Many senders (esp. PHPMailer-driven marketing) declare
- * `charset=iso-8859-1` but emit UTF-8 bytes. simpleParser honors the
- * declared charset and produces "â??" garbage for every non-ASCII
- * codepoint (em-dash, smart quotes, …). When the raw body bytes are
- * valid UTF-8, rewrite the charset header before parsing. We only
- * override the obviously-wrong legacy declarations; explicit utf-8 /
- * koi8 / etc. pass through.
+ *  1. Mis-declared UTF-8: PHPMailer-style senders declare `iso-8859-1`
+ *     (the PHP default) but emit UTF-8 bytes. When the raw body really is
+ *     valid UTF-8, rewrite the declaration to utf-8.
+ *
+ *  2. iso-8859-1 → windows-1252: senders that genuinely mean a legacy
+ *     8-bit charset overwhelmingly emit Windows-1252, not strict
+ *     ISO-8859-1 — smart quotes / em-dash / euro live in 0x80-0x9F, which
+ *     ISO-8859-1 leaves as unusable C1 control codes. The WHATWG / browser
+ *     standard is to decode `iso-8859-1` AS `windows-1252`; iconv-lite
+ *     does not, so we rewrite the declaration. windows-1252 is a strict
+ *     superset of printable ISO-8859-1, so a genuine ISO-8859-1 char is
+ *     never lost.
  */
 /** Returns either the original buffer (no change needed) or a copy with
- *  the leading charset declaration rewritten to utf-8. */
+ *  the leading charset declaration corrected. */
 export function sniffAndFixCharset(raw) {
     const HEAD_LIMIT = 16384;
     const head = raw.subarray(0, Math.min(HEAD_LIMIT, raw.length)).toString("latin1");
-    const re = /charset\s*=\s*"?(iso-8859-1|us-ascii|windows-1252|latin1)"?/gi;
-    if (!re.test(head))
-        return raw;
-    // The rewrite is only sound when the raw bytes ARE the body bytes — i.e.
-    // an 8bit / binary part. For a quoted-printable or base64 part the raw
-    // .eml is pure ASCII (the high bytes live inside `=XX` / base64 chars),
-    // so isValidUtf8(raw) passes vacuously and we would relabel a genuine
-    // Windows-1252 part as utf-8 — every smart-quote / em-dash then decodes
-    // as mojibake (Bob's 2026-05-21 report: a QP Windows-1252 Outlook mail).
-    // Requiring an actual non-ASCII byte in the raw gates the heuristic to
-    // the only case where the UTF-8 sniff is meaningful.
-    if (!hasNonAscii(raw))
-        return raw;
-    if (!isValidUtf8(raw))
+    const legacyRe = /charset\s*=\s*"?(iso-8859-1|us-ascii|windows-1252|latin1)"?/gi;
+    if (!legacyRe.test(head))
         return raw;
-    const fixed = head.replace(/charset\s*=\s*"?(iso-8859-1|us-ascii|windows-1252|latin1)"?/gi, "charset=utf-8");
-    return Buffer.concat([Buffer.from(fixed, "latin1"), raw.subarray(head.length)]);
+    // Correction 1 — mis-declared UTF-8. Only sound when the raw bytes ARE
+    // the body bytes (an 8bit part): for a quoted-printable / base64 part
+    // the raw .eml is pure ASCII, so isValidUtf8 passes vacuously and we'd
+    // wrongly relabel a real Windows-1252 part as utf-8 (Bob 2026-05-21).
+    // Requiring a real non-ASCII byte gates the UTF-8 sniff to where it
+    // means something.
+    if (hasNonAscii(raw) && isValidUtf8(raw)) {
+        const fixed = head.replace(legacyRe, "charset=utf-8");
+        return Buffer.concat([Buffer.from(fixed, "latin1"), raw.subarray(head.length)]);
+    }
+    // Correction 2 — iso-8859-1 / latin1 → windows-1252. Applies whether
+    // the part is QP or 8bit; the rewrite is purely on the declaration.
+    // (Bob 2026-05-22: an Intuit mail declared ISO-8859-1, QP-encoded a
+    // 0x92 apostrophe → latin1 decode produced a U+0092 control char.)
+    const isoRe = /charset\s*=\s*"?(iso-8859-1|latin1)"?/gi;
+    if (isoRe.test(head)) {
+        const fixed = head.replace(isoRe, "charset=windows-1252");
+        return Buffer.concat([Buffer.from(fixed, "latin1"), raw.subarray(head.length)]);
+    }
+    return raw;
+}
+/** String form of sniffAndFixCharset — for the preview path, which parses
+ *  the message as a string (extractPreview). Same two corrections, applied
+ *  to the charset declaration in the text. The UTF-8-misdeclare correction
+ *  can't be done on a string (the bytes are already decoded), so this only
+ *  does correction 2 (iso-8859-1 → windows-1252), which is a pure
+ *  declaration rewrite and the common case for previews. */
+export function fixCharsetDeclString(s) {
+    return s.replace(/charset\s*=\s*"?(iso-8859-1|latin1)"?/gi, "charset=windows-1252");
 }
 /** True if the buffer contains at least one byte >= 0x80. A pure-ASCII
  *  buffer is trivially valid UTF-8, so the isValidUtf8 sniff tells us

package/db.js

@@ -23,11 +23,18 @@ const _libmime = _libmimeMod.default || _libmimeMod;
 function decodeHeaderWords(s) {
     if (!s || s.indexOf("=?") < 0)
         return s || "";
+    // Normalize a POSIX-locale charset to the bare charset before decoding.
+    // Some mailers emit `=?en_US.UTF-8?Q?...?=` — `en_US.UTF-8` is a locale
+    // string (language_TERRITORY.CODESET), not an RFC 2047 charset; libmime /
+    // iconv-lite don't recognize it and the encoded-word stays raw in the
+    // subject (Bob 2026-05-22, NNSquad mail). The real charset is the CODESET
+    // after the dot. Rewrite `=?lang_REGION.CHARSET?` → `=?CHARSET?`.
+    const fixed = s.replace(/=\?[A-Za-z]{1,8}_[A-Za-z]{1,8}\.([A-Za-z0-9][A-Za-z0-9._-]*)((?:\*[^?]*)?\?)/g, "=?$1$2");
     try {
-        return _libmime.decodeWords(s);
+        return _libmime.decodeWords(fixed);
     }
     catch {
-        return s;
+        return fixed;
     }
 }
 /** Addresses that have no business in autocomplete. Patterns load from
@@ -2726,17 +2733,37 @@ export class MailxDB {
                         ftsQuery += `(${alts}) `;
                 }
                 else {
-                    term = ftsClean(term);
-                    if (term)
-                        ftsQuery += `${term}* `;
+                    // Match the FTS5 unicode61 tokenizer: split on any
+                    // non-(letter|number|underscore) so the user's term
+                    // shapes the same way the index did. Without this,
+                    // typing "192." built "192.*" and matched nothing —
+                    // FTS5 had indexed "192.55.226" as the three tokens
+                    // 192/55/226 (Bob 2026-05-22 "type 192. it no longer
+                    // matches"). AND the sub-tokens (FTS5 implicit AND
+                    // between space-separated terms); only the LAST gets
+                    // a `*` so incremental typing keeps narrowing.
+                    const sub = ftsClean(term).split(/[^\p{L}\p{N}_]+/u).filter(Boolean);
+                    if (sub.length === 1) {
+                        ftsQuery += `${sub[0]}* `;
+                    }
+                    else if (sub.length > 1) {
+                        const last = sub.pop();
+                        ftsQuery += `${sub.join(" ")} ${last}* `;
+                    }
                 }
             }
         }
         ftsQuery = ftsQuery.trim();
-        // If the user typed only qualifier-only terms (e.g. "is:flagged after:1w"),
-        // FTS query is empty — match-all surrogate.
-        if (!ftsQuery)
-            ftsQuery = "*";
+        // No real FTS term — either qualifier-only ("is:flagged after:1w") or
+        // every term sanitized away (user typed only punctuation). There is NO
+        // valid FTS5 "match everything" string — `MATCH '*'` is itself an FTS5
+        // error ("unknown special query"). So: with qualifiers, query the
+        // messages table directly (no FTS join); with neither, the query is
+        // garbage — return empty rather than dumping the whole mailbox.
+        const hasFts = ftsQuery.length > 0;
+        if (!hasFts && extraWhere.length === 0) {
+            return { items: [], total: 0, page, pageSize };
+        }
         const offset = (page - 1) * pageSize;
         try {
             let scopeWhere = "";
@@ -2761,6 +2788,13 @@ export class MailxDB {
                 scopeWhere += " AND " + extraWhere.join(" AND ");
                 scopeParams.push(...extraParams);
             }
+            // FTS path joins messages_fts + MATCH; qualifier-only path skips
+            // the FTS table entirely and filters `messages` directly.
+            const fromJoin = hasFts
+                ? "FROM messages m JOIN messages_fts fts ON m.id = fts.rowid LEFT JOIN folders f ON f.id = m.folder_id AND f.account_id = m.account_id"
+                : "FROM messages m LEFT JOIN folders f ON f.id = m.folder_id AND f.account_id = m.account_id";
+            const matchClause = hasFts ? "messages_fts MATCH ?" : "1=1";
+            const matchParams = hasFts ? [ftsQuery] : [];
             // Cap COUNT at 1001 — a bare `SELECT COUNT(*)` on FTS5 has to
             // enumerate every match, which for a common term ("ieee", "the")
             // can be tens of thousands of rows and several seconds. We never
@@ -2768,19 +2802,15 @@ export class MailxDB {
             // count is invisible to the user and turns search from
             // multi-second to sub-second. UI shows "1000+" when cnt === 1001.
             const countRow = this.db.prepare(`SELECT COUNT(*) as cnt FROM (
-                    SELECT 1 FROM messages m
-                    JOIN messages_fts fts ON m.id = fts.rowid
-                    LEFT JOIN folders f ON f.id = m.folder_id AND f.account_id = m.account_id
-                    WHERE messages_fts MATCH ?${scopeWhere}
+                    SELECT 1 ${fromJoin}
+                    WHERE ${matchClause}${scopeWhere}
                     LIMIT 1001
-                 )`).get(ftsQuery, ...scopeParams);
+                 )`).get(...matchParams, ...scopeParams);
             const total = countRow?.cnt || 0;
-            const rows = this.db.prepare(`SELECT m.*, f.name AS folder_name FROM messages m
-                 JOIN messages_fts fts ON m.id = fts.rowid
-                 LEFT JOIN folders f ON f.id = m.folder_id AND f.account_id = m.account_id
-                 WHERE messages_fts MATCH ?${scopeWhere}
+            const rows = this.db.prepare(`SELECT m.*, f.name AS folder_name ${fromJoin}
+                 WHERE ${matchClause}${scopeWhere}
                  ORDER BY m.date DESC
-                 LIMIT ? OFFSET ?`).all(ftsQuery, ...scopeParams, pageSize, offset);
+                 LIMIT ? OFFSET ?`).all(...matchParams, ...scopeParams, pageSize, offset);
             const items = rows.map(r => ({
                 id: r.id,
                 accountId: r.account_id,

package/index.d.ts

@@ -5,6 +5,7 @@
 export { MailxDB } from "./db.js";
 export { FileMessageStore } from "./file-store.js";
 export { parseSerial, prewarmParseWorker } from "./parse-serial.js";
+export { sniffAndFixCharset, fixCharsetDeclString } from "./charset.js";
 export { Store } from "./store.js";
 export type { StoreMessage } from "./store.js";
 export { StoreBus, storeBus } from "@bobfrankston/mailx-bus";

package/index.js

@@ -5,6 +5,7 @@
 export { MailxDB } from "./db.js";
 export { FileMessageStore } from "./file-store.js";
 export { parseSerial, prewarmParseWorker } from "./parse-serial.js";
+export { sniffAndFixCharset, fixCharsetDeclString } from "./charset.js";
 // Store — the nexus. Owns DB + .eml files + operations + bus.
 export { Store } from "./store.js";
 // Store-event bus lives in `@bobfrankston/mailx-bus` so the browser-side

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bobfrankston/mailx-store",
-  "version": "0.1.33",
+  "version": "0.1.35",
   "type": "module",
   "main": "index.js",
   "types": "index.d.ts",