@bobfrankston/mailx-store 0.1.31 → 0.1.33

package/charset.js

@@ -17,11 +17,31 @@ export function sniffAndFixCharset(raw) {
     const re = /charset\s*=\s*"?(iso-8859-1|us-ascii|windows-1252|latin1)"?/gi;
     if (!re.test(head))
         return raw;
+    // The rewrite is only sound when the raw bytes ARE the body bytes — i.e.
+    // an 8bit / binary part. For a quoted-printable or base64 part the raw
+    // .eml is pure ASCII (the high bytes live inside `=XX` / base64 chars),
+    // so isValidUtf8(raw) passes vacuously and we would relabel a genuine
+    // Windows-1252 part as utf-8 — every smart-quote / em-dash then decodes
+    // as mojibake (Bob's 2026-05-21 report: a QP Windows-1252 Outlook mail).
+    // Requiring an actual non-ASCII byte in the raw gates the heuristic to
+    // the only case where the UTF-8 sniff is meaningful.
+    if (!hasNonAscii(raw))
+        return raw;
     if (!isValidUtf8(raw))
         return raw;
     const fixed = head.replace(/charset\s*=\s*"?(iso-8859-1|us-ascii|windows-1252|latin1)"?/gi, "charset=utf-8");
     return Buffer.concat([Buffer.from(fixed, "latin1"), raw.subarray(head.length)]);
 }
+/** True if the buffer contains at least one byte >= 0x80. A pure-ASCII
+ *  buffer is trivially valid UTF-8, so the isValidUtf8 sniff tells us
+ *  nothing about a quoted-printable / base64 body — gate on this first. */
+function hasNonAscii(buf) {
+    for (let i = 0; i < buf.length; i++) {
+        if (buf[i] >= 0x80)
+            return true;
+    }
+    return false;
+}
 /** Strict UTF-8 validity check: rejects overlong forms, invalid start
  *  bytes, and dangling continuations. Used to confirm the body is really
  *  UTF-8 before overriding a Latin-1 declaration. */

package/db.d.ts

@@ -520,6 +520,7 @@ export declare class MailxDB {
         name: string;
         email: string;
         source: string;
+        sources: string[];
         useCount: number;
     }[];
     /** List all contacts (address-book view) with pagination + optional filter. */

package/db.js

@@ -2451,6 +2451,13 @@ export class MailxDB {
         const tokens = query.split(/\s+/).filter(Boolean);
         const firstSubstr = `%${tokens[0]}%`;
         const firstPrefix = `${tokens[0]}%`;
+        // Word-prefix patterns: the token at the start of ANY name word, not
+        // just the whole name. Catches "Frankston, Bob" / "Frankston,Bob"
+        // when the user types "bob" — a first-name match on a Last,First
+        // contact (Bob 2026-05-21). The space variant also covers the normal
+        // "Bob Frankston" middle/last-word case.
+        const firstWordSpace = `% ${tokens[0]}%`;
+        const firstWordComma = `%,${tokens[0]}%`;
         const tokenWhere = tokens.map(() => "(name LIKE ? OR email LIKE ?)").join(" AND ");
         const tokenParams = [];
         for (const t of tokens) {
@@ -2463,11 +2470,19 @@ export class MailxDB {
             // contacts.jsonc#preferred[]. The user's `source: "work"` /
             // `source: "family"` tags all rank +40 alongside the default
             // `preferred` label.
+            // Ranking: match POSITION dominates, source is only a tiebreaker
+            // within the same position tier. Position tiers are spaced 100
+            // apart and source tops out at 40, so a prefix match (300+) can
+            // never be outranked by a mid-string match (≤140) — Bob 2026-05-21:
+            // "the match should favor starting letters rather than random
+            // letters in the middle." The old 3/2/1 + 0/30/40 weighting let a
+            // google contact matched mid-string (1+30=31) beat a discovered
+            // contact matched on prefix (3+0=3).
             rows = this.db.prepare(`SELECT name, email, source, use_count, last_used,
                         (CASE
-                            WHEN lower(name) LIKE lower(?) THEN 3
-                            WHEN substr(email, 1, instr(email, '@') - 1) LIKE lower(?) THEN 2
-                            WHEN email LIKE ? OR name LIKE ? THEN 1
+                            WHEN lower(name) LIKE lower(?) OR lower(name) LIKE lower(?) OR lower(name) LIKE lower(?) THEN 300
+                            WHEN substr(email, 1, instr(email, '@') - 1) LIKE lower(?) THEN 200
+                            WHEN email LIKE ? OR name LIKE ? THEN 100
                             ELSE 0
                          END) +
                         (CASE
@@ -2478,7 +2493,7 @@ export class MailxDB {
                  FROM contacts
                  WHERE ${tokenWhere}
                  ORDER BY match_rank DESC, use_count DESC, last_used DESC
-                 LIMIT ?`).all(firstPrefix, firstPrefix, firstSubstr, firstSubstr, ...tokenParams, limit * 2);
+                 LIMIT ?`).all(firstPrefix, firstWordSpace, firstWordComma, firstPrefix, firstSubstr, firstSubstr, ...tokenParams, limit * 2);
         }
         catch (e) {
             console.error(`  [searchContacts] ranked query failed (${e?.message}) — falling back to simple LIKE`);
@@ -2499,23 +2514,38 @@ export class MailxDB {
         rows.sort((a, b) => score(b) - score(a));
         // Dedup by lowercased email — same address often appears as both
         // source='google' (synced from Google Contacts) and source='discovered'
-        // (auto-collected from sent mail). The user only wants to see the
-        // best entry; we keep the higher-ranked source (which the sort above
-        // has already put first), and silently drop the duplicate. Without
-        // this, the autocomplete dropdown showed two identical Kevin Healy
-        // rows just labeled GOOGLE and DISCOVERED.
-        const seenEmails = new Set();
-        rows = rows.filter(r => {
+        // (auto-collected from sent mail). Keep the higher-ranked row (already
+        // sorted first) and FOLD subsequent rows' sources into a `sources` array
+        // on the survivor so the dropdown can display "google, discovered"
+        // instead of showing two duplicate lines.
+        const merged = new Map();
+        for (const r of rows) {
             const k = (r.email || "").toLowerCase();
-            if (!k)
-                return true;
-            if (seenEmails.has(k))
-                return false;
-            seenEmails.add(k);
-            return true;
-        });
-        rows = rows.slice(0, limit);
-        return rows.map(r => ({ name: r.name, email: r.email, source: r.source, useCount: r.use_count }));
+            if (!k) {
+                merged.set(`__noemail_${merged.size}`, { ...r, _sources: new Set([r.source]) });
+                continue;
+            }
+            const existing = merged.get(k);
+            if (!existing) {
+                merged.set(k, { ...r, _sources: new Set([r.source]) });
+            }
+            else {
+                existing._sources.add(r.source);
+                // If this row has a non-empty name and the kept one didn't,
+                // promote the name — Google rows often have proper names while
+                // discovered rows are bare-email.
+                if (!existing.name && r.name)
+                    existing.name = r.name;
+            }
+        }
+        const out = Array.from(merged.values()).slice(0, limit);
+        return out.map(r => ({
+            name: r.name,
+            email: r.email,
+            source: r.source,
+            sources: Array.from(r._sources).filter(Boolean),
+            useCount: r.use_count,
+        }));
     }
     /** List all contacts (address-book view) with pagination + optional filter. */
     listContacts(query, page = 1, pageSize = 100) {
@@ -2605,6 +2635,11 @@ export class MailxDB {
             const ts = Date.parse(s);
             return isNaN(ts) ? null : ts;
         };
+        // Strip FTS5 metacharacters from a user term. A stray `;` `(` `:` `"`
+        // etc. is a hard `fts5: syntax error` (Bob 2026-05-21 log). FTS5's
+        // tokenizer splits on non-word chars anyway, so removing them loses no
+        // real matching ability.
+        const ftsClean = (t) => t.replace(/["';:(){}\[\]^~\\/]/g, "").trim();
         for (const part of parts) {
             const fromMatch = part.match(/^from:(.+)$/i);
             const toMatch = part.match(/^to:(.+)$/i);
@@ -2616,16 +2651,19 @@ export class MailxDB {
             const isMatch = part.match(/^is:(.+)$/i);
             const folderMatch = part.match(/^folder:(.+)$/i);
             if (fromMatch) {
-                const term = fromMatch[1].replace(/"/g, "");
-                ftsQuery += `(from_name:${term} OR from_address:${term}) `;
+                const term = ftsClean(fromMatch[1]);
+                if (term)
+                    ftsQuery += `(from_name:${term}* OR from_address:${term}*) `;
             }
             else if (toMatch) {
-                const term = toMatch[1].replace(/"/g, "");
-                ftsQuery += `(to_text:${term} OR cc_text:${term}) `;
+                const term = ftsClean(toMatch[1]);
+                if (term)
+                    ftsQuery += `(to_text:${term}* OR cc_text:${term}*) `;
             }
             else if (subjectMatch) {
-                const term = subjectMatch[1].replace(/"/g, "");
-                ftsQuery += `subject:${term} `;
+                const term = ftsClean(subjectMatch[1]);
+                if (term)
+                    ftsQuery += `subject:${term}* `;
             }
             else if (dateMatch || afterMatch || beforeMatch) {
                 const op = dateMatch ? (dateMatch[1] || "=") : (afterMatch ? ">" : "<");
@@ -2670,15 +2708,27 @@ export class MailxDB {
                 extraWhere.push("LOWER(f.name) LIKE ?");
                 extraParams.push(`%${v.toLowerCase()}%`);
             }
+            else if (/^(AND|OR|NOT)$/.test(part)) {
+                // FTS5 boolean operators — pass through verbatim (must be uppercase).
+                // Without this branch, `hoddie AND git` got wildcarded into
+                // `hoddie* AND* git*`, which FTS5 reads as three required terms
+                // (one of them being any word starting with "AND") — so a real
+                // match like "Peter Hoddie" + "github" returned zero hits.
+                ftsQuery += `${part} `;
+            }
             else {
-                // Unqualified — search everything.
+                // Unqualified — search everything. Strip /regex-literal/
+                // slashes and FTS5 metacharacters before wildcarding.
                 let term = part.replace(/^\/|\/$/g, "");
                 if (term.includes("|")) {
-                    const alts = term.split("|").filter(Boolean).map(t => `${t}*`).join(" OR ");
-                    ftsQuery += `(${alts}) `;
+                    const alts = term.split("|").map(t => ftsClean(t)).filter(Boolean).map(t => `${t}*`).join(" OR ");
+                    if (alts)
+                        ftsQuery += `(${alts}) `;
                 }
                 else {
-                    ftsQuery += `${term}* `;
+                    term = ftsClean(term);
+                    if (term)
+                        ftsQuery += `${term}* `;
                 }
             }
         }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bobfrankston/mailx-store",
-  "version": "0.1.31",
+  "version": "0.1.33",
   "type": "module",
   "main": "index.js",
   "types": "index.d.ts",
@@ -10,7 +10,7 @@
   "license": "ISC",
   "dependencies": {
     "@bobfrankston/mailx-types": "^0.1.18",
-    "@bobfrankston/mailx-settings": "^0.1.21",
+    "@bobfrankston/mailx-settings": "^0.1.22",
     "@bobfrankston/mailx-bus": "^0.1.2",
     "mailparser": "^3.7.2"
   },
@@ -30,7 +30,7 @@
   ".transformedSnapshot": {
     "dependencies": {
       "@bobfrankston/mailx-types": "^0.1.18",
-      "@bobfrankston/mailx-settings": "^0.1.21",
+      "@bobfrankston/mailx-settings": "^0.1.22",
       "@bobfrankston/mailx-bus": "^0.1.2",
       "mailparser": "^3.7.2"
     }

package/store.js

@@ -300,6 +300,29 @@ export class Store {
         }
         let bodyHtml = parsed.html || "";
         const bodyText = parsed.text || "";
+        // Charset diagnostic. If the parsed body has U+FFFD, distinguish the
+        // two very different causes:
+        //  - SOURCE already contains U+FFFD (raw EF BF BD bytes, or QP
+        //    =EF=BF=BD) → the message was composed/replied during the pre-fix
+        //    charset window; the original bytes are gone — UNRECOVERABLE, and
+        //    NOT a current bug.
+        //  - source is clean but the parse produced U+FFFD → a live
+        //    charset-decode bug worth chasing.
+        if (bodyHtml.includes("�") || bodyText.includes("�")) {
+            let sourceCorrupt = false;
+            try {
+                const src = await this.bodyStore.readByPath(storedPath);
+                sourceCorrupt = src.includes(Buffer.from([0xEF, 0xBF, 0xBD]))
+                    || src.toString("latin1").toUpperCase().includes("=EF=BF=BD");
+            }
+            catch { /* */ }
+            if (sourceCorrupt) {
+                console.error(`  [charset] U+FFFD baked into SOURCE — acct=${accountId} uid=${uid} path=${storedPath} — corrupt at compose time (pre-fix), unrecoverable`);
+            }
+            else {
+                console.error(`  [charset] U+FFFD from PARSE — acct=${accountId} uid=${uid} path=${storedPath} — LIVE charset-decode bug, source is clean`);
+            }
+        }
         // Backfill FTS body_text now that we've parsed the body. upsertMessage
         // only had the short `preview` snippet at index time; without this
         // backfill, searches miss any word that only appears deeper in the
@@ -345,17 +368,17 @@ export class Store {
             hasRemoteContent = result.hasRemoteContent;
         }
         // Header extraction — Delivered-To, Return-Path, List-Unsubscribe.
-        // mlproc preprocesses Delivered-To server-side, so the inner
-        // address is already clean by the time mailx sees it. We take the
-        // last entry of the chain (the final-delivery hop). The
-        // `relayDomains` filtering layer was retired 2026-05-13: nothing
-        // configures it in practice and the conditional made the code
-        // harder to reason about than the one-liner that replaces it.
+        // Each delivery agent PREPENDS its Delivered-To, so the FIRST
+        // (topmost) header is the final delivery to the user's actual
+        // mailbox; later ones are earlier forwarding hops / internal
+        // routing artifacts. Take [0]. (The old code took the LAST entry —
+        // on a forwarded message that grabbed a stale hop, e.g. a malformed
+        // `…@elkin.ws@trap-prot` address — Bob 2026-05-21.)
         let deliveredTo = "";
         const rawDelivered = parsed.headers.get("delivered-to");
         if (rawDelivered) {
             const deliveredList = Array.isArray(rawDelivered) ? rawDelivered : [rawDelivered];
-            const d = deliveredList[deliveredList.length - 1];
+            const d = deliveredList[0];
             deliveredTo = typeof d === "string" ? d : d?.text || d?.address || String(d);
         }
         const hdr = (key) => {