@bobfrankston/rmfmail 1.0.545 → 1.0.548

package/packages/mailx-service/index.js

@@ -8,8 +8,11 @@ import * as fs from "node:fs";
 import * as path from "node:path";
 const __dirname = import.meta.dirname;
 import * as gsync from "./google-sync.js";
+import { LocalStore } from "./local-store.js";
+import { SyncQueue } from "./sync-queue.js";
+import { Reconciler } from "./reconciler.js";
 import { loadSettings, saveSettings, loadAccounts, loadAccountsAsync, saveAccounts, initCloudConfig, loadAllowlist, saveAllowlist, loadAutocomplete, saveAutocomplete, loadKeys, getStorageInfo, getConfigDir } from "@bobfrankston/mailx-settings";
-import { sanitizeHtml, encodeQuotedPrintable, htmlToPlainText } from "@bobfrankston/mailx-types";
+import { encodeQuotedPrintable, htmlToPlainText } from "@bobfrankston/mailx-types";
 import { simpleParser } from "mailparser";
 /** Detect mis-labeled charset and rewrite the part header to `utf-8` when
  *  the body bytes are actually valid UTF-8. PHPMailer-driven senders are
@@ -21,88 +24,10 @@ import { simpleParser } from "mailparser";
  *  Operates byte-wise so MIME boundaries / base64 / etc. are preserved.
  *  The `utf-8` validity test rejects 0xC0–0xC1 / 0xF5–0xFF and continuation
  *  bytes out of place, which would be common in genuine Latin-1 text. */
-function sniffAndFixCharset(raw) {
-    const HEAD_LIMIT = 16384;
-    const head = raw.subarray(0, Math.min(HEAD_LIMIT, raw.length)).toString("latin1");
-    const re = /charset\s*=\s*"?(iso-8859-1|us-ascii|windows-1252|latin1)"?/gi;
-    if (!re.test(head))
-        return raw;
-    if (!isValidUtf8(raw))
-        return raw;
-    const fixed = head.replace(/charset\s*=\s*"?(iso-8859-1|us-ascii|windows-1252|latin1)"?/gi, "charset=utf-8");
-    return Buffer.concat([Buffer.from(fixed, "latin1"), raw.subarray(head.length)]);
-}
-/** Strict UTF-8 validity check: rejects overlong forms, invalid start
- *  bytes, and dangling continuations. Used to confirm the body is really
- *  UTF-8 before overriding a Latin-1 declaration. */
-function isValidUtf8(buf) {
-    let i = 0;
-    while (i < buf.length) {
-        const b = buf[i];
-        if (b < 0x80) {
-            i++;
-            continue;
-        }
-        let need;
-        if ((b & 0xE0) === 0xC0) {
-            if (b < 0xC2)
-                return false;
-            need = 1;
-        }
-        else if ((b & 0xF0) === 0xE0)
-            need = 2;
-        else if ((b & 0xF8) === 0xF0) {
-            if (b > 0xF4)
-                return false;
-            need = 3;
-        }
-        else
-            return false;
-        if (i + need >= buf.length)
-            return false;
-        for (let k = 1; k <= need; k++) {
-            if ((buf[i + k] & 0xC0) !== 0x80)
-                return false;
-        }
-        i += need + 1;
-    }
-    return true;
-}
-/** Parse `List-Unsubscribe` (RFC 2369) and `List-Unsubscribe-Post` (RFC 8058).
- *  mailparser only exposes ONE of mail/url even when both are present, so we
- *  also scan the raw header text for the full set of angle-bracketed URIs. */
-function parseListUnsubscribe(headers) {
-    let mail = "";
-    let http = "";
-    let oneClick = false;
-    const raw = headers.get("list-unsubscribe");
-    const rawStr = typeof raw === "string" ? raw : (raw && typeof raw.text === "string" ? raw.text : "");
-    if (rawStr) {
-        const matches = rawStr.match(/<([^>]+)>/g) || [];
-        for (const m of matches) {
-            const url = m.slice(1, -1).trim();
-            if (!mail && /^mailto:/i.test(url))
-                mail = url;
-            else if (!http && /^https?:/i.test(url))
-                http = url;
-        }
-    }
-    if (!mail && !http) {
-        const listHeaders = headers.get("list");
-        if (listHeaders?.unsubscribe) {
-            const unsub = listHeaders.unsubscribe;
-            if (unsub.url)
-                http = Array.isArray(unsub.url) ? unsub.url[0] : unsub.url;
-            if (unsub.mail)
-                mail = `mailto:${Array.isArray(unsub.mail) ? unsub.mail[0] : unsub.mail}`;
-        }
-    }
-    const post = headers.get("list-unsubscribe-post");
-    const postStr = typeof post === "string" ? post : (post && typeof post.text === "string" ? post.text : "");
-    if (postStr && /one-?click/i.test(postStr))
-        oneClick = true;
-    return { listUnsubscribeMail: mail, listUnsubscribeHttp: http, listUnsubscribeOneClick: oneClick };
-}
+// sniffAndFixCharset moved to ./charset.ts so LocalStore can use it
+// without creating a circular dep with index.ts.
+// parseListUnsubscribe moved to ./local-store.ts (only consumer is the
+// body-read path, which is now part of LocalStore).
 // ── Email provider detection (MX-based) ──
 const GOOGLE_DOMAINS = ["gmail.com", "googlemail.com"];
 const MS_DOMAINS = ["outlook.com", "hotmail.com", "live.com"];
@@ -165,9 +90,23 @@ export class MailxService {
      *  bin/mailx.ts with mailx-host's showMessageBox. Without this,
      *  showReminderPopup returns a "no host" reason. */
     setPopupFn(fn) { this.popupFn = fn; }
+    /** Local-first read/write facade. Every UI IPC handler that touches the
+     *  local DB or body store goes through this — no awaiting IMAP, no
+     *  awaiting Gmail API, no awaiting SMTP. See docs/local-first-plan.md. */
+    localStore;
+    /** Persistent (and in-memory body-fetch) queue. UI handlers commit
+     *  locally, then enqueue a server-mirror task here. */
+    syncQueue;
+    /** Background loop: drains body-fetch tasks, retries failed message
+     *  actions, emits sync-state events for the status pill. */
+    reconciler;
     constructor(db, imapManager) {
         this.db = db;
         this.imapManager = imapManager;
+        this.localStore = new LocalStore(db, imapManager.getBodyStore());
+        this.syncQueue = new SyncQueue(db, imapManager);
+        this.reconciler = new Reconciler(db, imapManager, this.syncQueue);
+        this.reconciler.start();
         // Invalidate account cache when accounts.jsonc changes on disk or GDrive.
         this.imapManager.on?.("configChanged", (filename) => {
             if (filename === "accounts.jsonc")
@@ -431,198 +370,49 @@ export class MailxService {
     }
     // ── Messages ──
     getUnifiedInbox(page = 1, pageSize = 50) {
-        return this.db.getUnifiedInbox(page, pageSize);
+        return this.localStore.getUnifiedInbox(page, pageSize);
     }
     getMessages(accountId, folderId, page = 1, pageSize = 50, sort = "date", sortDir = "desc", search, flaggedOnly = false) {
-        return this.db.getMessages({ accountId, folderId, page, pageSize, sort: sort, sortDir: sortDir, search, flaggedOnly });
+        return this.localStore.getMessages({ accountId, folderId, page, pageSize, sort: sort, sortDir: sortDir, search, flaggedOnly });
     }
+    /** UI body read — local-first. Returns immediately from the local cache:
+     *  `cached: true` with the full parsed body when on disk, or `cached: false`
+     *  with envelope-only when not. In the cache-miss case we kick off a
+     *  fire-and-forget IMAP fetch in the background and emit `bodyAvailable`
+     *  when the body lands; the UI listens for that event and re-requests.
+     *
+     *  No `await imap*` in the click → render path. The 60s body-fetch race
+     *  and structured `bodyError` shape that lived here previously are gone —
+     *  step 1 of the local-first refactor (docs/local-first-plan.md). */
     async getMessage(accountId, uid, allowRemote = false, folderId) {
-        const envelope = this.db.getMessageByUid(accountId, uid, folderId);
-        if (!envelope)
+        const local = await this.localStore.getMessage(accountId, uid, allowRemote, folderId);
+        if (!local)
             throw new Error("Message not found");
-        let bodyHtml = "";
-        let bodyText = "";
-        let hasRemoteContent = false;
-        let attachments = [];
-        // The per-account ops queue inside ImapManager has its own per-task
-        // timeout that destroys a wedged client and unblocks the queue. This
-        // outer race is a safety net only — the underlying timeout in
-        // withConnection should trigger first.
-        const BODY_FETCH_TIMEOUT_MS = 60_000;
-        let raw = null;
-        try {
-            raw = await Promise.race([
-                this.imapManager.fetchMessageBody(accountId, envelope.folderId, envelope.uid),
-                new Promise((_, reject) => setTimeout(() => reject(new Error("body fetch timed out — try again")), BODY_FETCH_TIMEOUT_MS)),
-            ]);
-        }
-        catch (fetchErr) {
-            // Message was deleted from the server (another device, expunge, etc.) —
-            // drop the stale local row so the UI removes it instead of showing a
-            // confusing error. Throwing a tagged error lets the client react.
-            if (fetchErr?.isNotFound) {
-                try {
-                    this.db.deleteMessage(accountId, envelope.uid);
-                    this.db.recalcFolderCounts(envelope.folderId);
-                }
-                catch { /* ignore */ }
-                const err = new Error("Message was deleted from the server");
-                err.isNotFound = true;
-                throw err;
-            }
-            // Don't stuff the error text into bodyText — it bleeds into the
-            // viewer's main content area. Surface as a structured error field
-            // so the UI can render a banner with retry UX above the (empty)
-            // body. The caller keeps the envelope so the header still shows.
-            const rawErr = fetchErr.message || "connection failed";
-            const isTransient = /connection|Too many|UNAVAILABLE|rate|429|5\d\d|timeout|ENOTFOUND|ECONNRESET|ETIMEDOUT/i.test(rawErr);
-            return {
-                ...envelope, bodyHtml: "", bodyText: "",
-                bodyError: rawErr,
-                bodyErrorTransient: isTransient,
-                hasRemoteContent: false, remoteAllowed: false, attachments: [], emlPath: "", deliveredTo: "", returnPath: "", listUnsubscribe: ""
-            };
-        }
-        if (!raw) {
-            // Same treatment as the thrown-error case: structured field, not body text.
-            return {
-                ...envelope, bodyHtml: "", bodyText: "",
-                bodyError: "Message body not cached locally and the server fetch returned nothing.",
-                bodyErrorTransient: true,
-                hasRemoteContent: false, remoteAllowed: false, attachments: [], emlPath: "", deliveredTo: "", returnPath: "", listUnsubscribe: ""
-            };
-        }
-        else {
-            // Many senders (esp. PHPMailer-driven marketing) declare
-            // `charset=iso-8859-1` but emit UTF-8 bytes. simpleParser
-            // honors the declared charset and produces "â??" garbage for
-            // every non-ASCII codepoint (em-dash, smart quotes, …). If
-            // the raw body bytes are valid UTF-8, rewrite the charset
-            // header before parsing. We only override the obviously-
-            // wrong legacy declarations; explicit utf-8 / koi8 / etc.
-            // pass through.
-            const adjusted = sniffAndFixCharset(raw);
-            const parsed = await simpleParser(adjusted);
-            bodyHtml = parsed.html || "";
-            bodyText = parsed.text || "";
-            attachments = (parsed.attachments || []).map((a, i) => ({
-                id: i,
-                filename: a.filename || `attachment-${i}`,
-                mimeType: a.contentType || "application/octet-stream",
-                size: a.size || 0,
-                contentId: a.contentId || ""
-            }));
-        }
-        // Sanitize HTML
-        if (bodyHtml && !allowRemote) {
-            const allowList = loadAllowlist();
-            const senderAddr = envelope.from?.address || "";
-            const senderDomain = senderAddr.split("@")[1] || "";
-            const toAddrs = (envelope.to || []).map((a) => a.address);
-            const isAllowed = allowList.senders.includes(senderAddr) ||
-                allowList.domains.includes(senderDomain) ||
-                toAddrs.some((a) => allowList.recipients?.includes(a));
-            if (isAllowed) {
-                allowRemote = true;
-            }
-            else {
-                const result = sanitizeHtml(bodyHtml);
-                bodyHtml = result.html;
-                hasRemoteContent = result.hasRemoteContent;
-            }
-        }
-        // Extract headers
-        let deliveredTo = "";
-        let returnPath = "";
-        let listUnsubscribe = "";
-        let listUnsubscribeMail = "";
-        let listUnsubscribeHttp = "";
-        let listUnsubscribeOneClick = false;
-        if (raw) {
-            const parsed2 = await simpleParser(raw);
-            const hdr = (key) => {
-                let v = parsed2.headers.get(key);
-                if (!v)
-                    return "";
-                if (Array.isArray(v))
-                    v = v[0];
-                if (typeof v === "string")
-                    return v;
-                if (typeof v === "object" && v !== null) {
-                    if ("text" in v)
-                        return v.text || "";
-                    if ("value" in v)
-                        return String(v.value);
-                    if ("address" in v)
-                        return v.address || "";
-                }
-                return String(v);
-            };
-            const msgSettings = loadSettings();
-            const acctConfig = msgSettings.accounts.find((a) => a.id === accountId);
-            const relayDomains = acctConfig?.relayDomains || [];
-            const prefixes = acctConfig?.deliveredToPrefix || [];
-            const rawDelivered = parsed2.headers.get("delivered-to");
-            if (rawDelivered) {
-                const deliveredList = Array.isArray(rawDelivered) ? rawDelivered : [rawDelivered];
-                for (let i = deliveredList.length - 1; i >= 0; i--) {
-                    const d = deliveredList[i];
-                    const addr = typeof d === "string" ? d : d?.text || d?.address || String(d);
-                    if (!relayDomains.some(rd => addr.includes(`@${rd}`))) {
-                        deliveredTo = addr;
-                        break;
-                    }
-                }
-                if (!deliveredTo && deliveredList.length > 0) {
-                    const d = deliveredList[deliveredList.length - 1];
-                    deliveredTo = typeof d === "string" ? d : d?.text || d?.address || String(d);
-                }
-                if (deliveredTo && prefixes.length > 0) {
-                    const [local, domain] = deliveredTo.split("@");
-                    for (const prefix of prefixes) {
-                        if (local.startsWith(prefix)) {
-                            deliveredTo = `${local.slice(prefix.length)}@${domain}`;
-                            break;
-                        }
-                    }
-                }
-            }
-            returnPath = hdr("return-path").replace(/[<>]/g, "");
-            ({ listUnsubscribeMail, listUnsubscribeHttp, listUnsubscribeOneClick } =
-                parseListUnsubscribe(parsed2.headers));
-            listUnsubscribe = listUnsubscribeHttp || listUnsubscribeMail;
-        }
-        // EML path: re-read the row after the fetch — `fetchMessageBody`
-        // writes the body to disk and updates `body_path` on success, but the
-        // `envelope` snapshot above pre-dates that write, so trusting it
-        // hides the Source button on every just-opened message.
-        const refreshed = this.db.getMessageByUid(accountId, uid, folderId);
-        const emlPath = refreshed?.bodyPath || envelope.bodyPath || "";
-        // Flag check — surfaced in the remote-content banner as a red
-        // warning when the sender's address or domain is on the user's
-        // flagged list. Cheap lookup; loadAllowlist is already cached.
-        const allowList = loadAllowlist();
-        const senderAddr = (envelope.from?.address || "").toLowerCase();
-        const senderDomain = senderAddr.split("@")[1] || "";
-        const isFlagged = !!((allowList.flaggedSenders || []).some((s) => (s || "").toLowerCase() === senderAddr) ||
-            (allowList.flaggedDomains || []).some((d) => (d || "").toLowerCase() === senderDomain));
-        // External reputation check — Spamhaus DBL + SURBL + URIBL in parallel.
-        // Off by default (privacy: the domain leaks to those DNSBLs and the
-        // user's local resolver). User opts in via Settings → "Check sender
-        // reputation". Each lookup is bounded at 500 ms; the whole check is
-        // bounded by the slowest, ~500 ms worst case.
+        if (!local.cached) {
+            // Body not on disk. Enqueue an interactive-lane fetch — the
+            // reconciler picks it up, emits `bodyAvailable` on success or
+            // `bodyFetchError` on failure. Return envelope-only; UI shows
+            // a placeholder and re-renders on the event.
+            this.syncQueue.enqueueBodyFetch(accountId, local.folderId, local.uid, "interactive");
+            this.reconciler.kick();
+            return local;
+        }
+        // Optional sender-reputation check. This is the one residual server
+        // touch in the read path (DNSBL lookup, ~500 ms). Opt-in via Settings;
+        // moves to the reconciler in step 4. See feedback_no_bandaids.md.
         let reputation = null;
         const settings = loadSettings() || {};
-        if (settings.checkDomainReputation && senderDomain && hasRemoteContent) {
+        const senderDomain = (local.from?.address || "").toLowerCase().split("@")[1] || "";
+        if (settings.checkDomainReputation && senderDomain && local.hasRemoteContent) {
             reputation = await this.checkDomainReputation(senderDomain);
         }
-        return {
-            ...envelope, bodyHtml, bodyText, hasRemoteContent, remoteAllowed: allowRemote,
-            attachments, emlPath, deliveredTo, returnPath,
-            listUnsubscribe, listUnsubscribeMail, listUnsubscribeHttp, listUnsubscribeOneClick,
-            isFlagged,
-            reputation,
-        };
+        return { ...local, reputation };
+    }
+    /** Diagnostic accessor — exposed for the sync-status pill / debug UI.
+     *  Returns the current queue counts so the UI can render
+     *  "Sync OK / Syncing N items" without polling per-account state. */
+    getSyncStatus() {
+        return this.syncQueue.pendingCount();
     }
     /** RFC 8058 one-click unsubscribe: POST `List-Unsubscribe=One-Click` to the
      *  HTTPS URL the message's List-Unsubscribe header advertised. Done server-
@@ -1135,10 +925,10 @@ export class MailxService {
             return { items: sliced, total, page, pageSize };
         }
         else if (scope === "current" && accountId && folderId) {
-            return this.db.searchMessages(q, page, pageSize, accountId, folderId);
+            return this.localStore.searchMessages(q, page, pageSize, accountId, folderId);
         }
         else {
-            return this.db.searchMessages(q, page, pageSize);
+            return this.localStore.searchMessages(q, page, pageSize);
         }
     }
     rebuildSearchIndex() {
@@ -1294,7 +1084,7 @@ export class MailxService {
                     .catch(e => this.handleGoogleRefreshError("calendar", e));
             }
         }
-        return this.db.getCalendarEvents(acctId, fromMs, toMs);
+        return this.localStore.getCalendarEvents(acctId, fromMs, toMs);
     }
     /** Returns true if the feature is currently in a quota-exceeded cooldown. */
     inQuotaCooldown(feature) {
@@ -1449,7 +1239,7 @@ export class MailxService {
                     .catch(e => this.handleGoogleRefreshError("tasks", e));
             }
         }
-        return this.db.getTasks(acctId, includeCompleted);
+        return this.localStore.getTasks(acctId, includeCompleted);
     }
     async refreshTasks(accountId, includeCompleted) {
         const tp = await this.primaryTokenProvider("tasks");
@@ -2146,7 +1936,24 @@ export class MailxService {
         const envelope = this.db.getMessageByUid(accountId, uid, folderId);
         if (!envelope)
             throw new Error("Message not found");
-        const raw = await this.imapManager.fetchMessageBody(accountId, envelope.folderId, envelope.uid);
+        // Prefer the on-disk body — when the user just opened the message,
+        // LocalStore.getMessage either returned cached:true (body already on
+        // disk) or kicked a background fetch that wrote it. Either way a
+        // re-fetch here would be wasted IMAP work and risks racing.
+        const bodyStore = this.imapManager.getBodyStore();
+        let raw = null;
+        const storedPath = envelope.bodyPath || this.db.getMessageBodyPath(accountId, uid) || "";
+        if (storedPath && await bodyStore.hasByPath(storedPath)) {
+            try {
+                raw = await bodyStore.readByPath(storedPath);
+            }
+            catch {
+                raw = null;
+            }
+        }
+        if (!raw) {
+            raw = await this.imapManager.fetchMessageBody(accountId, envelope.folderId, envelope.uid);
+        }
         if (!raw)
             throw new Error("Message body not available");
         const parsed = await simpleParser(raw);
@@ -2210,14 +2017,10 @@ export class MailxService {
             }
         }
         catch { /* non-fatal — draft stays in memory at least */ }
-        // Background reconcile to server Drafts folder. Fire-and-forget —
-        // the ACK to the client is already on its way.
-        this.imapManager.saveDraft(accountId, raw, previousDraftUid, id).catch((e) => {
-            console.error(`  [draft] background IMAP save failed for ${id}: ${e?.message || e}`);
-            // Surface as an event so the UI can show a status-bar hint without
-            // blocking the caller. Draft is preserved on disk regardless.
-            this.emit?.("draftSaveDeferred", { accountId, draftId: id, error: String(e?.message || e) });
-        });
+        // Background reconcile to server Drafts folder via the SyncQueue
+        // seam. Fire-and-forget; deferred failures surface as a `draftSaveDeferred`
+        // event the UI can render in the status bar.
+        this.syncQueue.enqueueDraftPush(accountId, raw, previousDraftUid, id);
         return { draftUid: null, draftId: id };
     }
     async deleteDraft(accountId, draftUid, draftId) {
@@ -2228,7 +2031,7 @@ export class MailxService {
         query = (query || "").trim();
         if (query.length < 1)
             return [];
-        return this.db.searchContacts(query);
+        return this.localStore.searchContacts(query);
     }
     /** Q49: boolean hint for compose to auto-expand Cc when replying to this
      *  address. True when at least one past sent message to the same recipient
@@ -2260,7 +2063,7 @@ export class MailxService {
     }
     /** Address-book listing — paginated, filterable. */
     listContacts(query, page = 1, pageSize = 100) {
-        return this.db.listContacts(query || "", page, pageSize);
+        return this.localStore.listContacts(query || "", page, pageSize);
     }
     /** Upsert a contact from the address book UI (edit name). Two-way cache:
      *  commits locally, queues a Google People push. */