@bobfrankston/mailx-store 0.1.24 → 0.1.26

package/parse-serial.js

@@ -1,40 +1,111 @@
 /**
- * Process-wide serialized simpleParser.
+ * Worker-thread-backed simpleParser dispatcher.
  *
  * mailparser's `simpleParser` is declared `async` but its work is CPU-bound
- * (Node Streams + libmime decoding). When N parses run concurrently on the
- * single-threaded event loop they share CPU and each finishes at roughly
- * N× wall-clock. Real evidence (2026-05-13): four near-concurrent parses
- * each reported `14691ms for 3 KB` — pure contention, not size.
+ * (Node Streams + libmime decoding). When it runs on the main event loop:
+ *   - The first parse after a fresh process takes 14-25 seconds (cold V8
+ *     JIT + libmime + iconv-lite + charset-table loading). See log
+ *     2026-05-14 01:11:44 — simpleParser 14524ms for 5 KB.
+ *   - Every parse, cold or warm, blocks the IPC stdin pump for its
+ *     duration. A 200 ms parse delays every queued IPC message by 200 ms.
  *
- * The downstream symptom is the IPC pipe: while the event loop is
- * saturated by interleaving parses, unrelated IPC operations (mark-as-spam,
- * move, delete) wait their turn and the WebView-side `mailxapi` shim
- * times out at 120s with a misleading "stayed in the list" alert.
+ * Moving the parse to a `worker_threads` Worker fixes both problems:
+ *   - The main event loop stays responsive — IPC, sync, timers all run
+ *     during the parse. Click-to-render latency is bounded by post-message
+ *     RTT (sub-ms) + parse time on the worker, but the rest of the app
+ *     stays alive.
+ *   - The worker absorbs cold-start once. Subsequent parses ride the
+ *     worker's warm JIT and run in their natural 50-500 ms budget.
  *
- * Serializing through a module-level promise chain bounds the damage with
- * minimal plumbing: a single parse runs at full CPU and finishes in its
- * natural ~50-500ms budget; the next parse starts when it's done. Each
- * UI click produces a clean preview latency instead of a 14-second stall.
+ * Priority queue: foreground (UI clicks) jumps over background (sync /
+ * prefetch) when more than one parse is pending. Each parse still runs
+ * sequentially on the worker — the single-worker design intentionally
+ * mirrors the prior in-process serialization to keep CPU bounded.
  *
- * Lives in mailx-store rather than mailx-service so both the UI-hot path
- * (mailx-service/local-store.ts) and the sync path (mailx-imap) can share
- * one queue — otherwise sync-time parses would still contend with UI
- * parses through the event loop, just not through this module's chain.
+ * Lazy spawn: the worker is created on first call. boot/setup code that
+ * runs no parses pays nothing.
+ *
+ * Worker lifecycle: the worker is process-singleton, never explicitly
+ * terminated. Node exits cleanly because the worker is `unref()`d so it
+ * doesn't keep the event loop alive.
  *
- * Future work: replace the in-process chain with a `node:worker_threads`
- * pool. The chain is the precursor that bounds the worst case while the
- * pool is built.
+ * Lives in mailx-store rather than mailx-service so both the UI-hot path
+ * (mailx-service/local-store.ts) and the sync path (mailx-imap) share one
+ * worker — otherwise each module would spawn its own (parallel workers
+ * would defeat the bound on CPU, and each pays its own cold start).
  */
-import { simpleParser } from "mailparser";
-const _head = []; // UI / foreground — drained first
-const _tail = []; // sync / background
+import { Worker } from "node:worker_threads";
+import { fileURLToPath } from "node:url";
+import { dirname, join } from "node:path";
+const _head = [];
+const _tail = [];
 let _pumping = false;
-async function pump() {
+let _worker = null;
+let _nextId = 1;
+const _inflight = new Map();
+function getWorker() {
+    if (_worker)
+        return _worker;
+    // Resolve parse-worker.js alongside the compiled parse-serial.js.
+    // import.meta.url is the running .js URL after tsc compilation, so
+    // the worker resolves cleanly from the package install location.
+    const here = dirname(fileURLToPath(import.meta.url));
+    const workerPath = join(here, "parse-worker.js");
+    const w = new Worker(workerPath);
+    w.on("message", (msg) => {
+        // Warmup heartbeat (no `id`). Logs the worker cold-start cost so
+        // we can confirm the absorber is working on each boot.
+        if (typeof msg.warmupMs === "number") {
+            console.log(`  [parse-worker] cold-start absorbed in worker in ${msg.warmupMs}ms`);
+            return;
+        }
+        if (typeof msg.id !== "number")
+            return;
+        const entry = _inflight.get(msg.id);
+        if (!entry)
+            return;
+        _inflight.delete(msg.id);
+        if (msg.ok)
+            entry.resolve(msg.result);
+        else
+            entry.reject(new Error(msg.error || "parse-worker error"));
+        signalCompletion();
+    });
+    w.on("error", (e) => {
+        // Fatal worker crash — fail every in-flight parse, then null the
+        // singleton so the next call respawns.
+        for (const entry of _inflight.values())
+            entry.reject(e);
+        _inflight.clear();
+        _worker = null;
+        signalCompletion();
+    });
+    w.on("exit", (code) => {
+        if (_inflight.size > 0) {
+            const err = new Error(`parse-worker exited (code=${code}) with ${_inflight.size} in-flight`);
+            for (const entry of _inflight.values())
+                entry.reject(err);
+            _inflight.clear();
+        }
+        _worker = null;
+        signalCompletion();
+    });
+    // Don't let the worker prevent process exit.
+    w.unref();
+    _worker = w;
+    return w;
+}
+/** In-main-thread fallback. Used when the worker spawn itself fails (e.g.,
+ *  a packaged build where parse-worker.js can't be resolved). Falls back
+ *  to the original behavior — blocks the event loop, but at least works.
+ *  Lazy-imports mailparser so the main thread doesn't pay the (heavy)
+ *  module-init cost unless the fallback actually fires. */
+async function pumpInMain() {
     if (_pumping)
         return;
     _pumping = true;
     try {
+        const { simpleParser } = await import("mailparser");
         for (;;) {
             const next = _head.shift() ?? _tail.shift();
             if (!next)
@@ -52,17 +123,78 @@ async function pump() {
         _pumping = false;
     }
 }
-/** Serialized wrapper around `mailparser.simpleParser`. `priority` defaults
- *  to "foreground" — only sync/background callers should pass "background"
- *  so user clicks aren't stuck behind a back-fill. */
+// Notifier the pump awaits to drive the next iteration. When a reply
+// arrives from the worker, completing the in-flight parse, this fires
+// and unblocks the pump so it can pop the next queued parse. Avoids the
+// busy-wait setTimeout-poll version.
+let _completionSignal = null;
+function signalCompletion() {
+    const fn = _completionSignal;
+    _completionSignal = null;
+    if (fn)
+        fn();
+}
+async function pumpViaWorker() {
+    if (_pumping)
+        return;
+    _pumping = true;
+    try {
+        for (;;) {
+            const next = _head.shift() ?? _tail.shift();
+            if (!next)
+                break;
+            _inflight.set(next.id, next);
+            try {
+                getWorker().postMessage({ id: next.id, source: next.source });
+            }
+            catch (e) {
+                _inflight.delete(next.id);
+                next.reject(e);
+                continue;
+            }
+            // Wait for THIS parse to complete before sending the next.
+            // Preserves the priority order — a foreground entry that arrives
+            // after we've already postMessage'd a background parse will be
+            // next in line, NOT after several already-queued backgrounds.
+            await new Promise(resolve => { _completionSignal = resolve; });
+        }
+    }
+    finally {
+        _pumping = false;
+    }
+}
+/** Spawn the parse worker early so its cold-start (mailparser module
+ *  loading, V8 JIT, libmime / iconv-lite tables — empirically 14-25 s)
+ *  runs in parallel with the rest of boot. Safe to call multiple times;
+ *  the worker is a process-singleton. Returns immediately; the actual
+ *  cold-start completes in the worker's internal self-warm. */
+export function prewarmParseWorker() {
+    try {
+        getWorker();
+    }
+    catch { /* fallback path handles it on first parseSerial */ }
+}
+/** Serialized wrapper around `mailparser.simpleParser` running in a worker
+ *  thread. `priority` defaults to "foreground" — only sync/background
+ *  callers should pass "background" so user clicks aren't stuck behind a
+ *  back-fill. If the worker can't be spawned (rare), falls back to an
+ *  in-main-thread serial pump so the system still functions. */
 export async function parseSerial(source, priority = "foreground") {
     return new Promise((resolve, reject) => {
-        const entry = { source, resolve, reject };
+        const entry = { id: _nextId++, source, resolve, reject };
         if (priority === "foreground")
             _head.push(entry);
         else
             _tail.push(entry);
-        pump();
+        // Try the worker first; fall back transparently if the spawn fails.
+        try {
+            getWorker();
+            pumpViaWorker();
+        }
+        catch (e) {
+            console.error(`  [parse-serial] worker unavailable, falling back to main-thread parse: ${e instanceof Error ? e.message : String(e)}`);
+            pumpInMain();
+        }
     });
 }
 //# sourceMappingURL=parse-serial.js.map

package/parse-worker.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=parse-worker.d.ts.map

package/parse-worker.js

@@ -0,0 +1,69 @@
+/**
+ * mailparser worker thread.
+ *
+ * Runs simpleParser off the main event loop so a 14-25 second cold-start
+ * (or any slow parse) can't block IPC. The main thread's `parse-serial.ts`
+ * spawns this worker once at first use, then dispatches every parse here
+ * via postMessage and awaits the reply.
+ *
+ * Protocol:
+ *   main → worker: { id: number, source: Buffer | string }
+ *   worker → main: { id: number, ok: true,  result: ParsedMail }
+ *                | { id: number, ok: false, error: string }
+ *
+ * The worker holds its own mailparser module instance — JIT cost and
+ * libmime/iconv-lite loading happen exactly once per worker process. The
+ * main thread sees only the postMessage round-trip (~1 ms on local
+ * structured-clone of a small Buffer + parsed object).
+ */
+import { parentPort } from "node:worker_threads";
+import { simpleParser } from "mailparser";
+if (!parentPort) {
+    throw new Error("parse-worker: must be spawned as a worker, parentPort is null");
+}
+// Self-warmup: parse a synthetic RFC 5322 message at worker startup so V8
+// JIT, libmime / iconv-lite module loading, and mailparser's lazy
+// initialisation all complete *before* the worker accepts its first real
+// request. Sub-50 ms parses become the norm even on the first user click,
+// instead of the 14-25 s cold-start observed when this work happened
+// on-demand. Buffered messages received during warmup queue behind it.
+const _warmupT0 = Date.now();
+const _warmupPromise = simpleParser(Buffer.from("From: warmup@mailx.local\r\nTo: warmup@mailx.local\r\n"
+    + "Subject: warmup\r\nMIME-Version: 1.0\r\n"
+    + "Content-Type: text/plain; charset=UTF-8\r\n\r\n"
+    + "parse-worker cold-start absorber. Discard.\r\n", "utf8")).then(() => {
+    // Optional: emit a heartbeat so the main thread can log the cost.
+    parentPort.postMessage({ warmupMs: Date.now() - _warmupT0 });
+}).catch(() => { });
+parentPort.on("message", async (msg) => {
+    const { id, source } = msg;
+    // If a real parse arrives during warmup, queue behind it. The await
+    // resolves immediately once warmup is done; trivially fast on hot
+    // worker since _warmupPromise is already resolved.
+    await _warmupPromise;
+    try {
+        // CRITICAL: `Buffer` sent across `worker_threads.postMessage` arrives
+        // as plain `Uint8Array` (Node docs: "Buffer instances passed to
+        // Worker.postMessage() are passed as Uint8Array instances when
+        // received"). mailparser's `simpleParser` uses `Buffer.isBuffer(input)`
+        // to discriminate Buffer / string / Stream. A Uint8Array fails that
+        // check and gets routed to the Stream branch, which calls
+        // `input.once("end", …)` and immediately throws
+        //   `input.once is not a function`
+        // Re-wrap with `Buffer.from(src.buffer, src.byteOffset, src.byteLength)`
+        // — zero-copy view over the same memory, but with the Buffer prototype
+        // so `Buffer.isBuffer` is true.
+        let normalized = source;
+        if (source instanceof Uint8Array && !Buffer.isBuffer(source)) {
+            const u = source;
+            normalized = Buffer.from(u.buffer, u.byteOffset, u.byteLength);
+        }
+        const result = await simpleParser(normalized);
+        parentPort.postMessage({ id, ok: true, result });
+    }
+    catch (e) {
+        const error = e instanceof Error ? e.message : String(e);
+        parentPort.postMessage({ id, ok: false, error });
+    }
+});
+//# sourceMappingURL=parse-worker.js.map

package/store.d.ts

@@ -0,0 +1,169 @@
+/**
+ * Store — the nexus. Owns the local database, the .eml file store, the
+ * operations API, and the event bus. Single source of truth for everything
+ * about a mailx account that lives on this device.
+ *
+ *   UI ──┐                                ┌── Sync clients (IMAP, Gmail API, …)
+ *        │  reads / writes / subscribes  │   read pending actions, commit
+ *        ↓                                ↓   server-discovered changes
+ *                          [ Store ]
+ *
+ * Contract: every method here mutates local-only state. No IMAP, no Gmail,
+ * no SMTP, no DNS, no Drive API. Mutations emit on the bus; subscribers
+ * (UI in another process, in-process reconciler, etc.) react. Sync clients
+ * (mailx-imap, mailx-sync) hold a Store reference and never touch the
+ * underlying MailxDB / FileMessageStore directly.
+ *
+ * This was previously named `Store` and lived in mailx-service. It
+ * moved to mailx-store because mailx-imap (and other sync clients) must
+ * be able to consume it without depending on mailx-service — that's the
+ * "arrow points the right way" property of the architecture.
+ */
+import { MailxDB } from "./db.js";
+import { FileMessageStore } from "./file-store.js";
+import type { StoreBus } from "./bus.js";
+import type { MessageEnvelope, MessageQuery, PagedResult } from "@bobfrankston/mailx-types";
+/** What the UI gets back from a body read. Mirrors the historical
+ *  `getMessage` shape so call-site migration is mechanical. `cached: false`
+ *  means the body isn't on disk yet — the UI shows a "downloading…"
+ *  placeholder and listens for `bodyAvailable` to re-render. The reconciler
+ *  is responsible for actually fetching and emitting the event. */
+export interface StoreMessage extends MessageEnvelope {
+    bodyHtml: string;
+    bodyText: string;
+    hasRemoteContent: boolean;
+    remoteAllowed: boolean;
+    attachments: Array<{
+        id: number;
+        filename: string;
+        mimeType: string;
+        size: number;
+        contentId: string;
+    }>;
+    cached: boolean;
+    deliveredTo: string;
+    returnPath: string;
+    listUnsubscribe: string;
+    listUnsubscribeMail: string;
+    listUnsubscribeHttp: string;
+    listUnsubscribeOneClick: boolean;
+    emlPath: string;
+    isFlagged: boolean;
+}
+export declare class Store {
+    /** SQLite metadata index. Exposed as a public field — sync clients
+     *  (mailx-imap, mailx-sync) read/write through it during the
+     *  refactor. Future state: every external mutation routes through
+     *  a Store method that emits a bus event; raw `store.db.X()` calls
+     *  shrink to zero. Today, this is read mostly. */
+    readonly db: MailxDB;
+    /** .eml file backend. Same migration story as `db` — sync clients
+     *  read/write directly today, will route through Store methods. */
+    readonly bodyStore: FileMessageStore;
+    /** Event bus for Store mutations. Defaults to the process-singleton
+     *  so cross-package subscribers (bin/mailx.ts forwarder → WebView,
+     *  in-process reconciler triggers) see all writes without explicit
+     *  wiring. Tests can pass a fresh StoreBus for isolation. */
+    readonly bus: StoreBus;
+    private static readonly PARSED_LRU_CAPACITY;
+    private parsedLru;
+    private parsedLruGet;
+    private parsedLruPut;
+    private _allowlistCache;
+    private _settingsCache;
+    private getCachedAllowlist;
+    private getCachedSettings;
+    invalidateConfigCaches(): void;
+    constructor(
+    /** SQLite metadata index. Exposed as a public field — sync clients
+     *  (mailx-imap, mailx-sync) read/write through it during the
+     *  refactor. Future state: every external mutation routes through
+     *  a Store method that emits a bus event; raw `store.db.X()` calls
+     *  shrink to zero. Today, this is read mostly. */
+    db: MailxDB, 
+    /** .eml file backend. Same migration story as `db` — sync clients
+     *  read/write directly today, will route through Store methods. */
+    bodyStore: FileMessageStore, 
+    /** Event bus for Store mutations. Defaults to the process-singleton
+     *  so cross-package subscribers (bin/mailx.ts forwarder → WebView,
+     *  in-process reconciler triggers) see all writes without explicit
+     *  wiring. Tests can pass a fresh StoreBus for isolation. */
+    bus?: StoreBus);
+    /** DB-shape account list (id/name/email/lastSync). The richer
+     *  AccountConfig (with imap/smtp/etc.) lives in accounts.jsonc and is
+     *  loaded by mailx-settings, not the DB — that path stays in
+     *  MailxService until step 3 of the local-first plan. */
+    getAccounts(): {
+        id: string;
+        name: string;
+        email: string;
+        lastSync: number;
+    }[];
+    getFolders(accountId: string): any[];
+    /** Look up a folder by RFC 6154 specialUse tag (`trash`, `drafts`, `sent`,
+     *  `junk`, etc.) for the given account. Falls back to a case-insensitive
+     *  path match for legacy rows where specialUse never got tagged.
+     *  Returns null when the account has no such folder configured. */
+    findSpecialFolder(accountId: string, specialUse: string): {
+        id: number;
+        path: string;
+    } | null;
+    /** Single envelope by (account, uid, folder). Null when the row isn't
+     *  in the DB — caller decides whether to show "deleted" or queue a
+     *  server lookup via the reconciler. */
+    getMessageEnvelope(accountId: string, uid: number, folderId?: number): MessageEnvelope | null;
+    /** Paginated message list for a (account, folder, ...) query. */
+    getMessages(query: MessageQuery): PagedResult<MessageEnvelope>;
+    /** All-Inboxes view: union of every account's INBOX, paginated. */
+    getUnifiedInbox(page?: number, pageSize?: number): PagedResult<MessageEnvelope>;
+    /** Local FTS5 search. Server-scope search is the reconciler's job. */
+    searchMessages(query: string, page?: number, pageSize?: number, accountId?: string, folderId?: number, includeTrashSpam?: boolean): PagedResult<MessageEnvelope>;
+    /** Read a fully-parsed message (envelope + body + attachments) entirely
+     *  from local state. Returns null when the envelope isn't known.
+     *  Returns `{ ...envelope, cached: false }` when the envelope is known
+     *  but the body file isn't on disk — UI shows a placeholder and the
+     *  reconciler queues the fetch.
+     *
+     *  `allowRemote=true` skips HTML sanitization. Used when the user has
+     *  explicitly allowed remote content for this sender / domain. */
+    getMessage(accountId: string, uid: number, allowRemote: boolean, folderId?: number): Promise<StoreMessage | null>;
+    getCalendarEvents(accountId: string, fromMs: number, toMs: number): any[];
+    getTasks(accountId: string, includeCompleted?: boolean): any[];
+    searchContacts(query: string, limit?: number): any[];
+    listContacts(query: string, page?: number, pageSize?: number): any;
+    /** Update a message's flag set. Local DB write completes synchronously;
+     *  the server-mirror enqueue is the caller's responsibility (typically
+     *  via SyncQueue.enqueueFlag) so callers that don't want a server push
+     *  — pure-local UI state like "pin in pane" — can skip it.
+     *
+     *  Publishes:
+     *    `message:<uuid>` { kind: "flagsChanged" }
+     *    `folder:<id>`    (auto fan-out)
+     */
+    updateFlags(accountId: string, uid: number, folderId: number, flags: string[]): void;
+    /** Move a message between folders in the same account. Adds a tombstone
+     *  on the Message-ID so the next sync doesn't re-import the pre-move row
+     *  in the source folder before the server-side MOVE completes; tombstone
+     *  is cleared on terminal IMAP failure (see processSyncActions).
+     *
+     *  Returns true if a local row existed and was moved, false otherwise.
+     *
+     *  Publishes:
+     *    `message:<uuid>` { kind: "messageMoved", folderId: source, targetFolderId }
+     *    `folder:<source>` and `folder:<target>` (auto fan-out + explicit count)
+     */
+    moveMessage(accountId: string, uid: number, fromFolderId: number, targetFolderId: number): boolean;
+    /** Trash a message. If a trash folder is configured and the message is
+     *  not already in it, this is a move-to-trash. If the message is already
+     *  in trash (or no trash exists), it's a hard delete + body unlink.
+     *
+     *  Returns "moved-to-trash" or "expunged" so the caller knows whether
+     *  to enqueue an IMAP MOVE or a DELETE+EXPUNGE on the queue.
+     */
+    trashMessage(accountId: string, uid: number, folderId: number, trashFolderId: number | null): "moved-to-trash" | "expunged";
+    /** Restore a message from trash back to its original folder. Local-only;
+     *  caller handles the queue (cancel-pending-MOVE vs queue-counter-MOVE).
+     *  Returns true if a local row was moved. */
+    undeleteMessage(accountId: string, uid: number, trashFolderId: number, originalFolderId: number): boolean;
+}
+//# sourceMappingURL=store.d.ts.map