@bobfrankston/mailx-store 0.1.72 → 0.1.74

package/db.d.ts

@@ -320,6 +320,15 @@ export declare class MailxDB {
      *  so that lookup could miss the row entirely and silently leave the
      *  body un-indexed (Bob 2026-05-18: "mcdade" in an opened message's
      *  body, search found nothing). */
+    /** Length of the indexed FTS body_text for a row (0 = never backfilled
+     *  or row missing). Cheap rowid lookup — used to skip redundant
+     *  re-index work on every message view and by the backfill sweep. */
+    ftsBodyLength(rowId: number): number;
+    /** Cursor page over messages for the FTS body backfill sweep. */
+    listBodyRowsAfter(afterId: number, limit: number): Array<{
+        id: number;
+        bodyPath: string;
+    }>;
     updateFtsBody(rowId: number, bodyText: string): void;
     /** List view: messages currently in (account, folder).
      *  Joins through `message_folders` so the UID + folder location come

package/db.js

@@ -1991,6 +1991,22 @@ export class MailxDB {
      *  so that lookup could miss the row entirely and silently leave the
      *  body un-indexed (Bob 2026-05-18: "mcdade" in an opened message's
      *  body, search found nothing). */
+    /** Length of the indexed FTS body_text for a row (0 = never backfilled
+     *  or row missing). Cheap rowid lookup — used to skip redundant
+     *  re-index work on every message view and by the backfill sweep. */
+    ftsBodyLength(rowId) {
+        try {
+            const r = this.db.prepare("SELECT length(body_text) AS l FROM messages_fts WHERE rowid = ?").get(rowId);
+            return Number(r?.l || 0);
+        }
+        catch {
+            return 0;
+        }
+    }
+    /** Cursor page over messages for the FTS body backfill sweep. */
+    listBodyRowsAfter(afterId, limit) {
+        return this.db.prepare("SELECT id, body_path AS bodyPath FROM messages WHERE id > ? ORDER BY id LIMIT ?").all(afterId, limit);
+    }
     updateFtsBody(rowId, bodyText) {
         if (!rowId)
             return;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bobfrankston/mailx-store",
-  "version": "0.1.72",
+  "version": "0.1.74",
   "type": "module",
   "main": "index.js",
   "types": "index.d.ts",

package/store.d.ts

@@ -89,6 +89,21 @@ export declare class Store {
     private getCachedAllowlist;
     private getCachedSettings;
     invalidateConfigCaches(): void;
+    /** Set on read-only Store instances (the db-worker serving UI reads):
+     *  receives (rowId, bodyText) when getMessage parses a body whose FTS
+     *  body_text is still empty, so the writable side can apply
+     *  updateFtsBody. Without this the worker's UPDATE fails on its
+     *  query_only connection — silently, for months (2026-07-27). */
+    onFtsBodyBackfill: ((rowId: number, bodyText: string) => void) | null;
+    /** Background sweep: index the body text of every message whose body is
+     *  on disk but whose FTS row has an empty body_text. Historical debt
+     *  from the read-only-worker backfill bug — 179k of 190k rows on Bob's
+     *  store. Paced (one parse per ~100 ms + chunk pauses) so the shared
+     *  parse-worker lane stays responsive for interactive views; safe to
+     *  re-run every boot — already-indexed rows cost one rowid lookup and
+     *  are skipped, so a completed sweep re-scans in seconds. Never runs on
+     *  a read-only connection. */
+    runFtsBodyBackfillSweep(): Promise<void>;
     constructor(
     /** SQLite metadata index. Exposed as a public field — sync clients
      *  (mailx-imap, mailx-sync) read/write through it during the

package/store.js

@@ -22,7 +22,7 @@
 import * as fs from "node:fs";
 import { parseSerial } from "./parse-serial.js";
 import { storeBus } from "./bus.js";
-import { sanitizeHtml } from "@bobfrankston/mailx-types";
+import { sanitizeHtml, htmlToPlainText } from "@bobfrankston/mailx-types";
 import { loadSettings, loadAllowlist } from "@bobfrankston/mailx-settings";
 import { sniffAndFixCharset } from "./charset.js";
 /** Parse `List-Unsubscribe` (RFC 2369) and `List-Unsubscribe-Post` (RFC 8058).
@@ -127,6 +127,61 @@ export class Store {
         this._allowlistCache = null;
         this._settingsCache = null;
     }
+    /** Set on read-only Store instances (the db-worker serving UI reads):
+     *  receives (rowId, bodyText) when getMessage parses a body whose FTS
+     *  body_text is still empty, so the writable side can apply
+     *  updateFtsBody. Without this the worker's UPDATE fails on its
+     *  query_only connection — silently, for months (2026-07-27). */
+    onFtsBodyBackfill = null;
+    /** Background sweep: index the body text of every message whose body is
+     *  on disk but whose FTS row has an empty body_text. Historical debt
+     *  from the read-only-worker backfill bug — 179k of 190k rows on Bob's
+     *  store. Paced (one parse per ~100 ms + chunk pauses) so the shared
+     *  parse-worker lane stays responsive for interactive views; safe to
+     *  re-run every boot — already-indexed rows cost one rowid lookup and
+     *  are skipped, so a completed sweep re-scans in seconds. Never runs on
+     *  a read-only connection. */
+    async runFtsBodyBackfillSweep() {
+        if (this.db.readOnly)
+            return;
+        const CHUNK = 200;
+        let last = 0, scanned = 0, indexed = 0, failed = 0;
+        const t0 = Date.now();
+        for (;;) {
+            const rows = this.db.listBodyRowsAfter(last, CHUNK);
+            if (rows.length === 0)
+                break;
+            for (const r of rows) {
+                last = r.id;
+                scanned++;
+                if (!r.bodyPath)
+                    continue;
+                if (this.db.ftsBodyLength(r.id) > 0)
+                    continue;
+                try {
+                    if (!await this.bodyStore.hasByPath(r.bodyPath))
+                        continue;
+                    const raw = await this.bodyStore.readByPath(r.bodyPath);
+                    const parsed = await parseSerial(sniffAndFixCharset(raw));
+                    const text = parsed.text || (parsed.html ? htmlToPlainText(parsed.html) : "");
+                    if (text) {
+                        this.db.updateFtsBody(r.id, text);
+                        indexed++;
+                    }
+                }
+                catch {
+                    failed++;
+                }
+                // Pace: yield the parse lane between messages so a user click
+                // never waits behind more than one sweep parse.
+                await new Promise(res => setTimeout(res, 100));
+            }
+            if (scanned % 5000 < CHUNK) {
+                console.log(`  [fts-backfill] ${scanned} scanned, ${indexed} indexed, ${failed} failed (${Math.round((Date.now() - t0) / 60000)} min)`);
+            }
+        }
+        console.log(`  [fts-backfill] sweep complete: ${indexed} bodies indexed, ${failed} failed, ${scanned} rows scanned in ${Math.round((Date.now() - t0) / 60000)} min`);
+    }
     constructor(
     /** SQLite metadata index. Exposed as a public field — sync clients
      *  (mailx-imap, mailx-sync) read/write through it during the
@@ -337,11 +392,30 @@ export class Store {
         // backfill, searches miss any word that only appears deeper in the
         // body. Fire-and-forget — failures are non-fatal, never block the
         // user's preview render.
-        if (bodyText && envelope.id) {
-            try {
-                this.db.updateFtsBody(envelope.id, bodyText);
+        //
+        // HTML-only mail has no parsed.text — derive it from the HTML so
+        // those bodies are searchable too (most marketing/newsletter mail).
+        //
+        // On the READ-ONLY connection (db-worker serving UI reads) the
+        // UPDATE cannot run — for months it failed silently here and left
+        // body_text empty for every message whose body arrived outside the
+        // prefetch path (Bob 2026-07-27: "orwell" found 1 of 6 thread
+        // messages; 179k of 190k FTS rows had no body text). Route through
+        // onFtsBodyBackfill so the writable main thread applies it.
+        const ftsText = bodyText || (bodyHtml ? htmlToPlainText(bodyHtml) : "");
+        if (ftsText && envelope.id && this.db.ftsBodyLength(envelope.id) === 0) {
+            if (this.db.readOnly) {
+                try {
+                    this.onFtsBodyBackfill?.(envelope.id, ftsText.slice(0, 64_000));
+                }
+                catch { /* */ }
+            }
+            else {
+                try {
+                    this.db.updateFtsBody(envelope.id, ftsText);
+                }
+                catch { /* */ }
             }
-            catch { /* */ }
         }
         let hasRemoteContent = false;
         // Filter out "spurious" attachments: mailing-list footers and signature