@bobfrankston/mailx-store 0.1.72 → 0.1.76

package/db.d.ts

@@ -274,8 +274,12 @@ export declare class MailxDB {
      *  During the additive migration we ALSO have to handle the legacy
      *  case where the row's folder_id matches but no message_folders row
      *  was migrated for it (race during initial population, etc.) — we
-     *  fall back to the messages.folder_id check to clean up consistently. */
-    dropFolderMembership(accountId: string, folderId: number, uid: number, folderPath?: string): boolean;
+     *  fall back to the messages.folder_id check to clean up consistently.
+     *
+     *  `reason`/`source` override the audit text — deleteMessage routes user
+     *  deletes through here, and "reconcile: server missing this UID" would
+     *  be a lie in that audit trail. */
+    dropFolderMembership(accountId: string, folderId: number, uid: number, folderPath?: string, reason?: string, source?: string): boolean;
     upsertMessage(msg: {
         accountId: string;
         folderId: number;
@@ -320,6 +324,15 @@ export declare class MailxDB {
      *  so that lookup could miss the row entirely and silently leave the
      *  body un-indexed (Bob 2026-05-18: "mcdade" in an opened message's
      *  body, search found nothing). */
+    /** Length of the indexed FTS body_text for a row (0 = never backfilled
+     *  or row missing). Cheap rowid lookup — used to skip redundant
+     *  re-index work on every message view and by the backfill sweep. */
+    ftsBodyLength(rowId: number): number;
+    /** Cursor page over messages for the FTS body backfill sweep. */
+    listBodyRowsAfter(afterId: number, limit: number): Array<{
+        id: number;
+        bodyPath: string;
+    }>;
     updateFtsBody(rowId: number, bodyText: string): void;
     /** List view: messages currently in (account, folder).
      *  Joins through `message_folders` so the UID + folder location come

package/db.js

@@ -1675,8 +1675,12 @@ export class MailxDB {
      *  During the additive migration we ALSO have to handle the legacy
      *  case where the row's folder_id matches but no message_folders row
      *  was migrated for it (race during initial population, etc.) — we
-     *  fall back to the messages.folder_id check to clean up consistently. */
-    dropFolderMembership(accountId, folderId, uid, folderPath) {
+     *  fall back to the messages.folder_id check to clean up consistently.
+     *
+     *  `reason`/`source` override the audit text — deleteMessage routes user
+     *  deletes through here, and "reconcile: server missing this UID" would
+     *  be a lie in that audit trail. */
+    dropFolderMembership(accountId, folderId, uid, folderPath, reason, source) {
         // First find the messages row this membership pointed at.
         const mf = this.db.prepare("SELECT message_row_id FROM message_folders WHERE folder_id = ? AND uid = ?").get(folderId, uid);
         let rowId = null;
@@ -1711,8 +1715,8 @@ export class MailxDB {
             this.audit({
                 kind: "delete-membership",
                 accountId, folderId, uid,
-                reason: `reconcile dropped folder membership; ${remaining.cnt} other location(s) remain`,
-                source: `dropFolderMembership (${folderPath || ""})`,
+                reason: reason || `reconcile dropped folder membership; ${remaining.cnt} other location(s) remain`,
+                source: source || `dropFolderMembership (${folderPath || ""})`,
             });
             return true;
         }
@@ -1726,8 +1730,8 @@ export class MailxDB {
                 accountId, folderId, uid,
                 messageId: env.message_id || undefined,
                 subject: env.subject || undefined,
-                reason: "reconcile: server missing this UID after grace, no other folder memberships",
-                source: `dropFolderMembership (${folderPath || ""})`,
+                reason: reason || "reconcile: server missing this UID after grace, no other folder memberships",
+                source: source || `dropFolderMembership (${folderPath || ""})`,
             });
             console.log(`  [reconcile-delete] ${accountId} ${folderPath || folderId}/${uid} msgid=${env.message_id || "?"} (no other memberships) — body ${env.body_path || "(none)"}`);
         }
@@ -1991,6 +1995,22 @@ export class MailxDB {
      *  so that lookup could miss the row entirely and silently leave the
      *  body un-indexed (Bob 2026-05-18: "mcdade" in an opened message's
      *  body, search found nothing). */
+    /** Length of the indexed FTS body_text for a row (0 = never backfilled
+     *  or row missing). Cheap rowid lookup — used to skip redundant
+     *  re-index work on every message view and by the backfill sweep. */
+    ftsBodyLength(rowId) {
+        try {
+            const r = this.db.prepare("SELECT length(body_text) AS l FROM messages_fts WHERE rowid = ?").get(rowId);
+            return Number(r?.l || 0);
+        }
+        catch {
+            return 0;
+        }
+    }
+    /** Cursor page over messages for the FTS body backfill sweep. */
+    listBodyRowsAfter(afterId, limit) {
+        return this.db.prepare("SELECT id, body_path AS bodyPath FROM messages WHERE id > ? ORDER BY id LIMIT ?").all(afterId, limit);
+    }
     updateFtsBody(rowId, bodyText) {
         if (!rowId)
             return;
@@ -2294,13 +2314,19 @@ export class MailxDB {
      *  hitting the message in `_Spam` (correct) and hitting nothing
      *  because the legacy messages.folder_id still points at INBOX. */
     getMessageByUid(accountId, uid, folderId) {
+        // Alias mf.folder_id over m.folder_id too — the envelope must carry
+        // the identity of the membership that MATCHED, not the legacy primary
+        // location, or a caller doing delete/move/flag with the returned
+        // folderId operates on a different folder than the one it looked up
+        // (the two drift: local moves keep the original uid, re-binds update
+        // one table but not the other).
         const sql = folderId != null
-            ? `SELECT m.*, mf.uid AS uid
+            ? `SELECT m.*, mf.uid AS uid, mf.folder_id AS folder_id
                FROM messages m
                JOIN message_folders mf ON mf.message_row_id = m.id
                WHERE m.account_id = ? AND mf.uid = ? AND mf.folder_id = ?
                LIMIT 1`
-            : `SELECT m.*, mf.uid AS uid
+            : `SELECT m.*, mf.uid AS uid, mf.folder_id AS folder_id
                FROM messages m
                JOIN message_folders mf ON mf.message_row_id = m.id
                WHERE m.account_id = ? AND mf.uid = ?
@@ -2868,14 +2894,31 @@ export class MailxDB {
      *  Reason is propagated into the audit_log so the DB carries an
      *  authoritative trail of every removal. */
     deleteMessage(accountId, folderId, uid, reason, source) {
-        // Get message_id + subject before deleting so the audit row carries
-        // enough context to identify what was removed.
-        const msg = folderId != null
-            ? this.db.prepare("SELECT folder_id, message_id, subject FROM messages WHERE account_id = ? AND folder_id = ? AND uid = ?").get(accountId, folderId, uid)
-            : this.db.prepare("SELECT folder_id, message_id, subject FROM messages WHERE account_id = ? AND uid = ?").get(accountId, uid);
-        const r = folderId != null
-            ? this.db.prepare("DELETE FROM messages WHERE account_id = ? AND folder_id = ? AND uid = ?").run(accountId, folderId, uid)
-            : this.db.prepare("DELETE FROM messages WHERE account_id = ? AND uid = ?").run(accountId, uid);
+        if (folderId != null) {
+            // Resolve through message_folders — the SAME identity that
+            // getMessageByUid and every list query use. The legacy
+            // messages.folder_id/uid columns can drift from the membership
+            // rows (local moves keep the original uid, server re-binds update
+            // one table but not the other), and a delete keyed only on the
+            // legacy columns silently matches ZERO rows on a drifted message:
+            // the membership keeps it visible in the UI and it becomes
+            // undeletable forever (Eleanor 2026-07-29: ElkinWs uid 125482 in
+            // Trash "refuses to delete" — every attempt no-op'd here while
+            // the sync action reported success). dropFolderMembership drops
+            // the membership, orphan-cleans the messages row, and falls back
+            // to the legacy columns for pre-migration rows.
+            const cleaned = this.dropFolderMembership(accountId, folderId, uid, undefined, reason || "deleteMessage (no reason)", source || "db.deleteMessage");
+            if (!cleaned) {
+                // A delete that deletes nothing is the "refuses to delete"
+                // symptom — never let it pass silently.
+                console.error(`  [delete-miss] ${accountId} folder=${folderId} uid=${uid}: no membership or legacy row matched — nothing deleted (source=${source || "db.deleteMessage"})`);
+            }
+            this.recalcFolderCounts(folderId);
+            return;
+        }
+        // No folderId (legacy callers): fall back to the messages-table key.
+        const msg = this.db.prepare("SELECT id, folder_id, message_id, subject FROM messages WHERE account_id = ? AND uid = ?").get(accountId, uid);
+        const r = this.db.prepare("DELETE FROM messages WHERE account_id = ? AND uid = ?").run(accountId, uid);
         if (r.changes && msg) {
             this.audit({
                 kind: "delete-msg",
@@ -2888,6 +2931,9 @@ export class MailxDB {
                 source: source || "db.deleteMessage",
             });
         }
+        else if (!r.changes) {
+            console.error(`  [delete-miss] ${accountId} uid=${uid} (no folderId): no messages row matched — nothing deleted (source=${source || "db.deleteMessage"})`);
+        }
         // Refresh folder counts
         if (msg)
             this.recalcFolderCounts(msg.folder_id);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bobfrankston/mailx-store",
-  "version": "0.1.72",
+  "version": "0.1.76",
   "type": "module",
   "main": "index.js",
   "types": "index.d.ts",

package/store.d.ts

@@ -89,6 +89,21 @@ export declare class Store {
     private getCachedAllowlist;
     private getCachedSettings;
     invalidateConfigCaches(): void;
+    /** Set on read-only Store instances (the db-worker serving UI reads):
+     *  receives (rowId, bodyText) when getMessage parses a body whose FTS
+     *  body_text is still empty, so the writable side can apply
+     *  updateFtsBody. Without this the worker's UPDATE fails on its
+     *  query_only connection — silently, for months (2026-07-27). */
+    onFtsBodyBackfill: ((rowId: number, bodyText: string) => void) | null;
+    /** Background sweep: index the body text of every message whose body is
+     *  on disk but whose FTS row has an empty body_text. Historical debt
+     *  from the read-only-worker backfill bug — 179k of 190k rows on Bob's
+     *  store. Paced (one parse per ~100 ms + chunk pauses) so the shared
+     *  parse-worker lane stays responsive for interactive views; safe to
+     *  re-run every boot — already-indexed rows cost one rowid lookup and
+     *  are skipped, so a completed sweep re-scans in seconds. Never runs on
+     *  a read-only connection. */
+    runFtsBodyBackfillSweep(): Promise<void>;
     constructor(
     /** SQLite metadata index. Exposed as a public field — sync clients
      *  (mailx-imap, mailx-sync) read/write through it during the

package/store.js

@@ -22,7 +22,7 @@
 import * as fs from "node:fs";
 import { parseSerial } from "./parse-serial.js";
 import { storeBus } from "./bus.js";
-import { sanitizeHtml } from "@bobfrankston/mailx-types";
+import { sanitizeHtml, htmlToPlainText } from "@bobfrankston/mailx-types";
 import { loadSettings, loadAllowlist } from "@bobfrankston/mailx-settings";
 import { sniffAndFixCharset } from "./charset.js";
 /** Parse `List-Unsubscribe` (RFC 2369) and `List-Unsubscribe-Post` (RFC 8058).
@@ -127,6 +127,61 @@ export class Store {
         this._allowlistCache = null;
         this._settingsCache = null;
     }
+    /** Set on read-only Store instances (the db-worker serving UI reads):
+     *  receives (rowId, bodyText) when getMessage parses a body whose FTS
+     *  body_text is still empty, so the writable side can apply
+     *  updateFtsBody. Without this the worker's UPDATE fails on its
+     *  query_only connection — silently, for months (2026-07-27). */
+    onFtsBodyBackfill = null;
+    /** Background sweep: index the body text of every message whose body is
+     *  on disk but whose FTS row has an empty body_text. Historical debt
+     *  from the read-only-worker backfill bug — 179k of 190k rows on Bob's
+     *  store. Paced (one parse per ~100 ms + chunk pauses) so the shared
+     *  parse-worker lane stays responsive for interactive views; safe to
+     *  re-run every boot — already-indexed rows cost one rowid lookup and
+     *  are skipped, so a completed sweep re-scans in seconds. Never runs on
+     *  a read-only connection. */
+    async runFtsBodyBackfillSweep() {
+        if (this.db.readOnly)
+            return;
+        const CHUNK = 200;
+        let last = 0, scanned = 0, indexed = 0, failed = 0;
+        const t0 = Date.now();
+        for (;;) {
+            const rows = this.db.listBodyRowsAfter(last, CHUNK);
+            if (rows.length === 0)
+                break;
+            for (const r of rows) {
+                last = r.id;
+                scanned++;
+                if (!r.bodyPath)
+                    continue;
+                if (this.db.ftsBodyLength(r.id) > 0)
+                    continue;
+                try {
+                    if (!await this.bodyStore.hasByPath(r.bodyPath))
+                        continue;
+                    const raw = await this.bodyStore.readByPath(r.bodyPath);
+                    const parsed = await parseSerial(sniffAndFixCharset(raw));
+                    const text = parsed.text || (parsed.html ? htmlToPlainText(parsed.html) : "");
+                    if (text) {
+                        this.db.updateFtsBody(r.id, text);
+                        indexed++;
+                    }
+                }
+                catch {
+                    failed++;
+                }
+                // Pace: yield the parse lane between messages so a user click
+                // never waits behind more than one sweep parse.
+                await new Promise(res => setTimeout(res, 100));
+            }
+            if (scanned % 5000 < CHUNK) {
+                console.log(`  [fts-backfill] ${scanned} scanned, ${indexed} indexed, ${failed} failed (${Math.round((Date.now() - t0) / 60000)} min)`);
+            }
+        }
+        console.log(`  [fts-backfill] sweep complete: ${indexed} bodies indexed, ${failed} failed, ${scanned} rows scanned in ${Math.round((Date.now() - t0) / 60000)} min`);
+    }
     constructor(
     /** SQLite metadata index. Exposed as a public field — sync clients
      *  (mailx-imap, mailx-sync) read/write through it during the
@@ -337,11 +392,30 @@ export class Store {
         // backfill, searches miss any word that only appears deeper in the
         // body. Fire-and-forget — failures are non-fatal, never block the
         // user's preview render.
-        if (bodyText && envelope.id) {
-            try {
-                this.db.updateFtsBody(envelope.id, bodyText);
+        //
+        // HTML-only mail has no parsed.text — derive it from the HTML so
+        // those bodies are searchable too (most marketing/newsletter mail).
+        //
+        // On the READ-ONLY connection (db-worker serving UI reads) the
+        // UPDATE cannot run — for months it failed silently here and left
+        // body_text empty for every message whose body arrived outside the
+        // prefetch path (Bob 2026-07-27: "orwell" found 1 of 6 thread
+        // messages; 179k of 190k FTS rows had no body text). Route through
+        // onFtsBodyBackfill so the writable main thread applies it.
+        const ftsText = bodyText || (bodyHtml ? htmlToPlainText(bodyHtml) : "");
+        if (ftsText && envelope.id && this.db.ftsBodyLength(envelope.id) === 0) {
+            if (this.db.readOnly) {
+                try {
+                    this.onFtsBodyBackfill?.(envelope.id, ftsText.slice(0, 64_000));
+                }
+                catch { /* */ }
+            }
+            else {
+                try {
+                    this.db.updateFtsBody(envelope.id, ftsText);
+                }
+                catch { /* */ }
             }
-            catch { /* */ }
         }
         let hasRemoteContent = false;
         // Filter out "spurious" attachments: mailing-list footers and signature