@bobfrankston/mailx-store 0.1.29 → 0.1.31

package/db.d.ts

@@ -259,9 +259,15 @@ export declare class MailxDB {
     /** Backfill the FTS5 `body_text` column for a message after its body
      *  has been parsed. Capped at ~64 KB of text per row — FTS5 stores the
      *  raw text and indexes tokens; we don't need every byte of a 1 MB
-     *  marketing email to find a word in the first paragraph. Called from
-     *  LocalStore.getMessage right after simpleParser succeeds. */
-    updateFtsBodyByUid(accountId: string, folderId: number, uid: number, bodyText: string): void;
+     *  marketing email to find a word in the first paragraph.
+     *
+     *  Keyed on the FTS rowid (= messages.id) directly. The earlier
+     *  (account, folder_id, uid) lookup keyed on the LEGACY messages.folder_id
+     *  column — a message can live in several folders via message_folders,
+     *  so that lookup could miss the row entirely and silently leave the
+     *  body un-indexed (Bob 2026-05-18: "mcdade" in an opened message's
+     *  body, search found nothing). */
+    updateFtsBody(rowId: number, bodyText: string): void;
     /** List view: messages currently in (account, folder).
      *  Joins through `message_folders` so the UID + folder location come
      *  from membership rows, not from the legacy messages.folder_id /
@@ -314,6 +320,33 @@ export declare class MailxDB {
         id: number;
     } | null;
     updateBodyPath(accountId: string, uid: number, bodyPath: string): void;
+    /** Write the post-download metadata in one go: body file path, parsed
+     *  attachment flag, preview text, and the parse timestamp. Called from
+     *  prefetch after the body has been written to disk and parsed — without
+     *  this, IMAP metadata-only sync leaves has_attachments=0 and preview=""
+     *  forever (the paperclip 📎 never renders for non-Gmail accounts).
+     *  Pass bodyPath = "" to update only the parsed fields (used by the
+     *  backfill worker re-parsing existing .eml files on disk). */
+    updateBodyMeta(accountId: string, uid: number, bodyPath: string, hasAttachments: boolean, preview: string): void;
+    /** Find rows whose body is on disk but was never run through extractPreview
+     *  (has_attachments/preview reflect "no body source at sync time", typical
+     *  of the IMAP metadata-only path). Used by the backfill worker to walk
+     *  every existing message once after the prefetch-reparse fix landed.
+     *  Returns id + body_path so the worker doesn't need a second lookup. */
+    getMessagesNeedingBodyReparse(accountId: string, limit?: number): {
+        id: number;
+        uid: number;
+        bodyPath: string;
+    }[];
+    /** Backfill variant that writes by row id (avoids re-resolving uid). */
+    updateBodyMetaById(rowId: number, hasAttachments: boolean, preview: string): void;
+    /** Mark a message as "replied to" by its RFC-5322 Message-ID. Source of
+     *  truth for the ↩ indicator: the local DB knows who replied to whom via
+     *  In-Reply-To headers, independent of whether the IMAP server propagates
+     *  \Answered (Gmail labels, cross-account replies, etc.). Called from the
+     *  send path when msg.inReplyTo is set, and from upsertMessage when a
+     *  reply arrives. Idempotent — UPDATE on an already-1 row is a no-op. */
+    markRepliedByMessageId(accountId: string, messageId: string): boolean;
     /** Get messages without cached bodies (for background prefetch) */
     getMessagesWithoutBody(accountId: string, limit?: number): {
         uid: number;

package/db.js

@@ -389,6 +389,44 @@ export class MailxDB {
             this.db.exec("CREATE INDEX IF NOT EXISTS idx_messages_thread_id ON messages(account_id, thread_id)");
         }
         catch { /* already exists */ }
+        // is_replied: set when ANY other message in this account has in_reply_to
+        // pointing at this row's message_id. Primary source of truth for the ↩
+        // marker — \Answered is plan B (some servers strip it, Gmail labels
+        // don't propagate it, cross-account replies don't update the source).
+        // Backward-compatible default 0; rebuilt on the next sync as upsertMessage
+        // walks references both forward (mark parent) and backward (set self).
+        this.addColumnIfMissing("messages", "is_replied", "INTEGER DEFAULT 0");
+        try {
+            this.db.exec("CREATE INDEX IF NOT EXISTS idx_messages_in_reply_to ON messages(account_id, in_reply_to)");
+        }
+        catch { /* already exists */ }
+        // body_parsed_at: timestamp of the last extractPreview run. Backfills
+        // use this to find rows whose body_path is populated but were never
+        // parsed (existing IMAP-synced rows that predate the prefetch-reparse
+        // fix, where has_attachments stays 0 forever). NULL = never parsed.
+        this.addColumnIfMissing("messages", "body_parsed_at", "INTEGER");
+        // One-shot is_replied backfill on every boot. SQL-only, indexed
+        // both sides via idx_messages_message_id and idx_messages_in_reply_to,
+        // so a 100k-row mailbox finishes in milliseconds. Idempotent — after
+        // the first run the WHERE-is_replied=0 filter prunes work down to any
+        // new linkages that arrived during the previous session.
+        try {
+            const r = this.db.prepare(`
+                UPDATE messages SET is_replied = 1
+                WHERE is_replied = 0
+                  AND message_id IS NOT NULL AND message_id != ''
+                  AND EXISTS (
+                    SELECT 1 FROM messages m2
+                    WHERE m2.account_id = messages.account_id
+                      AND m2.in_reply_to = messages.message_id
+                  )
+            `).run();
+            if (r.changes > 0)
+                console.log(`  [db] backfilled is_replied on ${r.changes} rows`);
+        }
+        catch (e) {
+            console.error(`  [db] is_replied backfill failed: ${e?.message || e}`);
+        }
         // provider_id: native server-side id for API-backed providers (Gmail
         // hex id, Outlook Graph id, etc.). Lets fetchOne look up the message
         // directly instead of paginating listMessageIds for every body fetch
@@ -1068,6 +1106,7 @@ export class MailxDB {
             flags: JSON.parse(r.flags_json),
             size: r.size,
             hasAttachments: !!r.has_attachments,
+            isReplied: !!r.is_replied,
             preview: r.preview,
             bodyPath: r.body_path || undefined,
         }));
@@ -1398,9 +1437,9 @@ export class MailxDB {
             // every message every cycle.
             if (msg.bodyPath) {
                 this.db.prepare(`
-                    UPDATE messages SET flags_json = ?, preview = ?, body_path = ?, cached_at = ?
+                    UPDATE messages SET flags_json = ?, preview = ?, has_attachments = ?, body_path = ?, body_parsed_at = ?, cached_at = ?
                     WHERE id = ?
-                `).run(flagsToWrite, msg.preview, msg.bodyPath, Date.now(), existing.id);
+                `).run(flagsToWrite, msg.preview, msg.hasAttachments ? 1 : 0, msg.bodyPath, Date.now(), Date.now(), existing.id);
             }
             else {
                 this.db.prepare(`
@@ -1471,14 +1510,26 @@ export class MailxDB {
         // the life of the row — survives server UID renumbers, folder moves
         // (sync rebinds folder_id/uid but keeps uuid), UIDVALIDITY bumps.
         const uuid = randomUUID().replace(/-/g, "");
+        // Backward direction for ↩: a reply may have arrived BEFORE its parent
+        // in sync order (very common — Gmail/IMAP fetch is date-DESC). If any
+        // existing row already references this new message, the new message is
+        // already-replied at insert time.
+        const alreadyReplied = msg.messageId
+            ? !!this.db.prepare("SELECT 1 FROM messages WHERE account_id = ? AND in_reply_to = ? LIMIT 1").get(msg.accountId, msg.messageId)
+            : false;
         const result = this.db.prepare(`
             INSERT INTO messages (
                 account_id, folder_id, uid, uuid, message_id, in_reply_to, refs, thread_id,
                 date, subject, from_address, from_name, to_json, cc_json,
-                flags_json, size, has_attachments, preview, body_path, cached_at, provider_id
-            ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
-        `).run(msg.accountId, msg.folderId, msg.uid, uuid, msg.messageId, msg.inReplyTo, JSON.stringify(msg.references), threadId, msg.date, msg.subject, msg.from.address, msg.from.name, JSON.stringify(msg.to), JSON.stringify(msg.cc), JSON.stringify(msg.flags), msg.size, msg.hasAttachments ? 1 : 0, msg.preview, msg.bodyPath, Date.now(), msg.providerId || null);
+                flags_json, size, has_attachments, preview, body_path, body_parsed_at, cached_at, provider_id, is_replied
+            ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+        `).run(msg.accountId, msg.folderId, msg.uid, uuid, msg.messageId, msg.inReplyTo, JSON.stringify(msg.references), threadId, msg.date, msg.subject, msg.from.address, msg.from.name, JSON.stringify(msg.to), JSON.stringify(msg.cc), JSON.stringify(msg.flags), msg.size, msg.hasAttachments ? 1 : 0, msg.preview, msg.bodyPath, msg.bodyPath ? Date.now() : null, Date.now(), msg.providerId || null, alreadyReplied ? 1 : 0);
         const rowId = Number(result.lastInsertRowid);
+        // Forward direction for ↩: if this message is a reply, mark the parent
+        // as replied-to. Cheap indexed UPDATE; no-op when the parent isn't in
+        // mailx's DB (cross-account, archived, never synced).
+        if (msg.inReplyTo)
+            this.markRepliedByMessageId(msg.accountId, msg.inReplyTo);
         // Record the (folder, uid) membership for the new row. New schema
         // source of truth — old folder_id/uid columns above are kept in
         // sync during the additive migration but reads will move to JOIN
@@ -1501,15 +1552,20 @@ export class MailxDB {
     /** Backfill the FTS5 `body_text` column for a message after its body
      *  has been parsed. Capped at ~64 KB of text per row — FTS5 stores the
      *  raw text and indexes tokens; we don't need every byte of a 1 MB
-     *  marketing email to find a word in the first paragraph. Called from
-     *  LocalStore.getMessage right after simpleParser succeeds. */
-    updateFtsBodyByUid(accountId, folderId, uid, bodyText) {
+     *  marketing email to find a word in the first paragraph.
+     *
+     *  Keyed on the FTS rowid (= messages.id) directly. The earlier
+     *  (account, folder_id, uid) lookup keyed on the LEGACY messages.folder_id
+     *  column — a message can live in several folders via message_folders,
+     *  so that lookup could miss the row entirely and silently leave the
+     *  body un-indexed (Bob 2026-05-18: "mcdade" in an opened message's
+     *  body, search found nothing). */
+    updateFtsBody(rowId, bodyText) {
+        if (!rowId)
+            return;
         try {
-            const row = this.db.prepare("SELECT id FROM messages WHERE account_id = ? AND folder_id = ? AND uid = ?").get(accountId, folderId, uid);
-            if (!row)
-                return;
             const capped = bodyText.length > 64_000 ? bodyText.slice(0, 64_000) : bodyText;
-            this.db.prepare("UPDATE messages_fts SET body_text = ? WHERE rowid = ?").run(capped, row.id);
+            this.db.prepare("UPDATE messages_fts SET body_text = ? WHERE rowid = ?").run(capped, rowId);
         }
         catch { /* FTS update is best-effort */ }
     }
@@ -1592,6 +1648,7 @@ export class MailxDB {
             flags: JSON.parse(r.flags_json),
             size: r.size,
             hasAttachments: !!r.has_attachments,
+            isReplied: !!r.is_replied,
             preview: r.preview,
             bodyPath: r.body_path || "",
             pending: !!r.pending,
@@ -1661,6 +1718,7 @@ export class MailxDB {
             flags: JSON.parse(r.flags_json),
             size: r.size,
             hasAttachments: !!r.has_attachments,
+            isReplied: !!r.is_replied,
             preview: r.preview,
             bodyPath: r.body_path || "",
             pending: !!r.pending,
@@ -1694,6 +1752,7 @@ export class MailxDB {
             flags: JSON.parse(r.flags_json),
             size: r.size,
             hasAttachments: !!r.has_attachments,
+            isReplied: !!r.is_replied,
             preview: r.preview,
             bodyPath: r.body_path || "",
             providerId: r.provider_id || undefined,
@@ -1828,6 +1887,45 @@ export class MailxDB {
     updateBodyPath(accountId, uid, bodyPath) {
         this.db.prepare("UPDATE messages SET body_path = ? WHERE account_id = ? AND uid = ?").run(bodyPath, accountId, uid);
     }
+    /** Write the post-download metadata in one go: body file path, parsed
+     *  attachment flag, preview text, and the parse timestamp. Called from
+     *  prefetch after the body has been written to disk and parsed — without
+     *  this, IMAP metadata-only sync leaves has_attachments=0 and preview=""
+     *  forever (the paperclip 📎 never renders for non-Gmail accounts).
+     *  Pass bodyPath = "" to update only the parsed fields (used by the
+     *  backfill worker re-parsing existing .eml files on disk). */
+    updateBodyMeta(accountId, uid, bodyPath, hasAttachments, preview) {
+        if (bodyPath) {
+            this.db.prepare("UPDATE messages SET body_path = ?, has_attachments = ?, preview = ?, body_parsed_at = ? WHERE account_id = ? AND uid = ?").run(bodyPath, hasAttachments ? 1 : 0, preview, Date.now(), accountId, uid);
+        }
+        else {
+            this.db.prepare("UPDATE messages SET has_attachments = ?, preview = ?, body_parsed_at = ? WHERE account_id = ? AND uid = ?").run(hasAttachments ? 1 : 0, preview, Date.now(), accountId, uid);
+        }
+    }
+    /** Find rows whose body is on disk but was never run through extractPreview
+     *  (has_attachments/preview reflect "no body source at sync time", typical
+     *  of the IMAP metadata-only path). Used by the backfill worker to walk
+     *  every existing message once after the prefetch-reparse fix landed.
+     *  Returns id + body_path so the worker doesn't need a second lookup. */
+    getMessagesNeedingBodyReparse(accountId, limit = 50) {
+        return this.db.prepare("SELECT id, uid, body_path AS bodyPath FROM messages WHERE account_id = ? AND body_path IS NOT NULL AND body_path != '' AND body_parsed_at IS NULL LIMIT ?").all(accountId, limit);
+    }
+    /** Backfill variant that writes by row id (avoids re-resolving uid). */
+    updateBodyMetaById(rowId, hasAttachments, preview) {
+        this.db.prepare("UPDATE messages SET has_attachments = ?, preview = ?, body_parsed_at = ? WHERE id = ?").run(hasAttachments ? 1 : 0, preview, Date.now(), rowId);
+    }
+    /** Mark a message as "replied to" by its RFC-5322 Message-ID. Source of
+     *  truth for the ↩ indicator: the local DB knows who replied to whom via
+     *  In-Reply-To headers, independent of whether the IMAP server propagates
+     *  \Answered (Gmail labels, cross-account replies, etc.). Called from the
+     *  send path when msg.inReplyTo is set, and from upsertMessage when a
+     *  reply arrives. Idempotent — UPDATE on an already-1 row is a no-op. */
+    markRepliedByMessageId(accountId, messageId) {
+        if (!messageId)
+            return false;
+        const r = this.db.prepare("UPDATE messages SET is_replied = 1 WHERE account_id = ? AND message_id = ? AND (is_replied IS NULL OR is_replied = 0)").run(accountId, messageId);
+        return r.changes > 0;
+    }
     /** Get messages without cached bodies (for background prefetch) */
     getMessagesWithoutBody(accountId, limit = 50) {
         // Prefetch order: smallest first, NULLs last, recent within tiebreak.
@@ -2650,6 +2748,7 @@ export class MailxDB {
                 flags: JSON.parse(r.flags_json),
                 size: r.size,
                 hasAttachments: !!r.has_attachments,
+                isReplied: !!r.is_replied,
                 preview: r.preview
             }));
             return { items, total, page, pageSize };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bobfrankston/mailx-store",
-  "version": "0.1.29",
+  "version": "0.1.31",
   "type": "module",
   "main": "index.js",
   "types": "index.d.ts",
@@ -9,8 +9,8 @@
   },
   "license": "ISC",
   "dependencies": {
-    "@bobfrankston/mailx-types": "^0.1.15",
-    "@bobfrankston/mailx-settings": "^0.1.18",
+    "@bobfrankston/mailx-types": "^0.1.18",
+    "@bobfrankston/mailx-settings": "^0.1.21",
     "@bobfrankston/mailx-bus": "^0.1.2",
     "mailparser": "^3.7.2"
   },
@@ -29,8 +29,8 @@
   },
   ".transformedSnapshot": {
     "dependencies": {
-      "@bobfrankston/mailx-types": "^0.1.15",
-      "@bobfrankston/mailx-settings": "^0.1.18",
+      "@bobfrankston/mailx-types": "^0.1.18",
+      "@bobfrankston/mailx-settings": "^0.1.21",
       "@bobfrankston/mailx-bus": "^0.1.2",
       "mailparser": "^3.7.2"
     }

package/store.js

@@ -305,9 +305,9 @@ export class Store {
         // backfill, searches miss any word that only appears deeper in the
         // body. Fire-and-forget — failures are non-fatal, never block the
         // user's preview render.
-        if (bodyText) {
+        if (bodyText && envelope.id) {
             try {
-                this.db.updateFtsBodyByUid(accountId, envelope.folderId, uid, bodyText);
+                this.db.updateFtsBody(envelope.id, bodyText);
             }
             catch { /* */ }
         }