@bobfrankston/mailx 1.0.360 → 1.0.362

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bobfrankston/mailx",
-  "version": "1.0.360",
+  "version": "1.0.362",
   "description": "Local-first email client with IMAP sync and standalone native app",
   "type": "module",
   "main": "bin/mailx.js",

package/packages/mailx-imap/index.d.ts

@@ -177,6 +177,10 @@ export declare class ImapManager extends EventEmitter {
      *  The persistent fetchClient can only handle one command at a time (IMAP protocol limitation). */
     private fetchQueues;
     /** Serialize body fetch operations per account — prevents concurrent IMAP commands on same connection */
+    /** Unlink the on-disk body file for a message by reading its `body_path`
+     *  from the DB. Safe to call either before or after `db.deleteMessage`
+     *  — read body_path first, store it, then unlink whenever. */
+    private unlinkBodyFile;
     private enqueueFetch;
     /** Fetch a single message body on demand, caching in the store.
      *  Uses its own fresh connection — never blocked by background prefetch. */

package/packages/mailx-imap/index.js

@@ -689,6 +689,14 @@ export class ImapManager extends EventEmitter {
                 }
                 if (msg.uid <= highestUid)
                     continue; // already have it
+                // Tombstone check: if the user locally deleted this Message-ID,
+                // don't re-import it. Server-side EXPUNGE may lag, or reconcile
+                // may find the message in an old list snapshot. Without this,
+                // deleted messages reappear on the next sync pass.
+                if (msg.messageId && this.db.hasTombstone(accountId, msg.messageId)) {
+                    console.log(`  [tombstone] ${accountId}: skipping ${msg.messageId} (locally deleted)`);
+                    continue;
+                }
                 const source = msg.source || "";
                 let bodyPath = "";
                 let preview = "";
@@ -931,8 +939,9 @@ export class ImapManager extends EventEmitter {
                 const localUids = this.db.getUidsForFolder(accountId, folderId);
                 for (const uid of localUids) {
                     if (!serverUids.has(uid)) {
+                        // Read body_path BEFORE deleting the row, then unlink.
+                        this.unlinkBodyFile(accountId, uid, folderId).catch(() => { });
                         this.db.deleteMessage(accountId, uid);
-                        this.bodyStore.deleteMessage(accountId, folderId, uid).catch(() => { });
                         deletedCount++;
                     }
                 }
@@ -1265,8 +1274,8 @@ export class ImapManager extends EventEmitter {
                 }
                 else {
                     for (const uid of toDelete) {
+                        this.unlinkBodyFile(accountId, uid, folder.id).catch(() => { });
                         this.db.deleteMessage(accountId, uid);
-                        this.bodyStore.deleteMessage(accountId, folder.id, uid).catch(() => { });
                     }
                     if (toDelete.length > 0)
                         console.log(`  [api] ${accountId}/${folder.path}: ${toDelete.length} deleted`);
@@ -1581,6 +1590,17 @@ export class ImapManager extends EventEmitter {
             this.syncIntervals.set("prefetch", prefetchInterval);
             console.log(`  [periodic] body prefetch every 60s (independent of sync)`);
         }
+        // Tombstone prune: age out local-delete records older than 30 days.
+        // Runs hourly — cheap (one indexed DELETE).
+        const TOMBSTONE_RETENTION_DAYS = 30;
+        const pruneTombstones = () => {
+            const cutoff = Date.now() - TOMBSTONE_RETENTION_DAYS * 86400_000;
+            const n = this.db.pruneTombstones(cutoff);
+            if (n > 0)
+                console.log(`  [tombstones] pruned ${n} older than ${TOMBSTONE_RETENTION_DAYS} days`);
+        };
+        setTimeout(pruneTombstones, 30_000); // first run after startup settles
+        this.syncIntervals.set("tombstone-prune", setInterval(pruneTombstones, 3600_000));
         // Full sync (all folders + IDLE restart) at configured interval
         const fullInterval = setInterval(async () => {
             console.log(`  [periodic] Full sync at ${new Date().toLocaleTimeString()}`);
@@ -1641,6 +1661,18 @@ export class ImapManager extends EventEmitter {
      *  The persistent fetchClient can only handle one command at a time (IMAP protocol limitation). */
     fetchQueues = new Map();
     /** Serialize body fetch operations per account — prevents concurrent IMAP commands on same connection */
+    /** Unlink the on-disk body file for a message by reading its `body_path`
+     *  from the DB. Safe to call either before or after `db.deleteMessage`
+     *  — read body_path first, store it, then unlink whenever. */
+    async unlinkBodyFile(accountId, uid, folderId) {
+        try {
+            const row = this.db.getMessageByUid(accountId, uid, folderId);
+            const p = row?.bodyPath;
+            if (p)
+                await this.bodyStore.unlinkByPath(p);
+        }
+        catch { /* row already gone / file already gone — both fine */ }
+    }
     enqueueFetch(accountId, fn) {
         const prev = this.fetchQueues.get(accountId) || Promise.resolve();
         const next = prev.then(fn, fn); // run fn after previous completes (regardless of success/failure)
@@ -1651,48 +1683,15 @@ export class ImapManager extends EventEmitter {
     /** Fetch a single message body on demand, caching in the store.
      *  Uses its own fresh connection — never blocked by background prefetch. */
     async fetchMessageBody(accountId, folderId, uid) {
-        // Already cached? If the file is on disk but body_path wasn't written to
-        // the DB (e.g. from an interrupted earlier run), the prefetch loop would
-        // otherwise keep returning the same missing rows forever — once saw
-        // "gmail: 17266796 bodies cached" in the logs, which is the counter
-        // spinning on the same 100 rows.
-        if (await this.bodyStore.hasMessage(accountId, folderId, uid)) {
-            // COMINGLING GUARD: verify the cached body's Message-ID matches the
-            // DB row's messageId. If UIDVALIDITY changed server-side (mailbox
-            // recreated, server quirk) the same integer UID can point at a
-            // different message — the on-disk .eml becomes stale but hasMessage()
-            // still returns true. User-reported: "Peter Hoddie letter comingled
-            // with a much older letter." Check fixes it regardless of root cause.
-            const cached = await this.bodyStore.getMessage(accountId, folderId, uid);
-            const envelope = this.db.getMessageByUid(accountId, uid, folderId);
-            const expectedId = envelope?.messageId || "";
-            if (expectedId) {
-                // Scan headers only — Message-ID should land in the first few KB.
-                const head = cached.subarray(0, Math.min(cached.length, 16 * 1024)).toString("utf-8");
-                const m = head.match(/^Message-ID:\s*<([^>\r\n]+)>/im);
-                const cachedId = m ? `<${m[1]}>` : "";
-                if (cachedId && expectedId && cachedId !== expectedId) {
-                    console.error(`  [body] COMINGLING DETECTED ${accountId}/${folderId}/${uid}: expected ${expectedId}, cached ${cachedId} — dropping cache, re-fetching`);
-                    try {
-                        await this.bodyStore.deleteMessage(accountId, folderId, uid);
-                    }
-                    catch { /* */ }
-                    // fall through to re-fetch path
-                }
-                else {
-                    const existingPath = this.bodyStore.getMessagePath?.(accountId, folderId, uid);
-                    if (existingPath)
-                        this.db.updateBodyPath(accountId, uid, existingPath);
-                    return cached;
-                }
-            }
-            else {
-                // No messageId on the DB row (shouldn't happen but be permissive).
-                const existingPath = this.bodyStore.getMessagePath?.(accountId, folderId, uid);
-                if (existingPath)
-                    this.db.updateBodyPath(accountId, uid, existingPath);
-                return cached;
-            }
+        // Already cached? Read the DB row's `body_path` and check the file
+        // exists there. No more `(folderId, uid)` path reconstruction — that
+        // was the source of the S49 comingling bug (UID reuse + folder move
+        // pointing two messages at one file). `body_path` is the sole
+        // authority on where a given message's body lives on disk.
+        const envelope = this.db.getMessageByUid(accountId, uid, folderId);
+        const storedPath = envelope?.bodyPath || "";
+        if (storedPath && await this.bodyStore.hasByPath(storedPath)) {
+            return this.bodyStore.readByPath(storedPath);
         }
         if (!this.configs.has(accountId))
             return null;
@@ -1948,8 +1947,8 @@ export class ImapManager extends EventEmitter {
                                 if (received.has(uid))
                                     continue;
                                 try {
+                                    this.unlinkBodyFile(accountId, uid, folderId).catch(() => { });
                                     this.db.deleteMessage(accountId, uid);
-                                    this.bodyStore.deleteMessage(accountId, folderId, uid).catch(() => { });
                                     counters.deleted++;
                                     madeProgress = true;
                                 }
@@ -2032,8 +2031,8 @@ export class ImapManager extends EventEmitter {
                                 if (received.has(uid))
                                     continue;
                                 try {
+                                    this.unlinkBodyFile(accountId, uid, folderId).catch(() => { });
                                     this.db.deleteMessage(accountId, uid);
-                                    this.bodyStore.deleteMessage(accountId, folderId, uid).catch(() => { });
                                     counters.deleted++;
                                     madeProgress = true;
                                 }
@@ -2077,8 +2076,8 @@ export class ImapManager extends EventEmitter {
         const trash = this.findFolder(accountId, "trash");
         // Local first — remove all from DB immediately
         for (const msg of messages) {
+            this.unlinkBodyFile(accountId, msg.uid, msg.folderId).catch(() => { });
             this.db.deleteMessage(accountId, msg.uid);
-            this.bodyStore.deleteMessage(accountId, msg.folderId, msg.uid).catch(() => { });
         }
         console.log(`  Deleted ${messages.length} messages locally`);
         // Queue IMAP actions
@@ -2130,8 +2129,8 @@ export class ImapManager extends EventEmitter {
     async trashMessage(accountId, folderId, uid) {
         const trash = this.findFolder(accountId, "trash");
         // Local first — remove from DB immediately
+        this.unlinkBodyFile(accountId, uid, folderId).catch(() => { });
         this.db.deleteMessage(accountId, uid);
-        this.bodyStore.deleteMessage(accountId, folderId, uid).catch(() => { });
         // Queue IMAP action + log the resolution so "I deleted a message and
         // now it's in neither trash nor deleted" is diagnosable from the log.
         if (trash && trash.id !== folderId) {

package/packages/mailx-service/index.js

@@ -646,6 +646,11 @@ export class MailxService {
         const envelope = this.db.getMessageByUid(accountId, uid);
         if (!envelope)
             throw new Error("Message not found");
+        // Tombstone the Message-ID so a subsequent sync pass can't resurrect
+        // the row if the server's EXPUNGE hasn't propagated yet. `undelete`
+        // removes the tombstone.
+        if (envelope.messageId)
+            this.db.addTombstone(accountId, envelope.messageId, envelope.subject || "");
         await this.imapManager.trashMessage(accountId, envelope.folderId, envelope.uid);
     }
     async deleteMessages(accountId, uids) {
@@ -653,6 +658,9 @@ export class MailxService {
             const env = this.db.getMessageByUid(accountId, uid);
             if (!env)
                 return null;
+            // Tombstone each — same reason as single-delete above.
+            if (env.messageId)
+                this.db.addTombstone(accountId, env.messageId, env.subject || "");
             return { uid: env.uid, folderId: env.folderId };
         }).filter(m => m !== null);
         await this.imapManager.trashMessages(accountId, messages);
@@ -711,6 +719,12 @@ export class MailxService {
         return { targetFolderId: target.id, moved: uids.length };
     }
     async undeleteMessage(accountId, uid, folderId) {
+        // Clear the tombstone first so a subsequent sync can re-import if
+        // the server still has the row. Messages with no Message-ID just
+        // didn't get a tombstone — this is a no-op for them.
+        const envelope = this.db.getMessageByUid(accountId, uid, folderId);
+        if (envelope?.messageId)
+            this.db.removeTombstone(accountId, envelope.messageId);
         await this.imapManager.undeleteMessage(accountId, uid, folderId);
     }
     async deleteOnServer(accountId, folderPath, uid) {

package/packages/mailx-store/db.d.ts

@@ -12,6 +12,20 @@ export declare class MailxDB {
     hasSentMessage(messageId: string): boolean;
     /** Record a successfully sent message so future attempts are skipped. */
     recordSent(messageId: string, accountId: string, subject: string, recipients: string[]): void;
+    /** Mark a Message-ID as locally-deleted for an account. No-op if messageId
+     *  is empty (e.g. provider stripped the header) — without a stable id we
+     *  can't check against future sync results anyway. */
+    addTombstone(accountId: string, messageId: string, subject?: string): void;
+    /** Is this Message-ID tombstoned for this account? */
+    hasTombstone(accountId: string, messageId: string): boolean;
+    /** Remove a tombstone — used by "undelete" (Ctrl-Z) so a subsequent sync
+     *  re-imports the message as normal. Also lets the user recover from a
+     *  mistaken local delete. */
+    removeTombstone(accountId: string, messageId: string): void;
+    /** Age-out tombstones older than the given cutoff. Keeps the table from
+     *  growing unboundedly. Default retention is 30 days; caller passes the
+     *  actual cutoff in ms since epoch. */
+    pruneTombstones(olderThanMs: number): number;
     /** Idempotently add a column to a table if it's missing. */
     private addColumnIfMissing;
     /** Compute a thread id for an incoming message. Strategy:

package/packages/mailx-store/db.js

@@ -126,6 +126,21 @@ const SCHEMA = `
         last_error TEXT,
         UNIQUE(account_id, action, uid, folder_id)
     );
+
+    -- Tombstones: messages the user deleted locally. Sync checks this table
+    -- before inserting a new row so a server-side delete that hasn't yet
+    -- propagated (or a stale server listing during the EXPUNGE race) can't
+    -- resurrect a message the user already removed. Keyed by Message-ID
+    -- because that's the only identifier stable across UID renumbers,
+    -- UIDVALIDITY bumps, and cross-folder moves.
+    CREATE TABLE IF NOT EXISTS tombstones (
+        account_id TEXT NOT NULL,
+        message_id TEXT NOT NULL,
+        deleted_at INTEGER NOT NULL,
+        subject TEXT DEFAULT '',
+        PRIMARY KEY (account_id, message_id)
+    );
+    CREATE INDEX IF NOT EXISTS idx_tombstones_deleted_at ON tombstones(deleted_at);
 `;
 export class MailxDB {
     db;
@@ -172,6 +187,53 @@ export class MailxDB {
             console.error(`  [sent_log] failed to record ${messageId}: ${e.message}`);
         }
     }
+    // ── Tombstones (local-delete record so server echo can't resurrect) ──
+    /** Mark a Message-ID as locally-deleted for an account. No-op if messageId
+     *  is empty (e.g. provider stripped the header) — without a stable id we
+     *  can't check against future sync results anyway. */
+    addTombstone(accountId, messageId, subject = "") {
+        if (!messageId)
+            return;
+        try {
+            this.db.prepare("INSERT INTO tombstones (account_id, message_id, deleted_at, subject) VALUES (?, ?, ?, ?) ON CONFLICT(account_id, message_id) DO UPDATE SET deleted_at = excluded.deleted_at").run(accountId, messageId, Date.now(), subject || "");
+        }
+        catch (e) {
+            console.error(`  [tombstones] failed to record ${messageId}: ${e.message}`);
+        }
+    }
+    /** Is this Message-ID tombstoned for this account? */
+    hasTombstone(accountId, messageId) {
+        if (!messageId)
+            return false;
+        const row = this.db.prepare("SELECT 1 FROM tombstones WHERE account_id = ? AND message_id = ? LIMIT 1").get(accountId, messageId);
+        return !!row;
+    }
+    /** Remove a tombstone — used by "undelete" (Ctrl-Z) so a subsequent sync
+     *  re-imports the message as normal. Also lets the user recover from a
+     *  mistaken local delete. */
+    removeTombstone(accountId, messageId) {
+        if (!messageId)
+            return;
+        try {
+            this.db.prepare("DELETE FROM tombstones WHERE account_id = ? AND message_id = ?").run(accountId, messageId);
+        }
+        catch (e) {
+            console.error(`  [tombstones] failed to remove ${messageId}: ${e.message}`);
+        }
+    }
+    /** Age-out tombstones older than the given cutoff. Keeps the table from
+     *  growing unboundedly. Default retention is 30 days; caller passes the
+     *  actual cutoff in ms since epoch. */
+    pruneTombstones(olderThanMs) {
+        try {
+            const res = this.db.prepare("DELETE FROM tombstones WHERE deleted_at < ?").run(olderThanMs);
+            return Number(res.changes || 0);
+        }
+        catch (e) {
+            console.error(`  [tombstones] prune failed: ${e.message}`);
+            return 0;
+        }
+    }
     /** Idempotently add a column to a table if it's missing. */
     addColumnIfMissing(table, column, sqlType) {
         try {

package/packages/mailx-store/file-store.d.ts

@@ -1,18 +1,36 @@
 /**
  * File-per-message body storage backend.
- * Messages stored as: {basePath}/{accountId}/{folderId}/{uid}.eml
- * Folder IDs are numeric (from SQLite), not names.
+ *
+ * Disk layout: {basePath}/{accountId}/<xx>/<uuid>.eml
+ * Filename is an opaque UUID. Two-char prefix dir for filesystem fan-out.
+ *
+ * CRITICAL: the on-disk filename carries NO semantic meaning — not folder
+ * id, not UID, not Message-ID. A new UUID is minted on every `putMessage`.
+ * Moves, UID renumbers, UIDVALIDITY bumps cannot shadow a body because no
+ * filename is ever reused. DB's `body_path` column is the sole authority
+ * on where a given message's body lives.
  */
 import type { MessageStore } from "@bobfrankston/mailx-types";
 export declare class FileMessageStore implements MessageStore {
     private basePath;
     constructor(basePath: string);
-    private messagePath;
-    /** Public lookup of the on-disk path without touching the file. */
-    getMessagePath(accountId: string, folderId: number, uid: number): string;
-    putMessage(accountId: string, folderId: number, uid: number, raw: Buffer): Promise<string>;
-    getMessage(accountId: string, folderId: number, uid: number): Promise<Buffer>;
-    deleteMessage(accountId: string, folderId: number, uid: number): Promise<void>;
-    hasMessage(accountId: string, folderId: number, uid: number): Promise<boolean>;
+    /** Fresh opaque path per call. No inputs from the caller affect the name. */
+    private newMessagePath;
+    /** Verify a given path resolves inside this store's basePath — refuses
+     *  any value that doesn't (cheap directory-traversal guard). */
+    private inStore;
+    /** Write a new body. Always a fresh UUID path. Caller MUST persist the
+     *  returned path in the DB (`body_path`) and use it for all reads. The
+     *  (folderId, uid) args are kept for interface compatibility; they do
+     *  NOT affect the filename. */
+    putMessage(accountId: string, _folderId: number, _uid: number, raw: Buffer): Promise<string>;
+    /** Read by absolute path (DB `body_path`). The primary read API. */
+    readByPath(fullPath: string): Promise<Buffer>;
+    hasByPath(fullPath: string): Promise<boolean>;
+    unlinkByPath(fullPath: string): Promise<void>;
+    getMessagePath(_accountId: string, _folderId: number, _uid: number): string;
+    getMessage(_accountId: string, _folderId: number, _uid: number): Promise<Buffer>;
+    hasMessage(_accountId: string, _folderId: number, _uid: number): Promise<boolean>;
+    deleteMessage(_accountId: string, _folderId: number, _uid: number): Promise<void>;
 }
 //# sourceMappingURL=file-store.d.ts.map

package/packages/mailx-store/file-store.js

@@ -1,39 +1,80 @@
 /**
  * File-per-message body storage backend.
- * Messages stored as: {basePath}/{accountId}/{folderId}/{uid}.eml
- * Folder IDs are numeric (from SQLite), not names.
+ *
+ * Disk layout: {basePath}/{accountId}/<xx>/<uuid>.eml
+ * Filename is an opaque UUID. Two-char prefix dir for filesystem fan-out.
+ *
+ * CRITICAL: the on-disk filename carries NO semantic meaning — not folder
+ * id, not UID, not Message-ID. A new UUID is minted on every `putMessage`.
+ * Moves, UID renumbers, UIDVALIDITY bumps cannot shadow a body because no
+ * filename is ever reused. DB's `body_path` column is the sole authority
+ * on where a given message's body lives.
  */
 import * as fs from "node:fs";
 import * as path from "node:path";
+import { randomUUID } from "node:crypto";
 export class FileMessageStore {
     basePath;
     constructor(basePath) {
         this.basePath = basePath;
         fs.mkdirSync(basePath, { recursive: true });
     }
-    messagePath(accountId, folderId, uid) {
-        return path.join(this.basePath, accountId, String(folderId), `${uid}.eml`);
+    /** Fresh opaque path per call. No inputs from the caller affect the name. */
+    newMessagePath(accountId) {
+        const uuid = randomUUID().replace(/-/g, "");
+        const prefix = uuid.slice(0, 2);
+        return path.join(this.basePath, accountId, prefix, `${uuid}.eml`);
     }
-    /** Public lookup of the on-disk path without touching the file. */
-    getMessagePath(accountId, folderId, uid) {
-        return this.messagePath(accountId, folderId, uid);
+    /** Verify a given path resolves inside this store's basePath — refuses
+     *  any value that doesn't (cheap directory-traversal guard). */
+    inStore(fullPath) {
+        if (!fullPath)
+            return false;
+        const rel = path.relative(path.resolve(this.basePath), path.resolve(fullPath));
+        return !rel.startsWith("..") && !path.isAbsolute(rel);
     }
-    async putMessage(accountId, folderId, uid, raw) {
-        const filePath = this.messagePath(accountId, folderId, uid);
+    /** Write a new body. Always a fresh UUID path. Caller MUST persist the
+     *  returned path in the DB (`body_path`) and use it for all reads. The
+     *  (folderId, uid) args are kept for interface compatibility; they do
+     *  NOT affect the filename. */
+    async putMessage(accountId, _folderId, _uid, raw) {
+        const filePath = this.newMessagePath(accountId);
         fs.mkdirSync(path.dirname(filePath), { recursive: true });
         fs.writeFileSync(filePath, raw);
         return filePath;
     }
-    async getMessage(accountId, folderId, uid) {
-        return fs.readFileSync(this.messagePath(accountId, folderId, uid));
+    /** Read by absolute path (DB `body_path`). The primary read API. */
+    async readByPath(fullPath) {
+        if (!this.inStore(fullPath))
+            throw new Error(`refusing to read outside store: ${fullPath}`);
+        return fs.readFileSync(fullPath);
     }
-    async deleteMessage(accountId, folderId, uid) {
-        const filePath = this.messagePath(accountId, folderId, uid);
-        if (fs.existsSync(filePath))
-            fs.unlinkSync(filePath);
+    async hasByPath(fullPath) {
+        if (!this.inStore(fullPath))
+            return false;
+        return fs.existsSync(fullPath);
     }
-    async hasMessage(accountId, folderId, uid) {
-        return fs.existsSync(this.messagePath(accountId, folderId, uid));
+    async unlinkByPath(fullPath) {
+        if (!this.inStore(fullPath))
+            return;
+        if (fs.existsSync(fullPath))
+            fs.unlinkSync(fullPath);
+    }
+    // MessageStore interface compatibility (unused once all callers migrate to
+    // path-based reads). These used to compose {folderId}/{uid}.eml and they
+    // would resurrect the comingling bug if restored. Kept as throwing stubs
+    // so any accidental caller surfaces loudly rather than silently misbehave.
+    getMessagePath(_accountId, _folderId, _uid) {
+        throw new Error("FileMessageStore.getMessagePath is retired — read body_path from DB");
+    }
+    async getMessage(_accountId, _folderId, _uid) {
+        throw new Error("FileMessageStore.getMessage(folder,uid) is retired — use readByPath(body_path)");
+    }
+    async hasMessage(_accountId, _folderId, _uid) {
+        throw new Error("FileMessageStore.hasMessage(folder,uid) is retired — use hasByPath(body_path)");
+    }
+    async deleteMessage(_accountId, _folderId, _uid) {
+        throw new Error("FileMessageStore.deleteMessage(folder,uid) is retired — use unlinkByPath(body_path)");
     }
 }
 //# sourceMappingURL=file-store.js.map