@bobfrankston/rmfmail 1.1.166 → 1.1.169

package/bin/mailx.ts

@@ -20,8 +20,12 @@
  *   rmfmail -setup           Interactive first-time account setup (CLI)
  *   rmfmail -add             Add another account (CLI)
  *   rmfmail -test            Test IMAP/SMTP connectivity for all accounts
- *   rmfmail -rebuild         Wipe local cache, re-sync everything from IMAP
- *   rmfmail -repair          Re-sync metadata (fix corrupt subjects), keeps .eml files
+ *   rmfmail -rebuild         WIPE local DB + .eml files, then re-sync from IMAP
+ *                            (slow; first-sync caps at 200 newest per folder)
+ *   rmfmail -repair          DELETE DB rows, KEEP .eml files, re-sync from IMAP
+ *                            (subject/encoding fixes; same 200-newest cap)
+ *   rmfmail -recover         DELETE bobma DB rows, REBUILD index from .eml files
+ *                            on disk (no network; recovers everything you had)
  *   rmfmail -reauth          Clear cached OAuth tokens; next start re-consents
  *                            (use when new Google scopes have been added)
  *   rmfmail -log             Print log file path and exit
@@ -395,7 +399,7 @@ function pidIsMailx(pid: number): boolean {
 // on an old UI with no indication that the install has been upgraded.
 // Skip this logic for command-only flags (kill, rebuild, setup, ...) and for
 // the internal --daemon respawn.
-const __commandFlags = ["kill", "v", "version", "setup", "add", "test", "rebuild", "repair", "import", "log", "reauth"];
+const __commandFlags = ["kill", "v", "version", "setup", "add", "test", "rebuild", "repair", "recover", "import", "log", "reauth"];
 const __isCommandInvocation = process.argv.slice(2).some(a => __commandFlags.includes(a.replace(/^--?/, "")));
 // `--another` opts out of the replace-on-launch sweep so the user can run
 // multiple rmfmail instances side by side (testing, parallel sessions, etc.).
@@ -465,10 +469,11 @@ const addMode = hasFlag("add");
 const testMode = hasFlag("test");
 const rebuildMode = hasFlag("rebuild");
 const repairMode = hasFlag("repair");
+const recoverMode = hasFlag("recover");
 const importMode = hasFlag("import");
 
 // Validate arguments
-const knownFlags = ["verbose", "kill", "v", "version", "setup", "add", "test", "rebuild", "repair", "log", "import", "email", "mail", "daemon", "reauth", "mailto", "register-mailto", "unregister-mailto", "allow-elevated", "another", "server", "no-browser", "debug-server", "send", "account"];
+const knownFlags = ["verbose", "kill", "v", "version", "setup", "add", "test", "rebuild", "repair", "recover", "log", "import", "email", "mail", "daemon", "reauth", "mailto", "register-mailto", "unregister-mailto", "allow-elevated", "another", "server", "no-browser", "debug-server", "send", "account"];
 for (const arg of args) {
     // Strip a leading -/-- and any `=value` suffix before checking.
     const flag = arg.replace(/^--?/, "").split("=")[0];
@@ -489,8 +494,12 @@ for (const arg of args) {
             "    -setup                 Interactive first-time account setup (CLI)\n" +
             "    -add                   Add another account (CLI)\n" +
             "    -test                  Test IMAP/SMTP connectivity for all accounts\n" +
-            "    -rebuild               Wipe local cache, re-sync everything\n" +
-            "    -repair                Re-sync metadata (keeps .eml files)\n" +
+            "    -rebuild               WIPE DB + .eml files, re-sync from IMAP\n" +
+            "                           (cap 200 newest per folder on first-sync)\n" +
+            "    -repair                DELETE DB rows, KEEP .eml files, re-sync\n" +
+            "                           (same 200-newest cap; fixes corrupt fields)\n" +
+            "    -recover               REBUILD DB index from .eml files on disk\n" +
+            "                           (no network; restores everything you had)\n" +
             "    -reauth                Clear cached OAuth tokens; re-consent on next start\n" +
             "    -log                   Print log file path and exit\n" +
             "    -email <addr>          First-time setup with this email (skips prompt)\n" +
@@ -726,6 +735,176 @@ if (repairMode) {
     process.exit(0);
 }
 
+// Recover: walk .eml files on disk and rebuild DB index from them. No network.
+// Use case: data loss event wiped DB rows but mailxstore .eml files survived
+// (2026-05-27 UID-collision DELETE bug). Each .eml is parsed for Message-ID +
+// envelope fields and inserted as a placeholder row with a negative UID in
+// INBOX so the user can read/search them immediately. Subsequent IMAP sync
+// rebinds these placeholders to real (folder, uid) when the server confirms
+// each message — at which point the synthetic-UID row is replaced or merged
+// by the existing move-detect-via-Message-ID logic.
+//
+// Skips messages whose Message-ID is already indexed (idempotent re-run).
+if (recoverMode) {
+    const { getConfigDir, getStorePath } = await import("@bobfrankston/mailx-settings");
+    const dbDir = getConfigDir();
+    const storePath = getStorePath();
+    const dbPath = path.join(dbDir, "mailx.db");
+    if (!fs.existsSync(dbPath)) {
+        console.error("No database found. Run 'rmfmail -setup' first.");
+        process.exit(1);
+    }
+    if (!fs.existsSync(storePath)) {
+        console.error(`No mailxstore directory found at ${storePath}.`);
+        process.exit(1);
+    }
+
+    console.log("Recovering rmfmail index from .eml files on disk...");
+    console.log(`  Store: ${storePath}`);
+    console.log("  No network access; existing DB rows preserved.");
+
+    const { DatabaseSync } = await import("node:sqlite");
+    const db = new DatabaseSync(dbPath);
+    db.exec("PRAGMA journal_mode = WAL");
+
+    // Build the Message-ID set already in the DB so we can skip duplicates.
+    const seenStmt = db.prepare("SELECT message_id FROM messages WHERE account_id = ? AND message_id IS NOT NULL AND message_id != ''");
+    // INBOX folder per account (target for recovered rows). If no INBOX row
+    // exists for an account we skip that account entirely — we have nowhere
+    // to put the recovered messages and creating a folder row from a CLI is
+    // out of scope here.
+    const inboxStmt = db.prepare("SELECT id FROM folders WHERE account_id = ? AND (LOWER(special_use) = 'inbox' OR LOWER(path) = 'inbox') LIMIT 1");
+    // Synthetic UID allocator: start well below the smallest real IMAP UID
+    // and decrement, so they're distinct from existing real or synthetic UIDs.
+    let synthUid = -2_000_000;
+    const insertMsg = db.prepare(`
+        INSERT INTO messages
+            (account_id, folder_id, uid, message_id, date, subject,
+             from_address, from_name, to_json, cc_json, flags_json, size,
+             has_attachments, preview, body_path, cached_at)
+        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+    `);
+
+    // Walk every account directory under storePath.
+    const accountDirs = fs.readdirSync(storePath, { withFileTypes: true })
+        .filter(d => d.isDirectory())
+        .map(d => d.name);
+    let totalScanned = 0;
+    let totalIndexed = 0;
+    let totalSkipped = 0;
+    for (const acct of accountDirs) {
+        const inboxRow = inboxStmt.get(acct) as any;
+        if (!inboxRow) {
+            console.log(`  [skip] ${acct}: no INBOX folder row in DB — skipping ${fs.readdirSync(path.join(storePath, acct)).length} entries`);
+            continue;
+        }
+        const inboxFolderId: number = inboxRow.id;
+        const seen = new Set<string>();
+        for (const r of seenStmt.iterate(acct) as any) {
+            if (r.message_id) seen.add(String(r.message_id));
+        }
+        console.log(`  [${acct}] INBOX folder_id=${inboxFolderId}; already indexed: ${seen.size}`);
+
+        const acctRoot = path.join(storePath, acct);
+        let scanned = 0;
+        let indexed = 0;
+        let skipped = 0;
+
+        // Iterate two-level layout: acct/<2hex>/<uuid>.eml. Also tolerate
+        // flat or three-level layouts that older builds might've used.
+        const walkDir = (dir: string): void => {
+            for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
+                const p = path.join(dir, entry.name);
+                if (entry.isDirectory()) { walkDir(p); continue; }
+                if (!entry.isFile() || !entry.name.endsWith(".eml")) continue;
+                scanned++;
+                try {
+                    // Read only the headers — first ~8 KB is plenty. Faster
+                    // than slurping a 100 KB body when we only need fields.
+                    const fd = fs.openSync(p, "r");
+                    const buf = Buffer.alloc(8192);
+                    const n = fs.readSync(fd, buf, 0, 8192, 0);
+                    fs.closeSync(fd);
+                    const head = buf.slice(0, n).toString("utf-8");
+                    // Header section ends at the first blank line. Stop there
+                    // so a `Subject:` inside the body can't shadow the real one.
+                    const headerEnd = head.search(/\r?\n\r?\n/);
+                    const hdrText = headerEnd >= 0 ? head.slice(0, headerEnd) : head;
+                    const getHdr = (name: string): string => {
+                        const re = new RegExp(`^${name}:[ \\t]*(.*(?:\\r?\\n[ \\t]+.*)*)`, "im");
+                        const m = hdrText.match(re);
+                        return m ? m[1].replace(/\r?\n[ \t]+/g, " ").trim() : "";
+                    };
+                    const messageId = (getHdr("Message-ID") || getHdr("Message-Id")).replace(/^<|>$/g, "");
+                    if (!messageId) { skipped++; continue; }
+                    if (seen.has(messageId) || seen.has(`<${messageId}>`)) { skipped++; continue; }
+                    const subject = getHdr("Subject");
+                    const dateStr = getHdr("Date");
+                    const fromRaw = getHdr("From");
+                    const toRaw = getHdr("To");
+                    const ccRaw = getHdr("Cc");
+                    const date = dateStr ? (Date.parse(dateStr) || 0) : 0;
+                    // Crude address split — good enough for index reconstruction;
+                    // the daemon's normal upsert will overwrite with parsed data
+                    // once IMAP rebinds. We just need something to display.
+                    const parseAddrs = (raw: string): { name: string; address: string }[] => {
+                        if (!raw) return [];
+                        return raw.split(",").map(s => {
+                            const m = s.match(/^\s*"?([^"<]*?)"?\s*<([^>]+)>\s*$/) || s.match(/^\s*([^\s<>"]+@[^\s<>"]+)\s*$/);
+                            if (!m) return { name: "", address: s.trim() };
+                            return m[2] ? { name: m[1].trim(), address: m[2].trim() } : { name: "", address: m[1].trim() };
+                        });
+                    };
+                    const fromAddrs = parseAddrs(fromRaw);
+                    const from = fromAddrs[0] || { name: "", address: "" };
+                    const toList = parseAddrs(toRaw);
+                    const ccList = parseAddrs(ccRaw);
+                    const stat = fs.statSync(p);
+                    const rel = path.relative(storePath, p).replace(/\\/g, "/");
+                    insertMsg.run(
+                        acct,
+                        inboxFolderId,
+                        synthUid--,
+                        messageId,
+                        date,
+                        subject,
+                        from.address,
+                        from.name,
+                        JSON.stringify(toList),
+                        JSON.stringify(ccList),
+                        "[]",                       // flags_json
+                        stat.size,                  // size
+                        0,                          // has_attachments — leave as 0; daemon refreshes on first view
+                        "",                         // preview — empty, daemon will fill on first view
+                        rel,                        // body_path (relative to storePath)
+                        Date.now(),                 // cached_at
+                    );
+                    seen.add(messageId);
+                    indexed++;
+                    if (indexed % 500 === 0) console.log(`  [${acct}] ${indexed} indexed (${scanned} scanned)`);
+                } catch (e: any) {
+                    skipped++;
+                    if (skipped < 5) console.error(`  [${acct}] parse error on ${p}: ${e?.message || e}`);
+                }
+            }
+        };
+        walkDir(acctRoot);
+        // Refresh folder counts so the INBOX row shows the new total.
+        const cnt = (db.prepare("SELECT COUNT(*) as c FROM messages WHERE folder_id = ?").get(inboxFolderId) as any).c;
+        db.prepare("UPDATE folders SET total_count = ? WHERE id = ?").run(cnt, inboxFolderId);
+        console.log(`  [${acct}] done — scanned ${scanned}, indexed ${indexed}, skipped ${skipped}. INBOX total now ${cnt}.`);
+        totalScanned += scanned;
+        totalIndexed += indexed;
+        totalSkipped += skipped;
+    }
+    db.close();
+    console.log(`Recovery complete: indexed ${totalIndexed} of ${totalScanned} .eml files (${totalSkipped} skipped).`);
+    console.log("  Recovered rows have synthetic negative UIDs and live in INBOX.");
+    console.log("  Run 'rmfmail' to start the daemon — normal IMAP sync will rebind");
+    console.log("  these placeholders to real (folder, uid) tuples by Message-ID.");
+    process.exit(0);
+}
+
 // Import accounts from a local file into GDrive
 if (importMode) {
     const importPath = args.find(a => !a.startsWith("-"));

package/client/android-bootstrap.bundle.js

@@ -5233,11 +5233,13 @@ var WebMailxDB = class {
   getUidsForFolder(accountId, folderId) {
     return this.all("SELECT uid FROM messages WHERE account_id = ? AND folder_id = ?", [accountId, folderId]).map((r) => r.uid);
   }
-  deleteMessage(accountId, uid) {
-    const msg = this.get("SELECT folder_id FROM messages WHERE account_id = ? AND uid = ?", [accountId, uid]);
-    this.run("DELETE FROM messages WHERE account_id = ? AND uid = ?", [accountId, uid]);
-    if (msg)
-      this.recalcFolderCounts(msg.folder_id);
+  /** folderId REQUIRED. See main mailx-store/db.ts for why: IMAP UIDs are
+   *  per-folder, so DELETE WHERE (account, uid) would wipe the same UID
+   *  out of every folder. The pre-2026-05-27 implementation took only
+   *  (accountId, uid) — that was the 70k-row data-loss bug. */
+  deleteMessage(accountId, folderId, uid) {
+    this.run("DELETE FROM messages WHERE account_id = ? AND folder_id = ? AND uid = ?", [accountId, folderId, uid]);
+    this.recalcFolderCounts(folderId);
   }
   recalcFolderCounts(folderId) {
     const counts = this.get(`SELECT COUNT(*) as total,
@@ -9887,7 +9889,7 @@ var AndroidSyncManager = class {
           console.log(`[sync] ${folder.path}: reconcile refused \u2014 would delete ${toDelete.length}/${localUids.length} (${Math.round(toDelete.length / localUids.length * 100)}%)`);
         } else {
           for (const uid of toDelete) {
-            this.db.deleteMessage(accountId, uid);
+            this.db.deleteMessage(accountId, folderId, uid);
             this.bodyStore.deleteMessage(accountId, folderId, uid).catch(() => {
             });
           }
@@ -9999,7 +10001,7 @@ var AndroidSyncManager = class {
     emitEvent({ type: "folderCountsChanged", accountId, counts: {} });
   }
   async trashMessage(accountId, folderId, uid) {
-    this.db.deleteMessage(accountId, uid);
+    this.db.deleteMessage(accountId, folderId, uid);
     this.db.queueSyncAction(accountId, "trash", uid, folderId);
     emitEvent({ type: "messageDeleted", accountId, folderId, uid });
     emitEvent({ type: "folderCountsChanged", accountId, counts: {} });