@bobfrankston/mailx-store 0.1.53 → 0.1.55

package/db.d.ts

@@ -435,6 +435,20 @@ export declare class MailxDB {
      *  here but NOT on the server gets its membership dropped (which
      *  may then GC the messages row if it has no other folders). */
     getUidsForFolder(_accountId: string, folderId: number): number[];
+    /** UIDs we actually have a stored MESSAGE ROW for in this folder — from the
+     *  `messages` table, NOT `message_folders`. The two can diverge: an "orphan
+     *  membership" (a message_folders row whose messages row was never written /
+     *  got deleted) makes getUidsForFolder report a UID we don't truly have.
+     *  The set-diff backfill must diff against THIS so it re-fetches the real
+     *  message data for orphans instead of treating them as present (Bob
+     *  2026-06-20: ~1k old INBOX messages stuck missing — membership said
+     *  "have it", the row didn't exist). */
+    getStoredUids(accountId: string, folderId: number): number[];
+    /** Count of stored MESSAGE ROWS in a folder (messages table). The deficit
+     *  gate compares this to the server's count to decide whether to reconcile;
+     *  using the messages table (not message_folders) means orphan memberships
+     *  don't mask a real "rows are missing" deficit. */
+    getStoredCount(accountId: string, folderId: number): number;
     /** List recent rows in a folder for a Sent-sweep / reconciliation pass.
      *  Returns only the columns the sweeper needs (uid, message_id, date,
      *  subject, size, has_attachments, preview, body_path) for every row in
@@ -559,7 +573,7 @@ export declare class MailxDB {
      *  click, queued behind it; that IS the "loading body takes forever"
      *  bug). Now keyset-paginated by `m.id` with a `setImmediate` yield
      *  between pages, so user IPC lands in the gaps. */
-    seedContactsFromMessages(): Promise<number>;
+    seedContactsFromMessages(force?: boolean): Promise<number>;
     /** Apply the contents of contacts.jsonc — replaces all preferred-tier rows
      *  with the entries in `preferred[]`, merges `discovered[]` into the local
      *  cache, sets the in-memory denylist, and purges any discovered rows

package/db.js

@@ -633,6 +633,10 @@ export class MailxDB {
         catch (e) {
             console.error(`  [db] membership collapse failed: ${e?.message || e}`);
         }
+        // (The per-boot cross-folder collapse for "exclusive" accounts was
+        // REMOVED 2026-06-16 — it deleted intentional copies. Cross-folder stale
+        // memberships are now cleaned by server-truth reconcile, which keeps real
+        // copies. See upsertMessageFolder's note.)
         // One-shot cleanup: the retired insertOptimisticSentRow path wrote
         // synthetic-negative-UID rows into Sent. Those rows are stale (the
         // real server-synced row eventually appears with a positive UID),
@@ -1495,27 +1499,24 @@ export class MailxDB {
      *  (when the server stops listing a UID in a folder). */
     upsertMessageFolder(messageRowId, folderId, uid, exclusive = false) {
         const now = Date.now();
-        if (exclusive) {
-            // EXCLUSIVE-folder account (IMAP — Bob's iecc): a message lives in
-            // exactly ONE folder. Drop EVERY other membership for this message
-            // before recording the current one; the folder that most recently
-            // saw it (this sync) is its real location. This both prevents the
-            // cross-folder "moved but still in source" accretion (Bob 2026-06-16
-            // "colleagues still in main folder") and self-heals the backlog as
-            // each folder syncs — only the folder that genuinely holds the
-            // message on the server will ever see+set it. NOT used for Gmail,
-            // where one message legitimately carries several label-folders.
-            this.db.prepare("DELETE FROM message_folders WHERE message_row_id = ? AND NOT (folder_id = ? AND uid = ?)").run(messageRowId, folderId, uid);
-        }
-        else {
-            // INVARIANT (all accounts): a message lives at exactly ONE uid per
-            // folder. UNIQUE(folder_id, uid) only stops the SAME (folder,uid)
-            // dup'ing — not the SAME message gaining N rows in ONE folder under N
-            // different uids (move-detect re-binding to fresh uids each sync →
-            // msg 7 "Re: FYI" had 458 Sent rows). Drop any OTHER uid this message
-            // held in THIS folder before inserting the new one.
-            this.db.prepare("DELETE FROM message_folders WHERE message_row_id = ? AND folder_id = ? AND uid != ?").run(messageRowId, folderId, uid);
-        }
+        // NOTE: `exclusive` (drop-all-other-folder-memberships) was REVERTED
+        // 2026-06-16. It deletes INTENTIONAL copies — Bob copies messages into
+        // multiple folders, so the same Message-ID legitimately lives in several
+        // places, and exclusivity would flip-flop a real copy between its folders
+        // every sync. Cross-folder stale-move-source memberships are cleaned by
+        // server-truth RECONCILE (the folder's sync drops uids the server no
+        // longer lists) — which preserves copies (server confirms both) while
+        // dropping accidents (server confirms one). The param is kept for the
+        // signature but no longer drops cross-folder rows.
+        void exclusive;
+        // INVARIANT (all accounts): a message lives at exactly ONE uid per
+        // folder. UNIQUE(folder_id, uid) only stops the SAME (folder,uid)
+        // dup'ing — not the SAME message gaining N rows in ONE folder under N
+        // different uids (move-detect re-binding to fresh uids each sync →
+        // msg 7 "Re: FYI" had 458 Sent rows). Drop any OTHER uid this message
+        // held in THIS folder before inserting the new one. (Same-folder only —
+        // never touches other folders, so copies survive.)
+        this.db.prepare("DELETE FROM message_folders WHERE message_row_id = ? AND folder_id = ? AND uid != ?").run(messageRowId, folderId, uid);
         // INSERT OR REPLACE on (folder_id, uid) — if some other message_row_id
         // somehow had this slot, replace it. In practice this happens after
         // an EXPUNGE+reinsert: a UID gets reused by the server for a different
@@ -1710,26 +1711,17 @@ export class MailxDB {
         // stacks) that point at the UUID. Only kicks in when messageId is
         // present (servers usually include it; if not we fall through to a
         // fresh insert which mints a new UUID).
+        //
+        // NOTE (2026-06-16): the instance-model refactor (docs/instance-model-plan.md)
+        // will REPLACE this collapse with per-(folder,uid) instances. That change
+        // is held until the full model + a rebuild land — shipping it alone would
+        // surface transient move-dups. Keeping the collapse for now.
         if (msg.messageId) {
             const moved = this.db.prepare("SELECT id, folder_id, uid FROM messages WHERE account_id = ? AND message_id = ? LIMIT 1").get(msg.accountId, msg.messageId);
             if (moved) {
                 console.log(`  [move-detect] ${msg.accountId} ${msg.messageId}: rebinding row ${moved.id} (folder ${moved.folder_id}/uid ${moved.uid} → folder ${msg.folderId}/uid ${msg.uid})`);
-                // Update folder_id + uid on the messages row (additive
-                // migration: keeps the "primary location" columns valid for
-                // existing reads). The new schema's source of truth is
-                // message_folders — see below.
                 this.db.prepare("UPDATE messages SET folder_id = ?, uid = ?, cached_at = ? WHERE id = ?").run(msg.folderId, msg.uid, Date.now(), moved.id);
-                // Add the new membership row. The OLD membership (source
-                // folder + old uid) stays — reconcile in the source folder
-                // is what eventually drops it (when the server stops
-                // listing the UID there). For Gmail multi-label messages,
-                // both memberships persist legitimately.
                 this.upsertMessageFolder(moved.id, msg.folderId, msg.uid, msg.exclusive);
-                // Notify subscribers so e.g. the reconciler can cancel any
-                // pending deferred-delete for the original (folder, uid) —
-                // otherwise the reconcile-delete grace timer fires for a
-                // row that's been rebound elsewhere, and the user sees the
-                // moved message vanish 30 minutes after the move.
                 if (this._onMoveDetected) {
                     try {
                         this._onMoveDetected({
@@ -2402,6 +2394,26 @@ export class MailxDB {
         const rows = this.db.prepare("SELECT uid FROM message_folders WHERE folder_id = ?").all(folderId);
         return rows.map(r => r.uid);
     }
+    /** UIDs we actually have a stored MESSAGE ROW for in this folder — from the
+     *  `messages` table, NOT `message_folders`. The two can diverge: an "orphan
+     *  membership" (a message_folders row whose messages row was never written /
+     *  got deleted) makes getUidsForFolder report a UID we don't truly have.
+     *  The set-diff backfill must diff against THIS so it re-fetches the real
+     *  message data for orphans instead of treating them as present (Bob
+     *  2026-06-20: ~1k old INBOX messages stuck missing — membership said
+     *  "have it", the row didn't exist). */
+    getStoredUids(accountId, folderId) {
+        const rows = this.db.prepare("SELECT uid FROM messages WHERE account_id = ? AND folder_id = ?").all(accountId, folderId);
+        return rows.map(r => r.uid);
+    }
+    /** Count of stored MESSAGE ROWS in a folder (messages table). The deficit
+     *  gate compares this to the server's count to decide whether to reconcile;
+     *  using the messages table (not message_folders) means orphan memberships
+     *  don't mask a real "rows are missing" deficit. */
+    getStoredCount(accountId, folderId) {
+        const r = this.db.prepare("SELECT count(*) as cnt FROM messages WHERE account_id = ? AND folder_id = ?").get(accountId, folderId);
+        return r?.cnt || 0;
+    }
     /** List recent rows in a folder for a Sent-sweep / reconciliation pass.
      *  Returns only the columns the sweeper needs (uid, message_id, date,
      *  subject, size, has_attachments, preview, body_path) for every row in
@@ -2656,9 +2668,19 @@ export class MailxDB {
      *  click, queued behind it; that IS the "loading body takes forever"
      *  bug). Now keyset-paginated by `m.id` with a `setImmediate` yield
      *  between pages, so user IPC lands in the gaps. */
-    async seedContactsFromMessages() {
+    async seedContactsFromMessages(force = false) {
         const VALID = /^[^\s<>@]+@[^\s<>@]+\.[^\s<>@]+$/;
         const now = Date.now();
+        // INCREMENTAL via a persisted high-water-mark on messages.id. The old
+        // behaviour re-aggregated ALL ~180k messages and re-wrote the ENTIRE
+        // contacts table every 30 min — 1–2s write txns that locked out the sync
+        // worker ("database is locked", Bob 2026-06-16 "locked"). Now each run
+        // only scans rows with id > the last-seeded id, so a periodic run touches
+        // just the handful of messages that arrived since — a tiny, fast write.
+        // First run (no watermark) and `force` do the full scan. The watermark
+        // resets to 0 on rebuild (cache wiped), so a rebuild re-seeds fully.
+        const startId = force ? 0 : Number(this.getKv("contacts", "seed_high_id") || "0");
+        let maxSeenId = startId;
         const agg = new Map();
         const bump = (name, address, date) => {
             const email = (address || "").trim().toLowerCase();
@@ -2709,7 +2731,7 @@ export class MailxDB {
                  JOIN folders f ON m.folder_id = f.id
                  WHERE f.special_use = 'sent' AND m.id > ?
                  ORDER BY m.id LIMIT ?`);
-            let lastId = 0;
+            let lastId = startId;
             for (;;) {
                 const rows = stmt.all(lastId, PAGE);
                 if (rows.length === 0)
@@ -2721,6 +2743,8 @@ export class MailxDB {
                     eatRecipients(r.bcc_json, date);
                 }
                 lastId = rows[rows.length - 1].id;
+                if (lastId > maxSeenId)
+                    maxSeenId = lastId;
                 if (rows.length < PAGE)
                     break;
                 await yieldLoop();
@@ -2733,7 +2757,7 @@ export class MailxDB {
                  LEFT JOIN folders f ON m.folder_id = f.id
                  WHERE (f.special_use IS NULL OR f.special_use != 'sent') AND m.id > ?
                  ORDER BY m.id LIMIT ?`);
-            let lastId = 0;
+            let lastId = startId;
             for (;;) {
                 const rows = stmt.all(lastId, PAGE);
                 if (rows.length === 0)
@@ -2746,6 +2770,8 @@ export class MailxDB {
                     eatRecipients(r.bcc_json, date);
                 }
                 lastId = rows[rows.length - 1].id;
+                if (lastId > maxSeenId)
+                    maxSeenId = lastId;
                 if (rows.length < PAGE)
                     break;
                 await yieldLoop();
@@ -2794,6 +2820,11 @@ export class MailxDB {
             console.log(`  [contacts] seed: ${added} new + ${bumped} refreshed (discovered)`);
             this.notifyContactsChanged();
         }
+        // Advance the high-water-mark so the next run only scans messages newer
+        // than the highest id we just processed. Monotonic — never regress (a
+        // concurrent insert mid-run keeps a higher max for the next pass).
+        if (maxSeenId > startId)
+            this.setKv("contacts", "seed_high_id", String(maxSeenId));
         return added;
     }
     /** Apply the contents of contacts.jsonc — replaces all preferred-tier rows

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bobfrankston/mailx-store",
-  "version": "0.1.53",
+  "version": "0.1.55",
   "type": "module",
   "main": "index.js",
   "types": "index.d.ts",
@@ -10,7 +10,7 @@
   "license": "ISC",
   "dependencies": {
     "@bobfrankston/mailx-types": "^0.1.19",
-    "@bobfrankston/mailx-settings": "^0.1.28",
+    "@bobfrankston/mailx-settings": "^0.1.30",
     "@bobfrankston/mailx-bus": "^0.1.2",
     "mailparser": "^3.7.2"
   },
@@ -30,7 +30,7 @@
   ".transformedSnapshot": {
     "dependencies": {
       "@bobfrankston/mailx-types": "^0.1.19",
-      "@bobfrankston/mailx-settings": "^0.1.28",
+      "@bobfrankston/mailx-settings": "^0.1.30",
       "@bobfrankston/mailx-bus": "^0.1.2",
       "mailparser": "^3.7.2"
     }