@bobfrankston/mailx-store 0.1.135 → 0.1.138

package/db.js

@@ -618,23 +618,34 @@ export class MailxDB {
         // Purge phantom uid=0 rows (invalid IMAP UID — empty stub letters, the
         // upsertMessage guard now prevents new ones; this clears any already on
         // disk, e.g. Bob's 00:03 INBOX phantom). Cascade to FTS + membership.
-        try {
-            const ph = this.db.prepare("SELECT id FROM messages WHERE uid = 0").all();
-            if (ph.length) {
-                for (const r of ph) {
-                    try {
-                        this.db.prepare("DELETE FROM messages_fts WHERE rowid = ?").run(r.id);
+        //
+        // 2026-09-30 — Claude Code (Opus 5.5): this and the other heals below
+        // that end in "idempotent" ran on EVERY start and found nothing — on
+        // Bob's 215k-message store the five passes cost ~1.8 s before the
+        // window could open (measured read-only: trim ids 786 ms, is_replied
+        // 546 ms, membership collapse 353 ms, this purge 80 ms). Bob: "took a
+        // long time to load". The cause of each is fixed at the write path, so
+        // each now runs once and records that in kv, like the migrations
+        // further down. is_replied is scoped instead (see there).
+        if (!this.getKv("schema", "purge_uid0_v1"))
+            try {
+                const ph = this.db.prepare("SELECT id FROM messages WHERE uid = 0").all();
+                if (ph.length) {
+                    for (const r of ph) {
+                        try {
+                            this.db.prepare("DELETE FROM messages_fts WHERE rowid = ?").run(r.id);
+                        }
+                        catch { /* */ }
+                        this.db.prepare("DELETE FROM message_folders WHERE message_row_id = ?").run(r.id);
                     }
-                    catch { /* */ }
-                    this.db.prepare("DELETE FROM message_folders WHERE message_row_id = ?").run(r.id);
+                    this.db.prepare("DELETE FROM messages WHERE uid = 0").run();
+                    console.log(`  [db] purged ${ph.length} phantom uid=0 row(s)`);
                 }
-                this.db.prepare("DELETE FROM messages WHERE uid = 0").run();
-                console.log(`  [db] purged ${ph.length} phantom uid=0 row(s)`);
+                this.setKv("schema", "purge_uid0_v1", "1");
+            }
+            catch (e) {
+                console.error(`  [db] phantom purge failed: ${e?.message || e}`);
             }
-        }
-        catch (e) {
-            console.error(`  [db] phantom purge failed: ${e?.message || e}`);
-        }
         // Idempotent migrations for older databases that predate new columns.
         // SQLite doesn't support "ADD COLUMN IF NOT EXISTS", so we just try the
         // ALTER and catch the "duplicate column" error. Simpler and more robust
@@ -705,12 +716,17 @@ export class MailxDB {
         // "missing" messages forever (3,707 duplicates of one reply,
         // 2026-07-02). Idempotent — after the first pass the WHERE matches
         // nothing. char(9/10/13)+space covers all RFC 5322 fold whitespace.
-        try {
-            const ws = "char(9)||char(10)||char(13)||' '";
-            this.db.exec(`UPDATE messages SET message_id = TRIM(message_id, ${ws}) WHERE message_id <> TRIM(message_id, ${ws})`);
-            this.db.exec(`UPDATE messages SET in_reply_to = TRIM(in_reply_to, ${ws}) WHERE in_reply_to IS NOT NULL AND in_reply_to <> TRIM(in_reply_to, ${ws})`);
-        }
-        catch { /* non-fatal */ }
+        // Once (kv, 2026-09-30): the envelope parser trims since 2026-07-02.
+        if (!this.getKv("schema", "trim_folded_ids_v1"))
+            try {
+                const ws = "char(9)||char(10)||char(13)||' '";
+                this.db.exec(`UPDATE messages SET message_id = TRIM(message_id, ${ws}) WHERE message_id <> TRIM(message_id, ${ws})`);
+                this.db.exec(`UPDATE messages SET in_reply_to = TRIM(in_reply_to, ${ws}) WHERE in_reply_to IS NOT NULL AND in_reply_to <> TRIM(in_reply_to, ${ws})`);
+                this.setKv("schema", "trim_folded_ids_v1", "1");
+            }
+            catch (e) {
+                console.error(`  [db] Message-ID trim heal failed (retried next start): ${e?.message || e}`);
+            }
         try {
             this.db.exec("CREATE INDEX IF NOT EXISTS idx_messages_folder_sentdate ON messages(account_id, folder_id, sent_date DESC)");
         }
@@ -731,19 +747,46 @@ export class MailxDB {
         // so a 100k-row mailbox finishes in milliseconds. Idempotent — after
         // the first run the WHERE-is_replied=0 filter prunes work down to any
         // new linkages that arrived during the previous session.
+        //
+        // 2026-09-30 — Claude Code (Opus 5.5): scoped to linkages that can be
+        // NEW since the last pass. It used to test every one of 215k rows each
+        // start (546 ms, zero changes). A new linkage needs a new row on one
+        // side: a reply that arrived (its parent may be old) or a parent that
+        // arrived after its reply. Both sides are covered from `cached_at`;
+        // the first run after this change (no kv yet) starts from 0, i.e. the
+        // old full pass. The stamp is taken BEFORE the pass so rows written
+        // while it runs are looked at again next time.
         try {
-            const r = this.db.prepare(`
+            const since = Number(this.getKv("schema", "is_replied_backfill_at") || 0);
+            const stamp = Date.now();
+            // A new reply marks its (possibly old) parent. CROSS JOIN pins the
+            // replies (c) as the outer loop, each seeking its parent through
+            // idx_messages_acct_msgid: 176 ms on Bob's store. A plain JOIN let
+            // the planner scan every parent instead — 2,344 ms, four times the
+            // pass it replaced (measured read-only 2026-09-30).
+            const byReply = this.db.prepare(`
+                UPDATE messages SET is_replied = 1 WHERE id IN (
+                    SELECT p.id FROM messages c
+                    CROSS JOIN messages p ON p.account_id = c.account_id AND p.message_id = c.in_reply_to
+                    WHERE c.cached_at >= ? AND c.in_reply_to IS NOT NULL AND c.in_reply_to != ''
+                      AND p.is_replied = 0
+                )
+            `).run(since);
+            // A new parent whose reply was already here.
+            const byParent = this.db.prepare(`
                 UPDATE messages SET is_replied = 1
-                WHERE is_replied = 0
+                WHERE cached_at >= ? AND is_replied = 0
                   AND message_id IS NOT NULL AND message_id != ''
                   AND EXISTS (
                     SELECT 1 FROM messages m2
                     WHERE m2.account_id = messages.account_id
                       AND m2.in_reply_to = messages.message_id
                   )
-            `).run();
-            if (r.changes > 0)
-                console.log(`  [db] backfilled is_replied on ${r.changes} rows`);
+            `).run(since);
+            const changes = Number(byReply.changes || 0) + Number(byParent.changes || 0);
+            if (changes > 0)
+                console.log(`  [db] backfilled is_replied on ${changes} rows`);
+            this.setKv("schema", "is_replied_backfill_at", String(stamp));
         }
         catch (e) {
             console.error(`  [db] is_replied backfill failed: ${e?.message || e}`);
@@ -811,17 +854,21 @@ export class MailxDB {
         // backlog without waiting for each folder to re-sync. Same-folder only —
         // cross-folder memberships (a real move's stale source) are NOT touched
         // here; those need server reconcile to resolve safely.
-        try {
-            const r = this.db.prepare(`DELETE FROM message_folders WHERE id NOT IN (
+        // Once (kv, 2026-09-30): upsertMessageFolder has kept the invariant
+        // since 2026-06-16; this was clearing a backlog that is long gone.
+        if (!this.getKv("schema", "collapse_dup_memberships_v1"))
+            try {
+                const r = this.db.prepare(`DELETE FROM message_folders WHERE id NOT IN (
                     SELECT MAX(id) FROM message_folders GROUP BY message_row_id, folder_id
                  )`).run();
-            const n = Number(r.changes || 0);
-            if (n > 0)
-                console.log(`  [db] collapsed ${n} duplicate same-folder membership row(s)`);
-        }
-        catch (e) {
-            console.error(`  [db] membership collapse failed: ${e?.message || e}`);
-        }
+                const n = Number(r.changes || 0);
+                if (n > 0)
+                    console.log(`  [db] collapsed ${n} duplicate same-folder membership row(s)`);
+                this.setKv("schema", "collapse_dup_memberships_v1", "1");
+            }
+            catch (e) {
+                console.error(`  [db] membership collapse failed: ${e?.message || e}`);
+            }
         // (The per-boot cross-folder collapse for "exclusive" accounts was
         // REMOVED 2026-06-16 — it deleted intentional copies. Cross-folder stale
         // memberships are now cleaned by server-truth reconcile, which keeps real

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bobfrankston/mailx-store",
-  "version": "0.1.135",
+  "version": "0.1.138",
   "type": "module",
   "main": "index.js",
   "types": "index.d.ts",
@@ -10,7 +10,7 @@
   "license": "ISC",
   "dependencies": {
     "@bobfrankston/mailx-types": "^0.1.108",
-    "@bobfrankston/mailx-settings": "^0.1.107",
+    "@bobfrankston/mailx-settings": "^0.1.108",
     "@bobfrankston/mailx-bus": "^0.1.2",
     "mailparser": "^3.7.2"
   },
@@ -30,7 +30,7 @@
   ".transformedSnapshot": {
     "dependencies": {
       "@bobfrankston/mailx-types": "^0.1.108",
-      "@bobfrankston/mailx-settings": "^0.1.107",
+      "@bobfrankston/mailx-settings": "^0.1.108",
       "@bobfrankston/mailx-bus": "^0.1.2",
       "mailparser": "^3.7.2"
     }