@bobfrankston/mailx-store 0.1.49 → 0.1.51

package/db.d.ts

@@ -400,6 +400,15 @@ export declare class MailxDB {
     } | null;
     /** Clear a UID's prefetch-failure record after a successful fetch. */
     clearPrefetchFailure(accountId: string, folderId: number, uid: number): void;
+    /** Wipe ALL prefetch-failure backoffs. Called once per boot: the backoff is
+     *  a within-SESSION optimisation (it stops a poison body being re-fetched
+     *  every 60s cycle), so a fresh session should give every body another try —
+     *  the network may have recovered, and crucially a body backed off by a
+     *  transient cause (a two-writer DB lock, a Dovecot 300s timeout) shouldn't
+     *  stay suppressed for up to 12h (Bob 2026-06-14: "lots of predownloading
+     *  not happening"). Genuine poison bodies simply re-fail once and re-back-off
+     *  for the session — no per-cycle loop. */
+    clearAllPrefetchFailures(): number;
     /** Highest server UID we've seen in this folder. After the
      *  message_folders refactor, this reads from the membership table
      *  rather than messages.uid — a server-side move from this folder
@@ -470,6 +479,17 @@ export declare class MailxDB {
      *  already open this just runs `fn` (its writes join the open txn);
      *  otherwise it owns a fresh BEGIN/COMMIT. `fn` MUST be synchronous so it
      *  can't span an await and leave a transaction open across a yield. */
+    /** Run a WAL checkpoint to bound the WAL file size. PASSIVE never blocks
+     *  on readers — it reclaims frames older than the oldest live snapshot and
+     *  returns immediately otherwise. Drive it from ONE connection (main) on a
+     *  timer so the WAL can't balloon while three connections share the file.
+     *  Returns {busy, log, checkpointed} (log = frames in WAL, checkpointed =
+     *  frames moved into the db) or null on error. */
+    checkpoint(mode?: "PASSIVE" | "FULL" | "RESTART" | "TRUNCATE"): {
+        busy: number;
+        log: number;
+        checkpointed: number;
+    } | null;
     runInTxn<T>(fn: () => T): T;
     /** Record an address used in sent mail */
     recordSentAddress(name: string, email: string): void;

package/db.js

@@ -419,7 +419,13 @@ export class MailxDB {
             // DDL/backfills concurrently would race on the WAL write lock.
             this.db.exec("PRAGMA journal_mode = WAL");
             this.db.exec("PRAGMA foreign_keys = ON");
-            this.db.exec("PRAGMA busy_timeout = 5000");
+            // 10s (vs the main thread's 5s): this is the BACKGROUND writer (sync,
+            // prefetch); waiting a bit longer for the foreground's lock is fine.
+            // NOT higher because busy_timeout BLOCKS this thread synchronously
+            // (it can't forward sync events while parked) — for longer holds the
+            // prefetch write path uses an ASYNC retry that yields between
+            // attempts instead. Fixes prefetch "database is locked" (Bob 2026-06-14).
+            this.db.exec("PRAGMA busy_timeout = 10000");
             return;
         }
         if (this.readOnly) {
@@ -446,6 +452,23 @@ export class MailxDB {
         }
         this.db.exec("PRAGMA journal_mode = WAL");
         this.db.exec("PRAGMA foreign_keys = ON");
+        // WAL hygiene. With THREE connections (main writer, sync-worker writer,
+        // read-worker reader) the WAL was never getting checkpointed — failed
+        // writes (two-writer contention) never commit, so the auto-checkpoint
+        // that normally fires on commit never ran, and the WAL ballooned to
+        // 71 MB (Bob 2026-06-14: every read/write crawled, `database is locked`
+        // everywhere, a bulk move took 51s and timed out). A bloated WAL is
+        // self-reinforcing: bigger WAL → slower writes → more lock failures →
+        // fewer commits → no checkpoint. Boot is the one moment main is the
+        // SOLE connection (read/sync workers spawn later), so TRUNCATE here
+        // resets the file to empty; `checkpoint()` on a timer keeps it small.
+        try {
+            const r = this.db.prepare("PRAGMA wal_checkpoint(TRUNCATE)").get();
+            console.log(`  [db] boot WAL checkpoint: busy=${r?.busy} log=${r?.log} checkpointed=${r?.checkpointed}`);
+        }
+        catch (e) {
+            console.error(`  [db] boot WAL checkpoint failed: ${e?.message || e}`);
+        }
         // Multi-writer foundation for the sync-worker migration: once sync runs
         // on its own thread with its own write connection, it and the main
         // thread are two writers on one WAL file. WAL serializes writers, so a
@@ -569,6 +592,15 @@ export class MailxDB {
         // this column landed. One UPDATE + an id roundtrip per row — cheap
         // at our row counts, runs once per DB upgrade.
         this.backfillUuids();
+        // Per-boot: clear prefetch-failure backoffs so transiently-failed bodies
+        // (two-writer DB lock, Dovecot timeout) get a fresh try this session
+        // instead of staying suppressed for up to 12h. See clearAllPrefetchFailures.
+        try {
+            const cleared = this.clearAllPrefetchFailures();
+            if (cleared > 0)
+                console.log(`  [db] cleared ${cleared} prefetch-failure backoff(s) for a fresh session`);
+        }
+        catch { /* non-fatal */ }
         // One-shot cleanup: the retired insertOptimisticSentRow path wrote
         // synthetic-negative-UID rows into Sent. Those rows are stale (the
         // real server-synced row eventually appears with a positive UID),
@@ -2235,6 +2267,22 @@ export class MailxDB {
     clearPrefetchFailure(accountId, folderId, uid) {
         this.db.prepare("DELETE FROM prefetch_failures WHERE account_id = ? AND folder_id = ? AND uid = ?").run(accountId, folderId, uid);
     }
+    /** Wipe ALL prefetch-failure backoffs. Called once per boot: the backoff is
+     *  a within-SESSION optimisation (it stops a poison body being re-fetched
+     *  every 60s cycle), so a fresh session should give every body another try —
+     *  the network may have recovered, and crucially a body backed off by a
+     *  transient cause (a two-writer DB lock, a Dovecot 300s timeout) shouldn't
+     *  stay suppressed for up to 12h (Bob 2026-06-14: "lots of predownloading
+     *  not happening"). Genuine poison bodies simply re-fail once and re-back-off
+     *  for the session — no per-cycle loop. */
+    clearAllPrefetchFailures() {
+        try {
+            return this.db.prepare("DELETE FROM prefetch_failures").run().changes;
+        }
+        catch {
+            return 0;
+        }
+    }
     /** Highest server UID we've seen in this folder. After the
      *  message_folders refactor, this reads from the membership table
      *  rather than messages.uid — a server-side move from this folder
@@ -2363,13 +2411,37 @@ export class MailxDB {
      *  already open this just runs `fn` (its writes join the open txn);
      *  otherwise it owns a fresh BEGIN/COMMIT. `fn` MUST be synchronous so it
      *  can't span an await and leave a transaction open across a yield. */
+    /** Run a WAL checkpoint to bound the WAL file size. PASSIVE never blocks
+     *  on readers — it reclaims frames older than the oldest live snapshot and
+     *  returns immediately otherwise. Drive it from ONE connection (main) on a
+     *  timer so the WAL can't balloon while three connections share the file.
+     *  Returns {busy, log, checkpointed} (log = frames in WAL, checkpointed =
+     *  frames moved into the db) or null on error. */
+    checkpoint(mode = "PASSIVE") {
+        try {
+            return this.db.prepare(`PRAGMA wal_checkpoint(${mode})`).get();
+        }
+        catch {
+            return null;
+        }
+    }
     runInTxn(fn) {
         if (this.db.isTransaction)
             return fn();
         this.db.exec("BEGIN");
+        const _t0 = Date.now();
         try {
             const r = fn();
             this.db.exec("COMMIT");
+            // Diagnostic: a write transaction held >1s on the MAIN connection is
+            // what locks out the sync-worker's prefetch write (Bob 2026-06-14
+            // "find the long main-thread transaction"). Log the call site so the
+            // next occurrence names itself instead of needing code archaeology.
+            const _ms = Date.now() - _t0;
+            if (_ms > 1000) {
+                const where = (new Error().stack || "").split("\n")[2]?.trim() || "?";
+                console.warn(`  [db] SLOW write txn ${_ms}ms (readOnly=${this.readOnly}) at ${where}`);
+            }
             return r;
         }
         catch (e) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bobfrankston/mailx-store",
-  "version": "0.1.49",
+  "version": "0.1.51",
   "type": "module",
   "main": "index.js",
   "types": "index.d.ts",
@@ -10,7 +10,7 @@
   "license": "ISC",
   "dependencies": {
     "@bobfrankston/mailx-types": "^0.1.19",
-    "@bobfrankston/mailx-settings": "^0.1.26",
+    "@bobfrankston/mailx-settings": "^0.1.28",
     "@bobfrankston/mailx-bus": "^0.1.2",
     "mailparser": "^3.7.2"
   },
@@ -30,7 +30,7 @@
   ".transformedSnapshot": {
     "dependencies": {
       "@bobfrankston/mailx-types": "^0.1.19",
-      "@bobfrankston/mailx-settings": "^0.1.26",
+      "@bobfrankston/mailx-settings": "^0.1.28",
       "@bobfrankston/mailx-bus": "^0.1.2",
       "mailparser": "^3.7.2"
     }