@bobfrankston/mailx-imap 0.1.148 → 0.1.150

package/index.d.ts

@@ -263,6 +263,15 @@ export declare class ImapManager extends EventEmitter {
     private noteRateLimited;
     /** Clear the cooldown after a connection succeeds. */
     private noteRateLimitCleared;
+    /**
+     * Fetch bodies for rows that carry IMAP uids on an account whose normal
+     * body path is a REST API (Gmail today). Uses the account's IMAP
+     * connection — the same one the server search used to find them.
+     *
+     * Returns how many bodies landed. Failures are logged per row and do not
+     * abort the rest: one unreadable message must not stall the queue.
+     */
+    private fetchBodiesOverImap;
     private newClient;
     /** Force-close every IMAP socket for an account — both lane clients
      *  (ops + fast) plus any lingering ones in openClients (e.g. an IDLE

package/index.js

@@ -27,6 +27,21 @@ const OUTBOX_RETRY_DELAY_MS = 60000;
  *  after boot always runs one (the map is empty); thereafter it's throttled to
  *  this interval so the QRESYNC speedup is preserved in steady state. */
 const RECONCILE_THROTTLE_MS = 15 * 60 * 1000;
+/**
+ * An IMAP UID is a 32-bit unsigned integer (RFC 3501 §2.3.1.1). The API
+ * providers mint their uid from the provider's own message id — Gmail uses
+ * `parseInt(id.slice(-12), 16)`, a 48-bit value — so the two uid spaces are
+ * disjoint and this bound tells them apart exactly, not heuristically.
+ *
+ * It matters because ONE ACCOUNT can hold rows from both: a Gmail account
+ * syncs bodies through the API, but a server search runs over IMAP and
+ * stores rows with IMAP UIDs. Those rows are unfetchable by the API path —
+ * it maps uid→id by listing the label and hashing each id, and an IMAP uid
+ * is not in that map at any depth. The body request simply returned nothing,
+ * forever, with the viewer counting seconds (Bob 2026-08-09: "why the
+ * forever fetch?" / "why did gmail search find them if they can't be read?").
+ */
+const IMAP_UID_MAX = 0xFFFFFFFF;
 /** Parse X-Mailx-Retry* tracking headers from a raw RFC822 message. */
 function parseRetryInfo(raw) {
     const headerEnd = raw.search(/\r?\n\r?\n/);
@@ -937,6 +952,60 @@ export class ImapManager extends EventEmitter {
             this.emit("rateLimitCleared", { accountId });
         }
     }
+    /**
+     * Fetch bodies for rows that carry IMAP uids on an account whose normal
+     * body path is a REST API (Gmail today). Uses the account's IMAP
+     * connection — the same one the server search used to find them.
+     *
+     * Returns how many bodies landed. Failures are logged per row and do not
+     * abort the rest: one unreadable message must not stall the queue.
+     */
+    async fetchBodiesOverImap(accountId, rows) {
+        const folders = this.db.getFolders(accountId);
+        let fetched = 0;
+        let client = null;
+        try {
+            client = await this.newClient(accountId, "fetch");
+            await client.connect?.();
+            for (const row of rows) {
+                const folder = folders.find(f => f.id === row.folderId);
+                if (!folder)
+                    continue;
+                try {
+                    const msg = await client.fetchMessageByUid(folder.path, row.uid, { source: true });
+                    const source = msg?.source;
+                    if (!source) {
+                        // Not on the server either — stop asking for it every
+                        // sweep, but say so once rather than silently.
+                        console.log(`  [prefetch] ${accountId}/${folder.path} uid ${row.uid}: not on the server over IMAP either — marking empty`);
+                        this.markPrefetchEmpty(accountId, row.folderId, row.uid);
+                        continue;
+                    }
+                    const raw = Buffer.from(source, "utf-8");
+                    const bodyPath = await this.bodyStore.putMessage(accountId, row.folderId, row.uid, raw);
+                    const parsed = await extractPreview(source);
+                    this.db.updateBodyMeta(accountId, row.folderId, row.uid, bodyPath, parsed.hasAttachments, parsed.preview);
+                    this.emit("bodyCached", accountId, row.uid);
+                    fetched++;
+                }
+                catch (e) {
+                    console.error(`  [prefetch] ${accountId}/${folder.path} uid ${row.uid}: IMAP body fetch failed: ${e?.message || e}`);
+                }
+            }
+        }
+        catch (e) {
+            console.error(`  [prefetch] ${accountId}: IMAP fallback could not open a connection: ${e?.message || e}`);
+        }
+        finally {
+            try {
+                await client?.logout?.();
+            }
+            catch { /* socket already gone */ }
+        }
+        if (fetched > 0)
+            console.log(`  [prefetch] ${accountId}: ${fetched} body(ies) fetched over IMAP for API-account rows`);
+        return fetched;
+    }
     async newClient(accountId, purpose = "?") {
         // Refuse to spend a LOGIN the server has already told us it will not
         // accept. Every lane funnels through newClient, so one check covers
@@ -1051,8 +1120,22 @@ export class ImapManager extends EventEmitter {
                         // a rate-limit refusal stop ALL of them instead of each
                         // discovering it separately, once a minute, forever.
                         const em = e?.message || String(e);
-                        if (/[LIMIT]|rate limit/i.test(em))
+                        // NOTE the escaping: [LIMIT] is a literal bracketed
+                        // response code. Written unescaped, [LIMIT] is a
+                        // CHARACTER CLASS matching any L/I/M/T — which is to
+                        // say almost every error message ever produced, so a
+                        // stray parse error would pause the whole account for
+                        // half an hour (shipped that way in v1.2.240, caught
+                        // 2026-08-10 when a connection-cap error tripped it).
+                        //
+                        // A connection cap belongs here too, deliberately:
+                        // Dovecot's "Maximum number of connections from
+                        // user+IP exceeded" is the same instruction as a rate
+                        // limit — stop opening sockets — and standing down for
+                        // a few minutes is what frees the slots.
+                        if (/\[LIMIT\]|rate limit|Maximum number of connections|too many connections/i.test(em)) {
                             this.noteRateLimited(accountId, em);
+                        }
                         throw e;
                     }
                 };
@@ -3864,7 +3947,16 @@ export class ImapManager extends EventEmitter {
                 // message in multiple labels gets fetched twice under current
                 // grouping. A deeper label-native redesign is a separate TODO.
                 const byFolder = new Map();
+                // Rows whose uid is in the IMAP space cannot be addressed by
+                // the API provider at all. Route them to the IMAP fetcher —
+                // the same transport that created them — instead of handing
+                // the API a name it cannot resolve.
+                const imapSpaceRows = [];
                 for (const m of missing) {
+                    if (m.uid <= IMAP_UID_MAX) {
+                        imapSpaceRows.push({ folderId: m.folderId, uid: m.uid });
+                        continue;
+                    }
                     let arr = byFolder.get(m.folderId);
                     if (!arr) {
                         arr = [];
@@ -3872,6 +3964,14 @@ export class ImapManager extends EventEmitter {
                     }
                     arr.push(m.uid);
                 }
+                if (imapSpaceRows.length > 0) {
+                    console.log(`  [prefetch] ${accountId}: ${imapSpaceRows.length} row(s) carry IMAP uids on an API account (server-search results) — fetching those over IMAP`);
+                    const fetchedOverImap = await this.fetchBodiesOverImap(accountId, imapSpaceRows);
+                    if (fetchedOverImap > 0) {
+                        madeProgress = true;
+                        counters.totalFetched += fetchedOverImap;
+                    }
+                }
                 const folders = this.db.getFolders(accountId);
                 const api = this.getApiProvider(accountId);
                 try {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bobfrankston/mailx-imap",
-  "version": "0.1.148",
+  "version": "0.1.150",
   "type": "module",
   "main": "index.js",
   "types": "index.d.ts",
@@ -9,8 +9,8 @@
   },
   "license": "ISC",
   "dependencies": {
-    "@bobfrankston/mailx-types": "^0.1.41",
-    "@bobfrankston/mailx-settings": "^0.1.50",
+    "@bobfrankston/mailx-types": "^0.1.43",
+    "@bobfrankston/mailx-settings": "^0.1.51",
     "@bobfrankston/mailx-store": "^0.1.83",
     "@bobfrankston/iflow-direct": "^0.1.65",
     "@bobfrankston/tcp-transport": "^0.1.8",
@@ -37,8 +37,8 @@
   },
   ".transformedSnapshot": {
     "dependencies": {
-      "@bobfrankston/mailx-types": "^0.1.41",
-      "@bobfrankston/mailx-settings": "^0.1.50",
+      "@bobfrankston/mailx-types": "^0.1.43",
+      "@bobfrankston/mailx-settings": "^0.1.51",
       "@bobfrankston/mailx-store": "^0.1.83",
       "@bobfrankston/iflow-direct": "^0.1.65",
       "@bobfrankston/tcp-transport": "^0.1.8",

package/test/rate-limit-backoff.test.mjs

@@ -59,3 +59,25 @@ for (const lane of ["sync", "quickCheck", "idle", "outbox", "fetch"]) {
 assert.strictEqual(attempted, 0, "no lane may reach the connect path during a cooldown");
 
 console.log("rate-limit backoff: 6 scenarios passed");
+
+// 7. THE CLASSIFIER ITSELF. Shipped in v1.2.240 as /[LIMIT]|rate limit/i —
+//    an unescaped character class matching any L, I, M or T, i.e. nearly
+//    every error string in existence. A stray parse error would have paused
+//    the account for up to 30 minutes. These assertions pin the intent:
+//    back off when the SERVER says stop, never on an ordinary failure.
+const shouldBackOff = (m) => /\[LIMIT\]|rate limit|Maximum number of connections|too many connections/i.test(m);
+for (const m of [
+    "Login failed: [LIMIT] LOGIN Rate limit hit.",
+    "Login failed: [UNAVAILABLE] Maximum number of connections from user+IP exceeded (mail_max_userip_connections=20)",
+    "429 rate limit exceeded",
+    "too many connections",
+]) assert.ok(shouldBackOff(m), `should back off: ${m}`);
+for (const m of [
+    "Login failed: AUTHENTICATE failed.",
+    "inactivity timeout",
+    "SELECT INBOX failed: [NONEXISTENT] Unknown Mailbox",
+    "socket hang up",
+    "ETIMEDOUT",
+    "simpleParser threw on MIME boundary",
+]) assert.ok(!shouldBackOff(m), `must NOT pause the account for: ${m}`);
+console.log("rate-limit classifier: 10 messages classified correctly");