@bobfrankston/mailx-imap 0.1.147 → 0.1.149

package/index.d.ts

@@ -77,6 +77,18 @@ export declare class ImapManager extends EventEmitter {
      *  (rate limits, network) are NOT recorded here — they keep the normal
      *  60s deadman cadence. */
     private watchAuthFailed;
+    /** Accounts the SERVER has told us to back off, with when they may be
+     *  tried again and how many consecutive refusals we've had.
+     *
+     *  AOL/Yahoo answer an over-quota LOGIN with `[LIMIT] LOGIN Rate limit
+     *  hit.` — an explicit instruction, not a transient blip. mailx used to
+     *  treat it as transient and keep the 60 s deadman cadence, so every
+     *  refusal was followed by another LOGIN a minute later: measured on
+     *  2026-08-09, four separate runs of 4–22 minutes, one failed attempt per
+     *  minute throughout, every one of them spending quota that keeps the
+     *  limiter hot. Backing off is not politeness, it is how the account
+     *  becomes usable again sooner. */
+    private rateLimited;
     private fetchClients;
     /** The Store is the architectural nexus — owner of MailxDB +
      *  FileMessageStore + the event bus. This package (mailx-imap) is a
@@ -243,6 +255,23 @@ export declare class ImapManager extends EventEmitter {
      *  client; the slot is released when logout() or destroy() runs.
      *  `purpose` is a short tag printed alongside the `[conn+]` log so we can
      *  tell which code path (ops/idle/etc.) opened each connection. */
+    /** Remaining cooldown in ms for an account the server rate-limited, or 0. */
+    rateLimitRemaining(accountId: string): number;
+    /** Record a server rate-limit refusal and widen the cooldown. 5 → 10 → 20
+     *  → 30 min (capped): long enough to actually clear AOL's window, short
+     *  enough that a mistake costs one poll cycle rather than an afternoon. */
+    private noteRateLimited;
+    /** Clear the cooldown after a connection succeeds. */
+    private noteRateLimitCleared;
+    /**
+     * Fetch bodies for rows that carry IMAP uids on an account whose normal
+     * body path is a REST API (Gmail today). Uses the account's IMAP
+     * connection — the same one the server search used to find them.
+     *
+     * Returns how many bodies landed. Failures are logged per row and do not
+     * abort the rest: one unreadable message must not stall the queue.
+     */
+    private fetchBodiesOverImap;
     private newClient;
     /** Force-close every IMAP socket for an account — both lane clients
      *  (ops + fast) plus any lingering ones in openClients (e.g. an IDLE

package/index.js

@@ -27,6 +27,21 @@ const OUTBOX_RETRY_DELAY_MS = 60000;
  *  after boot always runs one (the map is empty); thereafter it's throttled to
  *  this interval so the QRESYNC speedup is preserved in steady state. */
 const RECONCILE_THROTTLE_MS = 15 * 60 * 1000;
+/**
+ * An IMAP UID is a 32-bit unsigned integer (RFC 3501 §2.3.1.1). The API
+ * providers mint their uid from the provider's own message id — Gmail uses
+ * `parseInt(id.slice(-12), 16)`, a 48-bit value — so the two uid spaces are
+ * disjoint and this bound tells them apart exactly, not heuristically.
+ *
+ * It matters because ONE ACCOUNT can hold rows from both: a Gmail account
+ * syncs bodies through the API, but a server search runs over IMAP and
+ * stores rows with IMAP UIDs. Those rows are unfetchable by the API path —
+ * it maps uid→id by listing the label and hashing each id, and an IMAP uid
+ * is not in that map at any depth. The body request simply returned nothing,
+ * forever, with the viewer counting seconds (Bob 2026-08-09: "why the
+ * forever fetch?" / "why did gmail search find them if they can't be read?").
+ */
+const IMAP_UID_MAX = 0xFFFFFFFF;
 /** Parse X-Mailx-Retry* tracking headers from a raw RFC822 message. */
 function parseRetryInfo(raw) {
     const headerEnd = raw.search(/\r?\n\r?\n/);
@@ -219,6 +234,18 @@ export class ImapManager extends EventEmitter {
      *  (rate limits, network) are NOT recorded here — they keep the normal
      *  60s deadman cadence. */
     watchAuthFailed = new Map();
+    /** Accounts the SERVER has told us to back off, with when they may be
+     *  tried again and how many consecutive refusals we've had.
+     *
+     *  AOL/Yahoo answer an over-quota LOGIN with `[LIMIT] LOGIN Rate limit
+     *  hit.` — an explicit instruction, not a transient blip. mailx used to
+     *  treat it as transient and keep the 60 s deadman cadence, so every
+     *  refusal was followed by another LOGIN a minute later: measured on
+     *  2026-08-09, four separate runs of 4–22 minutes, one failed attempt per
+     *  minute throughout, every one of them spending quota that keeps the
+     *  limiter hot. Backing off is not politeness, it is how the account
+     *  becomes usable again sooner. */
+    rateLimited = new Map();
     fetchClients = new Map();
     /** The Store is the architectural nexus — owner of MailxDB +
      *  FileMessageStore + the event bus. This package (mailx-imap) is a
@@ -277,7 +304,7 @@ export class ImapManager extends EventEmitter {
         else if (/UNAVAILABLE|Maximum number of connections|too many connections/i.test(errMsg)) {
             d.connCapHits++;
         }
-        else if (/429|rate limit/i.test(errMsg)) {
+        else if (/429|rate limit|\[LIMIT\]/i.test(errMsg)) {
             d.rateLimitWaits++;
         }
         else {
@@ -894,7 +921,101 @@ export class ImapManager extends EventEmitter {
      *  client; the slot is released when logout() or destroy() runs.
      *  `purpose` is a short tag printed alongside the `[conn+]` log so we can
      *  tell which code path (ops/idle/etc.) opened each connection. */
+    /** Remaining cooldown in ms for an account the server rate-limited, or 0. */
+    rateLimitRemaining(accountId) {
+        const e = this.rateLimited.get(accountId);
+        if (!e)
+            return 0;
+        const left = e.until - Date.now();
+        if (left <= 0) {
+            this.rateLimited.delete(accountId);
+            return 0;
+        }
+        return left;
+    }
+    /** Record a server rate-limit refusal and widen the cooldown. 5 → 10 → 20
+     *  → 30 min (capped): long enough to actually clear AOL's window, short
+     *  enough that a mistake costs one poll cycle rather than an afternoon. */
+    noteRateLimited(accountId, errMsg) {
+        const prev = this.rateLimited.get(accountId);
+        const strikes = (prev?.strikes || 0) + 1;
+        const mins = Math.min(30, 5 * Math.pow(2, strikes - 1));
+        const until = Date.now() + mins * 60_000;
+        this.rateLimited.set(accountId, { until, strikes });
+        console.warn(`  [rate-limit] ${accountId}: server said "${errMsg.trim().slice(0, 60)}" — pausing ALL lanes for ${mins} min (strike ${strikes})`);
+        this.emit("rateLimited", { accountId, untilMs: until, minutes: mins, strikes });
+    }
+    /** Clear the cooldown after a connection succeeds. */
+    noteRateLimitCleared(accountId) {
+        if (this.rateLimited.delete(accountId)) {
+            console.log(`  [rate-limit] ${accountId}: connection accepted again — cooldown cleared`);
+            this.emit("rateLimitCleared", { accountId });
+        }
+    }
+    /**
+     * Fetch bodies for rows that carry IMAP uids on an account whose normal
+     * body path is a REST API (Gmail today). Uses the account's IMAP
+     * connection — the same one the server search used to find them.
+     *
+     * Returns how many bodies landed. Failures are logged per row and do not
+     * abort the rest: one unreadable message must not stall the queue.
+     */
+    async fetchBodiesOverImap(accountId, rows) {
+        const folders = this.db.getFolders(accountId);
+        let fetched = 0;
+        let client = null;
+        try {
+            client = await this.newClient(accountId, "fetch");
+            await client.connect?.();
+            for (const row of rows) {
+                const folder = folders.find(f => f.id === row.folderId);
+                if (!folder)
+                    continue;
+                try {
+                    const msg = await client.fetchMessageByUid(folder.path, row.uid, { source: true });
+                    const source = msg?.source;
+                    if (!source) {
+                        // Not on the server either — stop asking for it every
+                        // sweep, but say so once rather than silently.
+                        console.log(`  [prefetch] ${accountId}/${folder.path} uid ${row.uid}: not on the server over IMAP either — marking empty`);
+                        this.markPrefetchEmpty(accountId, row.folderId, row.uid);
+                        continue;
+                    }
+                    const raw = Buffer.from(source, "utf-8");
+                    const bodyPath = await this.bodyStore.putMessage(accountId, row.folderId, row.uid, raw);
+                    const parsed = await extractPreview(source);
+                    this.db.updateBodyMeta(accountId, row.folderId, row.uid, bodyPath, parsed.hasAttachments, parsed.preview);
+                    this.emit("bodyCached", accountId, row.uid);
+                    fetched++;
+                }
+                catch (e) {
+                    console.error(`  [prefetch] ${accountId}/${folder.path} uid ${row.uid}: IMAP body fetch failed: ${e?.message || e}`);
+                }
+            }
+        }
+        catch (e) {
+            console.error(`  [prefetch] ${accountId}: IMAP fallback could not open a connection: ${e?.message || e}`);
+        }
+        finally {
+            try {
+                await client?.logout?.();
+            }
+            catch { /* socket already gone */ }
+        }
+        if (fetched > 0)
+            console.log(`  [prefetch] ${accountId}: ${fetched} body(ies) fetched over IMAP for API-account rows`);
+        return fetched;
+    }
     async newClient(accountId, purpose = "?") {
+        // Refuse to spend a LOGIN the server has already told us it will not
+        // accept. Every lane funnels through newClient, so one check covers
+        // sync, quickCheck, IDLE, outbox and fetch alike — which matters,
+        // because the lanes retry independently and would otherwise take turns
+        // poking the limiter.
+        const cooldown = this.rateLimitRemaining(accountId);
+        if (cooldown > 0) {
+            throw new Error(`rate-limited by server — retrying in ${Math.ceil(cooldown / 60_000)} min (no LOGIN attempted)`);
+        }
         if (this.reauthenticating.has(accountId))
             throw new Error(`Account ${accountId} is re-authenticating`);
         const config = this.configs.get(accountId);
@@ -985,11 +1106,22 @@ export class ImapManager extends EventEmitter {
                         const sock = client?.native?.transport?.socket;
                         if (typeof sock?.once === "function")
                             sock.once("close", () => markClosed("socket-close"));
+                        // A LOGIN the server accepted is the only proof the
+                        // cooldown is over — clear it here rather than letting
+                        // it lapse on a timer alone.
+                        this.noteRateLimitCleared(accountId);
                         return r;
                     }
                     catch (e) {
                         emitPhase("failed", e?.message || String(e));
                         markClosed("connect-failed");
+                        // Every lane (sync, quickCheck, IDLE, outbox, fetch)
+                        // connects through here, so this one line is what makes
+                        // a rate-limit refusal stop ALL of them instead of each
+                        // discovering it separately, once a minute, forever.
+                        const em = e?.message || String(e);
+                        if (/[LIMIT]|rate limit/i.test(em))
+                            this.noteRateLimited(accountId, em);
                         throw e;
                     }
                 };
@@ -3801,7 +3933,16 @@ export class ImapManager extends EventEmitter {
                 // message in multiple labels gets fetched twice under current
                 // grouping. A deeper label-native redesign is a separate TODO.
                 const byFolder = new Map();
+                // Rows whose uid is in the IMAP space cannot be addressed by
+                // the API provider at all. Route them to the IMAP fetcher —
+                // the same transport that created them — instead of handing
+                // the API a name it cannot resolve.
+                const imapSpaceRows = [];
                 for (const m of missing) {
+                    if (m.uid <= IMAP_UID_MAX) {
+                        imapSpaceRows.push({ folderId: m.folderId, uid: m.uid });
+                        continue;
+                    }
                     let arr = byFolder.get(m.folderId);
                     if (!arr) {
                         arr = [];
@@ -3809,6 +3950,14 @@ export class ImapManager extends EventEmitter {
                     }
                     arr.push(m.uid);
                 }
+                if (imapSpaceRows.length > 0) {
+                    console.log(`  [prefetch] ${accountId}: ${imapSpaceRows.length} row(s) carry IMAP uids on an API account (server-search results) — fetching those over IMAP`);
+                    const fetchedOverImap = await this.fetchBodiesOverImap(accountId, imapSpaceRows);
+                    if (fetchedOverImap > 0) {
+                        madeProgress = true;
+                        counters.totalFetched += fetchedOverImap;
+                    }
+                }
                 const folders = this.db.getFolders(accountId);
                 const api = this.getApiProvider(accountId);
                 try {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bobfrankston/mailx-imap",
-  "version": "0.1.147",
+  "version": "0.1.149",
   "type": "module",
   "main": "index.js",
   "types": "index.d.ts",
@@ -9,9 +9,9 @@
   },
   "license": "ISC",
   "dependencies": {
-    "@bobfrankston/mailx-types": "^0.1.39",
-    "@bobfrankston/mailx-settings": "^0.1.48",
-    "@bobfrankston/mailx-store": "^0.1.82",
+    "@bobfrankston/mailx-types": "^0.1.42",
+    "@bobfrankston/mailx-settings": "^0.1.51",
+    "@bobfrankston/mailx-store": "^0.1.83",
     "@bobfrankston/iflow-direct": "^0.1.65",
     "@bobfrankston/tcp-transport": "^0.1.8",
     "@bobfrankston/smtp-direct": "^0.1.9",
@@ -37,9 +37,9 @@
   },
   ".transformedSnapshot": {
     "dependencies": {
-      "@bobfrankston/mailx-types": "^0.1.39",
-      "@bobfrankston/mailx-settings": "^0.1.48",
-      "@bobfrankston/mailx-store": "^0.1.82",
+      "@bobfrankston/mailx-types": "^0.1.42",
+      "@bobfrankston/mailx-settings": "^0.1.51",
+      "@bobfrankston/mailx-store": "^0.1.83",
       "@bobfrankston/iflow-direct": "^0.1.65",
       "@bobfrankston/tcp-transport": "^0.1.8",
       "@bobfrankston/smtp-direct": "^0.1.9",

package/test/rate-limit-backoff.test.mjs

@@ -0,0 +1,61 @@
+/**
+ * AOL/Yahoo answer an over-quota LOGIN with `[LIMIT] LOGIN Rate limit hit.`
+ * Before v1.2.240 every lane treated that as transient and retried on its own
+ * 60 s cadence, so a refusal was followed by another LOGIN a minute later for
+ * as long as it lasted (measured 2026-08-09: four runs, up to 22 minutes,
+ * ~one wasted LOGIN per minute).
+ *
+ * These checks drive the real ImapManager methods over the cooldown state.
+ * Run: node packages/mailx-imap/test/rate-limit-backoff.test.mjs
+ */
+import assert from "node:assert";
+
+const { ImapManager } = await import("file:///Y:/dev/email/mailx/app/packages/mailx-imap/index.js");
+
+const mgr = Object.create(ImapManager.prototype);
+mgr.rateLimited = new Map();
+const events = [];
+mgr.emit = (name, payload) => { events.push({ name, payload }); };
+
+const LIMIT = "Login failed: [LIMIT] LOGIN Rate limit hit.";
+
+// 1. A refusal starts a cooldown, and it is reported (not swallowed).
+assert.strictEqual(mgr.rateLimitRemaining("aol"), 0, "clean account has no cooldown");
+mgr.noteRateLimited("aol", LIMIT);
+const first = mgr.rateLimitRemaining("aol");
+assert.ok(first > 4 * 60_000 && first <= 5 * 60_000, `first cooldown should be ~5 min, got ${Math.round(first / 1000)}s`);
+assert.ok(events.some(e => e.name === "rateLimited" && e.payload.accountId === "aol"),
+    "a cooldown must be announced — a silently paused account looks broken");
+
+// 2. Repeated refusals widen it: 5 → 10 → 20 → 30 (capped).
+const mins = [];
+for (let i = 0; i < 4; i++) { mgr.noteRateLimited("aol", LIMIT); mins.push(Math.round(mgr.rateLimitRemaining("aol") / 60_000)); }
+assert.deepStrictEqual(mins, [10, 20, 30, 30], `backoff should widen and cap at 30, got ${mins.join(",")}`);
+
+// 3. Other accounts are untouched — the limit is per-account, not global.
+assert.strictEqual(mgr.rateLimitRemaining("bobma"), 0, "one account's cooldown must not pause another");
+
+// 4. A successful connection clears it, and says so.
+mgr.noteRateLimitCleared("aol");
+assert.strictEqual(mgr.rateLimitRemaining("aol"), 0, "success must clear the cooldown");
+assert.ok(events.some(e => e.name === "rateLimitCleared"), "recovery must be announced too");
+
+// 5. An expired cooldown lapses on its own (no success needed).
+mgr.rateLimited.set("aol", { until: Date.now() - 1, strikes: 3 });
+assert.strictEqual(mgr.rateLimitRemaining("aol"), 0, "a lapsed cooldown must not block forever");
+assert.ok(!mgr.rateLimited.has("aol"), "lapsed entry should be dropped, not left to accumulate");
+
+// 6. THE POINT: while cooling down, newClient must refuse to spend a LOGIN —
+//    for every lane, since each one used to discover the limit separately.
+mgr.rateLimited.set("aol", { until: Date.now() + 5 * 60_000, strikes: 1 });
+let attempted = 0;
+mgr.acquireHostSlot = async () => { attempted++; return () => {}; };   // reached only if the guard lets it through
+for (const lane of ["sync", "quickCheck", "idle", "outbox", "fetch"]) {
+    await assert.rejects(
+        () => ImapManager.prototype.newClient.call(mgr, "aol", lane),
+        /rate-limited by server/,
+        `lane "${lane}" must be refused while the account is cooling down`);
+}
+assert.strictEqual(attempted, 0, "no lane may reach the connect path during a cooldown");
+
+console.log("rate-limit backoff: 6 scenarios passed");