@bobfrankston/mailx-sync 0.1.3 → 0.1.5

package/gmail.d.ts

@@ -15,6 +15,9 @@ import type { MailProvider, ProviderFolder, ProviderMessage, FetchOptions } from
 export declare class GmailApiProvider implements MailProvider {
     private tokenProvider;
     constructor(tokenProvider: () => Promise<string>);
+    /** Block until (a) cooldown has elapsed and (b) a token is available.
+     *  Token-bucket refill happens lazily on each call. */
+    private acquireToken;
     private fetch;
     listFolders(): Promise<ProviderFolder[]>;
     /** List message IDs matching a query, handling pagination.
@@ -42,6 +45,12 @@ export declare class GmailApiProvider implements MailProvider {
      *  fetches of multi-labeled messages. */
     fetchBodiesBatch(folder: string, uids: number[], onBody: (uid: number, source: string) => void): Promise<void>;
     fetchOne(folder: string, uid: number, options?: FetchOptions): Promise<ProviderMessage | null>;
+    /** Apply the absolute flag state to a message.
+     *  Gmail model: flags are labels. `\Seen` is the *absence* of UNREAD;
+     *  `\Flagged` is the presence of STARRED. We always send both add and
+     *  remove so the end state matches regardless of what was there before,
+     *  which makes the call idempotent and safe to retry. */
+    setFlags(folder: string, uid: number, flags: string[]): Promise<void>;
     getUids(folder: string): Promise<number[]>;
     close(): Promise<void>;
     /** Map folder path to Gmail label query term */

package/gmail.js

@@ -47,11 +47,58 @@ function parseAddressList(raw) {
     // Split on commas that aren't inside quotes
     return raw.split(/,(?=(?:[^"]*"[^"]*")*[^"]*$)/).map(s => parseAddress(s.trim())).filter(a => a.address);
 }
+/** Rate-limit + cooldown state shared across all GmailApiProvider instances.
+ *  Gmail enforces rate limits per Google user, not per JS object, so sharing
+ *  state here prevents new `new GmailApiProvider(...)` from side-stepping the
+ *  cooldown by creating a fresh instance. Single-account setups won't notice;
+ *  multi-account setups are slightly over-throttled, which is safer than
+ *  burning through Google's tolerance. */
+const rateState = {
+    tokens: 0,
+    lastRefill: 0,
+    cooldownUntil: 0,
+    /** Tokens refilled per second (keeps us well under per-user-per-sec caps). */
+    rate: 8,
+    /** Max burst above steady state. */
+    burst: 8,
+};
+rateState.tokens = rateState.burst;
+rateState.lastRefill = Date.now();
+/** Extract Google's structured "reason" from a JSON error body when possible
+ *  (e.g. `userRateLimitExceeded`, `rateLimitExceeded`, `quotaExceeded`). */
+function parseGoogleReason(body) {
+    const m = body.match(/"reason"\s*:\s*"([^"]+)"/);
+    return m ? m[1] : "";
+}
 export class GmailApiProvider {
     tokenProvider;
     constructor(tokenProvider) {
         this.tokenProvider = tokenProvider;
     }
+    /** Block until (a) cooldown has elapsed and (b) a token is available.
+     *  Token-bucket refill happens lazily on each call. */
+    async acquireToken() {
+        // Respect an active cooldown first — refuse to even try until it ends.
+        const now = Date.now();
+        if (rateState.cooldownUntil > now) {
+            const waitMs = rateState.cooldownUntil - now;
+            console.log(`  [gmail] cooldown: waiting ${(waitMs / 1000).toFixed(1)}s before next request`);
+            await new Promise(r => setTimeout(r, waitMs));
+        }
+        // Lazy refill, then spin-wait in chunks until a token is ready.
+        while (true) {
+            const t = Date.now();
+            const elapsedSec = (t - rateState.lastRefill) / 1000;
+            rateState.tokens = Math.min(rateState.burst, rateState.tokens + elapsedSec * rateState.rate);
+            rateState.lastRefill = t;
+            if (rateState.tokens >= 1) {
+                rateState.tokens -= 1;
+                return;
+            }
+            const waitMs = Math.max(20, ((1 - rateState.tokens) / rateState.rate) * 1000);
+            await new Promise(r => setTimeout(r, waitMs));
+        }
+    }
     async fetch(path, options = {}) {
         const token = await this.tokenProvider();
         const maxAttempts = 6;
@@ -59,6 +106,7 @@ export class GmailApiProvider {
         const maxDelayMs = 60_000;
         let lastStatus = 0;
         for (let attempt = 0; attempt < maxAttempts; attempt++) {
+            await this.acquireToken();
             const res = await globalThis.fetch(`${API}${path}`, {
                 ...options,
                 headers: {
@@ -70,11 +118,13 @@ export class GmailApiProvider {
             // 403 with "Quota exceeded" body is Google's per-minute quota limit.
             // Not 429. Retry like a rate-limit, don't throw as a permanent failure.
             let isQuota403 = false;
-            if (res.status === 403) {
+            let reason = "";
+            if (res.status === 403 || res.status === 429) {
                 try {
                     const cloned = res.clone();
                     const body = await cloned.text();
-                    if (/quota exceeded|rate ?limit|user[-_ ]rate/i.test(body))
+                    reason = parseGoogleReason(body);
+                    if (res.status === 403 && /quota exceeded|rate ?limit|user[-_ ]rate/i.test(body))
                         isQuota403 = true;
                 }
                 catch { /* ignore */ }
@@ -101,7 +151,13 @@ export class GmailApiProvider {
                 // Full jitter to avoid synchronized retries
                 delay = Math.min(maxDelayMs, delay);
                 delay = Math.floor(delay * (0.5 + Math.random() * 0.5));
-                console.log(`  [gmail] ${res.status}${isQuota403 ? " (per-minute quota)" : ""} (attempt ${attempt + 1}/${maxAttempts}), waiting ${(delay / 1000).toFixed(1)}s${retryAfter ? ` (Retry-After: ${retryAfter})` : ""}...`);
+                // Install a shared cooldown so *other* in-flight workers wait too,
+                // not just this one. Prevents the common failure mode where 5
+                // parallel workers each retry independently and keep the
+                // per-user QPS pegged above the threshold.
+                rateState.cooldownUntil = Math.max(rateState.cooldownUntil, Date.now() + delay);
+                const reasonTag = reason ? ` reason=${reason}` : (isQuota403 ? " (per-minute quota)" : "");
+                console.log(`  [gmail] ${res.status}${reasonTag} (attempt ${attempt + 1}/${maxAttempts}), waiting ${(delay / 1000).toFixed(1)}s${retryAfter ? ` (Retry-After: ${retryAfter})` : ""}...`);
                 await new Promise(r => setTimeout(r, delay));
                 continue;
             }
@@ -111,6 +167,14 @@ export class GmailApiProvider {
             }
             return res.json();
         }
+        // Exhausted all retries — Google is clearly unhappy. Park every
+        // subsequent request behind a long cooldown so the next periodic sync
+        // (30s) doesn't immediately fire another 6 retries into the same
+        // limit. 5 minutes is a floor, not a ceiling — an incoming Retry-After
+        // that's longer will still win via the per-attempt cooldown set above.
+        const TERMINAL_COOLDOWN_MS = 5 * 60_000;
+        rateState.cooldownUntil = Math.max(rateState.cooldownUntil, Date.now() + TERMINAL_COOLDOWN_MS);
+        console.log(`  [gmail] exhausted ${maxAttempts} retries — parking further requests for ${TERMINAL_COOLDOWN_MS / 1000}s`);
         throw new Error(`Gmail API: failed after ${maxAttempts} retries (last status ${lastStatus})`);
     }
     async listFolders() {
@@ -285,12 +349,12 @@ export class GmailApiProvider {
         }
         if (wanted.length === 0)
             return;
-        // Bounded concurrency — 5 in-flight respects Gmail's PER-MINUTE quota
-        // (~15k queries/user/min shared across a project; with 10 concurrent we
-        // hit 403 "Quota exceeded for Queries per minute" on active accounts).
-        // The fetch() helper now retries 403-quota like 429, but staying under
-        // the threshold avoids the backoff pause in the first place.
-        const CONCURRENCY = 5;
+        // Bounded concurrency — 2 in-flight, combined with the shared token
+        // bucket in `fetch()` (8 req/sec steady) keeps us comfortably below
+        // Gmail's per-user-per-second cap. Earlier 5-wide prefetch kept Google
+        // in sustained-abuse territory for hours, which triggered cooldowns
+        // that rejected even 1-unit calls like labels.list on fresh startup.
+        const CONCURRENCY = 2;
         let cursor = 0;
         const worker = async () => {
             while (cursor < wanted.length) {
@@ -329,6 +393,33 @@ export class GmailApiProvider {
         const msg = await this.fetch(`/messages/${id}?${params}`);
         return this.parseMessage(msg, options);
     }
+    /** Apply the absolute flag state to a message.
+     *  Gmail model: flags are labels. `\Seen` is the *absence* of UNREAD;
+     *  `\Flagged` is the presence of STARRED. We always send both add and
+     *  remove so the end state matches regardless of what was there before,
+     *  which makes the call idempotent and safe to retry. */
+    async setFlags(folder, uid, flags) {
+        const query = `in:${this.folderToLabel(folder)}`;
+        const ids = await this.listMessageIds(query, 1000);
+        const id = ids.find(id => idToUid(id) === uid);
+        if (!id)
+            throw new Error(`Gmail setFlags: UID ${uid} not found in ${folder}`);
+        const flagSet = new Set(flags);
+        const addLabelIds = [];
+        const removeLabelIds = [];
+        if (flagSet.has("\\Flagged"))
+            addLabelIds.push("STARRED");
+        else
+            removeLabelIds.push("STARRED");
+        if (flagSet.has("\\Seen"))
+            removeLabelIds.push("UNREAD");
+        else
+            addLabelIds.push("UNREAD");
+        await this.fetch(`/messages/${id}/modify`, {
+            method: "POST",
+            body: JSON.stringify({ addLabelIds, removeLabelIds }),
+        });
+    }
     async getUids(folder) {
         const query = `in:${this.folderToLabel(folder)}`;
         const ids = await this.listMessageIds(query, 10000);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bobfrankston/mailx-sync",
-  "version": "0.1.3",
+  "version": "0.1.5",
   "description": "Platform-agnostic mail provider implementations + sync orchestration. Single source of truth for Gmail/IMAP/Outlook protocol code, consumed by both desktop (Node) and Android (WebView) — eliminates the parallel mailx-imap/mailx-store-web Gmail providers that drifted in practice.",
   "main": "index.js",
   "types": "index.ts",
@@ -19,8 +19,8 @@
   "author": "Bob Frankston",
   "license": "ISC",
   "dependencies": {
-    "@bobfrankston/tcp-transport": "^0.1.3",
-    "@bobfrankston/iflow-direct": "^0.1.20"
+    "@bobfrankston/iflow-direct": "^0.1.22",
+    "@bobfrankston/tcp-transport": "^0.1.4"
   },
   "exports": {
     ".": {
@@ -35,14 +35,17 @@
     "type": "git",
     "url": "https://github.com/BobFrankston/mailx-sync.git"
   },
+  "devDependencies": {
+    "@types/node": "^25.6.0"
+  },
   ".dependencies": {
-    "@bobfrankston/tcp-transport": "file:../tcp-transport",
-    "@bobfrankston/iflow-direct": "file:../iflow-direct"
+    "@bobfrankston/iflow-direct": "file:../iflow-direct",
+    "@bobfrankston/tcp-transport": "file:../tcp-transport"
   },
   ".transformedSnapshot": {
     "dependencies": {
-      "@bobfrankston/tcp-transport": "^0.1.3",
-      "@bobfrankston/iflow-direct": "^0.1.20"
+      "@bobfrankston/iflow-direct": "^0.1.22",
+      "@bobfrankston/tcp-transport": "^0.1.4"
     }
   }
 }

package/types.d.ts

@@ -60,6 +60,11 @@ export interface MailProvider {
     fetchOne(folder: string, uid: number, options?: FetchOptions): Promise<ProviderMessage | null>;
     /** Get all UIDs in a folder (for reconciliation) */
     getUids(folder: string): Promise<number[]>;
+    /** Replace the full flag set on a message (idempotent). The provider is
+     *  responsible for translating IMAP flags like "\\Seen" / "\\Flagged" to
+     *  its native model — e.g. Gmail's UNREAD / STARRED labels.
+     *  Optional: IMAP uses the existing STORE path in sync-manager code. */
+    setFlags?(folder: string, uid: number, flags: string[]): Promise<void>;
     /** Close/cleanup */
     close(): Promise<void>;
 }