@bobfrankston/iflow-direct 0.1.33 → 0.1.35

package/imap-native.d.ts

@@ -48,7 +48,15 @@ export declare class NativeImapClient {
     private transport;
     private transportFactory;
     private config;
+    /** Receive buffer + read offset. CRITICAL: never reassign with
+     *  `this.buffer = this.buffer.substring(N)` — that copies the whole
+     *  remainder per line and is O(n²) over a multi-line response (a 457KB
+     *  LIST response actually blocked the daemon's event loop for 10 minutes
+     *  in production). Use bufferOffset to advance the read cursor; only
+     *  compact the buffer string when the consumed portion is >= half its
+     *  length, keeping per-line work O(1). */
     private buffer;
+    private bufferOffset;
     private pendingCommand;
     private capabilities;
     private _connected;
@@ -63,6 +71,13 @@ export declare class NativeImapClient {
     private greetingResolve;
     /** Callback for waitForContinuation — set when waiting for "+" response */
     private continuationResolve;
+    /** Per-command heartbeat (30s). Hoisted to instance state so any path
+     *  that swaps pendingCommand can clear it cleanly — closure-scoped
+     *  timers leaked when handleData's inline replacement or IDLE
+     *  suspend/resume bypassed sendCommandCore's resolve/reject closures. */
+    private heartbeatTimer;
+    /** Hard wall-clock cap per command. Same hoist for the same reason. */
+    private wallClockTimer;
     constructor(config: ImapClientConfig, transportFactory: TransportFactory);
     get connected(): boolean;
     /** Check the underlying transport's connected state — catches silently dead sockets. */
@@ -203,6 +218,17 @@ export declare class NativeImapClient {
     private waitForContinuation;
     private waitForTagged;
     private handleData;
+    /** Single canonical "command finished or abandoned" cleanup. Every
+     *  path that ends a command's lifecycle — resolve, reject, timeout,
+     *  socket close, command-replace — calls this. Was a leak source:
+     *  closure-scoped timers in sendCommandCore couldn't be cleared from
+     *  handleData's inline-replacement timer or from IDLE suspend/resume. */
+    private clearAllCommandTimers;
+    /** Compact the buffer when bufferOffset has advanced past half. Cheap
+     *  amortized cost; eliminates the O(n²) substring-each-line antipattern
+     *  that synchronously blocked the daemon for 10 minutes on a large LIST
+     *  response. */
+    private compactBufferIfNeeded;
     private processBuffer;
     private handleUntaggedResponse;
     private parseFetchResponses;

package/imap-native.js

@@ -11,7 +11,15 @@ export class NativeImapClient {
     transport;
     transportFactory;
     config;
+    /** Receive buffer + read offset. CRITICAL: never reassign with
+     *  `this.buffer = this.buffer.substring(N)` — that copies the whole
+     *  remainder per line and is O(n²) over a multi-line response (a 457KB
+     *  LIST response actually blocked the daemon's event loop for 10 minutes
+     *  in production). Use bufferOffset to advance the read cursor; only
+     *  compact the buffer string when the consumed portion is >= half its
+     *  length, keeping per-line work O(1). */
     buffer = "";
+    bufferOffset = 0;
     pendingCommand = null;
     capabilities = new Set();
     _connected = false;
@@ -26,16 +34,29 @@ export class NativeImapClient {
     greetingResolve = null;
     /** Callback for waitForContinuation — set when waiting for "+" response */
     continuationResolve = null;
+    /** Per-command heartbeat (30s). Hoisted to instance state so any path
+     *  that swaps pendingCommand can clear it cleanly — closure-scoped
+     *  timers leaked when handleData's inline replacement or IDLE
+     *  suspend/resume bypassed sendCommandCore's resolve/reject closures. */
+    heartbeatTimer = null;
+    /** Hard wall-clock cap per command. Same hoist for the same reason. */
+    wallClockTimer = null;
     constructor(config, transportFactory) {
         this.config = config;
         this.transportFactory = transportFactory;
         this.transport = transportFactory();
         this.verbose = config.verbose || false;
         this.inactivityTimeout = config.inactivityTimeout ?? 60000;
-        // Hard wall-clock deadline per command (default 5 min). Distinct
-        // from inactivityTimeout — that one resets on each data chunk and
-        // can be deferred indefinitely by a slow-trickling server.
-        this.commandWallClockTimeout = config.commandWallClockTimeout ?? 300_000;
+        // Hard wall-clock deadline per command. Distinct from inactivityTimeout
+        // — that one resets on each data chunk and can be deferred indefinitely
+        // by a slow-trickling server. Default 90s: in production we see
+        // connections silently die with sinceLastRead climbing for 50s+ on
+        // FETCH; 5 min was leaving them wedged for too long, blocking sync /
+        // prefetch / new-mail behind a dead socket. 90s lets a slow but
+        // genuine FETCH complete (Sent in isolation: 26ms; even a 50-message
+        // FETCH was <1s) while killing dead sockets aggressively. Override
+        // via ImapClientConfig.commandWallClockTimeout for known-slow servers.
+        this.commandWallClockTimeout = config.commandWallClockTimeout ?? 90_000;
         this.fetchChunkSize = config.fetchChunkSize ?? 25;
         this.fetchChunkSizeMax = config.fetchChunkSizeMax ?? 500;
         this.greetingTimeout = config.greetingTimeout ?? 10000;
@@ -53,8 +74,13 @@ export class NativeImapClient {
         this.transport = this.transportFactory();
         this._connected = false;
         this.buffer = "";
+        this.bufferOffset = 0;
         this.pendingCommand = null;
         this.selectedMailbox = null;
+        // Old socket is gone — kill any timers still running for the
+        // command that was on it. Without this, a stale heartbeat/wall-clock
+        // continues firing into a transport that no longer exists.
+        this.clearAllCommandTimers();
     }
     // ── Connection ──
     async connect() {
@@ -70,6 +96,10 @@ export class NativeImapClient {
                 clearTimeout(this.idleRefreshTimer);
                 this.idleRefreshTimer = null;
             }
+            // Kill any per-command timers — the connection is gone, no
+            // command on it can ever complete. Otherwise heartbeat keeps
+            // firing for a connection that no longer exists.
+            this.clearAllCommandTimers();
             if (this.pendingCommand) {
                 const { reject } = this.pendingCommand;
                 this.pendingCommand = null;
@@ -793,69 +823,33 @@ export class NativeImapClient {
                     return "";
                 }
             };
-            // HEARTBEAT_INTERVAL_MS — periodic progress log while a command
-            // is pending. Catches wedges in flight rather than only at the
-            // 300s timeout edge. Key signal: if `sinceLastRead` keeps
-            // climbing, the socket is silently dead. If reads are happening
-            // but the tagged OK never arrives, the parser is stuck.
-            const HEARTBEAT_INTERVAL_MS = 30_000;
+            // Always start clean — defends against a prior pendingCommand
+            // having leaked timers (e.g. swap by suspendIdle/resumeIdle that
+            // bypassed resolve/reject).
+            this.clearAllCommandTimers();
             const cmdStart = Date.now();
             const cmdSummary = command.split("\r")[0].substring(0, 80);
-            let heartbeatTimer = setInterval(() => {
-                const waited = Date.now() - cmdStart;
-                console.log(`  [imap] still waiting for tag ${tag} after ${(waited / 1000).toFixed(1)}s — ${cmdSummary}${transportDiag()}`);
-            }, HEARTBEAT_INTERVAL_MS);
             const onTimeout = (kind) => {
-                this.commandTimer = null;
+                this.clearAllCommandTimers();
                 this.pendingCommand = null;
-                if (heartbeatTimer) {
-                    clearInterval(heartbeatTimer);
-                    heartbeatTimer = null;
-                }
-                if (wallClockTimer) {
-                    clearTimeout(wallClockTimer);
-                    wallClockTimer = null;
-                }
-                // Kill the connection — a timed-out connection has stale data in the pipe.
-                // Mark disconnected so ensureConnected() reconnects on the next call —
-                // transport.close() may or may not fire onClose synchronously.
                 this._connected = false;
                 this.transport.close?.();
                 reject(new Error(`${kind}: ${cmdSummary}${transportDiag()}`));
             };
-            // Two timers, two responsibilities:
-            //   commandTimer = "no bytes for N seconds" — handleData resets it
-            //                  on every chunk so a steadily-streaming FETCH
-            //                  doesn't get killed mid-flight.
-            //   wallClockTimer = "this command has been pending too long
-            //                  total" — fires regardless of trickling bytes.
-            //                  Catches the pathological case where a server
-            //                  drips one byte every <inactivityTimeout> seconds
-            //                  for hours (real bobma bug we hit overnight:
-            //                  3 connections wedged 2-3 hours each because
-            //                  the inactivity reset kept deferring the only
-            //                  timer in play).
+            // Three timers, three responsibilities — all instance-scoped now:
+            //   commandTimer  = inactivity (resets on each data chunk)
+            //   wallClockTimer = hard cap from command start, never resets
+            //   heartbeatTimer = periodic progress log every 30s
             this.commandTimer = setTimeout(() => onTimeout(`IMAP inactivity timeout (${this.inactivityTimeout / 1000}s)`), this.inactivityTimeout);
-            const COMMAND_WALL_CLOCK_TIMEOUT_MS = this.commandWallClockTimeout;
-            let wallClockTimer = setTimeout(() => onTimeout(`IMAP command wall-clock timeout (${COMMAND_WALL_CLOCK_TIMEOUT_MS / 1000}s)`), COMMAND_WALL_CLOCK_TIMEOUT_MS);
-            const clearTimers = () => {
-                if (this.commandTimer) {
-                    clearTimeout(this.commandTimer);
-                    this.commandTimer = null;
-                }
-                if (heartbeatTimer) {
-                    clearInterval(heartbeatTimer);
-                    heartbeatTimer = null;
-                }
-                if (wallClockTimer) {
-                    clearTimeout(wallClockTimer);
-                    wallClockTimer = null;
-                }
-            };
+            this.wallClockTimer = setTimeout(() => onTimeout(`IMAP command wall-clock timeout (${this.commandWallClockTimeout / 1000}s)`), this.commandWallClockTimeout);
+            this.heartbeatTimer = setInterval(() => {
+                const waited = Date.now() - cmdStart;
+                console.log(`  [imap] still waiting for tag ${tag} after ${(waited / 1000).toFixed(1)}s — ${cmdSummary}${transportDiag()}`);
+            }, 30_000);
             this.pendingCommand = {
                 tag, responses: [],
-                resolve: (responses) => { clearTimers(); resolve(responses); },
-                reject: (err) => { clearTimers(); reject(err); },
+                resolve: (responses) => { this.clearAllCommandTimers(); resolve(responses); },
+                reject: (err) => { this.clearAllCommandTimers(); reject(err); },
                 onUntagged,
             };
             // PRE-WRITE HEALTH CHECK — verify the socket is in a writable
@@ -865,7 +859,7 @@ export class NativeImapClient {
             try {
                 const h = this.transport?.socketHealth;
                 if (h && (h.destroyed || h.writableEnded || !h.writable)) {
-                    clearTimers();
+                    this.clearAllCommandTimers();
                     this.pendingCommand = null;
                     this._connected = false;
                     reject(new Error(`IMAP socket dead before write: destroyed=${h.destroyed} writableEnded=${h.writableEnded} writable=${h.writable}${transportDiag()}`));
@@ -874,7 +868,7 @@ export class NativeImapClient {
             }
             catch { /* transport may not expose socketHealth — fall through */ }
             this.transport.write(command).catch((err) => {
-                clearTimers();
+                this.clearAllCommandTimers();
                 // Write failure = dead socket. Clear pendingCommand + disconnect flag
                 // so the next ensureConnected() does a fresh connect instead of
                 // reusing a socket that the OS layer has already declared dead
@@ -904,7 +898,10 @@ export class NativeImapClient {
         });
     }
     handleData(data) {
-        // Reset inactivity timer — data is flowing, connection is alive
+        // Reset inactivity timer ONLY (heartbeat + wall-clock NEVER reset
+        // on data — that was the original bug: wall-clock got reset every
+        // byte, so a slow-trickling server's command never died). Inactivity
+        // is "no data for N seconds"; data IS arriving, so push it out.
         if (this.commandTimer) {
             clearTimeout(this.commandTimer);
             this.commandTimer = setTimeout(() => {
@@ -912,6 +909,11 @@ export class NativeImapClient {
                 if (this.pendingCommand) {
                     const cmd = this.pendingCommand;
                     this.pendingCommand = null;
+                    // Use the canonical clear so heartbeat + wall-clock
+                    // also stop. cmd.reject runs clearAllCommandTimers via
+                    // the closure binding — call it directly here too in
+                    // case cmd.reject was already replaced (defense in depth).
+                    this.clearAllCommandTimers();
                     this._connected = false;
                     this.transport.close?.();
                     cmd.reject(new Error(`IMAP inactivity timeout (${this.inactivityTimeout / 1000}s)`));
@@ -921,6 +923,35 @@ export class NativeImapClient {
         this.buffer += data;
         this.processBuffer();
     }
+    /** Single canonical "command finished or abandoned" cleanup. Every
+     *  path that ends a command's lifecycle — resolve, reject, timeout,
+     *  socket close, command-replace — calls this. Was a leak source:
+     *  closure-scoped timers in sendCommandCore couldn't be cleared from
+     *  handleData's inline-replacement timer or from IDLE suspend/resume. */
+    clearAllCommandTimers() {
+        if (this.commandTimer) {
+            clearTimeout(this.commandTimer);
+            this.commandTimer = null;
+        }
+        if (this.heartbeatTimer) {
+            clearInterval(this.heartbeatTimer);
+            this.heartbeatTimer = null;
+        }
+        if (this.wallClockTimer) {
+            clearTimeout(this.wallClockTimer);
+            this.wallClockTimer = null;
+        }
+    }
+    /** Compact the buffer when bufferOffset has advanced past half. Cheap
+     *  amortized cost; eliminates the O(n²) substring-each-line antipattern
+     *  that synchronously blocked the daemon for 10 minutes on a large LIST
+     *  response. */
+    compactBufferIfNeeded() {
+        if (this.bufferOffset > 0 && this.bufferOffset >= this.buffer.length / 2) {
+            this.buffer = this.buffer.substring(this.bufferOffset);
+            this.bufferOffset = 0;
+        }
+    }
     processBuffer() {
         while (true) {
             // Check for literal {size}\r\n — reading exact BYTE count of literal data
@@ -929,16 +960,20 @@ export class NativeImapClient {
             // We must use byte-accurate extraction (TextEncoder for browser, Buffer for Node).
             if (this.pendingCommand?.literalBytes != null) {
                 const neededBytes = this.pendingCommand.literalBytes;
-                const encoded = new TextEncoder().encode(this.buffer);
+                // Encode only the unread portion — was encoding the whole
+                // buffer (including already-consumed bytes) on every literal,
+                // which on a multi-MB buffer is a non-trivial chunk of work.
+                const unread = this.bufferOffset === 0 ? this.buffer : this.buffer.substring(this.bufferOffset);
+                const encoded = new TextEncoder().encode(unread);
                 const bufferBytes = encoded.byteLength;
                 if (bufferBytes >= neededBytes) {
                     const literal_bytes = encoded.subarray(0, neededBytes);
                     const rest_bytes = encoded.subarray(neededBytes);
                     let literal = new TextDecoder().decode(literal_bytes);
+                    // Reset buffer to just the rest — bufferOffset goes back
+                    // to 0 since we replaced the buffer content.
                     this.buffer = new TextDecoder().decode(rest_bytes);
-                    // if (neededBytes !== literal.length) {
-                    //     console.log(`iflow-direct:  [imap] literal: ${neededBytes} bytes → ${literal.length} chars (multi-byte corrected)`);
-                    // }
+                    this.bufferOffset = 0;
                     // For non-BODY literals (e.g. display names in ENVELOPE), wrap in quotes
                     // so tokenizeParenList treats them as a single token
                     if (!this.pendingCommand.currentLiteralKey) {
@@ -966,11 +1001,23 @@ export class NativeImapClient {
                 }
                 break; // Wait for more data
             }
-            const lineEnd = this.buffer.indexOf("\r\n");
-            if (lineEnd < 0)
+            // Find next CRLF starting from the read offset (NOT from 0 —
+            // the bytes before bufferOffset are already consumed and
+            // searching them is dead work that scales with response size).
+            const lineEnd = this.buffer.indexOf("\r\n", this.bufferOffset);
+            if (lineEnd < 0) {
+                // No complete line yet. Compact the buffer if we've advanced
+                // far enough that the unconsumed-prefix is dragging.
+                this.compactBufferIfNeeded();
                 break;
-            const line = this.buffer.substring(0, lineEnd + 2);
-            this.buffer = this.buffer.substring(lineEnd + 2);
+            }
+            // Extract the line and advance the read cursor — no substring
+            // copy of the remainder, no buffer reallocation.
+            const line = this.buffer.substring(this.bufferOffset, lineEnd + 2);
+            this.bufferOffset = lineEnd + 2;
+            // Periodic compaction so the consumed prefix doesn't keep
+            // growing forever in long-running connections (IDLE, big sync).
+            this.compactBufferIfNeeded();
             // Check for literal announcement {size}\r\n at end of line
             const literalMatch = line.match(/\{(\d+)\}\r\n$/);
             if (literalMatch && this.pendingCommand) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bobfrankston/iflow-direct",
-  "version": "0.1.33",
+  "version": "0.1.35",
   "description": "Direct IMAP client — transport-agnostic, no Node.js dependencies, browser-ready",
   "main": "index.js",
   "types": "index.ts",