@bobfrankston/iflow-direct 0.1.35 → 0.1.39

package/imap-compat.d.ts

@@ -66,6 +66,12 @@ export declare class CompatImapClient {
     getMessagesCount(mailbox: string): Promise<number>;
     /** Get all UIDs in a mailbox */
     getUids(mailbox: string): Promise<number[]>;
+    /** Get UIDs whose INTERNALDATE is on/after `since`. Bounded version of
+     *  getUids — returns only the date window the caller cares about
+     *  instead of the entire folder. Lets set-diff reconciliation scope
+     *  itself to "messages from the last N days" rather than enumerating
+     *  every UID in a 134k-message folder. */
+    getUidsSince(mailbox: string, since: Date): Promise<number[]>;
     /** Fetch messages — supports two calling conventions for compatibility:
      *
      *    New:  fetchMessages(mailbox, "100:200")           — UID range string

package/imap-compat.js

@@ -168,6 +168,20 @@ export class CompatImapClient {
         await this.native.closeMailbox();
         return uids;
     }
+    /** Get UIDs whose INTERNALDATE is on/after `since`. Bounded version of
+     *  getUids — returns only the date window the caller cares about
+     *  instead of the entire folder. Lets set-diff reconciliation scope
+     *  itself to "messages from the last N days" rather than enumerating
+     *  every UID in a 134k-message folder. */
+    async getUidsSince(mailbox, since) {
+        await this.ensureConnected();
+        await this.native.select(mailbox);
+        const months = ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"];
+        const d = `${since.getUTCDate()}-${months[since.getUTCMonth()]}-${since.getUTCFullYear()}`;
+        const uids = await this.native.search(`SINCE ${d}`);
+        await this.native.closeMailbox();
+        return uids;
+    }
     async fetchMessages(mailbox, rangeOrEnd, countOrOptions, maybeOptions) {
         let range;
         let options;

package/imap-native.d.ts

@@ -48,15 +48,29 @@ export declare class NativeImapClient {
     private transport;
     private transportFactory;
     private config;
-    /** Receive buffer + read offset. CRITICAL: never reassign with
-     *  `this.buffer = this.buffer.substring(N)` — that copies the whole
-     *  remainder per line and is O(n²) over a multi-line response (a 457KB
-     *  LIST response actually blocked the daemon's event loop for 10 minutes
-     *  in production). Use bufferOffset to advance the read cursor; only
-     *  compact the buffer string when the consumed portion is >= half its
-     *  length, keeping per-line work O(1). */
+    /** Byte-level receive buffer. IMAP literals (`{N}`) are octets — message
+     *  bodies, MIME parts with binary content, non-UTF-8 charsets — and the
+     *  parser MUST work in bytes to recover them with byte fidelity and
+     *  without paying O(n²) re-encoding cost on every chunk arrival. The
+     *  prior implementation kept `this.buffer` as a JS string, accumulated
+     *  UTF-8-decoded text from the transport, then `TextEncoder.encode`'d
+     *  the whole unread region on every TCP chunk while waiting for a
+     *  literal — pegged the event loop on multi-message FETCH responses.
+     *
+     *  Invariants:
+     *    - `buffer[0 .. bufferLength)` holds valid received bytes
+     *    - `buffer[bufferOffset .. bufferLength)` is unread
+     *    - capacity = `buffer.length` ≥ `bufferLength`; we grow on demand
+     *    - never substring — bufferOffset advances past consumed bytes;
+     *      compactBufferIfNeeded reclaims space when consumed >= half. */
     private buffer;
+    private bufferLength;
     private bufferOffset;
+    /** Cached decoder so per-line / per-literal decode doesn't re-construct
+     *  one. `fatal: false` (default) replaces invalid UTF-8 sequences with
+     *  U+FFFD — same lenient behavior as JavaScript's implicit string
+     *  decoding the parser used to rely on. */
+    private utf8Decoder;
     private pendingCommand;
     private capabilities;
     private _connected;
@@ -213,22 +227,47 @@ export declare class NativeImapClient {
      * EXISTS callback (e.g. mailpuller's pullMail) would be ignored by the
      * server until the inactivity timer killed the socket.
      */
+    /** Per-client command serialization. IMAP allows only one command in
+     *  flight at a time on a single connection (RFC 3501 §2.2.1). The
+     *  client's `pendingCommand` field tracks that command — if a second
+     *  caller concurrently invokes sendCommand, it overwrites
+     *  pendingCommand and the first caller's await is orphaned. We saw
+     *  this in production: the outbox poller's `withConnection`-queued
+     *  task ran on the same shared ops client as syncFolder's INBOX work,
+     *  and INBOX silently hung forever (no heartbeat fire, no wall-clock,
+     *  no reject — the promise just had no command tracking it anymore).
+     *
+     *  This chain serializes ALL sendCommand callers against this client
+     *  regardless of whether the caller bothered to coordinate.
+     *  Defensive — the right thing for the upper layer to do is also use
+     *  the queue (withConnection in mailx-imap), but the IMAP client
+     *  shouldn't allow itself to be corrupted by a careless caller. */
+    private commandChain;
     private sendCommand;
     private sendCommandCore;
     private waitForContinuation;
     private waitForTagged;
     private handleData;
+    /** Append received bytes to the receive buffer, growing capacity if
+     *  needed. O(n) over the chunk size, never over the buffer total —
+     *  doubling growth amortizes to O(1) per byte across the response. */
+    private appendToBuffer;
     /** Single canonical "command finished or abandoned" cleanup. Every
      *  path that ends a command's lifecycle — resolve, reject, timeout,
      *  socket close, command-replace — calls this. Was a leak source:
      *  closure-scoped timers in sendCommandCore couldn't be cleared from
      *  handleData's inline-replacement timer or from IDLE suspend/resume. */
     private clearAllCommandTimers;
-    /** Compact the buffer when bufferOffset has advanced past half. Cheap
-     *  amortized cost; eliminates the O(n²) substring-each-line antipattern
-     *  that synchronously blocked the daemon for 10 minutes on a large LIST
-     *  response. */
+    /** Compact the buffer when bufferOffset has advanced past half the used
+     *  region. copyWithin moves the unread bytes to the front in-place
+     *  (no new allocation). Amortized O(1) per byte; never the O(n²)
+     *  substring-each-line pathology that blocked the event loop on a
+     *  large LIST response. */
     private compactBufferIfNeeded;
+    /** Find the next CRLF (0x0D 0x0A) in the unread region, returning the
+     *  byte index of the 0x0D, or -1 if not found. Manual byte loop —
+     *  Uint8Array doesn't have a native indexOf for byte sequences. */
+    private indexOfCRLF;
     private processBuffer;
     private handleUntaggedResponse;
     private parseFetchResponses;

package/imap-native.js

@@ -11,15 +11,29 @@ export class NativeImapClient {
     transport;
     transportFactory;
     config;
-    /** Receive buffer + read offset. CRITICAL: never reassign with
-     *  `this.buffer = this.buffer.substring(N)` — that copies the whole
-     *  remainder per line and is O(n²) over a multi-line response (a 457KB
-     *  LIST response actually blocked the daemon's event loop for 10 minutes
-     *  in production). Use bufferOffset to advance the read cursor; only
-     *  compact the buffer string when the consumed portion is >= half its
-     *  length, keeping per-line work O(1). */
-    buffer = "";
+    /** Byte-level receive buffer. IMAP literals (`{N}`) are octets — message
+     *  bodies, MIME parts with binary content, non-UTF-8 charsets — and the
+     *  parser MUST work in bytes to recover them with byte fidelity and
+     *  without paying O(n²) re-encoding cost on every chunk arrival. The
+     *  prior implementation kept `this.buffer` as a JS string, accumulated
+     *  UTF-8-decoded text from the transport, then `TextEncoder.encode`'d
+     *  the whole unread region on every TCP chunk while waiting for a
+     *  literal — pegged the event loop on multi-message FETCH responses.
+     *
+     *  Invariants:
+     *    - `buffer[0 .. bufferLength)` holds valid received bytes
+     *    - `buffer[bufferOffset .. bufferLength)` is unread
+     *    - capacity = `buffer.length` ≥ `bufferLength`; we grow on demand
+     *    - never substring — bufferOffset advances past consumed bytes;
+     *      compactBufferIfNeeded reclaims space when consumed >= half. */
+    buffer = new Uint8Array(8192);
+    bufferLength = 0;
     bufferOffset = 0;
+    /** Cached decoder so per-line / per-literal decode doesn't re-construct
+     *  one. `fatal: false` (default) replaces invalid UTF-8 sequences with
+     *  U+FFFD — same lenient behavior as JavaScript's implicit string
+     *  decoding the parser used to rely on. */
+    utf8Decoder = new TextDecoder("utf-8");
     pendingCommand = null;
     capabilities = new Set();
     _connected = false;
@@ -73,7 +87,7 @@ export class NativeImapClient {
         catch { /* ignore */ }
         this.transport = this.transportFactory();
         this._connected = false;
-        this.buffer = "";
+        this.bufferLength = 0;
         this.bufferOffset = 0;
         this.pendingCommand = null;
         this.selectedMailbox = null;
@@ -788,22 +802,52 @@ export class NativeImapClient {
      * EXISTS callback (e.g. mailpuller's pullMail) would be ignored by the
      * server until the inactivity timer killed the socket.
      */
+    /** Per-client command serialization. IMAP allows only one command in
+     *  flight at a time on a single connection (RFC 3501 §2.2.1). The
+     *  client's `pendingCommand` field tracks that command — if a second
+     *  caller concurrently invokes sendCommand, it overwrites
+     *  pendingCommand and the first caller's await is orphaned. We saw
+     *  this in production: the outbox poller's `withConnection`-queued
+     *  task ran on the same shared ops client as syncFolder's INBOX work,
+     *  and INBOX silently hung forever (no heartbeat fire, no wall-clock,
+     *  no reject — the promise just had no command tracking it anymore).
+     *
+     *  This chain serializes ALL sendCommand callers against this client
+     *  regardless of whether the caller bothered to coordinate.
+     *  Defensive — the right thing for the upper layer to do is also use
+     *  the queue (withConnection in mailx-imap), but the IMAP client
+     *  shouldn't allow itself to be corrupted by a careless caller. */
+    commandChain = Promise.resolve();
     async sendCommand(tag, command, onUntagged) {
-        const idleState = await this.suspendIdleForCommand();
+        const prev = this.commandChain.catch(() => { });
+        let myResolve;
+        let myReject;
+        const myPromise = new Promise((res, rej) => { myResolve = res; myReject = rej; });
+        this.commandChain = myPromise.catch(() => { });
+        await prev;
         try {
-            return await this.sendCommandCore(tag, command, onUntagged);
-        }
-        finally {
-            if (idleState) {
-                try {
-                    await this.resumeIdleAfterCommand(idleState);
-                }
-                catch (err) {
-                    if (this.verbose)
-                        console.error(`  [imap] IDLE auto-resume failed: ${err.message}`);
+            const idleState = await this.suspendIdleForCommand();
+            try {
+                const out = await this.sendCommandCore(tag, command, onUntagged);
+                myResolve(out);
+                return out;
+            }
+            finally {
+                if (idleState) {
+                    try {
+                        await this.resumeIdleAfterCommand(idleState);
+                    }
+                    catch (err) {
+                        if (this.verbose)
+                            console.error(`  [imap] IDLE auto-resume failed: ${err.message}`);
+                    }
                 }
             }
         }
+        catch (e) {
+            myReject(e);
+            throw e;
+        }
     }
     sendCommandCore(tag, command, onUntagged) {
         return new Promise((resolve, reject) => {
@@ -920,9 +964,25 @@ export class NativeImapClient {
                 }
             }, this.inactivityTimeout);
         }
-        this.buffer += data;
+        this.appendToBuffer(data);
         this.processBuffer();
     }
+    /** Append received bytes to the receive buffer, growing capacity if
+     *  needed. O(n) over the chunk size, never over the buffer total —
+     *  doubling growth amortizes to O(1) per byte across the response. */
+    appendToBuffer(data) {
+        const need = this.bufferLength + data.length;
+        if (need > this.buffer.length) {
+            let cap = Math.max(this.buffer.length * 2, 8192);
+            while (cap < need)
+                cap *= 2;
+            const grown = new Uint8Array(cap);
+            grown.set(this.buffer.subarray(0, this.bufferLength));
+            this.buffer = grown;
+        }
+        this.buffer.set(data, this.bufferLength);
+        this.bufferLength += data.length;
+    }
     /** Single canonical "command finished or abandoned" cleanup. Every
      *  path that ends a command's lifecycle — resolve, reject, timeout,
      *  socket close, command-replace — calls this. Was a leak source:
@@ -942,78 +1002,99 @@ export class NativeImapClient {
             this.wallClockTimer = null;
         }
     }
-    /** Compact the buffer when bufferOffset has advanced past half. Cheap
-     *  amortized cost; eliminates the O(n²) substring-each-line antipattern
-     *  that synchronously blocked the daemon for 10 minutes on a large LIST
-     *  response. */
+    /** Compact the buffer when bufferOffset has advanced past half the used
+     *  region. copyWithin moves the unread bytes to the front in-place
+     *  (no new allocation). Amortized O(1) per byte; never the O(n²)
+     *  substring-each-line pathology that blocked the event loop on a
+     *  large LIST response. */
     compactBufferIfNeeded() {
-        if (this.bufferOffset > 0 && this.bufferOffset >= this.buffer.length / 2) {
-            this.buffer = this.buffer.substring(this.bufferOffset);
+        if (this.bufferOffset > 0 && this.bufferOffset >= this.bufferLength / 2) {
+            this.buffer.copyWithin(0, this.bufferOffset, this.bufferLength);
+            this.bufferLength -= this.bufferOffset;
             this.bufferOffset = 0;
         }
     }
+    /** Find the next CRLF (0x0D 0x0A) in the unread region, returning the
+     *  byte index of the 0x0D, or -1 if not found. Manual byte loop —
+     *  Uint8Array doesn't have a native indexOf for byte sequences. */
+    indexOfCRLF(from) {
+        const buf = this.buffer;
+        const end = this.bufferLength - 1;
+        for (let i = from; i < end; i++) {
+            if (buf[i] === 0x0D && buf[i + 1] === 0x0A)
+                return i;
+        }
+        return -1;
+    }
     processBuffer() {
         while (true) {
-            // Check for literal {size}\r\n — reading exact BYTE count of literal data
-            // CRITICAL: literalBytes is in octets (from IMAP {N}), but this.buffer is a
-            // JavaScript string where multi-byte UTF-8 characters count as 1 character.
-            // We must use byte-accurate extraction (TextEncoder for browser, Buffer for Node).
+            // Check for literal — `{N}` announces N octets of arbitrary data
+            // (message body, MIME part, attachment chunk). The buffer is
+            // bytes; the check is a direct length compare and the slice is
+            // O(N) over the literal's own size — never O(buffer²).
             if (this.pendingCommand?.literalBytes != null) {
                 const neededBytes = this.pendingCommand.literalBytes;
-                // Encode only the unread portion — was encoding the whole
-                // buffer (including already-consumed bytes) on every literal,
-                // which on a multi-MB buffer is a non-trivial chunk of work.
-                const unread = this.bufferOffset === 0 ? this.buffer : this.buffer.substring(this.bufferOffset);
-                const encoded = new TextEncoder().encode(unread);
-                const bufferBytes = encoded.byteLength;
-                if (bufferBytes >= neededBytes) {
-                    const literal_bytes = encoded.subarray(0, neededBytes);
-                    const rest_bytes = encoded.subarray(neededBytes);
-                    let literal = new TextDecoder().decode(literal_bytes);
-                    // Reset buffer to just the rest — bufferOffset goes back
-                    // to 0 since we replaced the buffer content.
-                    this.buffer = new TextDecoder().decode(rest_bytes);
-                    this.bufferOffset = 0;
-                    // For non-BODY literals (e.g. display names in ENVELOPE), wrap in quotes
-                    // so tokenizeParenList treats them as a single token
-                    if (!this.pendingCommand.currentLiteralKey) {
-                        literal = `"${literal.replace(/\\/g, "\\\\").replace(/"/g, '\\"').replace(/\r?\n/g, "")}"`;
+                const available = this.bufferLength - this.bufferOffset;
+                if (available < neededBytes) {
+                    if (this.verbose && this.pendingCommand.literalBytes > 0) {
+                        console.log(`  [imap] waiting for literal: need ${neededBytes} bytes, have ${available}`);
                     }
-                    this.pendingCommand.literalBuffer = (this.pendingCommand.literalBuffer || "") + literal;
-                    this.pendingCommand.literalBytes = undefined;
-                    // Store the literal data by its BODY key for parseFetchResponses
-                    if (this.pendingCommand.currentLiteralKey) {
-                        if (!this.pendingCommand.literals)
-                            this.pendingCommand.literals = new Map();
-                        this.pendingCommand.literals.set(this.pendingCommand.currentLiteralKey, literal);
-                        this.pendingCommand.currentLiteralKey = undefined;
-                        this.pendingCommand.currentLiteralSize = undefined;
-                    }
-                    if (this.verbose)
-                        console.log(`  [imap] literal consumed, ${literal.length} bytes, buffer remaining: ${this.buffer.length}`);
-                    // After consuming literal, check if the NEXT part has another literal
-                    // (e.g., BODY[HEADER] {500}\r\n<data>BODY[] {2000}\r\n<data>)\r\n)
-                    // Continue to process the next line/literal from the buffer
-                    continue;
+                    break; // Wait for more data — cheap, no decode.
+                }
+                // Slice the literal bytes; advance the read cursor by N.
+                // The bytes themselves stay in the underlying buffer for the
+                // moment, but bufferOffset advances past them and the next
+                // compactBufferIfNeeded will reclaim the space.
+                const literalBytes = this.buffer.subarray(this.bufferOffset, this.bufferOffset + neededBytes);
+                this.bufferOffset += neededBytes;
+                // Decode literal once (UTF-8 with replacement). For BODY[]
+                // literals carrying non-UTF-8 charsets this is still lossy —
+                // the proper fix is to keep literals as bytes through to the
+                // body store, tracked separately. The win HERE is that the
+                // decode happens once per literal, not once per chunk that
+                // arrives during the literal's transit.
+                let literal = this.utf8Decoder.decode(literalBytes);
+                // For non-BODY literals (e.g. display names in ENVELOPE), wrap in quotes
+                // so tokenizeParenList treats them as a single token
+                if (!this.pendingCommand.currentLiteralKey) {
+                    literal = `"${literal.replace(/\\/g, "\\\\").replace(/"/g, '\\"').replace(/\r?\n/g, "")}"`;
                 }
-                if (this.verbose && this.pendingCommand.literalBytes > 0) {
-                    console.log(`  [imap] waiting for literal: need ${neededBytes} bytes, have ${bufferBytes} bytes`);
+                this.pendingCommand.literalBuffer = (this.pendingCommand.literalBuffer || "") + literal;
+                this.pendingCommand.literalBytes = undefined;
+                // Store the literal data by its BODY key for parseFetchResponses
+                if (this.pendingCommand.currentLiteralKey) {
+                    if (!this.pendingCommand.literals)
+                        this.pendingCommand.literals = new Map();
+                    this.pendingCommand.literals.set(this.pendingCommand.currentLiteralKey, literal);
+                    this.pendingCommand.currentLiteralKey = undefined;
+                    this.pendingCommand.currentLiteralSize = undefined;
                 }
-                break; // Wait for more data
+                if (this.verbose)
+                    console.log(`  [imap] literal consumed, ${neededBytes} bytes, buffer remaining: ${this.bufferLength - this.bufferOffset}`);
+                // Allow the consumed-prefix to be reclaimed when half-full —
+                // long literal-heavy responses would otherwise grow the
+                // backing buffer indefinitely between commands.
+                this.compactBufferIfNeeded();
+                continue;
             }
-            // Find next CRLF starting from the read offset (NOT from 0 —
-            // the bytes before bufferOffset are already consumed and
-            // searching them is dead work that scales with response size).
-            const lineEnd = this.buffer.indexOf("\r\n", this.bufferOffset);
+            // Find next CRLF starting from the read offset on bytes (not
+            // chars). bufferOffset advances past consumed bytes; we only
+            // search the unread region so per-line work is O(line length),
+            // never O(response size).
+            const lineEnd = this.indexOfCRLF(this.bufferOffset);
             if (lineEnd < 0) {
                 // No complete line yet. Compact the buffer if we've advanced
                 // far enough that the unconsumed-prefix is dragging.
                 this.compactBufferIfNeeded();
                 break;
             }
-            // Extract the line and advance the read cursor — no substring
-            // copy of the remainder, no buffer reallocation.
-            const line = this.buffer.substring(this.bufferOffset, lineEnd + 2);
+            // Slice the line bytes (including trailing CRLF) and decode
+            // once. IMAP control lines are 7-bit ASCII per RFC 3501; UTF-8
+            // decode is a superset and produces the same characters. The
+            // existing string-based parser (parseResponseLine, tokenizers,
+            // ENVELOPE/FLAGS handling) consumes the decoded line below.
+            const lineBytes = this.buffer.subarray(this.bufferOffset, lineEnd + 2);
+            const line = this.utf8Decoder.decode(lineBytes);
             this.bufferOffset = lineEnd + 2;
             // Periodic compaction so the consumed prefix doesn't keep
             // growing forever in long-running connections (IDLE, big sync).

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bobfrankston/iflow-direct",
-  "version": "0.1.35",
+  "version": "0.1.39",
   "description": "Direct IMAP client — transport-agnostic, no Node.js dependencies, browser-ready",
   "main": "index.js",
   "types": "index.ts",
@@ -19,7 +19,7 @@
   "author": "Bob Frankston",
   "license": "ISC",
   "dependencies": {
-    "@bobfrankston/tcp-transport": "^0.1.5"
+    "@bobfrankston/tcp-transport": "^0.1.6"
   },
   "exports": {
     ".": {
@@ -50,7 +50,7 @@
   },
   ".transformedSnapshot": {
     "dependencies": {
-      "@bobfrankston/tcp-transport": "^0.1.5"
+      "@bobfrankston/tcp-transport": "^0.1.6"
     }
   }
 }