@bobfrankston/iflow 1.0.54 → 1.0.56

package/imaplib/imap-compat.d.ts

@@ -37,10 +37,10 @@ export declare class CompatImapClient {
     fetchMessagesSinceUid(mailbox: string, sinceUid: number, options?: {
         source?: boolean;
     }): Promise<any[]>;
-    /** Fetch messages by date range */
+    /** Fetch messages by date range. Optional onChunk callback for incremental processing. */
     fetchMessageByDate(mailbox: string, start: Date, end?: Date, options?: {
         source?: boolean;
-    }): Promise<any[]>;
+    }, onChunk?: (msgs: any[]) => void): Promise<any[]>;
     /** Fetch a single message by UID */
     fetchMessageByUid(mailbox: string, uid: number, options?: {
         source?: boolean;

package/imaplib/imap-compat.js

@@ -102,11 +102,12 @@ export class CompatImapClient {
         await this.native.closeMailbox();
         return msgs.map(toCompatMessage);
     }
-    /** Fetch messages by date range */
-    async fetchMessageByDate(mailbox, start, end, options) {
+    /** Fetch messages by date range. Optional onChunk callback for incremental processing. */
+    async fetchMessageByDate(mailbox, start, end, options, onChunk) {
         await this.ensureConnected();
         await this.native.select(mailbox);
-        const msgs = await this.native.fetchByDate(start, end, options);
+        const chunkCb = onChunk ? (raw) => onChunk(raw.map(toCompatMessage)) : undefined;
+        const msgs = await this.native.fetchByDate(start, end, options, chunkCb);
         await this.native.closeMailbox();
         return msgs.map(toCompatMessage);
     }

package/imaplib/imap-native.d.ts

@@ -86,11 +86,11 @@ export declare class NativeImapClient {
     /** Fetch messages since a UID */
     fetchSinceUid(sinceUid: number, options?: {
         source?: boolean;
-    }): Promise<NativeFetchedMessage[]>;
-    /** Fetch messages by date range */
+    }, onChunk?: (msgs: NativeFetchedMessage[]) => void): Promise<NativeFetchedMessage[]>;
+    /** Fetch messages by date range. Optional onChunk callback receives each batch as it arrives. */
     fetchByDate(since: Date, before?: Date, options?: {
         source?: boolean;
-    }): Promise<NativeFetchedMessage[]>;
+    }, onChunk?: (msgs: NativeFetchedMessage[]) => void): Promise<NativeFetchedMessage[]>;
     /** Fetch a single message by UID */
     fetchMessage(uid: number, options?: {
         source?: boolean;
@@ -119,6 +119,9 @@ export declare class NativeImapClient {
      *  This is NOT a wall-clock timeout. Timer resets every time data arrives from the server.
      *  A large FETCH returning data continuously will never timeout. */
     private inactivityTimeout;
+    /** Fetch chunk sizes — start small for quick first paint, ramp up for throughput */
+    private static INITIAL_CHUNK_SIZE;
+    private static MAX_CHUNK_SIZE;
     /** Active command timer — reset by handleData on every data arrival */
     private commandTimer;
     private sendCommand;

package/imaplib/imap-native.js

@@ -289,26 +289,33 @@ export class NativeImapClient {
         return this.parseFetchResponses(responses);
     }
     /** Fetch messages since a UID */
-    async fetchSinceUid(sinceUid, options = {}) {
-        // Search for UIDs first, then fetch in chunks (avoids unbounded single command)
+    async fetchSinceUid(sinceUid, options = {}, onChunk) {
         const uids = await this.search(`UID ${sinceUid + 1}:*`);
         if (uids.length === 0)
             return [];
-        if (uids.length <= 500) {
-            return this.fetchMessages(uids.join(","), options);
+        console.log(`  [fetch] ${uids.length} UIDs since ${sinceUid}`);
+        if (uids.length <= NativeImapClient.INITIAL_CHUNK_SIZE) {
+            const msgs = await this.fetchMessages(uids.join(","), options);
+            if (onChunk)
+                onChunk(msgs);
+            return msgs;
         }
-        // Chunk large fetches
         const allMessages = [];
-        for (let i = 0; i < uids.length; i += 500) {
-            const chunk = uids.slice(i, i + 500);
+        let chunkSize = NativeImapClient.INITIAL_CHUNK_SIZE;
+        for (let i = 0; i < uids.length; i += chunkSize) {
+            const chunk = uids.slice(i, i + chunkSize);
             const msgs = await this.fetchMessages(chunk.join(","), options);
             allMessages.push(...msgs);
+            console.log(`  [fetch] ${allMessages.length}/${uids.length} (chunk of ${chunk.length})`);
+            if (onChunk)
+                onChunk(msgs);
+            if (chunkSize < NativeImapClient.MAX_CHUNK_SIZE)
+                chunkSize = Math.min(chunkSize * 4, NativeImapClient.MAX_CHUNK_SIZE);
         }
         return allMessages;
     }
-    /** Fetch messages by date range */
-    async fetchByDate(since, before, options = {}) {
-        // First search for UIDs in date range, then fetch
+    /** Fetch messages by date range. Optional onChunk callback receives each batch as it arrives. */
+    async fetchByDate(since, before, options = {}, onChunk) {
         const criteria = proto.buildSearchCriteria({
             since,
             before: before || undefined,
@@ -316,14 +323,18 @@ export class NativeImapClient {
         const uids = await this.search(criteria);
         if (uids.length === 0)
             return [];
-        // Fetch in chunks to avoid very long command lines
-        const chunkSize = 500;
+        console.log(`  [fetch] ${uids.length} UIDs to fetch`);
         const allMessages = [];
+        let chunkSize = NativeImapClient.INITIAL_CHUNK_SIZE;
         for (let i = 0; i < uids.length; i += chunkSize) {
             const chunk = uids.slice(i, i + chunkSize);
-            const range = chunk.join(",");
-            const msgs = await this.fetchMessages(range, options);
+            const msgs = await this.fetchMessages(chunk.join(","), options);
             allMessages.push(...msgs);
+            console.log(`  [fetch] ${allMessages.length}/${uids.length} (chunk of ${chunk.length})`);
+            if (onChunk)
+                onChunk(msgs);
+            if (chunkSize < NativeImapClient.MAX_CHUNK_SIZE)
+                chunkSize = Math.min(chunkSize * 4, NativeImapClient.MAX_CHUNK_SIZE);
         }
         return allMessages;
     }
@@ -444,6 +455,9 @@ export class NativeImapClient {
      *  This is NOT a wall-clock timeout. Timer resets every time data arrives from the server.
      *  A large FETCH returning data continuously will never timeout. */
     inactivityTimeout = 30000;
+    /** Fetch chunk sizes — start small for quick first paint, ramp up for throughput */
+    static INITIAL_CHUNK_SIZE = 25;
+    static MAX_CHUNK_SIZE = 500;
     /** Active command timer — reset by handleData on every data arrival */
     commandTimer = null;
     sendCommand(tag, command) {
@@ -507,11 +521,18 @@ export class NativeImapClient {
     }
     processBuffer() {
         while (true) {
-            // Check for literal {size}\r\n — reading exact byte count of literal data
+            // Check for literal {size}\r\n — reading exact BYTE count of literal data
+            // CRITICAL: literalBytes is in octets (from IMAP {N}), but this.buffer is a
+            // JavaScript string where multi-byte UTF-8 characters count as 1 character.
+            // We must use Buffer.byteLength to find the correct character boundary.
             if (this.pendingCommand?.literalBytes != null) {
-                if (this.buffer.length >= this.pendingCommand.literalBytes) {
-                    let literal = this.buffer.substring(0, this.pendingCommand.literalBytes);
-                    this.buffer = this.buffer.substring(this.pendingCommand.literalBytes);
+                const neededBytes = this.pendingCommand.literalBytes;
+                const bufferBytes = Buffer.byteLength(this.buffer, "utf-8");
+                if (bufferBytes >= neededBytes) {
+                    // Find the character position that corresponds to the byte boundary
+                    const buf = Buffer.from(this.buffer, "utf-8");
+                    let literal = buf.subarray(0, neededBytes).toString("utf-8");
+                    this.buffer = buf.subarray(neededBytes).toString("utf-8");
                     // For non-BODY literals (e.g. display names in ENVELOPE), wrap in quotes
                     // so tokenizeParenList treats them as a single token
                     if (!this.pendingCommand.currentLiteralKey) {
@@ -535,7 +556,7 @@ export class NativeImapClient {
                     continue;
                 }
                 if (this.verbose && this.pendingCommand.literalBytes > 0) {
-                    console.log(`  [imap] waiting for literal: need ${this.pendingCommand.literalBytes}, have ${this.buffer.length}`);
+                    console.log(`  [imap] waiting for literal: need ${neededBytes} bytes, have ${bufferBytes} bytes`);
                 }
                 break; // Wait for more data
             }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bobfrankston/iflow",
-  "version": "1.0.54",
+  "version": "1.0.56",
   "description": "IMAP client wrapper library",
   "main": "index.js",
   "types": "index.ts",