@bobfrankston/iflow 1.0.53 → 1.0.55

package/imaplib/imap-compat.d.ts

@@ -37,10 +37,10 @@ export declare class CompatImapClient {
     fetchMessagesSinceUid(mailbox: string, sinceUid: number, options?: {
         source?: boolean;
     }): Promise<any[]>;
-    /** Fetch messages by date range */
+    /** Fetch messages by date range. Optional onChunk callback for incremental processing. */
     fetchMessageByDate(mailbox: string, start: Date, end?: Date, options?: {
         source?: boolean;
-    }): Promise<any[]>;
+    }, onChunk?: (msgs: any[]) => void): Promise<any[]>;
     /** Fetch a single message by UID */
     fetchMessageByUid(mailbox: string, uid: number, options?: {
         source?: boolean;

package/imaplib/imap-compat.js

@@ -102,11 +102,12 @@ export class CompatImapClient {
         await this.native.closeMailbox();
         return msgs.map(toCompatMessage);
     }
-    /** Fetch messages by date range */
-    async fetchMessageByDate(mailbox, start, end, options) {
+    /** Fetch messages by date range. Optional onChunk callback for incremental processing. */
+    async fetchMessageByDate(mailbox, start, end, options, onChunk) {
         await this.ensureConnected();
         await this.native.select(mailbox);
-        const msgs = await this.native.fetchByDate(start, end, options);
+        const chunkCb = onChunk ? (raw) => onChunk(raw.map(toCompatMessage)) : undefined;
+        const msgs = await this.native.fetchByDate(start, end, options, chunkCb);
         await this.native.closeMailbox();
         return msgs.map(toCompatMessage);
     }

package/imaplib/imap-native.d.ts

@@ -86,11 +86,11 @@ export declare class NativeImapClient {
     /** Fetch messages since a UID */
     fetchSinceUid(sinceUid: number, options?: {
         source?: boolean;
-    }): Promise<NativeFetchedMessage[]>;
-    /** Fetch messages by date range */
+    }, onChunk?: (msgs: NativeFetchedMessage[]) => void): Promise<NativeFetchedMessage[]>;
+    /** Fetch messages by date range. Optional onChunk callback receives each batch as it arrives. */
     fetchByDate(since: Date, before?: Date, options?: {
         source?: boolean;
-    }): Promise<NativeFetchedMessage[]>;
+    }, onChunk?: (msgs: NativeFetchedMessage[]) => void): Promise<NativeFetchedMessage[]>;
     /** Fetch a single message by UID */
     fetchMessage(uid: number, options?: {
         source?: boolean;
@@ -115,8 +115,15 @@ export declare class NativeImapClient {
     appendMessage(mailbox: string, message: string | Uint8Array, flags?: string[]): Promise<number | null>;
     startIdle(onNewMail: (count: number) => void): Promise<() => Promise<void>>;
     getMessageCount(mailbox: string): Promise<number>;
-    /** Default timeout for IMAP commands (30s). Prevents hanging connections. */
-    private commandTimeout;
+    /** Inactivity timeout — how long to wait with NO data before declaring the connection dead.
+     *  This is NOT a wall-clock timeout. Timer resets every time data arrives from the server.
+     *  A large FETCH returning data continuously will never timeout. */
+    private inactivityTimeout;
+    /** Fetch chunk sizes — start small for quick first paint, ramp up for throughput */
+    private static INITIAL_CHUNK_SIZE;
+    private static MAX_CHUNK_SIZE;
+    /** Active command timer — reset by handleData on every data arrival */
+    private commandTimer;
     private sendCommand;
     private waitForContinuation;
     private waitForTagged;

package/imaplib/imap-native.js

@@ -289,12 +289,33 @@ export class NativeImapClient {
         return this.parseFetchResponses(responses);
     }
     /** Fetch messages since a UID */
-    async fetchSinceUid(sinceUid, options = {}) {
-        return this.fetchMessages(`${sinceUid + 1}:*`, options);
+    async fetchSinceUid(sinceUid, options = {}, onChunk) {
+        const uids = await this.search(`UID ${sinceUid + 1}:*`);
+        if (uids.length === 0)
+            return [];
+        console.log(`  [fetch] ${uids.length} UIDs since ${sinceUid}`);
+        if (uids.length <= NativeImapClient.INITIAL_CHUNK_SIZE) {
+            const msgs = await this.fetchMessages(uids.join(","), options);
+            if (onChunk)
+                onChunk(msgs);
+            return msgs;
+        }
+        const allMessages = [];
+        let chunkSize = NativeImapClient.INITIAL_CHUNK_SIZE;
+        for (let i = 0; i < uids.length; i += chunkSize) {
+            const chunk = uids.slice(i, i + chunkSize);
+            const msgs = await this.fetchMessages(chunk.join(","), options);
+            allMessages.push(...msgs);
+            console.log(`  [fetch] ${allMessages.length}/${uids.length} (chunk of ${chunk.length})`);
+            if (onChunk)
+                onChunk(msgs);
+            if (chunkSize < NativeImapClient.MAX_CHUNK_SIZE)
+                chunkSize = Math.min(chunkSize * 4, NativeImapClient.MAX_CHUNK_SIZE);
+        }
+        return allMessages;
     }
-    /** Fetch messages by date range */
-    async fetchByDate(since, before, options = {}) {
-        // First search for UIDs in date range, then fetch
+    /** Fetch messages by date range. Optional onChunk callback receives each batch as it arrives. */
+    async fetchByDate(since, before, options = {}, onChunk) {
         const criteria = proto.buildSearchCriteria({
             since,
             before: before || undefined,
@@ -302,14 +323,18 @@ export class NativeImapClient {
         const uids = await this.search(criteria);
         if (uids.length === 0)
             return [];
-        // Fetch in chunks to avoid very long command lines
-        const chunkSize = 500;
+        console.log(`  [fetch] ${uids.length} UIDs to fetch`);
         const allMessages = [];
+        let chunkSize = NativeImapClient.INITIAL_CHUNK_SIZE;
         for (let i = 0; i < uids.length; i += chunkSize) {
             const chunk = uids.slice(i, i + chunkSize);
-            const range = chunk.join(",");
-            const msgs = await this.fetchMessages(range, options);
+            const msgs = await this.fetchMessages(chunk.join(","), options);
             allMessages.push(...msgs);
+            console.log(`  [fetch] ${allMessages.length}/${uids.length} (chunk of ${chunk.length})`);
+            if (onChunk)
+                onChunk(msgs);
+            if (chunkSize < NativeImapClient.MAX_CHUNK_SIZE)
+                chunkSize = Math.min(chunkSize * 4, NativeImapClient.MAX_CHUNK_SIZE);
         }
         return allMessages;
     }
@@ -426,23 +451,37 @@ export class NativeImapClient {
         return status.messages || 0;
     }
     // ── Low-level command handling ──
-    /** Default timeout for IMAP commands (30s). Prevents hanging connections. */
-    commandTimeout = 30000;
+    /** Inactivity timeout — how long to wait with NO data before declaring the connection dead.
+     *  This is NOT a wall-clock timeout. Timer resets every time data arrives from the server.
+     *  A large FETCH returning data continuously will never timeout. */
+    inactivityTimeout = 30000;
+    /** Fetch chunk sizes — start small for quick first paint, ramp up for throughput */
+    static INITIAL_CHUNK_SIZE = 25;
+    static MAX_CHUNK_SIZE = 500;
+    /** Active command timer — reset by handleData on every data arrival */
+    commandTimer = null;
     sendCommand(tag, command) {
         return new Promise((resolve, reject) => {
             if (this.verbose && !command.includes("LOGIN") && !command.includes("AUTHENTICATE")) {
                 console.log(`  [imap] > ${command.trimEnd()}`);
             }
-            const timer = setTimeout(() => {
+            const onTimeout = () => {
+                this.commandTimer = null;
                 this.pendingCommand = null;
-                reject(new Error(`IMAP command timeout (${this.commandTimeout / 1000}s): ${command.split("\r")[0].substring(0, 50)}`));
-            }, this.commandTimeout);
+                // Kill the connection — a timed-out connection has stale data in the pipe
+                this.transport.close?.();
+                reject(new Error(`IMAP inactivity timeout (${this.inactivityTimeout / 1000}s): ${command.split("\r")[0].substring(0, 80)}`));
+            };
+            this.commandTimer = setTimeout(onTimeout, this.inactivityTimeout);
             this.pendingCommand = {
                 tag, responses: [],
-                resolve: (responses) => { clearTimeout(timer); resolve(responses); },
-                reject: (err) => { clearTimeout(timer); reject(err); },
+                resolve: (responses) => { if (this.commandTimer)
+                    clearTimeout(this.commandTimer); this.commandTimer = null; resolve(responses); },
+                reject: (err) => { if (this.commandTimer)
+                    clearTimeout(this.commandTimer); this.commandTimer = null; reject(err); },
             };
-            this.transport.write(command).catch((err) => { clearTimeout(timer); reject(err); });
+            this.transport.write(command).catch((err) => { if (this.commandTimer)
+                clearTimeout(this.commandTimer); this.commandTimer = null; reject(err); });
         });
     }
     waitForContinuation(tag) {
@@ -464,6 +503,19 @@ export class NativeImapClient {
         });
     }
     handleData(data) {
+        // Reset inactivity timer — data is flowing, connection is alive
+        if (this.commandTimer) {
+            clearTimeout(this.commandTimer);
+            this.commandTimer = setTimeout(() => {
+                this.commandTimer = null;
+                if (this.pendingCommand) {
+                    const cmd = this.pendingCommand;
+                    this.pendingCommand = null;
+                    this.transport.close?.();
+                    cmd.reject(new Error(`IMAP inactivity timeout (${this.inactivityTimeout / 1000}s)`));
+                }
+            }, this.inactivityTimeout);
+        }
         this.buffer += data;
         this.processBuffer();
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bobfrankston/iflow",
-  "version": "1.0.53",
+  "version": "1.0.55",
   "description": "IMAP client wrapper library",
   "main": "index.js",
   "types": "index.ts",