@bobfrankston/iflow-direct 0.1.16 → 0.1.18

package/README.md

@@ -0,0 +1,122 @@
+# @bobfrankston/iflow-direct
+
+Transport-agnostic IMAP client. Zero Node.js deps — works in Node, Electron, and
+the browser (via a bridge transport). Caller supplies a `TransportFactory`, OAuth
+token provider (if needed), and drives the client.
+
+See [MIGRATION.md](./MIGRATION.md) for the split from legacy `@bobfrankston/iflow`
+and the desktop/browser architecture diagram.
+
+## Install
+
+```bash
+npm install @bobfrankston/iflow-direct
+# desktop TCP transport:
+npm install @bobfrankston/tcp-transport
+```
+
+## Quick start (desktop)
+
+```typescript
+import { NativeImapClient } from "@bobfrankston/iflow-direct";
+import { TcpTransport } from "@bobfrankston/tcp-transport";
+
+const client = new NativeImapClient(
+    { server: "imap.example.com", port: 993, username: "me", password: "..." },
+    () => new TcpTransport(),
+);
+await client.connect();
+await client.select("INBOX");
+const msgs = await client.fetchMessages("1:50", { source: false });
+await client.logout();
+```
+
+For the legacy API shape (`getFolderList`, etc.) use `CompatImapClient` from the
+same package.
+
+## Batch body retrieval
+
+Two APIs for pulling bodies across many UIDs in a single `UID FETCH` round trip,
+with per-message streaming as responses arrive (not buffered until the tagged OK).
+
+### `fetchMessagesStream(range, options, onMessage?)`
+
+Streaming variant of `fetchMessages`. `range` accepts any IMAP sequence set —
+single UID (`"42"`), range (`"100:200"`), comma list (`"1,5,10,42"`), or mixed
+(`"1:10,42,50:55"`). The server handles all forms.
+
+```typescript
+await client.select("INBOX");
+await client.fetchMessagesStream(
+    "1,5,10,42,100:150",
+    { source: true, headers: false },
+    (msg) => {
+        // Called once per message as soon as its FETCH response
+        // (literals included) is fully received. Tagged OK has not arrived yet.
+        console.log(msg.uid, msg.source.length);
+    },
+);
+```
+
+Returns the parsed messages at the tagged OK; if `onMessage` is omitted, behaves
+like `fetchMessages` (collect-all).
+
+### `fetchBodiesBatch(folderPath, uids, onBody)`
+
+Folder-scoped convenience. Selects the folder (skips if already selected), issues
+one `UID FETCH <comma-list> (UID FLAGS ENVELOPE RFC822.SIZE INTERNALDATE BODY.PEEK[])`,
+streams each `(uid, source)` through `onBody`, returns on tagged OK.
+
+```typescript
+await client.fetchBodiesBatch("INBOX", [1, 5, 10, 42], (uid, source) => {
+    store.saveBody(uid, source);
+});
+```
+
+Intended for prefetch pipelines — e.g. mailx-imap's body-prefetch path — that
+would otherwise issue one SELECT+FETCH per message.
+
+### Streaming guarantees
+
+- `onMessage` / `onBody` fires on the same event-loop turn that the server's
+  FETCH response (with any literal bodies) is fully parsed.
+- Ordering matches server response order. For a comma list, servers typically
+  return in the order requested, but RFC 3501 does not require it — treat order
+  as server-defined.
+- Callback exceptions are caught and logged (verbose mode) so one bad message
+  doesn't abort the batch. Throw only if you want that behavior and wrap your
+  own try/catch.
+- The tagged-OK promise still resolves with the full response list; callers
+  needing both streaming and a terminal "all done" signal should `await` the
+  returned promise.
+
+## Other APIs
+
+`NativeImapClient` exposes the usual IMAP surface: `select`, `examine`,
+`listFolders`, `getStatus`, `search`, `fetchMessage`, `fetchSinceUid`,
+`fetchByDate` (both with optional `onChunk` chunked callbacks), `addFlags`,
+`removeFlags`, `copyMessage`, `moveMessage`, `deleteMessage`, `expunge`,
+`appendMessage`, `startIdle`, `logout`. See `imap-native.ts` for signatures.
+
+IDLE auto-suspends when any other command is issued on the same connection and
+re-enters afterwards — safe to interleave commands with a long-running IDLE,
+including commands issued from the IDLE `onNewMail` callback itself (e.g. a
+mailpuller-style `pullMail` that reacts to EXISTS by running STATUS + FETCH on
+the same connection). Suspend sends DONE, awaits the tagged OK, runs the
+command, then re-enters IDLE. If the caller stops IDLE (via the `startIdle`
+stop function) during the command, IDLE is not re-entered.
+
+## Configuration
+
+`ImapClientConfig` fields relevant to throughput/resilience:
+
+| Field | Default | Notes |
+|---|---|---|
+| `inactivityTimeout` | 60000 | ms with no data before connection is declared dead. Timer resets on every byte. Slow Dovecot often needs 180000+. |
+| `fetchChunkSize` | 25 | Initial chunk size for `fetchSinceUid`/`fetchByDate`. |
+| `fetchChunkSizeMax` | 500 | Ramps up 4× per chunk. Batch APIs above bypass chunking — caller decides. |
+| `verbose` | false | Log every command/response line + literal bookkeeping. |
+
+## Files
+
+See [MIGRATION.md § Files in iflow-direct](./MIGRATION.md#files-in-iflow-direct).

package/imap-compat.d.ts

@@ -38,6 +38,11 @@ export declare class CompatImapClient {
     fetchMessagesSinceUid(mailbox: string, sinceUid: number, options?: {
         source?: boolean;
     }): Promise<FetchedMessage[]>;
+    /** Batch-fetch bodies for many UIDs in one folder on one connection. Streams
+     *  each body through `onBody` as it arrives. No per-message round trips —
+     *  one SELECT, one UID FETCH, streaming response. Required by mailx-imap's
+     *  batch prefetch path. */
+    fetchBodiesBatch(mailbox: string, uids: number[], onBody: (uid: number, source: string) => void): Promise<void>;
     /** Fetch messages by date range. Optional onChunk callback for incremental processing. */
     fetchMessageByDate(mailbox: string, start: Date, end?: Date, options?: {
         source?: boolean;

package/imap-compat.js

@@ -93,6 +93,16 @@ export class CompatImapClient {
         await this.native.closeMailbox();
         return msgs.map(m => new FetchedMessage(m));
     }
+    /** Batch-fetch bodies for many UIDs in one folder on one connection. Streams
+     *  each body through `onBody` as it arrives. No per-message round trips —
+     *  one SELECT, one UID FETCH, streaming response. Required by mailx-imap's
+     *  batch prefetch path. */
+    async fetchBodiesBatch(mailbox, uids, onBody) {
+        if (uids.length === 0)
+            return;
+        await this.ensureConnected();
+        await this.native.fetchBodiesBatch(mailbox, uids, onBody);
+    }
     /** Fetch messages by date range. Optional onChunk callback for incremental processing. */
     async fetchMessageByDate(mailbox, start, end, options, onChunk) {
         await this.ensureConnected();

package/imap-native.d.ts

@@ -55,6 +55,8 @@ export declare class NativeImapClient {
     private idleTag;
     private idleCallback;
     private idleRefreshTimer;
+    /** Set by startIdle's stop closure so auto-suspend knows not to resume after a command finishes. */
+    private idleStopped;
     private verbose;
     private selectedMailbox;
     private mailboxInfo;
@@ -135,6 +137,19 @@ export declare class NativeImapClient {
     /** Append a message to a mailbox */
     appendMessage(mailbox: string, message: string | Uint8Array, flags?: string[]): Promise<number | null>;
     startIdle(onNewMail: (count: number) => void): Promise<() => Promise<void>>;
+    /**
+     * If IDLE is currently active, send DONE and wait for its tagged OK so the
+     * connection is free to accept a new command. Saves the active callback so
+     * the companion `resumeIdleAfterCommand` can re-enter IDLE on the same
+     * mailbox afterwards. Returns the saved state, or null if IDLE was not active.
+     *
+     * Called automatically at the top of `sendCommand` whenever the caller issues
+     * an interleaved command on an IDLE-holding connection (e.g. pullMail reacting
+     * to an EXISTS notification).
+     */
+    private suspendIdleForCommand;
+    /** Re-enter IDLE with the saved callback after an auto-suspended command completes. */
+    private resumeIdleAfterCommand;
     /** Send DONE + re-IDLE. Called by the 28-minute refresh timer. */
     private refreshIdle;
     getMessageCount(mailbox: string): Promise<number>;
@@ -151,7 +166,17 @@ export declare class NativeImapClient {
     private fetchChunkSizeMax;
     /** Active command timer — reset by handleData on every data arrival */
     private commandTimer;
+    /**
+     * Issue an IMAP command and await its tagged response.
+     *
+     * If IDLE is currently active on this connection, DONE it first (and wait
+     * for its tagged OK) before writing the new command; re-enter IDLE after
+     * the command finishes. Without this, commands issued from within an IDLE
+     * EXISTS callback (e.g. mailpuller's pullMail) would be ignored by the
+     * server until the inactivity timer killed the socket.
+     */
     private sendCommand;
+    private sendCommandCore;
     private waitForContinuation;
     private waitForTagged;
     private handleData;

package/imap-native.js

@@ -18,6 +18,8 @@ export class NativeImapClient {
     idleTag = null;
     idleCallback = null;
     idleRefreshTimer = null;
+    /** Set by startIdle's stop closure so auto-suspend knows not to resume after a command finishes. */
+    idleStopped = true;
     verbose;
     selectedMailbox = null;
     mailboxInfo = { exists: 0, recent: 0, uidNext: 0, uidValidity: 0, flags: [], permanentFlags: [] };
@@ -480,17 +482,20 @@ export class NativeImapClient {
     // ── IDLE ──
     async startIdle(onNewMail) {
         this.idleCallback = onNewMail;
-        let stopped = false;
+        this.idleStopped = false;
         const beginIdleCycle = async () => {
             const tag = proto.nextTag();
             this.idleTag = tag;
+            // Arm continuation listener BEFORE writing — some servers reply fast enough
+            // that data can arrive in the same tick.
+            const waitCont = this.waitForContinuation(tag);
             await this.transport.write(proto.idleCommand(tag));
-            await this.waitForContinuation(tag);
+            await waitCont;
             // RFC 2177: re-IDLE every ~28 minutes to avoid servers that terminate
             // idle sessions at the 29-minute mark, and to give us a steady
             // application-level keepalive that detects silently-dropped sockets.
             this.idleRefreshTimer = setTimeout(() => {
-                if (stopped)
+                if (this.idleStopped)
                     return;
                 this.refreshIdle().catch(err => {
                     if (this.verbose)
@@ -500,7 +505,7 @@ export class NativeImapClient {
         };
         await beginIdleCycle();
         return async () => {
-            stopped = true;
+            this.idleStopped = true;
             if (this.idleRefreshTimer) {
                 clearTimeout(this.idleRefreshTimer);
                 this.idleRefreshTimer = null;
@@ -508,40 +513,108 @@ export class NativeImapClient {
             const tag = this.idleTag;
             this.idleTag = null;
             this.idleCallback = null;
-            try {
-                await this.transport.write(proto.doneCommand());
-            }
-            catch { /* ignore */ }
             if (tag) {
+                // Arm tagged-wait BEFORE DONE — avoid missing the OK.
+                const waitTag = this.waitForTagged(tag);
+                try {
+                    await this.transport.write(proto.doneCommand());
+                }
+                catch { /* ignore */ }
                 try {
-                    await this.waitForTagged(tag);
+                    await waitTag;
                 }
                 catch { /* ignore */ }
             }
         };
     }
+    /**
+     * If IDLE is currently active, send DONE and wait for its tagged OK so the
+     * connection is free to accept a new command. Saves the active callback so
+     * the companion `resumeIdleAfterCommand` can re-enter IDLE on the same
+     * mailbox afterwards. Returns the saved state, or null if IDLE was not active.
+     *
+     * Called automatically at the top of `sendCommand` whenever the caller issues
+     * an interleaved command on an IDLE-holding connection (e.g. pullMail reacting
+     * to an EXISTS notification).
+     */
+    async suspendIdleForCommand() {
+        const oldTag = this.idleTag;
+        const cb = this.idleCallback;
+        if (!oldTag || !cb)
+            return null;
+        if (this.idleRefreshTimer) {
+            clearTimeout(this.idleRefreshTimer);
+            this.idleRefreshTimer = null;
+        }
+        this.idleTag = null;
+        this.idleCallback = null;
+        if (this.verbose)
+            console.log(`  [imap] IDLE auto-suspending for interleaved command`);
+        // Arm tagged listener BEFORE DONE so we don't miss the OK if the server is fast.
+        const waitTag = this.waitForTagged(oldTag);
+        try {
+            await this.transport.write(proto.doneCommand());
+        }
+        catch (err) {
+            // Write failure — pendingCommand is still set by waitForTagged; clear it to prevent hang.
+            this.pendingCommand = null;
+            throw err;
+        }
+        try {
+            await waitTag;
+        }
+        catch { /* best-effort — proceed even if OK was lost */ }
+        return { callback: cb };
+    }
+    /** Re-enter IDLE with the saved callback after an auto-suspended command completes. */
+    async resumeIdleAfterCommand(info) {
+        // The caller may have stopped IDLE during the command window — respect that.
+        if (this.idleStopped)
+            return;
+        const newTag = proto.nextTag();
+        this.idleTag = newTag;
+        this.idleCallback = info.callback;
+        // Arm continuation listener BEFORE writing IDLE.
+        const waitCont = this.waitForContinuation(newTag);
+        await this.transport.write(proto.idleCommand(newTag));
+        await waitCont;
+        if (this.verbose)
+            console.log(`  [imap] IDLE auto-resumed (${newTag})`);
+        this.idleRefreshTimer = setTimeout(() => {
+            if (this.idleStopped)
+                return;
+            this.refreshIdle().catch(err => {
+                if (this.verbose)
+                    console.error(`  [imap] IDLE refresh failed: ${err.message}`);
+            });
+        }, 28 * 60 * 1000);
+    }
     /** Send DONE + re-IDLE. Called by the 28-minute refresh timer. */
     async refreshIdle() {
         const oldTag = this.idleTag;
         if (!oldTag || !this.idleCallback)
             return;
         const cb = this.idleCallback;
-        // DONE the current IDLE
+        // Arm tagged listener BEFORE writing DONE to avoid missing a fast OK.
+        const waitTag = this.waitForTagged(oldTag);
         await this.transport.write(proto.doneCommand());
         try {
-            await this.waitForTagged(oldTag);
+            await waitTag;
         }
         catch { /* best-effort */ }
         // Re-IDLE with a new tag
         this.idleCallback = cb; // waitForTagged resolved pendingCommand, reinstall callback
         const newTag = proto.nextTag();
         this.idleTag = newTag;
+        const waitCont = this.waitForContinuation(newTag);
         await this.transport.write(proto.idleCommand(newTag));
-        await this.waitForContinuation(newTag);
+        await waitCont;
         if (this.verbose)
             console.log(`  [imap] IDLE refreshed (${newTag})`);
         // Schedule the next refresh
         this.idleRefreshTimer = setTimeout(() => {
+            if (this.idleStopped)
+                return;
             this.refreshIdle().catch(err => {
                 if (this.verbose)
                     console.error(`  [imap] IDLE refresh failed: ${err.message}`);
@@ -567,7 +640,33 @@ export class NativeImapClient {
     fetchChunkSizeMax;
     /** Active command timer — reset by handleData on every data arrival */
     commandTimer = null;
-    sendCommand(tag, command, onUntagged) {
+    /**
+     * Issue an IMAP command and await its tagged response.
+     *
+     * If IDLE is currently active on this connection, DONE it first (and wait
+     * for its tagged OK) before writing the new command; re-enter IDLE after
+     * the command finishes. Without this, commands issued from within an IDLE
+     * EXISTS callback (e.g. mailpuller's pullMail) would be ignored by the
+     * server until the inactivity timer killed the socket.
+     */
+    async sendCommand(tag, command, onUntagged) {
+        const idleState = await this.suspendIdleForCommand();
+        try {
+            return await this.sendCommandCore(tag, command, onUntagged);
+        }
+        finally {
+            if (idleState) {
+                try {
+                    await this.resumeIdleAfterCommand(idleState);
+                }
+                catch (err) {
+                    if (this.verbose)
+                        console.error(`  [imap] IDLE auto-resume failed: ${err.message}`);
+                }
+            }
+        }
+    }
+    sendCommandCore(tag, command, onUntagged) {
         return new Promise((resolve, reject) => {
             if (this.verbose && !command.includes("LOGIN") && !command.includes("AUTHENTICATE")) {
                 console.log(`  [imap] > ${command.trimEnd()}`);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bobfrankston/iflow-direct",
-  "version": "0.1.16",
+  "version": "0.1.18",
   "description": "Direct IMAP client — transport-agnostic, no Node.js dependencies, browser-ready",
   "main": "index.js",
   "types": "index.ts",