emailengine-app 2.68.1 → 2.69.0

package/lib/export.js

@@ -114,6 +114,13 @@ async function getExportMaxAge() {
     return DEFAULT_EXPORT_MAX_AGE;
 }
 
+// Resolve the retention window into the Redis `expire` TTL (seconds) and an absolute `expiresAt`
+// timestamp (ms), used wherever an export key's expiry is (re)set.
+async function getExportExpiry() {
+    const maxAge = await getExportMaxAge();
+    return { ttl: Math.ceil(maxAge / 1000), expiresAt: Date.now() + maxAge };
+}
+
 function toTimestamp(date) {
     const ts = new Date(date).getTime();
     if (isNaN(ts)) {
@@ -168,7 +175,8 @@ class Export {
             startDate,
             endDate,
             textType: options.textType || '*',
-            maxBytes: options.maxBytes || 5 * 1024 * 1024,
+            // Preserve an explicit 0 ("unlimited" per the API contract); only fall back when unset.
+            maxBytes: Number.isInteger(options.maxBytes) ? options.maxBytes : 5 * 1024 * 1024,
             includeAttachments: options.includeAttachments ? '1' : '0',
             isEncrypted: isEncrypted ? '1' : '0',
             foldersScanned: 0,
@@ -178,7 +186,6 @@ class Export {
             messagesSkipped: 0,
             bytesWritten: 0,
             filePath,
-            lastProcessedScore: 0,
             created: now,
             expiresAt,
             error: ''
@@ -325,11 +332,39 @@ class Export {
 
     static async startProcessing(account, exportId) {
         const exportKey = getExportKey(account, exportId);
-        const maxAge = await getExportMaxAge();
-        const ttl = Math.ceil(maxAge / 1000);
-        const newExpiresAt = Date.now() + maxAge;
+        const queueKey = getExportQueueKey(account, exportId);
+        const { ttl, expiresAt } = await getExportExpiry();
 
-        await redis.multi().hmset(exportKey, { status: 'processing', phase: 'indexing', expiresAt: newExpiresAt }).expire(exportKey, ttl).exec();
+        // Reset progress and clear any previously indexed queue so that each (re)run -- including a
+        // BullMQ stalled-job reprocess -- rebuilds the queue and rewrites the (truncated) output file
+        // from scratch, keeping counters and file content consistent.
+        await redis
+            .multi()
+            .hmset(exportKey, {
+                status: 'processing',
+                phase: 'indexing',
+                expiresAt,
+                foldersScanned: 0,
+                foldersTotal: 0,
+                messagesQueued: 0,
+                messagesExported: 0,
+                messagesSkipped: 0,
+                bytesWritten: 0,
+                truncated: '0'
+            })
+            .del(queueKey)
+            .expire(exportKey, ttl)
+            .exec();
+    }
+
+    static async extendExpiry(account, exportId) {
+        const exportKey = getExportKey(account, exportId);
+        const queueKey = getExportQueueKey(account, exportId);
+        const { ttl, expiresAt } = await getExportExpiry();
+
+        // Keep both the export hash and the pending-message queue alive for the full retention window
+        // while a long export is still running, and surface the refreshed expiry to status readers.
+        await redis.multi().hset(exportKey, 'expiresAt', expiresAt).expire(exportKey, ttl).expire(queueKey, ttl).exec();
     }
 
     static async queueMessage(account, exportId, messageInfo) {
@@ -355,19 +390,23 @@ class Export {
         await multi.exec();
     }
 
-    static async getNextBatch(account, exportId, lastScore, limit) {
+    static async getNextBatch(account, exportId, limit) {
         const queueKey = getExportQueueKey(account, exportId);
-        // Use exclusive lower bound to avoid re-processing messages at batch boundaries
-        const minScore = lastScore > 0 ? '(' + lastScore : lastScore;
-        const results = await redis.zrangebyscore(queueKey, minScore, '+inf', 'WITHSCORES', 'LIMIT', 0, limit);
+        // Atomically pop the lowest-scored (oldest) messages so each is processed exactly once.
+        // This avoids cursor-based reprocessing and the boundary drops that an exclusive score bound
+        // caused when two messages shared the same score.
+        const results = await redis.zpopmin(queueKey, limit);
 
+        // ZPOPMIN returns [member, score, member, score, ...]; only the decoded member is needed.
         const messages = [];
         for (let i = 0; i < results.length; i += 2) {
             try {
-                const info = msgpack.decode(Buffer.from(results[i], 'base64url'));
-                messages.push({ ...info, score: Number(results[i + 1]) });
+                messages.push(msgpack.decode(Buffer.from(results[i], 'base64url')));
             } catch (err) {
                 logger.error({ msg: 'Failed to decode message info', account, exportId, err });
+                // ZPOPMIN already removed the entry, so it will never be exported;
+                // count it as skipped to keep messagesQueued === exported + skipped
+                await Export.incrementSkipped(account, exportId);
             }
         }
 
@@ -383,19 +422,20 @@ class Export {
         await redis.hincrby(getExportKey(account, exportId), 'messagesSkipped', 1);
     }
 
-    static async updateLastProcessedScore(account, exportId, score) {
-        await redis.hset(getExportKey(account, exportId), 'lastProcessedScore', score);
-    }
-
     static async complete(account, exportId) {
         const exportKey = getExportKey(account, exportId);
         const queueKey = getExportQueueKey(account, exportId);
 
+        // Start the retention window at completion time so a long-running export stays downloadable for
+        // the full retention period rather than the time left over from when processing started.
+        const { ttl, expiresAt } = await getExportExpiry();
+
         await redis
             .multi()
-            .hmset(exportKey, { status: 'completed', phase: 'complete' })
+            .hmset(exportKey, { status: 'completed', phase: 'complete', expiresAt })
             .del(queueKey)
             .srem(ACTIVE_EXPORTS_KEY, `${account}:${exportId}`)
+            .expire(exportKey, ttl)
             .exec();
 
         logger.info({ msg: 'Export completed', account, exportId });
@@ -437,8 +477,9 @@ class Export {
 
         for (const entry of activeExports) {
             try {
-                // Find ':exp_' as separator since account IDs may contain colons
-                const separatorIndex = entry.indexOf(':exp_');
+                // Find ':exp_' as separator since account IDs may contain colons. The export id is
+                // always the trailing ':exp_<hex>' segment, so match the last occurrence.
+                const separatorIndex = entry.lastIndexOf(':exp_');
                 if (separatorIndex === -1) continue;
                 const account = entry.substring(0, separatorIndex);
                 const exportId = entry.substring(separatorIndex + 1);

package/lib/imapproxy/imap-core/lib/commands/starttls.js

@@ -15,6 +15,20 @@ module.exports = {
             });
         }
 
+        if (this._server.options.disableSTARTTLS) {
+            // STARTTLS is not advertised in this mode (e.g. the EmailEngine proxy);
+            // refuse it instead of upgrading against the built-in fallback cert.
+            return callback(null, {
+                response: 'NO',
+                message: 'STARTTLS not available'
+            });
+        }
+
+        // Block any further (possibly pipelined / injected) commands until the TLS
+        // handshake completes. _onCommand ignores commands while _upgrading is set,
+        // so a command sent in the same chunk as STARTTLS cannot be executed.
+        this._upgrading = true;
+
         setImmediate(upgrade.bind(null, this));
 
         callback(null, {
@@ -94,6 +108,10 @@ function upgrade(connection) {
             (cipher && cipher.name) || 'N/A'
         );
 
+        // Discard any plaintext the parser buffered before the handshake so it
+        // cannot be injected into the now-encrypted session.
+        connection._parser.reset();
+
         connection._socket.pipe(connection._parser);
         connection.writeStream.pipe(connection._socket);
     });

package/lib/imapproxy/imap-core/lib/imap-command.js

@@ -369,9 +369,14 @@ class IMAPCommand {
     countBadResponses() {
         this.connection._badCount++;
         if (this.connection._badCount > MAX_BAD_COMMANDS) {
+            // Stop reading first so a command pipelined right after the offending input is not
+            // dispatched during the graceful-close window.
+            this.connection._stopReading();
             this.connection.clearNotificationListener();
             this.connection.send('* BYE Too many protocol errors');
-            setImmediate(() => this.connection.close(true));
+            // Graceful close so the BYE is flushed to the client before teardown (close() ends
+            // the socket and force-destroys via its own failsafe timer).
+            setImmediate(() => this.connection.close());
             return false;
         }
         return true;

package/lib/imapproxy/imap-core/lib/imap-connection.js

@@ -14,6 +14,12 @@ const packageInfo = require('../../../../package');
 
 const SOCKET_TIMEOUT = 5 * 60 * 1000;
 
+// Idle timeout applied to a proxied connection after auth handoff. Generous enough
+// to not interrupt IMAP IDLE (which can sit idle for ~29 minutes) while still
+// preventing a connection from being held open forever. Overridable via the
+// proxyTimeout server option; 0 disables it (legacy behaviour).
+const PROXY_SOCKET_TIMEOUT = 30 * 60 * 1000;
+
 /**
  * Creates a handler for new socket
  *
@@ -52,11 +58,15 @@ class IMAPConnection extends EventEmitter {
         this._upgrading = false;
 
         // Parser instance for the incoming stream
-        this._parser = new IMAPStream();
+        this._parser = new IMAPStream({ maxLineLength: this._server.options.maxLineLength });
 
         // Set handler for incoming commands
         this._parser.oncommand = this._onCommand.bind(this);
 
+        // Close the connection if the parser rejects the input (e.g. an over-long
+        // command line) instead of letting the error reach the global handler.
+        this._parser.on('error', err => this._onParserError(err));
+
         // Manage multi part command
         this._currentCommand = false;
 
@@ -194,8 +204,12 @@ class IMAPConnection extends EventEmitter {
     }
 
     unbind() {
+        // Cancel any armed graceful-close failsafe so it cannot destroy the socket after we
+        // have handed it off to the proxy backend.
+        clearTimeout(this._closingTimeout);
+
         this.writeStream.unpipe(this._socket);
-        this._socket.unpipe(this._parser);
+        this._stopReading();
 
         if (this._onCloseHandler) {
             this._socket.removeListener('close', this._onCloseHandler);
@@ -206,8 +220,29 @@ class IMAPConnection extends EventEmitter {
         if (this._onErrorHandler) {
             this._socket.removeListener('error', this._onErrorHandler);
         }
+        // Drop the auth-phase protocol timeout handler - after handoff it must not
+        // run, it would write IMAP responses into the proxied byte stream.
+        this._socket.setTimeout(0);
         if (this._onTimeoutHandler) {
-            this._socket.setTimeout(0, this._onTimeoutHandler);
+            this._socket.removeListener('timeout', this._onTimeoutHandler);
+        }
+
+        // Re-arm a generous idle timeout so the proxied connection cannot be held
+        // open indefinitely (fd/connection exhaustion). Use a plain socket destroy
+        // rather than the protocol-level handler. proxyTimeout: 0 disables it.
+        let proxyTimeout = this._server.options.proxyTimeout;
+        if (proxyTimeout === undefined || proxyTimeout === null) {
+            proxyTimeout = PROXY_SOCKET_TIMEOUT;
+        }
+        if (proxyTimeout) {
+            this._socket.setTimeout(proxyTimeout);
+            this._socket.once('timeout', () => {
+                try {
+                    this._socket.destroy();
+                } catch (err) {
+                    // ignore
+                }
+            });
         }
 
         // After handoff to proxy mode the connection's own close/end handlers are gone,
@@ -281,42 +316,53 @@ class IMAPConnection extends EventEmitter {
     /**
      * Close socket
      */
-    close(force) {
+    close() {
         if (this._closed || this._closing) {
             return;
         }
 
         if (!this._socket.destroyed && this._socket.writable) {
-            this._socket[!force ? 'end' : 'destroy']();
+            this._socket.end();
         }
 
         this._server.connections.delete(this);
 
-        if (!force) {
-            // allow socket to close in 1500ms or force it to close
-            this._closingTimeout = setTimeout(() => {
-                if (this._closed) {
-                    return;
-                }
+        // allow socket to close gracefully in 1500ms or force it to close
+        this._closingTimeout = setTimeout(() => {
+            if (this._closed) {
+                return;
+            }
 
-                try {
-                    this._socket.destroy();
-                } catch (err) {
-                    // ignore
-                }
+            try {
+                this._socket.destroy();
+            } catch (err) {
+                // ignore
+            }
 
-                setImmediate(() => this._onClose());
-            }, 1500);
-        }
+            setImmediate(() => this._onClose());
+        }, 1500);
 
         this._closing = true;
-        if (force) {
-            setImmediate(() => this._onClose());
-        }
     }
 
     // PRIVATE METHODS
 
+    /**
+     * Stop feeding the socket into the command parser. Called before teardown so a client
+     * cannot pipeline another command (e.g. LOGIN) after we have decided to close - which
+     * would otherwise be dispatched during the graceful-close window. Safe to call more than
+     * once; a no-op once the parser has been torn down in _onClose().
+     */
+    _stopReading() {
+        try {
+            if (this._parser) {
+                this._socket.unpipe(this._parser);
+            }
+        } catch (err) {
+            // ignore
+        }
+    }
+
     /**
      * Setup socket event handlers
      */
@@ -334,6 +380,34 @@ class IMAPConnection extends EventEmitter {
         this._socket.pipe(this._parser);
     }
 
+    /**
+     * Fired when the incoming-stream parser rejects the input (e.g. an over-long
+     * command line). Stops feeding the parser, tells the client, and closes.
+     * @param {Error} err - Parser error
+     */
+    _onParserError(err) {
+        this._stopReading();
+
+        this.logger.info(
+            {
+                tnx: 'parser',
+                cid: this.id
+            },
+            'Closing connection: %s',
+            err && err.message
+        );
+
+        try {
+            this.send('* BYE ' + ((err && err.message) || 'Protocol error'));
+        } catch (E) {
+            // ignore
+        }
+
+        // Graceful close so the BYE is flushed before teardown; close() force-destroys via
+        // its own failsafe timer if the socket does not close on its own.
+        setImmediate(() => this.close());
+    }
+
     /**
      * Fired when the socket is closed
      * @event
@@ -489,8 +563,12 @@ class IMAPConnection extends EventEmitter {
             return;
         }
 
+        // Stop reading first so a command pipelined right after the timeout is not dispatched
+        // during the graceful-close window. Graceful close still flushes the BYE before teardown
+        // (close() has a failsafe timer).
+        this._stopReading();
         this.send('* BYE Idle timeout, closing connection');
-        setImmediate(() => this.close(true));
+        setImmediate(() => this.close());
     }
 
     /**
@@ -504,6 +582,11 @@ class IMAPConnection extends EventEmitter {
 
         callback = callback || (() => false);
 
+        if (this._closing || this._closed) {
+            // connection is tearing down - do not dispatch further commands
+            return callback();
+        }
+
         if (this._upgrading) {
             // ignore any commands before TLS upgrade is finished
             return callback();

package/lib/imapproxy/imap-core/lib/imap-server.js

@@ -12,6 +12,10 @@ const base32 = require('base32.js');
 
 const CLOSE_TIMEOUT = 1 * 1000; // how much to wait until pending connections are terminated
 
+// A PROXY protocol v1 header is at most ~107 bytes. Bound the buffer so a trusted
+// proxy source that never sends a newline cannot grow memory without limit.
+const MAX_PROXY_HEADER_SIZE = 256;
+
 /**
  * Creates a IMAP server instance.
  *
@@ -273,6 +277,26 @@ class IMAPServer extends EventEmitter {
                 }
                 chunks.push(chunk);
                 chunklen += chunk.length;
+
+                if (chunklen > MAX_PROXY_HEADER_SIZE) {
+                    socket.removeListener('readable', socketReader);
+                    try {
+                        socket.end('* BAD Invalid PROXY header\r\n');
+                    } catch (E) {
+                        // ignore
+                    }
+                    // end() only half-closes the socket; arm a short timeout so the fd is
+                    // released even if the peer never closes its side (fd exhaustion).
+                    socket.setTimeout(CLOSE_TIMEOUT);
+                    socket.once('timeout', () => {
+                        try {
+                            socket.destroy();
+                        } catch (E) {
+                            // ignore
+                        }
+                    });
+                    return;
+                }
             }
         };
         socket.on('readable', socketReader);

package/lib/imapproxy/imap-core/lib/imap-stream.js

@@ -4,6 +4,11 @@ const stream = require('stream');
 const Writable = stream.Writable;
 const PassThrough = stream.PassThrough;
 
+// Upper bound for a single command line (no CRLF yet). Generous - real IMAP command
+// lines are short and bulk data is sent as size-bounded literals - but it prevents a
+// client that never sends a newline from growing memory without limit (OOM).
+const MAX_LINE_LENGTH = 1 * 1024 * 1024;
+
 /**
  * Incoming IMAP stream parser. Detects and emits command payloads.
  * If literal values are encountered the command payload is split into parts
@@ -21,6 +26,9 @@ class IMAPStream extends Writable {
         this.options = options || {};
         Writable.call(this, this.options);
 
+        // largest single command line we will buffer before giving up
+        this._maxLineLength = Number(this.options.maxLineLength) || MAX_LINE_LENGTH;
+
         // unprocessed chars from the last parsing iteration
         this._remainder = '';
         this._literal = false;
@@ -40,6 +48,18 @@ class IMAPStream extends Writable {
         throw new Error('Command handler is not set');
     }
 
+    /**
+     * Discards any buffered or partial parser state. Used on TLS upgrade so that
+     * plaintext pipelined before STARTTLS cannot be parsed as commands inside the
+     * encrypted session (STARTTLS command injection).
+     */
+    reset() {
+        this._remainder = '';
+        this._literal = false;
+        this._literalReady = false;
+        this._expecting = 0;
+    }
+
     // PRIVATE METHODS
 
     /**
@@ -107,6 +127,12 @@ class IMAPStream extends Writable {
             pos += line.length + match[0].length;
         } else {
             this._remainder = pos < data.length ? data.substr(pos) : '';
+            if (this._remainder.length > this._maxLineLength) {
+                // Drop the buffered data and fail the stream - the connection layer
+                // closes the socket on a parser error.
+                this._remainder = '';
+                return done(new Error('Command line too long'));
+            }
             return done();
         }
 

package/lib/message-port-stream.js

@@ -1,6 +1,6 @@
 'use strict';
 
-const { Writable, Readable } = require('stream');
+const { Writable, Readable, pipeline } = require('stream');
 
 // const { MessageChannel } = require('worker_threads');
 // const { port1, port2 } = new MessageChannel();
@@ -9,6 +9,41 @@ class MessagePortWritable extends Writable {
     constructor(messagePort) {
         super();
         this.messagePort = messagePort;
+        this.portClosed = false;
+
+        // The reader side posts { cancel: true } when it is torn down (e.g. the
+        // HTTP client aborted the download). Stop the transfer so the upstream
+        // source - and the IMAP lock it holds - is released.
+        this.onPortMessage = message => {
+            if (message && message.cancel) {
+                this.destroy();
+            }
+        };
+        this.messagePort.on('message', this.onPortMessage);
+    }
+
+    postToPort(message) {
+        if (this.portClosed) {
+            return;
+        }
+        try {
+            this.messagePort.postMessage(message);
+        } catch (err) {
+            // ignore - the channel may already be torn down
+        }
+    }
+
+    closePort() {
+        if (this.portClosed) {
+            return;
+        }
+        this.portClosed = true;
+        this.messagePort.removeListener('message', this.onPortMessage);
+        try {
+            this.messagePort.close();
+        } catch (err) {
+            // ignore
+        }
     }
 
     _write(chunk, encoding, done) {
@@ -20,24 +55,25 @@ class MessagePortWritable extends Writable {
             chunk = Buffer.from(chunk, encoding);
         }
 
-        this.messagePort.postMessage({
-            value: chunk,
-            done: false
-        });
+        this.postToPort({ value: chunk, done: false });
         done();
     }
 
     _final(done) {
-        this.messagePort.postMessage({
-            done: true
-        });
-        try {
-            this.messagePort.close();
-        } catch (err) {
-            //ignore
-        }
+        this.postToPort({ done: true });
+        this.closePort();
         done();
     }
+
+    _destroy(err, done) {
+        // Abnormal termination: tell the reader so a truncated transfer is not
+        // mistaken for a complete message, then release the port.
+        if (err) {
+            this.postToPort({ error: err.message || 'Stream error' });
+        }
+        this.closePort();
+        done(err);
+    }
 }
 
 class MessagePortReadable extends Readable {
@@ -46,17 +82,28 @@ class MessagePortReadable extends Readable {
         this.messagePort = messagePort;
 
         this.canRead = false;
+        this.portClosed = false;
 
         this.readableQueue = [];
-        this.messagePort.on('message', message => {
-            if (message && (message.done || message.value)) {
+        this.onPortMessage = message => {
+            if (!message) {
+                return;
+            }
+            if (message.error) {
+                // The producer failed mid-transfer - surface it as a stream error
+                // rather than letting the consumer believe it received everything.
+                this.destroy(new Error(message.error));
+                return;
+            }
+            if (message.done || message.value) {
                 this.readableQueue.push(message);
 
                 if (this.canRead && this.readableQueue.length === 1) {
                     this._processReading();
                 }
             }
-        });
+        };
+        this.messagePort.on('message', this.onPortMessage);
     }
 
     _processReading() {
@@ -73,7 +120,57 @@ class MessagePortReadable extends Readable {
         this.canRead = true;
         this._processReading();
     }
+
+    _destroy(err, done) {
+        // Consumer is gone (client aborted, error, or clean end): tell the producer
+        // to stop and release the port + listener so nothing leaks across threads.
+        if (!this.portClosed) {
+            this.portClosed = true;
+            try {
+                this.messagePort.postMessage({ cancel: true });
+            } catch (cancelErr) {
+                // ignore - the peer may already be closed
+            }
+            this.messagePort.removeListener('message', this.onPortMessage);
+            try {
+                this.messagePort.close();
+            } catch (closeErr) {
+                // ignore
+            }
+        }
+        done(err);
+    }
+}
+
+/**
+ * Pipes a readable source into a MessagePortWritable so that an error on either
+ * side tears the transfer down instead of surfacing as an unhandled 'error'
+ * event. A mid-download failure on an IMAP source stream would otherwise crash
+ * the worker (and every account assigned to it).
+ *
+ * @param {Readable} source - Source stream (e.g. an IMAP download stream)
+ * @param {MessagePortWritable} writable - Destination bound to a MessagePort
+ * @param {Object} [logger] - Optional logger used to report transfer failures
+ */
+function pipeToMessagePort(source, writable, logger) {
+    pipeline(source, writable, err => {
+        if (!err || !logger) {
+            return;
+        }
+        // A consumer that aborts mid-download closes the destination early; that
+        // is expected (not a failure) so it should not be logged at error level.
+        if (err.code === 'ERR_STREAM_PREMATURE_CLOSE') {
+            if (typeof logger.debug === 'function') {
+                logger.debug({ msg: 'Message stream transfer aborted by consumer', err });
+            }
+            return;
+        }
+        if (typeof logger.error === 'function') {
+            logger.error({ msg: 'Message stream transfer failed', err });
+        }
+    });
 }
 
 module.exports.MessagePortWritable = MessagePortWritable;
 module.exports.MessagePortReadable = MessagePortReadable;
+module.exports.pipeToMessagePort = pipeToMessagePort;