amqp-connection-manager 3.5.0 → 3.7.0

package/README.md

@@ -2,13 +2,8 @@
 
 [![NPM version](https://badge.fury.io/js/amqp-connection-manager.svg)](https://npmjs.org/package/amqp-connection-manager)
 ![Build Status](https://github.com/jwalton/node-amqp-connection-manager/workflows/GitHub%20CI/badge.svg)
-[![Coverage Status](https://coveralls.io/repos/jwalton/node-amqp-connection-manager/badge.svg?branch=master&service=github)](https://coveralls.io/github/jwalton/node-amqp-connection-manager?branch=master)
 [![semantic-release](https://img.shields.io/badge/%20%20%F0%9F%93%A6%F0%9F%9A%80-semantic--release-e10079.svg)](https://github.com/semantic-release/semantic-release)
 
-[![Dependency Status](https://david-dm.org/jwalton/node-amqp-connection-manager.svg)](https://david-dm.org/jwalton/node-amqp-connection-manager)
-[![devDependency Status](https://david-dm.org/jwalton/node-amqp-connection-manager/dev-status.svg)](https://david-dm.org/jwalton/node-amqp-connection-manager#info=devDependencies)
-[![peerDependency Status](https://david-dm.org/jwalton/node-amqp-connection-manager/peer-status.svg)](https://david-dm.org/jwalton/node-amqp-connection-manager#info=peerDependencies)
-
 Connection management for amqplib. This is a wrapper around [amqplib](http://www.squaremobius.net/amqp.node/) which provides automatic reconnects.
 
 ## Features
@@ -139,6 +134,7 @@ Options:
   arbitrary data in.
 - `options.json` if true, then ChannelWrapper assumes all messages passed to `publish()` and `sendToQueue()`
   are plain JSON objects. These will be encoded automatically before being sent.
+- `options.publishTimeout` - a default timeout for messages published to this channel.
 
 ### AmqpConnectionManager#isConnected()
 
@@ -183,6 +179,12 @@ callback) which resolves when the message is confirmed to have been delivered to
 either the broker refuses the message, or if `close()` is called on the ChannelWrapper before the message can be
 delivered.
 
+Both of these functions take an additional option when passing options:
+
+- `timeout` - If specified, if a messages is not acked by the amqp broker within the specified number of milliseconds,
+  the message will be rejected. Note that the message _may_ still end up getting delivered after the timeout, as we
+  have no way to cancel the in-flight request.
+
 ### ChannelWrapper#ack and ChannelWrapper#nack
 
 These are just aliases for calling `ack()` and `nack()` on the underlying channel. They do nothing if the underlying

package/dist/cjs/AmqpConnectionManager.js

@@ -72,7 +72,41 @@ class AmqpConnectionManager extends events_1.EventEmitter {
         // There will be one listener per channel, and there could be a lot of channels, so disable warnings from node.
         this.setMaxListeners(0);
         this._findServers = options.findServers || (() => Promise.resolve(urls));
+    }
+    /**
+     * Start the connect retries and await the first connect result. Even if the initial connect fails or timeouts, the
+     * reconnect attempts will continue in the background.
+     * @param [options={}] -
+     * @param [options.timeout] - Time to wait for initial connect
+     */
+    async connect({ timeout } = {}) {
         this._connect();
+        let reject;
+        const onDisconnect = ({ err }) => {
+            // Ignore disconnects caused by dead servers etc., but throw on operational errors like bad credentials.
+            if (err.isOperational) {
+                reject(err);
+            }
+        };
+        try {
+            await Promise.race([
+                (0, events_1.once)(this, 'connect'),
+                new Promise((_resolve, innerReject) => {
+                    reject = innerReject;
+                    this.on('disconnect', onDisconnect);
+                }),
+                ...(timeout
+                    ? [
+                        (0, helpers_js_1.wait)(timeout).promise.then(() => {
+                            throw new Error('amqp-connection-manager: connect timeout');
+                        }),
+                    ]
+                    : []),
+            ]);
+        }
+        finally {
+            this.removeListener('disconnect', onDisconnect);
+        }
     }
     // `options` here are any options that can be passed to ChannelWrapper.
     createChannel(options = {}) {
@@ -221,7 +255,7 @@ class AmqpConnectionManager extends events_1.EventEmitter {
                     this._currentConnection = undefined;
                     this._disconnectSent = true;
                     this.emit('disconnect', { err });
-                    const handle = helpers_js_1.wait(this.reconnectTimeInSeconds * 1000);
+                    const handle = (0, helpers_js_1.wait)(this.reconnectTimeInSeconds * 1000);
                     this._cancelRetriesHandler = handle.cancel;
                     handle.promise
                         .then(() => this._connect())
@@ -245,10 +279,10 @@ class AmqpConnectionManager extends events_1.EventEmitter {
             this._connectPromise = undefined;
             let handle;
             if (err.name === 'OperationalError' && err.message === 'connect ETIMEDOUT') {
-                handle = helpers_js_1.wait(0);
+                handle = (0, helpers_js_1.wait)(0);
             }
             else {
-                handle = helpers_js_1.wait(this.reconnectTimeInSeconds * 1000);
+                handle = (0, helpers_js_1.wait)(this.reconnectTimeInSeconds * 1000);
             }
             this._cancelRetriesHandler = handle.cancel;
             return handle.promise.then(() => this._connect());

package/dist/cjs/ChannelWrapper.js

@@ -5,6 +5,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 Object.defineProperty(exports, "__esModule", { value: true });
 const events_1 = require("events");
 const promise_breaker_1 = __importDefault(require("promise-breaker"));
+const MAX_MESSAGES_PER_BATCH = 1000;
 const IRRECOVERABLE_ERRORS = [
     403,
     404,
@@ -56,6 +57,8 @@ class ChannelWrapper extends events_1.EventEmitter {
         this._messages = [];
         /** Oublished, but not yet confirmed messages. */
         this._unconfirmedMessages = [];
+        /** Consumers which will be reconnected on channel errors etc. */
+        this._consumers = [];
         /**
          * True if the "worker" is busy sending messages.  False if we need to
          * start the worker to get stuff done.
@@ -67,13 +70,19 @@ class ChannelWrapper extends events_1.EventEmitter {
          * workers ever do wake up, they'll know to stop working.
          */
         this._workerNumber = 0;
+        /**
+         * True if the underlying channel has room for more messages.
+         */
+        this._channelHasRoom = true;
         this._onConnect = this._onConnect.bind(this);
         this._onDisconnect = this._onDisconnect.bind(this);
         this._connectionManager = connectionManager;
         this.name = options.name;
+        this._publishTimeout = options.publishTimeout;
         this._json = (_a = options.json) !== null && _a !== void 0 ? _a : false;
         // Array of setup functions to call.
         this._setups = [];
+        this._consumers = [];
         if (options.setup) {
             this._setups.push(options.setup);
         }
@@ -172,15 +181,17 @@ class ChannelWrapper extends events_1.EventEmitter {
      */
     publish(exchange, routingKey, content, options, done) {
         return promise_breaker_1.default.addCallback(done, new Promise((resolve, reject) => {
-            this._messages.push({
+            const { timeout, ...opts } = options || {};
+            this._enqueueMessage({
                 type: 'publish',
                 exchange,
                 routingKey,
                 content,
-                options,
                 resolve,
                 reject,
-            });
+                options: opts,
+                isTimedout: false,
+            }, timeout || this._publishTimeout);
             this._startWorker();
         }));
     }
@@ -195,24 +206,47 @@ class ChannelWrapper extends events_1.EventEmitter {
      */
     sendToQueue(queue, content, options, done) {
         return promise_breaker_1.default.addCallback(done, new Promise((resolve, reject) => {
-            this._messages.push({
+            const { timeout, ...opts } = options || {};
+            this._enqueueMessage({
                 type: 'sendToQueue',
                 queue,
                 content,
-                options,
                 resolve,
                 reject,
-            });
-            return this._startWorker();
+                options: opts,
+                isTimedout: false,
+            }, timeout || this._publishTimeout);
+            this._startWorker();
         }));
     }
+    _enqueueMessage(message, timeout) {
+        if (timeout) {
+            message.timeout = setTimeout(() => {
+                let idx = this._messages.indexOf(message);
+                if (idx !== -1) {
+                    this._messages.splice(idx, 1);
+                }
+                else {
+                    idx = this._unconfirmedMessages.indexOf(message);
+                    if (idx !== -1) {
+                        this._unconfirmedMessages.splice(idx, 1);
+                    }
+                }
+                message.isTimedout = true;
+                message.reject(new Error('timeout'));
+            }, timeout);
+        }
+        this._messages.push(message);
+    }
     // Called whenever we connect to the broker.
     async _onConnect({ connection }) {
         this._irrecoverableCode = undefined;
         try {
             const channel = await connection.createConfirmChannel();
             this._channel = channel;
+            this._channelHasRoom = true;
             channel.on('close', () => this._onChannelClose(channel));
+            channel.on('drain', () => this._onChannelDrain());
             this._settingUp = Promise.all(this._setups.map((setupFn) => 
             // TODO: Use a timeout here to guard against setupFns that never resolve?
             promise_breaker_1.default.call(setupFn, this, channel).catch((err) => {
@@ -221,7 +255,11 @@ class ChannelWrapper extends events_1.EventEmitter {
                     return;
                 }
                 this.emit('error', err, { name: this.name });
-            }))).then(() => {
+            })))
+                .then(() => {
+                return Promise.all(this._consumers.map((c) => this._reconnectConsumer(c)));
+            })
+                .then(() => {
                 this._settingUp = undefined;
             });
             await this._settingUp;
@@ -229,13 +267,6 @@ class ChannelWrapper extends events_1.EventEmitter {
                 // Can happen if channel closes while we're setting up.
                 return;
             }
-            if (this._unconfirmedMessages.length > 0) {
-                // requeue any messages that were left unconfirmed when connection was lost
-                let message;
-                while ((message = this._unconfirmedMessages.shift())) {
-                    this._messages.push(message);
-                }
-            }
             // Since we just connected, publish any queued messages
             this._startWorker();
             this.emit('connect');
@@ -253,6 +284,11 @@ class ChannelWrapper extends events_1.EventEmitter {
         }
         // Wait for another reconnect to create a new channel.
     }
+    /** Called whenever the channel drains. */
+    _onChannelDrain() {
+        this._channelHasRoom = true;
+        this._startWorker();
+    }
     // Called whenever we disconnect from the AMQP server.
     _onDisconnect(ex) {
         this._irrecoverableCode = ex.err instanceof Error ? ex.err.code : undefined;
@@ -275,11 +311,21 @@ class ChannelWrapper extends events_1.EventEmitter {
             this._working = false;
             if (this._messages.length !== 0) {
                 // Reject any unsent messages.
-                this._messages.forEach((message) => message.reject(new Error('Channel closed')));
+                this._messages.forEach((message) => {
+                    if (message.timeout) {
+                        clearTimeout(message.timeout);
+                    }
+                    message.reject(new Error('Channel closed'));
+                });
             }
             if (this._unconfirmedMessages.length !== 0) {
                 // Reject any unconfirmed messages.
-                this._unconfirmedMessages.forEach((message) => message.reject(new Error('Channel closed')));
+                this._unconfirmedMessages.forEach((message) => {
+                    if (message.timeout) {
+                        clearTimeout(message.timeout);
+                    }
+                    message.reject(new Error('Channel closed'));
+                });
             }
             this._connectionManager.removeListener('connect', this._onConnect);
             this._connectionManager.removeListener('disconnect', this._onDisconnect);
@@ -290,7 +336,7 @@ class ChannelWrapper extends events_1.EventEmitter {
         });
     }
     _shouldPublish() {
-        return this._messages.length > 0 && !this._settingUp && !!this._channel;
+        return (this._messages.length > 0 && !this._settingUp && !!this._channel && this._channelHasRoom);
     }
     // Start publishing queued messages, if there isn't already a worker doing this.
     _startWorker() {
@@ -304,6 +350,44 @@ class ChannelWrapper extends events_1.EventEmitter {
     _canWaitReconnection() {
         return !this._irrecoverableCode || !IRRECOVERABLE_ERRORS.includes(this._irrecoverableCode);
     }
+    _messageResolved(message, result) {
+        removeUnconfirmedMessage(this._unconfirmedMessages, message);
+        message.resolve(result);
+    }
+    _messageRejected(message, err) {
+        if (!this._channel && this._canWaitReconnection()) {
+            // Tried to write to a closed channel.  Leave the message in the queue and we'll try again when
+            // we reconnect.
+            removeUnconfirmedMessage(this._unconfirmedMessages, message);
+            this._messages.push(message);
+        }
+        else {
+            // Something went wrong trying to send this message - could be JSON.stringify failed, could be
+            // the broker rejected the message. Either way, reject it back
+            removeUnconfirmedMessage(this._unconfirmedMessages, message);
+            message.reject(err);
+        }
+    }
+    _getEncodedMessage(content) {
+        let encodedMessage;
+        if (this._json) {
+            encodedMessage = Buffer.from(JSON.stringify(content));
+        }
+        else if (typeof content === 'string') {
+            encodedMessage = Buffer.from(content);
+        }
+        else if (content instanceof Buffer) {
+            encodedMessage = content;
+        }
+        else if (typeof content === 'object' && typeof content.toString === 'function') {
+            encodedMessage = Buffer.from(content.toString());
+        }
+        else {
+            console.warn('amqp-connection-manager: Sending JSON message, but json option not speicifed');
+            encodedMessage = Buffer.from(JSON.stringify(content));
+        }
+        return encodedMessage;
+    }
     _publishQueuedMessages(workerNumber) {
         const channel = this._channel;
         if (!channel ||
@@ -314,89 +398,136 @@ class ChannelWrapper extends events_1.EventEmitter {
             this._working = false;
             return;
         }
-        const message = this._messages.shift();
-        if (message) {
-            this._unconfirmedMessages.push(message);
-            Promise.resolve()
-                .then(() => {
-                let encodedMessage;
-                if (this._json) {
-                    encodedMessage = Buffer.from(JSON.stringify(message.content));
-                }
-                else if (typeof message.content === 'string') {
-                    encodedMessage = Buffer.from(message.content);
-                }
-                else if (message.content instanceof Buffer) {
-                    encodedMessage = message.content;
-                }
-                else if (typeof message.content === 'object' &&
-                    typeof message.content.toString === 'function') {
-                    encodedMessage = Buffer.from(message.content.toString());
-                }
-                else {
-                    throw new Error('Invalid message content');
+        try {
+            // Send messages in batches of 1000 - don't want to starve the event loop.
+            let sendsLeft = MAX_MESSAGES_PER_BATCH;
+            while (this._channelHasRoom && this._messages.length > 0 && sendsLeft > 0) {
+                sendsLeft--;
+                const message = this._messages.shift();
+                if (!message) {
+                    break;
                 }
-                let result = true;
-                const sendPromise = (() => {
-                    switch (message.type) {
-                        case 'publish':
-                            return new Promise(function (resolve, reject) {
-                                result = channel.publish(message.exchange, message.routingKey, encodedMessage, message.options, (err) => {
-                                    if (err) {
-                                        reject(err);
-                                    }
-                                    else {
-                                        resolve(result);
-                                    }
-                                });
-                            });
-                        case 'sendToQueue':
-                            return new Promise(function (resolve, reject) {
-                                result = channel.sendToQueue(message.queue, encodedMessage, message.options, (err) => {
-                                    if (err) {
-                                        reject(err);
-                                    }
-                                    else {
-                                        resolve(result);
-                                    }
-                                });
-                            });
-                        /* istanbul ignore next */
-                        default:
-                            throw new Error(`Unhandled message type ${message.type}`);
+                this._unconfirmedMessages.push(message);
+                const encodedMessage = this._getEncodedMessage(message.content);
+                switch (message.type) {
+                    case 'publish': {
+                        let thisCanSend = true;
+                        thisCanSend = this._channelHasRoom = channel.publish(message.exchange, message.routingKey, encodedMessage, message.options, (err) => {
+                            if (message.isTimedout) {
+                                return;
+                            }
+                            if (message.timeout) {
+                                clearTimeout(message.timeout);
+                            }
+                            if (err) {
+                                this._messageRejected(message, err);
+                            }
+                            else {
+                                this._messageResolved(message, thisCanSend);
+                            }
+                        });
+                        break;
                     }
-                })();
-                if (result) {
-                    this._publishQueuedMessages(workerNumber);
-                }
-                else {
-                    channel.once('drain', () => this._publishQueuedMessages(workerNumber));
-                }
-                return sendPromise;
-            })
-                .then((result) => {
-                removeUnconfirmedMessage(this._unconfirmedMessages, message);
-                message.resolve(result);
-            }, (err) => {
-                if (!this._channel && this._canWaitReconnection()) {
-                    // Tried to write to a closed channel.  Leave the message in the queue and we'll try again when
-                    // we reconnect.
-                    removeUnconfirmedMessage(this._unconfirmedMessages, message);
-                    this._messages.push(message);
-                }
-                else {
-                    // Something went wrong trying to send this message - could be JSON.stringify failed, could be
-                    // the broker rejected the message. Either way, reject it back
-                    removeUnconfirmedMessage(this._unconfirmedMessages, message);
-                    message.reject(err);
+                    case 'sendToQueue': {
+                        let thisCanSend = true;
+                        thisCanSend = this._channelHasRoom = channel.sendToQueue(message.queue, encodedMessage, message.options, (err) => {
+                            if (message.isTimedout) {
+                                return;
+                            }
+                            if (message.timeout) {
+                                clearTimeout(message.timeout);
+                            }
+                            if (err) {
+                                this._messageRejected(message, err);
+                            }
+                            else {
+                                this._messageResolved(message, thisCanSend);
+                            }
+                        });
+                        break;
+                    }
+                    /* istanbul ignore next */
+                    default:
+                        throw new Error(`Unhandled message type ${message.type}`);
                 }
-            })
-                .catch(
-            /* istanbul ignore next */ (err) => {
-                this.emit('error', err);
-                this._working = false;
-            });
+            }
+            // If we didn't send all the messages, send some more...
+            if (this._channelHasRoom && this._messages.length > 0) {
+                setImmediate(() => this._publishQueuedMessages(workerNumber));
+            }
+            this._working = false;
+            /* istanbul ignore next */
+        }
+        catch (err) {
+            this._working = false;
+            this.emit('error', err);
+        }
+    }
+    /**
+     * Setup a consumer
+     * This consumer will be reconnected on cancellation and channel errors.
+     */
+    async consume(queue, onMessage, options = {}) {
+        const consumer = {
+            consumerTag: null,
+            queue,
+            onMessage,
+            options,
+        };
+        this._consumers.push(consumer);
+        await this._consume(consumer);
+    }
+    async _consume(consumer) {
+        if (!this._channel) {
+            return;
+        }
+        const { prefetch, ...options } = consumer.options;
+        if (typeof prefetch === 'number') {
+            this._channel.prefetch(prefetch, false);
+        }
+        const { consumerTag } = await this._channel.consume(consumer.queue, (msg) => {
+            if (!msg) {
+                consumer.consumerTag = null;
+                this._reconnectConsumer(consumer).catch((err) => {
+                    if (err.isOperational && err.message.includes('BasicConsume; 404')) {
+                        // Ignore errors caused by queue not declared. In
+                        // those cases the connection will reconnect and
+                        // then consumers reestablished. The full reconnect
+                        // might be avoided if we assert the queue again
+                        // before starting to consume.
+                        return;
+                    }
+                    throw err;
+                });
+                return;
+            }
+            consumer.onMessage(msg);
+        }, options);
+        consumer.consumerTag = consumerTag;
+    }
+    async _reconnectConsumer(consumer) {
+        if (!this._consumers.includes(consumer)) {
+            // Intentionally canceled
+            return;
         }
+        await this._consume(consumer);
+    }
+    /**
+     * Cancel all consumers
+     */
+    async cancelAll() {
+        const consumers = this._consumers;
+        this._consumers = [];
+        if (!this._channel) {
+            return;
+        }
+        const channel = this._channel;
+        await Promise.all(consumers.reduce((acc, consumer) => {
+            if (consumer.consumerTag) {
+                acc.push(channel.cancel(consumer.consumerTag));
+            }
+            return acc;
+        }, []));
     }
     /** Send an `ack` to the underlying channel. */
     ack(message, allUpTo) {

package/dist/cjs/index.js

@@ -3,10 +3,15 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.connect = void 0;
+exports.AmqpConnectionManagerClass = exports.connect = void 0;
 const AmqpConnectionManager_js_1 = __importDefault(require("./AmqpConnectionManager.js"));
+exports.AmqpConnectionManagerClass = AmqpConnectionManager_js_1.default;
 function connect(urls, options) {
-    return new AmqpConnectionManager_js_1.default(urls, options);
+    const conn = new AmqpConnectionManager_js_1.default(urls, options);
+    conn.connect().catch(() => {
+        /* noop */
+    });
+    return conn;
 }
 exports.connect = connect;
 const amqp = { connect };

package/dist/esm/AmqpConnectionManager.d.ts

@@ -59,6 +59,7 @@ export interface IAmqpConnectionManager {
     addListener(event: 'disconnect', listener: (arg: {
         err: Error;
     }) => void): this;
+    listeners(eventName: string | symbol): Function[];
     on(event: string, listener: (...args: any[]) => void): this;
     on(event: 'connect', listener: ConnectListener): this;
     on(event: 'blocked', listener: (arg: {
@@ -96,13 +97,17 @@ export interface IAmqpConnectionManager {
         err: Error;
     }) => void): this;
     removeListener(event: string, listener: (...args: any[]) => void): this;
+    connect(options?: {
+        timeout?: number;
+    }): Promise<void>;
+    reconnect(): void;
     createChannel(options?: CreateChannelOpts): ChannelWrapper;
     close(): Promise<void>;
     isConnected(): boolean;
     /** The current connection. */
-    get connection(): Connection | undefined;
+    readonly connection: Connection | undefined;
     /** Returns the number of registered channels. */
-    get channelCount(): number;
+    readonly channelCount: number;
 }
 export default class AmqpConnectionManager extends EventEmitter implements IAmqpConnectionManager {
     private _channels;
@@ -148,6 +153,15 @@ export default class AmqpConnectionManager extends EventEmitter implements IAmqp
      *   Note that if this is supplied, then `urls` is ignored.
      */
     constructor(urls: ConnectionUrl | ConnectionUrl[] | undefined | null, options?: AmqpConnectionManagerOptions);
+    /**
+     * Start the connect retries and await the first connect result. Even if the initial connect fails or timeouts, the
+     * reconnect attempts will continue in the background.
+     * @param [options={}] -
+     * @param [options.timeout] - Time to wait for initial connect
+     */
+    connect({ timeout }?: {
+        timeout?: number;
+    }): Promise<void>;
     createChannel(options?: CreateChannelOpts): ChannelWrapper;
     close(): Promise<void>;
     isConnected(): boolean;

package/dist/esm/AmqpConnectionManager.js

@@ -1,5 +1,5 @@
 import amqp from 'amqplib';
-import { EventEmitter } from 'events';
+import { EventEmitter, once } from 'events';
 import pb from 'promise-breaker';
 import { URL } from 'url';
 import ChannelWrapper from './ChannelWrapper.js';
@@ -67,7 +67,41 @@ export default class AmqpConnectionManager extends EventEmitter {
         // There will be one listener per channel, and there could be a lot of channels, so disable warnings from node.
         this.setMaxListeners(0);
         this._findServers = options.findServers || (() => Promise.resolve(urls));
+    }
+    /**
+     * Start the connect retries and await the first connect result. Even if the initial connect fails or timeouts, the
+     * reconnect attempts will continue in the background.
+     * @param [options={}] -
+     * @param [options.timeout] - Time to wait for initial connect
+     */
+    async connect({ timeout } = {}) {
         this._connect();
+        let reject;
+        const onDisconnect = ({ err }) => {
+            // Ignore disconnects caused by dead servers etc., but throw on operational errors like bad credentials.
+            if (err.isOperational) {
+                reject(err);
+            }
+        };
+        try {
+            await Promise.race([
+                once(this, 'connect'),
+                new Promise((_resolve, innerReject) => {
+                    reject = innerReject;
+                    this.on('disconnect', onDisconnect);
+                }),
+                ...(timeout
+                    ? [
+                        wait(timeout).promise.then(() => {
+                            throw new Error('amqp-connection-manager: connect timeout');
+                        }),
+                    ]
+                    : []),
+            ]);
+        }
+        finally {
+            this.removeListener('disconnect', onDisconnect);
+        }
     }
     // `options` here are any options that can be passed to ChannelWrapper.
     createChannel(options = {}) {