elasticio-sailor-nodejs 2.7.0-dev3 → 2.7.0

package/.nsprc

@@ -0,0 +1,6 @@
+{
+    "GHSA-wc69-rhjr-hc9g": {
+         "active": true,
+         "notes": "Bunyan library set only new Date to momentjs as parameter"
+     }
+}

package/CHANGELOG.md

@@ -0,0 +1,144 @@
+## 2.7.0 (September 15, 2022)
+
+* Add AMQP_PERSISTENT_MESSAGES configuration env var to enable persistent delivery mode.
+
+## 2.6.29 (July 14, 2022)
+
+* Enabled keep-alive for global HTTPS agent ([#6359](https://github.com/elasticio/elasticio/issues/6359))
+
+## 2.6.28 (June 21, 2022)
+
+* Fix: "sailor-nodejs ignores errors from maester during lightweight message upload" [#6233](https://github.com/elasticio/elasticio/issues/6233))
+
+## 2.6.27 (March 10, 2022)
+
+* Added npm audit to CI and fixed all dependencies
+
+## 2.6.26 (June 24, 2021)
+
+* Replaced exit on "unhandledRejection" with error logging
+
+## 2.6.25 (June 17, 2021)
+
+* Improve consume performance by switching back to pushing instead of polling, keeping reconnections mechanism
+  ([#5432](https://github.com/elasticio/elasticio/issues/5432))
+
+## 2.6.24 (January 27, 2021)
+
+* Fix "Big messages processing is not stable" errors ([#5051](https://github.com/elasticio/elasticio/issues/5051))
+
+## 2.6.23 (January 4, 2021)
+
+* Fix "Cannot read property 'publish' of undefined" error ([#5036](https://github.com/elasticio/elasticio/issues/5036))
+
+## 2.6.22 (December 23, 2020)
+
+* Enabled keep-alive for global HTTP agent ([#5012](#https://github.com/elasticio/elasticio/issues/5012))
+
+## 2.6.21 (December 3, 2020)
+
+* Fixed rebound message expiry time ([#4950](https://github.com/elasticio/elasticio/issues/4950))
+
+## 2.6.20 (December 1, 2020)
+
+* Fixed rebound message headers ([#4950](https://github.com/elasticio/elasticio/issues/4950))
+
+## 2.6.19 (November 23, 2020)
+
+* Separate connections for consuming and publishing
+* Consuming is done with polling instead of pushing
+* Reconnects on connection errors
+* Handling consumer cancel notification
+* Lowered log levels of some developers' log messages
+* Addded env vars:
+    * AMQP_RECONNECT_ATTEMPTS - number of retries on connection close
+    * AMQP_RECONNECT_TIMEOUT - delay between connection retries
+    * WAIT_MESSAGES_TIMEOUT - delay between next poll when queue is empty
+
+## 2.6.18 (October 26, 2020)
+
+* Remove the logging of triggers and actions processing errors stack
+
+## 2.6.17 (October 15, 2020)
+
+* Annual audit of the component code to check if it exposes a sensitive data in the logs
+
+## 2.6.16 (October 12, 2020)
+
+* Fix incoming headers appearance in the logs (part 2)
+
+## 2.6.15 (October 12, 2020)
+
+* Fix incoming headers appearance in the logs (part 1)
+
+## 2.6.14 (July 06, 2020)
+
+* Add Lightweight messages support
+* Sync this.emit() calls are not supported anymore. Use async process() interface and await this.emit() calls instead
+
+## 2.6.13 (July 01, 2020)
+
+* Error as incoming message in custom error handler
+
+## 2.6.10 (June 03, 2020)
+
+* Fix bug with incorrect publish retry policy.Dynamic flow control
+* Dynamic flow control
+
+## 2.6.9 (May 26, 2020)
+
+* Fix bugs with 'Retention policy' notification and 'Nodejs sailor return promise interface does not support dynamic flow control'
+
+## 2.6.8 (May 18, 2020)
+
+* Fix bug when Lookout throws exception if incoming message from error queue doesn't have errorInput property
+* From now on errors (description + stack) happening during component initialization won't be ignored and you will see them on frontend and in logs
+
+## 2.6.7 (May 07, 2020)
+
+* Add ability to publish to arbitrary exchange
+
+## 2.6.6 (May 06, 2020)
+
+* From now on Wiper will not suspend overloaded flows if all the components in the flow have the latest sailor (which supports dynamic flow control)
+
+## 2.6.5 (February 20, 2020)
+
+* Add support of non-base64 message in Admiral. This function is activated if 2 neighbour components' Sailors support a feature of non-base64 message
+
+## 2.6.4 (February 14, 2020)
+
+* Enable graceful restart for tasks pods
+
+## 2.6.2 (January 29, 2020)
+
+* Fix bug when Sailor does not reliably publish large messages
+
+## 2.6.1 (January 15, 2020)
+
+* Fix bug when publishing fail leads to fail in all subsequent pending messages
+
+## 2.6.0 (January 06, 2020)
+
+* A step must not put its own output message into the passthrough object
+
+## 2.5.4 (December 23, 2019)
+
+* Introduce new environment variable for Sailor: ELASTICIO_OUTGOING_MESSAGE_SIZE_LIMIT, which controls the outgoing message size limit
+
+## 2.5.3 (December 11, 2019)
+
+* Introduce new component environment variable ELASTICIO_ADDITIONAL_VARS_FOR_HEADERS. It contains comma-separated environment variables that will be passed to message headers
+
+## 2.4.1 (July 18, 2019)
+
+* Add additional information to RabbitMQ messages
+
+## 2.4.0 (June 11, 2019)
+
+* Add elastic's Threads functionality support
+* Add component custom logger. E.g. `this.logger.info('hello')`
+
+## 2.3.0 (October 19, 2018)
+
+* Sailor now handles RabbitMQ disconnects correctly

package/lib/amqp.js

@@ -1,10 +1,11 @@
 const log = require('./logging.js');
 const amqplib = require('amqplib');
 const { IllegalOperationError } = require('amqplib/lib/error');
-const encryptor = require('./encryptor.js');
+const Encryptor = require('./encryptor.js');
 const _ = require('lodash');
 const eventToPromise = require('event-to-promise');
 const uuid = require('uuid');
+const os = require('os');
 
 const HEADER_ROUTING_KEY = 'x-eio-routing-key';
 const HEADER_ERROR_RESPONSE = 'x-eio-error-response';
@@ -12,32 +13,34 @@ const HEADER_ERROR_RESPONSE = 'x-eio-error-response';
 class Amqp {
     constructor(settings) {
         this.settings = settings;
+        this._encryptor = new Encryptor(this.settings.MESSAGE_CRYPTO_PASSWORD, this.settings.MESSAGE_CRYPTO_IV);
+        this.closed = true;
+        this.consume = undefined;
     }
 
-    async connect(uri) {
-        this.consumerTag = null;
-        this.amqp = await amqplib.connect(uri);
-        if (process.env.NODE_ENV !== 'test') {
-            this.amqp.on('error', log.criticalErrorAndExit.bind(log, 'amqp.error'));
-            this.amqp.on('close', log.criticalErrorAndExit.bind(log, 'amqp.close'));
-        }
-        log.debug('Connected to AMQP');
-
-        this.subscribeChannel = await this.amqp.createChannel();
-        this.subscribeChannel.on('error', log.criticalErrorAndExit.bind(log, 'subscribeChannel.error'));
-        log.debug('Opened subscribe channel');
-
-        this.publishChannel = await this.amqp.createConfirmChannel();
-        this.publishChannel.on('error', log.criticalErrorAndExit.bind(log, 'publishChannel.error'));
-        log.debug('Opened publish channel');
+    async connect() {
+        await Promise.all([this._ensurePublishChannel(), this._ensureConsumerChannel()]);
+        this.closed = false;
     }
 
     async disconnect() {
+        await this.stopConsume();
+        this.closed = true;
         log.trace('Close AMQP connections');
-        this.amqp.removeAllListeners('close');
+        if (this._readConnection) {
+            this._readConnection.removeAllListeners('close');
+        }
+        if (this._writeConnection) {
+            this._writeConnection.removeAllListeners('close');
+        }
+        if (this.consumerChannel) {
+            this.consumerChannel.removeAllListeners('close');
+        }
+        if (this.publishChannel) {
+            this.publishChannel.removeAllListeners('close');
+        }
         try {
-            await this.listenQueueCancel();
-            await this.subscribeChannel.close();
+            await this.consumerChannel.close();
         } catch (alreadyClosed) {
             log.debug('Subscribe channel is closed already');
         }
@@ -47,41 +50,227 @@ class Amqp {
             log.debug('Publish channel is closed already');
         }
         try {
-            await this.amqp.close();
+            await this._readConnection.close();
         } catch (alreadyClosed) {
-            log.debug('AMQP connection is closed already');
+            log.debug('AMQP read connection is closed already');
+        }
+        try {
+            await this._writeConnection.close();
+        } catch (alreadyClosed) {
+            log.debug('AMQP write connection is closed already');
         }
         log.debug('Successfully closed AMQP connections');
     }
 
-    async listenQueue(queueName, callback) {
-        if (this.consumerTag) {
-            throw new Error('listenQueue MUST NOT be called more than once');
+    async _ensureReadConnection() {
+        if (this._readConnection) {
+            return this._readConnection;
         }
-        this.subscribeChannel.prefetch(this.settings.RABBITMQ_PREFETCH_SAILOR);
+        if (this._creatingReadConnection) {
+            do {
+                await new Promise(resolve => setImmediate(resolve));
+            } while (!this._readConnection);
+            return this._readConnection;
+        }
+        log.debug('Creating new read connection');
+        this._creatingReadConnection = true;
+        this._readConnection = await this._createConnection('read');
+        this._creatingReadConnection = false;
+        log.debug('Read connection created');
+        this._readConnection.on('error', err => log.error({ err }, 'AMQP read Connection error'));
+        this._readConnection.once('close', err => {
+            log.error({ err }, 'Unexpected connection close.');
+            delete this._readConnection;
+        });
+        return this._readConnection;
+    }
 
-        const result = await this.subscribeChannel.consume(queueName, this.decryptMessage.bind(this, callback));
-        this.consumerTag = result.consumerTag;
-        return result;
+    async _ensureWriteConnection() {
+        if (this._writeConnection) {
+            return this._writeConnection;
+        }
+        if (this._creatingWriteConnection) {
+            do {
+                await new Promise(resolve => setImmediate(resolve));
+            } while (!this._writeConnection);
+            return this._writeConnection;
+        }
+        log.debug('Creating new write connection');
+        this._creatingWriteConnection = true;
+        this._writeConnection = await this._createConnection('write');
+        this._creatingWriteConnection = false;
+        log.debug('Write connection created');
+        this._writeConnection.on('error', err => log.error({ err }, 'AMQP write Connection error'));
+        this._writeConnection.once('close', err => {
+            log.error({ err }, 'Unexpected connection close.');
+            delete this._writeConnection;
+        });
+        return this._writeConnection;
     }
 
-    async listenQueueCancel() {
-        log.trace('listenQueueCancel starting');
-        if (!this.consumerTag) {
-            log.debug('listenQueueCancel is not executed since this.consumerTag is falsy');
-            return;
+    async _ensureConsumerChannel() {
+        if (this.consumerChannel) {
+            return this.consumerChannel;
+        }
+        if (this._creatingConsumerChannel) {
+            do {
+                await new Promise(resolve => setImmediate(resolve));
+            } while (!this.consumerChannel);
+            return this.consumerChannel;
+        }
+        log.debug({ prefetch: this.settings.RABBITMQ_PREFETCH_SAILOR }, 'Creating new consume channel');
+        this._creatingConsumerChannel = true;
+        const amqp = await this._ensureReadConnection();
+        this.consumerChannel = await amqp.createChannel();
+        log.debug('Consume channel created');
+        this._creatingConsumerChannel = false;
+        this.consumerChannel.prefetch(this.settings.RABBITMQ_PREFETCH_SAILOR);
+        this.consumerChannel.on('error', err => log.error({ err }, 'Consumer channel error'));
+        this.consumerChannel.once('close', () => {
+            delete this.consumerChannel;
+            if (this.consume) {
+                log.warn('Channel unexpectedly closed, but we were listening. Reconnecting and re-listening queue');
+                this.consume.consumerTag = undefined;
+                // when RabbitMQ closes connection, amqplib will first emit 'close' for channel and then for connection
+                // we use setImmediate to wait for connection 'close' event which will unset connection property
+                // otherwise listenQueue will try to create channel on closing connection and fail hard
+                setImmediate(() => this.listenQueue(this.consume.queue, this.consume.messageHandler));
+            }
+        });
+        return this.consumerChannel;
+    }
+
+    async _ensurePublishChannel() {
+        if (this.publishChannel) {
+            return this.publishChannel;
         }
-        if (!this.subscribeChannel) {
-            log.warn('listenQueueCancel called when this.subscribeChannel is falsy');
+        if (this._creatingPublishChannel) {
+            do {
+                await new Promise(resolve => setImmediate(resolve));
+            } while (!this.publishChannel);
+            return this.publishChannel;
+        }
+        log.debug('Creating new publish connection and channel');
+        this._creatingPublishChannel = true;
+        const amqp = await this._ensureWriteConnection();
+        this.publishChannel = await amqp.createConfirmChannel();
+        log.debug('Publish connection and channel created');
+        this._creatingPublishChannel = false;
+        this.publishChannel.on('error', err => log.error({ err }, 'Publish channel error'));
+        this.publishChannel.once('close', () => {
+            delete this.publishChannel;
+        });
+        return this.publishChannel;
+    }
+
+    async _createConnection(name) {
+        const uri = this.settings.AMQP_URI;
+        const allowedAttempts = parseInt(this.settings.AMQP_RECONNECT_ATTEMPTS);
+        let lastErr;
+        let attempts = 0;
+        while (attempts <= allowedAttempts) {
+            if (attempts > 0) {
+                await new Promise(resolve => setTimeout(resolve, parseInt(this.settings.AMQP_RECONNECT_TIMEOUT)));
+                log.debug(
+                    {
+                        reconnectAttempt: attempts,
+                        AMQP_RECONNECT_ATTEMPTS: this.settings.AMQP_RECONNECT_ATTEMPTS
+                    },
+                    'AMQP Reconnecting'
+                );
+            }
+            try {
+                return await amqplib.connect(uri,
+                    { clientProperties: { connection_name: `${os.hostname()}-${name}` } });
+            } catch (err) {
+                lastErr = err;
+                log.error(err, 'AMQP Connection error');
+                attempts++;
+            }
+        }
+        throw lastErr;
+    }
+
+    async stopConsume() {
+        if (!this.consume) {
             return;
         }
-        await this.subscribeChannel.cancel(this.consumerTag);
-        this.consumerTag = null;
-        log.debug('listenQueueCancel success');
+        const consume = this.consume;
+        this.consume = undefined;
+        await this.consumerChannel.cancel(consume.consumerTag);
+        log.debug({ queue: consume.queue }, 'Stopped listening for messages');
+    }
+
+    async listenQueue(queue, messageHandler) {
+        await this._ensureConsumerChannel();
+
+        const { consumerTag } = await this.consumerChannel.consume(queue, async (amqpMessage) => {
+            if (!amqpMessage) {
+                log.warn('Consumer cancelled by rabbitmq');
+                return;
+            }
+            let message;
+            try {
+                message = this._decodeMessage(amqpMessage);
+            } catch (err) {
+                log.error({ err, deliveryTag: amqpMessage.fields.deliveryTag },
+                    'Error occurred while parsing message payload');
+                this.reject(amqpMessage);
+                return;
+            }
+            try {
+                await messageHandler(message, amqpMessage);
+            } catch (err) {
+                log.error({ err, deliveryTag: amqpMessage.fields.deliveryTag }, 'Failed to process message, reject');
+                this.reject(amqpMessage);
+            }
+        });
+        log.debug({ queue }, 'Started listening for messages');
+        this.consume = { queue, messageHandler, consumerTag };
+    }
+
+    _decodeMessage(amqpMessage) {
+        log.trace('Message received');
+        let message;
+        if (this.settings.INPUT_FORMAT === 'error') {
+            message = this._decodeErrorMessage(amqpMessage);
+        } else {
+            message = this._decodeDefaultMessage(amqpMessage);
+        }
+        message.headers = message.headers || {};
+        if (amqpMessage.properties.headers.reply_to) {
+            message.headers.reply_to = amqpMessage.properties.headers.reply_to;
+        }
+        return message;
+    }
+
+    _decodeDefaultMessage(amqpMessage) {
+        const protocolVersion = Number(amqpMessage.properties.headers.protocolVersion || 1);
+        return this._encryptor.decryptMessageContent(
+            amqpMessage.content,
+            protocolVersion < 2 ? 'base64' : undefined
+        );
+    }
+
+    _decodeErrorMessage(amqpMessage) {
+        const errorBody = JSON.parse(amqpMessage.content.toString());
+        // NOTICE both error and errorInput are transferred as base64 encoded.
+        // this does not depend on protocolVersion header of message (see _decodeDefault message)
+        // this should be fixed in future, but it's OK at this moment
+        if (errorBody.error) {
+            errorBody.error = this._encryptor.decryptMessageContent(Buffer.from(errorBody.error), 'base64');
+        }
+        if (errorBody.errorInput) {
+            errorBody.errorInput = this._encryptor.decryptMessageContent(errorBody.errorInput, 'base64');
+        }
+        return {
+            body: errorBody,
+            headers: amqpMessage.properties.headers
+        };
     }
 
     decryptMessage(callback, message) {
-        log.trace('Message received: %j', message);
+        log.trace('Message received');
 
         if (message === null) {
             log.warn('NULL message received');
@@ -91,7 +280,7 @@ class Amqp {
         const protocolVersion = Number(message.properties.headers.protocolVersion || 1);
         let decryptedContent;
         try {
-            decryptedContent = encryptor.decryptMessageContent(
+            decryptedContent = this._encryptor.decryptMessageContent(
                 message.content,
                 protocolVersion < 2 ? 'base64' : undefined
             );
@@ -118,16 +307,17 @@ class Amqp {
 
     ack(message) {
         log.debug('Message #%j ack', message.fields.deliveryTag);
-        this.subscribeChannel.ack(message);
+        this.consumerChannel.ack(message);
     }
 
     reject(message) {
         log.debug('Message #%j reject', message.fields.deliveryTag);
-        return this.subscribeChannel.reject(message, false);
+        return this.consumerChannel.reject(message, false);
     }
 
     async sendToExchange(exchangeName, routingKey, payload, options, throttle) {
         if (throttle) {
+            log.debug('Throttling outgoing message');
             await throttle();
         }
         const buffer = Buffer.from(payload);
@@ -140,6 +330,8 @@ class Amqp {
         if (iteration) {
             options.headers.retry = iteration;
         }
+        // AMQP_PERSISTENT_MESSAGES is false by default if not specified by env var
+        options.persistent = this.settings.AMQP_PERSISTENT_MESSAGES;
 
         log.debug('Current memory usage: %s Mb', process.memoryUsage().heapUsed / 1048576);
         log.trace('Pushing to exchange=%s, routingKey=%s, messageSize=%d, options=%j, iteration=%d',
@@ -185,6 +377,7 @@ class Amqp {
     }
 
     async _promisifiedPublish(exchangeName, routingKey, payloadBuffer, options) {
+        await this._ensurePublishChannel();
         try {
             let result;
             const publishChannel = this.publishChannel;
@@ -211,7 +404,7 @@ class Amqp {
     }
 
     encryptMessageContent(body, protocolVersion = 1) {
-        return encryptor.encryptMessageContent(
+        return this._encryptor.encryptMessageContent(
             body,
             protocolVersion < 2
                 ? 'base64'
@@ -241,7 +434,6 @@ class Amqp {
         const settings = this.settings;
         const routingKey = getRoutingKeyFromHeaders(data.headers) || settings.DATA_ROUTING_KEY;
         properties.headers.protocolVersion = settings.PROTOCOL_VERSION;
-
         return this.prepareMessageAndSendToExchange(data, properties, routingKey, throttle);
     }
 
@@ -257,10 +449,13 @@ class Amqp {
     }
 
     async sendError(err, headers, originalMessage, throttle) {
+        // NOTICE both error and errorInput are transferred as base64 encoded.
+        // this does not depend on protocolVersion header of message (see _decodeDefaultMessage or sendData methods)
+        // this should be fixed in future, but it's OK at this moment
         const properties = this._createPropsFromHeaders(headers);
         const settings = this.settings;
 
-        const encryptedError = encryptor.encryptMessageContent({
+        const encryptedError = this._encryptor.encryptMessageContent({
             name: err.name,
             message: err.message,
             stack: err.stack
@@ -270,12 +465,10 @@ class Amqp {
             error: encryptedError
         };
         if (originalMessage && originalMessage.content) {
-            // TODO inconsistency with sendData. sendData build protocol version from settings
-            // but send Error takes it from headers.
             const protocolVersion = Number(originalMessage.properties.headers.protocolVersion || 1);
             if (protocolVersion >= 2) {
-                payload.errorInput = encryptor.encryptMessageContent(
-                    encryptor.decryptMessageContent(originalMessage.content),
+                payload.errorInput = this._encryptor.encryptMessageContent(
+                    this._encryptor.decryptMessageContent(originalMessage.content),
                     'base64'
                 ).toString();
             } else {
@@ -284,10 +477,14 @@ class Amqp {
         }
         const errorPayload = JSON.stringify(payload);
 
-        let result = this.sendToExchange(settings.PUBLISH_MESSAGES_TO, settings.ERROR_ROUTING_KEY,
-            errorPayload, properties, throttle);
+        let result = this.sendToExchange(
+            settings.PUBLISH_MESSAGES_TO,
+            settings.ERROR_ROUTING_KEY,
+            errorPayload, properties,
+            throttle
+        );
 
-        if (headers.reply_to) {
+        if (!settings.NO_ERROR_REPLIES && headers.reply_to) {
             log.debug('Sending error to %s', headers.reply_to);
             const replyToOptions = _.cloneDeep(properties);
             replyToOptions.headers[HEADER_ERROR_RESPONSE] = true;
@@ -298,20 +495,20 @@ class Amqp {
         return result;
     }
 
-    async sendRebound(reboundError, originalMessage, headers) {
-        // TODO: inconsistency
-        // rebound message should be
-        // a) repacked to currently used protocol version
-        // b) passed as is
-        // not as it's done now: send rebound but take protocolVersion header from current context
-        // TODO double think about: why we need headers as argument here?
-        // seems that rebound should be published with same headers as received
-        // seems that answer is error, when rebound limit is exceeded.
-        const properties = this._createPropsFromHeaders(headers);
-        const settings = this.settings;
+    async sendRebound(reboundError, originalMessage) {
+        const { settings } = this;
+        let { properties: { headers } } = originalMessage;
+        headers = {
+            ...headers,
+            end: new Date().getTime(),
+            reboundReason: reboundError.message
+        };
+        log.trace('Rebound message');
+        let reboundIteration = 1;
 
-        log.trace('Rebound message: %j', originalMessage);
-        const reboundIteration = getReboundIteration(originalMessage.properties.headers.reboundIteration);
+        if (headers.reboundIteration && typeof headers.reboundIteration === 'number') {
+            reboundIteration = headers.reboundIteration + 1;
+        }
 
         if (reboundIteration > settings.REBOUND_LIMIT) {
             return this.sendError(
@@ -320,8 +517,15 @@ class Amqp {
                 originalMessage
             );
         } else {
-            properties.expiration = getExpiration(reboundIteration);
-            properties.headers.reboundIteration = reboundIteration;
+            const properties = {
+                ...originalMessage.properties,
+                // retry in 15 sec, 30 sec, 1 min, 2 min, 4 min, 8 min, etc.
+                expiration: Math.pow(2, reboundIteration - 1) * settings.REBOUND_INITIAL_EXPIRATION,
+                headers: {
+                    ...headers,
+                    reboundIteration
+                }
+            };
 
             return this.sendToExchange(
                 settings.PUBLISH_MESSAGES_TO,
@@ -330,18 +534,6 @@ class Amqp {
                 properties
             );
         }
-
-        function getReboundIteration(previousIteration) {
-            if (previousIteration && typeof previousIteration === 'number') {
-                return previousIteration + 1;
-            }
-            return 1;
-        }
-
-        // retry in 15 sec, 30 sec, 1 min, 2 min, 4 min, 8 min, etc.
-        function getExpiration(iteration) {
-            return Math.pow(2, iteration - 1) * settings.REBOUND_INITIAL_EXPIRATION;
-        }
     }
 
     async sendSnapshot(data, headers, throttle) {