elasticio-sailor-nodejs 2.6.24 → 2.6.26-dev1

package/CHANGELOG.md

@@ -1,3 +1,12 @@
+## 2.6.26 (June 24, 2021)
+
+* Replaced exit on "unhandledRejection" with error logging
+
+## 2.6.25 (June 17, 2021)
+
+* Improve consume performance by switching back to pushing instead of polling, keeping reconnections mechanism
+  ([#5432](https://github.com/elasticio/elasticio/issues/5432))
+
 ## 2.6.24 (January 27, 2021)
 
 * Fix "Big messages processing is not stable" errors ([#5051](https://github.com/elasticio/elasticio/issues/5051))
@@ -23,7 +32,7 @@
 * Separate connections for consuming and publishing
 * Consuming is done with polling instead of pushing
 * Reconnects on connection errors
-* Hadling consumer cancel notification
+* Handling consumer cancel notification
 * Lowered log levels of some developers' log messages
 * Addded env vars:
     * AMQP_RECONNECT_ATTEMPTS - number of retries on connection close

package/lib/amqp.js

@@ -5,18 +5,17 @@ const Encryptor = require('./encryptor.js');
 const _ = require('lodash');
 const eventToPromise = require('event-to-promise');
 const uuid = require('uuid');
+const os = require('os');
 
 const HEADER_ROUTING_KEY = 'x-eio-routing-key';
 const HEADER_ERROR_RESPONSE = 'x-eio-error-response';
 
-
 class Amqp {
     constructor(settings) {
-        this.reconectCount = 0;
         this.settings = settings;
         this._encryptor = new Encryptor(this.settings.MESSAGE_CRYPTO_PASSWORD, this.settings.MESSAGE_CRYPTO_IV);
         this.closed = true;
-        this.consume = false;
+        this.consume = undefined;
     }
 
     async connect() {
@@ -25,7 +24,7 @@ class Amqp {
     }
 
     async disconnect() {
-        this.stopConsume();
+        await this.stopConsume();
         this.closed = true;
         log.trace('Close AMQP connections');
         if (this._readConnection) {
@@ -53,7 +52,7 @@ class Amqp {
         try {
             await this._readConnection.close();
         } catch (alreadyClosed) {
-            log.debug('AMQP raed connection is closed already');
+            log.debug('AMQP read connection is closed already');
         }
         try {
             await this._writeConnection.close();
@@ -73,12 +72,14 @@ class Amqp {
             } while (!this._readConnection);
             return this._readConnection;
         }
+        log.debug('Creating new read connection');
         this._creatingReadConnection = true;
-        this._readConnection = await this._createConnection();
+        this._readConnection = await this._createConnection('read');
         this._creatingReadConnection = false;
+        log.debug('Read connection created');
         this._readConnection.on('error', err => log.error({ err }, 'AMQP read Connection error'));
         this._readConnection.once('close', err => {
-            log.error({ err }, 'Unexepected connection close.');
+            log.error({ err }, 'Unexpected connection close.');
             delete this._readConnection;
         });
         return this._readConnection;
@@ -94,12 +95,14 @@ class Amqp {
             } while (!this._writeConnection);
             return this._writeConnection;
         }
+        log.debug('Creating new write connection');
         this._creatingWriteConnection = true;
-        this._writeConnection = await this._createConnection();
+        this._writeConnection = await this._createConnection('write');
         this._creatingWriteConnection = false;
+        log.debug('Write connection created');
         this._writeConnection.on('error', err => log.error({ err }, 'AMQP write Connection error'));
         this._writeConnection.once('close', err => {
-            log.error({ err }, 'Unexepected connection close.');
+            log.error({ err }, 'Unexpected connection close.');
             delete this._writeConnection;
         });
         return this._writeConnection;
@@ -115,14 +118,24 @@ class Amqp {
             } while (!this.consumerChannel);
             return this.consumerChannel;
         }
+        log.debug({ prefetch: this.settings.RABBITMQ_PREFETCH_SAILOR }, 'Creating new consume channel');
         this._creatingConsumerChannel = true;
         const amqp = await this._ensureReadConnection();
         this.consumerChannel = await amqp.createChannel();
+        log.debug('Consume channel created');
         this._creatingConsumerChannel = false;
+        this.consumerChannel.prefetch(this.settings.RABBITMQ_PREFETCH_SAILOR);
         this.consumerChannel.on('error', err => log.error({ err }, 'Consumer channel error'));
         this.consumerChannel.once('close', () => {
-            this.consumerChannel.closed = true;
             delete this.consumerChannel;
+            if (this.consume) {
+                log.warn('Channel unexpectedly closed, but we were listening. Reconnecting and re-listening queue');
+                this.consume.consumerTag = undefined;
+                // when RabbitMQ closes connection, amqplib will first emit 'close' for channel and then for connection
+                // we use setImmediate to wait for connection 'close' event which will unset connection property
+                // otherwise listenQueue will try to create channel on closing connection and fail hard
+                setImmediate(() => this.listenQueue(this.consume.queue, this.consume.messageHandler));
+            }
         });
         return this.consumerChannel;
     }
@@ -137,11 +150,12 @@ class Amqp {
             } while (!this.publishChannel);
             return this.publishChannel;
         }
+        log.debug('Creating new publish connection and channel');
         this._creatingPublishChannel = true;
         const amqp = await this._ensureWriteConnection();
         this.publishChannel = await amqp.createConfirmChannel();
+        log.debug('Publish connection and channel created');
         this._creatingPublishChannel = false;
-        this.publishChannel.prefetch(this.settings.RABBITMQ_PREFETCH_SAILOR);
         this.publishChannel.on('error', err => log.error({ err }, 'Publish channel error'));
         this.publishChannel.once('close', () => {
             delete this.publishChannel;
@@ -149,104 +163,68 @@ class Amqp {
         return this.publishChannel;
     }
 
-    async *_qReader(queue) {
-        let channel;
-
-        while (!this.closed) {
-            channel = await this._ensureConsumerChannel();
-            let msg;
-
-            try {
-                msg = await channel.get(queue);
-            } catch (e) {
-                if (this.closed) {
-                    break;
-                }
-                if (channel.closed) {
-                    if (e.isOperational && e.fields && e.fields.replyCode === 404) { // deleted queue
-                        throw e;
-                    }
-                    delete this.consumerChannel;
-                } else {
-                    throw e;
-                }
-            }
-            if (msg) {
-                yield msg;
-            } else {
-                // need it here since if there's no message channel.get seems to be not true async
-                // and event loop is being blocked without this
-                // also to avoid CPU load if no queue is empty
-                await new Promise(resolve => setTimeout(resolve, parseInt(this.settings.WAIT_MESSAGES_TIMEOUT)));
-            }
-        }
-    }
-
-    async _createConnection() {
+    async _createConnection(name) {
         const uri = this.settings.AMQP_URI;
-        let connection;
-        const reconnectAttempts = parseInt(this.settings.AMQP_RECONNECT_ATTEMPTS);
-        for (let i = 0; i < reconnectAttempts; i++) {
+        const allowedAttempts = parseInt(this.settings.AMQP_RECONNECT_ATTEMPTS);
+        let lastErr;
+        let attempts = 0;
+        while (attempts <= allowedAttempts) {
+            if (attempts > 0) {
+                await new Promise(resolve => setTimeout(resolve, parseInt(this.settings.AMQP_RECONNECT_TIMEOUT)));
+                log.debug(
+                    {
+                        reconnectAttempt: attempts,
+                        AMQP_RECONNECT_ATTEMPTS: this.settings.AMQP_RECONNECT_ATTEMPTS
+                    },
+                    'AMQP Reconnecting'
+                );
+            }
             try {
-                connection = await amqplib.connect(uri);
-            } catch (e) {
-                log.error({ err: e }, 'AMQP Connection error');
-                await new Promise(resolve =>
-                    setTimeout(() => resolve(), parseInt(this.settings.AMQP_RECONNECT_TIMEOUT)));
-                if (this.reconnectCount >= reconnectAttempts) {
-                    log.debug(
-                        {
-                            reconnectCount: this.reconectCount,
-                            AMQP_RECONNECT_ATTEMPTS: this.settings.AMQP_RECONNECT_ATTEMPTS
-                        },
-                        'reconnecting'
-                    );
-                    continue;
-                }
-                throw e;
+                return await amqplib.connect(uri,
+                    { clientProperties: { connection_name: `${os.hostname()}-${name}` } });
+            } catch (err) {
+                lastErr = err;
+                log.error(err, 'AMQP Connection error');
+                attempts++;
             }
-            break;
         }
-        return connection;
+        throw lastErr;
     }
 
-    stopConsume() {
-        this.consume = false;
+    async stopConsume() {
+        if (!this.consume) {
+            return;
+        }
+        const consume = this.consume;
+        this.consume = undefined;
+        await this.consumerChannel.cancel(consume.consumerTag);
+        log.debug({ queue: consume.queue }, 'Stopped listening for messages');
     }
 
     async listenQueue(queue, messageHandler) {
-        const prefetch = parseInt(this.settings.RABBITMQ_PREFETCH_SAILOR);
-        this.consume = true;
+        await this._ensureConsumerChannel();
 
-        const readAndProcess = async () => {
-            for await (const amqpMessage of this._qReader(queue)) {
-                if (!this.consume) {
-                    break;
-                }
-                let message;
-                try {
-                    message = this._decodeMessage(amqpMessage);
-                } catch (e) {
-                    log.error(e,
-                        'Error occurred while parsing message #%j payload',
-                        amqpMessage.fields.deliveryTag
-                    );
-                    this.reject(amqpMessage);
-                    continue;
-                }
-                try {
-                    await messageHandler(message, amqpMessage);
-                } catch (e) {
-                    log.error(e, 'Failed to process message #%j, reject', amqpMessage.fields.deliveryTag);
-                    this.reject(amqpMessage);
-                }
+        const { consumerTag } = await this.consumerChannel.consume(queue, async (amqpMessage) => {
+            let message;
+            try {
+                message = this._decodeMessage(amqpMessage);
+            } catch (e) {
+                log.error(e,
+                    'Error occurred while parsing message #%j payload',
+                    amqpMessage.fields.deliveryTag
+                );
+                this.reject(amqpMessage);
+                return;
             }
-        };
-
-        // here we start readers count = configured prefetch
-        // if one of them fails then it's irrecoverable error (see _qReader)
-        // that's why Promise.all here - to fail if one is failed
-        await Promise.all((new Array(prefetch)).fill().map(() => readAndProcess()));
+            try {
+                await messageHandler(message, amqpMessage);
+            } catch (e) {
+                log.error(e, 'Failed to process message #%j, reject', amqpMessage.fields.deliveryTag);
+                this.reject(amqpMessage);
+            }
+        });
+        log.debug({ queue }, 'Started listening for messages');
+        this.consume = { queue, messageHandler, consumerTag };
     }
 
     _decodeMessage(amqpMessage) {
@@ -337,7 +315,7 @@ class Amqp {
 
     async sendToExchange(exchangeName, routingKey, payload, options, throttle) {
         if (throttle) {
-            log.debug('Throttlig outgoing message');
+            log.debug('Throttling outgoing message');
             await throttle();
         }
         const buffer = Buffer.from(payload);

package/lib/ipc.js

@@ -0,0 +1,13 @@
+/**
+ * Used to communicate with parent processes.
+ */
+class IPC {
+    send(event, data) {
+        if (!process.send) {
+            return;
+        }
+        process.send({ event, data });
+    }
+}
+
+exports.IPC = IPC;

package/lib/sailor.js

@@ -209,13 +209,13 @@ class Sailor {
         }
     }
 
-    scheduleShutdown() {
+    async scheduleShutdown() {
         if (this.shutdownCallback) {
             log.debug('scheduleShutdown – shutdown is already scheduled, do nothing');
             return new Promise(resolve => this.shutdownCallback = resolve);
         }
 
-        this.amqpConnection.stopConsume();
+        await this.amqpConnection.stopConsume();
         if (this.messagesCount === 0) {
             // there is no unfinished processMessage invocation, let's just resolve scheduleShutdown now
             log.debug('scheduleShutdown – about to shutdown immediately');

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "elasticio-sailor-nodejs",
   "description": "The official elastic.io library for bootstrapping and executing for Node.js connectors",
-  "version": "2.6.24",
+  "version": "2.6.26-dev1",
   "main": "run.js",
   "scripts": {
     "lint": "./node_modules/.bin/eslint lib spec mocha_spec lib run.js runService.js",
@@ -15,21 +15,22 @@
     "node": ">=12.13.0"
   },
   "dependencies": {
-    "@elastic.io/object-storage-client": "1.0.0",
-    "amqplib": "0.6.0",
+    "@elastic.io/object-storage-client": "1.0.1",
+    "amqplib": "0.8.0",
     "bunyan": "1.8.10",
     "co": "4.6.0",
     "debug": "3.1.0",
-    "elasticio-rest-node": "1.2.5",
+    "elasticio-rest-node": "1.2.7",
     "event-to-promise": "0.8.0",
-    "lodash": "4.17.15",
+    "lodash": "4.17.21",
     "p-throttle": "2.1.0",
-    "q": "1.4.1",
+    "q": "1.5.1",
     "request-promise-native": "1.0.5",
-    "requestretry": "3.1.0",
+    "requestretry": "7.0.2",
     "uuid": "3.0.1"
   },
   "devDependencies": {
+    "rabbitmq-stats": "1.2.3",
     "chai": "4.2.0",
     "del": "2.2.2",
     "eslint": "4.19.1",

package/run.js

@@ -1,10 +1,15 @@
+/**
+ * Entrypoint for starting task step.
+ */
 const logger = require('./lib/logging.js');
 const Sailor = require('./lib/sailor.js').Sailor;
 const settings = require('./lib/settings.js');
-const co = require('co');
+const { IPC } = require('./lib/ipc.js');
+const Q = require('q');
 const http = require('http');
 
 let sailor;
+let sailorInit;
 let disconnectRequired;
 
 // miserable try to workaround issue described in https://github.com/elasticio/elasticio/issues/4874
@@ -13,7 +18,10 @@ http.globalAgent = new Agent({
     keepAlive: true
 });
 
-async function putOutToSea(settings) {
+async function putOutToSea(settings, ipc) {
+    ipc.send('init:started');
+    const deferred = Q.defer();
+    sailorInit = deferred.promise;
     sailor = new Sailor(settings);
 
     //eslint-disable-next-line no-extra-boolean-cast
@@ -38,36 +46,28 @@ async function putOutToSea(settings) {
 
     await sailor.runHookInit();
     await sailor.run();
+    deferred.resolve();
+    ipc.send('init:ended');
 }
 
-function disconnectAndExit() {
+async function disconnectAndExit() {
     if (!disconnectRequired) {
         return;
     }
     disconnectRequired = false;
-    co(function* putIn() {
+
+    try {
         logger.info('Disconnecting...');
-        yield sailor.disconnect();
+        await sailor.disconnect();
         logger.info('Successfully disconnected');
         process.exit();
-    }).catch((err) => {
-        logger.error('Unable to disconnect', err.stack);
+    } catch (err) {
+        logger.error(err, 'Unable to disconnect');
         process.exit(-1);
-    });
-}
-
-function _disconnectOnly() {
-    if (!disconnectRequired) {
-        return Promise.resolve();
     }
-    return sailor.disconnect();
 }
 
-function _closeConsumerChannel() {
-    return sailor.amqpConnection.consumerChannel.close();
-}
-
-function gracefulShutdown() {
+async function gracefulShutdown() {
     if (!disconnectRequired) {
         return;
     }
@@ -77,12 +77,20 @@ function gracefulShutdown() {
         return;
     }
 
-    sailor.scheduleShutdown().then(disconnectAndExit);
+    // we connect to amqp, create channels, start listen a queue on init and interrupting this process with 'disconnect'
+    // will lead to undefined behaviour
+    logger.trace('Checking/waiting for init before graceful shutdown');
+    await sailorInit;
+    logger.trace('Waited an init before graceful shutdown');
+
+    await sailor.scheduleShutdown();
+    await disconnectAndExit();
 }
 
-async function run(settings) {
+async function run(settings, ipc) {
     try {
-        await putOutToSea(settings);
+        await putOutToSea(settings, ipc);
+        logger.info('Fully initialized and waiting for messages');
     } catch (e) {
         if (sailor && !sailor.amqpConnection.closed) {
             await sailor.reportError(e);
@@ -91,8 +99,17 @@ async function run(settings) {
     }
 }
 
-exports._disconnectOnly = _disconnectOnly;
-exports._closeConsumerChannel = _closeConsumerChannel;
+exports.__test__ = {
+    disconnectOnly: function disconnectOnly() {
+        if (!disconnectRequired) {
+            return Promise.resolve();
+        }
+        return sailor.disconnect();
+    },
+    closeConsumerChannel: function closeConsumerChannel() {
+        return sailor.amqpConnection.consumerChannel.close();
+    }
+};
 exports.run = run;
 exports.putOutToSea = putOutToSea;
 
@@ -108,6 +125,9 @@ if (require.main === module || process.mainModule.filename === __filename) {
     });
 
     process.on('uncaughtException', logger.criticalErrorAndExit.bind(logger, 'process.uncaughtException'));
+    process.on('unhandledRejection', (err) => logger.error(err, 'process.unhandledRejection'));
+
+    const ipc = new IPC();
 
-    run(settings.readFrom(process.env));
+    run(settings.readFrom(process.env), ipc);
 }

package/runService.js

@@ -1,3 +1,6 @@
+/**
+ * Entrypoint for starting service which will handle verifyCredentials and selectModel requests.
+ */
 const logger = require('./lib/logging');
 const service = require('./lib/service');
 const debug = require('debug')('sailor');
@@ -12,4 +15,5 @@ service.processService(serviceMethod, process.env)
         process.exit(0);
     });
 
-process.on('uncaughtException', logger.criticalErrorAndExit.bind(logger, 'runService.uncaughtException'));
+process.on('uncaughtException', logger.criticalErrorAndExit.bind(logger, 'process.uncaughtException'));
+process.on('unhandledRejection', (err) => logger.error(err, 'process.unhandledRejection'));

package/.travis.yml

@@ -1,8 +0,0 @@
-language: node_js
-node_js:
-  - "8"
-  - "9"
-  - "10"
-  - "11"
-script:
-  - npm run test:jasmine

package/Procfile

@@ -1,2 +0,0 @@
-sail: node run.js
-createQueues: node createQueues.js

package/createQueues.js

@@ -1,148 +0,0 @@
-var amqp = require('./lib/amqp.js');
-var settings = require('./lib/settings.js').readFrom(process.env);
-var Q = require('q');
-var util = require('util');
-
-var execId = "1432205514864";
-
-var task = {
-    "id" : "5559edd38968ec0736000003",
-    "user" : "5527f0ea43238e5d5f000001",
-    "data" : {
-        "step_2" : {
-            "_account" : "554b53aed5178d6540000001"
-        },
-        "step_3" : {
-            "mapper" : {
-                "data" : {
-                    "qty" : "2"
-                },
-                "product" : "Btestsku"
-            }
-        },
-        "step_1" : {
-            "interval" : "minute",
-            "_account" : "5559ed6b8968ec0736000002"
-        }
-    },
-    "recipe" : {
-        "nodes" : [
-            {
-                "first" : true,
-                "id" : "step_1",
-                "function" : "getProducts",
-                "compId" : "shopware"
-            },
-            {
-                "id" : "step_3",
-                "function" : "map",
-                "compId" : "mapper"
-            },
-            {
-                "id" : "step_2",
-                "function" : "updateInventory",
-                "compId" : "magento"
-            }
-        ],
-        "connections" : [
-            {
-                "to" : "step_3",
-                "from" : "step_1"
-            },
-            {
-                "to" : "step_2",
-                "from" : "step_3"
-            }
-        ]
-    }
-};
-
-
-var amqpConnection = new amqp.AMQPConnection(settings);
-amqpConnection.connect(settings.AMQP_URI).then(function(){
-    сreateQueuesAndExchanges(execId, task, "step_1", "step_2");
-});
-
-function сreateQueuesAndExchanges(execId, task, stepId, nextStepId){
-
-    var EXCHANGE_NAME = 'userexchange:' + task.user;
-
-    var MESSAGE_TAG = util.format('%s:%s:%s:message', task.id, stepId, execId);
-    var ERROR_TAG = util.format('%s:%s:%s:error', task.id, stepId, execId);
-    var REBOUND_TAG = util.format('%s:%s:%s:rebound', task.id, stepId, execId);
-
-    var MESSAGES_QUEUE = util.format('%s:%s:%s:messages', task.id, stepId, execId);
-    var MESSAGES_LISTENING_QUEUE = util.format('%s:%s:%s:messages', task.id, nextStepId, execId);
-    var REBOUNDS_QUEUE = util.format('%s:%s:%s:rebounds', task.id, stepId, execId);
-
-    console.log('INCOMING_MESSAGES_QUEUE=%s', MESSAGES_QUEUE);
-    console.log('EXCHANGE_NAME=%s', EXCHANGE_NAME);
-    console.log('MESSAGE_TAG=%s', MESSAGE_TAG);
-    console.log('ERROR_TAG=%s', ERROR_TAG);
-    console.log('REBOUND_TAG=%s', REBOUND_TAG);
-    console.log('MESSAGES_QUEUE=%s', MESSAGES_QUEUE);
-    console.log('REBOUNDS_QUEUE=%s', REBOUNDS_QUEUE);
-
-    var userExchange = {
-        name: EXCHANGE_NAME,
-        type: 'direct',
-        options: {
-            durable: true,
-            autoDelete: false
-        }
-    };
-
-    var messagesQueue = {
-        name: MESSAGES_QUEUE,
-        options: {
-            durable: true,
-            autoDelete: false
-        }
-    };
-
-    var messagesListeningQueue = {
-        name: MESSAGES_LISTENING_QUEUE,
-        options: {
-            durable: true,
-            autoDelete: false
-        }
-    };
-
-    var REBOUND_QUEUE_TTL = 10 * 60 * 1000; // 10 min
-
-    var reboundsQueue = {
-        name: REBOUNDS_QUEUE,
-        options: {
-            durable: true,
-            autoDelete: false,
-            arguments: {
-                'x-message-ttl': REBOUND_QUEUE_TTL,
-                'x-dead-letter-exchange': EXCHANGE_NAME, // send dead rebounded queues back to exchange
-                'x-dead-letter-routing-key': MESSAGE_TAG // with tag as message
-            }
-        }
-    };
-
-    return Q.all([
-        assertExchange(amqpConnection.publishChannel, userExchange), // check that exchange exists
-        assertQueue(amqpConnection.publishChannel, messagesQueue), // create messages queue
-        assertQueue(amqpConnection.publishChannel, messagesListeningQueue), // create messages queue
-        amqpConnection.publishChannel.bindQueue(messagesListeningQueue.name, userExchange.name, MESSAGE_TAG),
-        assertQueue(amqpConnection.publishChannel, reboundsQueue), // create rebounds queue
-        amqpConnection.publishChannel.bindQueue(reboundsQueue.name, userExchange.name, REBOUND_TAG)
-    ]).then(function(){
-        console.log('Successfully asserted all queues');
-    }).done();
-
-    function assertQueue(channel, queue) {
-        return channel.assertQueue(queue.name, queue.options).then(function assertQueueSuccess() {
-            console.log('Succesfully asserted queue: ' + queue.name);
-        });
-    }
-
-    function assertExchange(channel, exchange) {
-        return channel.assertExchange(exchange.name, exchange.type, exchange.options).then(function assertExchangeSuccess() {
-            console.log('Succesfully asserted exchange: ' + exchange.name);
-        });
-    }
-}

package/gulpfile.js

@@ -1,31 +0,0 @@
-var gulp = require('gulp');
-var jasmine = require('gulp-jasmine');
-var istanbul = require('gulp-istanbul');
-var del = require('del');
-
-var paths = {
-    code: ['./lib/*.js'],
-    spec: ['./spec/**/*.spec.js'],
-    coverageReport: './coverage/lcov.info',
-    coverage: './coverage'
-};
-
-gulp.task('clean:coverage', function (cb) {
-    del([paths.coverage], cb);
-});
-
-gulp.task('istanbul', function (cb) {
-    gulp
-        .src(paths.code)
-        .pipe(istanbul()) // Covering files
-        .pipe(istanbul.hookRequire()) // Force `require` to return covered files
-        .on('finish', function () {
-            gulp
-                .src(paths.spec)
-                .pipe(jasmine())
-                .pipe(istanbul.writeReports()) // Creating the reports after tests runned
-                .on('end', cb);
-        });
-});
-
-gulp.task('coverage', ['clean:coverage', 'istanbul']);