elasticio-sailor-nodejs 2.7.0-dev3 → 2.7.0

package/lib/settings.js

@@ -1,39 +1,8 @@
 const _ = require('lodash');
 
-
 const PREFIX = 'ELASTICIO_';
 
-function readFrom(envVars) {
-    const result = {};
-
-    // required settings
-    const requiredAlways = [
-        'FLOW_ID',
-        'EXEC_ID',
-        'STEP_ID',
-        'CONTAINER_ID',
-        'WORKSPACE_ID',
-
-        'USER_ID',
-        'COMP_ID',
-        'FUNCTION',
-
-        'API_URI',
-        'API_USERNAME',
-        'API_KEY'
-    ];
-
-    const requiredForMessageProcessing = [
-        'AMQP_URI',
-        'LISTEN_MESSAGES_ON',
-        'PUBLISH_MESSAGES_TO',
-
-        'DATA_ROUTING_KEY',
-        'ERROR_ROUTING_KEY',
-        'REBOUND_ROUTING_KEY',
-        'SNAPSHOT_ROUTING_KEY'
-    ];
-
+function getOptionalEnvVars(envVars) {
     const optional = {
         REBOUND_INITIAL_EXPIRATION: 15000,
         REBOUND_LIMIT: 20,
@@ -51,15 +20,40 @@ function readFrom(envVars) {
         AMQP_PUBLISH_RETRY_DELAY: 100, // 100ms
         AMQP_PUBLISH_RETRY_ATTEMPTS: Infinity,
         AMQP_PUBLISH_MAX_RETRY_DELAY: 5 * 60 * 1000, // 5 mins
+        AMQP_PERSISTENT_MESSAGES: false,
         OUTGOING_MESSAGE_SIZE_LIMIT: 10485760,
         NO_SELF_PASSTRHOUGH: false,
         PROTOCOL_VERSION: 1,
+        NO_ERROR_REPLIES: false,
+        INPUT_FORMAT: 'default',
         OBJECT_STORAGE_URI: null,
         OBJECT_STORAGE_TOKEN: null,
         OBJECT_STORAGE_SIZE_THRESHOLD: 1048576,
-        EMIT_LIGHTWEIGHT_MESSAGE: false
+        EMIT_LIGHTWEIGHT_MESSAGE: false,
+        AMQP_RECONNECT_ATTEMPTS: 3,
+        AMQP_RECONNECT_TIMEOUT: 100,
+        WAIT_MESSAGES_TIMEOUT: 50
     };
 
+    const result = {};
+    _.forEach(optional, function readOptional(defaultValue, key) {
+        const envVarName = PREFIX + key;
+        if (typeof defaultValue === 'number' && envVars[envVarName]) {
+            result[key] = parseInt(envVars[envVarName]) || defaultValue;
+        } else if (typeof defaultValue === 'boolean') {
+            if (envVars[envVarName] === undefined) {
+                result[key] = defaultValue;
+            } else {
+                result[key] = (!envVars[envVarName] || envVars[envVarName] === 'false') ? false : true;
+            }
+        } else {
+            result[key] = envVars[envVarName] || defaultValue;
+        }
+    });
+    return result;
+}
+
+function getAdditionalVars(envVars) {
     if (envVars.ELASTICIO_ADDITIONAL_VARS_FOR_HEADERS) {
         const vars = {};
         envVars.ELASTICIO_ADDITIONAL_VARS_FOR_HEADERS
@@ -69,39 +63,64 @@ function readFrom(envVars) {
                 const key = env.indexOf(PREFIX) === 0 ? env.slice(PREFIX.length) : env;
                 vars[key] = envVars[env];
             });
-        result.additionalVars = vars;
+
+        return vars;
     }
+}
+
+function getMandatoryEnvVars(envVars) {
+    // required settings
+    const requiredAlways = [
+        'FLOW_ID',
+        'EXEC_ID',
+        'STEP_ID',
+        'CONTAINER_ID',
+        'WORKSPACE_ID',
+
+        'USER_ID',
+        'COMP_ID',
+        'FUNCTION',
+
+        'API_URI',
+        'API_USERNAME',
+        'API_KEY'
+    ];
+
+    const requiredForMessageProcessing = [
+        'AMQP_URI',
+        'LISTEN_MESSAGES_ON',
+        'PUBLISH_MESSAGES_TO',
+
+        'DATA_ROUTING_KEY',
+        'ERROR_ROUTING_KEY',
+        'REBOUND_ROUTING_KEY',
+        'SNAPSHOT_ROUTING_KEY',
+        'MESSAGE_CRYPTO_IV',
+        'MESSAGE_CRYPTO_PASSWORD'
+    ];
+
     const envVarsList = requiredAlways.slice(0);
 
     if (!envVars.ELASTICIO_HOOK_SHUTDOWN) {
         envVarsList.push(...requiredForMessageProcessing);
     }
 
-    envVarsList.forEach(key => {
+    return envVarsList.reduce((result, key) => {
         const envVarName = PREFIX + key;
         if (!envVars[envVarName]) {
             throw new Error(`${envVarName} is missing`);
         }
         result[key] = envVars[envVarName];
-    });
-
-    _.forEach(optional, function readOptional(defaultValue, key) {
-        const envVarName = PREFIX + key;
-        const val = envVars[envVarName];
-        if (typeof defaultValue === 'number' && val) {
-            result[key] = parseInt(val) || defaultValue;
-        } else if (typeof defaultValue === 'boolean') {
-            if (typeof val === 'undefined') {
-                result[key] = defaultValue;
-            } else {
-                result[key] = val === 'false' ? false : Boolean(val);
-            }
-        } else {
-            result[key] = val || defaultValue;
-        }
-    });
+        return result;
+    }, {});
+}
 
-    return result;
+function readFrom(envVars) {
+    return {
+        ...getMandatoryEnvVars(envVars),
+        ...getOptionalEnvVars(envVars),
+        additionalVars: getAdditionalVars(envVars)
+    };
 }
 
 exports.readFrom = readFrom;

package/package.json

@@ -1,9 +1,10 @@
 {
   "name": "elasticio-sailor-nodejs",
   "description": "The official elastic.io library for bootstrapping and executing for Node.js connectors",
-  "version": "2.7.0-dev3",
+  "version": "2.7.0",
   "main": "run.js",
   "scripts": {
+    "audit": "better-npm-audit audit --level high --production",
     "lint": "./node_modules/.bin/eslint lib spec mocha_spec lib run.js runService.js",
     "pretest": "npm run lint",
     "test": "npm run test:jasmine && npm run test:mocha",
@@ -15,34 +16,36 @@
     "node": ">=12.13.0"
   },
   "dependencies": {
-    "@elastic.io/object-storage-client": "0.0.1-dev.12",
-    "amqplib": "0.5.1",
+    "@elastic.io/object-storage-client": "2.1.0",
+    "amqplib": "0.8.0",
     "bunyan": "1.8.10",
     "co": "4.6.0",
     "debug": "3.1.0",
-    "elasticio-rest-node": "1.2.5",
+    "elasticio-rest-node": "1.2.7",
     "event-to-promise": "0.8.0",
-    "lodash": "4.17.15",
+    "lodash": "4.17.21",
     "p-throttle": "2.1.0",
-    "q": "1.4.1",
+    "q": "1.5.1",
     "request-promise-native": "1.0.5",
-    "requestretry": "3.1.0",
+    "requestretry": "7.0.2",
     "uuid": "3.0.1"
   },
   "devDependencies": {
+    "better-npm-audit": "3.7.3",
     "chai": "4.2.0",
     "del": "2.2.2",
     "eslint": "4.19.1",
     "eslint-plugin-mocha": "4.12.1",
     "express": "4.16.4",
     "gulp": "3.9.1",
-    "gulp-istanbul": "1.1.1",
+    "gulp-istanbul": "1.1.3",
     "gulp-jasmine": "0.2.0",
     "jasmine-node": "3.0.0",
     "mocha": "7.1.2",
-    "nock": "^12.0.3",
+    "nock": "12.0.3",
+    "rabbitmq-stats": "1.2.4",
     "request": "2.88.0",
-    "sinon": "^9.0.2",
+    "sinon": "9.0.2",
     "sinon-chai": "3.5.0"
   },
   "repository": "elasticio/sailor-nodejs",

package/run.js

@@ -1,12 +1,32 @@
+/**
+ * Entrypoint for starting task step.
+ */
 const logger = require('./lib/logging.js');
 const Sailor = require('./lib/sailor.js').Sailor;
 const settings = require('./lib/settings.js');
-const co = require('co');
+const { IPC } = require('./lib/ipc.js');
+const Q = require('q');
+const http = require('http');
+const https = require('https');
 
 let sailor;
+let sailorInit;
 let disconnectRequired;
 
-async function putOutToSea(settings) {
+function prepareSandbox() {
+    // enable keep alive by default to handle issues like https://github.com/elasticio/elasticio/issues/4874
+    http.globalAgent = new http.Agent({
+        keepAlive: true
+    });
+    https.globalAgent = new https.Agent({
+        keepAlive: true
+    });
+}
+
+async function putOutToSea(settings, ipc) {
+    ipc.send('init:started');
+    const deferred = Q.defer();
+    sailorInit = deferred.promise;
     sailor = new Sailor(settings);
 
     //eslint-disable-next-line no-extra-boolean-cast
@@ -31,32 +51,28 @@ async function putOutToSea(settings) {
 
     await sailor.runHookInit();
     await sailor.run();
+    deferred.resolve();
+    ipc.send('init:ended');
 }
 
-function disconnectAndExit() {
+async function disconnectAndExit() {
     if (!disconnectRequired) {
         return;
     }
     disconnectRequired = false;
-    co(function* putIn() {
+
+    try {
         logger.info('Disconnecting...');
-        yield sailor.disconnect();
+        await sailor.disconnect();
         logger.info('Successfully disconnected');
         process.exit();
-    }).catch((err) => {
-        logger.error('Unable to disconnect', err.stack);
+    } catch (err) {
+        logger.error(err, 'Unable to disconnect');
         process.exit(-1);
-    });
-}
-
-function _disconnectOnly() {
-    if (!disconnectRequired) {
-        return Promise.resolve();
     }
-    return sailor.disconnect();
 }
 
-function gracefulShutdown() {
+async function gracefulShutdown() {
     if (!disconnectRequired) {
         return;
     }
@@ -66,22 +82,42 @@ function gracefulShutdown() {
         return;
     }
 
-    sailor.scheduleShutdown().then(disconnectAndExit);
+    // we connect to amqp, create channels, start listen a queue on init and interrupting this process with 'disconnect'
+    // will lead to undefined behaviour
+    logger.trace('Checking/waiting for init before graceful shutdown');
+    await sailorInit;
+    logger.trace('Waited an init before graceful shutdown');
+
+    await sailor.scheduleShutdown();
+    await disconnectAndExit();
 }
 
-async function run(settings) {
+async function run(settings, ipc) {
+    prepareSandbox();
     try {
-        await putOutToSea(settings);
+        await putOutToSea(settings, ipc);
+        logger.info('Fully initialized and waiting for messages');
     } catch (e) {
-        if (sailor) {
+        if (sailor && !sailor.amqpConnection.closed) {
             await sailor.reportError(e);
         }
         logger.criticalErrorAndExit('putOutToSea.catch', e);
     }
 }
 
-exports._disconnectOnly = _disconnectOnly;
+exports.__test__ = {
+    disconnectOnly: function disconnectOnly() {
+        if (!disconnectRequired) {
+            return Promise.resolve();
+        }
+        return sailor.disconnect();
+    },
+    closeConsumerChannel: function closeConsumerChannel() {
+        return sailor.amqpConnection.consumerChannel.close();
+    }
+};
 exports.run = run;
+exports.putOutToSea = putOutToSea;
 
 if (require.main === module || process.mainModule.filename === __filename) {
     process.on('SIGTERM', function onSigterm() {
@@ -95,6 +131,9 @@ if (require.main === module || process.mainModule.filename === __filename) {
     });
 
     process.on('uncaughtException', logger.criticalErrorAndExit.bind(logger, 'process.uncaughtException'));
+    process.on('unhandledRejection', (err) => logger.error(err, 'process.unhandledRejection'));
+
+    const ipc = new IPC();
 
-    run(settings.readFrom(process.env));
+    run(settings.readFrom(process.env), ipc);
 }

package/runService.js

@@ -1,3 +1,6 @@
+/**
+ * Entrypoint for starting service which will handle verifyCredentials and selectModel requests.
+ */
 const logger = require('./lib/logging');
 const service = require('./lib/service');
 const debug = require('debug')('sailor');
@@ -12,4 +15,5 @@ service.processService(serviceMethod, process.env)
         process.exit(0);
     });
 
-process.on('uncaughtException', logger.criticalErrorAndExit.bind(logger, 'runService.uncaughtException'));
+process.on('uncaughtException', logger.criticalErrorAndExit.bind(logger, 'process.uncaughtException'));
+process.on('unhandledRejection', (err) => logger.error(err, 'process.unhandledRejection'));

package/.travis.yml

@@ -1,8 +0,0 @@
-language: node_js
-node_js:
-  - "8"
-  - "9"
-  - "10"
-  - "11"
-script:
-  - npm run test:jasmine

package/Procfile

@@ -1,2 +0,0 @@
-sail: node run.js
-createQueues: node createQueues.js

package/createQueues.js

@@ -1,148 +0,0 @@
-var amqp = require('./lib/amqp.js');
-var settings = require('./lib/settings.js').readFrom(process.env);
-var Q = require('q');
-var util = require('util');
-
-var execId = "1432205514864";
-
-var task = {
-    "id" : "5559edd38968ec0736000003",
-    "user" : "5527f0ea43238e5d5f000001",
-    "data" : {
-        "step_2" : {
-            "_account" : "554b53aed5178d6540000001"
-        },
-        "step_3" : {
-            "mapper" : {
-                "data" : {
-                    "qty" : "2"
-                },
-                "product" : "Btestsku"
-            }
-        },
-        "step_1" : {
-            "interval" : "minute",
-            "_account" : "5559ed6b8968ec0736000002"
-        }
-    },
-    "recipe" : {
-        "nodes" : [
-            {
-                "first" : true,
-                "id" : "step_1",
-                "function" : "getProducts",
-                "compId" : "shopware"
-            },
-            {
-                "id" : "step_3",
-                "function" : "map",
-                "compId" : "mapper"
-            },
-            {
-                "id" : "step_2",
-                "function" : "updateInventory",
-                "compId" : "magento"
-            }
-        ],
-        "connections" : [
-            {
-                "to" : "step_3",
-                "from" : "step_1"
-            },
-            {
-                "to" : "step_2",
-                "from" : "step_3"
-            }
-        ]
-    }
-};
-
-
-var amqpConnection = new amqp.AMQPConnection(settings);
-amqpConnection.connect(settings.AMQP_URI).then(function(){
-    сreateQueuesAndExchanges(execId, task, "step_1", "step_2");
-});
-
-function сreateQueuesAndExchanges(execId, task, stepId, nextStepId){
-
-    var EXCHANGE_NAME = 'userexchange:' + task.user;
-
-    var MESSAGE_TAG = util.format('%s:%s:%s:message', task.id, stepId, execId);
-    var ERROR_TAG = util.format('%s:%s:%s:error', task.id, stepId, execId);
-    var REBOUND_TAG = util.format('%s:%s:%s:rebound', task.id, stepId, execId);
-
-    var MESSAGES_QUEUE = util.format('%s:%s:%s:messages', task.id, stepId, execId);
-    var MESSAGES_LISTENING_QUEUE = util.format('%s:%s:%s:messages', task.id, nextStepId, execId);
-    var REBOUNDS_QUEUE = util.format('%s:%s:%s:rebounds', task.id, stepId, execId);
-
-    console.log('INCOMING_MESSAGES_QUEUE=%s', MESSAGES_QUEUE);
-    console.log('EXCHANGE_NAME=%s', EXCHANGE_NAME);
-    console.log('MESSAGE_TAG=%s', MESSAGE_TAG);
-    console.log('ERROR_TAG=%s', ERROR_TAG);
-    console.log('REBOUND_TAG=%s', REBOUND_TAG);
-    console.log('MESSAGES_QUEUE=%s', MESSAGES_QUEUE);
-    console.log('REBOUNDS_QUEUE=%s', REBOUNDS_QUEUE);
-
-    var userExchange = {
-        name: EXCHANGE_NAME,
-        type: 'direct',
-        options: {
-            durable: true,
-            autoDelete: false
-        }
-    };
-
-    var messagesQueue = {
-        name: MESSAGES_QUEUE,
-        options: {
-            durable: true,
-            autoDelete: false
-        }
-    };
-
-    var messagesListeningQueue = {
-        name: MESSAGES_LISTENING_QUEUE,
-        options: {
-            durable: true,
-            autoDelete: false
-        }
-    };
-
-    var REBOUND_QUEUE_TTL = 10 * 60 * 1000; // 10 min
-
-    var reboundsQueue = {
-        name: REBOUNDS_QUEUE,
-        options: {
-            durable: true,
-            autoDelete: false,
-            arguments: {
-                'x-message-ttl': REBOUND_QUEUE_TTL,
-                'x-dead-letter-exchange': EXCHANGE_NAME, // send dead rebounded queues back to exchange
-                'x-dead-letter-routing-key': MESSAGE_TAG // with tag as message
-            }
-        }
-    };
-
-    return Q.all([
-        assertExchange(amqpConnection.publishChannel, userExchange), // check that exchange exists
-        assertQueue(amqpConnection.publishChannel, messagesQueue), // create messages queue
-        assertQueue(amqpConnection.publishChannel, messagesListeningQueue), // create messages queue
-        amqpConnection.publishChannel.bindQueue(messagesListeningQueue.name, userExchange.name, MESSAGE_TAG),
-        assertQueue(amqpConnection.publishChannel, reboundsQueue), // create rebounds queue
-        amqpConnection.publishChannel.bindQueue(reboundsQueue.name, userExchange.name, REBOUND_TAG)
-    ]).then(function(){
-        console.log('Successfully asserted all queues');
-    }).done();
-
-    function assertQueue(channel, queue) {
-        return channel.assertQueue(queue.name, queue.options).then(function assertQueueSuccess() {
-            console.log('Succesfully asserted queue: ' + queue.name);
-        });
-    }
-
-    function assertExchange(channel, exchange) {
-        return channel.assertExchange(exchange.name, exchange.type, exchange.options).then(function assertExchangeSuccess() {
-            console.log('Succesfully asserted exchange: ' + exchange.name);
-        });
-    }
-}

package/gulpfile.js

@@ -1,31 +0,0 @@
-var gulp = require('gulp');
-var jasmine = require('gulp-jasmine');
-var istanbul = require('gulp-istanbul');
-var del = require('del');
-
-var paths = {
-    code: ['./lib/*.js'],
-    spec: ['./spec/**/*.spec.js'],
-    coverageReport: './coverage/lcov.info',
-    coverage: './coverage'
-};
-
-gulp.task('clean:coverage', function (cb) {
-    del([paths.coverage], cb);
-});
-
-gulp.task('istanbul', function (cb) {
-    gulp
-        .src(paths.code)
-        .pipe(istanbul()) // Covering files
-        .pipe(istanbul.hookRequire()) // Force `require` to return covered files
-        .on('finish', function () {
-            gulp
-                .src(paths.spec)
-                .pipe(jasmine())
-                .pipe(istanbul.writeReports()) // Creating the reports after tests runned
-                .on('end', cb);
-        });
-});
-
-gulp.task('coverage', ['clean:coverage', 'istanbul']);