elasticio-sailor-nodejs 3.0.0-dev7 → 3.0.0-sailor-proxy-dev1

package/lib/sailor.js

@@ -1,28 +1,22 @@
 const uuid = require('uuid');
 const ComponentReader = require('./component_reader.js').ComponentReader;
-const amqp = require('./amqp.js');
+const { ProxyClient, MESSAGE_PROCESSING_STATUS } = require('./proxy-client.js');
 const TaskExec = require('./executor.js').TaskExec;
 const log = require('./logging.js');
 const _ = require('lodash');
 const hooksData = require('./hooksData');
-const Encryptor = require('../lib/encryptor');
 const RestApiClient = require('elasticio-rest-node');
 const assert = require('assert');
 const co = require('co');
-const pThrottle = require('p-throttle');
-const { ObjectStorage } = require('@elastic.io/maester-client');
-const { Readable } = require('stream');
-const messagesDB = require('./messagesDB.js');
 
-const AMQP_HEADER_META_PREFIX = 'x-eio-meta-';
 const OBJECT_ID_HEADER = 'x-ipaas-object-storage-id';
 
-function convertSettingsToCamelCase(settings) {
-    return _.mapKeys(settings, (value, key) => _.camelCase(key));
+function convertSettingsToSnakeCase(settings) {
+    return _.mapKeys(settings, (value, key) => _.snakeCase(key));
 }
 
 function getAdditionalHeadersFromSettings(settings) {
-    return convertSettingsToCamelCase(settings.additionalVars);
+    return convertSettingsToSnakeCase(settings.additionalVars);
 }
 
 class Sailor {
@@ -32,50 +26,35 @@ class Sailor {
     constructor(settings) {
         this.settings = settings;
         this.messagesCount = 0;
-        this.amqpConnection = new amqp.Amqp(settings);
+        this.proxyClient = new ProxyClient(settings);
         this.componentReader = new ComponentReader();
         this.snapshot = {};
         this.stepData = {};
         this.shutdownCallback = null;
+        // TODO move endpoint to proxy
         //eslint-disable-next-line new-cap
         this.apiClient = RestApiClient(
             settings.API_USERNAME,
+            // TODO find a way to make username and key consistent (without running api tests and looking up
+            // correct values in MongoDB)
             settings.API_KEY,
             {
                 retryCount: settings.API_REQUEST_RETRY_ATTEMPTS,
                 retryDelay: settings.API_REQUEST_RETRY_DELAY
-            });
-
-        const objectStorage = new ObjectStorage({
-            uri: settings.OBJECT_STORAGE_URI,
-            jwtSecret: settings.OBJECT_STORAGE_TOKEN
-        });
-
-        const encryptor = new Encryptor(settings.MESSAGE_CRYPTO_PASSWORD, settings.MESSAGE_CRYPTO_IV);
-        this.objectStorage = objectStorage.use(
-            () => encryptor.createCipher(),
-            () => encryptor.createDecipher()
+            }
         );
-
-        this.throttles = {
-            // 100 Messages per Second
-            data: pThrottle(() => Promise.resolve(true),
-                settings.DATA_RATE_LIMIT,
-                settings.RATE_INTERVAL),
-            error: pThrottle(() => Promise.resolve(true),
-                settings.ERROR_RATE_LIMIT,
-                settings.RATE_INTERVAL),
-            snapshot: pThrottle(() => Promise.resolve(true),
-                settings.SNAPSHOT_RATE_LIMIT,
-                settings.RATE_INTERVAL)
-        };
     }
 
     async connect() {
-        return this.amqpConnection.connect(this.settings.AMQP_URI);
+        return this.proxyClient.connect();
+    }
+
+    async isConnected() {
+        return this.proxyClient.isConnected();
     }
 
     async prepare() {
+        log.trace('prepare sailor');
         const {
             settings: {
                 COMPONENT_PATH: compPath,
@@ -101,11 +80,12 @@ class Sailor {
     }
 
     async disconnect() {
+        // TODO: delete if not needed (currently used only in old tests)
         log.debug('Disconnecting, %s messages in processing', this.messagesCount);
-        return this.amqpConnection.disconnect();
+        return this.proxyClient.disconnect();
     }
 
-    async reportError(err) {
+    reportError(err) {
         const headers = Object.assign({}, getAdditionalHeadersFromSettings(this.settings), {
             execId: this.settings.EXEC_ID,
             taskId: this.settings.FLOW_ID,
@@ -116,7 +96,7 @@ class Sailor {
             compId: this.settings.COMP_ID,
             function: this.settings.FUNCTION
         });
-        return this.amqpConnection.sendError(err, headers);
+        return this.proxyClient.sendError(err, headers);
     }
 
     startup() {
@@ -184,17 +164,16 @@ class Sailor {
     }
 
     run() {
-        const incomingQueue = this.settings.LISTEN_MESSAGES_ON;
         const handler = this.processMessageAndMaybeShutdownCallback.bind(this);
-        log.debug('Start listening for messages on %s', incomingQueue);
-        return this.amqpConnection.listenQueue(incomingQueue, handler);
+        log.debug('Start listening for messages');
+        return this.proxyClient.listenForMessages(handler);
     }
 
-    async processMessageAndMaybeShutdownCallback(payload, message) {
+    async processMessageAndMaybeShutdownCallback(headers, body) {
         try {
-            return await this.processMessage(payload, message);
+            return await this.processMessage(headers, body);
         } catch (e) {
-            log.error('Something very bad happened during message processing');
+            log.error(e, 'Something very bad happened during message processing');
         } finally {
             if (this.shutdownCallback) {
                 if (this.messagesCount === 0) {
@@ -216,7 +195,7 @@ class Sailor {
             return new Promise(resolve => this.shutdownCallback = resolve);
         }
 
-        await this.amqpConnection.stopConsume();
+        await this.proxyClient.disconnect();
         if (this.messagesCount === 0) {
             // there is no unfinished processMessage invocation, let's just resolve scheduleShutdown now
             log.debug('scheduleShutdown – about to shutdown immediately');
@@ -228,103 +207,23 @@ class Sailor {
         return new Promise(resolve => this.shutdownCallback = resolve);
     }
 
-
-    readIncomingMessageHeaders(message) {
-        const { headers } = message.properties;
-
-        // Get meta headers
-        const metaHeaderNames = Object.keys(headers)
-            .filter(key => key.toLowerCase().startsWith(AMQP_HEADER_META_PREFIX));
-
-        const metaHeaders = _.pick(headers, metaHeaderNames);
-        const metaHeadersLowerCased = _.mapKeys(metaHeaders, (value, key) => key.toLowerCase());
-
-        const result = {
-            stepId: headers.stepId, // the only use is passthrough mechanism
-            ...metaHeadersLowerCased,
-            threadId: headers.threadId || metaHeadersLowerCased['x-eio-meta-trace-id'],
-            messageId: headers.messageId,
-            parentMessageId: headers.parentMessageId
-        };
-        if (!result.threadId) {
-            const threadId = uuid.v4();
-            log.debug({ threadId }, 'Initiate new thread as it is not started ATM');
-            result.threadId = threadId;
-        }
-        if (headers.reply_to) {
-            result.reply_to = headers.reply_to;
-        }
-        return result;
-    }
-
-    async fetchMessageBody(message, logger) {
-        const { body, headers } = message;
-
-        logger.info('Checking if incoming messages is lightweight...');
-
-        if (!headers) {
-            logger.info('Empty headers so not lightweight.');
-            return body;
-        }
-
-        const { [OBJECT_ID_HEADER]: objectId } = headers;
-
-        if (!objectId) {
-            logger.trace('No object id header so not lightweight.');
-            return body;
-        }
-
-        logger.info('Object id header found, message is lightweight.', { objectId });
-
-        let object;
-
-        logger.info('Going to fetch message body.', { objectId });
-
-        try {
-            object = await this.objectStorage.getOne(
-                objectId,
-                { jwtPayloadOrToken: this.settings.OBJECT_STORAGE_TOKEN }
-            );
-        } catch (e) {
-            log.error(e);
-            throw new Error(`Failed to get message body with id=${objectId}`);
-        }
-
-        logger.info('Successfully obtained message body.', { objectId });
-        logger.trace('Message body object received');
-
-        return object.data;
-    }
-
-    uploadMessageBody(bodyBuf) {
-        const stream = () => Readable.from(bodyBuf);
-        return this.objectStorage.add(
-            stream,
-            { jwtPayloadOrToken: this.settings.OBJECT_STORAGE_TOKEN }
-        );
-    }
-
-    async runExec(module, payload, message, outgoingMessageHeaders, stepData, timeStart, logger) {
+    async runExec(module, payload, incomingMessageHeaders, outgoingMessageHeaders, stepData, timeStart, logger) {
+        log.debug({ incomingMessageHeaders }, 'runExec started');
         const origPassthrough = _.cloneDeep(payload.passthrough) || {};
-        const incomingMessageHeaders = this.readIncomingMessageHeaders(message);
-        const messageId = incomingMessageHeaders.messageId;
         const settings = this.settings;
         const cfg = _.cloneDeep(stepData.config) || {};
         const snapshot = _.cloneDeep(this.snapshot);
-        const { deliveryTag } = message.fields;
 
         const that = this;
 
         await new Promise(resolve => {
             let endWasEmitted;
 
-
             const taskExec = new TaskExec({
                 loggerOptions: _.pick(incomingMessageHeaders, ['threadId', 'messageId', 'parentMessageId']),
                 variables: stepData.variables,
                 services: {
                     apiClient: this.apiClient,
-                    amqp: this.amqpConnection,
                     config: this.settings
                 }
             });
@@ -391,9 +290,9 @@ class Sailor {
                         let passthroughIds;
                         try {
                             [bodyId, ...passthroughIds] = await Promise.all([
-                                that.uploadMessageBody(bodyBuf),
+                                that.proxyClient.uploadMessageBody(bodyBuf),
                                 ...passthroughBufs.map(async ({ stepId, body, id }) => {
-                                    const bodyId = id || await that.uploadMessageBody(body);
+                                    const bodyId = id || await that.proxyClient.uploadMessageBody(body);
                                     return { stepId, bodyId };
                                 })
                             ]);
@@ -402,7 +301,7 @@ class Sailor {
                             return onError(new Error('Lightweight message/passthrough body upload error'));
                         }
 
-                        logger.info('Message body uploaded', { id: bodyId });
+                        logger.info({ id: bodyId }, 'Message body uploaded');
                         const { headers } = data;
                         data.body = {};
                         data.headers = {
@@ -439,7 +338,7 @@ class Sailor {
                                 logger.trace('Body is not empty.', { stepId });
                                 return;
                             }
-                            data.passthrough[stepId].body = await that.fetchMessageBody(
+                            data.passthrough[stepId].body = await that.proxyClient.fetchMessageBody(
                                 passthrough[stepId],
                                 logger
                             );
@@ -458,7 +357,12 @@ class Sailor {
                 log.trace('Going to send outgoing message');
 
                 try {
-                    await that.amqpConnection.sendData(data, headers, that.throttles.data);
+                    await that.proxyClient.sendMessage({
+                        incomingMessageId: incomingMessageHeaders.messageId,
+                        data,
+                        headers,
+                        type: 'data'
+                    });
                     log.trace('Outgoing message sent');
                 } catch (err) {
                     return onError(err);
@@ -471,7 +375,12 @@ class Sailor {
                     messageProcessingTime: Date.now() - timeStart
                 }, 'processMessage emit HttpReply');
 
-                return that.amqpConnection.sendHttpReply(reply, headers);
+                return that.proxyClient.sendMessage({
+                    incomingMessageId: incomingMessageHeaders.messageId,
+                    data: reply,
+                    headers,
+                    type: 'http-reply'
+                });
             }
 
             async function onError(err) {
@@ -484,7 +393,7 @@ class Sailor {
                     messageProcessingTime: Date.now() - timeStart
                 }, 'processMessage emit error');
                 headers.end = new Date().getTime();
-                return that.amqpConnection.sendError(err, headers, message, that.throttles.error);
+                return that.proxyClient.sendError(err, headers, payload, incomingMessageHeaders);
             }
 
             async function onRebound(err) {
@@ -495,14 +404,14 @@ class Sailor {
                     messagesCount: that.messagesCount,
                     messageProcessingTime: Date.now() - timeStart
                 }, 'processMessage emit rebound');
-                return that.amqpConnection.sendRebound(err, message, outgoingHeaders);
+                return that.proxyClient.sendRebound(err, incomingMessageHeaders, outgoingHeaders);
             }
 
             async function onSnapshot(data) {
                 const headers = _.clone(outgoingMessageHeaders);
                 headers.snapshotEvent = 'snapshot';
                 that.snapshot = data; //replacing `local` snapshot
-                return that.amqpConnection.sendSnapshot(data, headers, that.throttles.snapshot);
+                return that.proxyClient.sendSnapshot(data, headers);
             }
 
             async function onUpdateSnapshot(data) {
@@ -514,7 +423,7 @@ class Sailor {
                         return log.warn('ERROR: $set is not supported any more in `updateSnapshot` event');
                     }
                     _.extend(that.snapshot, data); //updating `local` snapshot
-                    return that.amqpConnection.sendSnapshot(data, headers);
+                    return that.proxyClient.sendSnapshot(data, headers);
                 } else {
                     log.error('You should pass an object to the `updateSnapshot` event');
                 }
@@ -527,14 +436,14 @@ class Sailor {
 
                 try {
                     await that.apiClient.accounts.update(cfg._account, { keys: keys });
-                    logger.debug('Successfully updated keys #%s', deliveryTag);
+                    logger.debug({ messageId: incomingMessageHeaders.messageId }, 'Successfully updated keys');
                 } catch (error) {
-                    logger.error('Failed to updated keys #%s', deliveryTag);
+                    logger.debug({ messageId: incomingMessageHeaders.messageId }, 'Failed to update keys');
                     await onError(error);
                 }
             }
 
-            async function onEnd() {
+            function onEnd() {
                 if (endWasEmitted) {
                     logger.warn({
                         messagesCount: that.messagesCount,
@@ -546,11 +455,12 @@ class Sailor {
 
                 endWasEmitted = true;
 
-                if (taskExec.errorCount > 0) {
-                    await that.amqpConnection.reject(messageId);
-                } else {
-                    await that.amqpConnection.ack(messageId);
-                }
+                that.proxyClient.finishProcessing(
+                    incomingMessageHeaders,
+                    taskExec.errorCount > 0
+                        ? MESSAGE_PROCESSING_STATUS.ERROR
+                        : MESSAGE_PROCESSING_STATUS.SUCCESS
+                );
                 that.messagesCount -= 1;
                 logger.trace({
                     messagesCount: that.messagesCount,
@@ -579,43 +489,21 @@ class Sailor {
         }
     }
 
-    async processMessage(payload, message) {
+    async processMessage(headers, payload) {
         //eslint-disable-next-line consistent-this
         const self = this;
         const settings = this.settings;
-        const incomingMessageHeaders = this.readIncomingMessageHeaders(message);
 
         self.messagesCount += 1;
 
         const timeStart = Date.now();
 
-        const messageId = incomingMessageHeaders.messageId;
         const logger = log.child({
-            threadId: incomingMessageHeaders.threadId || 'unknown',
-            messageId: messageId || 'unknown',
-            parentMessageId: incomingMessageHeaders.parentMessageId || 'unknown',
-            ...message.fields
+            threadId: headers.threadId || 'unknown',
+            messageId: headers.messageId || 'unknown',
+            parentMessageId: headers.parentMessageId || 'unknown'
         });
 
-        if (messageId) {
-            const alreadyExists = messagesDB.getMessageById(messageId);
-            // Add message to DB even if it already exists
-            messagesDB.addMessage(messageId, message);
-            if (alreadyExists) {
-                logger.warn({ messageId }, 'Duplicate message detected. This'
-                    + ' delivery will be ignored; the handler that first received'
-                    + ' this message will process it as part of deduplication.');
-                // If message was in messagesDB, it means that the connection was closed
-                // and this message was redelivered. In this case, the process for original
-                // message is waiting for this message to be added to DB and then ack or
-                // nack the new message, instead of the one that was delivered by closed
-                // channel
-                return;
-            }
-        } else {
-            logger.warn('Message does not have messageId');
-        }
-
         logger.trace({ messagesCount: this.messagesCount }, 'processMessage received');
 
         const stepData = this.stepData;
@@ -623,10 +511,10 @@ class Sailor {
         log.debug('Trigger or action: %s', settings.FUNCTION);
         const outgoingMessageId = uuid.v4();
         const outgoingMessageHeaders = {
-            ...incomingMessageHeaders,
+            ...headers,
             ...getAdditionalHeadersFromSettings(settings),
-            parentMessageId: incomingMessageHeaders.messageId,
-            threadId: incomingMessageHeaders.threadId,
+            parentMessageId: headers.messageId,
+            threadId: headers.threadId,
             messageId: outgoingMessageId,
             execId: settings.EXEC_ID,
             taskId: settings.FLOW_ID,
@@ -644,10 +532,8 @@ class Sailor {
         } catch (e) {
             log.error(e);
             outgoingMessageHeaders.end = new Date().getTime();
-            await Promise.all([
-                self.amqpConnection.sendError(e, outgoingMessageHeaders, message),
-                self.amqpConnection.reject(messageId)
-            ]);
+            self.proxyClient.sendError(e, outgoingMessageHeaders, payload, headers);
+            self.proxyClient.finishProcessing(headers, MESSAGE_PROCESSING_STATUS.ERROR);
             return;
         }
 
@@ -660,12 +546,12 @@ class Sailor {
                 await Promise.all([
                     (async () => {
                         logger.trace('Going to check if incoming message body is lightweight.');
-                        payload.body = await this.fetchMessageBody(payload, logger);
+                        payload.body = await this.proxyClient.fetchMessageBody(payload, logger);
                     })(),
                     ...(passthrough
                         ? Object.keys(passthrough).map(async stepId => {
                             logger.trace('Going to check if passthrough for step is lightweight.', { stepId });
-                            payload.passthrough[stepId].body = await this.fetchMessageBody(
+                            payload.passthrough[stepId].body = await this.proxyClient.fetchMessageBody(
                                 payload.passthrough[stepId],
                                 logger
                             );
@@ -675,15 +561,13 @@ class Sailor {
             } catch (e) {
                 logger.error(e);
                 outgoingMessageHeaders.end = new Date().getTime();
-                await Promise.all([
-                    self.amqpConnection.sendError(e, outgoingMessageHeaders, message),
-                    self.amqpConnection.reject(messageId)
-                ]);
+                self.proxyClient.sendError(e, outgoingMessageHeaders, payload, headers);
+                self.proxyClient.finishProcessing(headers, MESSAGE_PROCESSING_STATUS.ERROR);
                 return;
             }
         }
 
-        await this.runExec(module, payload, message, outgoingMessageHeaders, stepData, timeStart, logger);
+        await this.runExec(module, payload, headers, outgoingMessageHeaders, stepData, timeStart, logger);
     }
 }
 

package/lib/settings.js

@@ -4,14 +4,22 @@ const PREFIX = 'ELASTICIO_';
 
 function getOptionalEnvVars(envVars) {
     const optional = {
-        REBOUND_INITIAL_EXPIRATION: 15000,
-        REBOUND_LIMIT: 20,
         COMPONENT_PATH: '',
-        RABBITMQ_PREFETCH_SAILOR: 1,
+        PROXY_PREFETCH_SAILOR: 1,
         STARTUP_REQUIRED: false,
         HOOK_SHUTDOWN: false,
         API_REQUEST_RETRY_ATTEMPTS: 3,
         API_REQUEST_RETRY_DELAY: 100,
+        PROXY_RECONNECT_MAX_RETRIES: Infinity,
+        PROXY_RECONNECT_INITIAL_DELAY: 1000,
+        PROXY_RECONNECT_MAX_DELAY: 30 * 1000, // 30 seconds
+        PROXY_RECONNECT_BACKOFF_MULTIPLIER: 2,
+        PROXY_RECONNECT_JITTER_FACTOR: 0.3,
+        PROXY_OBJECT_REQUEST_RETRY_ATTEMPTS: Infinity,
+        PROXY_OBJECT_REQUEST_RETRY_DELAY: 100,
+        PROXY_OBJECT_REQUEST_MAX_RETRY_DELAY: 5 * 60 * 1000, // 5 mins
+
+        // TODO: Move to proxy?
         DATA_RATE_LIMIT: 10, // 10 data events every 100ms
         ERROR_RATE_LIMIT: 2, // 2 errors every 100ms
         SNAPSHOT_RATE_LIMIT: 2, // 2 Snapshots every 100ms
@@ -20,19 +28,15 @@ function getOptionalEnvVars(envVars) {
         AMQP_PUBLISH_RETRY_DELAY: 100, // 100ms
         AMQP_PUBLISH_RETRY_ATTEMPTS: Infinity,
         AMQP_PUBLISH_MAX_RETRY_DELAY: 5 * 60 * 1000, // 5 mins
+        // Should be defaulted to true and moved to proxy
         AMQP_PERSISTENT_MESSAGES: false,
+
+        OBJECT_STORAGE_SIZE_THRESHOLD: 1048576,
         OUTGOING_MESSAGE_SIZE_LIMIT: 10485760,
         NO_SELF_PASSTRHOUGH: false,
         PROTOCOL_VERSION: 1,
-        NO_ERROR_REPLIES: false,
         INPUT_FORMAT: 'default',
-        OBJECT_STORAGE_URI: null,
-        OBJECT_STORAGE_TOKEN: null,
-        OBJECT_STORAGE_SIZE_THRESHOLD: 1048576,
         EMIT_LIGHTWEIGHT_MESSAGE: false,
-        AMQP_RECONNECT_ATTEMPTS: 3,
-        AMQP_RECONNECT_TIMEOUT: 100,
-        WAIT_MESSAGES_TIMEOUT: 50
     };
 
     const result = {};
@@ -87,16 +91,9 @@ function getMandatoryEnvVars(envVars) {
     ];
 
     const requiredForMessageProcessing = [
-        'AMQP_URI',
-        'LISTEN_MESSAGES_ON',
-        'PUBLISH_MESSAGES_TO',
-
-        'DATA_ROUTING_KEY',
-        'ERROR_ROUTING_KEY',
-        'REBOUND_ROUTING_KEY',
-        'SNAPSHOT_ROUTING_KEY',
-        'MESSAGE_CRYPTO_IV',
-        'MESSAGE_CRYPTO_PASSWORD'
+        'SAILOR_PROXY_URI',
+        'MESSAGE_CRYPTO_PASSWORD',
+        'MESSAGE_CRYPTO_IV'
     ];
 
     const envVarsList = requiredAlways.slice(0);

package/lib/utils.js

@@ -0,0 +1,8 @@
+function getJitteredDelay(baseDelay, jitterFactor) {
+    const minJitter = baseDelay * -jitterFactor;
+    const maxJitter = baseDelay * jitterFactor;
+    const jitter = Math.random() * (maxJitter - minJitter) + minJitter;
+    return baseDelay + jitter;
+}
+
+module.exports.getJitteredDelay = getJitteredDelay;

package/mise.toml

@@ -0,0 +1,2 @@
+[tools]
+node = "18.16.1"

package/package.json

@@ -1,23 +1,23 @@
 {
   "name": "elasticio-sailor-nodejs",
   "description": "The official elastic.io library for bootstrapping and executing for Node.js connectors",
-  "version": "3.0.0-dev7",
+  "version": "3.0.0-sailor-proxy-dev1",
   "main": "run.js",
   "scripts": {
+    "build": "tsc",
     "audit": "better-npm-audit audit --level high --production",
-    "lint": "./node_modules/.bin/eslint lib spec mocha_spec lib run.js runService.js",
     "pretest": "npm run lint",
+    "lint": "eslint --ext .ts .",
     "test": "npm run test:jasmine && npm run test:mocha",
     "test:jasmine": "NODE_ENV=test jasmine-node spec",
     "test:mocha": "NODE_ENV=test node_modules/.bin/mocha --recursive mocha_spec",
-    "postpublish": "./postpublish.js"
+    "postpublish": "./postpublish.js",
+    "dev:local": "node run.local.js | bunyan"
   },
   "engines": {
     "node": ">=12.13.0"
   },
   "dependencies": {
-    "@elastic.io/maester-client": "6.0.0",
-    "amqplib": "0.8.0",
     "bunyan": "1.8.10",
     "co": "4.6.0",
     "debug": "3.1.0",
@@ -45,7 +45,9 @@
     "request": "2.88.0",
     "request-promise-native": "1.0.5",
     "sinon": "9.0.2",
-    "sinon-chai": "3.5.0"
+    "sinon-chai": "3.5.0",
+    "ts-node": "10.4.0",
+    "typescript": "4.4.4"
   },
   "repository": "elasticio/sailor-nodejs",
   "license": "Apache-2.0"