elasticio-sailor-nodejs 2.7.0-dev2 → 2.7.0

package/lib/cipher.js

@@ -1,82 +0,0 @@
-const _ = require('lodash');
-const crypto = require('crypto');
-const debug = require('debug')('sailor:cipher');
-const { PassThrough } = require('stream');
-
-const ALGORYTHM = 'aes-256-cbc';
-
-exports.id = 1;
-exports.encrypt = encryptIV;
-exports.encryptStream = createCypher;
-exports.decrypt = decryptIV;
-exports.decryptStream = createDecipher;
-
-function createCypher() {
-    const PASSWORD = process.env.ELASTICIO_MESSAGE_CRYPTO_PASSWORD;
-    const VECTOR = process.env.ELASTICIO_MESSAGE_CRYPTO_IV;
-
-    if (!PASSWORD) {
-        //mimic cypher
-        return new class extends PassThrough {
-            update(data) {
-                return data;
-            }
-            final() {
-                return '';
-            }
-        };
-    }
-
-    if (!VECTOR) {
-        throw new Error('process.env.ELASTICIO_MESSAGE_CRYPTO_IV is not set');
-    }
-
-    const encodeKey = crypto.createHash('sha256').update(PASSWORD, 'utf-8').digest();
-    return crypto.createCipheriv(ALGORYTHM, encodeKey, VECTOR);
-}
-
-function createDecipher() {
-    const PASSWORD = process.env.ELASTICIO_MESSAGE_CRYPTO_PASSWORD;
-    const VECTOR = process.env.ELASTICIO_MESSAGE_CRYPTO_IV;
-
-    if (!PASSWORD) {
-        //mimic cypher
-        return new class extends PassThrough {
-            update(data) {
-                return data;
-            }
-            final() {
-                return '';
-            }
-        };
-    }
-
-    if (!VECTOR) {
-        throw new Error('process.env.ELASTICIO_MESSAGE_CRYPTO_IV is not set');
-    }
-
-    const decodeKey = crypto.createHash('sha256').update(PASSWORD, 'utf-8').digest();
-    return crypto.createDecipheriv(ALGORYTHM, decodeKey, VECTOR);
-}
-
-function encryptIV(rawData, outputEncoding) {
-    debug('About to encrypt:', rawData);
-
-    if (!_.isString(rawData)) {
-        throw new Error('RabbitMQ message cipher.encryptIV() accepts only string as parameter.');
-    }
-
-    const cipher = createCypher();
-    return Buffer.concat([
-        Buffer.from(cipher.update(rawData, 'utf8', outputEncoding)),
-        Buffer.from(cipher.final(outputEncoding))
-    ]);
-}
-
-function decryptIV(encData, inputEncoding) {
-    const decipher = createDecipher();
-
-    const data = inputEncoding ? encData.toString() : encData;
-
-    return decipher.update(data, inputEncoding, 'utf8') + decipher.final('utf8');
-}

package/lib/component_reader.js

@@ -34,21 +34,31 @@ ComponentReader.prototype.promiseLoadJson = function promiseLoadJson(jsonFilePat
     }
 };
 
-ComponentReader.prototype.findTriggerOrAction = function findTriggerOrAction(name) {
+ComponentReader.prototype.findTriggerOrActionDefinition = function findTriggerOrActionDefinition(name) {
     if (this.componentJson === null) {
         throw new Error('Component.json was not loaded');
     }
 
+    let def;
     if (this.componentJson.triggers && this.componentJson.triggers[name]) {
-        return this.componentJson.triggers[name].main;
+        def = this.componentJson.triggers[name];
     } else if (this.componentJson.actions && this.componentJson.actions[name]) {
-        return this.componentJson.actions[name].main;
+        def = this.componentJson.actions[name];
     } else {
         throw new Error('Trigger or action "' + name + '" is not found in component.json!');
     }
+
+    return {
+        autoResolveObjectReferences: true,
+        ...def
+    };
+};
+
+ComponentReader.prototype.findTriggerOrAction = function findTriggerOrAction(name) {
+    return this.findTriggerOrActionDefinition(name).main;
 };
 
-ComponentReader.prototype.loadTriggerOrAction = function loadTriggerOrAction(name) {
+ComponentReader.prototype.loadTriggerOrAction = async function loadTriggerOrAction(name) {
     var filename;
     var modulePath;
     var result;

package/lib/encryptor.js

@@ -1,21 +1,114 @@
-const cipher = require('./cipher.js');
+const crypto = require('crypto');
 const log = require('./logging.js');
 
-exports.encryptMessageContent = encryptMessageContent;
-exports.decryptMessageContent = decryptMessageContent;
+class Encryptor {
+    constructor(password, iv) {
+        this._cryptoPassword = password;
+        this._cryptoIV = iv;
+        if (this._cryptoPassword) {
+            this._encryptionKey = crypto
+                .createHash('sha256')
+                .update(this._cryptoPassword, 'utf-8')
+                .digest();
+            if (!this._cryptoIV) {
+                throw new Error(
+                    `missing crypt initialiazation vector,
+                    most likely ELASTICIO_MESSAGE_CRYPTO_PASSWORD env var is not set
+                    `
+                );
+            }
+        }
+        this._algorithm = 'aes-256-cbc';
+    }
+
+    createCipher() {
+        return crypto.createCipheriv(
+            this._algorithm,
+            this._encryptionKey,
+            this._cryptoIV
+        );
+    }
+
+    createDecipher() {
+        return crypto.createDecipheriv(
+            this._algorithm,
+            this._encryptionKey,
+            this._cryptoIV
+        );
+    }
 
-function encryptMessageContent(messagePayload, outputEncoding) {
-    return cipher.encrypt(JSON.stringify(messagePayload), outputEncoding);
-}
 
-function decryptMessageContent(messagePayload, inputEncoding) {
-    if (!messagePayload || messagePayload.length === 0) {
-        return null;
+    /**
+     * Encrypt message to proper format
+     * @param {*} messagePayload anything json-stringifiable
+     * @param {'hex'|'base64'|'utf-8',undefined} outputEncoding
+     * @returns {Buffer}
+     */
+    encryptMessageContent(messagePayload, outputEncoding) {
+        const encryptedBuffer = this._encryptToBuffer(JSON.stringify(messagePayload));
+        if (outputEncoding) {
+            return Buffer.from(encryptedBuffer.toString(outputEncoding));
+        } else {
+            return encryptedBuffer;
+        }
     }
-    try {
-        return JSON.parse(cipher.decrypt(messagePayload, inputEncoding));
-    } catch (err) {
-        log.error(err, 'Failed to decrypt message');
-        throw Error('Failed to decrypt message: ' + err.message);
+
+    /**
+     * Encrypt message to proper format
+     * @param {Buffer} messagePayload
+     * @param {'hex'|'base64'|'utf-8',undefined} inputEncoding
+     * @returns {*} anything, what have been encrypted
+     */
+    decryptMessageContent(messagePayload, inputEncoding) {
+        if (!messagePayload || messagePayload.length === 0) {
+            return null;
+        }
+        let encryptedBuffer;
+        if (inputEncoding) {
+            encryptedBuffer = Buffer.from(messagePayload.toString(), inputEncoding);
+        } else {
+            encryptedBuffer = messagePayload;
+        }
+        try {
+            const decryptedMessage = this._decryptFromBuffer(encryptedBuffer);
+            return JSON.parse(decryptedMessage);
+        } catch (err) {
+            log.error(err, 'Failed to decrypt message');
+            throw Error('Failed to decrypt message: ' + err.message);
+        }
+    }
+
+    /**
+     * Encrypt payload.
+     * @param {Buffer|String} message
+     * @returns {Buffer}
+     */
+    _encryptToBuffer(message) {
+        if (!this._encryptionKey) {
+            return Buffer.from(message);
+        }
+        const cipher = this.createCipher();
+        return Buffer.concat([
+            cipher.update(message),
+            cipher.final()
+        ]);
+    }
+
+    /**
+     * Decrypt payload.
+     * @param {Buffer} message
+     * @returns {String}
+     */
+    _decryptFromBuffer(message) {
+        if (!this._encryptionKey) {
+            return message.toString();
+        }
+        const decipher = this.createDecipher();
+
+        return Buffer.concat([
+            decipher.update(message),
+            decipher.final()
+        ]).toString('utf8');
     }
 }
+module.exports = Encryptor;

package/lib/executor.js

@@ -20,7 +20,7 @@ class TaskExec extends EventEmitter {
 
     process(triggerOrAction, payload, cfg, snapshot) {
         const onError = async (err) => {
-            this.logger.error(err);
+            this.logger.error('Error occurred during trigger or action processing');
             await this.emit('error', err);
             await this.emit('end');
         };

package/lib/ipc.js

@@ -0,0 +1,13 @@
+/**
+ * Used to communicate with parent processes.
+ */
+class IPC {
+    send(event, data) {
+        if (!process.send) {
+            return;
+        }
+        process.send({ event, data });
+    }
+}
+
+exports.IPC = IPC;

package/lib/sailor.js

@@ -4,8 +4,8 @@ const amqp = require('./amqp.js');
 const TaskExec = require('./executor.js').TaskExec;
 const log = require('./logging.js');
 const _ = require('lodash');
-const cipher = require('./cipher.js');
 const hooksData = require('./hooksData');
+const Encryptor = require('../lib/encryptor');
 const RestApiClient = require('elasticio-rest-node');
 const assert = require('assert');
 const co = require('co');
@@ -50,7 +50,11 @@ class Sailor {
             jwtSecret: settings.OBJECT_STORAGE_TOKEN
         });
 
-        this.objectStorage = objectStorage.use(cipher.encryptStream(), cipher.decryptStream());
+        const encryptor = new Encryptor(settings.MESSAGE_CRYPTO_PASSWORD, settings.MESSAGE_CRYPTO_IV);
+        this.objectStorage = objectStorage.use(
+            () => encryptor.createCipher(),
+            () => encryptor.createDecipher()
+        );
 
         this.throttles = {
             // 100 Messages per Second
@@ -82,7 +86,7 @@ class Sailor {
         } = this;
 
         const stepData = await apiClient.tasks.retrieveStep(flowId, stepId);
-        log.debug('Received step data: %j', stepData);
+        log.debug('Received step data');
         assert(stepData);
 
         Object.assign(this, {
@@ -118,7 +122,7 @@ class Sailor {
         return co(function* doStartup() {
             log.debug('Starting up component');
             const result = yield this.invokeModuleFunction('startup');
-            log.trace('Startup data', { result });
+            log.trace('Startup data received');
             const handle = hooksData.startup(this.settings);
             try {
                 const state = _.isEmpty(result) ? {} : result;
@@ -189,7 +193,7 @@ class Sailor {
         try {
             return await this.processMessage(payload, message);
         } catch (e) {
-            log.error(e, 'Something very bad happened during message processing');
+            log.error('Something very bad happened during message processing');
         } finally {
             if (this.shutdownCallback) {
                 if (this.messagesCount === 0) {
@@ -205,24 +209,22 @@ class Sailor {
         }
     }
 
-    scheduleShutdown() {
-        return co(function* doScheduleShutdown() {
-            if (this.shutdownCallback) {
-                log.debug('scheduleShutdown – shutdown is already scheduled, do nothing');
-                return new Promise(resolve => this.shutdownCallback = resolve);
-            }
-
-            yield this.amqpConnection.listenQueueCancel();
-            if (this.messagesCount === 0) {
-                // there is no unfinished processMessage invocation, let's just resolve scheduleShutdown now
-                log.debug('scheduleShutdown – about to shutdown immediately');
-                return;
-            }
-            // at least one processMessage invocation is not finished yet
-            // let's return a Promise, which will be resolved by processMessageAndMaybeShutdownCallback
-            log.debug('scheduleShutdown – shutdown is scheduled');
+    async scheduleShutdown() {
+        if (this.shutdownCallback) {
+            log.debug('scheduleShutdown – shutdown is already scheduled, do nothing');
             return new Promise(resolve => this.shutdownCallback = resolve);
-        }.bind(this));
+        }
+
+        await this.amqpConnection.stopConsume();
+        if (this.messagesCount === 0) {
+            // there is no unfinished processMessage invocation, let's just resolve scheduleShutdown now
+            log.debug('scheduleShutdown – about to shutdown immediately');
+            return Promise.resolve();
+        }
+        // at least one processMessage invocation is not finished yet
+        // let's return a Promise, which will be resolved by processMessageAndMaybeShutdownCallback
+        log.debug('scheduleShutdown – shutdown is scheduled');
+        return new Promise(resolve => this.shutdownCallback = resolve);
     }
 
 
@@ -257,17 +259,17 @@ class Sailor {
     async fetchMessageBody(message, logger) {
         const { body, headers } = message;
 
-        logger.info('Checking if incoming messages is lightweight...', { headers });
+        logger.info('Checking if incoming messages is lightweight...');
 
         if (!headers) {
-            logger.info('Empty headers so not lightweight.', { headers });
+            logger.info('Empty headers so not lightweight.');
             return body;
         }
 
         const { [OBJECT_ID_HEADER]: objectId } = headers;
 
         if (!objectId) {
-            logger.info('No object id header so not lightweight.', { headers });
+            logger.trace('No object id header so not lightweight.');
             return body;
         }
 
@@ -278,43 +280,27 @@ class Sailor {
         logger.info('Going to fetch message body.', { objectId });
 
         try {
-            object = await this.objectStorage.getAsJSON(objectId);
+            object = await this.objectStorage.getAsJSON(
+                objectId,
+                { jwtPayloadOrToken: this.settings.OBJECT_STORAGE_TOKEN }
+            );
         } catch (e) {
             log.error(e);
             throw new Error(`Failed to get message body with id=${objectId}`);
         }
 
         logger.info('Successfully obtained message body.', { objectId });
-        logger.trace('Message body object', { object });
+        logger.trace('Message body object received');
 
         return object;
     }
 
-    async uploadMessageBody(body, logger) {
-        const { settings: { OBJECT_STORAGE_SIZE_THRESHOLD } } = this;
-        const buf = Buffer.from(JSON.stringify(body), 'utf-8');
-
-        const size = buf.length;
-        if (size <= OBJECT_STORAGE_SIZE_THRESHOLD) {
-            logger.info('Message size is below threshold', { size, OBJECT_STORAGE_SIZE_THRESHOLD });
-            return;
-        }
-
-        const readable = Readable.from(buf.values());
-
-        let id;
-
-        logger.info('Message size is above threshold, going to upload', { size, OBJECT_STORAGE_SIZE_THRESHOLD });
-
-        try {
-            id = await this.objectStorage.addAsStream(readable);
-        } catch (e) {
-            log.error(e);
-        }
-
-        logger.info('Message uploaded', { id });
-
-        return id;
+    uploadMessageBody(bodyBuf) {
+        const stream = () => Readable.from(bodyBuf);
+        return this.objectStorage.addAsStream(
+            stream,
+            { jwtPayloadOrToken: this.settings.OBJECT_STORAGE_TOKEN }
+        );
     }
 
     async runExec(module, payload, message, outgoingMessageHeaders, stepData, timeStart, logger) {
@@ -376,49 +362,89 @@ class Sailor {
                 }
 
                 data.headers = data.headers || {};
-                const { body, passthrough } = data;
+                const { body, passthrough = {} } = data;
 
                 if (settings.EMIT_LIGHTWEIGHT_MESSAGE) {
-                    logger.info('Outgoing lightweight is enabled, going to check size and upload.');
-                    await Promise.all([
-                        (async () => {
-                            logger.info('Going to check size and upload message body.');
-                            const id = await that.uploadMessageBody(body, logger);
-                            if (id) {
-                                const { headers } = data;
-                                data.body = {};
-                                data.headers = {
-                                    ...(headers || {}),
-                                    [OBJECT_ID_HEADER]: id
-                                };
+                    logger.trace('Outgoing lightweight is enabled, going to check size.');
+                    const bodyBuf = Buffer.from(JSON.stringify(body), 'utf-8');
+                    const passthroughBufs = Object.keys(passthrough).map(stepId => ({
+                        stepId,
+                        body: Buffer.from(JSON.stringify(passthrough[stepId].body), 'utf-8'),
+                        id: passthrough[stepId].headers && passthrough[stepId].headers[OBJECT_ID_HEADER]
+                    }));
+
+                    const totalLength = passthroughBufs.reduce((len, { body }) =>
+                        len + body.length, bodyBuf.length);
+
+                    if (totalLength > settings.OBJECT_STORAGE_SIZE_THRESHOLD) {
+                        logger.info(
+                            'Message size is above threshold, going to upload',
+                            {
+                                totalLength,
+                                OBJECT_STORAGE_SIZE_THRESHOLD: settings.OBJECT_STORAGE_SIZE_THRESHOLD
                             }
-                        })(),
-                        ...(
-                            passthrough
-                                ? Object.keys(passthrough).map(async stepId => {
-                                    logger.info(
-                                        'Going to check size and upload message passthrough for step.', { stepId }
-                                    );
-                                    const { [stepId]: { headers } } = passthrough;
-                                    if (headers && headers[OBJECT_ID_HEADER]) {
-                                        logger.info(
-                                            'Passthrough is lightweight already',
-                                            { stepId, headers }
-                                        );
-                                        return;
-                                    }
-                                    const id = await that.uploadMessageBody(data.passthrough[stepId].body, logger);
-                                    if (id) {
-                                        data.passthrough[stepId].body = {};
-                                        data.passthrough[stepId].headers = {
-                                            ...(headers || {}),
-                                            [OBJECT_ID_HEADER]: id
-                                        };
-                                    }
+                        );
+
+                        let bodyId;
+                        let passthroughIds;
+                        try {
+                            [bodyId, ...passthroughIds] = await Promise.all([
+                                that.uploadMessageBody(bodyBuf),
+                                ...passthroughBufs.map(async ({ stepId, body, id }) => {
+                                    const bodyId = id || await that.uploadMessageBody(body);
+                                    return { stepId, bodyId };
                                 })
-                                : []
-                        )
-                    ]);
+                            ]);
+                        } catch (e) {
+                            logger.error(e, 'Error during message/passthrough body upload');
+                            return onError(new Error('Lightweight message/passthrough body upload error'));
+                        }
+
+                        logger.info('Message body uploaded', { id: bodyId });
+                        const { headers } = data;
+                        data.body = {};
+                        data.headers = {
+                            ...(headers || {}),
+                            [OBJECT_ID_HEADER]: bodyId
+                        };
+
+                        for (const { stepId, bodyId } of passthroughIds) {
+                            logger.info('Passthrough Message body uploaded', { stepId, id: bodyId });
+                            const { [stepId]: { headers } } = passthrough;
+                            data.passthrough[stepId].body = {};
+                            data.passthrough[stepId].headers = {
+                                ...(headers || {}),
+                                [OBJECT_ID_HEADER]: bodyId
+                            };
+                        }
+
+                    } else {
+                        logger.trace(
+                            'Message size is below threshold.',
+                            {
+                                totalLength,
+                                OBJECT_STORAGE_SIZE_THRESHOLD: settings.OBJECT_STORAGE_SIZE_THRESHOLD
+                            }
+                        );
+                    }
+                } else if (passthrough) {
+                    logger.trace('Outgoing lightweight is disabled, going to download all bodies.');
+                    try {
+                        await Promise.all(Object.keys(passthrough).map(async stepId => {
+                            logger.trace('Going to check if passthrough for step is lightweight.', { stepId });
+                            // if body is not empty then we've downloaded before processing, no need to redownload
+                            if (!_.isEmpty(data.passthrough[stepId].body)) {
+                                logger.trace('Body is not empty.', { stepId });
+                                return;
+                            }
+                            data.passthrough[stepId].body = await that.fetchMessageBody(
+                                passthrough[stepId],
+                                logger
+                            );
+                        }));
+                    } catch (e) {
+                        return onError(e);
+                    }
                 }
 
                 if (stepData.is_passthrough === true && !settings.NO_SELF_PASSTRHOUGH) {
@@ -427,8 +453,11 @@ class Sailor {
                     });
                 }
 
+                log.trace('Going to send outgoing message');
+
                 try {
                     await that.amqpConnection.sendData(data, headers, that.throttles.data);
+                    log.trace('Outgoing message sent');
                 } catch (err) {
                     return onError(err);
                 }
@@ -457,16 +486,13 @@ class Sailor {
             }
 
             async function onRebound(err) {
-                const headers = _.clone(outgoingMessageHeaders);
                 err = formatError(err);
                 logger.trace({
                     err,
                     messagesCount: that.messagesCount,
                     messageProcessingTime: Date.now() - timeStart
                 }, 'processMessage emit rebound');
-                headers.end = new Date().getTime();
-                headers.reboundReason = err.message;
-                return that.amqpConnection.sendRebound(err, message, headers);
+                return that.amqpConnection.sendRebound(err, message);
             }
 
             async function onSnapshot(data) {
@@ -528,7 +554,6 @@ class Sailor {
                     errorCount: taskExec.errorCount,
                     messageProcessingTime: Date.now() - timeStart
                 }, 'processMessage emit end');
-
                 resolve();
             }
         });
@@ -589,8 +614,7 @@ class Sailor {
             stepId: settings.STEP_ID,
             compId: settings.COMP_ID,
             function: settings.FUNCTION,
-            start: new Date().getTime(),
-            cid: cipher.id
+            start: new Date().getTime()
         };
         let module;
         try {
@@ -603,30 +627,34 @@ class Sailor {
             return;
         }
 
-        const { passthrough } = payload;
+        const method = this.componentReader.findTriggerOrActionDefinition(settings.FUNCTION);
 
-        try {
-            await Promise.all([
-                (async () => {
-                    logger.info('Going to check if incoming message body is lightweight.');
-                    payload.body = await this.fetchMessageBody(payload, logger);
-                })(),
-                ...(passthrough
-                    ? Object.keys(passthrough).map(async stepId => {
-                        logger.info('Going to check if passthrough for step is lightweight.', { stepId });
-                        payload.passthrough[stepId].body = await this.fetchMessageBody(
-                            payload.passthrough[stepId],
-                            logger
-                        );
-                    })
-                    : [])
-            ]);
-        } catch (e) {
-            logger.error(e);
-            outgoingMessageHeaders.end = new Date().getTime();
-            self.amqpConnection.sendError(e, outgoingMessageHeaders, message);
-            self.amqpConnection.reject(message);
-            return;
+        if (method.autoResolveObjectReferences) {
+            const { passthrough } = payload;
+
+            try {
+                await Promise.all([
+                    (async () => {
+                        logger.trace('Going to check if incoming message body is lightweight.');
+                        payload.body = await this.fetchMessageBody(payload, logger);
+                    })(),
+                    ...(passthrough
+                        ? Object.keys(passthrough).map(async stepId => {
+                            logger.trace('Going to check if passthrough for step is lightweight.', { stepId });
+                            payload.passthrough[stepId].body = await this.fetchMessageBody(
+                                payload.passthrough[stepId],
+                                logger
+                            );
+                        })
+                        : [])
+                ]);
+            } catch (e) {
+                logger.error(e);
+                outgoingMessageHeaders.end = new Date().getTime();
+                self.amqpConnection.sendError(e, outgoingMessageHeaders, message);
+                self.amqpConnection.reject(message);
+                return;
+            }
         }
 
         await this.runExec(module, payload, message, outgoingMessageHeaders, stepData, timeStart, logger);