elasticio-sailor-nodejs 3.0.0-dev2 → 3.0.0-dev4

package/config/local.json

@@ -1,6 +1,6 @@
 {
     "LOG_LEVEL": "trace",
-    "ELASTICIO_FLOW_ID": "692ee23d4ab5d34bb7321559",
+    "ELASTICIO_FLOW_ID": "697b56aca8e2e9b35e1952da",
     "ELASTICIO_EXEC_ID": "exec_67890",
     "ELASTICIO_STEP_ID": "step_1",
     "ELASTICIO_CONTAINER_ID": "container_abcde",
@@ -9,11 +9,11 @@
     "ELASTICIO_COMP_ID": "component_3344",
     "ELASTICIO_FUNCTION": "data_trigger",
     "ELASTICIO_API_URI": "http://localhost:9000",
-    "ELASTICIO_API_USERNAME": "task-692ee23d4ab5d34bb7321559",
-    "ELASTICIO_API_KEY": "976ffec8-455b-494e-9478-2d66761f4040",
+    "ELASTICIO_API_USERNAME": "task-697b56aca8e2e9b35e1952da",
+    "ELASTICIO_API_KEY": "9b2e0465-b00c-4b57-af91-bfc3a830306c",
     "ELASTICIO_MESSAGE_CRYPTO_IV": "0.03091345790184",
     "ELASTICIO_MESSAGE_CRYPTO_PASSWORD": "password",
-    "ELASTICIO_SAILOR_PROXY_URI": "http://localhost:5001",
+    "ELASTICIO_SAILOR_PROXY_URI": "http://localhost:4001",
     "ELASTICIO_COMPONENT_PATH": "./spec/component",
     "ELASTICIO_EMIT_LIGHTWEIGHT_MESSAGE": "true"
 }

package/lib/proxy-client.js

@@ -262,34 +262,74 @@ class ProxyClient {
 
         logger.info('Going to fetch message body.', { objectId });
 
-        try {
-            const getObjectStream = this.clientSession.request({
-                [HTTP2_HEADER_PATH]: `/object/${objectId}`,
-                [HTTP2_HEADER_METHOD]: 'GET',
-                [HTTP2_HEADER_AUTHORIZATION]: this.authHeader
-            }).pipe(this._encryptor.createDecipher());
-            // TODO: Add retries
-            object = await new Promise((resolve, reject) => {
-                const chunks = [];
-                getObjectStream.on('data', chunk => {
-                    chunks.push(chunk);
-                });
-                getObjectStream.on('error', (err) => {
-                    logger.error(err, 'Error during fetching message body');
-                    reject(err);
-                });
-                getObjectStream.on('end', () => {
-                    logger.info('Message stream ended by server');
-                    const buffer = Buffer.concat(chunks);
-                    logger.info({ messageSize: buffer.length }, 'Received complete message from server');
-                    resolve({ data: JSON.parse(buffer.toString()) });
+        const maxRetries = this.settings.PROXY_OBJECT_REQUEST_RETRY_ATTEMPTS;
+        const retryDelay = this.settings.PROXY_OBJECT_REQUEST_RETRY_DELAY;
+        for (let attempt = 0; attempt <= maxRetries; attempt++) {
+            try {
+                object = await new Promise((resolve, reject) => {
+                    const getObjectStream = this.clientSession.request({
+                        [HTTP2_HEADER_PATH]: `/object/${objectId}`,
+                        [HTTP2_HEADER_METHOD]: 'GET',
+                        [HTTP2_HEADER_AUTHORIZATION]: this.authHeader
+                    }).pipe(this._encryptor.createDecipher());
+
+                    const chunks = [];
+                    getObjectStream.on('data', chunk => {
+                        chunks.push(chunk);
+                    });
+                    getObjectStream.on('error', (err) => {
+                        logger.error(err, 'Error during fetching message body');
+                        reject(err);
+                    });
+                    getObjectStream.on('end', () => {
+                        logger.info('Message stream ended by server');
+                        const buffer = Buffer.concat(chunks);
+                        logger.info({ messageSize: buffer.length }, 'Received complete message from server');
+                        resolve({ data: JSON.parse(buffer.toString()) });
+                    });
                 });
-            });
-        } catch (e) {
-            log.error(e);
-            throw new Error(`Failed to get message body with id=${objectId}`);
-        }
 
+                if (attempt > 0) {
+                    log.info({ attempt, maxRetries }, 'Fetch message body succeeded after retry');
+                }
+                return objectId;
+            } catch (error) {
+                const isLastAttempt = attempt === maxRetries;
+                const isRetryable = error.isNetworkError ||
+                    (error.statusCode && error.statusCode >= 500) ||
+                    error.code === 'ECONNRESET' ||
+                    error.code === 'ETIMEDOUT';
+
+                if (!isRetryable || isLastAttempt) {
+                    log.error({
+                        attempt,
+                        maxRetries,
+                        error: error.message,
+                        isRetryable
+                    }, 'Fetch message body failed, no more retries');
+                    throw error;
+                }
+
+                const delay = Math.min(
+                    retryDelay * Math.pow(2, attempt),
+                    this.settings.PROXY_OBJECT_REQUEST_MAX_RETRY_DELAY
+                );
+                log.warn({
+                    attempt,
+                    maxRetries,
+                    error: error.message,
+                    nextRetryIn: delay
+                }, 'Fetch message body failed, retrying...');
+
+                await new Promise(resolve => setTimeout(resolve, delay));
+
+                // Ensure connection is still valid before retry
+                if (!this.isConnected()) {
+                    log.info('Reconnecting before retry...');
+                    await this.connect();
+                }
+            }
+        }
         logger.info('Successfully obtained message body.', { objectId });
         logger.trace('Message body object received');
 
@@ -299,43 +339,101 @@ class ProxyClient {
     async uploadMessageBody(bodyBuf) {
         await this._ensureConnection();
 
-        return new Promise((resolve, reject) => {
-            // TODO: Add retries
-            const postMessageStream = this.clientSession.request({
-                [HTTP2_HEADER_PATH]: '/object',
-                [HTTP2_HEADER_METHOD]: 'POST',
-                [HTTP2_HEADER_AUTHORIZATION]: this.authHeader
-            });
+        const maxRetries = this.settings.PROXY_OBJECT_REQUEST_RETRY_ATTEMPTS;
+        const retryDelay = this.settings.PROXY_OBJECT_REQUEST_RETRY_DELAY;
+        for (let attempt = 0; attempt <= maxRetries; attempt++) {
+            try {
+                const objectId = await new Promise((resolve, reject) => {
+                    const postMessageStream = this.clientSession.request({
+                        [HTTP2_HEADER_PATH]: '/object',
+                        [HTTP2_HEADER_METHOD]: 'POST',
+                        [HTTP2_HEADER_AUTHORIZATION]: this.authHeader
+                    });
 
-            postMessageStream.on('response', (headers, flags) => {
-                const status = headers[http2.constants.HTTP2_HEADER_STATUS];
-                if (status !== 200) {
-                    return reject(new Error(`Failed to upload message body, status code: ${status}`));
+                    let responseData = '';
+                    let statusCode = null;
+
+                    postMessageStream.on('response', (headers, flags) => {
+                        statusCode = headers[http2.constants.HTTP2_HEADER_STATUS];
+                        if (statusCode !== 200) {
+                            const error = new Error(`Failed to upload message body, status code: ${statusCode}`);
+                            error.statusCode = statusCode;
+                            return reject(error);
+                        }
+                    });
+
+                    postMessageStream.on('data', chunk => {
+                        responseData += chunk;
+                    });
+
+                    postMessageStream.on('error', (err) => {
+                        log.error(err, 'Error during upload message body');
+                        err.isNetworkError = true;
+                        reject(err);
+                    });
+
+                    postMessageStream.on('end', () => {
+                        if (!responseData) {
+                            return
+                        }
+                        try {
+                            const responseJson = JSON.parse(responseData);
+                            resolve(responseJson.objectId);
+                        } catch (e) {
+                            log.error(e, 'Failed to parse upload message body response');
+                            reject(e);
+                        }
+                    });
+
+                    const cipher = this._encryptor.createCipher();
+                    cipher.pipe(postMessageStream);
+                    cipher.write(bodyBuf);
+                    cipher.end();
+                });
+
+                // Success - return the objectId
+                if (attempt > 0) {
+                    log.info({ attempt, maxRetries }, 'Upload message body succeeded after retry');
                 }
-            });
-            let responseData = '';
-            postMessageStream.on('data', chunk => {
-                responseData += chunk;
-            });
-            postMessageStream.on('error', (err) => {
-                log.error(err, 'Error during upload message body');
-                reject(err);
-            });
-            postMessageStream.on('end', () => {
-                try {
-                    const responseJson = JSON.parse(responseData);
-                    resolve(responseJson.objectId);
-                } catch (e) {
-                    log.error(e, 'Failed to parse upload message body response');
-                    reject(e);
+                return objectId;
+
+            } catch (error) {
+                const isLastAttempt = attempt === maxRetries;
+                const isRetryable = error.isNetworkError ||
+                    (error.statusCode && error.statusCode >= 500) ||
+                    error.code === 'ECONNRESET' ||
+                    error.code === 'ETIMEDOUT';
+
+                if (!isRetryable || isLastAttempt) {
+                    log.error({
+                        attempt,
+                        maxRetries,
+                        error: error.message,
+                        isRetryable
+                    }, 'Upload message body failed, no more retries');
+                    throw error;
                 }
-            });
 
-            const cipher = this._encryptor.createCipher();
-            cipher.pipe(postMessageStream);
-            cipher.write(bodyBuf);
-            cipher.end();
-        });
+                const delay = Math.min(
+                    retryDelay * Math.pow(2, attempt),
+                    this.settings.PROXY_OBJECT_REQUEST_MAX_RETRY_DELAY
+                );
+                log.warn({
+                    attempt,
+                    maxRetries,
+                    error: error.message,
+                    nextRetryIn: delay
+                }, 'Upload message body failed, retrying...');
+
+                await new Promise(resolve => setTimeout(resolve, delay));
+
+                // Ensure connection is still valid before retry
+                if (!this.isConnected()) {
+                    log.info('Reconnecting before retry...');
+                    await this.connect();
+                }
+            }
+        }
     }
 
     async listenForMessages(messageHandler) {

package/lib/sailor.js

@@ -35,6 +35,8 @@ class Sailor {
         //eslint-disable-next-line new-cap
         this.apiClient = RestApiClient(
             settings.API_USERNAME,
+            // TODO find a way to make username and key consistent (without running api tests and looking up
+            // correct values in MongoDB)
             settings.API_KEY,
             {
                 retryCount: settings.API_REQUEST_RETRY_ATTEMPTS,

package/lib/settings.js

@@ -15,6 +15,9 @@ function getOptionalEnvVars(envVars) {
         PROXY_RECONNECT_MAX_DELAY: 30 * 1000, // 30 seconds
         PROXY_RECONNECT_BACKOFF_MULTIPLIER: 2,
         PROXY_RECONNECT_JITTER_FACTOR: 0.3,
+        PROXY_OBJECT_REQUEST_RETRY_ATTEMPTS: Infinity,
+        PROXY_OBJECT_REQUEST_RETRY_DELAY: 100,
+        PROXY_OBJECT_REQUEST_MAX_RETRY_DELAY: 5 * 60 * 1000, // 5 mins
 
         // TODO: Move to proxy?
         DATA_RATE_LIMIT: 10, // 10 data events every 100ms
@@ -28,6 +31,7 @@ function getOptionalEnvVars(envVars) {
         // Should be defaulted to true and moved to proxy
         AMQP_PERSISTENT_MESSAGES: false,
 
+        OBJECT_STORAGE_SIZE_THRESHOLD: 1048576,
         OUTGOING_MESSAGE_SIZE_LIMIT: 10485760,
         NO_SELF_PASSTRHOUGH: false,
         PROTOCOL_VERSION: 1,

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "elasticio-sailor-nodejs",
   "description": "The official elastic.io library for bootstrapping and executing for Node.js connectors",
-  "version": "3.0.0-dev2",
+  "version": "3.0.0-dev4",
   "main": "run.js",
   "scripts": {
     "build": "tsc",