elasticio-sailor-nodejs 3.0.0-dev3 → 3.0.0-dev5

package/lib/proxy-client.js

@@ -202,6 +202,7 @@ class ProxyClient {
                 return resolve();
             }
 
+            // TODO: what if incoming message is received during graceful shutdown?
             this.clientSession.close(() => {
                 log.debug('Successfully closed HTTP2 connection');
                 resolve();
@@ -262,34 +263,74 @@ class ProxyClient {
 
         logger.info('Going to fetch message body.', { objectId });
 
-        try {
-            const getObjectStream = this.clientSession.request({
-                [HTTP2_HEADER_PATH]: `/object/${objectId}`,
-                [HTTP2_HEADER_METHOD]: 'GET',
-                [HTTP2_HEADER_AUTHORIZATION]: this.authHeader
-            }).pipe(this._encryptor.createDecipher());
-            // TODO: Add retries
-            object = await new Promise((resolve, reject) => {
-                const chunks = [];
-                getObjectStream.on('data', chunk => {
-                    chunks.push(chunk);
-                });
-                getObjectStream.on('error', (err) => {
-                    logger.error(err, 'Error during fetching message body');
-                    reject(err);
-                });
-                getObjectStream.on('end', () => {
-                    logger.info('Message stream ended by server');
-                    const buffer = Buffer.concat(chunks);
-                    logger.info({ messageSize: buffer.length }, 'Received complete message from server');
-                    resolve({ data: JSON.parse(buffer.toString()) });
+        const maxRetries = this.settings.PROXY_OBJECT_REQUEST_RETRY_ATTEMPTS;
+        const retryDelay = this.settings.PROXY_OBJECT_REQUEST_RETRY_DELAY;
+        for (let attempt = 0; attempt <= maxRetries; attempt++) {
+            try {
+                object = await new Promise((resolve, reject) => {
+                    const getObjectStream = this.clientSession.request({
+                        [HTTP2_HEADER_PATH]: `/object/${objectId}`,
+                        [HTTP2_HEADER_METHOD]: 'GET',
+                        [HTTP2_HEADER_AUTHORIZATION]: this.authHeader
+                    }).pipe(this._encryptor.createDecipher());
+
+                    const chunks = [];
+                    getObjectStream.on('data', chunk => {
+                        chunks.push(chunk);
+                    });
+                    getObjectStream.on('error', (err) => {
+                        logger.error(err, 'Error during fetching message body');
+                        reject(err);
+                    });
+                    getObjectStream.on('end', () => {
+                        logger.info('Message stream ended by server');
+                        const buffer = Buffer.concat(chunks);
+                        logger.info({ messageSize: buffer.length }, 'Received complete message from server');
+                        resolve({ data: JSON.parse(buffer.toString()) });
+                    });
                 });
-            });
-        } catch (e) {
-            log.error(e);
-            throw new Error(`Failed to get message body with id=${objectId}`);
-        }
 
+                if (attempt > 0) {
+                    log.info({ attempt, maxRetries }, 'Fetch message body succeeded after retry');
+                }
+                return objectId;
+            } catch (error) {
+                const isLastAttempt = attempt === maxRetries;
+                const isRetryable = error.isNetworkError ||
+                    (error.statusCode && error.statusCode >= 500) ||
+                    error.code === 'ECONNRESET' ||
+                    error.code === 'ETIMEDOUT';
+
+                if (!isRetryable || isLastAttempt) {
+                    log.error({
+                        attempt,
+                        maxRetries,
+                        error: error.message,
+                        isRetryable
+                    }, 'Fetch message body failed, no more retries');
+                    throw error;
+                }
+
+                const delay = Math.min(
+                    retryDelay * Math.pow(2, attempt),
+                    this.settings.PROXY_OBJECT_REQUEST_MAX_RETRY_DELAY
+                );
+                log.warn({
+                    attempt,
+                    maxRetries,
+                    error: error.message,
+                    nextRetryIn: delay
+                }, 'Fetch message body failed, retrying...');
+
+                await new Promise(resolve => setTimeout(resolve, delay));
+
+                // Ensure connection is still valid before retry
+                if (!this.isConnected()) {
+                    log.info('Reconnecting before retry...');
+                    await this.connect();
+                }
+            }
+        }
         logger.info('Successfully obtained message body.', { objectId });
         logger.trace('Message body object received');
 
@@ -299,43 +340,101 @@ class ProxyClient {
     async uploadMessageBody(bodyBuf) {
         await this._ensureConnection();
 
-        return new Promise((resolve, reject) => {
-            // TODO: Add retries
-            const postMessageStream = this.clientSession.request({
-                [HTTP2_HEADER_PATH]: '/object',
-                [HTTP2_HEADER_METHOD]: 'POST',
-                [HTTP2_HEADER_AUTHORIZATION]: this.authHeader
-            });
+        const maxRetries = this.settings.PROXY_OBJECT_REQUEST_RETRY_ATTEMPTS;
+        const retryDelay = this.settings.PROXY_OBJECT_REQUEST_RETRY_DELAY;
+        for (let attempt = 0; attempt <= maxRetries; attempt++) {
+            try {
+                const objectId = await new Promise((resolve, reject) => {
+                    const postMessageStream = this.clientSession.request({
+                        [HTTP2_HEADER_PATH]: '/object',
+                        [HTTP2_HEADER_METHOD]: 'POST',
+                        [HTTP2_HEADER_AUTHORIZATION]: this.authHeader
+                    });
 
-            postMessageStream.on('response', (headers, flags) => {
-                const status = headers[http2.constants.HTTP2_HEADER_STATUS];
-                if (status !== 200) {
-                    return reject(new Error(`Failed to upload message body, status code: ${status}`));
+                    let responseData = '';
+                    let statusCode = null;
+
+                    postMessageStream.on('response', (headers, flags) => {
+                        statusCode = headers[http2.constants.HTTP2_HEADER_STATUS];
+                        if (statusCode !== 200) {
+                            const error = new Error(`Failed to upload message body, status code: ${statusCode}`);
+                            error.statusCode = statusCode;
+                            return reject(error);
+                        }
+                    });
+
+                    postMessageStream.on('data', chunk => {
+                        responseData += chunk;
+                    });
+
+                    postMessageStream.on('error', (err) => {
+                        log.error(err, 'Error during upload message body');
+                        err.isNetworkError = true;
+                        reject(err);
+                    });
+
+                    postMessageStream.on('end', () => {
+                        if (!responseData) {
+                            return
+                        }
+                        try {
+                            const responseJson = JSON.parse(responseData);
+                            resolve(responseJson.objectId);
+                        } catch (e) {
+                            log.error(e, 'Failed to parse upload message body response');
+                            reject(e);
+                        }
+                    });
+
+                    const cipher = this._encryptor.createCipher();
+                    cipher.pipe(postMessageStream);
+                    cipher.write(bodyBuf);
+                    cipher.end();
+                });
+
+                // Success - return the objectId
+                if (attempt > 0) {
+                    log.info({ attempt, maxRetries }, 'Upload message body succeeded after retry');
                 }
-            });
-            let responseData = '';
-            postMessageStream.on('data', chunk => {
-                responseData += chunk;
-            });
-            postMessageStream.on('error', (err) => {
-                log.error(err, 'Error during upload message body');
-                reject(err);
-            });
-            postMessageStream.on('end', () => {
-                try {
-                    const responseJson = JSON.parse(responseData);
-                    resolve(responseJson.objectId);
-                } catch (e) {
-                    log.error(e, 'Failed to parse upload message body response');
-                    reject(e);
+                return objectId;
+
+            } catch (error) {
+                const isLastAttempt = attempt === maxRetries;
+                const isRetryable = error.isNetworkError ||
+                    (error.statusCode && error.statusCode >= 500) ||
+                    error.code === 'ECONNRESET' ||
+                    error.code === 'ETIMEDOUT';
+
+                if (!isRetryable || isLastAttempt) {
+                    log.error({
+                        attempt,
+                        maxRetries,
+                        error: error.message,
+                        isRetryable
+                    }, 'Upload message body failed, no more retries');
+                    throw error;
                 }
-            });
 
-            const cipher = this._encryptor.createCipher();
-            cipher.pipe(postMessageStream);
-            cipher.write(bodyBuf);
-            cipher.end();
-        });
+                const delay = Math.min(
+                    retryDelay * Math.pow(2, attempt),
+                    this.settings.PROXY_OBJECT_REQUEST_MAX_RETRY_DELAY
+                );
+                log.warn({
+                    attempt,
+                    maxRetries,
+                    error: error.message,
+                    nextRetryIn: delay
+                }, 'Upload message body failed, retrying...');
+
+                await new Promise(resolve => setTimeout(resolve, delay));
+
+                // Ensure connection is still valid before retry
+                if (!this.isConnected()) {
+                    log.info('Reconnecting before retry...');
+                    await this.connect();
+                }
+            }
+        }
     }
 
     async listenForMessages(messageHandler) {

package/lib/sailor.js

@@ -35,6 +35,8 @@ class Sailor {
         //eslint-disable-next-line new-cap
         this.apiClient = RestApiClient(
             settings.API_USERNAME,
+            // TODO find a way to make username and key consistent (without running api tests and looking up
+            // correct values in MongoDB)
             settings.API_KEY,
             {
                 retryCount: settings.API_REQUEST_RETRY_ATTEMPTS,

package/lib/settings.js

@@ -15,6 +15,9 @@ function getOptionalEnvVars(envVars) {
         PROXY_RECONNECT_MAX_DELAY: 30 * 1000, // 30 seconds
         PROXY_RECONNECT_BACKOFF_MULTIPLIER: 2,
         PROXY_RECONNECT_JITTER_FACTOR: 0.3,
+        PROXY_OBJECT_REQUEST_RETRY_ATTEMPTS: Infinity,
+        PROXY_OBJECT_REQUEST_RETRY_DELAY: 100,
+        PROXY_OBJECT_REQUEST_MAX_RETRY_DELAY: 5 * 60 * 1000, // 5 mins
 
         // TODO: Move to proxy?
         DATA_RATE_LIMIT: 10, // 10 data events every 100ms

package/mise.toml

@@ -0,0 +1,2 @@
+[tools]
+node = "18.16.1"

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "elasticio-sailor-nodejs",
   "description": "The official elastic.io library for bootstrapping and executing for Node.js connectors",
-  "version": "3.0.0-dev3",
+  "version": "3.0.0-dev5",
   "main": "run.js",
   "scripts": {
     "build": "tsc",

package/run.js

@@ -57,23 +57,6 @@ async function putOutToSea(settings, ipc) {
     ipc.send('init:ended');
 }
 
-async function disconnectAndExit() {
-    if (!disconnectRequired) {
-        return;
-    }
-    disconnectRequired = false;
-
-    try {
-        logger.info('Disconnecting...');
-        await sailor.disconnect();
-        logger.info('Successfully disconnected');
-        process.exit();
-    } catch (err) {
-        logger.error(err, 'Unable to disconnect');
-        process.exit(-1);
-    }
-}
-
 async function gracefulShutdown() {
     if (!disconnectRequired) {
         return;
@@ -90,8 +73,15 @@ async function gracefulShutdown() {
     await sailorInit;
     logger.trace('Waited an init before graceful shutdown');
 
-    await sailor.scheduleShutdown();
-    await disconnectAndExit();
+    try {
+        logger.info('Disconnecting...');
+        await sailor.scheduleShutdown();
+        logger.info('Successfully disconnected');
+        process.exit();
+    } catch (err) {
+        logger.error(err, 'Unable to disconnect');
+        process.exit(-1);
+    }
 }
 
 async function run(settings, ipc) {