elasticio-sailor-nodejs 3.0.0-dev4 → 3.0.0-dev6

package/config/local.json

@@ -1,16 +1,16 @@
 {
     "LOG_LEVEL": "trace",
-    "ELASTICIO_FLOW_ID": "697b56aca8e2e9b35e1952da",
-    "ELASTICIO_EXEC_ID": "exec_67890",
+    "ELASTICIO_FLOW_ID": "692ee23d4ab5d34bb7321559",
+    "ELASTICIO_EXEC_ID": "69b26526c4796609ca0da12a",
     "ELASTICIO_STEP_ID": "step_1",
-    "ELASTICIO_CONTAINER_ID": "container_abcde",
-    "ELASTICIO_WORKSPACE_ID": "workspace_xyz",
-    "ELASTICIO_USER_ID": "user_1122",
-    "ELASTICIO_COMP_ID": "component_3344",
+    "ELASTICIO_CONTAINER_ID": "69b26526c4796609ca0da129",
+    "ELASTICIO_WORKSPACE_ID": "69b26526c4796609ca0da128",
+    "ELASTICIO_USER_ID": "69b26526c4796609ca0da127",
+    "ELASTICIO_COMP_ID": "69b26526c4796609ca0da126",
     "ELASTICIO_FUNCTION": "data_trigger",
     "ELASTICIO_API_URI": "http://localhost:9000",
-    "ELASTICIO_API_USERNAME": "task-697b56aca8e2e9b35e1952da",
-    "ELASTICIO_API_KEY": "9b2e0465-b00c-4b57-af91-bfc3a830306c",
+    "ELASTICIO_API_USERNAME": "task-692ee23d4ab5d34bb7321559",
+    "ELASTICIO_API_KEY": "976ffec8-455b-494e-9478-2d66761f4040",
     "ELASTICIO_MESSAGE_CRYPTO_IV": "0.03091345790184",
     "ELASTICIO_MESSAGE_CRYPTO_PASSWORD": "password",
     "ELASTICIO_SAILOR_PROXY_URI": "http://localhost:4001",

package/lib/proxy-client.js

@@ -202,6 +202,7 @@ class ProxyClient {
                 return resolve();
             }
 
+            // TODO: what if incoming message is received during graceful shutdown?
             this.clientSession.close(() => {
                 log.debug('Successfully closed HTTP2 connection');
                 resolve();
@@ -237,62 +238,16 @@ class ProxyClient {
         throw new Error('Connection lost and no reconnection in progress');
     }
 
-    async fetchMessageBody(message, logger) {
-        await this._ensureConnection();
-
-        const { body, headers } = message;
-
-        logger.info('Checking if incoming messages is lightweight...');
-
-        if (!headers) {
-            logger.info('Empty headers so not lightweight.');
-            return body;
-        }
-
-        const { [OBJECT_ID_HEADER]: objectId } = headers;
-
-        if (!objectId) {
-            logger.trace('No object id header so not lightweight.');
-            return body;
-        }
-
-        logger.info('Object id header found, message is lightweight.', { objectId });
-
-        let object;
-
-        logger.info('Going to fetch message body.', { objectId });
-
+    async _proxyRequestWithRetries(operationName, requestFn) {
         const maxRetries = this.settings.PROXY_OBJECT_REQUEST_RETRY_ATTEMPTS;
         const retryDelay = this.settings.PROXY_OBJECT_REQUEST_RETRY_DELAY;
         for (let attempt = 0; attempt <= maxRetries; attempt++) {
             try {
-                object = await new Promise((resolve, reject) => {
-                    const getObjectStream = this.clientSession.request({
-                        [HTTP2_HEADER_PATH]: `/object/${objectId}`,
-                        [HTTP2_HEADER_METHOD]: 'GET',
-                        [HTTP2_HEADER_AUTHORIZATION]: this.authHeader
-                    }).pipe(this._encryptor.createDecipher());
-
-                    const chunks = [];
-                    getObjectStream.on('data', chunk => {
-                        chunks.push(chunk);
-                    });
-                    getObjectStream.on('error', (err) => {
-                        logger.error(err, 'Error during fetching message body');
-                        reject(err);
-                    });
-                    getObjectStream.on('end', () => {
-                        logger.info('Message stream ended by server');
-                        const buffer = Buffer.concat(chunks);
-                        logger.info({ messageSize: buffer.length }, 'Received complete message from server');
-                        resolve({ data: JSON.parse(buffer.toString()) });
-                    });
-                });
-
+                const result = await requestFn();
                 if (attempt > 0) {
-                    log.info({ attempt, maxRetries }, 'Fetch message body succeeded after retry');
+                    log.info({ attempt, maxRetries }, `${operationName} succeeded after retry`);
                 }
-                return objectId;
+                return result;
             } catch (error) {
                 const isLastAttempt = attempt === maxRetries;
                 const isRetryable = error.isNetworkError ||
@@ -306,7 +261,7 @@ class ProxyClient {
                         maxRetries,
                         error: error.message,
                         isRetryable
-                    }, 'Fetch message body failed, no more retries');
+                    }, `${operationName} failed, no more retries`);
                     throw error;
                 }
 
@@ -319,121 +274,120 @@ class ProxyClient {
                     maxRetries,
                     error: error.message,
                     nextRetryIn: delay
-                }, 'Fetch message body failed, retrying...');
+                }, `${operationName} failed, retrying...`);
 
                 await new Promise(resolve => setTimeout(resolve, delay));
 
-                // Ensure connection is still valid before retry
                 if (!this.isConnected()) {
                     log.info('Reconnecting before retry...');
                     await this.connect();
                 }
             }
         }
-        logger.info('Successfully obtained message body.', { objectId });
-        logger.trace('Message body object received');
-
-        return object.data;
     }
 
-    async uploadMessageBody(bodyBuf) {
+    async fetchMessageBody(message, logger) {
         await this._ensureConnection();
 
-        const maxRetries = this.settings.PROXY_OBJECT_REQUEST_RETRY_ATTEMPTS;
-        const retryDelay = this.settings.PROXY_OBJECT_REQUEST_RETRY_DELAY;
-        for (let attempt = 0; attempt <= maxRetries; attempt++) {
-            try {
-                const objectId = await new Promise((resolve, reject) => {
-                    const postMessageStream = this.clientSession.request({
-                        [HTTP2_HEADER_PATH]: '/object',
-                        [HTTP2_HEADER_METHOD]: 'POST',
-                        [HTTP2_HEADER_AUTHORIZATION]: this.authHeader
-                    });
+        const { body, headers } = message;
 
-                    let responseData = '';
-                    let statusCode = null;
+        logger.info('Checking if incoming messages is lightweight...');
 
-                    postMessageStream.on('response', (headers, flags) => {
-                        statusCode = headers[http2.constants.HTTP2_HEADER_STATUS];
-                        if (statusCode !== 200) {
-                            const error = new Error(`Failed to upload message body, status code: ${statusCode}`);
-                            error.statusCode = statusCode;
-                            return reject(error);
-                        }
-                    });
+        if (!headers) {
+            logger.info('Empty headers so not lightweight.');
+            return body;
+        }
 
-                    postMessageStream.on('data', chunk => {
-                        responseData += chunk;
-                    });
+        const { [OBJECT_ID_HEADER]: objectId } = headers;
 
-                    postMessageStream.on('error', (err) => {
-                        log.error(err, 'Error during upload message body');
-                        err.isNetworkError = true;
-                        reject(err);
-                    });
+        if (!objectId) {
+            logger.trace('No object id header so not lightweight.');
+            return body;
+        }
 
-                    postMessageStream.on('end', () => {
-                        if (!responseData) {
-                            return
-                        }
-                        try {
-                            const responseJson = JSON.parse(responseData);
-                            resolve(responseJson.objectId);
-                        } catch (e) {
-                            log.error(e, 'Failed to parse upload message body response');
-                            reject(e);
-                        }
-                    });
+        logger.info('Object id header found, message is lightweight.', { objectId });
 
-                    const cipher = this._encryptor.createCipher();
-                    cipher.pipe(postMessageStream);
-                    cipher.write(bodyBuf);
-                    cipher.end();
-                });
+        logger.info('Going to fetch message body.', { objectId });
 
-                // Success - return the objectId
-                if (attempt > 0) {
-                    log.info({ attempt, maxRetries }, 'Upload message body succeeded after retry');
-                }
-                return objectId;
+        await this._proxyRequestWithRetries('Fetch message body', () => new Promise((resolve, reject) => {
+            const getObjectStream = this.clientSession.request({
+                [HTTP2_HEADER_PATH]: `/object/${objectId}`,
+                [HTTP2_HEADER_METHOD]: 'GET',
+                [HTTP2_HEADER_AUTHORIZATION]: this.authHeader
+            }).pipe(this._encryptor.createDecipher());
 
-            } catch (error) {
-                const isLastAttempt = attempt === maxRetries;
-                const isRetryable = error.isNetworkError ||
-                    (error.statusCode && error.statusCode >= 500) ||
-                    error.code === 'ECONNRESET' ||
-                    error.code === 'ETIMEDOUT';
+            const chunks = [];
+            getObjectStream.on('data', chunk => {
+                chunks.push(chunk);
+            });
+            getObjectStream.on('error', (err) => {
+                logger.error(err, 'Error during fetching message body');
+                reject(err);
+            });
+            getObjectStream.on('end', () => {
+                logger.info('Message stream ended by server');
+                const buffer = Buffer.concat(chunks);
+                logger.info({ messageSize: buffer.length }, 'Received complete message from server');
+                resolve({ data: JSON.parse(buffer.toString()) });
+            });
+        }));
 
-                if (!isRetryable || isLastAttempt) {
-                    log.error({
-                        attempt,
-                        maxRetries,
-                        error: error.message,
-                        isRetryable
-                    }, 'Upload message body failed, no more retries');
-                    throw error;
+        logger.info('Successfully obtained message body.', { objectId });
+        logger.trace('Message body object received');
+
+        return objectId;
+    }
+
+    async uploadMessageBody(bodyBuf) {
+        await this._ensureConnection();
+
+        return this._proxyRequestWithRetries('Upload message body', () => new Promise((resolve, reject) => {
+            const postMessageStream = this.clientSession.request({
+                [HTTP2_HEADER_PATH]: '/object',
+                [HTTP2_HEADER_METHOD]: 'POST',
+                [HTTP2_HEADER_AUTHORIZATION]: this.authHeader
+            });
+
+            let responseData = '';
+            let statusCode = null;
+
+            postMessageStream.on('response', (headers, flags) => {
+                statusCode = headers[http2.constants.HTTP2_HEADER_STATUS];
+                if (statusCode !== 200) {
+                    const error = new Error(`Failed to upload message body, status code: ${statusCode}`);
+                    error.statusCode = statusCode;
+                    return reject(error);
                 }
+            });
 
-                const delay = Math.min(
-                    retryDelay * Math.pow(2, attempt),
-                    this.settings.PROXY_OBJECT_REQUEST_MAX_RETRY_DELAY
-                );
-                log.warn({
-                    attempt,
-                    maxRetries,
-                    error: error.message,
-                    nextRetryIn: delay
-                }, 'Upload message body failed, retrying...');
+            postMessageStream.on('data', chunk => {
+                responseData += chunk;
+            });
 
-                await new Promise(resolve => setTimeout(resolve, delay));
+            postMessageStream.on('error', (err) => {
+                log.error(err, 'Error during upload message body');
+                err.isNetworkError = true;
+                reject(err);
+            });
 
-                // Ensure connection is still valid before retry
-                if (!this.isConnected()) {
-                    log.info('Reconnecting before retry...');
-                    await this.connect();
+            postMessageStream.on('end', () => {
+                if (!responseData) {
+                    return
                 }
-            }
-        }
+                try {
+                    const responseJson = JSON.parse(responseData);
+                    resolve(responseJson.objectId);
+                } catch (e) {
+                    log.error(e, 'Failed to parse upload message body response');
+                    reject(e);
+                }
+            });
+
+            const cipher = this._encryptor.createCipher();
+            cipher.pipe(postMessageStream);
+            cipher.write(bodyBuf);
+            cipher.end();
+        }));
     }
 
     async listenForMessages(messageHandler) {
@@ -542,33 +496,36 @@ class ProxyClient {
             type,
             ...(customRoutingKey ? { customRoutingKey } : {})
         }).toString();
-        // TODO: Add retries
-        const postMessageStream = this.clientSession.request({
-            ...proxyHeaders,
-            [HTTP2_HEADER_PATH]: `/message?${queryParams}`,
-            [HTTP2_HEADER_METHOD]: 'POST',
-            [HTTP2_HEADER_AUTHORIZATION]: this.authHeader
-        });
-        postMessageStream.write(encryptedData);
-        postMessageStream.end();
 
-        return new Promise((resolve, reject) => {
+        await this._proxyRequestWithRetries('Send message', () => new Promise((resolve, reject) => {
+            const postMessageStream = this.clientSession.request({
+                ...proxyHeaders,
+                [HTTP2_HEADER_PATH]: `/message?${queryParams}`,
+                [HTTP2_HEADER_METHOD]: 'POST',
+                [HTTP2_HEADER_AUTHORIZATION]: this.authHeader
+            });
+            postMessageStream.write(encryptedData);
+            postMessageStream.end();
+
             postMessageStream.on('response', (headers) => {
                 log.debug({ status: headers[HTTP2_HEADER_STATUS] }, 'Send message response');
                 if (headers[HTTP2_HEADER_STATUS] !== 200) {
                     log.error({ headers }, 'Failed to send message');
-                    return reject(new Error(`Failed to send message, status code: ${headers[HTTP2_HEADER_STATUS]}`));
+                    const error = new Error(`Failed to send message, status code: ${headers[HTTP2_HEADER_STATUS]}`);
+                    error.statusCode = headers[HTTP2_HEADER_STATUS];
+                    return reject(error);
                 }
             });
             postMessageStream.on('error', (err) => {
                 log.error(err, 'Error during sending message');
+                err.isNetworkError = true;
                 reject(err);
             });
             postMessageStream.on('end', () => {
                 log.debug('Send message end event');
                 resolve();
             });
-        });
+        }));
     }
 
     _decodeMessage(originalMessage, headers) {
@@ -620,30 +577,33 @@ class ProxyClient {
             incomingMessageId,
             status
         }).toString();
-        // TODO: Add retries
-        const postMessageStream = this.clientSession.request({
-            [HTTP2_HEADER_PATH]: `/finish-processing?${queryParams}`,
-            [HTTP2_HEADER_METHOD]: 'POST',
-            [HTTP2_HEADER_AUTHORIZATION]: this.authHeader
-        });
-        postMessageStream.end();
 
-        return new Promise((resolve, reject) => {
+        await this._proxyRequestWithRetries('Finish processing', () => new Promise((resolve, reject) => {
+            const postMessageStream = this.clientSession.request({
+                [HTTP2_HEADER_PATH]: `/finish-processing?${queryParams}`,
+                [HTTP2_HEADER_METHOD]: 'POST',
+                [HTTP2_HEADER_AUTHORIZATION]: this.authHeader
+            });
+            postMessageStream.end();
+
             postMessageStream.on('response', (headers) => {
                 log.debug({ status: headers[HTTP2_HEADER_STATUS] }, 'Finish processing response event');
                 if (headers[HTTP2_HEADER_STATUS] !== 200) {
                     log.error({ headers }, 'Failed to finish processing message');
-                    return reject(new Error(`Failed to finish processing message, status code: ${headers[HTTP2_HEADER_STATUS]}`));
+                    const error = new Error(`Failed to finish processing message, status code: ${headers[HTTP2_HEADER_STATUS]}`);
+                    error.statusCode = headers[HTTP2_HEADER_STATUS];
+                    return reject(error);
                 }
             });
-
             postMessageStream.on('end', () => {
                 log.debug('Finish processing end event');
                 resolve();
             });
-
-            postMessageStream.on('error', reject);
-        });
+            postMessageStream.on('error', (err) => {
+                err.isNetworkError = true;
+                reject(err);
+            });
+        }));
     }
 
     encryptMessageContent(body, protocolVersion = 1) {

package/lib/sailor.js

@@ -80,6 +80,7 @@ class Sailor {
     }
 
     async disconnect() {
+        // TODO: delete if not needed (currently used only in old tests)
         log.debug('Disconnecting, %s messages in processing', this.messagesCount);
         return this.proxyClient.disconnect();
     }
@@ -194,7 +195,6 @@ class Sailor {
             return new Promise(resolve => this.shutdownCallback = resolve);
         }
 
-        // TODO: remove duplicate disconnect call (also in run.js)
         await this.proxyClient.disconnect();
         if (this.messagesCount === 0) {
             // there is no unfinished processMessage invocation, let's just resolve scheduleShutdown now

package/mise.toml

@@ -0,0 +1,2 @@
+[tools]
+node = "18.16.1"

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "elasticio-sailor-nodejs",
   "description": "The official elastic.io library for bootstrapping and executing for Node.js connectors",
-  "version": "3.0.0-dev4",
+  "version": "3.0.0-dev6",
   "main": "run.js",
   "scripts": {
     "build": "tsc",

package/run.js

@@ -57,23 +57,6 @@ async function putOutToSea(settings, ipc) {
     ipc.send('init:ended');
 }
 
-async function disconnectAndExit() {
-    if (!disconnectRequired) {
-        return;
-    }
-    disconnectRequired = false;
-
-    try {
-        logger.info('Disconnecting...');
-        await sailor.disconnect();
-        logger.info('Successfully disconnected');
-        process.exit();
-    } catch (err) {
-        logger.error(err, 'Unable to disconnect');
-        process.exit(-1);
-    }
-}
-
 async function gracefulShutdown() {
     if (!disconnectRequired) {
         return;
@@ -90,8 +73,15 @@ async function gracefulShutdown() {
     await sailorInit;
     logger.trace('Waited an init before graceful shutdown');
 
-    await sailor.scheduleShutdown();
-    await disconnectAndExit();
+    try {
+        logger.info('Disconnecting...');
+        await sailor.scheduleShutdown();
+        logger.info('Successfully disconnected');
+        process.exit();
+    } catch (err) {
+        logger.error(err, 'Unable to disconnect');
+        process.exit(-1);
+    }
 }
 
 async function run(settings, ipc) {