elasticio-sailor-nodejs 2.7.6-dev.2 → 2.7.7-dev1

package/CHANGELOG.md

@@ -1,3 +1,8 @@
+## 2.7.7 (August 5, 2025)
+
+* Improve handling of cases when connection to RabbitMQ is re-established.
+[#7855](https://github.com/elasticio/elasticio/issues/7855)
+
 ## 2.7.6 (August 1, 2025)
 
 * Updated `elasticio-rest-node` to version 2.0.0 to address a vulnerability

package/lib/amqp.js

@@ -6,6 +6,8 @@ const _ = require('lodash');
 const eventToPromise = require('event-to-promise');
 const uuid = require('uuid');
 const os = require('os');
+const messagesDB = require('./messagesDB.js');
+const assert = require('assert');
 
 const HEADER_ROUTING_KEY = 'x-eio-routing-key';
 const HEADER_ERROR_RESPONSE = 'x-eio-error-response';
@@ -215,14 +217,14 @@ class Amqp {
             } catch (err) {
                 log.error({ err, deliveryTag: amqpMessage.fields.deliveryTag },
                     'Error occurred while parsing message payload');
-                this.reject(amqpMessage);
+                this.rejectOriginal(amqpMessage);
                 return;
             }
             try {
                 await messageHandler(message, amqpMessage);
             } catch (err) {
                 log.error({ err, deliveryTag: amqpMessage.fields.deliveryTag }, 'Failed to process message, reject');
-                this.reject(amqpMessage);
+                this.rejectOriginal(amqpMessage);
             }
         });
         log.debug({ queue }, 'Started listening for messages');
@@ -289,7 +291,10 @@ class Amqp {
                 'Error occurred while parsing message #%j payload',
                 message.fields.deliveryTag
             );
-            return this.reject(message);
+            // I didn't find a quick way to use reject that waits for new message when
+            // connection is re-established, so simply try to reject original message
+            // This will be fixed when we implement to Sailor Proxy
+            return this.rejectOriginal(message);
         }
         decryptedContent.headers = decryptedContent.headers || {};
         if (message.properties.headers.reply_to) {
@@ -301,16 +306,51 @@ class Amqp {
             callback(decryptedContent, message);
         } catch (err) {
             log.error(err, 'Failed to process message #%j, reject', message.fields.deliveryTag);
-            return this.reject(message);
+            // I didn't find a quick way to use reject that waits for new message when
+            // connection is re-established, so simply try to reject original message
+            // This will be fixed when we implement to Sailor Proxy
+            return this.rejectOriginal(message);
         }
     }
 
-    ack(message) {
-        log.debug('Message #%j ack', message.fields.deliveryTag);
+    async getMostRecentMessage(messageId) {
+        let message = messagesDB.getMessageById(messageId);
+        assert(message, `Message with ID ${messageId} not found in messagesDB`);
+        // If we stopped listening fo the new messages or the message was received
+        // from the same channel we can return it immediately
+        if (!this.consume || message.fields.consumerTag === this.consume.consumerTag) {
+            return message;
+        }
+        log.debug({ messageId }, 'Waiting for message from new channel');
+        message = await new Promise((resolve) => {
+            const onMessageUpdated = (updatedMessageId, updatedMessage) => {
+                if (updatedMessageId === messageId
+                    && updatedMessage.fields.consumerTag === this.consume.consumerTag) {
+                    messagesDB.off('message-updated', onMessageUpdated);
+                    resolve(updatedMessage);
+                }
+            };
+            messagesDB.on('message-updated', onMessageUpdated);
+        });
+        log.debug({ messageId }, 'Message received from new channel');
+        return message;
+    }
+
+    async ack(messageId) {
+        const message = await this.getMostRecentMessage(messageId);
+        log.debug(message.fields, 'Message ack');
         this.consumerChannel.ack(message);
+        messagesDB.deleteMessage(messageId);
+    }
+
+    async reject(messageId) {
+        const message = await this.getMostRecentMessage(messageId);
+        log.debug(message.fields, 'Message reject');
+        this.consumerChannel.reject(message, false);
+        messagesDB.deleteMessage(messageId);
     }
 
-    reject(message) {
+    rejectOriginal(message) {
         log.debug('Message #%j reject', message.fields.deliveryTag);
         return this.consumerChannel.reject(message, false);
     }

package/lib/messagesDB.js

@@ -0,0 +1,37 @@
+// Simple map to store messages by their IDs
+// This is useful when connection is re-established and we need to get the same
+// message again, but now from the new connection
+const EventEmitter = require('events');
+
+const messagesDB = (() => {
+    const messagesById = new Map();
+    const emitter = new EventEmitter();
+
+    return {
+        getMessageById: function getMessageById(id) {
+            return messagesById.get(id);
+        },
+        addMessage: function addMessage(id, message) {
+            const existingMessage = messagesById.get(id);
+            messagesById.set(id, message);
+            if (existingMessage) {
+                emitter.emit('message-updated', id, message);
+            }
+        },
+        deleteMessage: function deleteMessage(id) {
+            messagesById.delete(id);
+        },
+        on: function on(event, listener) {
+            emitter.on(event, listener);
+        },
+        off: function off(event, listener) {
+            emitter.off(event, listener);
+        },
+        __reset__: function reset() {
+            messagesById.clear();
+            emitter.removeAllListeners();
+        }
+    };
+})();
+
+module.exports = messagesDB;

package/lib/sailor.js

@@ -12,6 +12,7 @@ const co = require('co');
 const pThrottle = require('p-throttle');
 const { ObjectStorage } = require('@elastic.io/maester-client');
 const { Readable } = require('stream');
+const messagesDB = require('./messagesDB.js');
 
 const AMQP_HEADER_META_PREFIX = 'x-eio-meta-';
 const OBJECT_ID_HEADER = 'x-ipaas-object-storage-id';
@@ -104,7 +105,7 @@ class Sailor {
         return this.amqpConnection.disconnect();
     }
 
-    reportError(err) {
+    async reportError(err) {
         const headers = Object.assign({}, getAdditionalHeadersFromSettings(this.settings), {
             execId: this.settings.EXEC_ID,
             taskId: this.settings.FLOW_ID,
@@ -306,6 +307,7 @@ class Sailor {
     async runExec(module, payload, message, outgoingMessageHeaders, stepData, timeStart, logger) {
         const origPassthrough = _.cloneDeep(payload.passthrough) || {};
         const incomingMessageHeaders = this.readIncomingMessageHeaders(message);
+        const messageId = incomingMessageHeaders.messageId;
         const settings = this.settings;
         const cfg = _.cloneDeep(stepData.config) || {};
         const snapshot = _.cloneDeep(this.snapshot);
@@ -532,7 +534,7 @@ class Sailor {
                 }
             }
 
-            function onEnd() {
+            async function onEnd() {
                 if (endWasEmitted) {
                     logger.warn({
                         messagesCount: that.messagesCount,
@@ -545,9 +547,9 @@ class Sailor {
                 endWasEmitted = true;
 
                 if (taskExec.errorCount > 0) {
-                    that.amqpConnection.reject(message);
+                    await that.amqpConnection.reject(messageId);
                 } else {
-                    that.amqpConnection.ack(message);
+                    await that.amqpConnection.ack(messageId);
                 }
                 that.messagesCount -= 1;
                 logger.trace({
@@ -586,15 +588,34 @@ class Sailor {
         self.messagesCount += 1;
 
         const timeStart = Date.now();
-        const { deliveryTag } = message.fields;
 
+        const messageId = incomingMessageHeaders.messageId;
         const logger = log.child({
             threadId: incomingMessageHeaders.threadId || 'unknown',
-            messageId: incomingMessageHeaders.messageId || 'unknown',
+            messageId: messageId || 'unknown',
             parentMessageId: incomingMessageHeaders.parentMessageId || 'unknown',
-            deliveryTag
+            ...message.fields
         });
 
+        if (messageId) {
+            const alreadyExists = messagesDB.getMessageById(messageId);
+            // Add message to DB even if it already exists
+            messagesDB.addMessage(messageId, message);
+            if (alreadyExists) {
+                logger.warn({ messageId }, 'Duplicate message detected. This'
+                    + ' delivery will be ignored; the handler that first received'
+                    + ' this message will process it as part of deduplication.');
+                // If message was in messagesDB, it means that the connection was closed
+                // and this message was redelivered. In this case, the process for original
+                // message is waiting for this message to be added to DB and then ack or
+                // nack the new message, instead of the one that was delivered by closed
+                // channel
+                return;
+            }
+        } else {
+            logger.warn('Message does not have messageId');
+        }
+
         logger.trace({ messagesCount: this.messagesCount }, 'processMessage received');
 
         const stepData = this.stepData;
@@ -623,8 +644,10 @@ class Sailor {
         } catch (e) {
             log.error(e);
             outgoingMessageHeaders.end = new Date().getTime();
-            self.amqpConnection.sendError(e, outgoingMessageHeaders, message);
-            self.amqpConnection.reject(message);
+            await Promise.all([
+                self.amqpConnection.sendError(e, outgoingMessageHeaders, message),
+                self.amqpConnection.reject(messageId)
+            ]);
             return;
         }
 
@@ -652,8 +675,10 @@ class Sailor {
             } catch (e) {
                 logger.error(e);
                 outgoingMessageHeaders.end = new Date().getTime();
-                self.amqpConnection.sendError(e, outgoingMessageHeaders, message);
-                self.amqpConnection.reject(message);
+                await Promise.all([
+                    self.amqpConnection.sendError(e, outgoingMessageHeaders, message),
+                    self.amqpConnection.reject(messageId)
+                ]);
                 return;
             }
         }

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "elasticio-sailor-nodejs",
   "description": "The official elastic.io library for bootstrapping and executing for Node.js connectors",
-  "version": "2.7.6-dev.2",
+  "version": "2.7.7-dev1",
   "main": "run.js",
   "scripts": {
     "audit": "better-npm-audit audit --level high --production",
@@ -21,7 +21,7 @@
     "bunyan": "1.8.10",
     "co": "4.6.0",
     "debug": "3.1.0",
-    "elasticio-rest-node": "2.0.0-dev.3",
+    "elasticio-rest-node": "2.0.0",
     "event-to-promise": "0.8.0",
     "lodash": "4.17.21",
     "p-throttle": "2.1.0",