elasticio-sailor-nodejs 3.0.0-sailor-proxy-dev17 → 3.0.0-sailor-proxy-dev19

package/config/local.json

@@ -7,7 +7,7 @@
     "ELASTICIO_WORKSPACE_ID": "69b26526c4796609ca0da128",
     "ELASTICIO_USER_ID": "69b26526c4796609ca0da127",
     "ELASTICIO_COMP_ID": "69b26526c4796609ca0da126",
-    "ELASTICIO_FUNCTION": "rebound_trigger",
+    "ELASTICIO_FUNCTION": "data_trigger",
     "ELASTICIO_API_URI": "http://localhost:9000",
     "ELASTICIO_API_USERNAME": "task-692ee23d4ab5d34bb7321559",
     "ELASTICIO_API_KEY": "976ffec8-455b-494e-9478-2d66761f4040",

package/lib/proxy-client.js

@@ -17,7 +17,7 @@ const {
     NGHTTP2_NO_ERROR
 } = http2.constants;
 
-const MESSAGE_METADATA_QUERY_PARAM = 'message-metadata';
+const MESSAGE_METADATA_HEADER = 'message-metadata';
 const HEADER_ROUTING_KEY = 'x-eio-routing-key';
 const AMQP_HEADER_META_PREFIX = 'x-eio-meta-';
 const OBJECT_ID_HEADER = 'x-ipaas-object-storage-id';
@@ -155,6 +155,9 @@ class ProxyClient {
         log.warn({ reason, details }, 'Connection lost, initiating reconnection');
         this.reconnecting = true;
 
+        // Clean up existing message streams
+        this._cleanupMessageStreams();
+
         // Clean up the current session
         if (this.clientSession && !this.clientSession.destroyed) {
             this.clientSession.destroy();
@@ -228,7 +231,6 @@ class ProxyClient {
                 return resolve();
             }
 
-            // TODO: what if incoming message is received during graceful shutdown?
             this.clientSession.close(() => {
                 log.debug('Successfully closed HTTP2 connection');
                 resolve();
@@ -237,21 +239,24 @@ class ProxyClient {
     }
 
     async _ensureConnection() {
-        if (this.isConnected()) {
+        // Check if we have a valid connection (not just "isConnected" which includes reconnecting state)
+        if (!this.closed && this.clientSession && !this.clientSession.destroyed) {
             return;
         }
 
         if (this.reconnecting) {
-            // Wait for reconnection to complete
+            // Wait for reconnection to complete and ensure we have a valid session
             log.info('Waiting for reconnection to complete');
             const maxWait = 30000; // 30 seconds
             const startTime = Date.now();
-            while (this.reconnecting && (Date.now() - startTime) < maxWait) {
+            while ((this.reconnecting || !this.clientSession || this.clientSession.destroyed) &&
+                   !this.closed && (Date.now() - startTime) < maxWait) {
                 await new Promise(resolve => setTimeout(resolve, 100));
             }
 
-            if (!this.isConnected()) {
-                throw new Error('Failed to reconnect within timeout period');
+            // Double-check we have a valid session after waiting
+            if (this.closed || !this.clientSession || this.clientSession.destroyed) {
+                throw new Error('Failed to establish valid connection within timeout period');
             }
             return;
         }
@@ -269,6 +274,7 @@ class ProxyClient {
         const retryDelay = this.settings.PROXY_OBJECT_REQUEST_RETRY_DELAY;
         for (let attempt = 0; attempt <= maxRetries; attempt++) {
             try {
+                await this._ensureConnection();
                 const result = await requestFn();
                 if (attempt > 0) {
                     log.info({ attempt, maxRetries }, `${operationName} succeeded after retry`);
@@ -286,8 +292,9 @@ class ProxyClient {
                         attempt,
                         maxRetries,
                         error: error.message,
-                        isRetryable
-                    }, `${operationName} failed, no more retries`);
+                        errorCode: error.code,
+                        errorStatusCode: error.statusCode
+                    }, `${operationName} failed and will not be retried`);
                     throw error;
                 }
 
@@ -313,8 +320,6 @@ class ProxyClient {
     }
 
     async fetchMessageBody(message, logger) {
-        await this._ensureConnection();
-
         const { body, headers } = message;
 
         logger.info('Checking if incoming messages is lightweight...');
@@ -363,8 +368,6 @@ class ProxyClient {
     }
 
     async uploadMessageBody(bodyBuf) {
-        await this._ensureConnection();
-
         return this._proxyRequestWithRetries('Upload message body', () => new Promise((resolve, reject) => {
             const postMessageStream = this.clientSession.request({
                 [HTTP2_HEADER_PATH]: '/object',
@@ -427,7 +430,6 @@ class ProxyClient {
                         prefetch
                     }).toString();
                     log.info({ prefetch }, 'Requesting message from proxy');
-                    // TODO: what timeout is here? e.g. when flow is realtime - we might need to wait a long time...
                     const getMessageStream = this.clientSession.request({
                         [HTTP2_HEADER_PATH]: `/message?${queryParams}`,
                         [HTTP2_HEADER_METHOD]: 'GET',
@@ -435,61 +437,100 @@ class ProxyClient {
                     });
                     this.getMessageStreams.add(getMessageStream);
 
-                    const { headers, body } = await new Promise((resolve, reject) => {
-                        getMessageStream.on('response', (headers, flags) => {
-                            log.info({ headers, flags }, 'Connected to message stream');
-                            if (headers[HTTP2_HEADER_STATUS] !== 200) {
-                                return reject(new Error(`Failed to get message, status code: ${headers[HTTP2_HEADER_STATUS]}`));
-                            }
-                            const messageId = headers['x-message-id'];
-                            const chunks = [];
-                            getMessageStream.on('data', chunk => {
-                                chunks.push(chunk);
-                            });
-                            getMessageStream.on('end', () => {
-                                log.info('Message stream ended by server');
-                                const body = Buffer.concat(chunks);
-                                log.info({
-                                    messageId,
-                                    messageSize: body.length
-                                }, 'Received complete message from server');
-                                log.trace({ body: body.toString() }, 'Message body as string');
-                                resolve({ headers, body });
+                    try {
+                        const { headers, body } = await new Promise((resolve, reject) => {
+                            getMessageStream.on('response', (headers, flags) => {
+                                log.info({ headers, flags }, 'Connected to message stream');
+                                if (headers[HTTP2_HEADER_STATUS] === 204) {
+                                    log.info('Received empty message stream, closing stream and going to request again');
+                                    getMessageStream.close(NGHTTP2_NO_ERROR);
+                                    return;
+                                } else if (headers[HTTP2_HEADER_STATUS] !== 200) {
+                                    return reject(new Error(`Failed to get message, status code: ${headers[HTTP2_HEADER_STATUS]}`));
+                                }
+                                const chunks = [];
+                                getMessageStream.on('data', chunk => {
+                                    chunks.push(chunk);
+                                });
+                                getMessageStream.on('end', () => {
+                                    log.info('Message stream ended by server');
+                                    const body = Buffer.concat(chunks);
+                                    log.info({
+                                        messageId: headers.messageId,
+                                        messageSize: body.length
+                                    }, 'Received complete message from server');
+                                    log.trace({ body: body.toString() }, 'Message body as string');
+                                    resolve({ headers, body });
+                                });
                             });
-                        });
 
-                        getMessageStream.on('close', () => {
-                            log.warn('Message stream closed by server');
-                            reject(new Error('Message stream closed by server'));
-                        });
-                        getMessageStream.on('error', (err) => {
-                            log.error(err, 'Error on message stream');
-                            reject(err);
+                            getMessageStream.on('close', () => {
+                                log.warn('Message stream closed by server');
+                                if (getMessageStream.rstCode !== NGHTTP2_NO_ERROR) {
+                                    reject(new Error('Message stream closed by server'));
+                                } else {
+                                    resolve({ headers: null, body: null });
+                                }
+                            });
+                            getMessageStream.on('error', (err) => {
+                                log.error(err, 'Error on message stream');
+                                reject(err);
+                            });
                         });
-                    });
-
-                    const messageMetadata = this._extractMessageMetadata(headers);
-                    const message = this._decodeMessage(body, messageMetadata);
-                    log.debug({ messageMetadata, message }, 'Processing received message');
-                    await messageHandler(messageMetadata, message);
+                        if (headers === null && body === null) {
+                            // Stream was closed without data (e.g. 204 No Content), just return to listen for the next messageMetadata
+                            return;
+                        }
+
+                        const messageMetadata = this._extractMessageMetadata(headers);
+                        const message = this._decodeMessage(body, messageMetadata);
+                        log.debug({ messageMetadata, message }, 'Processing received message');
+                        await messageHandler(messageMetadata, message);
+                    } finally {
+                        // Remove this specific stream from the tracking set
+                        this.getMessageStreams.delete(getMessageStream);
+                        // Ensure the stream is properly closed
+                        if (!getMessageStream.closed && !getMessageStream.destroyed) {
+                            try {
+                                getMessageStream.destroy();
+                            } catch (err) {
+                                log.debug({ err }, 'Error closing message stream');
+                            }
+                        }
+                    }
                 }));
             } catch (err) {
+                log.error(err, 'Error while listening for messages');
                 if (this.closed) {
                     log.info('Connection closed, stopping message listener');
                     break;
                 }
+                // Clean up any invalid streams from the error
+                this._cleanupMessageStreams();
+            } finally {
+                // Note: _cleanupMessageStreams() is called in the catch block for errors
+                // and in _handleDisconnection() for disconnections, so we only clear here
+                // for normal completion (which shouldn't happen in the infinite loop)
+                if (this.closed || !this.listeningForMessages) {
+                    log.debug('Cleaning up message streams on exit');
+                    this._cleanupMessageStreams();
+                }
+            }
+        }
+    }
 
-                if (!this.reconnecting) {
-                    log.error(err, 'Error in listenForMessages, waiting for reconnection');
-                } else {
-                    log.debug('Currently reconnecting, will retry listening for messages after reconnection');
+    _cleanupMessageStreams() {
+        log.debug({ streamCount: this.getMessageStreams.size }, 'Cleaning up message streams due to disconnection');
+        for (const stream of this.getMessageStreams) {
+            if (stream && !stream.closed && !stream.destroyed) {
+                try {
+                    stream.destroy();
+                } catch (err) {
+                    log.debug({ err }, 'Error destroying message stream');
                 }
-                await new Promise(resolve => setTimeout(resolve, 1000));
-            } finally {
-                log.debug('Cleaning up message streams');
-                this.getMessageStreams.clear();
             }
         }
+        this.getMessageStreams.clear();
     }
 
     async stopListeningForMessages() {
@@ -522,6 +563,7 @@ class ProxyClient {
             ...metadata,
             messageId: metadata.messageId || uuid.v4()
         };
+        delete preparedMetadata.reboundIteration;
         let preparedData = data;
         if (preparedData && preparedData.headers) {
             preparedData.headers = _.omitBy(
@@ -546,8 +588,6 @@ class ProxyClient {
         metadata,
         forceProtocolVersion
     }) {
-        await this._ensureConnection();
-
         const throttledSend = this.throttles[type];
         if (throttledSend) {
             log.debug({ incomingMessageId, type, metadata }, 'Applying rate limiting for message send');
@@ -584,7 +624,7 @@ class ProxyClient {
                 [HTTP2_HEADER_PATH]: `/message?${queryParams}`,
                 [HTTP2_HEADER_METHOD]: 'POST',
                 [HTTP2_HEADER_AUTHORIZATION]: this.authHeader,
-                [MESSAGE_METADATA_QUERY_PARAM]: JSON.stringify(preparedMetadata)
+                [MESSAGE_METADATA_HEADER]: JSON.stringify(preparedMetadata)
             });
             if (preparedData) {
                 postMessageStream.write(preparedData);
@@ -652,16 +692,15 @@ class ProxyClient {
     }
 
     async finishProcessing(metadata, status) {
-        await this._ensureConnection();
-
         if (Object.values(MESSAGE_PROCESSING_STATUS).indexOf(status) === -1) {
             throw new Error(`Invalid message processing status: ${status}`);
         }
-        const incomingMessageId = metadata.messageId;
-        log.debug({ incomingMessageId, status }, 'Finishing processing of message');
+        const { messageId: incomingMessageId, reboundIteration } = metadata;
+        log.debug({ incomingMessageId, status, reboundIteration }, 'Finishing processing of message');
         const queryParams = new URLSearchParams({
             incomingMessageId,
-            status
+            status,
+            ...(typeof reboundIteration !== 'undefined' ? { reboundIteration } : {})
         }).toString();
 
         await this._proxyRequestWithRetries('Finish processing', () => new Promise((resolve, reject) => {
@@ -703,8 +742,6 @@ class ProxyClient {
     }
 
     async sendError(err, outgoingMetadata, originalMessage, metadata) {
-        await this._ensureConnection();
-
         const encryptedError = this._encryptor.encryptMessageContent({
             name: err.name,
             message: err.message,
@@ -742,8 +779,6 @@ class ProxyClient {
     }
 
     async sendRebound(reboundError, metadata, outgoingMetadata) {
-        await this._ensureConnection();
-
         outgoingMetadata.end = new Date().getTime();
         outgoingMetadata.reboundReason = reboundError.message;
         return this.sendMessage({
@@ -754,8 +789,6 @@ class ProxyClient {
     }
 
     async sendSnapshot(data, metadata) {
-        await this._ensureConnection();
-
         const payload = JSON.stringify(data);
         return this.sendMessage({
             type: 'snapshot',
@@ -766,20 +799,20 @@ class ProxyClient {
 
     _extractMessageMetadata(headers) {
         log.trace({ headers }, 'Extracting message metadata');
-        const messageMetadataQuery = headers[MESSAGE_METADATA_QUERY_PARAM];
-        if (!messageMetadataQuery) {
+        const messageMetadata = headers[MESSAGE_METADATA_HEADER];
+        if (!messageMetadata) {
             log.error({ headers }, 'Missing metadata in message stream response');
             throw new Error('Missing metadata in message stream response');
         }
 
         let parsedMessageMetadata;
         try {
-            parsedMessageMetadata = JSON.parse(messageMetadataQuery);
+            parsedMessageMetadata = JSON.parse(messageMetadata);
         } catch (e) {
-            log.error({ messageMetadataQuery }, 'Failed to parse metadata JSON');
+            log.error({ messageMetadata: messageMetadata }, 'Failed to parse metadata JSON');
             throw new Error('Failed to parse metadata JSON');
         }
-        log.debug({ parsedMessageMetadata }, 'Parsed metadata from query parameter');
+        log.debug({ parsedMessageMetadata }, 'Parsed metadata from header');
 
         // Get meta headers
         const metaHeaderNames = Object.keys(parsedMessageMetadata)
@@ -804,6 +837,9 @@ class ProxyClient {
         if (parsedMessageMetadata.reply_to) {
             result.reply_to = parsedMessageMetadata.reply_to;
         }
+        if (parsedMessageMetadata.reboundIteration) {
+            result.reboundIteration = parsedMessageMetadata.reboundIteration;
+        }
         log.debug({ result }, 'Extracted message metadata');
         return result;
     }

package/lib/settings.js

@@ -19,17 +19,10 @@ function getOptionalEnvVars(envVars) {
         PROXY_OBJECT_REQUEST_RETRY_DELAY: 100,
         PROXY_OBJECT_REQUEST_MAX_RETRY_DELAY: 5 * 60 * 1000, // 5 mins
 
-        // TODO: Move to proxy?
         DATA_RATE_LIMIT: 10, // 10 data events every 100ms
         ERROR_RATE_LIMIT: 2, // 2 errors every 100ms
         SNAPSHOT_RATE_LIMIT: 2, // 2 Snapshots every 100ms
         RATE_INTERVAL: 100, // 100ms
-        PROCESS_AMQP_DRAIN: true,
-        AMQP_PUBLISH_RETRY_DELAY: 100, // 100ms
-        AMQP_PUBLISH_RETRY_ATTEMPTS: Infinity,
-        AMQP_PUBLISH_MAX_RETRY_DELAY: 5 * 60 * 1000, // 5 mins
-        // Should be defaulted to true and moved to proxy
-        AMQP_PERSISTENT_MESSAGES: false,
 
         OBJECT_STORAGE_SIZE_THRESHOLD: 1048576,
         OUTGOING_MESSAGE_SIZE_LIMIT: 10485760,

package/package.json

@@ -1,7 +1,7 @@
 {
     "name": "elasticio-sailor-nodejs",
     "description": "The official elastic.io library for bootstrapping and executing for Node.js connectors",
-    "version": "3.0.0-sailor-proxy-dev17",
+    "version": "3.0.0-sailor-proxy-dev19",
     "main": "run.js",
     "scripts": {
         "build": "tsc",