@ti-engine/core 1.2.4 → 1.3.1

package/CHANGELOG.md

@@ -2,6 +2,32 @@
 
 This document will contain the list of changes made to the framework. The format is based on the [Conventional Commits](https://www.conventionalcommits.org/en/v1.0.0/) specification.
 
+## Version 1.3.1
+* feat(service caller): refactor the entire service call execution flow for clarity and better performance
+* feat(service caller): create a new private class `ServiceCallProcessor` to handle individual service calls in a contained scope
+* feat(message observer): add new property `priority` to the message observer class. It is used to determine the order in which the observers are notified about the messages
+* feat(message observer)!: change method `onMessage` to now return the message it received. This allows for the message to be modified before it is passed to the next observer based on `priority`
+* feat(message handler)!: change method `onMessage` to `notifyMessageObservers` for clarity. It now implements the `MessageObserver` functionality for prioritization and message modification
+* fix(service caller): fix an issue which did not allow for a service call to be marked as completed thus being entered into the trace log as still pending
+* fix(redis integration): remove hardcoded `#retryMaxAttempts` value in `#setupClient` method
+* fix(redis integration): fix an issue which was setting the client status to `DISCONNECTED` during a normal shut down procedure
+
+## Version 1.3.0
+* feat(redis integration)!: change `reconnectOnError` behavior to also resubmit the failed command in case the error was of type `READONLY`
+* feat(redis integration): improve the reliability and usage of the event notification mechanism for connection observers
+* feat(redis integration): implement listener to the `end` event on Redis connection to capture when connection can no longer be recovered
+* feat(redis integration): notify connection observers `onConnectionLost` event
+* feat(redis integration): add redis client platform-specific status. It is used internally by the platform and can also be accessed via `redisClient.clientStatus` property
+* feat(connection observer): add new event handler `onConnectionLost` that will be invoked when the observed connection is irrevocably lost
+* feat(cache)!: implement `onConnectionLost` handler that will cause the service instance to immediately stop since it cannot work without the cache
+* fix(service caller): fix multiple promise reject condition when a service call timed out and the service handler still attempted to complete with subsequent error
+* fix(service instance): limit health check reporting to one attempt at a time to avoid unnecessary cache requests and log spam
+* fix(redis integration): fix a multiple promise resolve condition on Redis `reconnect` event
+* fix(redis integration): fix broken event propagation on `disrupted` events to some connection observers
+
+## Version 1.2.5
+* feat(cache): extend method `expireValue` to work with has set fields as well
+
 ## Version 1.2.4
 * feat(cache): expose the cache module as export in `package.json`
 * feat(cache): add method `hashDeleteField` to remove a hash-set field. This implements the `hdel` Redis command

package/bin/localization/labels.json

@@ -1,65 +1,65 @@
-{
-  "labels": {
-    "general": {
-      "exceptions": {
-        "0": {
-          "en": "Unidentified error encountered or unrecognized exception code provided."
-        },
-        "1000": {
-          "en": "Error thrown by internal JS source."
-        },
-        "1001": {
-          "en": "Attempt to construct an abstract class detected."
-        },
-        "1002": {
-          "en": "Attempt to call an abstract method detected."
-        },
-        "1003": {
-          "en": "Invalid or no service domain name provided at microservice startup."
-        },
-        "1004": {
-          "en": "The system cache required for proper engine operation is unavailable."
-        },
-        "1005": {
-          "en": "The provided service handler is not a proper function."
-        },
-        "1006": {
-          "en": "The requested feature is not supported by current configuration or version."
-        },
-        "2000": {
-          "en": "Invalid authorization token provided."
-        },
-        "2001": {
-          "en": "Invalid or expired session encountered."
-        },
-        "2002": {
-          "en": "Attempt for unauthorized access detected."
-        },
-        "2003": {
-          "en": "The system detected tampering with the message received via message exchange."
-        },
-        "3000": {
-          "en": "General error during cross-application communication."
-        },
-        "3001": {
-          "en": "The message sender instance is currently unavailable."
-        },
-        "3002": {
-          "en": "The execution of a service could not complete within the allowed timeout."
-        },
-        "3003": {
-          "en": "The specified service is not found in the service registry."
-        },
-        "3004": {
-          "en": "The specified service is not found in the service definition interface."
-        },
-        "3005": {
-          "en": "No handler found in the interface for the specified service or service version."
-        },
-        "3010": {
-          "en": "Connection retry attempts exceeded the configured limit."
-        }
-      }
-    }
-  }
+{
+  "labels": {
+    "general": {
+      "exceptions": {
+        "0": {
+          "en": "Unidentified error encountered or unrecognized exception code provided."
+        },
+        "1000": {
+          "en": "Error thrown by internal JS source."
+        },
+        "1001": {
+          "en": "Attempt to construct an abstract class detected."
+        },
+        "1002": {
+          "en": "Attempt to call an abstract method detected."
+        },
+        "1003": {
+          "en": "Invalid or no service domain name provided at microservice startup."
+        },
+        "1004": {
+          "en": "The system cache required for proper engine operation is unavailable."
+        },
+        "1005": {
+          "en": "The provided service handler is not a proper function."
+        },
+        "1006": {
+          "en": "The requested feature is not supported by current configuration or version."
+        },
+        "2000": {
+          "en": "Invalid authorization token provided."
+        },
+        "2001": {
+          "en": "Invalid or expired session encountered."
+        },
+        "2002": {
+          "en": "Attempt for unauthorized access detected."
+        },
+        "2003": {
+          "en": "The system detected tampering with the message received via message exchange."
+        },
+        "3000": {
+          "en": "General error during cross-application communication."
+        },
+        "3001": {
+          "en": "The message sender instance is currently unavailable."
+        },
+        "3002": {
+          "en": "The execution of a service could not complete within the allowed timeout."
+        },
+        "3003": {
+          "en": "The specified service is not found in the service registry."
+        },
+        "3004": {
+          "en": "The specified service is not found in the service definition interface."
+        },
+        "3005": {
+          "en": "No handler found in the interface for the specified service or service version."
+        },
+        "3010": {
+          "en": "Connection retry attempts exceeded the configured limit."
+        }
+      }
+    }
+  }
 }

package/bin/start-instance.js

@@ -151,7 +151,7 @@ try {
             setImmediate( () => process.exit( 1 ) );
         } );
     } else {
-        logger.log( `Attempting to start a module that does not implement the ServiceInstance abstract class!`, logger.logSeverity.ERROR );
+        logger.log( `Attempting to start a module that does not implement the ServiceInstance abstract class!`, logger.logSeverity.CRITICAL );
         setImmediate( () => process.exit( 1 ) );
     }
 } catch ( error ) {

package/components/connection-observer.js

@@ -1,6 +1,6 @@
 /*
  * The ti-engine is an open source, free to use—both for personal and commercial projects—framework for the creation of microservice-based solutions using node.js.
- * Copyright © 2021-2023 Boris Kostadinov <kostadinov.boris@gmail.com>
+ * Copyright © 2021-2025 Boris Kostadinov <kostadinov.boris@gmail.com>
  * This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
  * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
  * You should have received a copy of the GNU General Public License along with this program. If not, see <https://www.gnu.org/licenses/>.
@@ -37,7 +37,8 @@ class ConnectionObserver {
      * @virtual
      * @public
      */
-    onConnectionDisrupted( identifier ) { }
+    onConnectionDisrupted( identifier ) {
+    }
 
     /**
      * Needs to be invoked by the connection handler when the connection is recovered.
@@ -49,8 +50,22 @@ class ConnectionObserver {
      * @virtual
      * @public
      */
-    onConnectionRecovered( identifier ) { }
+    onConnectionRecovered( identifier ) {
+    }
+
+    /**
+     * Needs to be invoked by the connection handler when the connection is irrevocably lost.
+     * <br/>
+     * NOTE: Override this to add custom functionality.
+     *
+     * @method
+     * @param {string} identifier The identifier of the observed connection.
+     * @virtual
+     * @public
+     */
+    onConnectionLost( identifier ) {
+    }
 
 }
 
-module.exports = ConnectionObserver;
+module.exports = ConnectionObserver;

package/components/exchange/message-exchange.js

@@ -74,7 +74,7 @@ class MessageExchange extends MessageObserver {
      * @param {string} serviceDomainName The domain name of the microservice using the message exchange.
      */
     constructor( instanceID, serviceDomainName ) {
-        super();
+        super( 9 );
 
         // make sure this abstract class cannot be instantiated:
         if ( new.target === MessageExchange ) {
@@ -394,12 +394,25 @@ class MessageExchange extends MessageObserver {
         _.pull( this.#disruptedConnections, identifier );
     }
 
+    /**
+     * Used to mark the connection with the provided identifier as disrupted.
+     *
+     * @method
+     * @param {string} identifier The identifier of the observed connection.
+     * @override
+     * @public
+     */
+    onConnectionLost( identifier ) {
+        // TODO: implement triggering of a graceful shutdown of the service instance
+    }
+
     /**
      * Used only for the purposes of the message tracer.
      *
      * @method
      * @param {string} identifier The identifier of the observed connection.
      * @param {Message} message The message for processing.
+     * @returns {Message} The message that was received.
      * @override
      * @public
      */
@@ -411,6 +424,8 @@ class MessageExchange extends MessageObserver {
         } else if ( MessageExchange.connectionNameResponsesIn === identifier ) {
             messageTracer.instance.recordTraceEntry( message, messageTracer.messageType.MESSAGE_RESPONSE, messageTracer.dispatchEvent.RECEIVED, messageTracer.messageState.PROCESSED );
         }
+
+        return message;
     }
 
     /**

package/components/exchange/message-handler.js

@@ -54,7 +54,9 @@ class MessageHandler extends ConnectionObserver {
      * @returns {boolean}
      * @public
      */
-    get isAvailable() { return this.#isAvailable; }
+    get isAvailable() {
+        return this.#isAvailable;
+    }
 
     /**
      * Used to set the isAvailable flag.
@@ -65,7 +67,9 @@ class MessageHandler extends ConnectionObserver {
      * @param {boolean} value
      * @public
      */
-    set isAvailable( value ) { this.#isAvailable = value; }
+    set isAvailable( value ) {
+        this.#isAvailable = value;
+    }
 
     /**
      * Returns the connection identifier.
@@ -74,7 +78,9 @@ class MessageHandler extends ConnectionObserver {
      * @returns {string}
      * @public
      */
-    get connectionIdentifier() { return this.#connectionIdentifier; }
+    get connectionIdentifier() {
+        return this.#connectionIdentifier;
+    }
 
     /**
      * Used to initialize and enable the communication capabilities of the handler.
@@ -128,9 +134,9 @@ class MessageHandler extends ConnectionObserver {
      */
     addMessageObserver( messageObserver ) {
         const MessageObserver = require( "#message-observer" );
-
         if ( messageObserver instanceof MessageObserver ) {
             this.#messageObservers.push( messageObserver );
+            this.#messageObservers = _.orderBy( this.#messageObservers, [ "priority" ], [ "desc" ] );
         } else {
             logger.log( `Attempting to add '${ messageObserver.constructor.name }' as message observer but it's not a child-class of 'MessageObserver'!`, logger.logSeverity.WARNING );
         }
@@ -138,19 +144,23 @@ class MessageHandler extends ConnectionObserver {
 
     /**
      * An event-triggered method that will notify any observers about a new message for handling.
+     * <br/>
+     * NOTE: Each observer will be notified in the order of their priority via their {@link MessageObserver.onMessage} method. Additionally, the message will be
+     * passed through each observer in the order of their priority. If the observer returns a modified message, it will be used instead of the original message!
      *
      * @method
      * @param {Message} message
      * @public
      */
-    onMessage( message ) {
+    notifyMessageObservers( message ) {
+        let modifiedMessage = message;
         _.forEach( this.#messageObservers, ( messageObserver ) => {
-            messageObserver.onMessage( this.#connectionIdentifier, message );
+            modifiedMessage = messageObserver.onMessage( this.#connectionIdentifier, modifiedMessage );
         } );
     }
 
     /**
-     * An event-triggered method that will notify any observers about primary connection recovered state.
+     * An event-triggered method that will notify any observers about the primary connection recovered state.
      * <br/>
      * NOTE: You can override this to add custom functionality but make sure to also call the base method
      * using: super.onConnectionRecovered( identifier )
@@ -189,6 +199,26 @@ class MessageHandler extends ConnectionObserver {
         }
     }
 
+    /**
+     * An event-triggered method that will notify any observers about the primary connection having been lost.
+     * <br/>
+     * NOTE: You can override this to add custom functionality but make sure to also call the base method
+     * using: super.onConnectionLost( identifier )
+     *
+     * @method
+     * @param {string} identifier The identifier of the observed connection.
+     * @override
+     * @private
+     */
+    onConnectionLost( identifier ) {
+        if ( identifier === this.#connectionIdentifier ) {
+            this.#isAvailable = false;
+            _.forEach( this.#messageObservers, ( messageObserver ) => {
+                messageObserver.onConnectionLost( this.#connectionIdentifier );
+            } );
+        }
+    }
+
 }
 
 module.exports = MessageHandler;

package/components/exchange/message-observer.js

@@ -1,12 +1,13 @@
 /*
  * The ti-engine is an open source, free to use—both for personal and commercial projects—framework for the creation of microservice-based solutions using node.js.
- * Copyright © 2021-2023 Boris Kostadinov <kostadinov.boris@gmail.com>
+ * Copyright © 2021-2025 Boris Kostadinov <kostadinov.boris@gmail.com>
  * This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
  * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
  * You should have received a copy of the GNU General Public License along with this program. If not, see <https://www.gnu.org/licenses/>.
 */
 
 const ConnectionObserver = require( "#connection-observer" );
+const _ = require( "lodash" );
 const exceptions = require( "#exceptions" );
 
 /**
@@ -21,16 +22,45 @@ const exceptions = require( "#exceptions" );
  */
 class MessageObserver extends ConnectionObserver {
 
+    #priority = 0;
+
     /**
+     * @param {number} [priority=0] The priority of this observer. Higher values indicate higher priority.
      * @constructor
      */
-    constructor() {
+    constructor( priority = 0 ) {
         super();
 
         // make sure this abstract class cannot be instantiated:
         if ( new.target === MessageObserver ) {
             throw exceptions.raise( exceptions.exceptionCode.E_GEN_ABSTRACT_CLASS_INIT, { name: this.constructor.name } );
         }
+
+        this.#priority = _.isNumber( priority ) ? priority : 0;
+    }
+
+    /**
+     * Returns the priority of this observer.
+     * <br/>
+     * NOTE: Higher values indicate higher priority.
+     *
+     * @property
+     * @returns {number}
+     */
+    get priority() {
+        return this.#priority;
+    }
+
+    /**
+     * Used to set the priority of this observer.
+     * <br/>
+     * NOTE: Higher values indicate higher priority.
+     *
+     * @property
+     * @param {number} value
+     */
+    set priority( value ) {
+        this.#priority = _.isNumber( value ) ? value : 0;
     }
 
     /**
@@ -41,10 +71,13 @@ class MessageObserver extends ConnectionObserver {
      * @method
      * @param {string} identifier The identifier of the observed connection.
      * @param {Message} message The message for processing.
+     * @returns {Message} The message that was received.
      * @virtual
      * @public
      */
-    onMessage( identifier, message ) { }
+    onMessage( identifier, message ) {
+        return message;
+    }
 
     /**
      * Needs to be invoked by the connection handler when the connection is disrupted.
@@ -74,6 +107,20 @@ class MessageObserver extends ConnectionObserver {
         super.onConnectionRecovered( identifier );
     }
 
+    /**
+     * Needs to be invoked by the connection handler when the connection is irrevocably lost.
+     * <br/>
+     * NOTE: Override this to add custom functionality.
+     *
+     * @method
+     * @param {string} identifier The identifier of the observed connection.
+     * @virtual
+     * @public
+     */
+    onConnectionLost( identifier ) {
+        super.onConnectionLost( identifier );
+    }
+
 }
 
-module.exports = MessageObserver;
+module.exports = MessageObserver;

package/components/exchange/message-receiver.js

@@ -117,7 +117,7 @@ class MessageReceiver extends MessageHandler {
             this.onReceive().then( ( message ) => {
                 return this.#postReceive( message );
             } ).then( ( message ) => {
-                this.onMessage( message );
+                this.notifyMessageObservers( message );
             } ).catch( ( error ) => {
                 if ( error.code !== exceptions.exceptionCode.E_COM_MESSAGE_RECEIVER_UNAVAILABLE ) {
                     logger.log( `Error while trying to receive the next pending message from memory cache in receiver '${ this.connectionIdentifier }'! Resuming operation...`, logger.logSeverity.ERROR, error );

package/components/service-caller.js

@@ -50,13 +50,215 @@ const messageDispatcher = require( "#message-dispatcher" );
  * @typedef {Object} ServiceCallResult
  * @property {Object|undefined} exception If there was exception during the service call processing, it will be set here. Otherwise, it will be 'undefined'.
  * @property {boolean} isSuccessful A flag indicating if this service call can be considered successful or not.
- * @property {Object|string|undefined} payload The payload containing the results from the service call processing. If string it is ID of the payload in the memory cache instead.
+ * @property {Object|string|undefined} payload The payload containing the results from the service call processing. If a string, it is ID of the payload in the memory cache instead.
  */
 
 /**
- * @callback TaskHandler
- * @param {ServiceCall} serviceCall The service call for processing.
+ * Used to assemble and prepare a new {@link ServiceCall} object.
+ *
+ * @method
+ * @param {string} messageID The message ID of the service call. This has to be unique across the whole service call tree, including the current service call, and will be used to identify the service call in the service call tree.
+ * @param {ServiceAddress} serviceAddress The service address has to define a valid service domain name, service alias, and optionally a service version.
+ * @param {Object} serviceParams Set of named parameters to provide to the called service.
+ * @param {ServiceExecContext} serviceExecContext The context in which the service call is being executed.
+ * @returns {Promise<ServiceCall>}
+ * @private
  */
+let prepareServiceCall = ( messageID, serviceAddress, serviceParams, serviceExecContext ) => {
+    return new Promise( ( resolve ) => {
+        const ServiceInstance = require( "#service-instance" );
+
+        // assemble the new service call:
+        let chainID = ( serviceExecContext.previousServiceCall ) ? serviceExecContext.previousServiceCall.chainID : tools.getUUID();
+        let chainLevel = ( serviceExecContext.previousServiceCall ) ? serviceExecContext.previousServiceCall.chainLevel + 1 : 0;
+        let source = {
+            instanceID: ServiceInstance.instanceID,
+            route: ServiceInstance.serviceDomainName
+        };
+        let destination = {
+            instanceID: undefined,
+            route: serviceAddress.serviceDomainName
+        };
+        /** @type ServiceCall */
+        let serviceCall = {
+            authToken: serviceExecContext.authToken,
+            chainID: chainID,
+            chainLevel: chainLevel,
+            createdOn: Date.now(),
+            destination: destination,
+            executionTime: 0,
+            exception: undefined,
+            finishedOn: undefined,
+            isCompleted: false,
+            isSuccessful: undefined,
+            messageID: messageID,
+            payload: undefined,
+            predecessor: ( serviceExecContext.previousServiceCall ) ? serviceExecContext.previousServiceCall.messageID : undefined,
+            serviceAddress: serviceAddress,
+            serviceParams: serviceParams,
+            source: source,
+            successors: undefined
+        };
+
+        resolve( serviceCall );
+    } );
+}
+
+/**
+ * Used to verify if the service is registered in the service registry.
+ *
+ * @method
+ * @param {ServiceAddress} serviceAddress
+ * @returns {Promise}
+ * @private
+ */
+let findServiceInRegistry = ( serviceAddress ) => {
+    return new Promise( ( resolve, reject ) => {
+        let serviceCatalog = config.getSetting( config.setting.SERVICE_REGISTRY_ADDRESS ) + serviceAddress.serviceDomainName;
+        cache.instance.isSetMember( serviceCatalog, serviceAddress.serviceAlias ).then( ( result ) => {
+            if ( result === true ) {
+                resolve();
+            } else {
+                reject( exceptions.raise( exceptions.exceptionCode.E_COM_SERVICE_NOT_REGISTERED ) );
+            }
+        } ).catch( ( error ) => {
+            reject( exceptions.raise( error ) );
+        } );
+    } );
+}
+
+/**
+ * A class defining a service call processor.
+ *
+ * @class ServiceCallProcessor
+ * @private
+ */
+class ServiceCallProcessor {
+
+    #messageID;
+    #serviceAddress;
+    #serviceParams;
+    #serviceExecContext;
+    #timeoutHandle;
+    #taskCompletionHandler;
+    #isProcessed = false;
+
+    /**
+     * @constructor
+     * @param {ServiceAddress} serviceAddress The service address has to define a valid service domain name, service alias, and optionally a service version.
+     * @param {Object} serviceParams Set of named parameters to provide to the called service.
+     * @param {ServiceExecContext} serviceExecContext The context in which the service call is being executed.
+     * @returns {ServiceCallProcessor}
+     */
+    constructor( serviceAddress, serviceParams, serviceExecContext ) {
+        this.#messageID = tools.getUUID();
+        this.#serviceAddress = serviceAddress;
+        this.#serviceParams = serviceParams;
+        this.#serviceExecContext = serviceExecContext;
+    }
+
+    /* Public interface */
+
+    /**
+     * The unique identifier of the service call.
+     *
+     * @property
+     * @returns {string}
+     * @public
+     */
+    get messageID() {
+        return this.#messageID;
+    }
+
+    /**
+     * Used to start the execution of the service call.
+     * <br/>
+     * NOTE: This method will time out after specific preconfigured time, in which case it will resolve with {@link E_COM_SERVICE_EXEC_TIMEOUT} error.
+     *
+     * @method
+     * @returns {Promise<ServiceCallResult>}
+     * @public
+     */
+    process() {
+        return Promise.race( [ this.#execute(), this.#timeout() ] ).then( ( result ) => {
+            clearTimeout( this.#timeoutHandle );
+            this.#isProcessed = true;
+            return result;
+        } ).catch( ( error ) => {
+            clearTimeout( this.#timeoutHandle );
+            this.#isProcessed = true;
+            logger.log( `Error during service call execution!`, logger.logSeverity.ERROR, error );
+            return {
+                isSuccessful: false,
+                exception: exceptions.raise( error ),
+                payload: undefined
+            };
+        } );
+    }
+
+    /**
+     * Used to complete the execution of the service call that was started within the {@link process} method.
+     *
+     * @method
+     * @param {ServiceCall} serviceCall
+     * @public
+     */
+    complete( serviceCall ) {
+        if ( this.#isProcessed !== true ) {
+            let serviceCallResult = {
+                exception: serviceCall.exception,
+                isSuccessful: ( serviceCall.isSuccessful !== undefined ) ? tools.toBool( serviceCall.isSuccessful ) : true,
+                payload: serviceCall.payload
+            };
+            this.#taskCompletionHandler( serviceCallResult );
+        }
+    }
+
+    /* Private interface */
+
+    /**
+     * Used to execute the service call.
+     *
+     * @method
+     * @returns {Promise<ServiceCallResult>}
+     * @private
+     */
+    #execute() {
+        return new Promise( ( resolve, reject ) => {
+            findServiceInRegistry( this.#serviceAddress ).then( () => {
+                return prepareServiceCall( this.#messageID, this.#serviceAddress, this.#serviceParams, this.#serviceExecContext );
+            } ).then( ( serviceCall ) => {
+                return messageDispatcher.instance.sendRequest( serviceCall );
+            } ).then( () => {
+                this.#taskCompletionHandler = ( serviceCallResult ) => {
+                    resolve( serviceCallResult );
+                };
+            } ).catch( ( error ) => {
+                if ( this.#isProcessed !== true ) {
+                    reject( exceptions.raise( error ) );
+                }
+            } );
+        } );
+    }
+
+    /**
+     * Used to time out the service call execution.
+     *
+     * @method
+     * @returns {Promise<ServiceCallResult>}
+     * @private
+     */
+    #timeout() {
+        return new Promise( ( resolve, reject ) => {
+            this.#timeoutHandle = setTimeout( () => {
+                if ( this.#isProcessed !== true ) {
+                    reject( exceptions.raise( exceptions.exceptionCode.E_COM_SERVICE_EXEC_TIMEOUT ) );
+                }
+            }, config.getSetting( config.setting.SERVICE_EXECUTION_TIMEOUT ) );
+        } );
+    }
+
+}
 
 /**
  * A class defining a service caller behavior.
@@ -67,13 +269,13 @@ const messageDispatcher = require( "#message-dispatcher" );
  */
 class ServiceCaller extends MessageObserver {
 
-    #serviceCallTasks = {};
+    #serviceCallProcessors = {};
 
     /**
      * @constructor
      */
     constructor() {
-        super();
+        super( 10 );
     }
 
     /* Public interface */
@@ -91,44 +293,14 @@ class ServiceCaller extends MessageObserver {
      * @public
      */
     executeServiceCall( serviceAddress, serviceParams, serviceExecContext ) {
-        let execution = new Promise( ( resolve, reject ) => {
-            this.#findServiceInRegistry( serviceAddress ).then( () => {
-                return this.#prepareServiceCall( serviceAddress, serviceParams, serviceExecContext );
-            } ).then( ( serviceCall ) => {
-                return messageDispatcher.instance.sendRequest( serviceCall );
-            } ).then( ( messageID ) => {
-                this.#addTaskHandler( messageID, ( serviceCall ) => {
-                    this.#completeServiceCall( serviceCall ).then( ( serviceCall ) => {
-                        let serviceCallResult = {
-                            exception: serviceCall.exception,
-                            isSuccessful: ( serviceCall.isSuccessful !== undefined ) ? serviceCall.isSuccessful : true,
-                            payload: serviceCall.payload
-                        };
-
-                        resolve( serviceCallResult );
-                    } ).catch( ( error ) => {
-                        reject( exceptions.raise( error ) );
-                    } );
-                } );
-            } ).catch( ( error ) => {
-                reject( exceptions.raise( error ) );
+        return new Promise( ( resolve, reject ) => {
+            let processor = new ServiceCallProcessor( serviceAddress, serviceParams, serviceExecContext );
+            this.#addProcessor( processor.messageID, processor );
+            processor.process().then( ( serviceCallResult ) => {
+                this.#removeProcessor( processor.messageID );
+                resolve( serviceCallResult );
             } );
         } );
-        let timeoutHandle;
-        let timeout = new Promise( ( resolve, reject ) => {
-            timeoutHandle = setTimeout( () => reject( exceptions.raise( exceptions.exceptionCode.E_COM_SERVICE_EXEC_TIMEOUT ) ), config.getSetting( config.setting.SERVICE_EXECUTION_TIMEOUT ) );
-        } );
-
-        return Promise.race( [ execution, timeout ] ).then( ( result ) => {
-            clearTimeout( timeoutHandle );
-            return result;
-        } ).catch( ( error ) => {
-            logger.log( `Error during service call execution!`, logger.logSeverity.ERROR, error );
-            return {
-                isSuccessful: false,
-                exception: exceptions.raise( error )
-            };
-        } );
     }
 
     /**
@@ -136,18 +308,26 @@ class ServiceCaller extends MessageObserver {
      *
      * @method
      * @param {string} identifier The identifier of the observed connection.
-     * @param {Message} message The message for processing.
+     * @param {ServiceCall} serviceCall The service call message for processing.
+     * @returns {ServiceCall} The service call message that was received.
      * @override
      * @public
      */
-    onMessage( identifier, message ) {
-        let execution = this.#getTaskHandler( message.messageID );
-        if ( typeof ( execution ) === "function" ) {
-            execution( message );
-            this.#removeTaskHandler( message.messageID );
+    onMessage( identifier, serviceCall ) {
+        // Complete the service call:
+        serviceCall.finishedOn = Date.now();
+        serviceCall.executionTime = serviceCall.finishedOn - serviceCall.createdOn;
+        serviceCall.isCompleted = true;
+
+        let processor = this.#getProcessor( serviceCall.messageID );
+        if ( processor ) {
+            this.#removeProcessor( serviceCall.messageID );
+            processor.complete( serviceCall );
         } else {
-            logger.log( `Received message with ID '${ message.messageID }' in ServiceCaller that has no registered handler. This is probably a software bug!`, logger.logSeverity.WARNING );
+            logger.log( `Received service call message with ID '${ serviceCall.messageID }' without registered processor! This may be caused by a service call timeout.`, logger.logSeverity.DEBUG, serviceCall.exception || undefined );
         }
+
+        return serviceCall;
     }
 
     /**
@@ -174,133 +354,54 @@ class ServiceCaller extends MessageObserver {
         super.onConnectionRecovered( identifier );
     }
 
-    /* Private interface */
-
-    /**
-     * Used to verify if the service is registered in the service registry.
-     *
-     * @method
-     * @param {ServiceAddress} serviceAddress
-     * @returns {Promise}
-     * @private
-     */
-    #findServiceInRegistry( serviceAddress ) {
-        return new Promise( ( resolve, reject ) => {
-            let serviceCatalog = config.getSetting( config.setting.SERVICE_REGISTRY_ADDRESS ) + serviceAddress.serviceDomainName;
-            cache.instance.isSetMember( serviceCatalog, serviceAddress.serviceAlias ).then( ( result ) => {
-                if ( result === true ) {
-                    resolve();
-                } else {
-                    reject( exceptions.raise( exceptions.exceptionCode.E_COM_SERVICE_NOT_REGISTERED ) );
-                }
-            } ).catch( ( error ) => {
-                reject( exceptions.raise( error ) );
-            } );
-        } );
-    }
-
     /**
-     * Used to assemble and prepare a new {@link ServiceCall} object.
+     * Needs to be invoked by the connection handler when the connection is irrevocably lost.
      *
      * @method
-     * @param {ServiceAddress} serviceAddress The service address has to define a valid service domain name, service alias, and optionally a service version.
-     * @param {Object} serviceParams Set of named parameters to provide to the called service.
-     * @param {ServiceExecContext} serviceExecContext The context in which the service call is being executed.
-     * @returns {Promise<ServiceCall>}
-     * @private
+     * @param {string} identifier The identifier of the observed connection.
+     * @override
+     * @public
      */
-    #prepareServiceCall( serviceAddress, serviceParams, serviceExecContext ) {
-        return new Promise( ( resolve ) => {
-            const ServiceInstance = require( "#service-instance" );
-
-            // assemble the new service call:
-            let chainID = ( serviceExecContext.previousServiceCall ) ? serviceExecContext.previousServiceCall.chainID : tools.getUUID();
-            let chainLevel = ( serviceExecContext.previousServiceCall ) ? serviceExecContext.previousServiceCall.chainLevel + 1 : 0;
-            let source = {
-                instanceID: ServiceInstance.instanceID,
-                route: ServiceInstance.serviceDomainName
-            };
-            let destination = {
-                instanceID: undefined,
-                route: serviceAddress.serviceDomainName
-            };
-            /** @type ServiceCall */
-            let serviceCall = {
-                authToken: serviceExecContext.authToken,
-                chainID: chainID,
-                chainLevel: chainLevel,
-                createdOn: Date.now(),
-                destination: destination,
-                executionTime: 0,
-                exception: undefined,
-                finishedOn: undefined,
-                isCompleted: false,
-                isSuccessful: undefined,
-                messageID: tools.getUUID(),
-                payload: undefined,
-                predecessor: ( serviceExecContext.previousServiceCall ) ? serviceExecContext.previousServiceCall.messageID : undefined,
-                serviceAddress: serviceAddress,
-                serviceParams: serviceParams,
-                source: source,
-                successors: undefined
-            };
-
-            resolve( serviceCall );
-        } );
+    onConnectionLost( identifier ) {
+        super.onConnectionLost( identifier );
     }
 
-    /**
-     * Used to complete the provided {@link ServiceCall} object by setting all required properties to their correct values.
-     *
-     * @method
-     * @param {ServiceCall} serviceCall
-     * @returns {Promise<ServiceCall>}
-     * @private
-     */
-    #completeServiceCall( serviceCall ) {
-        return new Promise( ( resolve ) => {
-            serviceCall.finishedOn = Date.now();
-            serviceCall.executionTime = serviceCall.finishedOn - serviceCall.createdOn;
-            serviceCall.isCompleted = true;
-
-            resolve( serviceCall );
-        } );
-    }
+    /* Private interface */
 
     /**
      * Used to add a new task handler to the list of current tasks.
      *
      * @method
-     * @param {string} taskID
-     * @param {TaskHandler} taskHandler
+     * @param {string} messageID
+     * @param {ServiceCallProcessor} processor
      * @private
      */
-    #addTaskHandler( taskID, taskHandler ) {
-        this.#serviceCallTasks[ taskID ] = taskHandler;
+    #addProcessor( messageID, processor ) {
+        this.#serviceCallProcessors[ messageID ] = processor;
     }
 
     /**
      * Used to fetch a task handler from the list of current tasks.
      *
      * @method
-     * @param {string} taskID
-     * @returns {TaskHandler}
+     * @param {string} messageID
+     * @returns {ServiceCallProcessor}
      * @private
      */
-    #getTaskHandler( taskID ) {
-        return this.#serviceCallTasks[ taskID ];
+    #getProcessor( messageID ) {
+        return this.#serviceCallProcessors[ messageID ];
     }
 
     /**
      * Used to remove a task handler from the list of current tasks.
      *
      * @method
-     * @param {string} taskID
+     * @param {string} messageID
      * @private
      */
-    #removeTaskHandler( taskID ) {
-        if ( this.#serviceCallTasks[ taskID ] ) {
-            delete this.#serviceCallTasks[ taskID ];
+    #removeProcessor( messageID ) {
+        if ( this.#serviceCallProcessors[ messageID ] ) {
+            delete this.#serviceCallProcessors[ messageID ];
         }
     }
 

package/components/service-executor.js

@@ -91,16 +91,19 @@ class ServiceExecutor extends MessageObserver {
      *
      * @method
      * @param {string} identifier The identifier of the observed connection.
-     * @param {Message} message The message for processing.
+     * @param {ServiceCall} serviceCall The service call message for processing.
+     * @returns {ServiceCall} The service call message that was received.
      * @override
      * @public
      */
-    onMessage( identifier, message ) {
-        this.#processServiceCall( message ).then( ( serviceCall ) => {
+    onMessage( identifier, serviceCall ) {
+        this.#processServiceCall( serviceCall ).then( ( serviceCall ) => {
             return messageDispatcher.instance.sendResponse( serviceCall );
         } ).catch( ( error ) => {
-            logger.log( `Failed to send service call response after processing! Service call ID was: '${ message.messageID }'`, logger.logSeverity.ERROR, error );
+            logger.log( `Failed to send service call response after processing! Service call message ID was: '${ serviceCall.messageID }'`, logger.logSeverity.ERROR, error );
         } );
+
+        return serviceCall;
     }
 
     /**
@@ -127,6 +130,18 @@ class ServiceExecutor extends MessageObserver {
         super.onConnectionRecovered( identifier );
     }
 
+    /**
+     * Needs to be invoked by the connection handler when the connection is irrevocably lost.
+     *
+     * @method
+     * @param {string} identifier The identifier of the observed connection.
+     * @override
+     * @public
+     */
+    onConnectionLost( identifier ) {
+        super.onConnectionLost( identifier );
+    }
+
     /**
      * Used to set up the method for service access verification.
      *

package/components/service-instance.js

@@ -37,6 +37,7 @@ class ServiceInstance {
     #serviceConfig;
     #serviceHealthCheck;
     #reportHealthyJob;
+    #healthReportUnderway = false;
 
     /**
      * @constructor
@@ -225,10 +226,14 @@ class ServiceInstance {
      * @public
      */
     reportHealthy() {
-        if ( cache.instance.isOperational ) {
+        if ( cache.instance.isOperational && !this.#healthReportUnderway ) {
+            this.#healthReportUnderway = true;
             let timestamp = new Date();
-            cache.instance.setValue( this.#serviceHealthCheck, timestamp.toISOString(), config.getSetting( config.setting.SERVICE_HEALTH_CHECK_TIMEOUT ) ).catch( ( error ) => {
-                logger.log( `Error while trying to report for health check from '${ ServiceInstance.instanceID }'!`, logger.logSeverity.WARNING, error );
+            cache.instance.setValue( this.#serviceHealthCheck, timestamp.toISOString(), config.getSetting( config.setting.SERVICE_HEALTH_CHECK_TIMEOUT ) ).then( () => {
+                this.#healthReportUnderway = false;
+            } ).catch( ( error ) => {
+                this.#healthReportUnderway = false;
+                logger.log( `Failed to report for health check from instance '${ ServiceInstance.instanceID }'!`, logger.logSeverity.WARNING, error );
             } );
         }
     }

package/integrations/redis-integration.js

@@ -28,6 +28,7 @@ let cacheCommandsEnum = tools.enum( {
     HASH_GET: [ "hget", "hash get", "https://redis.io/docs/latest/commands/hget/" ],
     HASH_GET_ALL: [ "hgetall", "hash get all", "https://redis.io/docs/latest/commands/hgetall/" ],
     HASH_REMOVE: [ "hdel", "hash remove", "https://redis.io/docs/latest/commands/hdel/" ],
+    HASH_EXPIRE: [ "hexpire", "hash expire", "https://redis.io/docs/latest/commands/hexpire/" ],
     HASH_SET: [ "hset", "", "https://redis.io/docs/latest/commands/hset/" ],
     HASH_SET_MANY: [ "hmset", "(deprecated) use HSET with multiple fields", "https://redis.io/docs/latest/commands/hmset/" ],
     IS_SET_MEMBER: [ "sismember", "", "https://redis.io/docs/latest/commands/sismember/" ],
@@ -43,6 +44,21 @@ let cacheCommandsEnum = tools.enum( {
     UNION_OF_SETS: [ "sunion", "union of sets", "https://redis.io/docs/latest/commands/sunion/" ]
 } );
 
+/**
+ * Enum for listing all client statuses.
+ *
+ * @readonly
+ * @enum {number}
+ */
+let clientStatusEnum = tools.enum( {
+    UNINITIALIZED: [ 0, "uninitialized", "Redis client is offline and not yet initialized." ],
+    CONNECTED: [ 1, "connected", "Redis client is connected and online." ],
+    CONNECTING: [ 2, "connecting", "Redis client is connecting to server." ],
+    DISRUPTED: [ 3, "disrupted", "Redis client is temporarily disconnected from server due to a disruption." ],
+    SHUTTING_DOWN: [ 4, "shutting down", "Redis client is shutting down." ],
+    DISCONNECTED: [ 5, "disconnected", "Redis client is permanently disconnected from server." ]
+} );
+
 /**
  * Enum for listing the Redis key override modes.
  *
@@ -63,6 +79,10 @@ module.exports.cacheCommands = cacheCommandsEnum;
  * @typedef {string} TiRedisOverrideMode
  */
 module.exports.cacheOverrideMode = cacheOverrideModeEnum;
+/**
+ * @typedef {number} TiRedisClientStatus
+ */
+module.exports.clientStatus = clientStatusEnum;
 
 /**
  * Used to create a Redis Cache client.
@@ -76,6 +96,7 @@ module.exports.cacheOverrideMode = cacheOverrideModeEnum;
 class RedisClient {
 
     #clientIdentifier;
+    #clientStatus = clientStatusEnum.UNINITIALIZED;
     #retryMaxInterval = 1000;
     #retryMaxAttempts = undefined;
     #redisConnection = undefined;
@@ -118,6 +139,17 @@ class RedisClient {
         return this.#redisClientID;
     }
 
+    /**
+     * Used to return the Redis client status.
+     *
+     * @property
+     * @returns {number}
+     * @public
+     */
+    get clientStatus() {
+        return this.#clientStatus;
+    }
+
     /**
      * Used to return the Redis server version.
      *
@@ -167,12 +199,6 @@ class RedisClient {
                 } ).then( ( clientID ) => {
                     // Store the client ID:
                     this.#redisClientID = clientID;
-
-                    // Notify all connection observers about the initialization success:
-                    _.forEach( this.#connectionObservers, ( connectionObservers ) => {
-                        connectionObservers.onConnectionRecovered( this.#clientIdentifier );
-                    } );
-
                     resolve();
                 } ).catch( ( error ) => {
                     reject( exceptions.raise( error ) );
@@ -359,6 +385,7 @@ class RedisClient {
     shutDown( timeoutMs = 1000 ) {
         return new Promise( ( resolve ) => {
             let finished = false;
+            this.#clientStatus = clientStatusEnum.SHUTTING_DOWN;
             const done = () => {
                 if ( !finished ) {
                     finished = true;
@@ -403,7 +430,8 @@ class RedisClient {
      * @param {number} defaultDB
      * @param {number} [retryMaxIntervalMs=1000] Optional max backoff interval.
      * @param {number|undefined} [retryMaxAttempts=undefined] Optional max (re)connection attempts before abort.
-     * @public
+     * @returns {Promise}
+     * @private
      */
     #setupClient( host, port, authKey, user, defaultDB, retryMaxIntervalMs, retryMaxAttempts ) {
         return new Promise( ( resolve, reject ) => {
@@ -412,19 +440,22 @@ class RedisClient {
                 this.#retryMaxAttempts = retryMaxAttempts;
 
                 let retryStrategy = ( attempt ) => {
-                    let result = Math.min( attempt * 50, this.#retryMaxInterval );
-
+                    let retryInterval = Math.min( attempt * 50, this.#retryMaxInterval );
                     if ( this.#retryMaxAttempts != null && attempt > this.#retryMaxAttempts ) {
                         logger.log( "In Redis retry strategy: reached max attempts for command retry. Aborting...", logger.logSeverity.WARNING, { attempts: attempt } );
-                        result = exceptions.raise( exceptions.exceptionCode.E_COM_RETRY_ATTEMPTS_EXCEEDED );
+                        retryInterval = exceptions.raise( exceptions.exceptionCode.E_COM_RETRY_ATTEMPTS_EXCEEDED );
                     }
-
-                    return result;
+                    return retryInterval;
                 };
 
                 let reconnectOnError = ( error ) => {
                     logger.log( `In Redis reconnect on error strategy: ${ error.message }`, logger.logSeverity.ERROR, error );
-                    return !!error.message.includes( "READONLY" );
+                    if ( error.message.includes( "READONLY" ) ) {
+                        // Returning 2 will also resubmit the failed command:
+                        return 2;
+                    } else {
+                        return 0;
+                    }
                 };
 
                 let options = {
@@ -441,22 +472,33 @@ class RedisClient {
 
                 /** @type Redis */
                 this.#redisConnection = new Redis( options );
+                this.#clientStatus = clientStatusEnum.CONNECTING;
+
+                this.#redisConnection.once( "ready", () => {
+                    this.#clientStatus = clientStatusEnum.CONNECTED;
+                    this.#notifyConnectionObservers();
+
+                    this.#redisConnection.on( "ready", () => {
+                        this.#clientStatus = clientStatusEnum.CONNECTED;
+                        this.#notifyConnectionObservers();
+                    } );
 
-                this.#redisConnection.on( "ready", () => {
-                    logger.log( `Connection to Redis server ${ host }:${ port } (re)established by client '${ this.identifier }' and is ready to be used.`, logger.logSeverity.INFO );
                     resolve();
                 } );
                 this.#redisConnection.on( "error", ( error ) => {
+                    this.#clientStatus = clientStatusEnum.DISRUPTED;
                     logger.log( `Error received in Redis client '${ this.identifier }'.`, logger.logSeverity.ERROR, error );
-
-                    // Notify all connection observers about this event:
-                    _.forEach( this.#connectionObservers, ( connectionObservers ) => {
-                        connectionObservers.onConnectionDisrupted( this.#clientIdentifier );
-                    } );
+                    this.#notifyConnectionObservers();
+                } );
+                this.#redisConnection.on( "reconnecting", ( retryInterval ) => {
+                    this.#clientStatus = clientStatusEnum.CONNECTING;
+                    logger.log( `Client '${ this.identifier }' reconnecting to Redis server after ${ retryInterval } ms.`, logger.logSeverity.DEBUG );
                 } );
-                this.#redisConnection.on( "reconnecting", ( info ) => {
-                    if ( info.attempt > 1 ) {
-                        logger.log( `Client '${ this.identifier }' reconnecting to Redis server after ${ info.delay } ms. This is attempt ${ info.attempt }.`, logger.logSeverity.DEBUG );
+                this.#redisConnection.on( "end", () => {
+                    if ( this.#clientStatus !== clientStatusEnum.SHUTTING_DOWN ) {
+                        this.#clientStatus = clientStatusEnum.DISCONNECTED;
+                        logger.log( `Client '${ this.identifier }' cannot reconnect to Redis server and has been shut down.`, logger.logSeverity.WARNING );
+                        this.#notifyConnectionObservers();
                     }
                 } );
             } catch ( error ) {
@@ -465,10 +507,31 @@ class RedisClient {
         } );
     }
 
+    /**
+     * Used to notify all connection observers about the current connection state.
+     *
+     * @method
+     * @private
+     */
+    #notifyConnectionObservers() {
+        // Notify all connection observers about the event:
+        _.forEach( this.#connectionObservers, ( connectionObservers ) => {
+            if ( this.#clientStatus === clientStatusEnum.CONNECTED ) {
+                logger.log( `Connection to Redis server ${ this.#redisConnection.options.host }:${ this.#redisConnection.options.port } (re)established by client '${ this.identifier }' and is ready to be used.`, logger.logSeverity.INFO );
+                connectionObservers.onConnectionRecovered( this.#clientIdentifier );
+            } else if ( this.#clientStatus === clientStatusEnum.DISRUPTED ) {
+                connectionObservers.onConnectionDisrupted( this.#clientIdentifier );
+            } else if ( this.#clientStatus === clientStatusEnum.DISCONNECTED ) {
+                connectionObservers.onConnectionLost( this.#clientIdentifier );
+            }
+        } );
+    }
+
     /**
      * Used to fetch and store Redis server information.
      *
      * @method
+     * @returns {Promise}
      * @private
      */
     #fetchServerInfo() {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ti-engine/core",
-  "version": "1.2.4",
+  "version": "1.3.1",
   "description": "The ti-engine is an open source, free to use—both for personal and commercial projects—framework for the creation of microservice-based solutions using node.js.",
   "author": "Boris Kostadinov <kostadinov.boris@gmail.com>",
   "license": "GPL-3.0-or-later",

package/utils/cache.js

@@ -124,6 +124,22 @@ class CommonMemoryCache extends ConnectionObserver {
         }
     }
 
+    /**
+     * Needs to be invoked by the connection handler when the connection is irrevocably lost.
+     *
+     * @method
+     * @param {string} identifier The identifier of the observed connection.
+     * @override
+     * @public
+     */
+    onConnectionLost( identifier ) {
+        if ( identifier === this.#connectionIdentifier ) {
+            this.#isOperational = false;
+            // TODO: implement forced shut down of the service instance instead of crashing it outright
+            throw exceptions.raise( exceptions.exceptionCode.E_GEN_SYSTEM_CACHE_UNAVAILABLE );
+        }
+    }
+
     /**
      * Used to register a new {@link ConnectionObserver} for events related to the underlying Redis connection state.
      *
@@ -317,13 +333,19 @@ class CommonMemoryCache extends ConnectionObserver {
      * @method
      * @param {string} key
      * @param {number} seconds
+     * @param {string} [name] If you need to expire a field in a hash set instead, provide the name of the set here.
      * @returns {Promise<number>} This will resolve with the seconds as provided initially by the caller.
      * @public
      */
-    expireValue( key, seconds ) {
+    expireValue( key, seconds, name ) {
         return new Promise( ( resolve, reject ) => {
             if ( this.#isOperational === true ) {
-                let commandExpire = [ redis.cacheCommands.EXPIRE, key, seconds ];
+                let commandExpire = [];
+                if ( name ) {
+                    commandExpire = [ redis.cacheCommands.HASH_EXPIRE, name, seconds, "FIELDS", 1, key ];
+                } else {
+                    commandExpire = [ redis.cacheCommands.EXPIRE, key, seconds ];
+                }
                 this.#redisClient.executeCommands( [ commandExpire ] ).then( () => {
                     resolve( seconds );
                 } ).catch( ( error ) => {

package/utils/exceptions.js

@@ -38,6 +38,7 @@ let exceptionCodeEnum = tools.enum( {
     E_COM_SERVICE_NOT_FOUND: [ 3004, "service not found", "The specified service is not found in the service definition interface." ],
     E_COM_SERVICE_HANDLER_NOT_FOUND: [ 3005, "service handler not found", "No handler found in the interface for the specified service or service version." ],
     E_COM_MESSAGE_RECEIVER_UNAVAILABLE: [ 3006, "message receiver unavailable", "The message receiver instance is currently unavailable." ],
+    E_COM_MESSAGE_EXCHANGE_BROKEN: [ 3007, "message exchange broken", "The message exchange is irrevocably broken and cannot be used any longer." ],
     // E_COM_UNRECOGNIZED_API_URL: [ 301, "unrecognized api url", "Attempt to access unrecognized or invalid API URL." ],
     // E_COM_MISSING_REQUIRED_ARGUMENTS: [ 302, "missing required arguments", "Attempt to execute operation without all required arguments." ],
     // E_COM_UNRECOGNIZED_RESPONSE_STRUCTURE: [ 303, "unrecognized response structure", "The received response has unrecognized structure and cannot be parsed or examined." ],