@celerispay/hazelcast-client 3.12.5 → 3.12.7-3

package/lib/invocation/ClusterService.js

@@ -24,6 +24,8 @@ var Util_1 = require("../Util");
 var MemberAttributeEvent_1 = require("../core/MemberAttributeEvent");
 var MembershipEvent_1 = require("../core/MembershipEvent");
 var UuidUtil_1 = require("../util/UuidUtil");
+var Address = require("../Address");
+var HazelcastFailoverManager_1 = require("./HazelcastFailoverManager");
 var MemberEvent;
 (function (MemberEvent) {
     MemberEvent[MemberEvent["ADDED"] = 1] = "ADDED";
@@ -49,10 +51,15 @@ var ClusterService = /** @class */ (function () {
         this.lastFailoverAttempt = 0;
         this.failoverCooldown = 5000; // 5 seconds cooldown between failover attempts
         this.downAddresses = new Map(); // address -> timestamp when marked down
-        this.addressBlockDuration = 30000; // 30 seconds block duration for down addresses
+        this.addressBlockDuration = 15000; // Reduced from 30000ms to 15000ms
+        this.reconnectionTask = null;
+        this.reconnectionInterval = 10000; // 10 seconds between reconnection attempts
         this.client = client;
         this.logger = this.client.getLoggingService().getLogger();
         this.members = [];
+        this.startReconnectionTask();
+        this.startStateLoggingTask();
+        this.failoverManager = new HazelcastFailoverManager_1.HazelcastFailoverManager(client, this.logger);
     }
     /**
      * Starts cluster service.
@@ -98,6 +105,24 @@ var ClusterService = /** @class */ (function () {
             return Array.from(new Set(Array.from(addresses).concat(Array.from(providerAddresses))));
         });
     };
+    /**
+     * Returns the owner connection if available
+     */
+    ClusterService.prototype.getOwnerConnection = function () {
+        return this.ownerConnection;
+    };
+    /**
+     * Returns whether failover is currently in progress
+     */
+    ClusterService.prototype.isFailoverInProgress = function () {
+        return this.failoverInProgress;
+    };
+    /**
+     * Returns the list of known addresses in the cluster
+     */
+    ClusterService.prototype.getKnownAddresses = function () {
+        return this.knownAddresses.slice(); // Return a copy to prevent external modification
+    };
     /**
      * Returns the list of members in the cluster.
      * @returns
@@ -139,16 +164,15 @@ var ClusterService = /** @class */ (function () {
     ClusterService.prototype.getClientInfo = function () {
         var info = new ClientInfo_1.ClientInfo();
         info.uuid = this.uuid;
-        info.localAddress = this.getOwnerConnection().getLocalAddress();
+        var ownerConnection = this.getOwnerConnection();
+        if (ownerConnection) {
+            info.localAddress = ownerConnection.getLocalAddress();
+        }
+        else {
+            info.localAddress = null;
+        }
         return info;
     };
-    /**
-     * Returns the connection associated with owner node of this client.
-     * @returns {ClientConnection}
-     */
-    ClusterService.prototype.getOwnerConnection = function () {
-        return this.ownerConnection;
-    };
     /**
      * Adds MembershipListener to listen for membership updates. There is no check for duplicate registrations,
      * so if you register the listener twice, it will get events twice.
@@ -180,7 +204,11 @@ var ClusterService = /** @class */ (function () {
             var handleAttributeChange = _this.handleMemberAttributeChange.bind(_this);
             ClientAddMembershipListenerCodec_1.ClientAddMembershipListenerCodec.handle(m, handleMember, handleMemberList, handleAttributeChange, null);
         };
-        return this.client.getInvocationService().invokeOnConnection(this.getOwnerConnection(), request, handler)
+        var ownerConnection = this.getOwnerConnection();
+        if (!ownerConnection) {
+            return Promise.reject(new Error('Cannot initialize membership listener: no owner connection available'));
+        }
+        return this.client.getInvocationService().invokeOnConnection(ownerConnection, request, handler)
             .then(function (resp) {
             _this.logger.trace('ClusterService', 'Registered listener with id '
                 + ClientAddMembershipListenerCodec_1.ClientAddMembershipListenerCodec.decodeResponse(resp).response);
@@ -196,6 +224,8 @@ var ClusterService = /** @class */ (function () {
     };
     ClusterService.prototype.onConnectionClosed = function (connection) {
         this.logger.warn('ClusterService', 'Connection closed to ' + connection.toString());
+        // Mark the address as down when connection is closed
+        this.markAddressAsDown(connection.getAddress());
         if (connection.isAuthenticatedAsOwner()) {
             this.ownerConnection = null;
             this.triggerFailover();
@@ -203,6 +233,8 @@ var ClusterService = /** @class */ (function () {
     };
     ClusterService.prototype.onHeartbeatStopped = function (connection) {
         this.logger.warn('ClusterService', connection.toString() + ' stopped heartbeating.');
+        // Mark the address as down when heartbeat stops
+        this.markAddressAsDown(connection.getAddress());
         if (connection.isAuthenticatedAsOwner()) {
             this.client.getConnectionManager().destroyConnection(connection.getAddress());
             this.ownerConnection = null;
@@ -210,7 +242,6 @@ var ClusterService = /** @class */ (function () {
         }
     };
     ClusterService.prototype.triggerFailover = function () {
-        var _this = this;
         var now = Date.now();
         if (this.failoverInProgress || (now - this.lastFailoverAttempt) < this.failoverCooldown) {
             this.logger.debug('ClusterService', 'Failover already in progress or too soon since last attempt');
@@ -218,23 +249,116 @@ var ClusterService = /** @class */ (function () {
         }
         this.failoverInProgress = true;
         this.lastFailoverAttempt = now;
-        this.logger.info('ClusterService', 'Starting failover process...');
+        // Check if this is a single-node scenario
+        var isSingleNode = this.knownAddresses.length === 1;
+        if (isSingleNode) {
+            this.logger.info('ClusterService', '🔄 SINGLE-NODE CLUSTER RESET: Node restart detected, starting fresh...');
+            this.handleSingleNodeClusterReset();
+        }
+        else {
+            this.logger.info('ClusterService', '🚀 Starting failover process - SERVER-FIRST APPROACH...');
+            // SERVER-FIRST: No credential preservation needed
+            // We trust the server will provide correct member information
+            this.logger.info('ClusterService', '🎯 SERVER-FIRST: No credential management - trusting server data');
+            this.handleMultiNodeFailover();
+        }
+    };
+    /**
+     * Handles single-node cluster reset - treats node restart as fresh cluster
+     */
+    ClusterService.prototype.handleSingleNodeClusterReset = function () {
+        var _this = this;
+        this.logger.info('ClusterService', '🧹 SINGLE-NODE RESET: Clearing all credentials and state...');
+        // Clear all stored credentials - node restart means fresh cluster
+        this.client.getConnectionManager().clearAllCredentials();
+        // Reset client UUIDs - will be assigned fresh by server
+        this.uuid = null;
+        this.ownerUuid = null;
+        // Log state before reset
+        this.logCurrentState();
+        // Force cleanup of all dead connections
+        this.client.getConnectionManager().forceCleanupDeadConnections();
+        // Clear partition information
+        this.client.getPartitionService().clearPartitionTable();
+        // Direct reconnection without waiting for member events
+        this.logger.info('ClusterService', '🔄 SINGLE-NODE RESET: Attempting direct reconnection...');
+        this.connectToCluster()
+            .then(function () {
+            _this.logger.info('ClusterService', '✅ Single-node cluster reset completed successfully');
+            _this.logCurrentState();
+        })
+            .catch(function (error) {
+            _this.logger.error('ClusterService', 'Single-node cluster reset failed', error);
+            _this.logCurrentState();
+        })
+            .finally(function () {
+            _this.failoverInProgress = false;
+        });
+    };
+    /**
+     * Handles multi-node failover - preserves existing logic
+     */
+    ClusterService.prototype.handleMultiNodeFailover = function () {
+        var _this = this;
+        // Log state before failover
+        this.logCurrentState();
+        // Force cleanup of all dead connections to prevent leakage
+        this.client.getConnectionManager().forceCleanupDeadConnections();
         // Clear any stale partition information
         this.client.getPartitionService().clearPartitionTable();
         // Attempt to reconnect to cluster
         this.connectToCluster()
             .then(function () {
-            _this.logger.info('ClusterService', 'Failover completed successfully');
+            _this.logger.info('ClusterService', '✅ Failover completed successfully - SERVER-FIRST approach');
+            // No credential management needed - server handles everything
+            _this.logCurrentState(); // Log state after successful failover
         })
             .catch(function (error) {
             _this.logger.error('ClusterService', 'Failover failed', error);
-            // If failover fails, shutdown the client to prevent further issues
-            _this.client.shutdown();
+            // If failover fails, try to unblock at least one address to allow recovery
+            _this.attemptEmergencyRecovery();
+            _this.logCurrentState(); // Log state after failed failover
+            // Don't shutdown immediately, give recovery a chance
         })
             .finally(function () {
             _this.failoverInProgress = false;
         });
     };
+    /**
+     * Attempts emergency recovery when failover fails
+     */
+    ClusterService.prototype.attemptEmergencyRecovery = function () {
+        var _this = this;
+        this.logger.warn('ClusterService', 'Attempting emergency recovery...');
+        // Unblock at least one address to allow recovery
+        if (this.downAddresses.size > 0) {
+            var firstBlockedAddress_1 = Array.from(this.downAddresses.keys())[0];
+            this.logger.info('ClusterService', "Emergency unblocking address " + firstBlockedAddress_1);
+            this.downAddresses.delete(firstBlockedAddress_1);
+            // Try to connect to the unblocked address
+            try {
+                var _a = firstBlockedAddress_1.split(':'), host = _a[0], portStr = _a[1];
+                var port = parseInt(portStr, 10);
+                if (host && !isNaN(port)) {
+                    var address_1 = new Address(host, port);
+                    this.logger.info('ClusterService', "Attempting emergency connection to " + firstBlockedAddress_1);
+                    // Try to connect without blocking
+                    this.client.getConnectionManager().getOrConnect(address_1, false)
+                        .then(function (connection) {
+                        _this.logger.info('ClusterService', "Emergency connection successful to " + firstBlockedAddress_1);
+                        _this.evaluateOwnershipChange(address_1, connection);
+                        _this.client.getPartitionService().refresh();
+                    })
+                        .catch(function (error) {
+                        _this.logger.warn('ClusterService', "Emergency connection failed to " + firstBlockedAddress_1 + ":", error);
+                    });
+                }
+            }
+            catch (error) {
+                this.logger.error('ClusterService', 'Error during emergency recovery:', error);
+            }
+        }
+    };
     ClusterService.prototype.isAddressKnownDown = function (address) {
         var addressStr = address.toString();
         var downTime = this.downAddresses.get(addressStr);
@@ -285,7 +409,7 @@ var ClusterService = /** @class */ (function () {
             + ', attempt period: ' + attemptPeriod + ', down addresses: ' + this.getDownAddressesInfo());
         if (this.knownAddresses.length <= index) {
             remainingAttemptLimit = remainingAttemptLimit - 1;
-            if (remainingAttemptLimit === 0) {
+            if (remainingAttemptLimit <= 0) {
                 var errorMessage = 'Unable to connect to any of the following addresses: ' +
                     this.knownAddresses.map(function (element) {
                         return element.toString();
@@ -341,12 +465,48 @@ var ClusterService = /** @class */ (function () {
         this.members = members;
         this.client.getPartitionService().refresh();
         this.logger.info('ClusterService', 'Members received.', this.members);
+        // Log current state after member list update
+        this.logCurrentState();
+        // In smart routing mode, proactively open connections to all non-owner members
+        // so that partition-aware routing can work across all nodes.
+        if (this.client.getConfig().networkConfig.smartRouting) {
+            this.connectToNonOwnerMembers(members);
+        }
         var events = this.detectMembershipEvents(prevMembers);
         for (var _i = 0, events_1 = events; _i < events_1.length; _i++) {
             var event = events_1[_i];
             this.fireMembershipEvent(event);
         }
     };
+    /**
+     * Proactively opens connections to all non-owner cluster members.
+     * Required for smart routing: the client needs a live connection to every
+     * node so it can route operations to the correct partition owner.
+     * Failures are silently ignored — the periodic reconnection task will retry.
+     */
+    ClusterService.prototype.connectToNonOwnerMembers = function (members) {
+        var _this = this;
+        var ownerAddress = this.ownerConnection ? this.ownerConnection.getAddress().toString() : null;
+        members.forEach(function (member) {
+            var memberAddressStr = member.address.toString();
+            // Skip the owner — already connected
+            if (memberAddressStr === ownerAddress) {
+                return;
+            }
+            // Skip if already connected
+            if (_this.client.getConnectionManager().hasConnection(member.address)) {
+                return;
+            }
+            _this.logger.info('ClusterService', "\uD83D\uDD17 SMART-ROUTING: Opening connection to non-owner member " + memberAddressStr);
+            _this.client.getConnectionManager().getOrConnect(member.address, false)
+                .then(function () {
+                _this.logger.info('ClusterService', "\u2705 SMART-ROUTING: Connected to member " + memberAddressStr);
+            })
+                .catch(function (err) {
+                _this.logger.warn('ClusterService', "\u26A0\uFE0F SMART-ROUTING: Could not connect to member " + memberAddressStr + ": " + err.message);
+            });
+        });
+    };
     ClusterService.prototype.detectMembershipEvents = function (prevMembers) {
         var events = [];
         var eventMembers = Array.from(this.members);
@@ -399,6 +559,8 @@ var ClusterService = /** @class */ (function () {
     };
     ClusterService.prototype.memberAdded = function (member) {
         this.members.push(member);
+        // Handle member added and update preserved credentials
+        this.handleMemberAdded(member);
         var membershipEvent = new MembershipEvent_1.MembershipEvent(member, MemberEvent.ADDED, this.members);
         this.fireMembershipEvent(membershipEvent);
     };
@@ -408,10 +570,372 @@ var ClusterService = /** @class */ (function () {
             var removedMemberList = this.members.splice(memberIndex, 1);
             assert(removedMemberList.length === 1);
         }
-        this.client.getConnectionManager().destroyConnection(member.address);
+        // Check if we have a healthy connection to this member
+        var connectionManager = this.client.getConnectionManager();
+        var existingConnection = connectionManager.getConnection(member.address);
+        if (existingConnection && existingConnection.isHealthy()) {
+            // If the connection is healthy, don't destroy it immediately
+            // This prevents unnecessary disconnections during temporary network issues
+            this.logger.info('ClusterService', "Member removed but connection is healthy: " + member.address.toString() + ", preserving connection");
+            // Only destroy if we're not in failover mode
+            if (!this.failoverInProgress) {
+                this.logger.debug('ClusterService', "Destroying healthy connection to removed member: " + member.address.toString());
+                connectionManager.destroyConnection(member.address);
+            }
+            else {
+                this.logger.debug('ClusterService', "Preserving healthy connection during failover: " + member.address.toString());
+            }
+        }
+        else {
+            // If connection is unhealthy, destroy it
+            this.logger.info('ClusterService', "Member removed with unhealthy connection: " + member.address.toString() + ", destroying connection");
+            connectionManager.destroyConnection(member.address);
+        }
         var membershipEvent = new MembershipEvent_1.MembershipEvent(member, MemberEvent.REMOVED, this.members);
         this.fireMembershipEvent(membershipEvent);
     };
+    /**
+     * Performs comprehensive credential cleanup when cluster membership changes
+     * This ensures ALL credentials are consistent with the current cluster owner UUID
+     * @param connectionManager The connection manager instance
+     * @param currentClusterOwnerUuid The current cluster owner UUID
+     */
+    /**
+     * Handles member added event - SERVER-FIRST APPROACH
+     * We trust what the server tells us and store it as credentials
+     * @param member The member that was added
+     */
+    ClusterService.prototype.handleMemberAdded = function (member) {
+        var _this = this;
+        this.logger.info('ClusterService', "\u2705 SERVER CONFIRMED: Member[ uuid: " + member.uuid + ", address: " + member.address.toString() + "] added to cluster");
+        // SERVER-FIRST: Store server data as credentials
+        // The server is the authority - we store what it tells us
+        this.logger.info('ClusterService', "\uD83C\uDFAF SERVER-FIRST: Storing server member data as credentials - server is authority");
+        var connectionManager = this.client.getConnectionManager();
+        // Store the server-provided UUID as the authoritative credential
+        if (connectionManager && typeof connectionManager.updatePreservedCredentials === 'function') {
+            connectionManager.updatePreservedCredentials(member.address, member.uuid);
+        }
+        // Record that we received a member added event for this address
+        if (connectionManager && typeof connectionManager.recordMemberAddedEvent === 'function') {
+            connectionManager.recordMemberAddedEvent(member.address);
+        }
+        // Find the current owner from the cluster state
+        var currentOwner = this.findCurrentOwner();
+        if (currentOwner) {
+            this.logger.info('ClusterService', "\uD83D\uDD04 SERVER-FIRST: Updating ALL credentials with current owner UUID: " + currentOwner.uuid);
+            // Update all credentials with the current owner UUID from server
+            if (connectionManager && typeof connectionManager.updateAllCredentialsWithNewOwnerUuid === 'function') {
+                connectionManager.updateAllCredentialsWithNewOwnerUuid(currentOwner.uuid);
+            }
+            // CRITICAL FIX: Update client's own UUIDs to match server expectations
+            this.logger.info('ClusterService', "\uD83D\uDD04 SERVER-FIRST: Updating client UUIDs to match server state");
+            this.logger.info('ClusterService', "   - Old Client UUID: " + (this.uuid || 'NOT SET'));
+            this.logger.info('ClusterService', "   - Old Owner UUID: " + (this.ownerUuid || 'NOT SET'));
+            // Update client's own UUIDs with server-provided data
+            // The client UUID should match the owner's UUID for authentication
+            this.uuid = currentOwner.uuid;
+            this.ownerUuid = currentOwner.uuid;
+            this.logger.info('ClusterService', "   - New Client UUID: " + this.uuid);
+            this.logger.info('ClusterService', "   - New Owner UUID: " + this.ownerUuid);
+        }
+        // Refresh partition table (KEEPING REFRESH METHOD UNTOUCHED as requested)
+        this.client.getPartitionService().refresh();
+        this.logger.info('ClusterService', "\u2705 SERVER-FIRST: Member " + member.uuid + " at " + member.address.toString() + " credentials stored from server data");
+        // In smart routing mode, proactively open a connection to this member
+        // so it is immediately available for partition-aware routing.
+        if (this.client.getConfig().networkConfig.smartRouting) {
+            var ownerAddress = this.ownerConnection ? this.ownerConnection.getAddress().toString() : null;
+            var memberAddressStr_1 = member.address.toString();
+            if (memberAddressStr_1 !== ownerAddress && !this.client.getConnectionManager().hasConnection(member.address)) {
+                this.logger.info('ClusterService', "\uD83D\uDD17 SMART-ROUTING: Opening connection to newly joined member " + memberAddressStr_1);
+                this.client.getConnectionManager().getOrConnect(member.address, false)
+                    .then(function () {
+                    _this.logger.info('ClusterService', "\u2705 SMART-ROUTING: Connected to newly joined member " + memberAddressStr_1);
+                })
+                    .catch(function (err) {
+                    _this.logger.warn('ClusterService', "\u26A0\uFE0F SMART-ROUTING: Could not connect to newly joined member " + memberAddressStr_1 + ": " + err.message);
+                });
+            }
+        }
+    };
+    /**
+     * Finds the current owner from the cluster state
+     * @returns The current owner member or null if not found
+     */
+    ClusterService.prototype.findCurrentOwner = function () {
+        // Check if we have an active owner connection
+        var ownerConnection = this.ownerConnection;
+        if (ownerConnection && ownerConnection.isAlive()) {
+            var ownerAddress = ownerConnection.getAddress();
+            // Find the member with this address
+            for (var _i = 0, _a = this.members; _i < _a.length; _i++) {
+                var member = _a[_i];
+                if (member.address.toString() === ownerAddress.toString()) {
+                    this.logger.debug('ClusterService', "Found current owner: " + member.uuid + " at " + member.address.toString());
+                    return member;
+                }
+            }
+        }
+        // Fallback: look for any member that might be the owner
+        this.logger.debug('ClusterService', "No active owner connection found, checking member list for potential owner");
+        return null;
+    };
+    /**
+     * Logs the current state for debugging purposes
+     */
+    ClusterService.prototype.logCurrentState = function () {
+        var _this = this;
+        var activeConnections = Object.keys(this.client.getConnectionManager().getEstablishedConnections()).length;
+        var memberCount = this.members.length;
+        var downAddressesCount = this.downAddresses.size;
+        var hasOwner = !!this.ownerConnection;
+        this.logger.info('ClusterService', "Current State - Members: " + memberCount + ", Active Connections: " + activeConnections + ", Down Addresses: " + downAddressesCount + ", Has Owner: " + hasOwner);
+        if (this.ownerConnection) {
+            this.logger.info('ClusterService', "Owner Connection: " + this.ownerConnection.getAddress().toString() + ", Alive: " + this.ownerConnection.isAlive());
+        }
+        // Log all active connections
+        var connections = this.client.getConnectionManager().getEstablishedConnections();
+        Object.keys(connections).forEach(function (addressStr) {
+            var connection = connections[addressStr];
+            _this.logger.debug('ClusterService', "Connection to " + addressStr + ": Alive=" + connection.isAlive() + ", Owner=" + connection.isAuthenticatedAsOwner());
+        });
+        // Log down addresses
+        if (downAddressesCount > 0) {
+            var downAddresses = Array.from(this.downAddresses.keys());
+            this.logger.debug('ClusterService', "Down Addresses: " + downAddresses.join(', '));
+        }
+    };
+    ClusterService.prototype.startReconnectionTask = function () {
+        var _this = this;
+        // Periodically attempt to reconnect to previously failed addresses
+        this.reconnectionTask = setInterval(function () {
+            _this.attemptReconnectionToFailedNodes();
+        }, this.reconnectionInterval);
+    };
+    /**
+     * Starts a periodic task to log the current state for debugging
+     */
+    ClusterService.prototype.startStateLoggingTask = function () {
+        var _this = this;
+        // Log state every 30 seconds for debugging
+        setInterval(function () {
+            if (_this.client.getLifecycleService().isRunning()) {
+                _this.logCurrentState();
+            }
+        }, 30000);
+    };
+    ClusterService.prototype.attemptReconnectionToFailedNodes = function () {
+        var _this = this;
+        // Allow reconnection even during failover, but be more careful
+        if (this.failoverInProgress) {
+            this.logger.debug('ClusterService', 'Skipping reconnection attempt during failover');
+            return;
+        }
+        var now = Date.now();
+        var addressesToReconnect = [];
+        var totalDownAddresses = this.downAddresses.size;
+        // If we have no down addresses, we can skip
+        if (totalDownAddresses === 0) {
+            return;
+        }
+        // Find addresses that are no longer blocked
+        this.downAddresses.forEach(function (downTime, addressStr) {
+            var timeSinceDown = now - downTime;
+            if (timeSinceDown > _this.addressBlockDuration) {
+                // Parse the address string back to Address object
+                try {
+                    var _a = addressStr.split(':'), host = _a[0], portStr = _a[1];
+                    var port = parseInt(portStr, 10);
+                    if (host && !isNaN(port)) {
+                        var address = new Address(host, port);
+                        // Check if we already have a connection to this address
+                        if (_this.client.getConnectionManager().hasConnection(address)) {
+                            _this.logger.debug('ClusterService', "Already have active connection to " + addressStr + ", removing from down addresses");
+                            _this.downAddresses.delete(addressStr);
+                            return;
+                        }
+                        addressesToReconnect.push(address);
+                    }
+                }
+                catch (error) {
+                    _this.logger.warn('ClusterService', "Failed to parse address " + addressStr + " for reconnection:", error);
+                }
+            }
+        });
+        if (addressesToReconnect.length > 0) {
+            this.logger.info('ClusterService', "Attempting to reconnect to " + addressesToReconnect.length + " previously failed nodes: " + addressesToReconnect.map(function (addr) { return addr.toString(); }).join(', '));
+            // Attempt to establish connections to each unblocked address
+            addressesToReconnect.forEach(function (address) {
+                _this.attemptReconnectionToAddress(address);
+            });
+        }
+        else if (totalDownAddresses > 0) {
+            // Log remaining blocked addresses for debugging
+            var remainingBlocked = Array.from(this.downAddresses.keys()).map(function (addr) {
+                var downTime = _this.downAddresses.get(addr);
+                var timeSinceDown = now - downTime;
+                var remainingTime = Math.max(0, _this.addressBlockDuration - timeSinceDown);
+                return addr + " (" + Math.ceil(remainingTime / 1000) + "s remaining)";
+            });
+            this.logger.debug('ClusterService', "Still waiting for " + totalDownAddresses + " addresses to unblock: " + remainingBlocked.join(', '));
+        }
+        // Log current state after reconnection attempts
+        this.logCurrentState();
+    };
+    /**
+     * Attempts to establish a connection to a specific address
+     * @param address The address to reconnect to
+     */
+    ClusterService.prototype.attemptReconnectionToAddress = function (address) {
+        var _this = this;
+        var addressStr = address.toString();
+        // Check if we already have a connection to this address
+        if (this.client.getConnectionManager().hasConnection(address)) {
+            this.logger.debug('ClusterService', "Already have active connection to " + addressStr + ", skipping reconnection");
+            this.downAddresses.delete(addressStr);
+            return;
+        }
+        // Check if we're already trying to connect to this address
+        var connectionManager = this.client.getConnectionManager();
+        var establishedConnections = connectionManager.getEstablishedConnections();
+        var pendingConnections = Object.keys(connectionManager.getPendingConnections()).length;
+        if (pendingConnections > 0) {
+            this.logger.debug('ClusterService', "Already have pending connections, skipping reconnection to " + addressStr);
+            return;
+        }
+        // Remove from down addresses to allow connection attempt
+        this.downAddresses.delete(addressStr);
+        this.logger.debug('ClusterService', "Attempting reconnection to " + addressStr);
+        // Attempt to establish connection (not as owner, just as regular member connection)
+        this.client.getConnectionManager().getOrConnect(address, false)
+            .then(function (connection) {
+            _this.logger.info('ClusterService', "Successfully reconnected to " + addressStr);
+            // Only evaluate ownership change if we don't have an owner or current owner is unhealthy
+            if (!_this.ownerConnection || !_this.ownerConnection.isHealthy()) {
+                _this.logger.info('ClusterService', "Evaluating ownership change for " + addressStr);
+                _this.evaluateOwnershipChange(address, connection);
+            }
+            else {
+                _this.logger.debug('ClusterService', "Keeping " + addressStr + " as member connection, current owner is healthy");
+            }
+            // Trigger partition service refresh to update routing information
+            _this.client.getPartitionService().refresh();
+        }).catch(function (error) {
+            _this.logger.warn('ClusterService', "Reconnection attempt to " + addressStr + " failed:", error);
+            // Mark the address as down again, but with a shorter block duration for reconnection attempts
+            var shorterBlockDuration = Math.min(_this.addressBlockDuration / 2, 15000); // Max 15 seconds
+            _this.markAddressAsDownWithDuration(address, shorterBlockDuration);
+        });
+    };
+    /**
+     * Evaluates whether we should switch ownership to a reconnected node
+     * @param address The address of the reconnected node
+     * @param connection The connection to the reconnected node
+     */
+    ClusterService.prototype.evaluateOwnershipChange = function (address, connection) {
+        // If we don't have an owner connection, this reconnected node becomes the owner
+        if (!this.ownerConnection) {
+            this.logger.info('ClusterService', "Promoting reconnected node " + address.toString() + " to owner status");
+            this.promoteToOwner(connection, address);
+            return;
+        }
+        // If our current owner connection is having issues, consider switching
+        if (this.ownerConnection && !this.ownerConnection.isAlive()) {
+            this.logger.info('ClusterService', "Current owner is unhealthy, switching to reconnected node " + address.toString());
+            this.promoteToOwner(connection, address);
+            return;
+        }
+        // Don't switch ownership if current owner is healthy
+        this.logger.debug('ClusterService', "Current owner is healthy, keeping " + address.toString() + " as member connection");
+    };
+    /**
+     * Promotes a connection to owner status
+     * @param connection The connection to promote
+     * @param address The address of the promoted connection
+     */
+    ClusterService.prototype.promoteToOwner = function (connection, address) {
+        try {
+            // Close the old owner connection if it exists
+            if (this.ownerConnection && this.ownerConnection !== connection) {
+                this.logger.info('ClusterService', "Closing previous owner connection to " + this.ownerConnection.getAddress().toString());
+                this.client.getConnectionManager().destroyConnection(this.ownerConnection.getAddress());
+            }
+            // Set the new owner connection
+            connection.setAuthenticatedAsOwner(true);
+            this.ownerConnection = connection;
+            this.logger.info('ClusterService', "Successfully promoted " + address.toString() + " to owner status");
+            // Refresh partition information with the new owner
+            this.client.getPartitionService().refresh();
+        }
+        catch (error) {
+            this.logger.error('ClusterService', "Failed to promote " + address.toString() + " to owner:", error);
+            // If promotion fails, mark the address as down again
+            this.markAddressAsDown(address);
+        }
+    };
+    /**
+     * Marks an address as down with a custom block duration
+     * @param address The address to mark as down
+     * @param blockDuration The duration to block the address (in milliseconds)
+     */
+    ClusterService.prototype.markAddressAsDownWithDuration = function (address, blockDuration) {
+        var _this = this;
+        var addressStr = address.toString();
+        var now = Date.now();
+        // Don't block if we already have a healthy connection to this address
+        if (this.client.getConnectionManager().hasConnection(address)) {
+            this.logger.debug('ClusterService', "Not blocking " + addressStr + " as we have a healthy connection");
+            return;
+        }
+        // Don't block if this would leave us with no available nodes
+        var totalDownAddresses = this.downAddresses.size;
+        var totalMembers = this.members.length;
+        if (totalDownAddresses >= totalMembers - 1) {
+            this.logger.warn('ClusterService', "Not blocking " + addressStr + " as it would leave us with no available nodes");
+            return;
+        }
+        this.downAddresses.set(addressStr, now);
+        this.logger.warn('ClusterService', "Marked address " + addressStr + " as down, will be blocked for " + blockDuration + "ms");
+        // Schedule cleanup of this address after block duration
+        setTimeout(function () {
+            if (_this.downAddresses.has(addressStr)) {
+                _this.logger.info('ClusterService', "Unblocking address " + addressStr + " after block duration");
+                _this.downAddresses.delete(addressStr);
+            }
+        }, blockDuration);
+    };
+    /**
+     * Handles ownership change when failover occurs
+     * @param newOwnerAddress The address of the new owner
+     * @param newOwnerConnection The connection to the new owner
+     */
+    ClusterService.prototype.handleOwnershipChange = function (newOwnerAddress, newOwnerConnection) {
+        this.logger.info('ClusterService', "Handling ownership change to " + newOwnerAddress.toString());
+        // Update owner connection
+        this.ownerConnection = newOwnerConnection;
+        // Note: Owner UUID will be updated when authentication completes
+        // For now, we'll keep the existing owner UUID
+        // Clear any stale partition information
+        this.client.getPartitionService().clearPartitionTable();
+        // Refresh partition information with the new owner
+        this.client.getPartitionService().refresh();
+        this.logger.info('ClusterService', "Ownership change completed, new owner: " + newOwnerAddress.toString());
+    };
+    /**
+     * Gets the failover manager for external access
+     * @returns The failover manager instance
+     */
+    ClusterService.prototype.getFailoverManager = function () {
+        return this.failoverManager;
+    };
+    ClusterService.prototype.shutdown = function () {
+        if (this.reconnectionTask) {
+            clearInterval(this.reconnectionTask);
+            this.reconnectionTask = null;
+        }
+        this.downAddresses.clear();
+    };
     return ClusterService;
 }());
 exports.ClusterService = ClusterService;