aws_advanced_ruby_driver_wrapper 1.0.0 → 1.1.0

data/lib/aws_advanced_ruby_driver_wrapper/plugins/blue_green/drop_blue_connections_marker.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+#  Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License").
+#  You may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+require 'securerandom'
+
+module AwsAdvancedRubyDriverWrapper
+  module Plugins
+    module BlueGreen
+      # Records that a Blue/Green switchover is retiring the blue environment, so that connections
+      # opened to blue before the switchover can be invalidated the next time they are used.
+      #
+      # Ruby cannot safely abort a connection from another thread, so instead of dropping blue connections
+      # from the monitor thread, each connection checks this marker on its own thread the next time it is
+      # used. See the +bg_drop_blue_connections+ property and BlueGreenPlugin#execute.
+      #
+      # A marker is immutable. It becomes +pending+ when the switchover reaches IN_PROGRESS and is
+      # replaced by an +active+ copy (see #activated) once the switchover is confirmed by POST or
+      # COMPLETED. It is cleared on rollback.
+      class DropBlueConnectionsMarker
+        # @return [String] a unique identity for this switchover's marker, so a connection can tell
+        #   whether it has already been checked against it.
+        attr_reader :id
+
+        # @return [Integer] monotonic time (nanoseconds) at which the switchover reached IN_PROGRESS.
+        attr_reader :switchover_time_ns
+
+        # @return [Set<String>] the blue host set: downcased cluster and instance endpoint names plus
+        #   their pre-switchover IP addresses.
+        attr_reader :blue_hosts
+
+        # @param blue_hosts [Enumerable<String>] blue endpoint names and their pre-switchover IPs
+        # @param switchover_time_ns [Integer] monotonic time the switchover reached IN_PROGRESS
+        # @param id [String] unique identity for this marker
+        # @param active [Boolean] false while the switchover is in progress, true once it is confirmed
+        def initialize(blue_hosts:, switchover_time_ns:, id: SecureRandom.uuid, active: false)
+          @id = id
+          @switchover_time_ns = switchover_time_ns
+          @blue_hosts = blue_hosts.to_set.freeze
+          @active = active
+          freeze
+        end
+
+        def active?
+          @active
+        end
+
+        # @return [DropBlueConnectionsMarker] an active copy of this marker, keeping its identity.
+        def activated
+          return self if @active
+
+          DropBlueConnectionsMarker.new(
+            blue_hosts: @blue_hosts, switchover_time_ns: @switchover_time_ns, id: @id, active: true
+          )
+        end
+
+        # Whether a connection opened to the given host at the given time is now a stale blue connection.
+        #
+        # @param host [String, nil] the downcased connect-time host of the connection
+        # @param open_time_ns [Integer, nil] the monotonic time the connection was opened
+        # @return [Boolean]
+        def stale?(host, open_time_ns)
+          return false unless @active
+          return false if host.nil? || open_time_ns.nil?
+          return false unless open_time_ns < @switchover_time_ns
+
+          @blue_hosts.include?(host)
+        end
+      end
+    end
+  end
+end

data/lib/aws_advanced_ruby_driver_wrapper/plugins/blue_green/host_mapper.rb

@@ -39,8 +39,10 @@ module AwsAdvancedRubyDriverWrapper
           build_cluster_pairs(source_status, target_status)
         end
 
+        # Keyed by downcased host so lookups line up with role_by_host (also downcased); the blue host
+        # set the drop-blue-connections marker is built from indexes this map with downcased endpoints.
         def merge_ips(ip_map)
-          @host_ip_addresses.merge!(ip_map)
+          ip_map.each { |host, ip| @host_ip_addresses[host.downcase] = ip }
         end
 
         def register_role(host_names, role)

data/lib/aws_advanced_ruby_driver_wrapper/plugins/blue_green/routing/base_routing.rb

@@ -15,6 +15,7 @@
 #  limitations under the License.
 
 require_relative '../../../logging'
+require_relative '../../../utils/telemetry/telemetry_const'
 
 module AwsAdvancedRubyDriverWrapper
   module Plugins
@@ -24,6 +25,7 @@ module AwsAdvancedRubyDriverWrapper
           include Logging
 
           MINIMUM_SLEEP_CHUNK_SEC = 0.05
+          TELEMETRY_SWITCHOVER = 'Blue/Green Deployment switchover'
 
           def match?(host_info, role)
             (@host.nil? || @host.downcase == host_info&.host&.downcase) &&
@@ -39,6 +41,11 @@ module AwsAdvancedRubyDriverWrapper
               "bgd_id: #{defined?(@bgd_id) ? (@bgd_id || '<null>') : '<n/a>'}]"
           end
 
+          # Records the time a call is held while the switchover is in progress as a span.
+          def in_switchover_span(telemetry_factory, &)
+            telemetry_factory.in_context(TELEMETRY_SWITCHOVER, Utils::Telemetry::TraceLevel::NESTED, &)
+          end
+
           # Blocks until the Blue/Green switchover phase is no longer IN_PROGRESS, or until the
           # configured timeout elapses.
           def wait_while_in_progress(wrapper_props, storage_service, bgd_id, sleep_time_sec = MINIMUM_SLEEP_CHUNK_SEC)

data/lib/aws_advanced_ruby_driver_wrapper/plugins/blue_green/routing/substitute_connect_routing.rb

@@ -52,57 +52,37 @@ module AwsAdvancedRubyDriverWrapper
                     bg_plugin: nil)
             plugin_manager = service_container.plugin_manager
             dialect_service = service_container.dialect_service
+            driver_dialect = dialect_service.driver_dialect
 
             unless Utils::RdsUtils.ip?(@substitute_host.host)
               return open_connection(plugin_manager, @substitute_host, driver_props, wrapper_props, is_initial_connection, is_internal,
                                      bg_plugin)
             end
 
-            # mysql2 uses :host for both socket connection and TLS CN verification.
-            # Connecting to a raw IP with TLS active causes a CN mismatch.
-            # Fall back to connecting via hostname (DNS will resolve to the correct IP post-switchover),
-            # matching the effective behaviour of the Java/Go MySQL drivers which use the original
-            # hostname for TLS regardless of the socket address.
-            if driver_props[:sslca] && !plugin_manager.plugin_in_use?(Plugins::IamAuthPlugin)
+            # The substitute host is a raw IP, so a connection made by hostname could resolve (via DNS)
+            # to the old instance during the switchover window. Connect to the IP directly instead.
+            if plugin_manager.iam_token_plugin_in_use?
+              return apply_iam(service_container, driver_props, wrapper_props, is_initial_connection, bg_plugin)
+            end
+
+            # pg can pin the socket to the IP (hostaddr) while keeping the original hostname for TLS
+            # verification, so the pinned connection reaches the right instance and still passes
+            # verify-full whatever the CA source.
+            pinned_props = driver_dialect.props_pinned_to_ip(driver_props, @substitute_host.host)
+            if pinned_props
               hostname_host = @substitute_host.deep_dup(host: @host)
-              return open_connection(plugin_manager, hostname_host, driver_props, wrapper_props, is_initial_connection, is_internal,
+              return open_connection(plugin_manager, hostname_host, pinned_props, wrapper_props, is_initial_connection, is_internal,
                                      bg_plugin)
             end
 
-            if plugin_manager.plugin_in_use?(Plugins::IamAuthPlugin)
-              if @iam_hosts.nil? || @iam_hosts.empty?
-                raise StandardError,
-                      'Connecting with IP address when IAM authentication is enabled requires an \'iam_host\' parameter.'
-              end
-
-              @iam_hosts.each do |iam_host|
-                rerouted_host = @substitute_host.deep_dup
-                rerouted_host.id = iam_host.id
-                rerouted_host.availability = Host::HostAvailability::AVAILABLE
-
-                rerouted_wrapper_props = wrapper_props.dup
-                PropertyDefinition::IAM_HOST.set(rerouted_wrapper_props, iam_host.host)
-                PropertyDefinition::IAM_PORT.set(rerouted_wrapper_props, iam_host.port.to_s) if iam_host.port_specified?
-
-                begin
-                  conn = plugin_manager.internal_connect(
-                    rerouted_host,
-                    driver_props,
-                    rerouted_wrapper_props,
-                    is_initial_connection,
-                    plugin_to_skip: bg_plugin
-                  )
-                  begin
-                    @iam_successful_connect_notify&.call(iam_host.host)
-                  rescue StandardError
-                    nil
-                  end
-                  return conn
-                rescue StandardError => e
-                  raise unless dialect_service.login_error?(e)
-                end
-              end
-              return nil
+            # mysql2/trilogy use :host for both the socket and TLS CN verification, so connecting to a
+            # raw IP with TLS active causes a CN mismatch. When TLS is active, fall back to the hostname
+            # (DNS resolves to the correct IP post-switchover), matching the effective behaviour of the
+            # Java/Go drivers which use the original hostname for TLS regardless of the socket address.
+            if driver_props[driver_dialect.tls_ca_property_key]
+              hostname_host = @substitute_host.deep_dup(host: @host)
+              return open_connection(plugin_manager, hostname_host, driver_props, wrapper_props, is_initial_connection, is_internal,
+                                     bg_plugin)
             end
 
             open_connection(plugin_manager, @substitute_host, driver_props, wrapper_props, is_initial_connection, is_internal, bg_plugin)
@@ -114,6 +94,53 @@ module AwsAdvancedRubyDriverWrapper
 
           private
 
+          # Connects to the substitute IP with IAM auth active, trying each configured IAM host in turn
+          # so the IAM plugin can resolve the correct token hostname. pg pins the socket to the IP
+          # (hostaddr) while presenting the IAM host name for TLS verification; the IAM token is still
+          # generated for the IAM host via the IAM_HOST property. Other drivers connect to the IP as-is.
+          def apply_iam(service_container, driver_props, wrapper_props, is_initial_connection, bg_plugin)
+            plugin_manager = service_container.plugin_manager
+            dialect_service = service_container.dialect_service
+            driver_dialect = dialect_service.driver_dialect
+
+            if @iam_hosts.nil? || @iam_hosts.empty?
+              raise StandardError,
+                    'Connecting with IP address when IAM authentication is enabled requires an \'iam_host\' parameter.'
+            end
+
+            @iam_hosts.each do |iam_host|
+              rerouted_host = @substitute_host.deep_dup
+              rerouted_host.id = iam_host.id
+              rerouted_host.availability = Host::HostAvailability::AVAILABLE
+
+              rerouted_wrapper_props = wrapper_props.dup
+              PropertyDefinition::IAM_HOST.set(rerouted_wrapper_props, iam_host.host)
+              PropertyDefinition::IAM_PORT.set(rerouted_wrapper_props, iam_host.port.to_s) if iam_host.port_specified?
+
+              connect_props = driver_props
+              pinned_props = driver_dialect.props_pinned_to_ip(driver_props, @substitute_host.host)
+              if pinned_props
+                connect_props = pinned_props
+                rerouted_host = rerouted_host.deep_dup(host: iam_host.host)
+              end
+
+              begin
+                conn = plugin_manager.internal_connect(
+                  rerouted_host, connect_props, rerouted_wrapper_props, is_initial_connection, plugin_to_skip: bg_plugin
+                )
+                begin
+                  @iam_successful_connect_notify&.call(iam_host.host)
+                rescue StandardError
+                  nil
+                end
+                return conn
+              rescue StandardError => e
+                raise unless dialect_service.login_error?(e)
+              end
+            end
+            nil
+          end
+
           # Opens a connection through the plugin pipeline. Monitoring connections (is_internal: true)
           # must use internal_connect so they do NOT mutate the shared current connection — otherwise
           # one monitor thread's connect would close another monitor thread's live connection via

data/lib/aws_advanced_ruby_driver_wrapper/plugins/blue_green/routing/suspend_connect_routing.rb

@@ -35,8 +35,10 @@ module AwsAdvancedRubyDriverWrapper
             storage_service = service_container.storage_service
             hold_start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
 
-            wait_while_in_progress(wrapper_props, storage_service, @bgd_id, SLEEP_TIME_SEC) do |timeout_sec|
-              "Blue/Green Deployment switchover is still in progress after #{timeout_sec} s. Try to connect again later."
+            in_switchover_span(service_container.telemetry_factory) do
+              wait_while_in_progress(wrapper_props, storage_service, @bgd_id, SLEEP_TIME_SEC) do |timeout_sec|
+                "Blue/Green Deployment switchover is still in progress after #{timeout_sec} s. Try to connect again later."
+              end
             end
 
             elapsed_ms = ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - hold_start_time) * 1000).round

data/lib/aws_advanced_ruby_driver_wrapper/plugins/blue_green/routing/suspend_execute_routing.rb

@@ -31,11 +31,13 @@ module AwsAdvancedRubyDriverWrapper
           end
 
           # Blocks until switchover is no longer IN_PROGRESS
-          def apply(method_name, wrapper_props, storage_service)
+          def apply(method_name, wrapper_props, storage_service, telemetry_factory:)
             hold_start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
 
-            wait_while_in_progress(wrapper_props, storage_service, @bgd_id, SLEEP_TIME_SEC) do |timeout_sec|
-              "Blue/Green Deployment switchover is still in progress after #{timeout_sec} s. Try '#{method_name}' again later."
+            in_switchover_span(telemetry_factory) do
+              wait_while_in_progress(wrapper_props, storage_service, @bgd_id, SLEEP_TIME_SEC) do |timeout_sec|
+                "Blue/Green Deployment switchover is still in progress after #{timeout_sec} s. Try '#{method_name}' again later."
+              end
             end
 
             elapsed_ms = ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - hold_start_time) * 1000).round

data/lib/aws_advanced_ruby_driver_wrapper/plugins/blue_green/routing/suspend_until_corresponding_host_found_connect_routing.rb

@@ -35,8 +35,14 @@ module AwsAdvancedRubyDriverWrapper
           # Blocks until a corresponding green host is found for host_info.host, the switchover
           # completes, or the timeout elapses.
           def apply(host_info, _, wrapper_props, _is_initial_connection, service_container, is_internal: false, bg_plugin: nil)
-            storage_service = service_container.storage_service
+            in_switchover_span(service_container.telemetry_factory) do
+              hold_until_corresponding_host_found(host_info, wrapper_props, service_container.storage_service)
+            end
+          end
+
+          private
 
+          def hold_until_corresponding_host_found(host_info, wrapper_props, storage_service)
             bg_status = storage_service.get(BlueGreenPlugin::BLUE_GREEN_NAME, @bgd_id)
             corresponding_pair = bg_status&.corresponding_hosts&.[](host_info.host)
 

data/lib/aws_advanced_ruby_driver_wrapper/plugins/blue_green/status_builder.rb

@@ -223,7 +223,7 @@ module AwsAdvancedRubyDriverWrapper
             next [] if host_pair.nil?
 
             blue_host_spec    = host_pair[0]
-            blue_ip_entry     = @host_mapper.host_ip_addresses[blue_host_spec.host]
+            blue_ip_entry     = @host_mapper.host_ip_addresses[blue_host_spec.host.downcase]
             blue_ip_host_spec = blue_ip_entry ? Host::HostInfo.new(host: blue_ip_entry, port: blue_host_spec.port,
                                                                    role: blue_host_spec.role) : blue_host_spec
 

data/lib/aws_advanced_ruby_driver_wrapper/plugins/blue_green/status_monitor.rb

@@ -338,6 +338,10 @@ module AwsAdvancedRubyDriverWrapper
           @event.reset
         end
 
+        def abandon_connections
+          @connection.abandon
+        end
+
         private
 
         def attempt_open_connection
@@ -390,7 +394,7 @@ module AwsAdvancedRubyDriverWrapper
         end
 
         def iam_enabled?
-          @service_container.plugin_manager.plugin_in_use?(Plugins::IamAuthPlugin)
+          @service_container.plugin_manager.iam_token_plugin_in_use?
         end
 
         def driver_dialect
@@ -417,14 +421,6 @@ module AwsAdvancedRubyDriverWrapper
           true
         end
 
-        def syntax_error?(err)
-          driver_dialect.sql_state(err)&.start_with?('42') || false
-        end
-
-        def db_error?(err)
-          !driver_dialect.sql_state(err).nil?
-        end
-
         # @note Callers must hold @connection_mutex (invoke within a @connection_mutex.synchronize block).
         def status_available?(conn)
           dialect = @service_container.dialect_service.db_dialect
@@ -504,11 +500,11 @@ module AwsAdvancedRubyDriverWrapper
         end
 
         def handle_collect_status_error(err, conn)
-          if syntax_error?(err)
+          if @service_container.dialect_service.schema_error?(err)
             @current_phase = Phase::NOT_CREATED
             logger.warn { "[#{@role}] current_phase: #{@current_phase}, exception while querying for blue/green status." }
-          elsif db_error?(err)
-            if err.message.include?('An error occurred while retrieving the blue/green fast switchover metadata')
+          elsif @service_container.dialect_service.database_error?(err)
+            if err.message.include?('while retrieving the blue/green fast switchover metadata')
               @current_phase = Phase::NOT_CREATED
               return
             end
@@ -553,7 +549,8 @@ module AwsAdvancedRubyDriverWrapper
             @service_container.plugin_manager,
             @service_container.session_state_service,
             @service_container.storage_service,
-            @service_container.monitor_service
+            @service_container.monitor_service,
+            @service_container.telemetry_factory
           )
 
           @host_list_provider = @service_container.dialect_service.db_dialect.create_host_list_provider(scoped_container)

data/lib/aws_advanced_ruby_driver_wrapper/plugins/blue_green/status_provider.rb

@@ -18,6 +18,7 @@ require 'concurrent'
 require_relative '../../logging'
 require_relative '../../property_definition'
 require_relative '../../utils/rds_utils'
+require_relative 'drop_blue_connections_marker'
 require_relative 'iam_host_tracker'
 require_relative 'interval_rate'
 require_relative 'host_mapper'
@@ -61,6 +62,10 @@ module AwsAdvancedRubyDriverWrapper
           @process_status_lock   = Mutex.new
           @monitor_generation    = 0
 
+          @drop_blue_connections = PropertyDefinition::BG_DROP_BLUE_CONNECTIONS.get_bool(wrapper_props)
+          # Identity of the drop-blue-connections marker created for the current switchover, if any.
+          @drop_marker_id        = nil
+
           @status_check_interval_map = {
             IntervalRate::BASELINE => PropertyDefinition::BG_INTERVAL_BASELINE_SEC.get_float(wrapper_props) * 1000,
             IntervalRate::INCREASED => PropertyDefinition::BG_INTERVAL_INCREASED_SEC.get_float(wrapper_props) * 1000,
@@ -93,6 +98,11 @@ module AwsAdvancedRubyDriverWrapper
           @monitors.each_value { |m| m&.stop }
         end
 
+        # Detaches the monitors inherited by a forked child without closing the parent's connections.
+        def release_after_fork
+          @monitors.each_value { |m| m&.release_after_fork }
+        end
+
         def log_switchover_final_summary
           return unless switchover_finalized?
 
@@ -171,6 +181,18 @@ module AwsAdvancedRubyDriverWrapper
 
           logger.warn { 'Blue/Green switchover has timed out.' }
           @summary_status = @rollback ? @builder.created : @builder.completed(@rollback)
+          # A timeout means the wrapper stopped waiting, not that the switchover is confirmed. Only
+          # activate the marker if the switchover was last observed at POST (past the point of no
+          # return); timing out in PREPARATION/IN_PROGRESS could still roll back, and dropping blue
+          # connections then would hit a blue that is still primary. In that case discard the pending
+          # marker so it does not linger unactivated for the full marker TTL: we have given up waiting
+          # on this switchover, and no later phase will arrive to activate it.
+          if @latest_status_phase == Phase::POST
+            activate_drop_marker
+          else
+            remove_drop_marker
+            @drop_marker_id = nil
+          end
           update_monitors
           update_status_cache
           log_current_context
@@ -186,7 +208,15 @@ module AwsAdvancedRubyDriverWrapper
           @monitors = { Role::SOURCE => nil, Role::TARGET => nil }
           old_monitors.each_value { |m| m&.stop }
 
-          Plugins::IamAuthPlugin.clear_cache(@storage_service) if iam_enabled?
+          # A completed switchover's marker must survive this reset so connections opened to the old
+          # blue environment are still invalidated on their next use. A rollback leaves blue as primary,
+          # so its marker is discarded instead. Either way a later switchover starts a fresh marker.
+          remove_drop_marker if @rollback
+          @drop_marker_id = nil
+
+          # IamAuthPlugin, FederatedAuthPlugin, and OktaAuthPlugin all cache tokens under the same
+          # shared IAM_TOKEN_CACHE_NAME, so clearing through any one of them clears all of them.
+          Plugins::IamAuthPlugin.clear_cache(@storage_service) if iam_token_cache_in_use?
 
           @rollback = false
           @green_topology_recognized_logged = false
@@ -220,12 +250,15 @@ module AwsAdvancedRubyDriverWrapper
             @switchover_state.update_dns_flags(@bgd_id, role, interim_status)
             @summary_status = @builder.in_progress(@rollback)
             @switchover_state.trigger_in_progress_monitor_reset
+            ensure_drop_marker_pending
           when Phase::POST
             @switchover_state.update_dns_flags(@bgd_id, role, interim_status)
             @summary_status = @builder.post(@rollback)
+            activate_drop_marker
           when Phase::COMPLETED
             @switchover_state.update_dns_flags(@bgd_id, role, interim_status)
             @summary_status = @builder.completed(@rollback)
+            activate_drop_marker
           else
             raise ArgumentError, "[bgd_id: '#{@bgd_id}'] Unknown BG phase '#{@latest_status_phase}'."
           end
@@ -345,14 +378,73 @@ module AwsAdvancedRubyDriverWrapper
           )
         end
 
+        # Creates a pending drop-blue-connections marker the first time a switchover reaches IN_PROGRESS.
+        # The marker is only confirmed (see #activate_drop_marker) once the switchover passes the point
+        # of no return, so nothing is invalidated while it is still in progress and could roll back.
+        def ensure_drop_marker_pending
+          return unless @drop_blue_connections
+          return if @rollback
+          return if @drop_marker_id
+
+          hosts = blue_host_set
+          return if hosts.empty?
+
+          marker = DropBlueConnectionsMarker.new(blue_hosts: hosts, switchover_time_ns: nano_time)
+          @drop_marker_id = marker.id
+          store_drop_marker(marker)
+        end
+
+        # Confirms the switchover's pending marker so stale blue connections begin to be invalidated.
+        # A marker is only created when IN_PROGRESS is observed (see #ensure_drop_marker_pending), since
+        # that is the only point with a reliable switchover time. A provider that first sees the
+        # deployment at POST/COMPLETED cannot tell which of its connections predate the switchover and
+        # would wrongly cover connections opened after switchover, including to green.
+        def activate_drop_marker
+          return unless @drop_blue_connections
+          return if @rollback
+
+          marker = read_drop_marker
+          return if marker.nil? || marker.active?
+
+          store_drop_marker(marker.activated)
+        end
+
+        # The blue host set: downcased SOURCE endpoint names plus their pre-switchover IP addresses.
+        def blue_host_set
+          return Set.new unless @summary_status
+
+          blue = @summary_status.role_by_host.select { |_host, role| role == Role::SOURCE }.keys
+          ips  = blue.filter_map { |host| @host_mapper.host_ip_addresses[host] }
+          (blue + ips).to_set(&:downcase)
+        end
+
+        def store_drop_marker(marker)
+          @storage_service.set(BlueGreenPlugin::DROP_MARKER_NAME, BlueGreenPlugin.drop_marker_key(@bgd_id), marker)
+        end
+
+        def read_drop_marker
+          @storage_service.get(BlueGreenPlugin::DROP_MARKER_NAME, BlueGreenPlugin.drop_marker_key(@bgd_id), register_access: false)
+        end
+
+        def remove_drop_marker
+          @storage_service.remove(BlueGreenPlugin::DROP_MARKER_NAME, BlueGreenPlugin.drop_marker_key(@bgd_id))
+        end
+
+        def nano_time
+          Process.clock_gettime(Process::CLOCK_MONOTONIC, :nanosecond)
+        end
+
         def switchover_finalized?
           completed = (!@rollback && @summary_status.current_phase == Phase::COMPLETED) ||
                       (@rollback && @summary_status.current_phase == Phase::CREATED)
           completed && @event_log.any? { |_, v| v.phase&.active_switchover_or_completed? }
         end
 
-        def iam_enabled?
-          @service_container.plugin_manager&.plugin_in_use?(Plugins::IamAuthPlugin) || false
+        # True when any auth plugin backed by the shared IAM token cache (IAM or AD FS
+        # or Okta) is in use, so a switchover knows to purge that cache regardless
+        # of which one is active.
+        def iam_token_cache_in_use?
+          @service_container.plugin_manager&.iam_token_plugin_in_use? || false
         end
 
         def context_hash

data/lib/aws_advanced_ruby_driver_wrapper/plugins/custom_endpoint/custom_endpoint_monitor.rb

@@ -41,7 +41,14 @@ module AwsAdvancedRubyDriverWrapper
           refresh_rate_sec,
           refresh_rate_backoff_factor,
           max_refresh_rate_sec,
-          rds_client_func: ->(_, region) { Aws::RDS::Client.new(region: region) }
+          credentials_provider: nil,
+          rds_client_func: lambda { |_, client_region|
+            # Pass the configured credentials provider when one is set; otherwise omit :credentials
+            # so the SDK resolves from its default chain.
+            options = { region: client_region }
+            options[:credentials] = credentials_provider if credentials_provider
+            Aws::RDS::Client.new(**options)
+          }
         )
           super(termination_timeout_sec: TERMINATION_TIMEOUT_SEC)
 
@@ -58,6 +65,7 @@ module AwsAdvancedRubyDriverWrapper
           @refresh_cv = ConditionVariable.new
           @refresh_required = false
           @connection_issue = false
+          @info_changed_counter = service_container.telemetry_factory.create_counter('custom_endpoint.info_changed.count')
         end
 
         def endpoint_info?
@@ -161,6 +169,7 @@ module AwsAdvancedRubyDriverWrapper
           logger.debug("[#{@endpoint_id}] Custom endpoint info changed: #{endpoint_info}")
           cache_allowed_blocked(endpoint_info)
           cache_info(endpoint_info)
+          @info_changed_counter.inc
           speedup_refresh_rate
 
           elapsed = monotonic_time - start
@@ -241,6 +250,7 @@ module AwsAdvancedRubyDriverWrapper
         end
 
         def cached_info
+          # Explicitly unregistered: a monitor reading its own data must not keep itself alive.
           storage_service.get(ENDPOINT_INFO_CACHE_NAME, @custom_endpoint_host.url, register_access: false)
         end
 

data/lib/aws_advanced_ruby_driver_wrapper/plugins/custom_endpoint/custom_endpoint_plugin.rb

@@ -38,6 +38,7 @@ module AwsAdvancedRubyDriverWrapper
           @props = props
           @should_wait_for_info = PropertyDefinition::WAIT_FOR_CUSTOM_ENDPOINT_INFO.get_bool(props)
           @wait_timeout_sec = PropertyDefinition::WAIT_FOR_CUSTOM_ENDPOINT_INFO_TIMEOUT_SEC.get_float(props)
+          @wait_for_info_counter = service_container.telemetry_factory.create_counter('custom_endpoint.wait_for_info.count')
 
           @custom_endpoint_host = nil
           @endpoint_id = nil
@@ -128,7 +129,8 @@ module AwsAdvancedRubyDriverWrapper
               @region,
               PropertyDefinition::CUSTOM_ENDPOINT_INFO_REFRESH_RATE_SEC.get_float(props),
               PropertyDefinition::CUSTOM_ENDPOINT_INFO_REFRESH_RATE_BACKOFF_FACTOR.get_int(props),
-              PropertyDefinition::CUSTOM_ENDPOINT_INFO_MAX_REFRESH_RATE_SEC.get_float(props)
+              PropertyDefinition::CUSTOM_ENDPOINT_INFO_MAX_REFRESH_RATE_SEC.get_float(props),
+              credentials_provider: PropertyDefinition::AWS_CREDENTIALS_PROVIDER.get(props)
             )
           end
         end
@@ -137,6 +139,7 @@ module AwsAdvancedRubyDriverWrapper
           return if monitor.endpoint_info?
 
           monitor.request_endpoint_info_update
+          @wait_for_info_counter.inc
           logger.debug("CustomEndpointPlugin: waiting up to #{@wait_timeout_sec}s for endpoint info on #{@custom_endpoint_host.url}")
 
           return if monitor.wait_for_info?(@wait_timeout_sec)

data/lib/aws_advanced_ruby_driver_wrapper/plugins/default_plugin.rb

@@ -18,6 +18,7 @@ require 'concurrent'
 require_relative '../errors'
 require_relative '../host/host_info'
 require_relative '../host/host_availability'
+require_relative '../utils/telemetry/telemetry_const'
 
 module AwsAdvancedRubyDriverWrapper
   module Plugins
@@ -44,7 +45,7 @@ module AwsAdvancedRubyDriverWrapper
                              host_info
                            end
 
-        conn = driver_dialect.connect(target_host_info, driver_props)
+        conn = with_driver_span { driver_dialect.connect(target_host_info, driver_props) }
 
         if conn.nil?
           raise Errors::AwsError,
@@ -83,7 +84,7 @@ module AwsAdvancedRubyDriverWrapper
 
       def internal_connect(host_info, driver_props, _wrapper_props, _is_initial_connection, _pipeline_callable)
         driver_dialect = @service_container.dialect_service.driver_dialect
-        driver_dialect.connect(host_info, driver_props)
+        with_driver_span { driver_dialect.connect(host_info, driver_props) }
       end
 
       def execute(target_method_name, target_callable, *args, **, &)
@@ -92,7 +93,7 @@ module AwsAdvancedRubyDriverWrapper
         succeeded = false
 
         begin
-          result = target_callable.call(*args, **, &)
+          result = with_driver_span { target_callable.call(*args, **, &) }
           succeeded = true
           result
         ensure
@@ -103,6 +104,16 @@ module AwsAdvancedRubyDriverWrapper
           end
         end
       end
+
+      private
+
+      # Records the time spent in the driver itself as a span named after the driver.
+      def with_driver_span(&)
+        telemetry_factory = @service_container.telemetry_factory
+        return yield unless telemetry_factory.traces_in_use?
+
+        telemetry_factory.in_context(@service_container.dialect_service.driver_dialect.name, Utils::Telemetry::TraceLevel::NESTED, &)
+      end
     end
   end
 end