mysql_framework 2.1.9 → 2.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 91ca6641d16e5f4d344b6749fe9f5291c8f24b23f4f8efb28ac8e76443d61750
-  data.tar.gz: 961c49140b91c9ea461a1512d63bb89a5861ed5375386e21e21be1c0e1691f2b
+  metadata.gz: 2e86d1afb0cbbd571d728e469bf5fffa14ad3e0f89f268dcf951564aca98043a
+  data.tar.gz: be53e02e828f37e5d7c651b1526b353e8dc599df0396b6b28f50632efecbfd9f
 SHA512:
-  metadata.gz: 3b3b3b5ebf141f85110733c54cf6c190e8c091bb681d0e9fc555643efaca73cc29f61964f7ce06142ed48fa466bb49c1f616ff48ba3d44223db12d414e6bcc81
-  data.tar.gz: a0119d3cb52bc3d513054fe803f9627dfd6ca0c20f51dc602fcfe36897227bb6ea29933cff88599f1fbb5d7019bad8cf971aedf887b216fbd5a07c55e39f8b33
+  metadata.gz: 2e4fae53d9831f913be250ace9a5b4d5ad85e89433561b7be804a11b0e30f6faf434195fb34b2b4c8d31b9669ac486da07e5dd5a65ee17ca1d52d172bd13b824
+  data.tar.gz: 4ee13ac87b94966188ddd33a2d9b090192f7bddac0e4bf7623d14553176fa1706fa58b83728fc89746383c60e12165f4f092052013bffb51a03218895269ec21

data/lib/mysql_framework/connector.rb

@@ -1,133 +1,171 @@
 # frozen_string_literal: true
 
+require_relative 'mysql_connection_pool'
+
 module MysqlFramework
   class Connector
+    attr_reader :connection_pool
+
+    # Initializes a connector instance with MySQL client options.
+    #
+    # @param options [Hash] custom MySQL client options that override defaults
+    # @return [void]
     def initialize(options = {})
       @options = default_options.merge(options)
-      @mutex = Mutex.new
-
       Mysql2::Client.default_query_options.merge!(symbolize_keys: true, cast_booleans: true)
     end
 
-    # This method is called to setup a pool of MySQL connections.
+    # Sets up the MySQL connection pool when pooling is enabled.
+    #
+    # @return [ConnectionPool, nil] configured pool, or nil when pooling is disabled
     def setup
       return unless connection_pool_enabled?
 
-      @connection_pool = ::Queue.new
-
-      start_pool_size.times { @connection_pool.push(new_client) }
-
-      @created_connections = start_pool_size
+      @connection_pool = MysqlFramework::MysqlConnectionPool.new(@options)
+      @connection_pool.setup
     end
 
-    # This method is called to close all MySQL connections in the pool and dispose of the pool itself.
+    # Disposes of the connection pool and closes pooled connections.
+    #
+    # @return [void]
     def dispose
-      return if @connection_pool.nil?
-
-      until @connection_pool.empty?
-        conn = @connection_pool.pop(true)
-        conn&.close
-      end
+      return unless connection_pool_enabled?
 
+      @connection_pool&.dispose
       @connection_pool = nil
     end
 
-    # This method is called to get the idle connection queue for this connector.
-    def connections
-      @connection_pool
-    end
-
-    # This method is called to fetch a client from the connection pool.
+    # Checks out a MySQL client, sanitizing it before use.
+    #
+    # @return [Mysql2::Client] checked-out client
+    # @raise [ConnectionSanitizationError] when sanitization repeatedly fails
+    # @raise [Mysql2::Error] when checkout or sanitization fails due to MySQL errors
     def check_out
-      @mutex.synchronize do
-        begin
-          return new_client unless connection_pool_enabled?
-
-          client = @connection_pool.pop(true)
-
-          client.ping if @options[:reconnect]
+      return new_client unless connection_pool_enabled?
 
-          client
-        rescue ThreadError
-          if @created_connections < max_pool_size
-            client = new_client
-            @created_connections += 1
-            return client
-          end
-
-          MysqlFramework.logger.error { "[#{self.class}] - Database connection pool depleted." }
-
-          raise 'Database connection pool depleted.'
-        end
-      end
+      @connection_pool.check_out
     end
 
-    # This method is called to check a client back in to the connection when no longer needed.
+    # Returns a MySQL client back to the pool or closes it when pooling is disabled.
+    #
+    # @param client [Mysql2::Client, nil] client to return or close
+    # @return [void]
     def check_in(client)
-      @mutex.synchronize do
-        return client&.close unless connection_pool_enabled?
+      return client&.close unless connection_pool_enabled?
 
-        client = new_client if client&.closed?
-        @connection_pool.push(client)
-      end
+      @connection_pool.check_in(client)
     end
 
-    # This method is called to use a client from the connection pool.
-    def with_client(provided = nil)
-      client = provided || check_out
-      yield client
-    ensure
-      check_in(client) unless provided
+    # Yields a MySQL client from the pool, or yields the provided client directly.
+    #
+    # @param provided_client [Mysql2::Client, nil] existing client to yield without pool checkout
+    # @param discard_current_pool_connection [Boolean] whether to discard the pooled connection after use
+    # @yield [client] block that performs work with a MySQL client
+    # @yieldparam client [Mysql2::Client]
+    # @return [Object] block result
+    # @raise [Mysql2::Error] re-raises MySQL errors from the block
+    def with_client(provided_client = nil, discard_current_pool_connection: false)
+      return yield provided_client if provided_client
+      return with_new_client { |c| yield c } unless connection_pool_enabled?
+
+      @connection_pool.with_client(discard_current_pool_connection:) { |c| yield c }
     end
 
-    # This method is called to execute a prepared statement
+    # Executes a prepared statement.
+    #
+    # @param query [Object] query object responding to +sql+ and +params+
+    # @param provided_client [Mysql2::Client, nil] optional existing client
+    # @return [Array<Hash>, nil] query result rows
+    # @raise [Mysql2::Error] when statement preparation or execution fails
     #
-    # @note Ensure we free any result and close each statement, otherwise we
-    # can run into a 'Commands out of sync' error if multiple threads are
-    # running different queries at the same time.
+    # NOTE:
+    # We must always free the result and close the prepared statement.
+    # Otherwise MySQL may raise "Commands out of sync" when the same
+    # connection is reused (e.g. via connection pooling).
+    #
+    # The connection itself must NOT be closed here because it is
+    # managed by the connection pool.
     def execute(query, provided_client = nil)
       with_client(provided_client) do |client|
+        statement = nil
+        result = nil
+
         begin
           statement = client.prepare(query.sql)
-          result = statement.execute(*query.params)
-          result&.to_a
+          result = statement.execute(
+            *query.params, symbolize_keys: true, cast_booleans: true
+          )
+          final = result&.to_a
+          final
         ensure
+          client&.abandon_results!
           result&.free
           statement&.close
         end
       end
     end
 
-    # This method is called to execute a query
+    # Executes a SQL query.
+    #
+    # @param query_string [String] SQL query to execute
+    # @param provided_client [Mysql2::Client, nil] optional existing client
+    # @return [Mysql2::Result] raw MySQL result
+    # @raise [Mysql2::Error] when query execution fails
     def query(query_string, provided_client = nil)
-      with_client(provided_client) { |client| client.query(query_string) }
+      with_client(provided_client) { |conn| conn.query(query_string) }
     end
 
-    # This method is called to execute a query which will return multiple result sets in an array
+    # Executes a multi-statement SQL query and collects all result sets.
+    #
+    # @param query_string [String] multi-statement SQL query
+    # @param provided_client [Mysql2::Client, nil] optional existing client
+    # @return [Array<Array<Hash>>] list of result sets
+    # @raise [Mysql2::Error] when query execution or result fetching fails
     def query_multiple_results(query_string, provided_client = nil)
-      results = with_client(provided_client) do |client|
-        result = []
-        result << client.query(query_string)
-        result << client.store_result while client.next_result
-        result.compact
+      results = nil
+
+      # Multiple statement query is buggy and client cannot be reused after calling next_result/store_result
+      # Client's state gets corrupted and leaks into next queries. The reason is unknown.
+      # As a result we do not return client back to the pool but instead close connection which is not optimal.
+      with_client(provided_client, discard_current_pool_connection: true) do |client|
+        raw_results = []
+        query_call = client.query(query_string)
+        raw_results << query_call&.to_a
+        query_call&.free
+
+        while client.more_results?
+          client.next_result
+          query_call = client.store_result
+          raw_results << query_call&.to_a
+          query_call&.free
+        end
+
+        results = raw_results.compact
+        results
+      ensure
+        client&.abandon_results!
       end
 
-      results.map(&:to_a)
+      results
     end
 
-    # This method is called to use a client within a transaction
+    # Executes a block within a database transaction.
+    #
+    # @yield [client] block executed between BEGIN and COMMIT
+    # @yieldparam client [Mysql2::Client]
+    # @return [Object] block result
+    # @raise [ArgumentError] when no block is given
+    # @raise [StandardError] re-raises any exception after rollback
     def transaction
       raise ArgumentError, 'No block was given' unless block_given?
 
       with_client do |client|
-        begin
-          client.query('BEGIN')
-          yield client
-          client.query('COMMIT')
-        rescue StandardError => e
-          client.query('ROLLBACK')
-          raise e
-        end
+        client.query('BEGIN')
+        yield client
+        client.query('COMMIT')
+      rescue StandardError => e
+        client.query('ROLLBACK')
+        raise e
       end
     end
 
@@ -146,20 +184,21 @@ module MysqlFramework
       }
     end
 
+    def with_new_client
+      client = new_client
+      yield client
+    ensure
+      client&.close
+    end
+
     def new_client
       Mysql2::Client.new(@options)
     end
 
     def connection_pool_enabled?
-      @connection_pool_enabled ||= ENV.fetch('MYSQL_CONNECTION_POOL_ENABLED', 'true').casecmp?('true')
-    end
-
-    def start_pool_size
-      @start_pool_size ||= Integer(ENV.fetch('MYSQL_START_POOL_SIZE', 1))
-    end
+      return @connection_pool_enabled unless @connection_pool_enabled.nil?
 
-    def max_pool_size
-      @max_pool_size ||= Integer(ENV.fetch('MYSQL_MAX_POOL_SIZE', 5))
+      @connection_pool_enabled = ENV.fetch('MYSQL_CONNECTION_POOL_ENABLED', 'true').casecmp?('true')
     end
   end
 end

data/lib/mysql_framework/mysql_connection_pool.rb

@@ -0,0 +1,176 @@
+# frozen_string_literal: true
+
+require 'connection_pool'
+
+module MysqlFramework
+  class MysqlConnectionPool
+    class ConnectionSanitizationError < StandardError; end
+
+    CLEAN_IDLE_CONNECTIONS_THREAD_NAME = 'clean-idle-connections'
+
+    attr_reader :connections
+
+    # Initializes a connection pool instance with MySQL client options.
+    #
+    # @param options [Hash] MySQL client options passed to each pooled connection
+    # @return [void]
+    def initialize(options)
+      @options = options
+      @setup_mutex = Mutex.new
+    end
+
+    # Sets up the MySQL connection pool. Idempotent — safe to call more than once.
+    #
+    # @return [ConnectionPool] configured pool
+    def setup
+      @setup_mutex.synchronize do
+        return if connections
+
+        @connections = ConnectionPool.new(size: max_pool_size, timeout: pool_timeout) do
+          Mysql2::Client.new(@options)
+        end
+
+        start_clean_idle_connections_thread
+      end
+    end
+
+    # Disposes of the connection pool and closes pooled connections.
+    #
+    # @return [void]
+    def dispose
+      @setup_mutex.synchronize do
+        dispose_clean_idle_connections_thread
+        connections&.shutdown(&:close)
+        @connections = nil
+      end
+    end
+
+    # Returns key connection-pool metrics for monitoring.
+    #
+    # @return [Hash{Symbol => Integer}] pool size and availability metrics
+    def pool_stats
+      return { size: 0, available: 0, idle: 0 } if connections.nil?
+
+      {
+        size: connections.size,
+        available: connections.available,
+        idle: connections.idle
+      }
+    end
+
+    # Checks out a MySQL client, sanitizing it before use.
+    #
+    # @return [Mysql2::Client] checked-out client
+    # @raise [ConnectionSanitizationError] when sanitization repeatedly fails
+    # @raise [Mysql2::Error] when checkout or sanitization fails due to MySQL errors
+    def check_out
+      sanitization_retries = 0
+      begin
+        conn = connections.checkout
+        sanitize_connection!(conn)
+        conn
+      rescue ConnectionSanitizationError
+        discard_current_connection!
+        sanitization_retries += 1
+        retry if sanitization_retries <= 1
+        raise
+      rescue Mysql2::Error
+        discard_current_connection!
+        raise
+      end
+    end
+
+    # Returns a MySQL client back to the pool or closes it when pooling is disabled.
+    #
+    # @param client [Mysql2::Client, nil] client to return or close
+    # @return [void]
+    def check_in(client)
+      return if client.nil?
+
+      discard_current_connection! if client.closed?
+      connections.checkin
+    end
+
+    # Yields a MySQL client from the pool, or yields the provided client directly.
+    #
+    # @param provided_client [Mysql2::Client, nil] existing client to yield without pool checkout
+    # @param discard_current_pool_connection [Boolean] whether to discard the pooled connection after use
+    # @yield [client] block that performs work with a MySQL client
+    # @yieldparam client [Mysql2::Client]
+    # @return [Object] block result
+    # @raise [Mysql2::Error] re-raises MySQL errors from the block
+    def with_client(discard_current_pool_connection: false)
+      sanitization_retries = 0
+
+      begin
+        connections.with do |conn|
+          sanitize_connection!(conn)
+          yield conn
+        rescue ConnectionSanitizationError, Mysql2::Error
+          discard_current_connection!
+          raise
+        ensure
+          discard_current_connection! if discard_current_pool_connection
+        end
+      rescue ConnectionSanitizationError
+        sanitization_retries += 1
+        retry if sanitization_retries <= 1
+        raise
+      end
+    end
+
+    private
+
+    def start_clean_idle_connections_thread
+      thread_name = "#{CLEAN_IDLE_CONNECTIONS_THREAD_NAME}-#{object_id}"
+      @idle_connections_thread = Thread.new do
+        Thread.current.name = thread_name
+        loop do
+          sleep idle_reap_loop_time
+          break unless Thread.current == @idle_connections_thread
+
+          connections&.reap(idle_seconds: idle_timeout, &:close)
+        end
+      end
+
+      @idle_connections_thread.abort_on_exception # = false
+      @idle_connections_thread
+    end
+
+    def dispose_clean_idle_connections_thread
+      @idle_connections_thread&.join(5)
+      @idle_connections_thread&.kill
+      @idle_connections_thread = nil
+    end
+
+    def sanitize_connection!(conn)
+      conn.ping
+      conn.abandon_results!
+      conn.query('ROLLBACK')
+    rescue Mysql2::Error => e
+      raise ConnectionSanitizationError, "Connection sanitization failed: #{e.message}"
+    end
+
+    def discard_current_connection!
+      connections&.discard_current_connection(&:close)
+    rescue StandardError
+      nil
+    end
+
+    def max_pool_size
+      @max_pool_size ||= Integer(ENV.fetch('MYSQL_MAX_POOL_SIZE', 5))
+    end
+
+    def pool_timeout
+      @pool_timeout ||= Integer(ENV.fetch('MYSQL_POOL_TIMEOUT', 5))
+    end
+
+    def idle_timeout
+      @idle_timeout ||= Integer(ENV.fetch('MYSQL_POOL_IDLE_TIMEOUT', 300))
+    end
+
+    def idle_reap_loop_time
+      @idle_reap_loop_time ||= Integer(ENV.fetch('MYSQL_POOL_IDLE_REAP_TIME', 60))
+    end
+  end
+end

data/lib/mysql_framework/scripts/manager.rb

@@ -27,7 +27,7 @@ module MysqlFramework
       end
 
       def apply_by_tag(tags)
-        lock_manager.with_lock(key: self.class) do
+        lock_manager.with_lock(key: self.class.name) do
           initialize_script_history
 
           mysql_connector.transaction do |client|

data/lib/mysql_framework/stats/aws_metric_publisher.rb

@@ -0,0 +1,124 @@
+# frozen_string_literal: true
+
+require 'aws-sdk-cloudwatch'
+require_relative 'dimension_map'
+
+module MysqlFramework
+  module Stats
+    class AwsMetricPublisher
+      THREAD_NAME = 'mysql-connector-pool-stats'
+      JOIN_TIMEOUT = 5 # seconds to wait for clean thread exit before force-killing
+      METRIC_UNIT = 'Count'
+      METRIC_NAME_MAP = {
+        size: 'MysqlConnectionPoolSize',
+        available: 'MysqlConnectionPoolAvailable',
+        idle: 'MysqlConnectionPoolIdle'
+      }.freeze
+
+      # Initializes AWS metric publishing dependencies.
+      #
+      # @param connector [MysqlFramework::Connector, nil] connector used to read connection-pool stats
+      # @param dimension_map [MysqlFramework::Stats::DimensionMap, nil] CloudWatch namespace and dimensions
+      # @param cloudwatch_client [Aws::CloudWatch::Client, nil] CloudWatch client instance
+      # @param publish_interval [Integer] metric publish interval in seconds
+      # @return [void]
+      def initialize(
+        connector: nil,
+        dimension_map: nil,
+        cloudwatch_client: nil,
+        publish_interval: 300
+      )
+        @thread = nil
+        @connector = connector
+        @cloudwatch_client = cloudwatch_client
+        @dimension_map = dimension_map || MysqlFramework::Stats::DimensionMap.new
+        @publish_interval = publish_interval
+      end
+
+      # Spawns the background sampling thread. Safe to call more than once –
+      # subsequent calls are no-ops while the thread is already running.
+      #
+      # @return [Thread, nil] reporter thread when started, or nil when already running
+      def start
+        return if running?
+
+        thread_name = "#{THREAD_NAME}-#{object_id}"
+        thread = Thread.new do
+          Thread.current.name = thread_name
+          loop do
+            sleep @publish_interval
+            break unless Thread.current == @thread
+
+            sample
+          end
+        end
+
+        thread.abort_on_exception = false
+        @thread = thread
+      end
+
+      # Cooperatively stops the background thread and waits up to JOIN_TIMEOUT
+      # seconds for it to exit before force-killing it.
+      #
+      # @return [void]
+      def stop
+        thread = @thread
+        @thread = nil # cooperative stop signal: loop checks this after each sleep
+        thread&.join(JOIN_TIMEOUT)
+        thread&.kill # force-kill only if still alive after timeout
+      end
+
+      # Returns true when the reporter thread is alive.
+      #
+      # @return [Boolean]
+      def running?
+        @thread&.alive? || false
+      end
+
+      private
+
+      # Reads pool stats and publishes them to CloudWatch using a low-cardinality
+      # dimension set so all ECS tasks for the same service aggregate together.
+      # Errors are swallowed and logged so that a reporting failure never
+      # propagates to the caller.
+      def sample
+        connection_pool = @connector&.connection_pool
+        return if connection_pool.nil?
+
+        stats = connection_pool.pool_stats
+        metric_data = build_metric_data(stats)
+        return if metric_data.empty?
+
+        MysqlFramework.logger.debug { "[#{self.class}] - CloudWatch/#{@dimension_map.namespace} - #{stats.inspect}" }
+
+        cloudwatch_client.put_metric_data(
+          namespace: @dimension_map.namespace,
+          metric_data: metric_data
+        )
+      rescue StandardError => e
+        MysqlFramework.logger.error { "[#{self.class}] - Failed to record pool stats: #{e.message}" }
+      end
+
+      def build_metric_data(stats)
+        timestamp = Time.now.utc
+
+        METRIC_NAME_MAP.filter_map do |key, metric_name|
+          value = stats[key]
+          next if value.nil?
+
+          {
+            metric_name: metric_name,
+            dimensions: @dimension_map.to_cloudwatch_dimensions,
+            timestamp: timestamp,
+            unit: METRIC_UNIT,
+            value: value.to_f
+          }
+        end
+      end
+
+      def cloudwatch_client
+        @cloudwatch_client ||= Aws::CloudWatch::Client.new
+      end
+    end
+  end
+end

data/lib/mysql_framework/stats/dimension_map.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module MysqlFramework
+  module Stats
+    # Class to handle dimensions for AWS reporting
+    class DimensionMap
+      attr_accessor :service_name, :application, :environment, :landscape, :namespace
+
+      # Initializes dimension values used for CloudWatch metrics.
+      #
+      # @param service_name [String, nil] service dimension
+      # @param application [String, nil] application dimension
+      # @param environment [String, nil] environment dimension
+      # @param landscape [String, nil] landscape dimension
+      # @param namespace [String, nil] CloudWatch namespace override
+      # @return [void]
+      def initialize(
+        service_name: nil,
+        application: nil,
+        environment: nil,
+        landscape: nil,
+        namespace: nil
+      )
+        @service_name = service_name
+        @application = application
+        @environment = environment
+        @landscape = landscape
+        @namespace = namespace
+      end
+
+      # Builds CloudWatch dimensions from configured values or environment variables.
+      #
+      # @return [Array<Hash{Symbol => String}>] dimensions with non-nil values only
+      def to_cloudwatch_dimensions
+        [
+          { name: 'ServiceName', value: service_name || ENV.fetch('SERVICE_NAME', nil) },
+          { name: 'Application', value: application || ENV.fetch('APPLICATION', nil) },
+          { name: 'Environment', value: environment || ENV.fetch('ENVIRONMENT', nil) },
+          { name: 'Landscape', value: landscape || ENV.fetch('LANDSCAPE', nil) }
+        ].reject { |dimension| dimension[:value].nil? }
+      end
+
+      # Returns the CloudWatch namespace.
+      #
+      # @return [String] configured namespace or default namespace value
+      def namespace
+        @namespace || ENV.fetch('AWS_METRICS_NAMESPACE', 'MysqlFramework')
+      end
+    end
+  end
+end

data/lib/mysql_framework/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module MysqlFramework
-  VERSION = '2.1.9'
+  VERSION = '2.3.0'
 end