sequel 4.36.0 → 5.61.0

data/lib/sequel/connection_pool/threaded.rb

@@ -3,34 +3,26 @@
 # A connection pool allowing multi-threaded access to a pool of connections.
 # This is the default connection pool used by Sequel.
 class Sequel::ThreadedConnectionPool < Sequel::ConnectionPool
-  # Whether or not a ConditionVariable should be used to wait for connections.
-  # True except on ruby 1.8, where ConditionVariable#wait does not support a
-  # timeout.
-  unless defined?(USE_WAITER)
-    USE_WAITER = RUBY_VERSION >= '1.9'
-  end
+  USE_WAITER = true
+  Sequel::Deprecation.deprecate_constant(self, :USE_WAITER)
 
   # The maximum number of connections this pool will create (per shard/server
   # if sharding).
   attr_reader :max_size
   
   # An array of connections that are available for use by the pool.
+  # The calling code should already have the mutex before calling this.
   attr_reader :available_connections
   
-  # A hash with thread keys and connection values for currently allocated
-  # connections.
+  # A hash with thread keys and connection values for currently allocated connections.
+  # The calling code should already have the mutex before calling this.
   attr_reader :allocated
 
   # The following additional options are respected:
-  # * :connection_handling - Set how to handle available connections.  By default,
-  #   uses a a queue for fairness.  Can be set to :stack to use a stack, which may
-  #   offer better performance.
-  # * :max_connections - The maximum number of connections the connection pool
-  #   will open (default 4)
-  # * :pool_sleep_time - The amount of time to sleep before attempting to acquire
-  #   a connection again, only used on ruby 1.8. (default 0.001)
-  # * :pool_timeout - The amount of seconds to wait to acquire a connection
-  #   before raising a PoolTimeoutError (default 5)
+  # :max_connections :: The maximum number of connections the connection pool
+  #                     will open (default 4)
+  # :pool_timeout :: The amount of seconds to wait to acquire a connection
+  #                  before raising a PoolTimeoutError (default 5)
   def initialize(db, opts = OPTS)
     super
     @max_size = Integer(opts[:max_connections] || 4)
@@ -40,14 +32,7 @@ class Sequel::ThreadedConnectionPool < Sequel::ConnectionPool
     @available_connections = []
     @allocated = {}
     @timeout = Float(opts[:pool_timeout] || 5)
-
-    if USE_WAITER
-      @waiter = ConditionVariable.new
-    else
-      # :nocov:
-      @sleep_time = Float(opts[:pool_sleep_time] || 0.001)
-      # :nocov:
-    end
+    @waiter = ConditionVariable.new
   end
   
   # Yield all of the available connections, and the one currently allocated to
@@ -70,7 +55,7 @@ class Sequel::ThreadedConnectionPool < Sequel::ConnectionPool
   # being used.  If you want to be able to disconnect connections that are
   # currently in use, use the ShardedThreadedConnectionPool, which can do that.
   # This connection pool does not, for performance reasons. To use the sharded pool,
-  # pass the <tt>:servers=>{}</tt> option when connecting to the database.
+  # pass the <tt>servers: {}</tt> option when connecting to the database.
   # 
   # Once a connection is requested using #hold, the connection pool
   # creates new connections to the database.
@@ -79,6 +64,7 @@ class Sequel::ThreadedConnectionPool < Sequel::ConnectionPool
     sync do
       conns = @available_connections.dup
       @available_connections.clear
+      @waiter.signal
     end
     conns.each{|conn| disconnect_connection(conn)}
   end
@@ -95,24 +81,33 @@ class Sequel::ThreadedConnectionPool < Sequel::ConnectionPool
   # If no connection is immediately available and the pool is already using the maximum
   # number of connections, Pool#hold will block until a connection
   # is available or the timeout expires.  If the timeout expires before a
-  # connection can be acquired, a Sequel::PoolTimeout is 
-  # raised.
+  # connection can be acquired, a Sequel::PoolTimeout is raised.
   def hold(server=nil)
-    t = Thread.current
+    t = Sequel.current
     if conn = owned_connection(t)
       return yield(conn)
     end
     begin
       conn = acquire(t)
       yield conn
-    rescue Sequel::DatabaseDisconnectError
-      oconn = conn
-      conn = nil
-      disconnect_connection(oconn) if oconn
-      @allocated.delete(t)
+    rescue Sequel::DatabaseDisconnectError, *@error_classes => e
+      if disconnect_error?(e)
+        oconn = conn
+        conn = nil
+        disconnect_connection(oconn) if oconn
+        sync do 
+          @allocated.delete(t)
+          @waiter.signal
+        end
+      end
       raise
     ensure
-      sync{release(t)} if conn
+      if conn
+        sync{release(t)}
+        if @connection_handling == :disconnect
+          disconnect_connection(conn)
+        end
+      end
     end
   end
 
@@ -121,111 +116,115 @@ class Sequel::ThreadedConnectionPool < Sequel::ConnectionPool
   end
   
   # The total number of connections opened, either available or allocated.
-  # This may not be completely accurate as it isn't protected by the mutex.
+  # The calling code should not have the mutex before calling this.
   def size
-    @allocated.length + @available_connections.length
+    @mutex.synchronize{_size}
   end
   
   private
 
+  # The total number of connections opened, either available or allocated.
+  # The calling code should already have the mutex before calling this.
+  def _size
+    @allocated.length + @available_connections.length
+  end
+
   # Assigns a connection to the supplied thread, if one
-  # is available. The calling code should already have the mutex when
+  # is available. The calling code should NOT already have the mutex when
   # calling this.
-  def _acquire(thread)
-    if conn = available
-      @allocated[thread] = conn
+  #
+  # This should return a connection is one is available within the timeout,
+  # or nil if a connection could not be acquired within the timeout.
+  def acquire(thread)
+    if conn = assign_connection(thread)
+      return conn
     end
-  end
-  
-  if USE_WAITER
-    # Assigns a connection to the supplied thread, if one
-    # is available. The calling code should NOT already have the mutex when
-    # calling this.
-    #
-    # This should return a connection is one is available within the timeout,
-    # or nil if a connection could not be acquired within the timeout.
-    def acquire(thread)
+
+    timeout = @timeout
+    timer = Sequel.start_timer
+
+    sync do
+      @waiter.wait(@mutex, timeout)
+      if conn = next_available
+        return(@allocated[thread] = conn)
+      end
+    end
+
+    until conn = assign_connection(thread)
+      elapsed = Sequel.elapsed_seconds_since(timer)
+      # :nocov:
+      raise_pool_timeout(elapsed) if elapsed > timeout
+
+      # It's difficult to get to this point, it can only happen if there is a race condition
+      # where a connection cannot be acquired even after the thread is signalled by the condition variable
       sync do
-        if conn = _acquire(thread)
-          return conn
+        @waiter.wait(@mutex, timeout - elapsed)
+        if conn = next_available
+          return(@allocated[thread] = conn)
         end
+      end
+      # :nocov:
+    end
+
+    conn
+  end
 
-        time = Time.now
-        @waiter.wait(@mutex, @timeout)
+  # Assign a connection to the thread, or return nil if one cannot be assigned.
+  # The caller should NOT have the mutex before calling this.
+  def assign_connection(thread)
+    # Thread safe as instance variable is only assigned to local variable
+    # and not operated on outside mutex.
+    allocated = @allocated
+    do_make_new = false
+    to_disconnect = nil
 
-        # Not sure why this is helpful, but calling Thread.pass after conditional
-        # variable access dramatically increases reliability when under heavy
-        # resource contention (almost eliminating timeouts), at a small cost to
-        # runtime performance.
-        Thread.pass
+    sync do
+      if conn = next_available
+        return(allocated[thread] = conn)
+      end
 
-        until conn = _acquire(thread)
-          deadline ||= time + @timeout
-          current_time = Time.now
-          raise_pool_timeout(current_time - time) if current_time > deadline
-          # :nocov:
-          # It's difficult to get to this point, it can only happen if there is a race condition
-          # where a connection cannot be acquired even after the thread is signalled by the condition
-          @waiter.wait(@mutex, deadline - current_time)
-          Thread.pass
-          # :nocov:
+      if (n = _size) >= (max = @max_size)
+        allocated.keys.each do |t|
+          unless t.alive?
+            (to_disconnect ||= []) << allocated.delete(t)
+          end
         end
+        n = nil
+      end
 
-        conn
+      if (n || _size) < max
+        do_make_new = allocated[thread] = true
       end
     end
-  else
-    # :nocov:
-    def acquire(thread)
-      unless conn = sync{_acquire(thread)}
-        time = Time.now
-        timeout = time + @timeout
-        sleep_time = @sleep_time
-        sleep sleep_time
-        until conn = sync{_acquire(thread)}
-          raise_pool_timeout(Time.now - time) if Time.now > timeout
-          sleep sleep_time
+
+    if to_disconnect
+      to_disconnect.each{|dconn| disconnect_connection(dconn)}
+    end
+
+    # Connect to the database outside of the connection pool mutex,
+    # as that can take a long time and the connection pool mutex
+    # shouldn't be locked while the connection takes place.
+    if do_make_new
+      begin
+        conn = make_new(:default)
+        sync{allocated[thread] = conn}
+      ensure
+        unless conn
+          sync{allocated.delete(thread)}
         end
       end
-      conn
     end
-    # :nocov:
-  end
 
-  # Returns an available connection. If no connection is
-  # available, tries to create a new connection. The calling code should already
-  # have the mutex before calling this.
-  def available
-    next_available || make_new(DEFAULT_SERVER)
+    conn
   end
 
   # Return a connection to the pool of available connections, returns the connection.
   # The calling code should already have the mutex before calling this.
   def checkin_connection(conn)
     @available_connections << conn
-    if USE_WAITER
-      @waiter.signal
-      Thread.pass
-    end
     conn
   end
 
-  unless method_defined?(:default_make_new)
-    # Alias the default make_new method, so subclasses can call it directly.
-    alias default_make_new make_new
-  end
-  
-  # Creates a new connection to the given server if the size of the pool for
-  # the server is less than the maximum size of the pool. The calling code
-  # should already have the mutex before calling this.
-  def make_new(server)
-    if (n = size) >= @max_size
-      @allocated.keys.each{|t| release(t) unless t.alive?}
-      n = nil
-    end
-    super if (n || size) < @max_size
-  end
-
   # Return the next available connection in the pool, or nil if there
   # is not currently an available connection.  The calling code should already
   # have the mutex before calling this.
@@ -244,15 +243,18 @@ class Sequel::ThreadedConnectionPool < Sequel::ConnectionPool
     sync{@allocated[thread]}
   end
   
-  # Create the maximum number of connections immediately.
+  # Create the maximum number of connections immediately.  The calling code should
+  # NOT have the mutex before calling this.
   def preconnect(concurrent = false)
-    enum = (max_size - size).times
+    enum = (max_size - _size).times
 
-    if concurrent
-      enum.map{Thread.new{make_new(nil)}}.map(&:join).each{|t| checkin_connection(t.value)}
+    conns = if concurrent
+      enum.map{Thread.new{make_new(:default)}}.map(&:value)
     else
-      enum.each{checkin_connection(make_new(nil))}
+      enum.map{make_new(:default)}
     end
+
+    sync{conns.each{|conn| checkin_connection(conn)}}
   end
 
   # Raise a PoolTimeout error showing the current timeout, the elapsed time, and the
@@ -267,11 +269,12 @@ class Sequel::ThreadedConnectionPool < Sequel::ConnectionPool
   def release(thread)
     conn = @allocated.delete(thread)
 
-    if @connection_handling == :disconnect
-      disconnect_connection(conn)
-    else
+    unless @connection_handling == :disconnect
       checkin_connection(conn)
     end
+
+    @waiter.signal
+    nil
   end
 
   # Yield to the block while inside the mutex. The calling code should NOT
@@ -279,6 +282,4 @@ class Sequel::ThreadedConnectionPool < Sequel::ConnectionPool
   def sync
     @mutex.synchronize{yield}
   end
-  
-  CONNECTION_POOL_MAP[[false, false]] = self
 end

data/lib/sequel/connection_pool.rb

@@ -26,16 +26,13 @@
 #                                     specified by the array of symbols.
 class Sequel::ConnectionPool
   OPTS = Sequel::OPTS
+  POOL_CLASS_MAP = {
+    :threaded => :ThreadedConnectionPool,
+    :single => :SingleConnectionPool,
+    :sharded_threaded => :ShardedThreadedConnectionPool,
+    :sharded_single => :ShardedSingleConnectionPool
+  }.freeze
 
-  # The default server to use
-  DEFAULT_SERVER = :default
-  
-  # A map of [single threaded, sharded] values to symbols or ConnectionPool subclasses.
-  CONNECTION_POOL_MAP = {[true, false] => :single, 
-    [true, true] => :sharded_single,
-    [false, false] => :threaded,
-    [false, true] => :sharded_threaded}
-  
   # Class methods used to return an appropriate pool subclass, separated
   # into a module for easier overridding by extensions.
   module ClassMethods
@@ -44,20 +41,33 @@ class Sequel::ConnectionPool
     # use a new instance of an appropriate pool subclass based on the
     # <tt>:single_threaded</tt> and <tt>:servers</tt> options.
     def get_pool(db, opts = OPTS)
-      case v = connection_pool_class(opts)
-      when Class
-        v.new(db, opts)
-      when Symbol
-        require("sequel/connection_pool/#{v}")
-        connection_pool_class(opts).new(db, opts) || raise(Sequel::Error, "No connection pool class found")
-      end
+      connection_pool_class(opts).new(db, opts)
     end
     
     private
     
     # Return a connection pool class based on the given options.
     def connection_pool_class(opts)
-      CONNECTION_POOL_MAP[opts[:pool_class]] || opts[:pool_class] || CONNECTION_POOL_MAP[[!!opts[:single_threaded], !!opts[:servers]]]
+      if pc = opts[:pool_class]
+        unless pc.is_a?(Class)
+          unless name = POOL_CLASS_MAP[pc]
+            raise Sequel::Error, "unsupported connection pool type, please pass appropriate class as the :pool_class option"
+          end
+
+          require_relative "connection_pool/#{pc}"
+          pc = Sequel.const_get(name)
+        end
+
+        pc
+      else
+        pc = if opts[:single_threaded]
+          opts[:servers] ? :sharded_single : :single
+        else
+          opts[:servers] ? :sharded_threaded : :threaded
+        end
+
+        connection_pool_class(:pool_class=>pc)
+      end
     end
   end
   extend ClassMethods
@@ -66,6 +76,9 @@ class Sequel::ConnectionPool
   # connection made, and is usually used to set custom per-connection settings.
   attr_accessor :after_connect
 
+  # An array of sql strings to execute on each new connection.
+  attr_accessor :connect_sqls
+
   # The Sequel::Database object tied to this connection pool.
   attr_accessor :db
 
@@ -76,38 +89,38 @@ class Sequel::ConnectionPool
   # :after_connect :: A callable object called after each new connection is made, with the
   #                   connection object (and server argument if the callable accepts 2 arguments),
   #                   useful for customizations that you want to apply to all connections.
-  # :preconnect :: Automatically create the maximum number of connections, so that they don't
-  #                need to be created as needed.  This is useful when connecting takes a long time
-  #                and you want to avoid possible latency during runtime.
-  #                Set to :concurrently to create the connections in separate threads. Otherwise
-  #                they'll be created sequentially.
+  # :connect_sqls :: An array of sql strings to execute on each new connection, after :after_connect runs.
   def initialize(db, opts=OPTS)
     @db = db
     @after_connect = opts[:after_connect]
-  end
-  
-  # Alias for +size+, not aliased directly for ease of subclass implementation
-  def created_count(*args)
-    size(*args)
+    @connect_sqls = opts[:connect_sqls]
+    @error_classes = db.send(:database_error_classes).dup.freeze
   end
   
   # An array of symbols for all shards/servers, which is a single <tt>:default</tt> by default.
   def servers
-    [DEFAULT_SERVER]
+    [:default]
   end
   
   private
 
-  # Remove the connection from the pool.
+  # Remove the connection from the pool.  For threaded connections, this should be
+  # called without the mutex, because the disconnection may block.
   def disconnect_connection(conn)
     db.disconnect_connection(conn)
   end
+
+  # Whether the given exception is a disconnect exception.
+  def disconnect_error?(exception)
+    exception.is_a?(Sequel::DatabaseDisconnectError) || db.send(:disconnect_error?, exception, OPTS)
+  end
   
   # Return a new connection by calling the connection proc with the given server name,
   # and checking for connection errors.
   def make_new(server)
     begin
       conn = @db.connect(server)
+
       if ac = @after_connect
         if ac.arity == 2
           ac.call(conn, server)
@@ -115,6 +128,12 @@ class Sequel::ConnectionPool
           ac.call(conn)
         end
       end
+
+      if cs = @connect_sqls
+        cs.each do |sql|
+          db.send(:log_connection_execute, conn, sql)
+        end
+      end
     rescue Exception=>exception
       raise Sequel.convert_exception_class(exception, Sequel::DatabaseConnectionError)
     end