sequel 4.26.0 → 5.37.0

data/lib/sequel/connection_pool/sharded_threaded.rb

@@ -1,4 +1,6 @@
-require 'sequel/connection_pool/threaded'
+# frozen-string-literal: true
+
+require_relative 'threaded'
 
 # The slowest and most advanced connection, dealing with both multi-threaded
 # access and configurations with multiple shards/servers.
@@ -17,20 +19,17 @@ class Sequel::ShardedThreadedConnectionPool < Sequel::ThreadedConnectionPool
     super
     @available_connections = {}
     @connections_to_remove = []
+    @connections_to_disconnect = []
     @servers = opts.fetch(:servers_hash, Hash.new(:default))
-
-    if USE_WAITER
-      @waiter = nil
-      @waiters = {}
-    end
+    remove_instance_variable(:@waiter)
+    @waiters = {}
 
     add_servers([:default])
     add_servers(opts[:servers].keys) if opts[:servers]
   end
   
-  # Adds new servers to the connection pool. Primarily used in conjunction with master/slave
-  # or shard configurations. Allows for dynamic expansion of the potential slaves/shards
-  # at runtime. servers argument should be an array of symbols. 
+  # Adds new servers to the connection pool.  Allows for dynamic expansion of the potential replicas/shards
+  # at runtime. +servers+ argument should be an array of symbols. 
   def add_servers(servers)
     sync do
       servers.each do |server|
@@ -38,7 +37,7 @@ class Sequel::ShardedThreadedConnectionPool < Sequel::ThreadedConnectionPool
           @servers[server] = server
           @available_connections[server] = []
           @allocated[server] = {}
-          @waiters[server] = ConditionVariable.new if USE_WAITER
+          @waiters[server] = ConditionVariable.new
         end
       end
     end
@@ -47,6 +46,7 @@ class Sequel::ShardedThreadedConnectionPool < Sequel::ThreadedConnectionPool
   # A hash of connections currently being used for the given server, key is the
   # Thread, value is the connection.  Nonexistent servers will return nil.  Treat
   # this as read only, do not modify the resulting object.
+  # The calling code should already have the mutex before calling this.
   def allocated(server=:default)
     @allocated[server]
   end
@@ -57,7 +57,7 @@ class Sequel::ShardedThreadedConnectionPool < Sequel::ThreadedConnectionPool
   # it is yielding all of the connections, which means that until
   # the method's block returns, the pool is locked.
   def all_connections
-    t = Thread.current
+    t = Sequel.current
     sync do
       @allocated.values.each do |threads|
         threads.each do |thread, conn|
@@ -71,16 +71,16 @@ class Sequel::ShardedThreadedConnectionPool < Sequel::ThreadedConnectionPool
   # An array of connections opened but not currently used, for the given
   # server. Nonexistent servers will return nil. Treat this as read only, do
   # not modify the resulting object.
+  # The calling code should already have the mutex before calling this.
   def available_connections(server=:default)
     @available_connections[server]
   end
   
-  # The total number of connections opened for the given server, should
-  # be equal to available_connections.length + allocated.length.  Nonexistent
-  # servers will return the created count of the default server.
+  # The total number of connections opened for the given server.
+  # Nonexistent servers will return the created count of the default server.
+  # The calling code should NOT have the mutex before calling this.
   def size(server=:default)
-    server = @servers[server]
-    @allocated[server].length + @available_connections[server].length
+    @mutex.synchronize{_size(server)}
   end
   
   # Removes all connections currently available on all servers, optionally
@@ -94,13 +94,16 @@ class Sequel::ShardedThreadedConnectionPool < Sequel::ThreadedConnectionPool
   # :server :: Should be a symbol specifing the server to disconnect from,
   #            or an array of symbols to specify multiple servers.
   def disconnect(opts=OPTS)
-    sync do
-      (opts[:server] ? Array(opts[:server]) : @servers.keys).each do |s|
-        disconnect_server(s)
-      end
+    (opts[:server] ? Array(opts[:server]) : sync{@servers.keys}).each do |s|
+      disconnect_connections(sync{disconnect_server_connections(s)})
     end
   end
   
+  def freeze
+    @servers.freeze
+    super
+  end
+  
   # Chooses the first available connection to the given server, or if none are
   # available, creates a new connection.  Passes the connection to the supplied
   # block:
@@ -113,42 +116,48 @@ class Sequel::ShardedThreadedConnectionPool < Sequel::ThreadedConnectionPool
   # If no connection is immediately available and the pool is already using the maximum
   # number of connections, Pool#hold will block until a connection
   # is available or the timeout expires.  If the timeout expires before a
-  # connection can be acquired, a Sequel::PoolTimeout is 
-  # raised.
+  # connection can be acquired, a Sequel::PoolTimeout is raised.
   def hold(server=:default)
     server = pick_server(server)
-    t = Thread.current
+    t = Sequel.current
     if conn = owned_connection(t, server)
       return yield(conn)
     end
     begin
       conn = acquire(t, server)
       yield conn
-    rescue Sequel::DatabaseDisconnectError
-      sync{@connections_to_remove << conn} if conn
+    rescue Sequel::DatabaseDisconnectError, *@error_classes => e
+      sync{@connections_to_remove << conn} if conn && disconnect_error?(e)
       raise
     ensure
       sync{release(t, conn, server)} if conn
+      while dconn = sync{@connections_to_disconnect.shift}
+        disconnect_connection(dconn)
+      end
     end
   end
 
-  # Remove servers from the connection pool. Primarily used in conjunction with master/slave
-  # or shard configurations.  Similar to disconnecting from all given servers,
+  # Remove servers from the connection pool. Similar to disconnecting from all given servers,
   # except that after it is used, future requests for the server will use the
   # :default server instead.
   def remove_servers(servers)
+    conns = nil
     sync do
       raise(Sequel::Error, "cannot remove default server") if servers.include?(:default)
       servers.each do |server|
         if @servers.include?(server)
-          disconnect_server(server)
-          @waiters.delete(server) if USE_WAITER
+          conns = disconnect_server_connections(server)
+          @waiters.delete(server)
           @available_connections.delete(server)
           @allocated.delete(server)
           @servers.delete(server)
         end
       end
     end
+
+    if conns
+      disconnect_connections(conns)
+    end
   end
 
   # Return an array of symbols for servers in the connection pool.
@@ -162,71 +171,95 @@ class Sequel::ShardedThreadedConnectionPool < Sequel::ThreadedConnectionPool
   
   private
 
-  # Assigns a connection to the supplied thread for the given server, if one
-  # is available. The calling code should already have the mutex when
-  # calling this.
-  def _acquire(thread, server)
-    if conn = available(server)
-      allocated(server)[thread] = conn
-    end
+  # The total number of connections opened for the given server.
+  # The calling code should already have the mutex before calling this.
+  def _size(server)
+    server = @servers[server]
+    @allocated[server].length + @available_connections[server].length
   end
   
-  if USE_WAITER
-    # Assigns a connection to the supplied thread, if one
-    # is available. The calling code should NOT already have the mutex when
-    # calling this.
-    #
-    # This should return a connection is one is available within the timeout,
-    # or nil if a connection could not be acquired within the timeout.
-    def acquire(thread, server)
+  # Assigns a connection to the supplied thread, if one
+  # is available. The calling code should NOT already have the mutex when
+  # calling this.
+  #
+  # This should return a connection is one is available within the timeout,
+  # or nil if a connection could not be acquired within the timeout.
+  def acquire(thread, server)
+    if conn = assign_connection(thread, server)
+      return conn
+    end
+
+    timeout = @timeout
+    timer = Sequel.start_timer
+
+    sync do
+      @waiters[server].wait(@mutex, timeout)
+      if conn = next_available(server)
+        return(allocated(server)[thread] = conn)
+      end
+    end
+
+    until conn = assign_connection(thread, server)
+      elapsed = Sequel.elapsed_seconds_since(timer)
+      # :nocov:
+      raise_pool_timeout(elapsed, server) if elapsed > timeout
+
+      # It's difficult to get to this point, it can only happen if there is a race condition
+      # where a connection cannot be acquired even after the thread is signalled by the condition variable
       sync do
-        if conn = _acquire(thread, server)
-          return conn
+        @waiters[server].wait(@mutex, timeout - elapsed)
+        if conn = next_available(server)
+          return(allocated(server)[thread] = conn)
         end
+      end
+      # :nocov:
+    end
+
+    conn
+  end
+
+  # Assign a connection to the thread, or return nil if one cannot be assigned.
+  # The caller should NOT have the mutex before calling this.
+  def assign_connection(thread, server)
+    alloc = nil
 
-        time = Time.now
-        @waiters[server].wait(@mutex, @timeout)
-        Thread.pass
-
-        until conn = _acquire(thread, server)
-          deadline ||= time + @timeout
-          current_time = Time.now
-          raise(::Sequel::PoolTimeout, "timeout: #{@timeout}, elapsed: #{current_time - time}") if current_time > deadline
-          # :nocov:
-          # It's difficult to get to this point, it can only happen if there is a race condition
-          # where a connection cannot be acquired even after the thread is signalled by the condition
-          # variable that a connection is ready.
-          @waiters[server].wait(@mutex, deadline - current_time)
-          Thread.pass
-          # :nocov:
+    do_make_new = false
+    sync do
+      alloc = allocated(server)
+      if conn = next_available(server)
+        alloc[thread] = conn
+        return conn
+      end
+
+      if (n = _size(server)) >= (max = @max_size)
+        alloc.to_a.each do |t,c|
+          unless t.alive?
+            remove(t, c, server)
+          end
         end
+        n = nil
+      end
 
-        conn
+      if (n || _size(server)) < max
+        do_make_new = alloc[thread] = true
       end
     end
-  else
-    # :nocov:
-    def acquire(thread, server)
-      unless conn = sync{_acquire(thread, server)}
-        time = Time.now
-        timeout = time + @timeout
-        sleep_time = @sleep_time
-        sleep sleep_time
-        until conn = sync{_acquire(thread, server)}
-          raise(::Sequel::PoolTimeout, "timeout: #{@timeout}, elapsed: #{Time.now - time}") if Time.now > timeout
-          sleep sleep_time
+
+    # Connect to the database outside of the connection pool mutex,
+    # as that can take a long time and the connection pool mutex
+    # shouldn't be locked while the connection takes place.
+    if do_make_new
+      begin
+        conn = make_new(server)
+        sync{alloc[thread] = conn}
+      ensure
+        unless conn
+          sync{alloc.delete(thread)}
         end
       end
-      conn
     end
-    # :nocov:
-  end
 
-  # Returns an available connection to the given server. If no connection is
-  # available, tries to create a new connection. The calling code should already
-  # have the mutex before calling this.
-  def available(server)
-    next_available(server) || make_new(server)
+    conn
   end
 
   # Return a connection to the pool of available connections for the server,
@@ -234,34 +267,32 @@ class Sequel::ShardedThreadedConnectionPool < Sequel::ThreadedConnectionPool
   # before calling this.
   def checkin_connection(server, conn)
     available_connections(server) << conn
-    if USE_WAITER
-      @waiters[server].signal
-      Thread.pass
-    end
+    @waiters[server].signal
     conn
   end
 
-  # Disconnect from the given server.  Disconnects available connections
-  # immediately, and schedules currently allocated connections for disconnection
-  # as soon as they are returned to the pool. The calling code should already
-  # have the mutex before calling this.
-  def disconnect_server(server)
-    if conns = available_connections(server)
-      conns.each{|conn| db.disconnect_connection(conn)}
-      conns.clear
-    end
-    @connections_to_remove.concat(allocated(server).values)
+  # Clear the array of available connections for the server, returning an array
+  # of previous available connections that should be disconnected (or nil if none should be).
+  # Mark any allocated connections to be removed when they are checked back in. The calling
+  # code should already have the mutex before calling this.
+  def disconnect_server_connections(server)
+    remove_conns = allocated(server)
+    dis_conns = available_connections(server)
+    raise Sequel::Error, "invalid server: #{server}" unless remove_conns && dis_conns
+
+    @connections_to_remove.concat(remove_conns.values)
+
+    conns = dis_conns.dup
+    dis_conns.clear
+    @waiters[server].signal
+    conns
   end
 
-  # Creates a new connection to the given server if the size of the pool for
-  # the server is less than the maximum size of the pool. The calling code
-  # should already have the mutex before calling this.
-  def make_new(server)
-    if (n = size(server)) >= @max_size
-      allocated(server).to_a.each{|t, c| release(t, c, server) unless t.alive?}
-      n = nil
-    end
-    default_make_new(server) if (n || size(server)) < @max_size
+  # Disconnect all available connections immediately, and schedule currently allocated connections for disconnection
+  # as soon as they are returned to the pool. The calling code should NOT
+  # have the mutex before calling this.
+  def disconnect_connections(conns)
+    conns.each{|conn| disconnect_connection(conn)}
   end
 
   # Return the next available connection in the pool for the given server, or nil
@@ -287,9 +318,25 @@ class Sequel::ShardedThreadedConnectionPool < Sequel::ThreadedConnectionPool
     sync{@servers[server]}
   end
   
-  # Create the maximum number of connections to each server immediately.
-  def preconnect
-    servers.each{|s| (max_size - size(s)).times{checkin_connection(s, make_new(s))}}
+  # Create the maximum number of connections immediately.  The calling code should
+  # NOT have the mutex before calling this.
+  def preconnect(concurrent = false)
+    conn_servers = @servers.keys.map!{|s| Array.new(max_size - _size(s), s)}.flatten!
+
+    if concurrent
+      conn_servers.map!{|s| Thread.new{[s, make_new(s)]}}.map!(&:value)
+    else
+      conn_servers.map!{|s| [s, make_new(s)]}
+    end
+
+    sync{conn_servers.each{|s, conn| checkin_connection(s, conn)}}
+  end
+
+  # Raise a PoolTimeout error showing the current timeout, the elapsed time, the server
+  # the connection attempt was made to, and the database's name (if any).
+  def raise_pool_timeout(elapsed, server)
+    name = db.opts[:name]
+    raise ::Sequel::PoolTimeout, "timeout: #{@timeout}, elapsed: #{elapsed}, server: #{server}#{", database name: #{name}" if name}"
   end
   
   # Releases the connection assigned to the supplied thread and server. If the
@@ -303,11 +350,15 @@ class Sequel::ShardedThreadedConnectionPool < Sequel::ThreadedConnectionPool
       conn = allocated(server).delete(thread)
 
       if @connection_handling == :disconnect
-        db.disconnect_connection(conn)
+        @connections_to_disconnect << conn
       else
         checkin_connection(server, conn)
       end
     end
+
+    if waiter = @waiters[server]
+      waiter.signal
+    end
   end
 
   # Removes the currently allocated connection from the connection pool. The
@@ -315,8 +366,6 @@ class Sequel::ShardedThreadedConnectionPool < Sequel::ThreadedConnectionPool
   def remove(thread, conn, server)
     @connections_to_remove.delete(conn)
     allocated(server).delete(thread) if @servers.include?(server)
-    db.disconnect_connection(conn)
+    @connections_to_disconnect << conn
   end
-  
-  CONNECTION_POOL_MAP[[false, true]] = self
 end

data/lib/sequel/connection_pool/single.rb

@@ -1,25 +1,36 @@
+# frozen-string-literal: true
+
 # This is the fastest connection pool, since it isn't a connection pool at all.
 # It is just a wrapper around a single connection that uses the connection pool
 # API.
 class Sequel::SingleConnectionPool < Sequel::ConnectionPool  
+  def initialize(db, opts=OPTS)
+    super
+    @conn = []
+  end
+
   # Yield the connection if one has been made.
   def all_connections
-    yield @conn if @conn
+    yield @conn.first unless @conn.empty?
   end
 
   # Disconnect the connection from the database.
   def disconnect(opts=nil)
-    return unless @conn
-    db.disconnect_connection(@conn)
-    @conn = nil
+    return unless c = @conn.first
+    disconnect_connection(c)
+    @conn.clear
+    nil
   end
-  
+
   # Yield the connection to the block.
   def hold(server=nil)
     begin
-      yield(@conn ||= make_new(DEFAULT_SERVER))
-    rescue Sequel::DatabaseDisconnectError
-      disconnect
+      unless c = @conn.first
+        @conn.replace([c = make_new(:default)])
+      end
+      yield c
+    rescue Sequel::DatabaseDisconnectError, *@error_classes => e
+      disconnect if disconnect_error?(e)
       raise
     end
   end
@@ -36,15 +47,13 @@ class Sequel::SingleConnectionPool < Sequel::ConnectionPool
   # The SingleConnectionPool always has a size of 1 if connected
   # and 0 if not.
   def size
-    @conn ? 1 : 0
+    @conn.empty? ? 0 : 1
   end
 
   private
 
   # Make sure there is a valid connection.
-  def preconnect
+  def preconnect(concurrent = nil)
     hold{}
   end
-
-  CONNECTION_POOL_MAP[[true, false]] = self
 end