sequel 5.39.0 → 5.72.0

data/lib/sequel/connection_pool/timed_queue.rb

@@ -0,0 +1,270 @@
+# frozen-string-literal: true
+
+# :nocov:
+raise LoadError, "Sequel::TimedQueueConnectionPool is only available on Ruby 3.2+" unless RUBY_VERSION >= '3.2'
+# :nocov:
+
+# A connection pool allowing multi-threaded access to a pool of connections,
+# using a timed queue (only available in Ruby 3.2+).
+class Sequel::TimedQueueConnectionPool < Sequel::ConnectionPool
+  # The maximum number of connections this pool will create.
+  attr_reader :max_size
+  
+  # The following additional options are respected:
+  # :max_connections :: The maximum number of connections the connection pool
+  #                     will open (default 4)
+  # :pool_timeout :: The amount of seconds to wait to acquire a connection
+  #                  before raising a PoolTimeout (default 5)
+  def initialize(db, opts = OPTS)
+    super
+    @max_size = Integer(opts[:max_connections] || 4)
+    raise(Sequel::Error, ':max_connections must be positive') if @max_size < 1
+    @mutex = Mutex.new  
+    # Size inside array so this still works while the pool is frozen.
+    @size = [0]
+    @allocated = {}
+    @allocated.compare_by_identity
+    @timeout = Float(opts[:pool_timeout] || 5)
+    @queue = Queue.new
+  end
+
+  # Yield all of the available connections, and the one currently allocated to
+  # this thread.  This will not yield connections currently allocated to other
+  # threads, as it is not safe to operate on them.
+  def all_connections
+    hold do |conn|
+      yield conn
+
+      # Use a hash to record all connections already seen.  As soon as we
+      # come across a connection we've already seen, we stop the loop.
+      conns = {}
+      conns.compare_by_identity
+      while true
+        conn = nil
+        begin
+          break unless (conn = @queue.pop(timeout: 0)) && !conns[conn]
+          conns[conn] = true
+          yield conn
+        ensure
+          @queue.push(conn) if conn
+        end
+      end
+    end
+  end
+  
+  # Removes all connections currently in the pool's queue. This method has the effect of 
+  # disconnecting from the database, assuming that no connections are currently
+  # being used.
+  # 
+  # Once a connection is requested using #hold, the connection pool
+  # creates new connections to the database.
+  def disconnect(opts=OPTS)
+    while conn = @queue.pop(timeout: 0)
+      disconnect_connection(conn)
+    end
+    fill_queue
+    nil
+  end
+
+  # Chooses the first available connection, or if none are
+  # available, creates a new connection.  Passes the connection to the supplied
+  # block:
+  # 
+  #   pool.hold {|conn| conn.execute('DROP TABLE posts')}
+  # 
+  # Pool#hold is re-entrant, meaning it can be called recursively in
+  # the same thread without blocking.
+  #
+  # If no connection is immediately available and the pool is already using the maximum
+  # number of connections, Pool#hold will block until a connection
+  # is available or the timeout expires.  If the timeout expires before a
+  # connection can be acquired, a Sequel::PoolTimeout is raised.
+  def hold(server=nil)
+    t = Sequel.current
+    if conn = owned_connection(t)
+      return yield(conn)
+    end
+
+    begin
+      conn = acquire(t)
+      yield conn
+    rescue Sequel::DatabaseDisconnectError, *@error_classes => e
+      if disconnect_error?(e)
+        oconn = conn
+        conn = nil
+        disconnect_connection(oconn) if oconn
+        sync{@allocated.delete(t)}
+        fill_queue
+      end
+      raise
+    ensure
+      release(t) if conn
+    end
+  end
+
+  def pool_type
+    :timed_queue
+  end
+  
+  # The total number of connections in the pool.
+  def size
+    sync{@size[0]}
+  end
+  
+  private
+
+  # Create a new connection, after the pool's current size has already
+  # been updated to account for the new connection.  If there is an exception
+  # when creating the connection, decrement the current size.
+  #
+  # This should only be called after can_make_new?.  If there is an exception
+  # between when can_make_new? is called and when preallocated_make_new
+  # is called, it has the effect of reducing the maximum size of the
+  # connection pool by 1, since the current size of the pool will show a
+  # higher number than the number of connections allocated or
+  # in the queue.
+  #
+  # Calling code should not have the mutex when calling this.
+  def preallocated_make_new
+    make_new(:default)
+  rescue Exception
+    sync{@size[0] -= 1}
+    raise
+  end
+
+  # Decrement the current size of the pool when disconnecting connections.
+  #
+  # Calling code should not have the mutex when calling this.
+  def disconnect_connection(conn)
+    sync{@size[0] -= 1}
+    super
+  end
+
+  # If there are any threads waiting on the queue, try to create
+  # new connections in a separate thread if the pool is not yet at the
+  # maximum size.
+  #
+  # The reason for this method is to handle cases where acquire
+  # could not retrieve a connection immediately, and the pool
+  # was already at the maximum size.  In that case, the acquire will
+  # wait on the queue until the timeout.  This method is called
+  # after disconnecting to potentially add new connections to the
+  # pool, so the threads that are currently waiting for connections
+  # do not timeout after the pool is no longer full.
+  def fill_queue
+    if @queue.num_waiting > 0
+      Thread.new do
+        while @queue.num_waiting > 0 && (conn = try_make_new)
+          @queue.push(conn)
+        end
+      end
+    end
+  end
+
+  # Whether the given size is less than the maximum size of the pool.
+  # In that case, the pool's current size is incremented.  If this
+  # method returns true, space in the pool for the connection is
+  # preallocated, and preallocated_make_new should be called to
+  # create the connection.
+  #
+  # Calling code should have the mutex when calling this.
+  def can_make_new?(current_size)
+    if @max_size > current_size
+      @size[0] += 1
+    end
+  end
+
+  # Try to make a new connection if there is space in the pool.
+  # If the pool is already full, look for dead threads/fibers and
+  # disconnect the related connections.
+  #
+  # Calling code should not have the mutex when calling this.
+  def try_make_new
+    return preallocated_make_new if sync{can_make_new?(@size[0])}
+
+    to_disconnect = nil
+    do_make_new = false
+
+    sync do
+      current_size = @size[0]
+      @allocated.keys.each do |t|
+        unless t.alive?
+          (to_disconnect ||= []) << @allocated.delete(t)
+          current_size -= 1
+        end
+      end
+    
+      do_make_new = true if can_make_new?(current_size)
+    end
+
+    begin
+      preallocated_make_new if do_make_new
+    ensure
+      if to_disconnect
+        to_disconnect.each{|conn| disconnect_connection(conn)}
+        fill_queue
+      end
+    end
+  end
+  
+  # Assigns a connection to the supplied thread, if one
+  # is available.
+  #
+  # This should return a connection is one is available within the timeout,
+  # or raise PoolTimeout if a connection could not be acquired within the timeout.
+  #
+  # Calling code should not have the mutex when calling this.
+  def acquire(thread)
+    if conn = @queue.pop(timeout: 0) || try_make_new || @queue.pop(timeout: @timeout)
+      sync{@allocated[thread] = conn}
+    else
+      name = db.opts[:name]
+      raise ::Sequel::PoolTimeout, "timeout: #{@timeout}#{", database name: #{name}" if name}"
+    end
+  end
+
+  # Returns the connection owned by the supplied thread,
+  # if any. The calling code should NOT already have the mutex before calling this.
+  def owned_connection(thread)
+    sync{@allocated[thread]}
+  end
+  
+  # Create the maximum number of connections immediately. This should not be called
+  # with a true argument unless no code is currently operating on the database.
+  #
+  # Calling code should not have the mutex when calling this.
+  def preconnect(concurrent = false)
+    if concurrent
+      if times = sync{@max_size > (size = @size[0]) ? @max_size - size : false}
+        times.times.map{Thread.new{if conn = try_make_new; @queue.push(conn) end}}.map(&:value)
+      end
+    else
+      while conn = try_make_new
+        @queue.push(conn)
+      end
+    end
+
+    nil
+  end
+
+  # Releases the connection assigned to the supplied thread back to the pool.
+  #
+  # Calling code should not have the mutex when calling this.
+  def release(thread)
+    checkin_connection(sync{@allocated.delete(thread)})
+    nil
+  end
+
+  # Adds a connection to the queue of available connections, returns the connection.
+  def checkin_connection(conn)
+    @queue.push(conn)
+    conn
+  end
+
+  # Yield to the block while inside the mutex.
+  #
+  # Calling code should not have the mutex when calling this.
+  def sync
+    @mutex.synchronize{yield}
+  end
+end

data/lib/sequel/connection_pool.rb

@@ -30,8 +30,12 @@ class Sequel::ConnectionPool
     :threaded => :ThreadedConnectionPool,
     :single => :SingleConnectionPool,
     :sharded_threaded => :ShardedThreadedConnectionPool,
-    :sharded_single => :ShardedSingleConnectionPool
-  }.freeze
+    :sharded_single => :ShardedSingleConnectionPool,
+    :timed_queue => :TimedQueueConnectionPool,
+    :sharded_timed_queue => :ShardedTimedQueueConnectionPool,
+  }
+  POOL_CLASS_MAP.to_a.each{|k, v| POOL_CLASS_MAP[k.to_s] = v}
+  POOL_CLASS_MAP.freeze
 
   # Class methods used to return an appropriate pool subclass, separated
   # into a module for easier overridding by extensions.
@@ -39,7 +43,8 @@ class Sequel::ConnectionPool
     # Return a pool subclass instance based on the given options.  If a <tt>:pool_class</tt>
     # option is provided is provided, use that pool class, otherwise
     # use a new instance of an appropriate pool subclass based on the
-    # <tt>:single_threaded</tt> and <tt>:servers</tt> options.
+    # +SEQUEL_DEFAULT_CONNECTION_POOL+ environment variable if set, or
+    # the <tt>:single_threaded</tt> and <tt>:servers</tt> options, otherwise.
     def get_pool(db, opts = OPTS)
       connection_pool_class(opts).new(db, opts)
     end
@@ -59,9 +64,14 @@ class Sequel::ConnectionPool
         end
 
         pc
+      elsif pc = ENV['SEQUEL_DEFAULT_CONNECTION_POOL']
+        pc = "sharded_#{pc}" if opts[:servers] && !pc.start_with?('sharded_')
+        connection_pool_class(:pool_class=>pc)
       else
         pc = if opts[:single_threaded]
           opts[:servers] ? :sharded_single : :single
+        #elsif RUBY_VERSION >= '3.2' # SEQUEL6 or maybe earlier
+        #  opts[:servers] ? :sharded_timed_queue : :timed_queue
         else
           opts[:servers] ? :sharded_threaded : :threaded
         end
@@ -74,26 +84,35 @@ class Sequel::ConnectionPool
 
   # The after_connect proc used for this pool.  This is called with each new
   # connection made, and is usually used to set custom per-connection settings.
-  attr_accessor :after_connect
+  # Deprecated.
+  attr_reader :after_connect # SEQUEL6: Remove
+
+  # Override the after_connect proc for the connection pool. Deprecated.
+  # Disables support for shard-specific :after_connect and :connect_sqls if used.
+  def after_connect=(v) # SEQUEL6: Remove
+    @use_old_connect_api = true
+    @after_connect = v
+  end
 
-  # An array of sql strings to execute on each new connection.
-  attr_accessor :connect_sqls
+  # An array of sql strings to execute on each new connection. Deprecated.
+  attr_reader :connect_sqls # SEQUEL6: Remove
+
+  # Override the connect_sqls for the connection pool. Deprecated.
+  # Disables support for shard-specific :after_connect and :connect_sqls if used.
+  def connect_sqls=(v) # SEQUEL6: Remove
+    @use_old_connect_api = true
+    @connect_sqls = v
+  end
 
   # The Sequel::Database object tied to this connection pool.
   attr_accessor :db
 
-  # Instantiates a connection pool with the given options.  The block is called
-  # with a single symbol (specifying the server/shard to use) every time a new
-  # connection is needed.  The following options are respected for all connection
-  # pools:
-  # :after_connect :: A callable object called after each new connection is made, with the
-  #                   connection object (and server argument if the callable accepts 2 arguments),
-  #                   useful for customizations that you want to apply to all connections.
-  # :connect_sqls :: An array of sql strings to execute on each new connection, after :after_connect runs.
-  def initialize(db, opts=OPTS)
+  # Instantiates a connection pool with the given Database and options.
+  def initialize(db, opts=OPTS) # SEQUEL6: Remove second argument, always use db.opts
     @db = db
-    @after_connect = opts[:after_connect]
-    @connect_sqls = opts[:connect_sqls]
+    @use_old_connect_api = false # SEQUEL6: Remove
+    @after_connect = opts[:after_connect] # SEQUEL6: Remove
+    @connect_sqls = opts[:connect_sqls] # SEQUEL6: Remove
     @error_classes = db.send(:database_error_classes).dup.freeze
   end
   
@@ -119,25 +138,30 @@ class Sequel::ConnectionPool
   # and checking for connection errors.
   def make_new(server)
     begin
-      conn = @db.connect(server)
+      if @use_old_connect_api
+        # SEQUEL6: Remove block
+        conn = @db.connect(server)
 
-      if ac = @after_connect
-        if ac.arity == 2
-          ac.call(conn, server)
-        else
-          ac.call(conn)
+        if ac = @after_connect
+          if ac.arity == 2
+            ac.call(conn, server)
+          else
+            ac.call(conn)
+          end
         end
-      end
-
-      if cs = @connect_sqls
-        cs.each do |sql|
-          db.send(:log_connection_execute, conn, sql)
+  
+        if cs = @connect_sqls
+          cs.each do |sql|
+            db.send(:log_connection_execute, conn, sql)
+          end
         end
+
+        conn
+      else
+        @db.new_connection(server)
       end
     rescue Exception=>exception
       raise Sequel.convert_exception_class(exception, Sequel::DatabaseConnectionError)
-    end
-    raise(Sequel::DatabaseConnectionError, "Connection parameters not valid") unless conn
-    conn
+    end || raise(Sequel::DatabaseConnectionError, "Connection parameters not valid")
   end
 end

data/lib/sequel/core.rb

@@ -176,6 +176,17 @@ module Sequel
       JSON.parse(json, :create_additions=>false)
     end
 
+    # If a mutex is given, synchronize access using it.  If nil is given, just
+    # yield to the block.  This is designed for cases where a mutex may or may
+    # not be provided.
+    def synchronize_with(mutex)
+      if mutex
+        mutex.synchronize{yield}
+      else
+        yield
+      end
+    end
+
     # Convert each item in the array to the correct type, handling multi-dimensional
     # arrays.  For each element in the array or subarrays, call the converter,
     # unless the value is nil.
@@ -267,11 +278,9 @@ module Sequel
     #
     #   Sequel.string_to_date('2010-09-10') # Date.civil(2010, 09, 10)
     def string_to_date(string)
-      begin
-        Date.parse(string, Sequel.convert_two_digit_years)
-      rescue => e
-        raise convert_exception_class(e, InvalidValue)
-      end
+      Date.parse(string, Sequel.convert_two_digit_years)
+    rescue => e
+      raise convert_exception_class(e, InvalidValue)
     end
 
     # Converts the given +string+ into a +Time+ or +DateTime+ object, depending on the
@@ -279,15 +288,13 @@ module Sequel
     #
     #   Sequel.string_to_datetime('2010-09-10 10:20:30') # Time.local(2010, 09, 10, 10, 20, 30)
     def string_to_datetime(string)
-      begin
-        if datetime_class == DateTime
-          DateTime.parse(string, convert_two_digit_years)
-        else
-          datetime_class.parse(string)
-        end
-      rescue => e
-        raise convert_exception_class(e, InvalidValue)
+      if datetime_class == DateTime
+        DateTime.parse(string, convert_two_digit_years)
+      else
+        datetime_class.parse(string)
       end
+    rescue => e
+      raise convert_exception_class(e, InvalidValue)
     end
 
     # Converts the given +string+ into a <tt>Sequel::SQLTime</tt> object.
@@ -295,11 +302,9 @@ module Sequel
     #   v = Sequel.string_to_time('10:20:30') # Sequel::SQLTime.parse('10:20:30')
     #   DB.literal(v) # => '10:20:30'
     def string_to_time(string)
-      begin
-        SQLTime.parse(string)
-      rescue => e
-        raise convert_exception_class(e, InvalidValue)
-      end
+      SQLTime.parse(string)
+    rescue => e
+      raise convert_exception_class(e, InvalidValue)
     end
 
     # Unless in single threaded mode, protects access to any mutable
@@ -389,6 +394,11 @@ module Sequel
 
     private
 
+    # Return a hash of date information parsed from the given string.
+    def _date_parse(string)
+      Date._parse(string)
+    end
+
     # Helper method that the database adapter class methods that are added to Sequel via
     # metaprogramming use to parse arguments.
     def adapter_method(adapter, *args, &block)

data/lib/sequel/database/connecting.rb

@@ -8,7 +8,7 @@ module Sequel
     # ---------------------
 
     # Array of supported database adapters
-    ADAPTERS = %w'ado amalgalite ibmdb jdbc mock mysql mysql2 odbc oracle postgres sqlanywhere sqlite tinytds'.map(&:to_sym)
+    ADAPTERS = %w'ado amalgalite ibmdb jdbc mock mysql mysql2 odbc oracle postgres sqlanywhere sqlite tinytds trilogy'.map(&:to_sym)
 
     # The Database subclass for the given adapter scheme.
     # Raises Sequel::AdapterNotFound if the adapter
@@ -55,11 +55,11 @@ module Sequel
 
       begin
         db = c.new(opts)
-        if block_given?
+        if defined?(yield)
           return yield(db)
         end
       ensure
-        if block_given?
+        if defined?(yield)
           db.disconnect if db
           Sequel.synchronize{::Sequel::DATABASES.delete(db)}
         end
@@ -241,6 +241,30 @@ module Sequel
       pool.servers
     end
 
+    # Connect to the given server/shard. Handles database-generic post-connection
+    # setup not handled by #connect, using the :after_connect and :connect_sqls
+    # options.
+    def new_connection(server)
+      conn = connect(server)
+      opts = server_opts(server)
+
+      if ac = opts[:after_connect]
+        if ac.arity == 2
+          ac.call(conn, server)
+        else
+          ac.call(conn)
+        end
+      end
+
+      if cs = opts[:connect_sqls]
+        cs.each do |sql|
+          log_connection_execute(conn, sql)
+        end
+      end
+
+      conn
+    end
+
     # Returns true if the database is using a single-threaded connection pool.
     def single_threaded?
       @single_threaded

data/lib/sequel/database/dataset.rb

@@ -30,19 +30,29 @@ module Sequel
       @dataset_class.new(self)
     end
 
-    # Fetches records for an arbitrary SQL statement. If a block is given,
-    # it is used to iterate over the records:
+    # Returns a dataset instance for the given SQL string:
     #
-    #   DB.fetch('SELECT * FROM items'){|r| p r}
+    #   ds = DB.fetch('SELECT * FROM items')
+    #
+    # You can then call methods on the dataset to retrieve results:
     #
-    # The +fetch+ method returns a dataset instance:
+    #   ds.all
+    #   # SELECT * FROM items
+    #   # => [{:column=>value, ...}, ...]
     #
-    #   DB.fetch('SELECT * FROM items').all
+    # If a block is given, it is passed to #each on the resulting dataset to
+    # iterate over the records returned by the query:
+    #
+    #   DB.fetch('SELECT * FROM items'){|r| p r}
+    #   # {:column=>value, ...}
+    #   # ...
     #
     # +fetch+ can also perform parameterized queries for protection against SQL
     # injection:
     #
-    #   DB.fetch('SELECT * FROM items WHERE name = ?', my_name).all
+    #   ds = DB.fetch('SELECT * FROM items WHERE name = ?', "my name")
+    #   ds.all
+    #   # SELECT * FROM items WHERE name = 'my name'
     #
     # See caveats listed in Dataset#with_sql regarding datasets using custom
     # SQL and the methods that can be called on them.