sequel 5.39.0 → 5.63.0

data/lib/sequel/connection_pool/timed_queue.rb

@@ -0,0 +1,257 @@
+# frozen-string-literal: true
+
+# :nocov:
+raise LoadError, "Sequel::TimedQueueConnectionPool is only available on Ruby 3.2+" unless RUBY_VERSION >= '3.2'
+# :nocov:
+
+# A connection pool allowing multi-threaded access to a pool of connections,
+# using a timed queue (only available in Ruby 3.2+).
+class Sequel::TimedQueueConnectionPool < Sequel::ConnectionPool
+  # The maximum number of connections this pool will create.
+  attr_reader :max_size
+  
+  # The following additional options are respected:
+  # :max_connections :: The maximum number of connections the connection pool
+  #                     will open (default 4)
+  # :pool_timeout :: The amount of seconds to wait to acquire a connection
+  #                  before raising a PoolTimeout (default 5)
+  def initialize(db, opts = OPTS)
+    super
+    @max_size = Integer(opts[:max_connections] || 4)
+    raise(Sequel::Error, ':max_connections must be positive') if @max_size < 1
+    @mutex = Mutex.new  
+    # Size inside array so this still works while the pool is frozen.
+    @size = [0]
+    @allocated = {}
+    @allocated.compare_by_identity
+    @timeout = Float(opts[:pool_timeout] || 5)
+    @queue = Queue.new
+  end
+
+  # Yield all of the available connections, and the one currently allocated to
+  # this thread.  This will not yield connections currently allocated to other
+  # threads, as it is not safe to operate on them.
+  def all_connections
+    hold do |conn|
+      yield conn
+
+      # Use a hash to record all connections already seen.  As soon as we
+      # come across a connection we've already seen, we stop the loop.
+      conns = {}
+      conns.compare_by_identity
+      while true
+        conn = nil
+        begin
+          break unless (conn = @queue.pop(timeout: 0)) && !conns[conn]
+          conns[conn] = true
+          yield conn
+        ensure
+          @queue.push(conn) if conn
+        end
+      end
+    end
+  end
+  
+  # Removes all connections currently in the pool's queue. This method has the effect of 
+  # disconnecting from the database, assuming that no connections are currently
+  # being used.
+  # 
+  # Once a connection is requested using #hold, the connection pool
+  # creates new connections to the database.
+  def disconnect(opts=OPTS)
+    while conn = @queue.pop(timeout: 0)
+      disconnect_connection(conn)
+    end
+    fill_queue
+    nil
+  end
+
+  # Chooses the first available connection, or if none are
+  # available, creates a new connection.  Passes the connection to the supplied
+  # block:
+  # 
+  #   pool.hold {|conn| conn.execute('DROP TABLE posts')}
+  # 
+  # Pool#hold is re-entrant, meaning it can be called recursively in
+  # the same thread without blocking.
+  #
+  # If no connection is immediately available and the pool is already using the maximum
+  # number of connections, Pool#hold will block until a connection
+  # is available or the timeout expires.  If the timeout expires before a
+  # connection can be acquired, a Sequel::PoolTimeout is raised.
+  def hold(server=nil)
+    t = Sequel.current
+    if conn = sync{@allocated[t]}
+      return yield(conn)
+    end
+
+    begin
+      conn = acquire(t)
+      yield conn
+    rescue Sequel::DatabaseDisconnectError, *@error_classes => e
+      if disconnect_error?(e)
+        oconn = conn
+        conn = nil
+        disconnect_connection(oconn) if oconn
+        sync{@allocated.delete(t)}
+        fill_queue
+      end
+      raise
+    ensure
+      release(t) if conn
+    end
+  end
+
+  def pool_type
+    :timed_queue
+  end
+  
+  # The total number of connections in the pool.
+  def size
+    sync{@size[0]}
+  end
+  
+  private
+
+  # Create a new connection, after the pool's current size has already
+  # been updated to account for the new connection.  If there is an exception
+  # when creating the connection, decrement the current size.
+  #
+  # This should only be called after can_make_new?.  If there is an exception
+  # between when can_make_new? is called and when preallocated_make_new
+  # is called, it has the effect of reducing the maximum size of the
+  # connection pool by 1, since the current size of the pool will show a
+  # higher number than the number of connections allocated or
+  # in the queue.
+  #
+  # Calling code should not have the mutex when calling this.
+  def preallocated_make_new
+    make_new(:default)
+  rescue Exception
+    sync{@size[0] -= 1}
+    raise
+  end
+
+  # Decrement the current size of the pool when disconnecting connections.
+  #
+  # Calling code should not have the mutex when calling this.
+  def disconnect_connection(conn)
+    sync{@size[0] -= 1}
+    super
+  end
+
+  # If there are any threads waiting on the queue, try to create
+  # new connections in a separate thread if the pool is not yet at the
+  # maximum size.
+  #
+  # The reason for this method is to handle cases where acquire
+  # could not retrieve a connection immediately, and the pool
+  # was already at the maximum size.  In that case, the acquire will
+  # wait on the queue until the timeout.  This method is called
+  # after disconnecting to potentially add new connections to the
+  # pool, so the threads that are currently waiting for connections
+  # do not timeout after the pool is no longer full.
+  def fill_queue
+    if @queue.num_waiting > 0
+      Thread.new do
+        while @queue.num_waiting > 0 && (conn = try_make_new)
+          @queue.push(conn)
+        end
+      end
+    end
+  end
+
+  # Whether the given size is less than the maximum size of the pool.
+  # In that case, the pool's current size is incremented.  If this
+  # method returns true, space in the pool for the connection is
+  # preallocated, and preallocated_make_new should be called to
+  # create the connection.
+  #
+  # Calling code should have the mutex when calling this.
+  def can_make_new?(current_size)
+    if @max_size > current_size
+      @size[0] += 1
+    end
+  end
+
+  # Try to make a new connection if there is space in the pool.
+  # If the pool is already full, look for dead threads/fibers and
+  # disconnect the related connections.
+  #
+  # Calling code should not have the mutex when calling this.
+  def try_make_new
+    return preallocated_make_new if sync{can_make_new?(@size[0])}
+
+    to_disconnect = nil
+    do_make_new = false
+
+    sync do
+      current_size = @size[0]
+      @allocated.keys.each do |t|
+        unless t.alive?
+          (to_disconnect ||= []) << @allocated.delete(t)
+          current_size -= 1
+        end
+      end
+    
+      do_make_new = true if can_make_new?(current_size)
+    end
+
+    begin
+      preallocated_make_new if do_make_new
+    ensure
+      if to_disconnect
+        to_disconnect.each{|conn| disconnect_connection(conn)}
+        fill_queue
+      end
+    end
+  end
+  
+  # Assigns a connection to the supplied thread, if one
+  # is available.
+  #
+  # This should return a connection is one is available within the timeout,
+  # or raise PoolTimeout if a connection could not be acquired within the timeout.
+  #
+  # Calling code should not have the mutex when calling this.
+  def acquire(thread)
+    if conn = @queue.pop(timeout: 0) || try_make_new || @queue.pop(timeout: @timeout)
+      sync{@allocated[thread] = conn}
+    else
+      name = db.opts[:name]
+      raise ::Sequel::PoolTimeout, "timeout: #{@timeout}#{", database name: #{name}" if name}"
+    end
+  end
+
+  # Create the maximum number of connections immediately. This should not be called
+  # with a true argument unles no code is currently operating on the database.
+  #
+  # Calling code should not have the mutex when calling this.
+  def preconnect(concurrent = false)
+    if concurrent
+      if times = sync{@max_size > (size = @size[0]) ? @max_size - size : false}
+        times.times.map{Thread.new{if conn = try_make_new; @queue.push(conn) end}}.map(&:value)
+      end
+    else
+      while conn = try_make_new
+        @queue.push(conn)
+      end
+    end
+
+    nil
+  end
+
+  # Releases the connection assigned to the supplied thread back to the pool.
+  #
+  # Calling code should not have the mutex when calling this.
+  def release(thread)
+    @queue.push(sync{@allocated.delete(thread)})
+  end
+
+  # Yield to the block while inside the mutex.
+  #
+  # Calling code should not have the mutex when calling this.
+  def sync
+    @mutex.synchronize{yield}
+  end
+end

data/lib/sequel/connection_pool.rb

@@ -30,8 +30,11 @@ class Sequel::ConnectionPool
     :threaded => :ThreadedConnectionPool,
     :single => :SingleConnectionPool,
     :sharded_threaded => :ShardedThreadedConnectionPool,
-    :sharded_single => :ShardedSingleConnectionPool
-  }.freeze
+    :sharded_single => :ShardedSingleConnectionPool,
+    :timed_queue => :TimedQueueConnectionPool,
+  }
+  POOL_CLASS_MAP.to_a.each{|k, v| POOL_CLASS_MAP[k.to_s] = v}
+  POOL_CLASS_MAP.freeze
 
   # Class methods used to return an appropriate pool subclass, separated
   # into a module for easier overridding by extensions.
@@ -74,26 +77,35 @@ class Sequel::ConnectionPool
 
   # The after_connect proc used for this pool.  This is called with each new
   # connection made, and is usually used to set custom per-connection settings.
-  attr_accessor :after_connect
+  # Deprecated.
+  attr_reader :after_connect # SEQUEL6: Remove
 
-  # An array of sql strings to execute on each new connection.
-  attr_accessor :connect_sqls
+  # Override the after_connect proc for the connection pool. Deprecated.
+  # Disables support for shard-specific :after_connect and :connect_sqls if used.
+  def after_connect=(v) # SEQUEL6: Remove
+    @use_old_connect_api = true
+    @after_connect = v
+  end
+
+  # An array of sql strings to execute on each new connection. Deprecated.
+  attr_reader :connect_sqls # SEQUEL6: Remove
+
+  # Override the connect_sqls for the connection pool. Deprecated.
+  # Disables support for shard-specific :after_connect and :connect_sqls if used.
+  def connect_sqls=(v) # SEQUEL6: Remove
+    @use_old_connect_api = true
+    @connect_sqls = v
+  end
 
   # The Sequel::Database object tied to this connection pool.
   attr_accessor :db
 
-  # Instantiates a connection pool with the given options.  The block is called
-  # with a single symbol (specifying the server/shard to use) every time a new
-  # connection is needed.  The following options are respected for all connection
-  # pools:
-  # :after_connect :: A callable object called after each new connection is made, with the
-  #                   connection object (and server argument if the callable accepts 2 arguments),
-  #                   useful for customizations that you want to apply to all connections.
-  # :connect_sqls :: An array of sql strings to execute on each new connection, after :after_connect runs.
-  def initialize(db, opts=OPTS)
+  # Instantiates a connection pool with the given Database and options.
+  def initialize(db, opts=OPTS) # SEQUEL6: Remove second argument, always use db.opts
     @db = db
-    @after_connect = opts[:after_connect]
-    @connect_sqls = opts[:connect_sqls]
+    @use_old_connect_api = false # SEQUEL6: Remove
+    @after_connect = opts[:after_connect] # SEQUEL6: Remove
+    @connect_sqls = opts[:connect_sqls] # SEQUEL6: Remove
     @error_classes = db.send(:database_error_classes).dup.freeze
   end
   
@@ -119,25 +131,30 @@ class Sequel::ConnectionPool
   # and checking for connection errors.
   def make_new(server)
     begin
-      conn = @db.connect(server)
+      if @use_old_connect_api
+        # SEQUEL6: Remove block
+        conn = @db.connect(server)
 
-      if ac = @after_connect
-        if ac.arity == 2
-          ac.call(conn, server)
-        else
-          ac.call(conn)
+        if ac = @after_connect
+          if ac.arity == 2
+            ac.call(conn, server)
+          else
+            ac.call(conn)
+          end
         end
-      end
-
-      if cs = @connect_sqls
-        cs.each do |sql|
-          db.send(:log_connection_execute, conn, sql)
+  
+        if cs = @connect_sqls
+          cs.each do |sql|
+            db.send(:log_connection_execute, conn, sql)
+          end
         end
+
+        conn
+      else
+        @db.new_connection(server)
       end
     rescue Exception=>exception
       raise Sequel.convert_exception_class(exception, Sequel::DatabaseConnectionError)
-    end
-    raise(Sequel::DatabaseConnectionError, "Connection parameters not valid") unless conn
-    conn
+    end || raise(Sequel::DatabaseConnectionError, "Connection parameters not valid")
   end
 end

data/lib/sequel/core.rb

@@ -176,6 +176,17 @@ module Sequel
       JSON.parse(json, :create_additions=>false)
     end
 
+    # If a mutex is given, synchronize access using it.  If nil is given, just
+    # yield to the block.  This is designed for cases where a mutex may or may
+    # not be provided.
+    def synchronize_with(mutex)
+      if mutex
+        mutex.synchronize{yield}
+      else
+        yield
+      end
+    end
+
     # Convert each item in the array to the correct type, handling multi-dimensional
     # arrays.  For each element in the array or subarrays, call the converter,
     # unless the value is nil.
@@ -267,11 +278,9 @@ module Sequel
     #
     #   Sequel.string_to_date('2010-09-10') # Date.civil(2010, 09, 10)
     def string_to_date(string)
-      begin
-        Date.parse(string, Sequel.convert_two_digit_years)
-      rescue => e
-        raise convert_exception_class(e, InvalidValue)
-      end
+      Date.parse(string, Sequel.convert_two_digit_years)
+    rescue => e
+      raise convert_exception_class(e, InvalidValue)
     end
 
     # Converts the given +string+ into a +Time+ or +DateTime+ object, depending on the
@@ -279,15 +288,13 @@ module Sequel
     #
     #   Sequel.string_to_datetime('2010-09-10 10:20:30') # Time.local(2010, 09, 10, 10, 20, 30)
     def string_to_datetime(string)
-      begin
-        if datetime_class == DateTime
-          DateTime.parse(string, convert_two_digit_years)
-        else
-          datetime_class.parse(string)
-        end
-      rescue => e
-        raise convert_exception_class(e, InvalidValue)
+      if datetime_class == DateTime
+        DateTime.parse(string, convert_two_digit_years)
+      else
+        datetime_class.parse(string)
       end
+    rescue => e
+      raise convert_exception_class(e, InvalidValue)
     end
 
     # Converts the given +string+ into a <tt>Sequel::SQLTime</tt> object.
@@ -295,11 +302,9 @@ module Sequel
     #   v = Sequel.string_to_time('10:20:30') # Sequel::SQLTime.parse('10:20:30')
     #   DB.literal(v) # => '10:20:30'
     def string_to_time(string)
-      begin
-        SQLTime.parse(string)
-      rescue => e
-        raise convert_exception_class(e, InvalidValue)
-      end
+      SQLTime.parse(string)
+    rescue => e
+      raise convert_exception_class(e, InvalidValue)
     end
 
     # Unless in single threaded mode, protects access to any mutable
@@ -389,6 +394,11 @@ module Sequel
 
     private
 
+    # Return a hash of date information parsed from the given string.
+    def _date_parse(string)
+      Date._parse(string)
+    end
+
     # Helper method that the database adapter class methods that are added to Sequel via
     # metaprogramming use to parse arguments.
     def adapter_method(adapter, *args, &block)

data/lib/sequel/database/connecting.rb

@@ -55,11 +55,11 @@ module Sequel
 
       begin
         db = c.new(opts)
-        if block_given?
+        if defined?(yield)
           return yield(db)
         end
       ensure
-        if block_given?
+        if defined?(yield)
           db.disconnect if db
           Sequel.synchronize{::Sequel::DATABASES.delete(db)}
         end
@@ -241,6 +241,30 @@ module Sequel
       pool.servers
     end
 
+    # Connect to the given server/shard. Handles database-generic post-connection
+    # setup not handled by #connect, using the :after_connect and :connect_sqls
+    # options.
+    def new_connection(server)
+      conn = connect(server)
+      opts = server_opts(server)
+
+      if ac = opts[:after_connect]
+        if ac.arity == 2
+          ac.call(conn, server)
+        else
+          ac.call(conn)
+        end
+      end
+
+      if cs = opts[:connect_sqls]
+        cs.each do |sql|
+          log_connection_execute(conn, sql)
+        end
+      end
+
+      conn
+    end
+
     # Returns true if the database is using a single-threaded connection pool.
     def single_threaded?
       @single_threaded

data/lib/sequel/database/misc.rb

@@ -91,11 +91,23 @@ module Sequel
     # The specific default size of string columns for this Sequel::Database, usually 255 by default.
     attr_accessor :default_string_column_size
 
+    # Whether to check the bytesize of strings before typecasting (to avoid typecasting strings that
+    # would be too long for the given type), true by default. Strings that are too long will raise
+    # a typecasting error.
+    attr_accessor :check_string_typecast_bytesize
+
     # Constructs a new instance of a database connection with the specified
     # options hash.
     #
     # Accepts the following options:
+    # :after_connect :: A callable object called after each new connection is made, with the
+    #                   connection object (and server argument if the callable accepts 2 arguments),
+    #                   useful for customizations that you want to apply to all connections.
+    # :before_preconnect :: Callable that runs after extensions from :preconnect_extensions are loaded,
+    #                       but before any connections are created.
     # :cache_schema :: Whether schema should be cached for this Database instance
+    # :check_string_typecast_bytesize :: Whether to check the bytesize of strings before typecasting.
+    # :connect_sqls :: An array of sql strings to execute on each new connection, after :after_connect runs.
     # :default_string_column_size :: The default size of string columns, 255 by default.
     # :extensions :: Extensions to load into this Database instance.  Can be a symbol, array of symbols,
     #                or string with extensions separated by columns.  These extensions are loaded after
@@ -105,7 +117,7 @@ module Sequel
     # :loggers :: An array of loggers to use.
     # :log_connection_info :: Whether connection information should be logged when logging queries.
     # :log_warn_duration :: The number of elapsed seconds after which queries should be logged at warn level.
-    # :name :: A name to use for the Database object, displayed in PoolTimeout .
+    # :name :: A name to use for the Database object, displayed in PoolTimeout.
     # :preconnect :: Automatically create the maximum number of connections, so that they don't
     #                need to be created as needed.  This is useful when connecting takes a long time
     #                and you want to avoid possible latency during runtime.
@@ -114,13 +126,15 @@ module Sequel
     # :preconnect_extensions :: Similar to the :extensions option, but loads the extensions before the
     #                           connections are made by the :preconnect option.
     # :quote_identifiers :: Whether to quote identifiers.
-    # :servers :: A hash specifying a server/shard specific options, keyed by shard symbol .
+    # :servers :: A hash specifying a server/shard specific options, keyed by shard symbol.
     # :single_threaded :: Whether to use a single-threaded connection pool.
     # :sql_log_level :: Method to use to log SQL to a logger, :info by default.
     #
+    # For sharded connection pools, :after_connect and :connect_sqls can be specified per-shard.
+    #
     # All options given are also passed to the connection pool.  Additional options respected by
-    # the connection pool are :after_connect, :connect_sqls, :max_connections, :pool_timeout,
-    # :servers, and :servers_hash.  See the connection pool documentation for details.
+    # the connection pool are :max_connections, :pool_timeout, :servers, and :servers_hash.  See the
+    # connection pool documentation for details.
     def initialize(opts = OPTS)
       @opts ||= opts
       @opts = connection_pool_default_options.merge(@opts)
@@ -130,6 +144,7 @@ module Sequel
       @opts[:adapter_class] = self.class
       @opts[:single_threaded] = @single_threaded = typecast_value_boolean(@opts.fetch(:single_threaded, Sequel.single_threaded))
       @default_string_column_size = @opts[:default_string_column_size] || DEFAULT_STRING_COLUMN_SIZE
+      @check_string_typecast_bytesize = typecast_value_boolean(@opts.fetch(:check_string_typecast_bytesize, true))
 
       @schemas = {}
       @prepared_statements = {}
@@ -160,6 +175,10 @@ module Sequel
 
         initialize_load_extensions(:preconnect_extensions)
 
+        if before_preconnect = @opts[:before_preconnect]
+          before_preconnect.call(self)
+        end
+
         if typecast_value_boolean(@opts[:preconnect]) && @pool.respond_to?(:preconnect, true)
           concurrent = typecast_value_string(@opts[:preconnect]) == "concurrently"
           @pool.send(:preconnect, concurrent)
@@ -213,8 +232,7 @@ module Sequel
       Sequel.extension(*exts)
       exts.each do |ext|
         if pr = Sequel.synchronize{EXTENSIONS[ext]}
-          unless Sequel.synchronize{@loaded_extensions.include?(ext)}
-            Sequel.synchronize{@loaded_extensions << ext}
+          if Sequel.synchronize{@loaded_extensions.include?(ext) ? false : (@loaded_extensions << ext)}
             pr.call(self)
           end
         else
@@ -460,6 +478,21 @@ module Sequel
     #   Don't rescue other exceptions, they will be raised normally.
     end
 
+    # Check the bytesize of a string before conversion. There is no point
+    # trying to typecast strings that would be way too long.
+    def typecast_check_string_length(string, max_size)
+      if @check_string_typecast_bytesize && string.bytesize > max_size
+        raise InvalidValue, "string too long to typecast (bytesize: #{string.bytesize}, max: #{max_size})"
+      end
+      string
+    end
+
+    # Check the bytesize of the string value, if value is a string.
+    def typecast_check_length(value, max_size)
+      typecast_check_string_length(value, max_size) if String === value
+      value
+    end
+
     # Typecast the value to an SQL::Blob
     def typecast_value_blob(value)
       value.is_a?(Sequel::SQL::Blob) ? value : Sequel::SQL::Blob.new(value)
@@ -483,9 +516,9 @@ module Sequel
       when Date
         value
       when String
-        Sequel.string_to_date(value)
+        Sequel.string_to_date(typecast_check_string_length(value, 100))
       when Hash
-        Date.new(*[:year, :month, :day].map{|x| (value[x] || value[x.to_s]).to_i})
+        Date.new(*[:year, :month, :day].map{|x| typecast_check_length(value[x] || value[x.to_s], 100).to_i})
       else
         raise InvalidValue, "invalid value for Date: #{value.inspect}"
       end
@@ -493,7 +526,17 @@ module Sequel
 
     # Typecast the value to a DateTime or Time depending on Sequel.datetime_class
     def typecast_value_datetime(value)
-      Sequel.typecast_to_application_timestamp(value)
+      case value
+      when String
+        Sequel.typecast_to_application_timestamp(typecast_check_string_length(value, 100))
+      when Hash
+        [:year, :month, :day, :hour, :minute, :second, :nanos, :offset].each do |x|
+          typecast_check_length(value[x] || value[x.to_s], 100)
+        end
+        Sequel.typecast_to_application_timestamp(value)
+      else
+        Sequel.typecast_to_application_timestamp(value)
+      end
     end
     
     if RUBY_VERSION >= '2.4'
@@ -526,18 +569,30 @@ module Sequel
       when Numeric
         BigDecimal(value.to_s)
       when String
-        _typecast_value_string_to_decimal(value)
+        _typecast_value_string_to_decimal(typecast_check_string_length(value, 1000))
       else
         raise InvalidValue, "invalid value for BigDecimal: #{value.inspect}"
       end
     end
 
     # Typecast the value to a Float
-    alias typecast_value_float Float
+    def typecast_value_float(value)
+      Float(typecast_check_length(value, 1000))
+    end
 
     # Typecast the value to an Integer
     def typecast_value_integer(value)
-      (value.is_a?(String) && value =~ /\A0+(\d)/) ? Integer(value, 10) : Integer(value)
+      case value
+      when String
+        typecast_check_string_length(value, 100)
+        if value =~ /\A-?0+(\d)/
+          Integer(value, 10)
+        else
+          Integer(value)
+        end
+      else
+        Integer(value)
+      end
     end
 
     # Typecast the value to a String
@@ -560,9 +615,9 @@ module Sequel
           SQLTime.create(value.hour, value.min, value.sec, value.nsec/1000.0)
         end
       when String
-        Sequel.string_to_time(value)
+        Sequel.string_to_time(typecast_check_string_length(value, 100))
       when Hash
-        SQLTime.create(*[:hour, :minute, :second].map{|x| (value[x] || value[x.to_s]).to_i})
+        SQLTime.create(*[:hour, :minute, :second].map{|x| typecast_check_length(value[x] || value[x.to_s], 100).to_i})
       else
         raise Sequel::InvalidValue, "invalid value for Time: #{value.inspect}"
       end