sequel 5.61.0 → 5.63.0

data/lib/sequel/connection_pool/timed_queue.rb

@@ -0,0 +1,257 @@
+# frozen-string-literal: true
+
+# :nocov:
+raise LoadError, "Sequel::TimedQueueConnectionPool is only available on Ruby 3.2+" unless RUBY_VERSION >= '3.2'
+# :nocov:
+
+# A connection pool allowing multi-threaded access to a pool of connections,
+# using a timed queue (only available in Ruby 3.2+).
+class Sequel::TimedQueueConnectionPool < Sequel::ConnectionPool
+  # The maximum number of connections this pool will create.
+  attr_reader :max_size
+  
+  # The following additional options are respected:
+  # :max_connections :: The maximum number of connections the connection pool
+  #                     will open (default 4)
+  # :pool_timeout :: The amount of seconds to wait to acquire a connection
+  #                  before raising a PoolTimeout (default 5)
+  def initialize(db, opts = OPTS)
+    super
+    @max_size = Integer(opts[:max_connections] || 4)
+    raise(Sequel::Error, ':max_connections must be positive') if @max_size < 1
+    @mutex = Mutex.new  
+    # Size inside array so this still works while the pool is frozen.
+    @size = [0]
+    @allocated = {}
+    @allocated.compare_by_identity
+    @timeout = Float(opts[:pool_timeout] || 5)
+    @queue = Queue.new
+  end
+
+  # Yield all of the available connections, and the one currently allocated to
+  # this thread.  This will not yield connections currently allocated to other
+  # threads, as it is not safe to operate on them.
+  def all_connections
+    hold do |conn|
+      yield conn
+
+      # Use a hash to record all connections already seen.  As soon as we
+      # come across a connection we've already seen, we stop the loop.
+      conns = {}
+      conns.compare_by_identity
+      while true
+        conn = nil
+        begin
+          break unless (conn = @queue.pop(timeout: 0)) && !conns[conn]
+          conns[conn] = true
+          yield conn
+        ensure
+          @queue.push(conn) if conn
+        end
+      end
+    end
+  end
+  
+  # Removes all connections currently in the pool's queue. This method has the effect of 
+  # disconnecting from the database, assuming that no connections are currently
+  # being used.
+  # 
+  # Once a connection is requested using #hold, the connection pool
+  # creates new connections to the database.
+  def disconnect(opts=OPTS)
+    while conn = @queue.pop(timeout: 0)
+      disconnect_connection(conn)
+    end
+    fill_queue
+    nil
+  end
+
+  # Chooses the first available connection, or if none are
+  # available, creates a new connection.  Passes the connection to the supplied
+  # block:
+  # 
+  #   pool.hold {|conn| conn.execute('DROP TABLE posts')}
+  # 
+  # Pool#hold is re-entrant, meaning it can be called recursively in
+  # the same thread without blocking.
+  #
+  # If no connection is immediately available and the pool is already using the maximum
+  # number of connections, Pool#hold will block until a connection
+  # is available or the timeout expires.  If the timeout expires before a
+  # connection can be acquired, a Sequel::PoolTimeout is raised.
+  def hold(server=nil)
+    t = Sequel.current
+    if conn = sync{@allocated[t]}
+      return yield(conn)
+    end
+
+    begin
+      conn = acquire(t)
+      yield conn
+    rescue Sequel::DatabaseDisconnectError, *@error_classes => e
+      if disconnect_error?(e)
+        oconn = conn
+        conn = nil
+        disconnect_connection(oconn) if oconn
+        sync{@allocated.delete(t)}
+        fill_queue
+      end
+      raise
+    ensure
+      release(t) if conn
+    end
+  end
+
+  def pool_type
+    :timed_queue
+  end
+  
+  # The total number of connections in the pool.
+  def size
+    sync{@size[0]}
+  end
+  
+  private
+
+  # Create a new connection, after the pool's current size has already
+  # been updated to account for the new connection.  If there is an exception
+  # when creating the connection, decrement the current size.
+  #
+  # This should only be called after can_make_new?.  If there is an exception
+  # between when can_make_new? is called and when preallocated_make_new
+  # is called, it has the effect of reducing the maximum size of the
+  # connection pool by 1, since the current size of the pool will show a
+  # higher number than the number of connections allocated or
+  # in the queue.
+  #
+  # Calling code should not have the mutex when calling this.
+  def preallocated_make_new
+    make_new(:default)
+  rescue Exception
+    sync{@size[0] -= 1}
+    raise
+  end
+
+  # Decrement the current size of the pool when disconnecting connections.
+  #
+  # Calling code should not have the mutex when calling this.
+  def disconnect_connection(conn)
+    sync{@size[0] -= 1}
+    super
+  end
+
+  # If there are any threads waiting on the queue, try to create
+  # new connections in a separate thread if the pool is not yet at the
+  # maximum size.
+  #
+  # The reason for this method is to handle cases where acquire
+  # could not retrieve a connection immediately, and the pool
+  # was already at the maximum size.  In that case, the acquire will
+  # wait on the queue until the timeout.  This method is called
+  # after disconnecting to potentially add new connections to the
+  # pool, so the threads that are currently waiting for connections
+  # do not timeout after the pool is no longer full.
+  def fill_queue
+    if @queue.num_waiting > 0
+      Thread.new do
+        while @queue.num_waiting > 0 && (conn = try_make_new)
+          @queue.push(conn)
+        end
+      end
+    end
+  end
+
+  # Whether the given size is less than the maximum size of the pool.
+  # In that case, the pool's current size is incremented.  If this
+  # method returns true, space in the pool for the connection is
+  # preallocated, and preallocated_make_new should be called to
+  # create the connection.
+  #
+  # Calling code should have the mutex when calling this.
+  def can_make_new?(current_size)
+    if @max_size > current_size
+      @size[0] += 1
+    end
+  end
+
+  # Try to make a new connection if there is space in the pool.
+  # If the pool is already full, look for dead threads/fibers and
+  # disconnect the related connections.
+  #
+  # Calling code should not have the mutex when calling this.
+  def try_make_new
+    return preallocated_make_new if sync{can_make_new?(@size[0])}
+
+    to_disconnect = nil
+    do_make_new = false
+
+    sync do
+      current_size = @size[0]
+      @allocated.keys.each do |t|
+        unless t.alive?
+          (to_disconnect ||= []) << @allocated.delete(t)
+          current_size -= 1
+        end
+      end
+    
+      do_make_new = true if can_make_new?(current_size)
+    end
+
+    begin
+      preallocated_make_new if do_make_new
+    ensure
+      if to_disconnect
+        to_disconnect.each{|conn| disconnect_connection(conn)}
+        fill_queue
+      end
+    end
+  end
+  
+  # Assigns a connection to the supplied thread, if one
+  # is available.
+  #
+  # This should return a connection is one is available within the timeout,
+  # or raise PoolTimeout if a connection could not be acquired within the timeout.
+  #
+  # Calling code should not have the mutex when calling this.
+  def acquire(thread)
+    if conn = @queue.pop(timeout: 0) || try_make_new || @queue.pop(timeout: @timeout)
+      sync{@allocated[thread] = conn}
+    else
+      name = db.opts[:name]
+      raise ::Sequel::PoolTimeout, "timeout: #{@timeout}#{", database name: #{name}" if name}"
+    end
+  end
+
+  # Create the maximum number of connections immediately. This should not be called
+  # with a true argument unles no code is currently operating on the database.
+  #
+  # Calling code should not have the mutex when calling this.
+  def preconnect(concurrent = false)
+    if concurrent
+      if times = sync{@max_size > (size = @size[0]) ? @max_size - size : false}
+        times.times.map{Thread.new{if conn = try_make_new; @queue.push(conn) end}}.map(&:value)
+      end
+    else
+      while conn = try_make_new
+        @queue.push(conn)
+      end
+    end
+
+    nil
+  end
+
+  # Releases the connection assigned to the supplied thread back to the pool.
+  #
+  # Calling code should not have the mutex when calling this.
+  def release(thread)
+    @queue.push(sync{@allocated.delete(thread)})
+  end
+
+  # Yield to the block while inside the mutex.
+  #
+  # Calling code should not have the mutex when calling this.
+  def sync
+    @mutex.synchronize{yield}
+  end
+end

data/lib/sequel/connection_pool.rb

@@ -30,8 +30,11 @@ class Sequel::ConnectionPool
     :threaded => :ThreadedConnectionPool,
     :single => :SingleConnectionPool,
     :sharded_threaded => :ShardedThreadedConnectionPool,
-    :sharded_single => :ShardedSingleConnectionPool
-  }.freeze
+    :sharded_single => :ShardedSingleConnectionPool,
+    :timed_queue => :TimedQueueConnectionPool,
+  }
+  POOL_CLASS_MAP.to_a.each{|k, v| POOL_CLASS_MAP[k.to_s] = v}
+  POOL_CLASS_MAP.freeze
 
   # Class methods used to return an appropriate pool subclass, separated
   # into a module for easier overridding by extensions.
@@ -74,26 +77,35 @@ class Sequel::ConnectionPool
 
   # The after_connect proc used for this pool.  This is called with each new
   # connection made, and is usually used to set custom per-connection settings.
-  attr_accessor :after_connect
+  # Deprecated.
+  attr_reader :after_connect # SEQUEL6: Remove
 
-  # An array of sql strings to execute on each new connection.
-  attr_accessor :connect_sqls
+  # Override the after_connect proc for the connection pool. Deprecated.
+  # Disables support for shard-specific :after_connect and :connect_sqls if used.
+  def after_connect=(v) # SEQUEL6: Remove
+    @use_old_connect_api = true
+    @after_connect = v
+  end
+
+  # An array of sql strings to execute on each new connection. Deprecated.
+  attr_reader :connect_sqls # SEQUEL6: Remove
+
+  # Override the connect_sqls for the connection pool. Deprecated.
+  # Disables support for shard-specific :after_connect and :connect_sqls if used.
+  def connect_sqls=(v) # SEQUEL6: Remove
+    @use_old_connect_api = true
+    @connect_sqls = v
+  end
 
   # The Sequel::Database object tied to this connection pool.
   attr_accessor :db
 
-  # Instantiates a connection pool with the given options.  The block is called
-  # with a single symbol (specifying the server/shard to use) every time a new
-  # connection is needed.  The following options are respected for all connection
-  # pools:
-  # :after_connect :: A callable object called after each new connection is made, with the
-  #                   connection object (and server argument if the callable accepts 2 arguments),
-  #                   useful for customizations that you want to apply to all connections.
-  # :connect_sqls :: An array of sql strings to execute on each new connection, after :after_connect runs.
-  def initialize(db, opts=OPTS)
+  # Instantiates a connection pool with the given Database and options.
+  def initialize(db, opts=OPTS) # SEQUEL6: Remove second argument, always use db.opts
     @db = db
-    @after_connect = opts[:after_connect]
-    @connect_sqls = opts[:connect_sqls]
+    @use_old_connect_api = false # SEQUEL6: Remove
+    @after_connect = opts[:after_connect] # SEQUEL6: Remove
+    @connect_sqls = opts[:connect_sqls] # SEQUEL6: Remove
     @error_classes = db.send(:database_error_classes).dup.freeze
   end
   
@@ -119,25 +131,30 @@ class Sequel::ConnectionPool
   # and checking for connection errors.
   def make_new(server)
     begin
-      conn = @db.connect(server)
+      if @use_old_connect_api
+        # SEQUEL6: Remove block
+        conn = @db.connect(server)
 
-      if ac = @after_connect
-        if ac.arity == 2
-          ac.call(conn, server)
-        else
-          ac.call(conn)
+        if ac = @after_connect
+          if ac.arity == 2
+            ac.call(conn, server)
+          else
+            ac.call(conn)
+          end
         end
-      end
-
-      if cs = @connect_sqls
-        cs.each do |sql|
-          db.send(:log_connection_execute, conn, sql)
+  
+        if cs = @connect_sqls
+          cs.each do |sql|
+            db.send(:log_connection_execute, conn, sql)
+          end
         end
+
+        conn
+      else
+        @db.new_connection(server)
       end
     rescue Exception=>exception
       raise Sequel.convert_exception_class(exception, Sequel::DatabaseConnectionError)
-    end
-    raise(Sequel::DatabaseConnectionError, "Connection parameters not valid") unless conn
-    conn
+    end || raise(Sequel::DatabaseConnectionError, "Connection parameters not valid")
   end
 end

data/lib/sequel/database/connecting.rb

@@ -241,6 +241,30 @@ module Sequel
       pool.servers
     end
 
+    # Connect to the given server/shard. Handles database-generic post-connection
+    # setup not handled by #connect, using the :after_connect and :connect_sqls
+    # options.
+    def new_connection(server)
+      conn = connect(server)
+      opts = server_opts(server)
+
+      if ac = opts[:after_connect]
+        if ac.arity == 2
+          ac.call(conn, server)
+        else
+          ac.call(conn)
+        end
+      end
+
+      if cs = opts[:connect_sqls]
+        cs.each do |sql|
+          log_connection_execute(conn, sql)
+        end
+      end
+
+      conn
+    end
+
     # Returns true if the database is using a single-threaded connection pool.
     def single_threaded?
       @single_threaded

data/lib/sequel/database/misc.rb

@@ -100,10 +100,14 @@ module Sequel
     # options hash.
     #
     # Accepts the following options:
+    # :after_connect :: A callable object called after each new connection is made, with the
+    #                   connection object (and server argument if the callable accepts 2 arguments),
+    #                   useful for customizations that you want to apply to all connections.
     # :before_preconnect :: Callable that runs after extensions from :preconnect_extensions are loaded,
     #                       but before any connections are created.
     # :cache_schema :: Whether schema should be cached for this Database instance
     # :check_string_typecast_bytesize :: Whether to check the bytesize of strings before typecasting.
+    # :connect_sqls :: An array of sql strings to execute on each new connection, after :after_connect runs.
     # :default_string_column_size :: The default size of string columns, 255 by default.
     # :extensions :: Extensions to load into this Database instance.  Can be a symbol, array of symbols,
     #                or string with extensions separated by columns.  These extensions are loaded after
@@ -126,9 +130,11 @@ module Sequel
     # :single_threaded :: Whether to use a single-threaded connection pool.
     # :sql_log_level :: Method to use to log SQL to a logger, :info by default.
     #
+    # For sharded connection pools, :after_connect and :connect_sqls can be specified per-shard.
+    #
     # All options given are also passed to the connection pool.  Additional options respected by
-    # the connection pool are :after_connect, :connect_sqls, :max_connections, :pool_timeout,
-    # :servers, and :servers_hash.  See the connection pool documentation for details.
+    # the connection pool are :max_connections, :pool_timeout, :servers, and :servers_hash.  See the
+    # connection pool documentation for details.
     def initialize(opts = OPTS)
       @opts ||= opts
       @opts = connection_pool_default_options.merge(@opts)

data/lib/sequel/database/query.rb

@@ -175,6 +175,9 @@ module Sequel
         if !c[:max_length] && c[:type] == :string && (max_length = column_schema_max_length(c[:db_type]))
           c[:max_length] = max_length
         end
+        if !c[:max_value] && !c[:min_value] && c[:type] == :integer && (min_max = column_schema_integer_min_max_values(c[:db_type]))
+          c[:min_value], c[:max_value] = min_max
+        end
       end
       schema_post_process(cols)
 
@@ -272,6 +275,40 @@ module Sequel
       column_schema_default_to_ruby_value(default, type) rescue nil
     end
 
+    INTEGER1_MIN_MAX = [-128, 127].freeze
+    INTEGER2_MIN_MAX = [-32768, 32767].freeze
+    INTEGER3_MIN_MAX = [-8388608, 8388607].freeze
+    INTEGER4_MIN_MAX = [-2147483648, 2147483647].freeze
+    INTEGER8_MIN_MAX = [-9223372036854775808, 9223372036854775807].freeze
+    UNSIGNED_INTEGER1_MIN_MAX = [0, 255].freeze
+    UNSIGNED_INTEGER2_MIN_MAX = [0, 65535].freeze
+    UNSIGNED_INTEGER3_MIN_MAX = [0, 16777215].freeze
+    UNSIGNED_INTEGER4_MIN_MAX = [0, 4294967295].freeze
+    UNSIGNED_INTEGER8_MIN_MAX = [0, 18446744073709551615].freeze
+
+    # Look at the db_type and guess the minimum and maximum integer values for
+    # the column.
+    def column_schema_integer_min_max_values(db_type)
+      unsigned = /unsigned/i =~ db_type
+      case db_type
+      when /big|int8/i
+        unsigned ? UNSIGNED_INTEGER8_MIN_MAX : INTEGER8_MIN_MAX
+      when /medium/i
+        unsigned ? UNSIGNED_INTEGER3_MIN_MAX : INTEGER3_MIN_MAX
+      when /small|int2/i
+        unsigned ? UNSIGNED_INTEGER2_MIN_MAX : INTEGER2_MIN_MAX
+      when /tiny/i
+        (unsigned || column_schema_tinyint_type_is_unsigned?) ? UNSIGNED_INTEGER1_MIN_MAX : INTEGER1_MIN_MAX
+      else
+        unsigned ? UNSIGNED_INTEGER4_MIN_MAX : INTEGER4_MIN_MAX
+      end
+    end
+
+    # Whether the tinyint type (if supported by the database) is unsigned by default.
+    def column_schema_tinyint_type_is_unsigned?
+      false
+    end
+
     # Look at the db_type and guess the maximum length of the column.
     # This assumes types such as varchar(255).
     def column_schema_max_length(db_type)

data/lib/sequel/dataset/actions.rb

@@ -127,6 +127,18 @@ module Sequel
     #
     #   DB[:table].delete # DELETE * FROM table
     #   # => 3
+    #
+    # Some databases support using multiple tables in a DELETE query. This requires
+    # multiple FROM tables (JOINs can also be used). As multiple FROM tables use
+    # an implicit CROSS JOIN, you should make sure your WHERE condition uses the
+    # appropriate filters for the FROM tables:
+    #
+    #  DB.from(:a, :b).join(:c, :d=>Sequel[:b][:e]).where{{a[:f]=>b[:g], a[:id]=>c[:h]}}.
+    #    delete
+    #  # DELETE FROM a
+    #  # USING b
+    #  # INNER JOIN c ON (c.d = b.e)
+    #  # WHERE ((a.f = b.g) AND (a.id = c.h))
     def delete(&block)
       sql = delete_sql
       if uses_returning?(:delete)
@@ -313,14 +325,18 @@ module Sequel
     
     # Inserts multiple records into the associated table. This method can be
     # used to efficiently insert a large number of records into a table in a
-    # single query if the database supports it. Inserts
-    # are automatically wrapped in a transaction.
+    # single query if the database supports it. Inserts are automatically
+    # wrapped in a transaction if necessary.
     # 
     # This method is called with a columns array and an array of value arrays:
     #
     #   DB[:table].import([:x, :y], [[1, 2], [3, 4]])
     #   # INSERT INTO table (x, y) VALUES (1, 2) 
-    #   # INSERT INTO table (x, y) VALUES (3, 4) 
+    #   # INSERT INTO table (x, y) VALUES (3, 4)
+    #
+    # or, if the database supports it:
+    #
+    #   # INSERT INTO table (x, y) VALUES (1, 2), (3, 4)
     #
     # This method also accepts a dataset instead of an array of value arrays:
     #
@@ -328,9 +344,13 @@ module Sequel
     #   # INSERT INTO table (x, y) SELECT a, b FROM table2 
     #
     # Options:
-    # :commit_every :: Open a new transaction for every given number of records.
-    #                  For example, if you provide a value of 50, will commit
-    #                  after every 50 records.
+    # :commit_every :: Open a new transaction for every given number of
+    #                  records. For example, if you provide a value of 50,
+    #                  will commit after every 50 records. When a
+    #                  transaction is not required, this option controls
+    #                  the maximum number of values to insert with a single
+    #                  statement; it does not force the use of a
+    #                  transaction.
     # :return :: When this is set to :primary_key, returns an array of
     #            autoincremented primary key values for the rows inserted.
     #            This does not have an effect if +values+ is a Dataset.
@@ -576,7 +596,7 @@ module Sequel
     #   # SELECT * FROM table ORDER BY id LIMIT 1000 OFFSET 1000
     #   # ...
     #
-    #   DB[:table].order(:id).paged_each(:rows_per_fetch=>100){|row| }
+    #   DB[:table].order(:id).paged_each(rows_per_fetch: 100){|row| }
     #   # SELECT * FROM table ORDER BY id LIMIT 100
     #   # SELECT * FROM table ORDER BY id LIMIT 100 OFFSET 100
     #   # ...
@@ -918,6 +938,19 @@ module Sequel
     #
     #   DB[:table].update(x: Sequel[:x]+1, y: 0) # UPDATE table SET x = (x + 1), y = 0
     #   # => 10
+    #
+    # Some databases support using multiple tables in an UPDATE query. This requires
+    # multiple FROM tables (JOINs can also be used). As multiple FROM tables use
+    # an implicit CROSS JOIN, you should make sure your WHERE condition uses the
+    # appropriate filters for the FROM tables:
+    #
+    #  DB.from(:a, :b).join(:c, :d=>Sequel[:b][:e]).where{{a[:f]=>b[:g], a[:id]=>10}}.
+    #    update(:f=>Sequel[:c][:h])
+    #  # UPDATE a
+    #  # SET f = c.h
+    #  # FROM b
+    #  # INNER JOIN c ON (c.d = b.e)
+    #  # WHERE ((a.f = b.g) AND (a.id = 10))
     def update(values=OPTS, &block)
       sql = update_sql(values)
       if uses_returning?(:update)
@@ -1023,18 +1056,19 @@ module Sequel
 
     # Internals of #import.  If primary key values are requested, use
     # separate insert commands for each row.  Otherwise, call #multi_insert_sql
-    # and execute each statement it gives separately.
+    # and execute each statement it gives separately. A transaction is only used
+    # if there are multiple statements to execute.
     def _import(columns, values, opts)
       trans_opts = Hash[opts]
       trans_opts[:server] = @opts[:server]
       if opts[:return] == :primary_key
-        @db.transaction(trans_opts){values.map{|v| insert(columns, v)}}
+        _import_transaction(values, trans_opts){values.map{|v| insert(columns, v)}}
       else
         stmts = multi_insert_sql(columns, values)
-        @db.transaction(trans_opts){stmts.each{|st| execute_dui(st)}}
+        _import_transaction(stmts, trans_opts){stmts.each{|st| execute_dui(st)}}
       end
     end
-  
+
     # Return an array of arrays of values given by the symbols in ret_cols.
     def _select_map_multiple(ret_cols)
       map{|r| r.values_at(*ret_cols)}
@@ -1073,6 +1107,17 @@ module Sequel
       end
     end
     
+    # Use a transaction when yielding to the block if multiple values/statements
+    # are provided. When only a single value or statement is provided, then yield
+    # without using a transaction.
+    def _import_transaction(values, trans_opts, &block)
+      if values.length > 1
+        @db.transaction(trans_opts, &block)
+      else
+        yield
+      end
+    end
+  
     # Internals of +select_hash+ and +select_hash_groups+
     def _select_hash(meth, key_column, value_column, opts=OPTS)
       select(*(key_column.is_a?(Array) ? key_column : [key_column]) + (value_column.is_a?(Array) ? value_column : [value_column])).

data/lib/sequel/dataset/features.rb

@@ -152,6 +152,11 @@ module Sequel
       supports_distinct_on?
     end
     
+    # Whether placeholder literalizers are supported, true by default.
+    def supports_placeholder_literalizer?
+      true
+    end
+
     # Whether the dataset supports pattern matching by regular expressions, false by default.
     def supports_regexp?
       false

data/lib/sequel/dataset/misc.rb

@@ -302,7 +302,7 @@ module Sequel
           cache_set(key, loader + 1)
           loader = nil
         end
-      elsif cache_sql?
+      elsif cache_sql? && supports_placeholder_literalizer?
         cache_set(key, 1)
       end