sequel 3.40.0 → 3.41.0

data/lib/sequel/connection_pool/sharded_threaded.rb

@@ -8,13 +8,12 @@ Sequel.require 'connection_pool/threaded'
 class Sequel::ShardedThreadedConnectionPool < Sequel::ThreadedConnectionPool
   # The following additional options are respected:
   # * :servers - A hash of servers to use.  Keys should be symbols.  If not
-  #   present, will use a single :default server.  The server name symbol will
-  #   be passed to the connection_proc.
+  #   present, will use a single :default server.
   # * :servers_hash - The base hash to use for the servers.  By default,
   #   Sequel uses Hash.new(:default).  You can use a hash with a default proc
   #   that raises an error if you want to catch all cases where a nonexistent
   #   server is used.
-  def initialize(opts = {}, &block)
+  def initialize(db, opts = {})
     super
     @available_connections = {}
     @connections_to_remove = []
@@ -87,11 +86,10 @@ class Sequel::ShardedThreadedConnectionPool < Sequel::ThreadedConnectionPool
   # creates new connections to the database. Options:
   # * :server - Should be a symbol specifing the server to disconnect from,
   #   or an array of symbols to specify multiple servers.
-  def disconnect(opts={}, &block)
-    block ||= @disconnection_proc
+  def disconnect(opts={})
     sync do
       (opts[:server] ? Array(opts[:server]) : @servers.keys).each do |s|
-        disconnect_server(s, &block)
+        disconnect_server(s)
       end
     end
   end
@@ -145,7 +143,7 @@ class Sequel::ShardedThreadedConnectionPool < Sequel::ThreadedConnectionPool
       raise(Sequel::Error, "cannot remove default server") if servers.include?(:default)
       servers.each do |server|
         if @servers.include?(server)
-          disconnect_server(server, &@disconnection_proc)
+          disconnect_server(server)
           @available_connections.delete(server)
           @allocated.delete(server)
           @servers.delete(server)
@@ -159,6 +157,10 @@ class Sequel::ShardedThreadedConnectionPool < Sequel::ThreadedConnectionPool
     sync{@servers.keys}
   end
 
+  def pool_type
+    :sharded_threaded
+  end
+  
   private
 
   # Assigns a connection to the supplied thread for the given server, if one
@@ -176,16 +178,30 @@ class Sequel::ShardedThreadedConnectionPool < Sequel::ThreadedConnectionPool
   # available, tries to create a new connection. The calling code should already
   # have the mutex before calling this.
   def available(server)
-    available_connections(server).pop || make_new(server)
+    next_available(server) || make_new(server)
+  end
+
+  # Return a connection to the pool of available connections for the server,
+  # returns the connection. The calling code should already have the mutex
+  # before calling this.
+  def checkin_connection(server, conn)
+    case @connection_handling
+    when :queue
+      available_connections(server).unshift(conn)
+    else
+      available_connections(server) << conn
+    end
+
+    conn
   end
 
   # Disconnect from the given server.  Disconnects available connections
   # immediately, and schedules currently allocated connections for disconnection
   # as soon as they are returned to the pool. The calling code should already
   # have the mutex before calling this.
-  def disconnect_server(server, &block)
+  def disconnect_server(server)
     if conns = available_connections(server)
-      conns.each{|conn| block.call(conn)} if block
+      conns.each{|conn| db.disconnect_connection(conn)}
       conns.clear
     end
     @connections_to_remove.concat(allocated(server).values)
@@ -201,6 +217,13 @@ class Sequel::ShardedThreadedConnectionPool < Sequel::ThreadedConnectionPool
     end
     default_make_new(server) if (n || size(server)) < @max_size
   end
+
+  # Return the next available connection in the pool for the given server, or nil
+  # if there is not currently an available connection for the server.
+  # The calling code should already have the mutex before calling this.
+  def next_available(server)
+    available_connections(server).pop
+  end
   
   # Returns the connection owned by the supplied thread for the given server,
   # if any. The calling code should NOT already have the mutex before calling this.
@@ -223,13 +246,10 @@ class Sequel::ShardedThreadedConnectionPool < Sequel::ThreadedConnectionPool
     else
       conn = allocated(server).delete(thread)
 
-      case @connection_handling
-      when :queue
-        available_connections(server).unshift(conn)
-      when :disconnect
-        @disconnection_proc.call(conn) if @disconnection_proc
+      if @connection_handling == :disconnect
+        db.disconnect_connection(conn)
       else
-        available_connections(server) << conn
+        checkin_connection(server, conn)
       end
     end
   end
@@ -239,7 +259,7 @@ class Sequel::ShardedThreadedConnectionPool < Sequel::ThreadedConnectionPool
   def remove(thread, conn, server)
     @connections_to_remove.delete(conn)
     allocated(server).delete(thread) if @servers.include?(server)
-    @disconnection_proc.call(conn) if @disconnection_proc
+    db.disconnect_connection(conn)
   end
   
   CONNECTION_POOL_MAP[[false, true]] = self

data/lib/sequel/connection_pool/single.rb

@@ -14,10 +14,9 @@ class Sequel::SingleConnectionPool < Sequel::ConnectionPool
   end
   
   # Disconnect the connection from the database.
-  def disconnect(opts=nil, &block)
+  def disconnect(opts=nil)
     return unless @conn
-    block ||= @disconnection_proc
-    block.call(@conn) if block
+    db.disconnect_connection(@conn)
     @conn = nil
   end
   
@@ -31,5 +30,9 @@ class Sequel::SingleConnectionPool < Sequel::ConnectionPool
     end
   end
 
+  def pool_type
+    :single
+  end
+  
   CONNECTION_POOL_MAP[[true, false]] = self
 end

data/lib/sequel/connection_pool/threaded.rb

@@ -22,7 +22,7 @@ class Sequel::ThreadedConnectionPool < Sequel::ConnectionPool
   #   a connection again (default 0.001)
   # * :pool_timeout - The amount of seconds to wait to acquire a connection
   #   before raising a PoolTimeoutError (default 5)
-  def initialize(opts = {}, &block)
+  def initialize(db, opts = {})
     super
     @max_size = Integer(opts[:max_connections] || 4)
     raise(Sequel::Error, ':max_connections must be positive') if @max_size < 1
@@ -64,10 +64,9 @@ class Sequel::ThreadedConnectionPool < Sequel::ConnectionPool
   # 
   # Once a connection is requested using #hold, the connection pool
   # creates new connections to the database.
-  def disconnect(opts={}, &block)
-    block ||= @disconnection_proc
+  def disconnect(opts={})
     sync do
-      @available_connections.each{|conn| block.call(conn)} if block
+      @available_connections.each{|conn| db.disconnect_connection(conn)}
       @available_connections.clear
     end
   end
@@ -106,13 +105,17 @@ class Sequel::ThreadedConnectionPool < Sequel::ConnectionPool
     rescue Sequel::DatabaseDisconnectError
       oconn = conn
       conn = nil
-      @disconnection_proc.call(oconn) if @disconnection_proc && oconn
+      db.disconnect_connection(oconn) if oconn
       @allocated.delete(t)
       raise
     ensure
       sync{release(t)} if conn
     end
   end
+
+  def pool_type
+    :threaded
+  end
   
   private
 
@@ -131,7 +134,20 @@ class Sequel::ThreadedConnectionPool < Sequel::ConnectionPool
   # available, tries to create a new connection. The calling code should already
   # have the mutex before calling this.
   def available
-    @available_connections.pop || make_new(DEFAULT_SERVER)
+    next_available || make_new(DEFAULT_SERVER)
+  end
+
+  # Return a connection to the pool of available connections, returns the connection.
+  # The calling code should already have the mutex before calling this.
+  def checkin_connection(conn)
+    case @connection_handling
+    when :queue
+      @available_connections.unshift(conn)
+    else
+      @available_connections << conn
+    end
+
+    conn
   end
 
   # Alias the default make_new method, so subclasses can call it directly.
@@ -147,7 +163,14 @@ class Sequel::ThreadedConnectionPool < Sequel::ConnectionPool
     end
     super if (n || size) < @max_size
   end
-  
+
+  # Return the next available connection in the pool, or nil if there
+  # is not currently an available connection.  The calling code should already
+  # have the mutex before calling this.
+  def next_available
+    @available_connections.pop
+  end
+
   # Returns the connection owned by the supplied thread,
   # if any. The calling code should NOT already have the mutex before calling this.
   def owned_connection(thread)
@@ -159,13 +182,10 @@ class Sequel::ThreadedConnectionPool < Sequel::ConnectionPool
   def release(thread)
     conn = @allocated.delete(thread)
 
-    case @connection_handling
-    when :queue
-      @available_connections.unshift(conn)
-    when :disconnect
-      @disconnection_proc.call(conn) if @disconnection_proc
+    if @connection_handling == :disconnect
+      db.disconnect_connection(conn)
     else
-      @available_connections << conn
+      checkin_connection(conn)
     end
   end
 

data/lib/sequel/database/connecting.rb

@@ -140,7 +140,7 @@ module Sequel
         @pool.add_servers(servers.keys)
       end
     end
-    
+
     # Connects to the database. This method should be overridden by descendants.
     def connect(server)
       raise NotImplemented, "#connect should be overridden by adapters"
@@ -173,6 +173,13 @@ module Sequel
       pool.disconnect(opts)
     end
 
+    # Should only be called by the connection pool code to disconnect a connection.
+    # By default, calls the close method on the connection object, since most
+    # adapters use that, but should be overwritten on other adapters.
+    def disconnect_connection(conn)
+      conn.close
+    end
+
     # Yield a new Database instance for every server in the connection pool.
     # Intended for use in sharded environments where there is a need to make schema
     # modifications (DDL queries) on each shard.
@@ -243,6 +250,21 @@ module Sequel
       true
     end
 
+    # Check whether the given connection is currently valid, by
+    # running a query against it.  If the query fails, the
+    # connection should probably be removed from the connection
+    # pool.
+    def valid_connection?(conn)
+      sql = valid_connection_sql
+      begin
+        log_connection_execute(conn, sql)
+      rescue Sequel::DatabaseError, *database_error_classes
+        false
+      else
+        true
+      end
+    end
+
     private
     
     # The default options for the connection pool.
@@ -271,5 +293,10 @@ module Sequel
       opts.delete(:servers)
       opts
     end
+
+    # The SQL query to issue to check if a connection is valid.
+    def valid_connection_sql
+      @valid_connection_sql ||= select(nil).sql
+    end
   end
 end

data/lib/sequel/database/logging.rb

@@ -19,7 +19,7 @@ module Sequel
 
     # Log a message at error level, with information about the exception.
     def log_exception(exception, message)
-      log_each(:error, "#{exception.class}: #{exception.message.strip}: #{message}")
+      log_each(:error, "#{exception.class}: #{exception.message.strip if exception.message}: #{message}")
     end
 
     # Log a message at level info to all loggers.

data/lib/sequel/database/misc.rb

@@ -49,7 +49,6 @@ module Sequel
     #
     # Accepts the following options:
     # :default_schema :: The default schema to use, see #default_schema.
-    # :disconnection_proc :: A proc used to disconnect the connection
     # :identifier_input_method :: A string method symbol to call on identifiers going into the database
     # :identifier_output_method :: A string method symbol to call on identifiers coming from the database
     # :logger :: A specific logger to use
@@ -59,14 +58,12 @@ module Sequel
     # :single_threaded :: Whether to use a single-threaded connection pool
     # :sql_log_level :: Method to use to log SQL to a logger, :info by default.
     #
-    # All options given are also passed to the connection pool.  If a block
-    # is given, it is used as the connection_proc for the ConnectionPool.
+    # All options given are also passed to the connection pool.
     def initialize(opts = {}, &block)
       @opts ||= opts
       @opts = connection_pool_default_options.merge(@opts)
       @loggers = Array(@opts[:logger]) + Array(@opts[:loggers])
       self.log_warn_duration = @opts[:log_warn_duration]
-      @opts[:disconnection_proc] ||= proc{|conn| disconnect_connection(conn)}
       block ||= proc{|server| connect(server)}
       @opts[:servers] = {} if @opts[:servers].is_a?(String)
       @opts[:adapter_class] = self.class
@@ -84,7 +81,7 @@ module Sequel
       @cache_schema = typecast_value_boolean(@opts.fetch(:cache_schema, true))
       @dataset_modules = []
       self.sql_log_level = @opts[:sql_log_level] ? @opts[:sql_log_level].to_sym : :info
-      @pool = ConnectionPool.get_pool(@opts, &block)
+      @pool = ConnectionPool.get_pool(self, @opts)
 
       Sequel.synchronize{::Sequel::DATABASES.push(self)}
     end

data/lib/sequel/database/query.rb

@@ -605,7 +605,7 @@ module Sequel
     # such as :integer or :string.
     def schema_column_type(db_type)
       case db_type
-      when /\A(character( varying)?|n?(var)?char|n?text|string)/io
+      when /\A(character( varying)?|n?(var)?char|n?text|string|clob)/io
         :string
       when /\A(int(eger)?|(big|small|tiny)int)/io
         :integer
@@ -621,7 +621,7 @@ module Sequel
         :float
       when /\A(?:(?:(?:num(?:ber|eric)?|decimal)(?:\(\d+,\s*(\d+|false|true)\))?)|(?:small)?money)\z/io
         $1 && ['0', 'false'].include?($1) ? :integer : :decimal
-      when /bytea|[bc]lob|image|(var)?binary/io
+      when /bytea|blob|image|(var)?binary/io
         :blob
       when /\Aenum/io
         :enum

data/lib/sequel/database/schema_generator.rb

@@ -177,7 +177,7 @@ module Sequel
       # :concurrently :: Create the index concurrently, so it doesn't block
       #                  operations on the table while the index is being
       #                  built.
-      # :op_class :: Use a specific operator class in the index.
+      # :opclass :: Use a specific operator class in the index.
       #
       # Microsoft SQL Server specific options:
       #

data/lib/sequel/database/schema_methods.rb

@@ -154,6 +154,9 @@ module Sequel
     # :collate :: The collation to use for the table.
     # :engine :: The table engine to use for the table.
     #
+    # PostgreSQL specific options:
+    # :unlogged :: Create the table as an unlogged table.
+    #
     # See <tt>Schema::Generator</tt> and the {"Schema Modification" guide}[link:files/doc/schema_modification_rdoc.html].
     def create_table(name, options={}, &block)
       remove_cached_schema(name)

data/lib/sequel/dataset/query.rb

@@ -592,7 +592,7 @@ module Sequel
     #   DB[:items].limit(10...20) # SELECT * FROM items LIMIT 10 OFFSET 10
     #   DB[:items].limit(10..20) # SELECT * FROM items LIMIT 11 OFFSET 10
     #   DB[:items].limit(nil, 20) # SELECT * FROM items OFFSET 20
-    def limit(l, o = nil)
+    def limit(l, o = (no_offset = true; nil))
       return from_self.limit(l, o) if @opts[:sql]
 
       if Range === l
@@ -610,6 +610,8 @@ module Sequel
           raise(Error, 'Offsets must be greater than or equal to 0') unless o >= 0
         end
         opts[:offset] = o
+      elsif !no_offset
+        opts[:offset] = nil
       end
       clone(opts)
     end
@@ -753,15 +755,17 @@ module Sequel
     # given, the existing order is inverted.
     #
     #   DB[:items].reverse(:id) # SELECT * FROM items ORDER BY id DESC
+    #   DB[:items].reverse{foo(bar)} # SELECT * FROM items ORDER BY foo(bar) DESC
     #   DB[:items].order(:id).reverse # SELECT * FROM items ORDER BY id DESC
     #   DB[:items].order(:id).reverse(Sequel.desc(:name)) # SELECT * FROM items ORDER BY name ASC
-    def reverse(*order)
+    def reverse(*order, &block)
+      virtual_row_columns(order, block)
       order(*invert_order(order.empty? ? @opts[:order] : order))
     end
 
     # Alias of +reverse+
-    def reverse_order(*order)
-      reverse(*order)
+    def reverse_order(*order, &block)
+      reverse(*order, &block)
     end
 
     # Returns a copy of the dataset with the columns selected changed

data/lib/sequel/dataset/sql.rb

@@ -290,6 +290,7 @@ module Sequel
       column_all_sql
       complex_expression_sql
       constant_sql
+      delayed_evaluation_sql
       function_sql
       join_clause_sql
       join_on_clause_sql
@@ -495,6 +496,12 @@ module Sequel
       sql << constant.to_s
     end
 
+    # SQL fragment for delayed evaluations, evaluating the
+    # object and literalizing the returned value.
+    def delayed_evaluation_sql_append(sql, callable)
+      literal_append(sql, callable.call)
+    end
+
     # SQL fragment specifying an emulated SQL function call.
     # By default, assumes just the function name may need to
     # be emulated, adapters should set an EMULATED_FUNCTION_MAP

data/lib/sequel/extensions/arbitrary_servers.rb

@@ -97,7 +97,7 @@ module Sequel
         a = @allocated[thread]
         a.delete(server)
         @allocated.delete(thread) if a.empty?
-        @disconnection_proc.call(conn) if @disconnection_proc
+        db.disconnect_connection(conn)
       else  
         super
       end

data/lib/sequel/extensions/connection_validator.rb

@@ -0,0 +1,109 @@
+# The connection_validator extension modifies a database's
+# connection pool to validate that connections checked out
+# from the pool are still valid, before yielding them for
+# use.  If it detects an invalid connection, it removes it
+# from the pool and tries the next available connection,
+# creating a new connection if no available connection is
+# valid.  Example of use:
+#
+#   DB.extension(:connection_validator)
+#
+# As checking connections for validity involves issuing a
+# query, which is potentially an expensive operation,
+# the validation checks are only run if the connection has
+# been idle for longer than a certain threshold. By default,
+# that threshold is 3600 seconds (1 hour), but it can be
+# modified by the user, set to -1 to always validate
+# connections on checkout:
+#
+#   DB.pool.connection_validation_timeout = -1
+#
+# Note that if you set the timeout to validate connections
+# on every checkout, you should probably manually control
+# connection checkouts on a coarse basis, using
+# Database#synchonrize.  In a web application, the optimal
+# place for that would be a rack middleware.  Validating
+# connections on every checkout without setting up coarse
+# connection checkouts will hurt performance, in some cases
+# significantly.  Note that setting up coarse connection
+# checkouts reduces the concurrency level acheivable.  For
+# example, in a web application, using Database#synchronize
+# in a rack middleware will limit the number of concurrent
+# web requests to the number to connections in the database
+# connection pool.
+#
+# Note that this extension only affects the default threaded
+# and the sharded threaded connection pool.  The single
+# threaded and sharded single threaded connection pools are
+# not affected.  As the only reason to use the single threaded
+# pools is for speed, and this extension makes the connection
+# pool slower, there's not much point in modifying this
+# extension to work with the single threaded pools.  The
+# threaded pools work fine even in single threaded code, so if
+# you are currently using a single threaded pool and want to
+# use this extension, switch to using a threaded pool.
+
+module Sequel
+  module ConnectionValidator
+    class Retry < Error; end
+
+    # The number of seconds that need to pass since
+    # connection checkin before attempting to validate
+    # the connection when checking it out from the pool.
+    # Defaults to 3600 seconds (1 hour).
+    attr_accessor :connection_validation_timeout
+
+    # Initialize the data structures used by this extension.
+    def self.extended(pool)
+      pool.instance_eval do
+        @connection_timestamps ||= {}
+        @connection_validation_timeout = 3600
+      end
+
+      # Make sure the valid connection SQL query is precached,
+      # otherwise it's possible it will happen at runtime. While
+      # it should work correctly at runtime, it's better to avoid
+      # the possibility of failure altogether.
+      pool.db.send(:valid_connection_sql)
+    end
+
+    private
+
+    # Record the time the connection was checked back into the pool.
+    def checkin_connection(*)
+      conn = super
+      @connection_timestamps[conn] = Time.now
+      conn
+    end
+
+    # When acquiring a connection, if it has been
+    # idle for longer than the connection validation timeout,
+    # test the connection for validity.  If it is not valid,
+    # disconnect the connection, and retry with a new connection.
+    def acquire(*a)
+      begin
+        if (conn = super) &&
+           (t = sync{@connection_timestamps.delete(conn)}) &&
+           Time.now - t > @connection_validation_timeout &&
+           !db.valid_connection?(conn)
+
+          if pool_type == :sharded_threaded
+            sync{allocated(a.last).delete(Thread.current)}
+          else
+            sync{@allocated.delete(Thread.current)}
+          end
+
+          db.disconnect_connection(conn)
+          raise Retry
+        end
+      rescue Retry
+        retry
+      end
+
+      conn
+    end
+  end
+
+  Database.register_extension(:connection_validator){|db| db.pool.extend(ConnectionValidator)}
+end
+