sequel 3.21.0 → 3.36.0

data/lib/sequel/connection_pool/threaded.rb

@@ -13,6 +13,9 @@ class Sequel::ThreadedConnectionPool < Sequel::ConnectionPool
   attr_reader :allocated
 
   # The following additional options are respected:
+  # * :connection_handling - Set how to handle available connections.  By default,
+  #   uses a a stack for performance.  Can be set to :queue to use a queue, which reduces
+  #   the chances of connections becoming stale.
   # * :max_connections - The maximum number of connections the connection pool
   #   will open (default 4)
   # * :pool_sleep_time - The amount of time to sleep before attempting to acquire
@@ -24,22 +27,40 @@ class Sequel::ThreadedConnectionPool < Sequel::ConnectionPool
     @max_size = Integer(opts[:max_connections] || 4)
     raise(Sequel::Error, ':max_connections must be positive') if @max_size < 1
     @mutex = Mutex.new  
+    @queue = opts[:connection_handling] == :queue
     @available_connections = []
     @allocated = {}
     @timeout = Integer(opts[:pool_timeout] || 5)
     @sleep_time = Float(opts[:pool_sleep_time] || 0.001)
   end
   
-  # The total number of connections opened for the given server, should
-  # be equal to available_connections.length + allocated.length.
+  # The total number of connections opened, either available or allocated.
+  # This may not be completely accurate as it isn't protected by the mutex.
   def size
     @allocated.length + @available_connections.length
   end
   
-  # Removes all connections currently available on all servers, optionally
+  # Yield all of the available connections, and the one currently allocated to
+  # this thread.  This will not yield connections currently allocated to other
+  # threads, as it is not safe to operate on them.  This holds the mutex while
+  # it is yielding all of the available connections, which means that until
+  # the method's block returns, the pool is locked.
+  def all_connections
+    hold do |c|
+      sync do
+        yield c
+        @available_connections.each{|c| yield c}
+      end
+    end
+  end
+  
+  # Removes all connections currently available, optionally
   # yielding each connection to the given block. This method has the effect of 
   # disconnecting from the database, assuming that no connections are currently
-  # being used.
+  # being used.  If you want to be able to disconnect connections that are
+  # currently in use, use the ShardedThreadedConnectionPool, which can do that.
+  # This connection pool does not, for performance reasons. To use the sharded pool,
+  # pass the <tt>:servers=>{}</tt> option when connecting to the database.
   # 
   # Once a connection is requested using #hold, the connection pool
   # creates new connections to the database.
@@ -51,7 +72,7 @@ class Sequel::ThreadedConnectionPool < Sequel::ConnectionPool
     end
   end
   
-  # Chooses the first available connection to the given server, or if none are
+  # Chooses the first available connection, or if none are
   # available, creates a new connection.  Passes the connection to the supplied
   # block:
   # 
@@ -83,9 +104,10 @@ class Sequel::ThreadedConnectionPool < Sequel::ConnectionPool
       end
       yield conn
     rescue Sequel::DatabaseDisconnectError
-      @disconnection_proc.call(conn) if @disconnection_proc && conn
-      @allocated.delete(t)
+      oconn = conn
       conn = nil
+      @disconnection_proc.call(oconn) if @disconnection_proc && oconn
+      @allocated.delete(t)
       raise
     ensure
       sync{release(t)} if conn
@@ -94,7 +116,7 @@ class Sequel::ThreadedConnectionPool < Sequel::ConnectionPool
   
   private
 
-  # Assigns a connection to the supplied thread for the given server, if one
+  # Assigns a connection to the supplied thread, if one
   # is available. The calling code should NOT already have the mutex when
   # calling this.
   def acquire(thread)
@@ -105,7 +127,7 @@ class Sequel::ThreadedConnectionPool < Sequel::ConnectionPool
     end
   end
   
-  # Returns an available connection to the given server. If no connection is
+  # Returns an available connection. If no connection is
   # available, tries to create a new connection. The calling code should already
   # have the mutex before calling this.
   def available
@@ -126,18 +148,20 @@ class Sequel::ThreadedConnectionPool < Sequel::ConnectionPool
     super if (n || size) < @max_size
   end
   
-  # Returns the connection owned by the supplied thread for the given server,
+  # Returns the connection owned by the supplied thread,
   # if any. The calling code should NOT already have the mutex before calling this.
   def owned_connection(thread)
     sync{@allocated[thread]}
   end
   
-  # Releases the connection assigned to the supplied thread and server. If the
-  # server or connection given is scheduled for disconnection, remove the
-  # connection instead of releasing it back to the pool.
+  # Releases the connection assigned to the supplied thread back to the pool.
   # The calling code should already have the mutex before calling this.
   def release(thread)
-    @available_connections << @allocated.delete(thread)
+    if @queue
+      @available_connections.unshift(@allocated.delete(thread))
+    else
+      @available_connections << @allocated.delete(thread)
+    end
   end
 
   # Yield to the block while inside the mutex. The calling code should NOT

data/lib/sequel/connection_pool.rb

@@ -53,10 +53,18 @@ class Sequel::ConnectionPool
     
     # Return a connection pool class based on the given options.
     def connection_pool_class(opts)
-      opts[:pool_class] || CONNECTION_POOL_MAP[[!!opts[:single_threaded], !!opts[:servers]]]
+      CONNECTION_POOL_MAP[opts[:pool_class]] || opts[:pool_class] || CONNECTION_POOL_MAP[[!!opts[:single_threaded], !!opts[:servers]]]
     end
   end
   extend ClassMethods
+
+  # The after_connect proc used for this pool.  This is called with each new
+  # connection made, and is usually used to set custom per-connection settings.
+  attr_accessor :after_connect
+
+  # The disconnect_proc used for the pool.  This is called with each connection
+  # that is disconnected, usually to clean up related resources.
+  attr_accessor :disconnection_proc
   
   # Instantiates a connection pool with the given options.  The block is called
   # with a single symbol (specifying the server/shard to use) every time a new
@@ -80,7 +88,7 @@ class Sequel::ConnectionPool
   
   # An array of symbols for all shards/servers, which is a single <tt>:default</tt> by default.
   def servers
-    [:default]
+    [DEFAULT_SERVER]
   end
   
   private

data/lib/sequel/core.rb

@@ -17,20 +17,26 @@
 #
 #   Sequel.sqlite('blog.db'){|db| puts db[:users].count} 
 #
-# You can set the +SEQUEL_NO_CORE_EXTENSIONS+ constant or environment variable to have
-# Sequel not extend the core classes.
+# Sequel currently adds methods to the Array, Hash, String and Symbol classes by
+# default.  You can either require 'sequel/no_core_ext' or set the
+# +SEQUEL_NO_CORE_EXTENSIONS+ constant or environment variable before requiring
+# sequel to have # Sequel not add methods to those classes.
 #
 # For a more expanded introduction, see the {README}[link:files/README_rdoc.html].
 # For a quicker introduction, see the {cheat sheet}[link:files/doc/cheat_sheet_rdoc.html].
 module Sequel
   @convert_two_digit_years = true
   @datetime_class = Time
+  @empty_array_handle_nulls = true
   @virtual_row_instance_eval = true
   @require_thread = nil
   
   # Mutex used to protect file loading/requireing
   @require_mutex = Mutex.new
   
+  # Whether Sequel is being run in single threaded mode
+  @single_threaded = false
+  
   class << self
     # Sequel converts two digit years in <tt>Date</tt>s and <tt>DateTime</tt>s by default,
     # so 01/02/03 is interpreted at January 2nd, 2003, and 12/13/99 is interpreted
@@ -54,6 +60,27 @@ module Sequel
     # days on +DateTime+).
     attr_accessor :datetime_class
 
+    # Sets whether or not to attempt to handle NULL values correctly when given
+    # an empty array.  By default:
+    #
+    #   DB[:a].filter(:b=>[])
+    #   # SELECT * FROM a WHERE (b != b)
+    #   DB[:a].exclude(:b=>[])
+    #   # SELECT * FROM a WHERE (b = b)
+    #
+    # However, some databases (e.g. MySQL) will perform very poorly
+    # with this type of query.  You can set this to false to get the
+    # following behavior:
+    #
+    #   DB[:a].filter(:b=>[])
+    #   # SELECT * FROM a WHERE 1 = 0
+    #   DB[:a].exclude(:b=>[])
+    #   # SELECT * FROM a WHERE 1 = 1
+    # 
+    # This may not handle NULLs correctly, but can be much faster on
+    # some databases.
+    attr_accessor :empty_array_handle_nulls
+
     # For backwards compatibility, has no effect.
     attr_accessor :virtual_row_instance_eval
     
@@ -64,6 +91,7 @@ module Sequel
 
     # Make thread safe requiring reentrant to prevent deadlocks.
     def check_requiring_thread
+      return yield if @single_threaded
       t = Thread.current
       return(yield) if @require_thread == t
       @require_mutex.synchronize do
@@ -118,6 +146,24 @@ module Sequel
   def self.connect(*args, &block)
     Database.connect(*args, &block)
   end
+
+  if !defined?(::SEQUEL_NO_CORE_EXTENSIONS) && !ENV.has_key?('SEQUEL_NO_CORE_EXTENSIONS')
+    # Whether the core extensions are enabled.  The core extensions are enabled by
+    # default for backwards compatibility, but can be disabled using the SEQUEL_NO_CORE_EXTENSIONS
+    # constant or environment variable.
+    def self.core_extensions?
+      # We override this method to return true inside the core_extensions.rb file,
+      # but we also set it here because that file is not loaded until most of Sequel
+      # is finished loading, and parts of Sequel check whether the core extensions
+      # are loaded.
+      true
+    end
+  else
+    def self.core_extensions?
+      false
+    end
+  end
+
   
   # Convert the +exception+ to the given class.  The given class should be
   # <tt>Sequel::Error</tt> or a subclass.  Returns an instance of +klass+ with
@@ -188,16 +234,18 @@ module Sequel
   # This is used to ensure that the files loaded are from the same version of
   # Sequel as this file.
   def self.require(files, subdir=nil)
-    Array(files).each{|f| super("#{File.dirname(__FILE__)}/#{"#{subdir}/" if subdir}#{f}")}
+    Array(files).each{|f| super("#{File.dirname(__FILE__).untaint}/#{"#{subdir}/" if subdir}#{f}")}
   end
   
-  # Set whether to set the single threaded mode for all databases by default. By default,
+  # Set whether Sequel is being used in single threaded mode. By default,
   # Sequel uses a thread-safe connection pool, which isn't as fast as the
-  # single threaded connection pool.  If your program will only have one thread,
-  # and speed is a priority, you may want to set this to true:
+  # single threaded connection pool, and also has some additional thread
+  # safety checks.  If your program will only have one thread,
+  # and speed is a priority, you should set this to true:
   #
   #   Sequel.single_threaded = true
   def self.single_threaded=(value)
+    @single_threaded = value
     Database.single_threaded = value
   end
 
@@ -228,17 +276,76 @@ module Sequel
     end
   end
 
-  # Converts the given +string+ into a +Time+ object.
+  # Converts the given +string+ into a <tt>Sequel::SQLTime</tt> object.
   #
-  #   Sequel.string_to_datetime('10:20:30') # Time.parse('10:20:30')
+  #   v = Sequel.string_to_time('10:20:30') # Sequel::SQLTime.parse('10:20:30')
+  #   DB.literal(v) # => '10:20:30'
   def self.string_to_time(string)
     begin
-      Time.parse(string)
+      SQLTime.parse(string)
     rescue => e
       raise convert_exception_class(e, InvalidValue)
     end
   end
 
+  if defined?(RUBY_ENGINE) && RUBY_ENGINE != 'ruby'
+    # Mutex used to protect mutable data structures
+    @data_mutex = Mutex.new
+
+    # Unless in single threaded mode, protects access to any mutable
+    # global data structure in Sequel.
+    # Uses a non-reentrant mutex, so calling code should be careful.
+    def self.synchronize(&block)
+      @single_threaded ? yield : @data_mutex.synchronize(&block)
+    end
+  else
+    # Yield directly to the block.  You don't need to synchronize
+    # access on MRI because the GVL makes certain methods atomic.
+    def self.synchronize(&block)
+      yield
+    end
+  end
+
+  # Uses a transaction on all given databases with the given options. This:
+  #
+  #   Sequel.transaction([DB1, DB2, DB3]){...}
+  #
+  # is equivalent to:
+  #
+  #   DB1.transaction do
+  #     DB2.transaction do
+  #       DB3.transaction do
+  #         ...
+  #       end
+  #     end
+  #   end
+  #
+  # except that if Sequel::Rollback is raised by the block, the transaction is
+  # rolled back on all databases instead of just the last one.
+  #
+  # Note that this method cannot guarantee that all databases will commit or
+  # rollback.  For example, if DB3 commits but attempting to commit on DB2
+  # fails (maybe because foreign key checks are deferred), there is no way
+  # to uncommit the changes on DB3.  For that kind of support, you need to
+  # have two-phase commit/prepared transactions (which Sequel supports on
+  # some databases).
+  def self.transaction(dbs, opts={}, &block)
+    unless opts[:rollback]
+      rescue_rollback = true
+      opts = opts.merge(:rollback=>:reraise)
+    end
+    pr = dbs.reverse.inject(block){|bl, db| proc{db.transaction(opts, &bl)}}
+    if rescue_rollback
+      begin
+        pr.call
+      rescue Sequel::Rollback => e
+        nil
+      end
+    else
+      pr.call
+    end
+  end
+
   # Same as Sequel.require, but wrapped in a mutex in order to be thread safe.
   def self.ts_require(*args)
     check_requiring_thread{require(*args)}
@@ -292,8 +399,8 @@ module Sequel
 
   private_class_method :adapter_method, :def_adapter_method
   
-  require(%w"metaprogramming sql connection_pool exceptions dataset database timezones version")
-  require('core_sql') if !defined?(::SEQUEL_NO_CORE_EXTENSIONS) && !ENV.has_key?('SEQUEL_NO_CORE_EXTENSIONS')
+  require(%w"metaprogramming sql connection_pool exceptions dataset database timezones ast_transformer version")
+  extension(:core_extensions) if Sequel.core_extensions?
 
   # Add the database adapter class methods to Sequel via metaprogramming
   def_adapter_method(*Database::ADAPTERS)

data/lib/sequel/database/connecting.rb

@@ -1,12 +1,12 @@
 module Sequel
   class Database
     # ---------------------
-    # :section: Methods relating to adapters, connecting, disconnecting, and sharding
+    # :section: 4 - Methods relating to adapters, connecting, disconnecting, and sharding
     # This methods involve the Database's connection pool.
     # ---------------------
 
     # Array of supported database adapters
-    ADAPTERS = %w'ado amalgalite db2 dbi do firebird informix jdbc mysql mysql2 odbc openbase oracle postgres sqlite swift tinytds'.collect{|x| x.to_sym}
+    ADAPTERS = %w'ado amalgalite db2 dbi do firebird ibmdb informix jdbc mock mysql mysql2 odbc openbase oracle postgres sqlite swift tinytds'.collect{|x| x.to_sym}
 
     # Whether to use the single threaded connection pool by default
     @@single_threaded = false
@@ -46,6 +46,7 @@ module Sequel
       when String
         if match = /\A(jdbc|do):/o.match(conn_string)
           c = adapter_class(match[1].to_sym)
+          opts = opts.merge(:orig_opts=>opts.dup)
           opts = {:uri=>conn_string}.merge(opts)
         else
           uri = URI.parse(conn_string)
@@ -54,12 +55,15 @@ module Sequel
           c = adapter_class(scheme)
           uri_options = c.send(:uri_to_options, uri)
           uri.query.split('&').collect{|s| s.split('=')}.each{|k,v| uri_options[k.to_sym] = v if k && !k.empty?} unless uri.query.to_s.strip.empty?
-          uri_options.entries.each{|k,v| uri_options[k] = URI.unescape(v) if v.is_a?(String)}
+          uri_options.to_a.each{|k,v| uri_options[k] = URI.unescape(v) if v.is_a?(String)}
+          opts = opts.merge(:orig_opts=>opts.dup)
+          opts[:uri] = conn_string
           opts = uri_options.merge(opts)
           opts[:adapter] = scheme
         end
       when Hash
         opts = conn_string.merge(opts)
+        opts = opts.merge(:orig_opts=>opts.dup)
         c = adapter_class(opts[:adapter_class] || opts[:adapter] || opts['adapter'])
       else
         raise Error, "Sequel::Database.connect takes either a Hash or a String, given: #{conn_string.inspect}"
@@ -77,7 +81,7 @@ module Sequel
       ensure
         if block_given?
           db.disconnect if db
-          ::Sequel::DATABASES.delete(db)
+          Sequel.synchronize{::Sequel::DATABASES.delete(db)}
         end
       end
       block_given? ? result : db
@@ -131,8 +135,10 @@ module Sequel
     #
     #   DB.add_servers(:f=>{:host=>"hash_host_f"})
     def add_servers(servers)
-      @opts[:servers] = @opts[:servers] ? @opts[:servers].merge(servers) : servers
-      @pool.add_servers(servers.keys)
+      if h = @opts[:servers]
+        Sequel.synchronize{h.merge!(servers)}
+        @pool.add_servers(servers.keys)
+      end
     end
     
     # Connects to the database. This method should be overridden by descendants.
@@ -155,8 +161,8 @@ module Sequel
     
     # Disconnects all available connections from the connection pool.  Any
     # connections currently in use will not be disconnected. Options:
-    # * :servers - Should be a symbol specifing the server to disconnect from,
-    #   or an array of symbols to specify multiple servers.
+    # :servers :: Should be a symbol specifing the server to disconnect from,
+    #             or an array of symbols to specify multiple servers.
     #
     # Example:
     #
@@ -187,11 +193,8 @@ module Sequel
     #
     #   DB.remove_servers(:f1, :f2)
     def remove_servers(*servers)
-      if @opts[:servers] && !@opts[:servers].empty?
-        servs = @opts[:servers].dup
-        servers.flatten!
-        servers.each{|s| servs.delete(s)}
-        @opts[:servers] = servs
+      if h = @opts[:servers]
+        servers.flatten.each{|s| Sequel.synchronize{h.delete(s)}}
         @pool.remove_servers(servers)
       end
     end
@@ -209,21 +212,26 @@ module Sequel
       @single_threaded
     end
     
-    # Acquires a database connection, yielding it to the passed block. This is
-    # useful if you want to make sure the same connection is used for all
-    # database queries in the block.  It is also useful if you want to gain
-    # direct access to the underlying connection object if you need to do
-    # something Sequel does not natively support.
-    #
-    # If a server option is given, acquires a connection for that specific
-    # server, instead of the :default server.
-    #
-    #
-    #   DB.synchronize do |conn|
-    #     ...
-    #   end
-    def synchronize(server=nil, &block)
-      @pool.hold(server || :default, &block)
+    if !defined?(RUBY_ENGINE) || RUBY_ENGINE == 'ruby'
+      # Acquires a database connection, yielding it to the passed block. This is
+      # useful if you want to make sure the same connection is used for all
+      # database queries in the block.  It is also useful if you want to gain
+      # direct access to the underlying connection object if you need to do
+      # something Sequel does not natively support.
+      #
+      # If a server option is given, acquires a connection for that specific
+      # server, instead of the :default server.
+      #
+      #   DB.synchronize do |conn|
+      #     ...
+      #   end
+      def synchronize(server=nil)
+        @pool.hold(server || :default){|conn| yield conn}
+      end
+    else
+      def synchronize(server=nil, &block)
+        @pool.hold(server || :default, &block)
+      end
     end
     
     # Attempts to acquire a database connection.  Returns true if successful.
@@ -255,6 +263,8 @@ module Sequel
         else
           raise Error, 'Server opts should be a hash or proc'
         end
+      elsif server.is_a?(Hash)
+        @opts.merge(server)
       else
         @opts.dup
       end

data/lib/sequel/database/dataset.rb

@@ -1,7 +1,7 @@
 module Sequel
   class Database
     # ---------------------
-    # :section: Methods that create datasets
+    # :section: 3 - Methods that create datasets
     # These methods all return instances of this database's dataset class.
     # ---------------------
 
@@ -24,10 +24,10 @@ module Sequel
     #
     #   DB.dataset # SELECT *
     #   DB.dataset.from(:items) # SELECT * FROM items
-    def dataset
-      ds = Sequel::Dataset.new(self)
+    def dataset(opts=nil)
+      @dataset_class.new(self, opts)
     end
-    
+
     # Fetches records for an arbitrary SQL statement. If a block is given,
     # it is used to iterate over the records:
     #

data/lib/sequel/database/dataset_defaults.rb

@@ -1,10 +1,13 @@
 module Sequel
   class Database
     # ---------------------
-    # :section: Methods that set defaults for created datasets
+    # :section: 5 - Methods that set defaults for created datasets
     # This methods change the default behavior of this database's datasets.
     # ---------------------
 
+    # The default class to use for datasets
+    DatasetClass = Sequel::Dataset
+
     # The identifier input method to use by default
     @@identifier_input_method = nil
 
@@ -42,9 +45,62 @@ module Sequel
       @@quote_identifiers = value
     end
 
-    # The default schema to use, generally should be nil.
+    # The class to use for creating datasets.  Should respond to
+    # new with the Database argument as the first argument, and
+    # an optional options hash.
+    attr_reader :dataset_class
+
+    # The default schema to use, generally should be nil.  This sets
+    # the default schema used for some schema modification and
+    # introspection queries, but does not effect most dataset code.
     attr_accessor :default_schema
 
+    # If the database has any dataset modules associated with it,
+    # use a subclass of the given class that includes the modules
+    # as the dataset class.
+    def dataset_class=(c)
+      unless @dataset_modules.empty?
+        c = Class.new(c)
+        @dataset_modules.each{|m| c.send(:include, m)}
+      end
+      @dataset_class = c
+    end
+
+    # Equivalent to extending all datasets produced by the database with a
+    # module.  What it actually does is use a subclass of the current dataset_class
+    # as the new dataset_class, and include the module in the subclass.
+    # Instead of a module, you can provide a block that is used to create an
+    # anonymous module.
+    #
+    # This allows you to override any of the dataset methods even if they are
+    # defined directly on the dataset class that this Database object uses.
+    #
+    # Examples:
+    #
+    #   # Introspec columns for all of DB's datasets
+    #   DB.extend_datasets(Sequel::ColumnsIntrospection)
+    #   
+    #   # Trace all SELECT queries by printing the SQL and the full backtrace
+    #   DB.extend_datasets do
+    #     def fetch_rows(sql)
+    #       puts sql
+    #       puts caller
+    #       super
+    #     end
+    #   end
+    def extend_datasets(mod=nil, &block)
+      raise(Error, "must provide either mod or block, not both") if mod && block
+      reset_schema_utility_dataset
+      mod = Module.new(&block) if block
+      if @dataset_modules.empty?
+       @dataset_modules = [mod]
+       @dataset_class = Class.new(@dataset_class)
+      else
+       @dataset_modules << mod
+      end
+      @dataset_class.send(:include, mod)
+    end
+
     # The method to call on identifiers going into the database
     def identifier_input_method
       case @identifier_input_method
@@ -109,6 +165,11 @@ module Sequel
     
     private
     
+    # The default dataset class to use for the database
+    def dataset_class_default
+      self.class.const_get(:DatasetClass)
+    end
+    
     # The default value for default_schema.
     def default_schema_default
       nil

data/lib/sequel/database/logging.rb

@@ -1,7 +1,7 @@
 module Sequel
   class Database
     # ---------------------
-    # :section: Methods relating to logging
+    # :section: 6 - Methods relating to logging
     # This methods affect relating to the logging of executed SQL.
     # ---------------------
 
@@ -17,6 +17,11 @@ module Sequel
     # is :info, it can be set to :debug to log at DEBUG level.
     attr_accessor :sql_log_level
 
+    # Log a message at error level, with information about the exception.
+    def log_exception(exception, message)
+      log_each(:error, "#{exception.class}: #{exception.message.strip}: #{message}")
+    end
+
     # Log a message at level info to all loggers.
     def log_info(message, args=nil)
       log_each(:info, args ? "#{message}; #{args.inspect}" : message)
@@ -31,7 +36,7 @@ module Sequel
       begin
         yield
       rescue => e
-        log_each(:error, "#{e.class}: #{e.message.strip}: #{sql}")
+        log_exception(e, sql)
         raise
       ensure
         log_duration(Time.now - start, sql) unless e