sequel 3.21.0 → 3.39.0

data/lib/sequel/connection_pool/sharded_threaded.rb

@@ -45,6 +45,23 @@ class Sequel::ShardedThreadedConnectionPool < Sequel::ThreadedConnectionPool
     @allocated[server]
   end
   
+  # Yield all of the available connections, and the ones currently allocated to
+  # this thread.  This will not yield connections currently allocated to other
+  # threads, as it is not safe to operate on them.  This holds the mutex while
+  # it is yielding all of the connections, which means that until
+  # the method's block returns, the pool is locked.
+  def all_connections
+    t = Thread.current
+    sync do
+      @allocated.values.each do |threads|
+        threads.each do |thread, conn|
+          yield conn if t == thread
+        end
+      end
+      @available_connections.values.each{|v| v.each{|c| yield c}}
+    end
+  end
+  
   # An array of connections opened but not currently used, for the given
   # server. Nonexistent servers will return nil. Treat this as read only, do
   # not modify the resulting object.
@@ -94,7 +111,7 @@ class Sequel::ShardedThreadedConnectionPool < Sequel::ThreadedConnectionPool
   # connection can be acquired, a Sequel::PoolTimeout is 
   # raised.
   def hold(server=:default)
-    sync{server = @servers[server]}
+    server = pick_server(server)
     t = Thread.current
     if conn = owned_connection(t, server)
       return yield(conn)
@@ -190,6 +207,11 @@ class Sequel::ShardedThreadedConnectionPool < Sequel::ThreadedConnectionPool
   def owned_connection(thread, server)
     sync{@allocated[server][thread]}
   end
+
+  # If the server given is in the hash, return it, otherwise, return the default server.
+  def pick_server(server)
+    sync{@servers[server]}
+  end
   
   # Releases the connection assigned to the supplied thread and server. If the
   # server or connection given is scheduled for disconnection, remove the
@@ -199,7 +221,16 @@ class Sequel::ShardedThreadedConnectionPool < Sequel::ThreadedConnectionPool
     if @connections_to_remove.include?(conn)
       remove(thread, conn, server)
     else
-      available_connections(server) << allocated(server).delete(thread)
+      conn = allocated(server).delete(thread)
+
+      case @connection_handling
+      when :queue
+        available_connections(server).unshift(conn)
+      when :disconnect
+        @disconnection_proc.call(conn) if @disconnection_proc
+      else
+        available_connections(server) << conn
+      end
     end
   end
 

data/lib/sequel/connection_pool/single.rb

@@ -8,6 +8,11 @@ class Sequel::SingleConnectionPool < Sequel::ConnectionPool
     @conn ? 1 : 0
   end
   
+  # Yield the connection if one has been made.
+  def all_connections
+    yield @conn if @conn
+  end
+  
   # Disconnect the connection from the database.
   def disconnect(opts=nil, &block)
     return unless @conn

data/lib/sequel/connection_pool/threaded.rb

@@ -13,6 +13,9 @@ class Sequel::ThreadedConnectionPool < Sequel::ConnectionPool
   attr_reader :allocated
 
   # The following additional options are respected:
+  # * :connection_handling - Set how to handle available connections.  By default,
+  #   uses a a stack for performance.  Can be set to :queue to use a queue, which reduces
+  #   the chances of connections becoming stale.
   # * :max_connections - The maximum number of connections the connection pool
   #   will open (default 4)
   # * :pool_sleep_time - The amount of time to sleep before attempting to acquire
@@ -24,22 +27,40 @@ class Sequel::ThreadedConnectionPool < Sequel::ConnectionPool
     @max_size = Integer(opts[:max_connections] || 4)
     raise(Sequel::Error, ':max_connections must be positive') if @max_size < 1
     @mutex = Mutex.new  
+    @connection_handling = opts[:connection_handling]
     @available_connections = []
     @allocated = {}
     @timeout = Integer(opts[:pool_timeout] || 5)
     @sleep_time = Float(opts[:pool_sleep_time] || 0.001)
   end
   
-  # The total number of connections opened for the given server, should
-  # be equal to available_connections.length + allocated.length.
+  # The total number of connections opened, either available or allocated.
+  # This may not be completely accurate as it isn't protected by the mutex.
   def size
     @allocated.length + @available_connections.length
   end
   
-  # Removes all connections currently available on all servers, optionally
+  # Yield all of the available connections, and the one currently allocated to
+  # this thread.  This will not yield connections currently allocated to other
+  # threads, as it is not safe to operate on them.  This holds the mutex while
+  # it is yielding all of the available connections, which means that until
+  # the method's block returns, the pool is locked.
+  def all_connections
+    hold do |c|
+      sync do
+        yield c
+        @available_connections.each{|c| yield c}
+      end
+    end
+  end
+  
+  # Removes all connections currently available, optionally
   # yielding each connection to the given block. This method has the effect of 
   # disconnecting from the database, assuming that no connections are currently
-  # being used.
+  # being used.  If you want to be able to disconnect connections that are
+  # currently in use, use the ShardedThreadedConnectionPool, which can do that.
+  # This connection pool does not, for performance reasons. To use the sharded pool,
+  # pass the <tt>:servers=>{}</tt> option when connecting to the database.
   # 
   # Once a connection is requested using #hold, the connection pool
   # creates new connections to the database.
@@ -51,7 +72,7 @@ class Sequel::ThreadedConnectionPool < Sequel::ConnectionPool
     end
   end
   
-  # Chooses the first available connection to the given server, or if none are
+  # Chooses the first available connection, or if none are
   # available, creates a new connection.  Passes the connection to the supplied
   # block:
   # 
@@ -83,9 +104,10 @@ class Sequel::ThreadedConnectionPool < Sequel::ConnectionPool
       end
       yield conn
     rescue Sequel::DatabaseDisconnectError
-      @disconnection_proc.call(conn) if @disconnection_proc && conn
-      @allocated.delete(t)
+      oconn = conn
       conn = nil
+      @disconnection_proc.call(oconn) if @disconnection_proc && oconn
+      @allocated.delete(t)
       raise
     ensure
       sync{release(t)} if conn
@@ -94,7 +116,7 @@ class Sequel::ThreadedConnectionPool < Sequel::ConnectionPool
   
   private
 
-  # Assigns a connection to the supplied thread for the given server, if one
+  # Assigns a connection to the supplied thread, if one
   # is available. The calling code should NOT already have the mutex when
   # calling this.
   def acquire(thread)
@@ -105,7 +127,7 @@ class Sequel::ThreadedConnectionPool < Sequel::ConnectionPool
     end
   end
   
-  # Returns an available connection to the given server. If no connection is
+  # Returns an available connection. If no connection is
   # available, tries to create a new connection. The calling code should already
   # have the mutex before calling this.
   def available
@@ -126,18 +148,25 @@ class Sequel::ThreadedConnectionPool < Sequel::ConnectionPool
     super if (n || size) < @max_size
   end
   
-  # Returns the connection owned by the supplied thread for the given server,
+  # Returns the connection owned by the supplied thread,
   # if any. The calling code should NOT already have the mutex before calling this.
   def owned_connection(thread)
     sync{@allocated[thread]}
   end
   
-  # Releases the connection assigned to the supplied thread and server. If the
-  # server or connection given is scheduled for disconnection, remove the
-  # connection instead of releasing it back to the pool.
+  # Releases the connection assigned to the supplied thread back to the pool.
   # The calling code should already have the mutex before calling this.
   def release(thread)
-    @available_connections << @allocated.delete(thread)
+    conn = @allocated.delete(thread)
+
+    case @connection_handling
+    when :queue
+      @available_connections.unshift(conn)
+    when :disconnect
+      @disconnection_proc.call(conn) if @disconnection_proc
+    else
+      @available_connections << conn
+    end
   end
 
   # Yield to the block while inside the mutex. The calling code should NOT

data/lib/sequel/connection_pool.rb

@@ -53,10 +53,18 @@ class Sequel::ConnectionPool
     
     # Return a connection pool class based on the given options.
     def connection_pool_class(opts)
-      opts[:pool_class] || CONNECTION_POOL_MAP[[!!opts[:single_threaded], !!opts[:servers]]]
+      CONNECTION_POOL_MAP[opts[:pool_class]] || opts[:pool_class] || CONNECTION_POOL_MAP[[!!opts[:single_threaded], !!opts[:servers]]]
     end
   end
   extend ClassMethods
+
+  # The after_connect proc used for this pool.  This is called with each new
+  # connection made, and is usually used to set custom per-connection settings.
+  attr_accessor :after_connect
+
+  # The disconnect_proc used for the pool.  This is called with each connection
+  # that is disconnected, usually to clean up related resources.
+  attr_accessor :disconnection_proc
   
   # Instantiates a connection pool with the given options.  The block is called
   # with a single symbol (specifying the server/shard to use) every time a new
@@ -80,7 +88,7 @@ class Sequel::ConnectionPool
   
   # An array of symbols for all shards/servers, which is a single <tt>:default</tt> by default.
   def servers
-    [:default]
+    [DEFAULT_SERVER]
   end
   
   private

data/lib/sequel/core.rb

@@ -17,20 +17,26 @@
 #
 #   Sequel.sqlite('blog.db'){|db| puts db[:users].count} 
 #
-# You can set the +SEQUEL_NO_CORE_EXTENSIONS+ constant or environment variable to have
-# Sequel not extend the core classes.
+# Sequel currently adds methods to the Array, Hash, String and Symbol classes by
+# default.  You can either require 'sequel/no_core_ext' or set the
+# +SEQUEL_NO_CORE_EXTENSIONS+ constant or environment variable before requiring
+# sequel to have # Sequel not add methods to those classes.
 #
 # For a more expanded introduction, see the {README}[link:files/README_rdoc.html].
 # For a quicker introduction, see the {cheat sheet}[link:files/doc/cheat_sheet_rdoc.html].
 module Sequel
   @convert_two_digit_years = true
   @datetime_class = Time
+  @empty_array_handle_nulls = true
   @virtual_row_instance_eval = true
   @require_thread = nil
   
   # Mutex used to protect file loading/requireing
   @require_mutex = Mutex.new
   
+  # Whether Sequel is being run in single threaded mode
+  @single_threaded = false
+  
   class << self
     # Sequel converts two digit years in <tt>Date</tt>s and <tt>DateTime</tt>s by default,
     # so 01/02/03 is interpreted at January 2nd, 2003, and 12/13/99 is interpreted
@@ -54,6 +60,27 @@ module Sequel
     # days on +DateTime+).
     attr_accessor :datetime_class
 
+    # Sets whether or not to attempt to handle NULL values correctly when given
+    # an empty array.  By default:
+    #
+    #   DB[:a].filter(:b=>[])
+    #   # SELECT * FROM a WHERE (b != b)
+    #   DB[:a].exclude(:b=>[])
+    #   # SELECT * FROM a WHERE (b = b)
+    #
+    # However, some databases (e.g. MySQL) will perform very poorly
+    # with this type of query.  You can set this to false to get the
+    # following behavior:
+    #
+    #   DB[:a].filter(:b=>[])
+    #   # SELECT * FROM a WHERE 1 = 0
+    #   DB[:a].exclude(:b=>[])
+    #   # SELECT * FROM a WHERE 1 = 1
+    # 
+    # This may not handle NULLs correctly, but can be much faster on
+    # some databases.
+    attr_accessor :empty_array_handle_nulls
+
     # For backwards compatibility, has no effect.
     attr_accessor :virtual_row_instance_eval
     
@@ -64,6 +91,7 @@ module Sequel
 
     # Make thread safe requiring reentrant to prevent deadlocks.
     def check_requiring_thread
+      return yield if @single_threaded
       t = Thread.current
       return(yield) if @require_thread == t
       @require_mutex.synchronize do
@@ -118,6 +146,12 @@ module Sequel
   def self.connect(*args, &block)
     Database.connect(*args, &block)
   end
+
+  # Assume the core extensions are not loaded by default, if the core_extensions
+  # extension is loaded, this will be overridden.
+  def self.core_extensions?
+    false
+  end
   
   # Convert the +exception+ to the given class.  The given class should be
   # <tt>Sequel::Error</tt> or a subclass.  Returns an instance of +klass+ with
@@ -182,25 +216,62 @@ module Sequel
   def self.quote_identifiers=(value)
     Database.quote_identifiers = value
   end
+
+  # Convert each item in the array to the correct type, handling multi-dimensional
+  # arrays.  For each element in the array or subarrays, call the converter,
+  # unless the value is nil.
+  def self.recursive_map(array, converter)
+    array.map do |i|
+      if i.is_a?(Array)
+        recursive_map(i, converter)
+      elsif i
+        converter.call(i)
+      end
+    end
+  end
   
   # Require all given +files+ which should be in the same or a subdirectory of
   # this file.  If a +subdir+ is given, assume all +files+ are in that subdir.
   # This is used to ensure that the files loaded are from the same version of
   # Sequel as this file.
   def self.require(files, subdir=nil)
-    Array(files).each{|f| super("#{File.dirname(__FILE__)}/#{"#{subdir}/" if subdir}#{f}")}
+    Array(files).each{|f| super("#{File.dirname(__FILE__).untaint}/#{"#{subdir}/" if subdir}#{f}")}
   end
   
-  # Set whether to set the single threaded mode for all databases by default. By default,
+  # Set whether Sequel is being used in single threaded mode. By default,
   # Sequel uses a thread-safe connection pool, which isn't as fast as the
-  # single threaded connection pool.  If your program will only have one thread,
-  # and speed is a priority, you may want to set this to true:
+  # single threaded connection pool, and also has some additional thread
+  # safety checks.  If your program will only have one thread,
+  # and speed is a priority, you should set this to true:
   #
   #   Sequel.single_threaded = true
   def self.single_threaded=(value)
+    @single_threaded = value
     Database.single_threaded = value
   end
 
+  COLUMN_REF_RE1 = /\A((?:(?!__).)+)__((?:(?!___).)+)___(.+)\z/.freeze
+  COLUMN_REF_RE2 = /\A((?:(?!___).)+)___(.+)\z/.freeze
+  COLUMN_REF_RE3 = /\A((?:(?!__).)+)__(.+)\z/.freeze
+
+  # Splits the symbol into three parts.  Each part will
+  # either be a string or nil.
+  #
+  # For columns, these parts are the table, column, and alias.
+  # For tables, these parts are the schema, table, and alias.
+  def self.split_symbol(sym)
+    case s = sym.to_s
+    when COLUMN_REF_RE1
+      [$1, $2, $3]
+    when COLUMN_REF_RE2
+      [nil, $1, $2]
+    when COLUMN_REF_RE3
+      [$1, $2, nil]
+    else
+      [nil, s, nil]
+    end
+  end
+
   # Converts the given +string+ into a +Date+ object.
   #
   #   Sequel.string_to_date('2010-09-10') # Date.civil(2010, 09, 10)
@@ -228,17 +299,76 @@ module Sequel
     end
   end
 
-  # Converts the given +string+ into a +Time+ object.
+  # Converts the given +string+ into a <tt>Sequel::SQLTime</tt> object.
   #
-  #   Sequel.string_to_datetime('10:20:30') # Time.parse('10:20:30')
+  #   v = Sequel.string_to_time('10:20:30') # Sequel::SQLTime.parse('10:20:30')
+  #   DB.literal(v) # => '10:20:30'
   def self.string_to_time(string)
     begin
-      Time.parse(string)
+      SQLTime.parse(string)
     rescue => e
       raise convert_exception_class(e, InvalidValue)
     end
   end
 
+  if defined?(RUBY_ENGINE) && RUBY_ENGINE != 'ruby'
+    # Mutex used to protect mutable data structures
+    @data_mutex = Mutex.new
+
+    # Unless in single threaded mode, protects access to any mutable
+    # global data structure in Sequel.
+    # Uses a non-reentrant mutex, so calling code should be careful.
+    def self.synchronize(&block)
+      @single_threaded ? yield : @data_mutex.synchronize(&block)
+    end
+  else
+    # Yield directly to the block.  You don't need to synchronize
+    # access on MRI because the GVL makes certain methods atomic.
+    def self.synchronize(&block)
+      yield
+    end
+  end
+
+  # Uses a transaction on all given databases with the given options. This:
+  #
+  #   Sequel.transaction([DB1, DB2, DB3]){...}
+  #
+  # is equivalent to:
+  #
+  #   DB1.transaction do
+  #     DB2.transaction do
+  #       DB3.transaction do
+  #         ...
+  #       end
+  #     end
+  #   end
+  #
+  # except that if Sequel::Rollback is raised by the block, the transaction is
+  # rolled back on all databases instead of just the last one.
+  #
+  # Note that this method cannot guarantee that all databases will commit or
+  # rollback.  For example, if DB3 commits but attempting to commit on DB2
+  # fails (maybe because foreign key checks are deferred), there is no way
+  # to uncommit the changes on DB3.  For that kind of support, you need to
+  # have two-phase commit/prepared transactions (which Sequel supports on
+  # some databases).
+  def self.transaction(dbs, opts={}, &block)
+    unless opts[:rollback]
+      rescue_rollback = true
+      opts = opts.merge(:rollback=>:reraise)
+    end
+    pr = dbs.reverse.inject(block){|bl, db| proc{db.transaction(opts, &bl)}}
+    if rescue_rollback
+      begin
+        pr.call
+      rescue Sequel::Rollback => e
+        nil
+      end
+    else
+      pr.call
+    end
+  end
+
   # Same as Sequel.require, but wrapped in a mutex in order to be thread safe.
   def self.ts_require(*args)
     check_requiring_thread{require(*args)}
@@ -292,8 +422,10 @@ module Sequel
 
   private_class_method :adapter_method, :def_adapter_method
   
-  require(%w"metaprogramming sql connection_pool exceptions dataset database timezones version")
-  require('core_sql') if !defined?(::SEQUEL_NO_CORE_EXTENSIONS) && !ENV.has_key?('SEQUEL_NO_CORE_EXTENSIONS')
+  require(%w"metaprogramming sql connection_pool exceptions dataset database timezones ast_transformer version")
+  if !defined?(::SEQUEL_NO_CORE_EXTENSIONS) && !ENV.has_key?('SEQUEL_NO_CORE_EXTENSIONS')
+    extension(:core_extensions)
+  end
 
   # Add the database adapter class methods to Sequel via metaprogramming
   def_adapter_method(*Database::ADAPTERS)

data/lib/sequel/database/connecting.rb

@@ -1,12 +1,12 @@
 module Sequel
   class Database
     # ---------------------
-    # :section: Methods relating to adapters, connecting, disconnecting, and sharding
+    # :section: 4 - Methods relating to adapters, connecting, disconnecting, and sharding
     # This methods involve the Database's connection pool.
     # ---------------------
 
     # Array of supported database adapters
-    ADAPTERS = %w'ado amalgalite db2 dbi do firebird informix jdbc mysql mysql2 odbc openbase oracle postgres sqlite swift tinytds'.collect{|x| x.to_sym}
+    ADAPTERS = %w'ado amalgalite db2 dbi do firebird ibmdb informix jdbc mock mysql mysql2 odbc openbase oracle postgres sqlite swift tinytds'.collect{|x| x.to_sym}
 
     # Whether to use the single threaded connection pool by default
     @@single_threaded = false
@@ -46,6 +46,7 @@ module Sequel
       when String
         if match = /\A(jdbc|do):/o.match(conn_string)
           c = adapter_class(match[1].to_sym)
+          opts = opts.merge(:orig_opts=>opts.dup)
           opts = {:uri=>conn_string}.merge(opts)
         else
           uri = URI.parse(conn_string)
@@ -54,12 +55,15 @@ module Sequel
           c = adapter_class(scheme)
           uri_options = c.send(:uri_to_options, uri)
           uri.query.split('&').collect{|s| s.split('=')}.each{|k,v| uri_options[k.to_sym] = v if k && !k.empty?} unless uri.query.to_s.strip.empty?
-          uri_options.entries.each{|k,v| uri_options[k] = URI.unescape(v) if v.is_a?(String)}
+          uri_options.to_a.each{|k,v| uri_options[k] = URI.unescape(v) if v.is_a?(String)}
+          opts = opts.merge(:orig_opts=>opts.dup)
+          opts[:uri] = conn_string
           opts = uri_options.merge(opts)
           opts[:adapter] = scheme
         end
       when Hash
         opts = conn_string.merge(opts)
+        opts = opts.merge(:orig_opts=>opts.dup)
         c = adapter_class(opts[:adapter_class] || opts[:adapter] || opts['adapter'])
       else
         raise Error, "Sequel::Database.connect takes either a Hash or a String, given: #{conn_string.inspect}"
@@ -77,7 +81,7 @@ module Sequel
       ensure
         if block_given?
           db.disconnect if db
-          ::Sequel::DATABASES.delete(db)
+          Sequel.synchronize{::Sequel::DATABASES.delete(db)}
         end
       end
       block_given? ? result : db
@@ -131,8 +135,10 @@ module Sequel
     #
     #   DB.add_servers(:f=>{:host=>"hash_host_f"})
     def add_servers(servers)
-      @opts[:servers] = @opts[:servers] ? @opts[:servers].merge(servers) : servers
-      @pool.add_servers(servers.keys)
+      if h = @opts[:servers]
+        Sequel.synchronize{h.merge!(servers)}
+        @pool.add_servers(servers.keys)
+      end
     end
     
     # Connects to the database. This method should be overridden by descendants.
@@ -155,8 +161,8 @@ module Sequel
     
     # Disconnects all available connections from the connection pool.  Any
     # connections currently in use will not be disconnected. Options:
-    # * :servers - Should be a symbol specifing the server to disconnect from,
-    #   or an array of symbols to specify multiple servers.
+    # :servers :: Should be a symbol specifing the server to disconnect from,
+    #             or an array of symbols to specify multiple servers.
     #
     # Example:
     #
@@ -187,11 +193,8 @@ module Sequel
     #
     #   DB.remove_servers(:f1, :f2)
     def remove_servers(*servers)
-      if @opts[:servers] && !@opts[:servers].empty?
-        servs = @opts[:servers].dup
-        servers.flatten!
-        servers.each{|s| servs.delete(s)}
-        @opts[:servers] = servs
+      if h = @opts[:servers]
+        servers.flatten.each{|s| Sequel.synchronize{h.delete(s)}}
         @pool.remove_servers(servers)
       end
     end
@@ -209,21 +212,26 @@ module Sequel
       @single_threaded
     end
     
-    # Acquires a database connection, yielding it to the passed block. This is
-    # useful if you want to make sure the same connection is used for all
-    # database queries in the block.  It is also useful if you want to gain
-    # direct access to the underlying connection object if you need to do
-    # something Sequel does not natively support.
-    #
-    # If a server option is given, acquires a connection for that specific
-    # server, instead of the :default server.
-    #
-    #
-    #   DB.synchronize do |conn|
-    #     ...
-    #   end
-    def synchronize(server=nil, &block)
-      @pool.hold(server || :default, &block)
+    if !defined?(RUBY_ENGINE) || RUBY_ENGINE == 'ruby'
+      # Acquires a database connection, yielding it to the passed block. This is
+      # useful if you want to make sure the same connection is used for all
+      # database queries in the block.  It is also useful if you want to gain
+      # direct access to the underlying connection object if you need to do
+      # something Sequel does not natively support.
+      #
+      # If a server option is given, acquires a connection for that specific
+      # server, instead of the :default server.
+      #
+      #   DB.synchronize do |conn|
+      #     ...
+      #   end
+      def synchronize(server=nil)
+        @pool.hold(server || :default){|conn| yield conn}
+      end
+    else
+      def synchronize(server=nil, &block)
+        @pool.hold(server || :default, &block)
+      end
     end
     
     # Attempts to acquire a database connection.  Returns true if successful.
@@ -255,6 +263,8 @@ module Sequel
         else
           raise Error, 'Server opts should be a hash or proc'
         end
+      elsif server.is_a?(Hash)
+        @opts.merge(server)
       else
         @opts.dup
       end

data/lib/sequel/database/dataset.rb

@@ -1,7 +1,7 @@
 module Sequel
   class Database
     # ---------------------
-    # :section: Methods that create datasets
+    # :section: 3 - Methods that create datasets
     # These methods all return instances of this database's dataset class.
     # ---------------------
 
@@ -24,10 +24,10 @@ module Sequel
     #
     #   DB.dataset # SELECT *
     #   DB.dataset.from(:items) # SELECT * FROM items
-    def dataset
-      ds = Sequel::Dataset.new(self)
+    def dataset(opts=nil)
+      @dataset_class.new(self, opts)
     end
-    
+
     # Fetches records for an arbitrary SQL statement. If a block is given,
     # it is used to iterate over the records:
     #