sequel 4.49.0 → 5.0.0

data/lib/sequel/connection_pool/single.rb

@@ -56,6 +56,4 @@ class Sequel::SingleConnectionPool < Sequel::ConnectionPool
   def preconnect(concurrent = nil)
     hold{}
   end
-
-  CONNECTION_POOL__MAP[[true, false]] = self
 end

data/lib/sequel/connection_pool/threaded.rb

@@ -3,12 +3,8 @@
 # A connection pool allowing multi-threaded access to a pool of connections.
 # This is the default connection pool used by Sequel.
 class Sequel::ThreadedConnectionPool < Sequel::ConnectionPool
-  # Whether or not a ConditionVariable should be used to wait for connections.
-  # True except on ruby 1.8, where ConditionVariable#wait does not support a
-  # timeout.
-  unless defined?(USE_WAITER)
-    USE_WAITER = RUBY_VERSION >= '1.9'
-  end
+  USE_WAITER = true
+  Sequel::Deprecation.deprecate_constant(self, :USE_WAITER)
 
   # The maximum number of connections this pool will create (per shard/server
   # if sharding).
@@ -22,15 +18,10 @@ class Sequel::ThreadedConnectionPool < Sequel::ConnectionPool
   attr_reader :allocated
 
   # The following additional options are respected:
-  # * :connection_handling - Set how to handle available connections.  By default,
-  #   uses a a queue for fairness.  Can be set to :stack to use a stack, which may
-  #   offer better performance.
-  # * :max_connections - The maximum number of connections the connection pool
-  #   will open (default 4)
-  # * :pool_sleep_time - The amount of time to sleep before attempting to acquire
-  #   a connection again, only used on ruby 1.8. (default 0.001)
-  # * :pool_timeout - The amount of seconds to wait to acquire a connection
-  #   before raising a PoolTimeoutError (default 5)
+  # :max_connections :: The maximum number of connections the connection pool
+  #                     will open (default 4)
+  # :pool_timeout :: The amount of seconds to wait to acquire a connection
+  #                  before raising a PoolTimeoutError (default 5)
   def initialize(db, opts = OPTS)
     super
     @max_size = Integer(opts[:max_connections] || 4)
@@ -40,14 +31,7 @@ class Sequel::ThreadedConnectionPool < Sequel::ConnectionPool
     @available_connections = []
     @allocated = {}
     @timeout = Float(opts[:pool_timeout] || 5)
-
-    if USE_WAITER
-      @waiter = ConditionVariable.new
-    else
-      # :nocov:
-      @sleep_time = Float(opts[:pool_sleep_time] || 0.001)
-      # :nocov:
-    end
+    @waiter = ConditionVariable.new
   end
   
   # Yield all of the available connections, and the one currently allocated to
@@ -70,7 +54,7 @@ class Sequel::ThreadedConnectionPool < Sequel::ConnectionPool
   # being used.  If you want to be able to disconnect connections that are
   # currently in use, use the ShardedThreadedConnectionPool, which can do that.
   # This connection pool does not, for performance reasons. To use the sharded pool,
-  # pass the <tt>:servers=>{}</tt> option when connecting to the database.
+  # pass the <tt>servers: {}</tt> option when connecting to the database.
   # 
   # Once a connection is requested using #hold, the connection pool
   # creates new connections to the database.
@@ -79,6 +63,7 @@ class Sequel::ThreadedConnectionPool < Sequel::ConnectionPool
     sync do
       conns = @available_connections.dup
       @available_connections.clear
+      @waiter.signal
     end
     conns.each{|conn| disconnect_connection(conn)}
   end
@@ -95,8 +80,7 @@ class Sequel::ThreadedConnectionPool < Sequel::ConnectionPool
   # If no connection is immediately available and the pool is already using the maximum
   # number of connections, Pool#hold will block until a connection
   # is available or the timeout expires.  If the timeout expires before a
-  # connection can be acquired, a Sequel::PoolTimeout is 
-  # raised.
+  # connection can be acquired, a Sequel::PoolTimeout is raised.
   def hold(server=nil)
     t = Thread.current
     if conn = owned_connection(t)
@@ -110,7 +94,10 @@ class Sequel::ThreadedConnectionPool < Sequel::ConnectionPool
         oconn = conn
         conn = nil
         disconnect_connection(oconn) if oconn
-        @allocated.delete(t)
+        sync do 
+          @allocated.delete(t)
+          @waiter.signal
+        end
       end
       raise
     ensure
@@ -123,111 +110,104 @@ class Sequel::ThreadedConnectionPool < Sequel::ConnectionPool
   end
   
   # The total number of connections opened, either available or allocated.
-  # This may not be completely accurate as it isn't protected by the mutex.
+  # The calling code should not have the mutex before calling this.
   def size
-    @allocated.length + @available_connections.length
+    @mutex.synchronize{_size}
   end
   
   private
 
+  # The total number of connections opened, either available or allocated.
+  # The calling code should already have the mutex before calling this.
+  def _size
+    @allocated.length + @available_connections.length
+  end
+
   # Assigns a connection to the supplied thread, if one
-  # is available. The calling code should already have the mutex when
+  # is available. The calling code should NOT already have the mutex when
   # calling this.
-  def _acquire(thread)
-    if conn = available
-      @allocated[thread] = conn
+  #
+  # This should return a connection is one is available within the timeout,
+  # or nil if a connection could not be acquired within the timeout.
+  def acquire(thread)
+    if conn = assign_connection(thread)
+      return conn
     end
-  end
-  
-  if USE_WAITER
-    # Assigns a connection to the supplied thread, if one
-    # is available. The calling code should NOT already have the mutex when
-    # calling this.
-    #
-    # This should return a connection is one is available within the timeout,
-    # or nil if a connection could not be acquired within the timeout.
-    def acquire(thread)
+
+    time = Time.now
+
+    sync do
+      @waiter.wait(@mutex, @timeout)
+      if conn = next_available
+        return(@allocated[thread] = conn)
+      end
+    end
+
+    until conn = assign_connection(thread)
+      deadline ||= time + @timeout
+      current_time = Time.now
+      raise_pool_timeout(current_time - time) if current_time > deadline
+
+      # :nocov:
+      # It's difficult to get to this point, it can only happen if there is a race condition
+      # where a connection cannot be acquired even after the thread is signalled by the condition
       sync do
-        if conn = _acquire(thread)
-          return conn
+        @waiter.wait(@mutex, deadline - current_time)
+        if conn = next_available
+          return(@allocated[thread] = conn)
         end
+      end
+      # :nocov:
+    end
 
-        time = Time.now
-        @waiter.wait(@mutex, @timeout)
+    conn
+  end
 
-        # Not sure why this is helpful, but calling Thread.pass after conditional
-        # variable access dramatically increases reliability when under heavy
-        # resource contention (almost eliminating timeouts), at a small cost to
-        # runtime performance.
-        Thread.pass
+  # Assign a connection to the thread, or return nil if one cannot be assigned.
+  # The caller should NOT have the mutex before calling this.
+  def assign_connection(thread)
+    allocated = @allocated
 
-        until conn = _acquire(thread)
-          deadline ||= time + @timeout
-          current_time = Time.now
-          raise_pool_timeout(current_time - time) if current_time > deadline
-          # :nocov:
-          # It's difficult to get to this point, it can only happen if there is a race condition
-          # where a connection cannot be acquired even after the thread is signalled by the condition
-          @waiter.wait(@mutex, deadline - current_time)
-          Thread.pass
-          # :nocov:
-        end
+    do_make_new = false
+    sync do
+      if conn = next_available
+        return(allocated[thread] = conn)
+      end
+
+      if (n = _size) >= (max = @max_size)
+        allocated.keys.each{|t| release(t) unless t.alive?}
+        n = nil
+      end
 
-        conn
+      if (n || _size) < max
+        do_make_new = allocated[thread] = true
       end
     end
-  else
-    # :nocov:
-    def acquire(thread)
-      unless conn = sync{_acquire(thread)}
-        time = Time.now
-        timeout = time + @timeout
-        sleep_time = @sleep_time
-        sleep sleep_time
-        until conn = sync{_acquire(thread)}
-          raise_pool_timeout(Time.now - time) if Time.now > timeout
-          sleep sleep_time
+
+    # Connect to the database outside of the connection pool mutex,
+    # as that can take a long time and the connection pool mutex
+    # shouldn't be locked while the connection takes place.
+    if do_make_new
+      begin
+        conn = make_new(:default)
+        sync{allocated[thread] = conn}
+      ensure
+        unless conn
+          sync{allocated.delete(thread)}
         end
       end
-      conn
     end
-    # :nocov:
-  end
 
-  # Returns an available connection. If no connection is
-  # available, tries to create a new connection. The calling code should already
-  # have the mutex before calling this.
-  def available
-    next_available || make_new(:default)
+    conn
   end
 
   # Return a connection to the pool of available connections, returns the connection.
   # The calling code should already have the mutex before calling this.
   def checkin_connection(conn)
     @available_connections << conn
-    if USE_WAITER
-      @waiter.signal
-      Thread.pass
-    end
     conn
   end
 
-  unless method_defined?(:default_make_new)
-    # Alias the default make_new method, so subclasses can call it directly.
-    alias default_make_new make_new
-  end
-  
-  # Creates a new connection to the given server if the size of the pool for
-  # the server is less than the maximum size of the pool. The calling code
-  # should already have the mutex before calling this.
-  def make_new(server)
-    if (n = size) >= @max_size
-      @allocated.keys.each{|t| release(t) unless t.alive?}
-      n = nil
-    end
-    super if (n || size) < @max_size
-  end
-
   # Return the next available connection in the pool, or nil if there
   # is not currently an available connection.  The calling code should already
   # have the mutex before calling this.
@@ -246,15 +226,18 @@ class Sequel::ThreadedConnectionPool < Sequel::ConnectionPool
     sync{@allocated[thread]}
   end
   
-  # Create the maximum number of connections immediately.
+  # Create the maximum number of connections immediately.  The calling code should
+  # NOT have the mutex before calling this.
   def preconnect(concurrent = false)
-    enum = (max_size - size).times
+    enum = (max_size - _size).times
 
-    if concurrent
-      enum.map{Thread.new{make_new(nil)}}.map(&:join).each{|t| checkin_connection(t.value)}
+    conns = if concurrent
+      enum.map{Thread.new{make_new(:default)}}.map(&:value)
     else
-      enum.each{checkin_connection(make_new(nil))}
+      enum.map{make_new(:default)}
     end
+
+    sync{conns.each{|conn| checkin_connection(conn)}}
   end
 
   # Raise a PoolTimeout error showing the current timeout, the elapsed time, and the
@@ -274,6 +257,9 @@ class Sequel::ThreadedConnectionPool < Sequel::ConnectionPool
     else
       checkin_connection(conn)
     end
+
+    @waiter.signal
+    nil
   end
 
   # Yield to the block while inside the mutex. The calling code should NOT
@@ -281,6 +267,4 @@ class Sequel::ThreadedConnectionPool < Sequel::ConnectionPool
   def sync
     @mutex.synchronize{yield}
   end
-  
-  CONNECTION_POOL__MAP[[false, false]] = self
 end

data/lib/sequel/core.rb

@@ -9,13 +9,16 @@
 #
 #   DB = Sequel.sqlite # Memory database
 #   DB = Sequel.sqlite('blog.db')
-#   DB = Sequel.postgres('database_name', :user=>'user', 
-#          :password=>'password', :host=>'host', :port=>5432, 
-#          :max_connections=>10)
+#   DB = Sequel.postgres('database_name',
+#          user:'user', 
+#          password: 'password',
+#          host: 'host'
+#          port: 5432, 
+#          max_connections: 10)
 #
 # If a block is given to these methods, it is passed the opened Database
 # object, which is closed (disconnected) when the block exits, just
-# like a block passed to connect.  For example:
+# like a block passed to Sequel.connect.  For example:
 #
 #   Sequel.sqlite('blog.db'){|db| puts db[:users].count} 
 #
@@ -24,7 +27,7 @@
 module Sequel
   @convert_two_digit_years = true
   @datetime_class = Time
-  @split_symbols = :deprecated
+  @split_symbols = false
   @single_threaded = false
 
   class << self
@@ -41,16 +44,13 @@ module Sequel
     #
     #   Sequel.datetime_class = DateTime
     #
-    # For ruby versions less than 1.9.2, +Time+ has a limited range (1901 to
-    # 2038), so if you use datetimes out of that range, you need to switch
-    # to +DateTime+.  Also, before 1.9.2, +Time+ can only handle local and UTC
-    # times, not other timezones.  Note that +Time+ and +DateTime+ objects
+    # Note that +Time+ and +DateTime+ objects
     # have a different API, and in cases where they implement the same methods,
     # they often implement them differently (e.g. + using seconds on +Time+ and
     # days on +DateTime+).
     attr_accessor :datetime_class
 
-    # Set whether sequel is being used in single threaded mode. by default,
+    # Set whether Sequel is being used in single threaded mode. by default,
     # Sequel uses a thread-safe connection pool, which isn't as fast as the
     # single threaded connection pool, and also has some additional thread
     # safety checks.  If your program will only have one thread,
@@ -92,11 +92,11 @@ module Sequel
   #   DB = Sequel.connect('sqlite://blog.db') # ./blog.db
   #   DB = Sequel.connect('sqlite:///blog.db') # /blog.db
   #   DB = Sequel.connect('postgres://user:password@host:port/database_name')
-  #   DB = Sequel.connect('sqlite:///blog.db', :max_connections=>10)
+  #   DB = Sequel.connect('sqlite:///blog.db', max_connections: 10)
   #
   # You can also pass a single options hash:
   #
-  #   DB = Sequel.connect(:adapter=>'sqlite', :database=>'./blog.db')
+  #   DB = Sequel.connect(adapter: 'sqlite', database: './blog.db')
   #
   # If a block is given, it is passed the opened +Database+ object, which is
   # closed when the block exits.  For example:
@@ -108,7 +108,7 @@ module Sequel
   # design, and used by <tt>Sequel::Model</tt> to pick the default
   # database.  It is recommended to pass a block if you do not want the
   # resulting Database object to remain in memory until the process
-  # terminates.
+  # terminates, or use the <tt>keep_reference: false</tt> Database option.
   #
   # For details, see the {"Connecting to a Database" guide}[rdoc-ref:doc/opening_databases.rdoc].
   # To set up a master/slave or sharded database connection, see the {"Master/Slave Databases and Sharding" guide}[rdoc-ref:doc/sharding.rdoc].
@@ -134,9 +134,7 @@ module Sequel
   end
 
   # Load all Sequel extensions given.  Extensions are just files that exist under
-  # <tt>sequel/extensions</tt> in the load path, and are just required.  Generally,
-  # extensions modify the behavior of +Database+ and/or +Dataset+, but Sequel ships
-  # with some extensions that modify other classes that exist for backwards compatibility.
+  # <tt>sequel/extensions</tt> in the load path, and are just required.  
   # In some cases, requiring an extension modifies classes directly, and in others,
   # it just loads a module that you can extend other classes with.  Consult the documentation
   # for each extension you plan on using for usage.
@@ -147,39 +145,6 @@ module Sequel
     extensions.each{|e| Kernel.require "sequel/extensions/#{e}"}
   end
   
-  # Set the method to call on identifiers going into the database.  This affects
-  # the literalization of identifiers by calling this method on them before they are input.
-  # Sequel upcases identifiers in all SQL strings for most databases, so to turn that off:
-  #
-  #   Sequel.identifier_input_method = nil
-  # 
-  # to downcase instead:
-  #
-  #   Sequel.identifier_input_method = :downcase
-  #
-  # Other String instance methods work as well.
-  def self.identifier_input_method=(value)
-    # SEQUEL5: Remove
-    Database.identifier_input_method = value
-  end
-
-  # Set the method to call on identifiers coming out of the database.  This affects
-  # the literalization of identifiers by calling this method on them when they are
-  # retrieved from the database.  Sequel downcases identifiers retrieved for most
-  # databases, so to turn that off:
-  #
-  #   Sequel.identifier_output_method = nil
-  # 
-  # to upcase instead:
-  #
-  #   Sequel.identifier_output_method = :upcase
-  #
-  # Other String instance methods work as well.
-  def self.identifier_output_method=(value)
-    # SEQUEL5: Remove
-    Database.identifier_output_method = value
-  end
-
   # The exception classed raised if there is an error parsing JSON.
   # This can be overridden to use an alternative json implementation.
   def self.json_parser_error_class
@@ -198,15 +163,6 @@ module Sequel
     JSON.parse(json, :create_additions=>false)
   end
 
-  # Set whether to quote identifiers for all databases by default. By default,
-  # Sequel quotes identifiers in all SQL strings, so to turn that off:
-  #
-  #   Sequel.quote_identifiers = false
-  def self.quote_identifiers=(value)
-    # SEQUEL5: Remove
-    Database.quote_identifiers = value
-  end
-
   # Convert each item in the array to the correct type, handling multi-dimensional
   # arrays.  For each element in the array or subarrays, call the converter,
   # unless the value is nil.
@@ -220,17 +176,15 @@ module Sequel
     end
   end
 
-  # Require all given +files+ which should be in the same or a subdirectory of
-  # this file.  If a +subdir+ is given, assume all +files+ are in that subdir.
-  # This is used to ensure that the files loaded are from the same version of
-  # Sequel as this file.
+  # For backwards compatibility only.  require_relative should be used instead.
   def self.require(files, subdir=nil)
-    Array(files).each{|f| super("#{File.dirname(__FILE__).untaint}/#{"#{subdir}/" if subdir}#{f}")}
+    # Use Kernel.require_relative to work around JRuby 9.0 bug
+    Array(files).each{|f| Kernel.require_relative "#{"#{subdir}/" if subdir}#{f}"}
   end
 
   SPLIT_SYMBOL_CACHE = {}
 
-  # Splits the symbol into three parts, if symbol splitting is enabled.
+  # Splits the symbol into three parts, if symbol splitting is enabled (not the default).
   # Each part will either be a string or nil. If symbol splitting
   # is disabled, returns an array with the first and third parts
   # being nil, and the second part beind a string version of the symbol.
@@ -239,22 +193,13 @@ module Sequel
   # For tables, these parts are the schema, table, and alias.
   def self.split_symbol(sym)
     unless v = Sequel.synchronize{SPLIT_SYMBOL_CACHE[sym]}
-      if split = split_symbols?
+      if split_symbols?
         v = case s = sym.to_s
         when /\A((?:(?!__).)+)__((?:(?!___).)+)___(.+)\z/
-          if split == :deprecated
-            Sequel::Deprecation.deprecate("Symbol splitting", "Either set Sequel.split_symbols = true, or change #{sym.inspect} to Sequel.qualify(#{$1.inspect}, #{$2.inspect}).as(#{$3.inspect})")
-          end
           [$1.freeze, $2.freeze, $3.freeze].freeze
         when /\A((?:(?!___).)+)___(.+)\z/
-          if split == :deprecated
-            Sequel::Deprecation.deprecate("Symbol splitting", "Either set Sequel.split_symbols = true, or change #{sym.inspect} to Sequel.identifier(#{$1.inspect}).as(#{$2.inspect})")
-          end
           [nil, $1.freeze, $2.freeze].freeze
         when /\A((?:(?!__).)+)__(.+)\z/
-          if split == :deprecated
-            Sequel::Deprecation.deprecate("Symbol splitting", "Either set Sequel.split_symbols = true, or change #{sym.inspect} to Sequel.qualify(#{$1.inspect}, #{$2.inspect})")
-          end
           [$1.freeze, $2.freeze, nil].freeze
         else
           [nil, s.freeze, nil].freeze
@@ -267,32 +212,28 @@ module Sequel
     v
   end
 
-  # Sequel by default will split symbols, treating:
+  # Setting this to true enables Sequel's historical behavior of splitting
+  # symbols on double or triple underscores:
   #
   #   :table__column         # table.column
   #   :column___alias        # column AS alias
   #   :table__column___alias # table.column AS alias
   #
-  # This can cause problems if any identifiers in the database use a double
-  # or triple underscore.  When Sequel was first created, using symbols with
-  # double or triple underscores was the only way to represent qualified or
-  # aliased identifiers.  Sequel now offers many ways to create qualified and
-  # aliased identifiers, so there is less of a need for this now.  This allows
-  # you to turn off symbol splitting, potentially avoiding problems if you
-  # have identifiers that use double underscores:
+  # It is only recommended to turn this on for backwards compatibility until
+  # such symbols have been converted to use newer Sequel APIs such as:
   #
-  #   Sequel.split_symbols = false
+  #   Sequel[:table][:column]            # table.column
+  #   Sequel[:column].as(:alias)         # column AS alias
+  #   Sequel[:table][:column].as(:alias) # table.column AS alias
   #
-  # Note that Sequel::Database instances do their own caching of literalized
+  # Sequel::Database instances do their own caching of literalized
   # symbols, and changing this setting does not affect those caches.  It is
   # recommended that if you want to change this setting, you do so directly
   # after requiring Sequel, before creating any Sequel::Database instances.
   #
-  # Also note that disabling symbol splitting will also disable the handling
+  # Disabling symbol splitting will also disable the handling
   # of double underscores in virtual row methods, causing such methods to
-  # yield regular identifers instead of qualified identifiers. To make sure
-  # the code works when splitting symbols is both disabled and enabled, you
-  # can use Sequel::SQL::Identifier#[].
+  # yield regular identifers instead of qualified identifiers:
   #
   #   # Sequel.split_symbols = true
   #   Sequel.expr{table__column}  # table.column
@@ -356,6 +297,8 @@ module Sequel
   # Unless in single threaded mode, protects access to any mutable
   # global data structure in Sequel.
   # Uses a non-reentrant mutex, so calling code should be careful.
+  # In general, this should only be used around the minimal possible code
+  # such as Hash#[], Hash#[]=, Hash#delete, Array#<<, and Array#delete.
   def self.synchronize(&block)
     @single_threaded ? yield : @data_mutex.synchronize(&block)
   end
@@ -433,18 +376,23 @@ module Sequel
 
   # Method that adds a database adapter class method to Sequel that calls
   # Sequel.adapter_method.
-  #
-  # Do not call this method with untrusted input, as that can result in
-  # arbitrary code execution.
   def self.def_adapter_method(*adapters) # :nodoc:
     adapters.each do |adapter|
-      instance_eval("def #{adapter}(*args, &block); adapter_method('#{adapter}', *args, &block) end", __FILE__, __LINE__)
+      define_singleton_method(adapter){|*args, &block| adapter_method(adapter, *args, &block)}
     end
   end
 
   private_class_method :adapter_method, :def_adapter_method
 
-  require(%w"deprecated sql connection_pool exceptions dataset database timezones ast_transformer version")
+  require_relative "deprecated"
+  require_relative "sql"
+  require_relative "connection_pool"
+  require_relative "exceptions"
+  require_relative "dataset"
+  require_relative "database"
+  require_relative "timezones"
+  require_relative "ast_transformer"
+  require_relative "version"
 
   class << self
     # Allow nicer syntax for creating Sequel expressions:
@@ -452,17 +400,10 @@ module Sequel
     #   Sequel[1]     # => Sequel::SQL::NumericExpression: 1
     #   Sequel["a"]   # => Sequel::SQL::StringExpression: 'a'
     #   Sequel[:a]    # => Sequel::SQL::Identifier: "a"
-    #   Sequel[:a=>1] # => Sequel::SQL::BooleanExpression: ("a" = 1)
+    #   Sequel[a: 1]  # => Sequel::SQL::BooleanExpression: ("a" = 1)
     alias_method :[], :expr
   end
 
   # Add the database adapter class methods to Sequel via metaprogramming
   def_adapter_method(*Database::ADAPTERS)
-
-  COLUMN_REF_RE1 = /\A((?:(?!__).)+)__((?:(?!___).)+)___(.+)\z/.freeze
-  Sequel::Deprecation.deprecate_constant(self, :COLUMN_REF_RE1)
-  COLUMN_REF_RE2 = /\A((?:(?!___).)+)___(.+)\z/.freeze
-  Sequel::Deprecation.deprecate_constant(self, :COLUMN_REF_RE2)
-  COLUMN_REF_RE3 = /\A((?:(?!__).)+)__(.+)\z/.freeze
-  Sequel::Deprecation.deprecate_constant(self, :COLUMN_REF_RE3)
 end