sequel 3.21.0 → 3.41.0

data/lib/sequel/core.rb

@@ -17,20 +17,26 @@
 #
 #   Sequel.sqlite('blog.db'){|db| puts db[:users].count} 
 #
-# You can set the +SEQUEL_NO_CORE_EXTENSIONS+ constant or environment variable to have
-# Sequel not extend the core classes.
+# Sequel currently adds methods to the Array, Hash, String and Symbol classes by
+# default.  You can either require 'sequel/no_core_ext' or set the
+# +SEQUEL_NO_CORE_EXTENSIONS+ constant or environment variable before requiring
+# sequel to have # Sequel not add methods to those classes.
 #
 # For a more expanded introduction, see the {README}[link:files/README_rdoc.html].
 # For a quicker introduction, see the {cheat sheet}[link:files/doc/cheat_sheet_rdoc.html].
 module Sequel
   @convert_two_digit_years = true
   @datetime_class = Time
+  @empty_array_handle_nulls = true
   @virtual_row_instance_eval = true
   @require_thread = nil
   
   # Mutex used to protect file loading/requireing
   @require_mutex = Mutex.new
   
+  # Whether Sequel is being run in single threaded mode
+  @single_threaded = false
+  
   class << self
     # Sequel converts two digit years in <tt>Date</tt>s and <tt>DateTime</tt>s by default,
     # so 01/02/03 is interpreted at January 2nd, 2003, and 12/13/99 is interpreted
@@ -54,6 +60,27 @@ module Sequel
     # days on +DateTime+).
     attr_accessor :datetime_class
 
+    # Sets whether or not to attempt to handle NULL values correctly when given
+    # an empty array.  By default:
+    #
+    #   DB[:a].filter(:b=>[])
+    #   # SELECT * FROM a WHERE (b != b)
+    #   DB[:a].exclude(:b=>[])
+    #   # SELECT * FROM a WHERE (b = b)
+    #
+    # However, some databases (e.g. MySQL) will perform very poorly
+    # with this type of query.  You can set this to false to get the
+    # following behavior:
+    #
+    #   DB[:a].filter(:b=>[])
+    #   # SELECT * FROM a WHERE 1 = 0
+    #   DB[:a].exclude(:b=>[])
+    #   # SELECT * FROM a WHERE 1 = 1
+    # 
+    # This may not handle NULLs correctly, but can be much faster on
+    # some databases.
+    attr_accessor :empty_array_handle_nulls
+
     # For backwards compatibility, has no effect.
     attr_accessor :virtual_row_instance_eval
     
@@ -64,6 +91,7 @@ module Sequel
 
     # Make thread safe requiring reentrant to prevent deadlocks.
     def check_requiring_thread
+      return yield if @single_threaded
       t = Thread.current
       return(yield) if @require_thread == t
       @require_mutex.synchronize do
@@ -118,6 +146,12 @@ module Sequel
   def self.connect(*args, &block)
     Database.connect(*args, &block)
   end
+
+  # Assume the core extensions are not loaded by default, if the core_extensions
+  # extension is loaded, this will be overridden.
+  def self.core_extensions?
+    false
+  end
   
   # Convert the +exception+ to the given class.  The given class should be
   # <tt>Sequel::Error</tt> or a subclass.  Returns an instance of +klass+ with
@@ -182,25 +216,62 @@ module Sequel
   def self.quote_identifiers=(value)
     Database.quote_identifiers = value
   end
+
+  # Convert each item in the array to the correct type, handling multi-dimensional
+  # arrays.  For each element in the array or subarrays, call the converter,
+  # unless the value is nil.
+  def self.recursive_map(array, converter)
+    array.map do |i|
+      if i.is_a?(Array)
+        recursive_map(i, converter)
+      elsif i
+        converter.call(i)
+      end
+    end
+  end
   
   # Require all given +files+ which should be in the same or a subdirectory of
   # this file.  If a +subdir+ is given, assume all +files+ are in that subdir.
   # This is used to ensure that the files loaded are from the same version of
   # Sequel as this file.
   def self.require(files, subdir=nil)
-    Array(files).each{|f| super("#{File.dirname(__FILE__)}/#{"#{subdir}/" if subdir}#{f}")}
+    Array(files).each{|f| super("#{File.dirname(__FILE__).untaint}/#{"#{subdir}/" if subdir}#{f}")}
   end
   
-  # Set whether to set the single threaded mode for all databases by default. By default,
+  # Set whether Sequel is being used in single threaded mode. By default,
   # Sequel uses a thread-safe connection pool, which isn't as fast as the
-  # single threaded connection pool.  If your program will only have one thread,
-  # and speed is a priority, you may want to set this to true:
+  # single threaded connection pool, and also has some additional thread
+  # safety checks.  If your program will only have one thread,
+  # and speed is a priority, you should set this to true:
   #
   #   Sequel.single_threaded = true
   def self.single_threaded=(value)
+    @single_threaded = value
     Database.single_threaded = value
   end
 
+  COLUMN_REF_RE1 = /\A((?:(?!__).)+)__((?:(?!___).)+)___(.+)\z/.freeze
+  COLUMN_REF_RE2 = /\A((?:(?!___).)+)___(.+)\z/.freeze
+  COLUMN_REF_RE3 = /\A((?:(?!__).)+)__(.+)\z/.freeze
+
+  # Splits the symbol into three parts.  Each part will
+  # either be a string or nil.
+  #
+  # For columns, these parts are the table, column, and alias.
+  # For tables, these parts are the schema, table, and alias.
+  def self.split_symbol(sym)
+    case s = sym.to_s
+    when COLUMN_REF_RE1
+      [$1, $2, $3]
+    when COLUMN_REF_RE2
+      [nil, $1, $2]
+    when COLUMN_REF_RE3
+      [$1, $2, nil]
+    else
+      [nil, s, nil]
+    end
+  end
+
   # Converts the given +string+ into a +Date+ object.
   #
   #   Sequel.string_to_date('2010-09-10') # Date.civil(2010, 09, 10)
@@ -228,17 +299,76 @@ module Sequel
     end
   end
 
-  # Converts the given +string+ into a +Time+ object.
+  # Converts the given +string+ into a <tt>Sequel::SQLTime</tt> object.
   #
-  #   Sequel.string_to_datetime('10:20:30') # Time.parse('10:20:30')
+  #   v = Sequel.string_to_time('10:20:30') # Sequel::SQLTime.parse('10:20:30')
+  #   DB.literal(v) # => '10:20:30'
   def self.string_to_time(string)
     begin
-      Time.parse(string)
+      SQLTime.parse(string)
     rescue => e
       raise convert_exception_class(e, InvalidValue)
     end
   end
 
+  if defined?(RUBY_ENGINE) && RUBY_ENGINE != 'ruby'
+    # Mutex used to protect mutable data structures
+    @data_mutex = Mutex.new
+
+    # Unless in single threaded mode, protects access to any mutable
+    # global data structure in Sequel.
+    # Uses a non-reentrant mutex, so calling code should be careful.
+    def self.synchronize(&block)
+      @single_threaded ? yield : @data_mutex.synchronize(&block)
+    end
+  else
+    # Yield directly to the block.  You don't need to synchronize
+    # access on MRI because the GVL makes certain methods atomic.
+    def self.synchronize(&block)
+      yield
+    end
+  end
+
+  # Uses a transaction on all given databases with the given options. This:
+  #
+  #   Sequel.transaction([DB1, DB2, DB3]){...}
+  #
+  # is equivalent to:
+  #
+  #   DB1.transaction do
+  #     DB2.transaction do
+  #       DB3.transaction do
+  #         ...
+  #       end
+  #     end
+  #   end
+  #
+  # except that if Sequel::Rollback is raised by the block, the transaction is
+  # rolled back on all databases instead of just the last one.
+  #
+  # Note that this method cannot guarantee that all databases will commit or
+  # rollback.  For example, if DB3 commits but attempting to commit on DB2
+  # fails (maybe because foreign key checks are deferred), there is no way
+  # to uncommit the changes on DB3.  For that kind of support, you need to
+  # have two-phase commit/prepared transactions (which Sequel supports on
+  # some databases).
+  def self.transaction(dbs, opts={}, &block)
+    unless opts[:rollback]
+      rescue_rollback = true
+      opts = opts.merge(:rollback=>:reraise)
+    end
+    pr = dbs.reverse.inject(block){|bl, db| proc{db.transaction(opts, &bl)}}
+    if rescue_rollback
+      begin
+        pr.call
+      rescue Sequel::Rollback => e
+        nil
+      end
+    else
+      pr.call
+    end
+  end
+
   # Same as Sequel.require, but wrapped in a mutex in order to be thread safe.
   def self.ts_require(*args)
     check_requiring_thread{require(*args)}
@@ -292,8 +422,10 @@ module Sequel
 
   private_class_method :adapter_method, :def_adapter_method
   
-  require(%w"metaprogramming sql connection_pool exceptions dataset database timezones version")
-  require('core_sql') if !defined?(::SEQUEL_NO_CORE_EXTENSIONS) && !ENV.has_key?('SEQUEL_NO_CORE_EXTENSIONS')
+  require(%w"metaprogramming sql connection_pool exceptions dataset database timezones ast_transformer version")
+  if !defined?(::SEQUEL_NO_CORE_EXTENSIONS) && !ENV.has_key?('SEQUEL_NO_CORE_EXTENSIONS')
+    extension(:core_extensions)
+  end
 
   # Add the database adapter class methods to Sequel via metaprogramming
   def_adapter_method(*Database::ADAPTERS)

data/lib/sequel/database/connecting.rb

@@ -1,12 +1,12 @@
 module Sequel
   class Database
     # ---------------------
-    # :section: Methods relating to adapters, connecting, disconnecting, and sharding
+    # :section: 4 - Methods relating to adapters, connecting, disconnecting, and sharding
     # This methods involve the Database's connection pool.
     # ---------------------
 
     # Array of supported database adapters
-    ADAPTERS = %w'ado amalgalite db2 dbi do firebird informix jdbc mysql mysql2 odbc openbase oracle postgres sqlite swift tinytds'.collect{|x| x.to_sym}
+    ADAPTERS = %w'ado amalgalite cubrid db2 dbi do firebird ibmdb informix jdbc mock mysql mysql2 odbc openbase oracle postgres sqlite swift tinytds'.collect{|x| x.to_sym}
 
     # Whether to use the single threaded connection pool by default
     @@single_threaded = false
@@ -46,6 +46,7 @@ module Sequel
       when String
         if match = /\A(jdbc|do):/o.match(conn_string)
           c = adapter_class(match[1].to_sym)
+          opts = opts.merge(:orig_opts=>opts.dup)
           opts = {:uri=>conn_string}.merge(opts)
         else
           uri = URI.parse(conn_string)
@@ -54,12 +55,15 @@ module Sequel
           c = adapter_class(scheme)
           uri_options = c.send(:uri_to_options, uri)
           uri.query.split('&').collect{|s| s.split('=')}.each{|k,v| uri_options[k.to_sym] = v if k && !k.empty?} unless uri.query.to_s.strip.empty?
-          uri_options.entries.each{|k,v| uri_options[k] = URI.unescape(v) if v.is_a?(String)}
+          uri_options.to_a.each{|k,v| uri_options[k] = URI.unescape(v) if v.is_a?(String)}
+          opts = opts.merge(:orig_opts=>opts.dup)
+          opts[:uri] = conn_string
           opts = uri_options.merge(opts)
           opts[:adapter] = scheme
         end
       when Hash
         opts = conn_string.merge(opts)
+        opts = opts.merge(:orig_opts=>opts.dup)
         c = adapter_class(opts[:adapter_class] || opts[:adapter] || opts['adapter'])
       else
         raise Error, "Sequel::Database.connect takes either a Hash or a String, given: #{conn_string.inspect}"
@@ -77,7 +81,7 @@ module Sequel
       ensure
         if block_given?
           db.disconnect if db
-          ::Sequel::DATABASES.delete(db)
+          Sequel.synchronize{::Sequel::DATABASES.delete(db)}
         end
       end
       block_given? ? result : db
@@ -131,10 +135,12 @@ module Sequel
     #
     #   DB.add_servers(:f=>{:host=>"hash_host_f"})
     def add_servers(servers)
-      @opts[:servers] = @opts[:servers] ? @opts[:servers].merge(servers) : servers
-      @pool.add_servers(servers.keys)
+      if h = @opts[:servers]
+        Sequel.synchronize{h.merge!(servers)}
+        @pool.add_servers(servers.keys)
+      end
     end
-    
+
     # Connects to the database. This method should be overridden by descendants.
     def connect(server)
       raise NotImplemented, "#connect should be overridden by adapters"
@@ -155,8 +161,8 @@ module Sequel
     
     # Disconnects all available connections from the connection pool.  Any
     # connections currently in use will not be disconnected. Options:
-    # * :servers - Should be a symbol specifing the server to disconnect from,
-    #   or an array of symbols to specify multiple servers.
+    # :servers :: Should be a symbol specifing the server to disconnect from,
+    #             or an array of symbols to specify multiple servers.
     #
     # Example:
     #
@@ -167,6 +173,13 @@ module Sequel
       pool.disconnect(opts)
     end
 
+    # Should only be called by the connection pool code to disconnect a connection.
+    # By default, calls the close method on the connection object, since most
+    # adapters use that, but should be overwritten on other adapters.
+    def disconnect_connection(conn)
+      conn.close
+    end
+
     # Yield a new Database instance for every server in the connection pool.
     # Intended for use in sharded environments where there is a need to make schema
     # modifications (DDL queries) on each shard.
@@ -187,11 +200,8 @@ module Sequel
     #
     #   DB.remove_servers(:f1, :f2)
     def remove_servers(*servers)
-      if @opts[:servers] && !@opts[:servers].empty?
-        servs = @opts[:servers].dup
-        servers.flatten!
-        servers.each{|s| servs.delete(s)}
-        @opts[:servers] = servs
+      if h = @opts[:servers]
+        servers.flatten.each{|s| Sequel.synchronize{h.delete(s)}}
         @pool.remove_servers(servers)
       end
     end
@@ -209,21 +219,26 @@ module Sequel
       @single_threaded
     end
     
-    # Acquires a database connection, yielding it to the passed block. This is
-    # useful if you want to make sure the same connection is used for all
-    # database queries in the block.  It is also useful if you want to gain
-    # direct access to the underlying connection object if you need to do
-    # something Sequel does not natively support.
-    #
-    # If a server option is given, acquires a connection for that specific
-    # server, instead of the :default server.
-    #
-    #
-    #   DB.synchronize do |conn|
-    #     ...
-    #   end
-    def synchronize(server=nil, &block)
-      @pool.hold(server || :default, &block)
+    if !defined?(RUBY_ENGINE) || RUBY_ENGINE == 'ruby'
+      # Acquires a database connection, yielding it to the passed block. This is
+      # useful if you want to make sure the same connection is used for all
+      # database queries in the block.  It is also useful if you want to gain
+      # direct access to the underlying connection object if you need to do
+      # something Sequel does not natively support.
+      #
+      # If a server option is given, acquires a connection for that specific
+      # server, instead of the :default server.
+      #
+      #   DB.synchronize do |conn|
+      #     ...
+      #   end
+      def synchronize(server=nil)
+        @pool.hold(server || :default){|conn| yield conn}
+      end
+    else
+      def synchronize(server=nil, &block)
+        @pool.hold(server || :default, &block)
+      end
     end
     
     # Attempts to acquire a database connection.  Returns true if successful.
@@ -235,6 +250,21 @@ module Sequel
       true
     end
 
+    # Check whether the given connection is currently valid, by
+    # running a query against it.  If the query fails, the
+    # connection should probably be removed from the connection
+    # pool.
+    def valid_connection?(conn)
+      sql = valid_connection_sql
+      begin
+        log_connection_execute(conn, sql)
+      rescue Sequel::DatabaseError, *database_error_classes
+        false
+      else
+        true
+      end
+    end
+
     private
     
     # The default options for the connection pool.
@@ -255,11 +285,18 @@ module Sequel
         else
           raise Error, 'Server opts should be a hash or proc'
         end
+      elsif server.is_a?(Hash)
+        @opts.merge(server)
       else
         @opts.dup
       end
       opts.delete(:servers)
       opts
     end
+
+    # The SQL query to issue to check if a connection is valid.
+    def valid_connection_sql
+      @valid_connection_sql ||= select(nil).sql
+    end
   end
 end

data/lib/sequel/database/dataset.rb

@@ -1,7 +1,7 @@
 module Sequel
   class Database
     # ---------------------
-    # :section: Methods that create datasets
+    # :section: 3 - Methods that create datasets
     # These methods all return instances of this database's dataset class.
     # ---------------------
 
@@ -24,10 +24,10 @@ module Sequel
     #
     #   DB.dataset # SELECT *
     #   DB.dataset.from(:items) # SELECT * FROM items
-    def dataset
-      ds = Sequel::Dataset.new(self)
+    def dataset(opts=nil)
+      @dataset_class.new(self, opts)
     end
-    
+
     # Fetches records for an arbitrary SQL statement. If a block is given,
     # it is used to iterate over the records:
     #

data/lib/sequel/database/dataset_defaults.rb

@@ -1,10 +1,13 @@
 module Sequel
   class Database
     # ---------------------
-    # :section: Methods that set defaults for created datasets
+    # :section: 5 - Methods that set defaults for created datasets
     # This methods change the default behavior of this database's datasets.
     # ---------------------
 
+    # The default class to use for datasets
+    DatasetClass = Sequel::Dataset
+
     # The identifier input method to use by default
     @@identifier_input_method = nil
 
@@ -42,9 +45,62 @@ module Sequel
       @@quote_identifiers = value
     end
 
-    # The default schema to use, generally should be nil.
+    # The class to use for creating datasets.  Should respond to
+    # new with the Database argument as the first argument, and
+    # an optional options hash.
+    attr_reader :dataset_class
+
+    # The default schema to use, generally should be nil.  This sets
+    # the default schema used for some schema modification and
+    # introspection queries, but does not effect most dataset code.
     attr_accessor :default_schema
 
+    # If the database has any dataset modules associated with it,
+    # use a subclass of the given class that includes the modules
+    # as the dataset class.
+    def dataset_class=(c)
+      unless @dataset_modules.empty?
+        c = Class.new(c)
+        @dataset_modules.each{|m| c.send(:include, m)}
+      end
+      @dataset_class = c
+    end
+
+    # Equivalent to extending all datasets produced by the database with a
+    # module.  What it actually does is use a subclass of the current dataset_class
+    # as the new dataset_class, and include the module in the subclass.
+    # Instead of a module, you can provide a block that is used to create an
+    # anonymous module.
+    #
+    # This allows you to override any of the dataset methods even if they are
+    # defined directly on the dataset class that this Database object uses.
+    #
+    # Examples:
+    #
+    #   # Introspec columns for all of DB's datasets
+    #   DB.extend_datasets(Sequel::ColumnsIntrospection)
+    #   
+    #   # Trace all SELECT queries by printing the SQL and the full backtrace
+    #   DB.extend_datasets do
+    #     def fetch_rows(sql)
+    #       puts sql
+    #       puts caller
+    #       super
+    #     end
+    #   end
+    def extend_datasets(mod=nil, &block)
+      raise(Error, "must provide either mod or block, not both") if mod && block
+      reset_schema_utility_dataset
+      mod = Module.new(&block) if block
+      if @dataset_modules.empty?
+       @dataset_modules = [mod]
+       @dataset_class = Class.new(@dataset_class)
+      else
+       @dataset_modules << mod
+      end
+      @dataset_class.send(:include, mod)
+    end
+
     # The method to call on identifiers going into the database
     def identifier_input_method
       case @identifier_input_method
@@ -109,6 +165,11 @@ module Sequel
     
     private
     
+    # The default dataset class to use for the database
+    def dataset_class_default
+      self.class.const_get(:DatasetClass)
+    end
+    
     # The default value for default_schema.
     def default_schema_default
       nil

data/lib/sequel/database/logging.rb

@@ -1,7 +1,7 @@
 module Sequel
   class Database
     # ---------------------
-    # :section: Methods relating to logging
+    # :section: 6 - Methods relating to logging
     # This methods affect relating to the logging of executed SQL.
     # ---------------------
 
@@ -17,6 +17,11 @@ module Sequel
     # is :info, it can be set to :debug to log at DEBUG level.
     attr_accessor :sql_log_level
 
+    # Log a message at error level, with information about the exception.
+    def log_exception(exception, message)
+      log_each(:error, "#{exception.class}: #{exception.message.strip if exception.message}: #{message}")
+    end
+
     # Log a message at level info to all loggers.
     def log_info(message, args=nil)
       log_each(:info, args ? "#{message}; #{args.inspect}" : message)
@@ -31,7 +36,7 @@ module Sequel
       begin
         yield
       rescue => e
-        log_each(:error, "#{e.class}: #{e.message.strip}: #{sql}")
+        log_exception(e, sql)
         raise
       ensure
         log_duration(Time.now - start, sql) unless e