sequel 3.21.0 → 3.24.0

data/lib/sequel/ast_transformer.rb

@@ -0,0 +1,190 @@
+module Sequel
+  # The +ASTTransformer+ class is designed to handle the abstract syntax trees
+  # that Sequel uses internally and produce modified copies of them.  By itself
+  # it only produces a straight copy.  It's designed to be subclassed and have
+  # subclasses returned modified copies of the specific nodes that need to
+  # be modified.
+  class ASTTransformer
+    # Return +obj+ or a potentially transformed version of it.
+    def transform(obj)
+      v(obj)
+    end
+
+    private
+
+    # Recursive version that handles all of Sequel's internal object types
+    # and produces copies of them.
+    def v(o)
+      case o
+      when Symbol, Numeric, String, Class, TrueClass, FalseClass, NilClass
+        o
+      when Array
+        o.map{|x| v(x)}
+      when Hash
+        h = {}
+        o.each{|k, val| h[v(k)] = v(val)}
+        h
+      when SQL::ComplexExpression
+        SQL::ComplexExpression.new(o.op, *v(o.args))
+      when SQL::Identifier
+        SQL::Identifier.new(v(o.value))
+      when SQL::QualifiedIdentifier
+        SQL::QualifiedIdentifier.new(v(o.table), v(o.column))
+      when SQL::OrderedExpression
+        SQL::OrderedExpression.new(v(o.expression), o.descending, :nulls=>o.nulls)
+      when SQL::AliasedExpression
+        SQL::AliasedExpression.new(v(o.expression), o.aliaz)
+      when SQL::CaseExpression
+        args = [v(o.conditions), v(o.default)]
+        args << v(o.expression) if o.expression?
+        SQL::CaseExpression.new(*args)
+      when SQL::Cast
+        SQL::Cast.new(v(o.expr), o.type)
+      when SQL::Function
+        SQL::Function.new(o.f, *v(o.args))
+      when SQL::Subscript
+        SQL::Subscript.new(v(o.f), v(o.sub))
+      when SQL::WindowFunction
+        SQL::WindowFunction.new(v(o.function), v(o.window))
+      when SQL::Window
+        opts = o.opts.dup
+        opts[:partition] = v(opts[:partition]) if opts[:partition]
+        opts[:order] = v(opts[:order]) if opts[:order]
+        SQL::Window.new(opts)
+      when SQL::PlaceholderLiteralString
+        args = if o.args.is_a?(Hash)
+          h = {}
+          o.args.each{|k,val| h[k] = v(val)}
+          h
+        else
+          v(o.args)
+        end
+        SQL::PlaceholderLiteralString.new(o.str, args, o.parens)
+      when SQL::JoinOnClause
+        SQL::JoinOnClause.new(v(o.on), o.join_type, v(o.table), v(o.table_alias))
+      when SQL::JoinUsingClause
+        SQL::JoinUsingClause.new(v(o.using), o.join_type, v(o.table), v(o.table_alias))
+      when SQL::JoinClause
+        SQL::JoinClause.new(o.join_type, v(o.table), v(o.table_alias))
+      else
+        o
+      end
+    end
+  end
+
+  # Handles qualifying existing datasets, so that unqualified columns
+  # in the dataset are qualified with a given table name.
+  class Qualifier < ASTTransformer
+    # Store the dataset to use as the basis for qualification, 
+    # and the table used to qualify unqualified columns. 
+    def initialize(ds, table)
+      @ds = ds
+      @table = table
+    end
+
+    private
+
+    # Turn <tt>SQL::Identifier</tt>s and symbols that aren't implicitly
+    # qualified into <tt>SQL::QualifiedIdentifier</tt>s.  For symbols that
+    # are not implicitly qualified by are implicitly aliased, return an
+    # <tt>SQL::AliasedExpression</tt>s with a qualified version of the symbol.
+    def v(o)
+      case o
+      when Symbol
+        t, column, aliaz = @ds.send(:split_symbol, o)
+        if t
+          o
+        elsif aliaz
+          SQL::AliasedExpression.new(SQL::QualifiedIdentifier.new(@table, SQL::Identifier.new(column)), aliaz)
+        else
+          SQL::QualifiedIdentifier.new(@table, o)
+        end
+      when SQL::Identifier
+        SQL::QualifiedIdentifier.new(@table, o)
+      when SQL::QualifiedIdentifier, SQL::JoinClause
+        # Return these directly, so we don't accidentally qualify symbols in them.
+        o
+      else
+        super
+      end
+    end
+  end
+
+  # +Unbinder+ is used to take a dataset filter and return a modified version
+  # that unbinds already bound values and returns a dataset with bound value
+  # placeholders and a hash of bind values.  You can then prepare the dataset
+  # and use the bound variables to execute it with the same values.
+  #
+  # This class only does a limited form of unbinding where the variable names
+  # and values can be associated unambiguously.  The only cases it handles
+  # are <tt>SQL::ComplexExpression<tt> with an operator in +UNBIND_OPS+, a
+  # first argument that's an instance of a member of +UNBIND_KEY_CLASSES+, and
+  # a second argument that's an instance of a member of +UNBIND_VALUE_CLASSES+.
+  #
+  # So it can handle cases like:
+  #
+  #   DB.filter(:a=>1).exclude(:b=>2).where{c > 3}
+  #
+  # But it cannot handle cases like:
+  #
+  #   DB.filter(:a + 1 < 0)
+  class Unbinder < ASTTransformer
+    # The <tt>SQL::ComplexExpression<tt> operates that will be considered
+    # for transformation.
+    UNBIND_OPS = [:'=', :'!=', :<, :>, :<=, :>=]
+
+    # The key classes (first argument of the ComplexExpression) that will
+    # considered for transformation.
+    UNBIND_KEY_CLASSES = [Symbol, SQL::Identifier, SQL::QualifiedIdentifier]
+
+    # The value classes (second argument of the ComplexExpression) that
+    # will be considered for transformation.
+    UNBIND_VALUE_CLASSES = [Numeric, String, Date, Time]
+
+    # The hash of bind variables that were extracted from the dataset filter.
+    attr_reader :binds
+
+    # Intialize an empty +binds+ hash.
+    def initialize
+      @binds = {}
+    end
+
+    private
+
+    # Create a suitable bound variable key for the object, which should be
+    # an instance of one of the +UNBIND_KEY_CLASSES+.
+    def bind_key(obj)
+      case obj
+      when Symbol, String
+        obj
+      when SQL::Identifier
+        bind_key(obj.value)
+      when SQL::QualifiedIdentifier
+        :"#{bind_key(obj.table)}.#{bind_key(obj.column)}"
+      else
+        raise Error, "unhandled object in Sequel::Unbinder#bind_key: #{obj}"
+      end
+    end
+
+    # Handle <tt>SQL::ComplexExpression</tt> instances with suitable ops
+    # and arguments, substituting the value with a bound variable placeholder
+    # and assigning it an entry in the +binds+ hash with a matching key.
+    def v(o)
+      if o.is_a?(SQL::ComplexExpression) && UNBIND_OPS.include?(o.op)
+        l, r = o.args
+        if UNBIND_KEY_CLASSES.any?{|c| l.is_a?(c)} && UNBIND_VALUE_CLASSES.any?{|c| r.is_a?(c)} && !r.is_a?(LiteralString)
+          key = bind_key(l)
+          if (old = binds[key]) && old != r
+            raise UnbindDuplicate, "two different values for #{key.inspect}: #{[r, old].inspect}" 
+          end
+          binds[key] = r
+          SQL::ComplexExpression.new(o.op, l, :"$#{key}")
+        else
+          super
+        end
+      else
+        super
+      end
+    end
+  end
+end

data/lib/sequel/core.rb

@@ -292,7 +292,7 @@ module Sequel
 
   private_class_method :adapter_method, :def_adapter_method
   
-  require(%w"metaprogramming sql connection_pool exceptions dataset database timezones version")
+  require(%w"metaprogramming sql connection_pool exceptions dataset database timezones ast_transformer version")
   require('core_sql') if !defined?(::SEQUEL_NO_CORE_EXTENSIONS) && !ENV.has_key?('SEQUEL_NO_CORE_EXTENSIONS')
 
   # Add the database adapter class methods to Sequel via metaprogramming

data/lib/sequel/database/connecting.rb

@@ -54,7 +54,7 @@ module Sequel
           c = adapter_class(scheme)
           uri_options = c.send(:uri_to_options, uri)
           uri.query.split('&').collect{|s| s.split('=')}.each{|k,v| uri_options[k.to_sym] = v if k && !k.empty?} unless uri.query.to_s.strip.empty?
-          uri_options.entries.each{|k,v| uri_options[k] = URI.unescape(v) if v.is_a?(String)}
+          uri_options.to_a.each{|k,v| uri_options[k] = URI.unescape(v) if v.is_a?(String)}
           opts = uri_options.merge(opts)
           opts[:adapter] = scheme
         end

data/lib/sequel/database/misc.rb

@@ -90,6 +90,12 @@ module Sequel
       {:primary_key => true, :type => Integer, :auto_increment => true}
     end
 
+    # Whether the database supports CREATE TABLE IF NOT EXISTS syntax,
+    # false by default.
+    def supports_create_table_if_not_exists?
+      false
+    end
+
     # Whether the database and adapter support prepared transactions
     # (two-phase commit), false by default.
     def supports_prepared_transactions?

data/lib/sequel/database/query.rb

@@ -210,6 +210,13 @@ module Sequel
       end
     end
     
+    # Return all views in the database as an array of symbols.
+    #
+    #   DB.views # => [:gold_albums, :artists_with_many_albums]
+    def views(opts={})
+      raise NotImplemented, "#views should be overridden by adapters"
+    end
+    
     private
     
     # Internal generic transaction method.  Any exception raised by the given
@@ -226,7 +233,7 @@ module Sequel
         transaction_error(e)
       ensure
         begin
-          commit_transaction(t, opts) unless e
+          commit_or_rollback_transaction(e, t, opts)
         rescue Exception => e
           raise_error(e, :classes=>database_error_classes)
         ensure
@@ -234,7 +241,7 @@ module Sequel
         end
       end
     end
-    
+
     # Add the current thread to the list of active transactions
     def add_transaction
       th = Thread.current
@@ -335,6 +342,29 @@ module Sequel
       end
     end
    
+    if (! defined?(RUBY_ENGINE) or RUBY_ENGINE == 'ruby' or RUBY_ENGINE == 'rbx') and RUBY_VERSION < '1.9'
+      # Whether to commit the current transaction. On ruby 1.8 and rubinius,
+      # Thread.current.status is checked because Thread#kill skips rescue
+      # blocks (so exception would be nil), but the transaction should
+      # still be rolled back.
+      def commit_or_rollback_transaction(exception, thread, opts)
+        unless exception
+          if Thread.current.status == 'aborting'
+            rollback_transaction(thread, opts)
+          else
+            commit_transaction(thread, opts)
+          end
+        end
+      end
+    else
+      # Whether to commit the current transaction.  On ruby 1.9 and JRuby,
+      # transactions will be committed if Thread#kill is used on an thread
+      # that has a transaction open, and there isn't a work around.
+      def commit_or_rollback_transaction(exception, thread, opts)
+        commit_transaction(thread, opts) unless exception
+      end
+    end
+    
     # SQL to commit a savepoint
     def commit_savepoint_sql(depth)
       SQL_RELEASE_SAVEPOINT % depth
@@ -434,7 +464,7 @@ module Sequel
         :datetime
       when /\Atime( with(out)? time zone)?\z/io
         :time
-      when /\A(boolean|bit)\z/io
+      when /\A(bool(ean)?)\z/io
         :boolean
       when /\A(real|float|double( precision)?)\z/io
         :float

data/lib/sequel/database/schema_methods.rb

@@ -21,7 +21,7 @@ module Sequel
     UNSIGNED = ' UNSIGNED'.freeze
 
     # The order of column modifiers to use when defining a column.
-    COLUMN_DEFINITION_ORDER = [:default, :null, :unique, :primary_key, :auto_increment, :references]
+    COLUMN_DEFINITION_ORDER = [:collate, :default, :null, :unique, :primary_key, :auto_increment, :references]
 
     # Adds a column to the specified table. This method expects a column name,
     # a datatype and optionally a hash with additional constraints and options:
@@ -111,7 +111,11 @@ module Sequel
     
     # Creates the table unless the table already exists
     def create_table?(name, options={}, &block)
-      create_table(name, options, &block) unless table_exists?(name)
+      if supports_create_table_if_not_exists?
+        create_table(name, options.merge(:if_not_exists=>true), &block)
+      elsif !table_exists?(name)
+        create_table(name, options, &block)
+      end
     end
     
     # Creates a view, replacing it if it already exists:
@@ -275,7 +279,12 @@ module Sequel
     def column_definition_auto_increment_sql(sql, column)
       sql << " #{auto_increment_sql}" if column[:auto_increment]
     end
-    
+
+    # Add collate SQL fragment to column creation SQL.
+    def column_definition_collate_sql(sql, column)
+      sql << " COLLATE #{column[:collate]}" if column[:collate]
+    end
+
     # Add default SQL fragment to column creation SQL.
     def column_definition_default_sql(sql, column)
       sql << " DEFAULT #{literal(column[:default])}" if column.include?(:default)
@@ -366,7 +375,7 @@ module Sequel
 
     # DDL statement for creating a table with the given name, columns, and options
     def create_table_sql(name, generator, options)
-      "CREATE #{temporary_table_sql if options[:temp]}TABLE #{quote_schema_table(name)} (#{column_list_sql(generator)})"
+      "CREATE #{temporary_table_sql if options[:temp]}TABLE#{' IF NOT EXISTS' if options[:if_not_exists]} #{options[:temp] ? quote_identifier(name) : quote_schema_table(name)} (#{column_list_sql(generator)})"
     end
 
     # Default index name for the table and columns, may be too long

data/lib/sequel/dataset/features.rb

@@ -37,6 +37,12 @@ module Sequel
       false
     end
 
+    # Whether this dataset supports the +insert_select+ method for returning all columns values
+    # directly from an insert query.
+    def supports_insert_select?
+      false
+    end
+
     # Whether the dataset supports the INTERSECT and EXCEPT compound operations, true by default.
     def supports_intersect_except?
       true

data/lib/sequel/dataset/prepared_statements.rb

@@ -81,6 +81,8 @@ module Sequel
           select_sql
         when :first
           clone(:limit=>1).select_sql
+        when :insert_select
+          clone(:returning=>nil).insert_sql(*@prepared_modify_values)
         when :insert
           insert_sql(*@prepared_modify_values)
         when :update
@@ -95,8 +97,8 @@ module Sequel
       # and they are substituted using prepared_arg.
       def literal_symbol(v)
         if @opts[:bind_vars] and match = PLACEHOLDER_RE.match(v.to_s)
-          v2 = prepared_arg(match[1].to_sym)
-          v2 ? literal(v2) : v
+          s = match[1].to_sym
+          prepared_arg?(s) ? literal(prepared_arg(s)) : v
         else
           super
         end
@@ -118,6 +120,9 @@ module Sequel
         case @prepared_type
         when :select, :all
           all(&block)
+        when :insert_select
+          meta_def(:select_sql){prepared_sql}
+          first
         when :first
           first
         when :insert
@@ -136,6 +141,11 @@ module Sequel
         @opts[:bind_vars][k]
       end
 
+      # Whether there is a bound value for the given key.
+      def prepared_arg?(k)
+        @opts[:bind_vars].has_key?(k)
+      end
+
       # Use a clone of the dataset extended with prepared statement
       # support and using the same argument hash so that you can use
       # bind variables/prepared arguments in subselects.
@@ -171,6 +181,11 @@ module Sequel
         prepared_args << k
         prepared_arg_placeholder
       end
+      
+      # Always assume there is a prepared arg in the argument mapper.
+      def prepared_arg?(k)
+        true
+      end
     end
     
     # Set the bind variables to use for the call.  If bind variables have

data/lib/sequel/dataset/query.rb

@@ -431,6 +431,7 @@ module Sequel
             v = qualified_column_name(v, last_alias) if v.is_a?(Symbol)
             [k,v]
           end
+          expr = SQL::BooleanExpression.from_value_pairs(expr)
         end
         if block
           expr2 = yield(table_name, last_alias, @opts[:join] || [])
@@ -702,6 +703,22 @@ module Sequel
       clone(:overrides=>hash.merge(@opts[:overrides]||{}))
     end
     
+    # Unbind bound variables from this dataset's filter and return an array of two
+    # objects.  The first object is a modified dataset where the filter has been
+    # replaced with one that uses bound variable placeholders.  The second object
+    # is the hash of unbound variables.  You can then prepare and execute (or just
+    # call) the dataset with the bound variables to get results.
+    #
+    #   ds, bv = DB[:items].filter(:a=>1).unbind
+    #   ds # SELECT * FROM items WHERE (a = $a)
+    #   bv #  {:a => 1}
+    #   ds.call(:select, bv)
+    def unbind
+      u = Unbinder.new
+      ds = clone(:where=>u.transform(opts[:where]), :join=>u.transform(opts[:join]))
+      [ds, u.binds]
+    end
+
     # Returns a copy of the dataset with no filters (HAVING or WHERE clause) applied.
     # 
     #   DB[:items].group(:a).having(:a=>1).where(:b).unfiltered

data/lib/sequel/dataset/sql.rb

@@ -269,6 +269,7 @@ module Sequel
               literal(op == :IN ? expr : ~expr)
             else
               old_vals = vals
+              vals = vals.naked if vals.is_a?(Sequel::Dataset)
               vals = vals.to_a
               val_cols = old_vals.columns
               complex_expression_sql(op, [cols, vals.map!{|x| x.values_at(*val_cols)}])
@@ -774,59 +775,7 @@ module Sequel
     
     # Qualify the given expression e to the given table.
     def qualified_expression(e, table)
-      case e
-      when Symbol
-        t, column, aliaz = split_symbol(e)
-        if t
-          e
-        elsif aliaz
-          SQL::AliasedExpression.new(SQL::QualifiedIdentifier.new(table, SQL::Identifier.new(column)), aliaz)
-        else
-          SQL::QualifiedIdentifier.new(table, e)
-        end
-      when Array
-        e.map{|a| qualified_expression(a, table)}
-      when Hash
-        h = {}
-        e.each{|k,v| h[qualified_expression(k, table)] = qualified_expression(v, table)}
-        h
-      when SQL::Identifier
-        SQL::QualifiedIdentifier.new(table, e)
-      when SQL::OrderedExpression
-        SQL::OrderedExpression.new(qualified_expression(e.expression, table), e.descending, :nulls=>e.nulls)
-      when SQL::AliasedExpression
-        SQL::AliasedExpression.new(qualified_expression(e.expression, table), e.aliaz)
-      when SQL::CaseExpression
-        args = [qualified_expression(e.conditions, table), qualified_expression(e.default, table)]
-        args << qualified_expression(e.expression, table) if e.expression?
-        SQL::CaseExpression.new(*args)
-      when SQL::Cast
-        SQL::Cast.new(qualified_expression(e.expr, table), e.type)
-      when SQL::Function
-        SQL::Function.new(e.f, *qualified_expression(e.args, table))
-      when SQL::ComplexExpression 
-        SQL::ComplexExpression.new(e.op, *qualified_expression(e.args, table))
-      when SQL::Subscript 
-        SQL::Subscript.new(qualified_expression(e.f, table), qualified_expression(e.sub, table))
-      when SQL::WindowFunction
-        SQL::WindowFunction.new(qualified_expression(e.function, table), qualified_expression(e.window, table))
-      when SQL::Window
-        o = e.opts.dup
-        o[:partition] = qualified_expression(o[:partition], table) if o[:partition]
-        o[:order] = qualified_expression(o[:order], table) if o[:order]
-        SQL::Window.new(o)
-      when SQL::PlaceholderLiteralString
-        args = if e.args.is_a?(Hash)
-          h = {}
-          e.args.each{|k,v| h[k] = qualified_expression(v, table)}
-          h
-        else
-          qualified_expression(e.args, table)
-        end
-        SQL::PlaceholderLiteralString.new(e.str, args, e.parens)
-      else
-        e
-      end
+      Qualifier.new(self, table).transform(e)
     end
 
     # The order of methods to call to build the SELECT SQL statement

data/lib/sequel/exceptions.rb

@@ -44,6 +44,10 @@ module Sequel
   # and won't reraise it.
   class Rollback < Error ; end
 
+  # Exception that occurs when unbinding a dataset that has multiple different values
+  # for a given variable.
+  class UnbindDuplicate < Error; end
+
   class Error
     AdapterNotFound = Sequel::AdapterNotFound
     InvalidOperation = Sequel::InvalidOperation

data/lib/sequel/extensions/columns_introspection.rb

@@ -0,0 +1,61 @@
+# The columns_introspection extension attempts to introspect the
+# selected columns for a dataset before issuing a query.  If it
+# thinks it can guess correctly at the columns the query will use,
+# it will return the columns without issuing a database query.
+# This method is not fool-proof, it's possible that some databases
+# will use column names that Sequel does not expect.
+#
+# To enable this for a single dataset, extend the dataset with
+# Sequel::ColumnIntrospection.  To enable this for all datasets, run:
+#
+#   Sequel::Dataset.introspect_all_columns
+
+module Sequel
+  module ColumnsIntrospection
+    # Attempt to guess the columns that will be returned
+    # if there are columns selected, in order to skip a database
+    # query to retrieve the columns.  This should work with
+    # Symbols, SQL::Identifiers, SQL::QualifiedIdentifiers, and
+    # SQL::AliasedExpressions.
+    def columns
+      return @columns if @columns
+      return columns_without_introspection unless cols = opts[:select] and !cols.empty?
+      probable_columns = cols.map{|c| probable_column_name(c)}
+      if probable_columns.all?
+        @columns = probable_columns
+      else
+        columns_without_introspection
+      end
+    end
+
+    private
+
+    # Return the probable name of the column, or nil if one
+    # cannot be determined.
+    def probable_column_name(c)
+      case c
+      when Symbol
+        _, c, a = split_symbol(c)
+        (a || c).to_sym
+      when SQL::Identifier
+        c.value.to_sym
+      when SQL::QualifiedIdentifier
+        col = c.column
+        col.is_a?(SQL::Identifier) ? col.value.to_sym : col.to_sym
+      when SQL::AliasedExpression
+        a = c.aliaz
+        a.is_a?(SQL::Identifier) ? a.value.to_sym : a.to_sym
+      end
+    end
+  end
+
+  class Dataset
+    alias columns_without_introspection columns
+
+    # Enable column introspection for every dataset.
+    def self.introspect_all_columns
+      include ColumnsIntrospection
+      remove_method(:columns) if instance_methods(false).map{|x| x.to_s}.include?('columns')
+    end
+  end
+end

data/lib/sequel/extensions/migration.rb

@@ -302,7 +302,7 @@ module Sequel
   #
   # Part of the +migration+ extension.
   class Migrator
-    MIGRATION_FILE_PATTERN = /\A\d+_.+\.rb\z/i.freeze
+    MIGRATION_FILE_PATTERN = /\A(\d+)_.+\.rb\z/i.freeze
     MIGRATION_SPLITTER = '_'.freeze
     MINIMUM_TIMESTAMP = 20000101
 
@@ -375,7 +375,8 @@ module Sequel
       @db = db
       @directory = directory
       @files = get_migration_files
-      @table = opts[:table]  || self.class.const_get(:DEFAULT_SCHEMA_TABLE)
+      schema, table = @db.send(:schema_and_table, opts[:table]  || self.class.const_get(:DEFAULT_SCHEMA_TABLE))
+      @table = schema ? Sequel::SQL::QualifiedIdentifier.new(schema, table) : table
       @column = opts[:column] || self.class.const_get(:DEFAULT_SCHEMA_COLUMN)
       @ds = schema_dataset
     end
@@ -588,7 +589,7 @@ module Sequel
         next unless MIGRATION_FILE_PATTERN.match(file)
         files << File.join(directory, file)
       end
-      files.sort
+      files.sort_by{|f| MIGRATION_FILE_PATTERN.match(File.basename(f))[1].to_i}
     end
     
     # Returns tuples of migration, filename, and direction