sequel 3.1.0 → 3.2.0

data/lib/sequel/dataset/sql.rb

@@ -7,6 +7,7 @@ module Sequel
     COLUMN_REF_RE2 = /\A([\w ]+)___([\w ]+)\z/.freeze
     COLUMN_REF_RE3 = /\A([\w ]+)__([\w ]+)\z/.freeze
     COUNT_FROM_SELF_OPTS = [:distinct, :group, :sql, :limit, :compounds]
+    DATASET_ALIAS_BASE_NAME = 't'.freeze
     INSERT_SQL_BASE="INSERT INTO ".freeze
     IS_LITERALS = {nil=>'NULL'.freeze, true=>'TRUE'.freeze, false=>'FALSE'.freeze}.freeze
     IS_OPERATORS = ::Sequel::SQL::ComplexExpression::IS_OPERATORS
@@ -15,9 +16,10 @@ module Sequel
     QUALIFY_KEYS = [:select, :where, :having, :order, :group]
     QUESTION_MARK = '?'.freeze
     STOCK_COUNT_OPTS = {:select => [SQL::AliasedExpression.new(LiteralString.new("COUNT(*)").freeze, :count)], :order => nil}.freeze
-    SELECT_CLAUSE_ORDER = %w'distinct columns from join where group having compounds order limit'.freeze
+    SELECT_CLAUSE_ORDER = %w'with distinct columns from join where group having compounds order limit'.freeze
     TWO_ARITY_OPERATORS = ::Sequel::SQL::ComplexExpression::TWO_ARITY_OPERATORS
     WILDCARD = '*'.freeze
+    SQL_WITH = "WITH ".freeze
 
     # Adds an further filter to an existing filter using AND. If no filter 
     # exists an error is raised. This method is identical to #filter except
@@ -63,8 +65,15 @@ module Sequel
     def complex_expression_sql(op, args)
       case op
       when *IS_OPERATORS
-        v = IS_LITERALS[args.at(1)] || raise(Error, 'Invalid argument used for IS operator')
-        "(#{literal(args.at(0))} #{op} #{v})"
+        r = args.at(1)
+        if r.nil? || supports_is_true?
+          raise(InvalidOperation, 'Invalid argument used for IS operator') unless v = IS_LITERALS[r]
+          "(#{literal(args.at(0))} #{op} #{v})"
+        elsif op == :IS
+          complex_expression_sql(:"=", args)
+        else
+          complex_expression_sql(:OR, [SQL::BooleanExpression.new(:"!=", *args), SQL::BooleanExpression.new(:IS, args.at(0), nil)])
+        end
       when *TWO_ARITY_OPERATORS
         "(#{literal(args.at(0))} #{op} #{literal(args.at(1))})"
       when *N_ARITY_OPERATORS
@@ -76,7 +85,7 @@ module Sequel
       when :'B~'
         "~#{literal(args.at(0))}"
       else
-        raise(Sequel::Error, "invalid operator #{op}")
+        raise(InvalidOperation, "invalid operator #{op}")
       end
     end
 
@@ -113,21 +122,31 @@ module Sequel
     # The DISTINCT clause is used to remove duplicate rows from the
     # output.  If arguments are provided, uses a DISTINCT ON clause,
     # in which case it will only be distinct on those columns, instead
-    # of all returned columns.
+    # of all returned columns.  Raises an error if arguments
+    # are given and DISTINCT ON is not supported.
     #
     #  dataset.distinct # SQL: SELECT DISTINCT * FROM items
     #  dataset.order(:id).distinct(:id) # SQL: SELECT DISTINCT ON (id) * FROM items ORDER BY id
     def distinct(*args)
+      raise(InvalidOperation, "DISTINCT ON not supported") if !args.empty? && !supports_distinct_on?
       clone(:distinct => args)
     end
 
-    # Adds an EXCEPT clause using a second dataset object. If all is true the
-    # clause used is EXCEPT ALL, which may return duplicate rows.
+    # Adds an EXCEPT clause using a second dataset object.
+    # An EXCEPT compound dataset returns all rows in the current dataset
+    # that are not in the given dataset.
+    # Raises an InvalidOperation if the operation is not supported.
+    # Options:
+    # * :all - Set to true to use EXCEPT ALL instead of EXCEPT, so duplicate rows can occur
+    # * :from_self - Set to false to not wrap the returned dataset in a from_self, use with care.
     #
     #   DB[:items].except(DB[:other_items]).sql
     #   #=> "SELECT * FROM items EXCEPT SELECT * FROM other_items"
-    def except(dataset, all = false)
-      compound_clone(:except, dataset, all)
+    def except(dataset, opts={})
+      opts = {:all=>opts} unless opts.is_a?(Hash)
+      raise(InvalidOperation, "EXCEPT not supported") unless supports_intersect_except?
+      raise(InvalidOperation, "EXCEPT ALL not supported") if opts[:all] && !supports_intersect_except_all?
+      compound_clone(:except, dataset, opts)
     end
 
     # Performs the inverse of Dataset#filter.
@@ -204,14 +223,14 @@ module Sequel
 
     # The first source (primary table) for this dataset.  If the dataset doesn't
     # have a table, raises an error.  If the table is aliased, returns the aliased name.
-    def first_source
+    def first_source_alias
       source = @opts[:from]
       if source.nil? || source.empty?
         raise Error, 'No source specified for query'
       end
       case s = source.first
-      when Hash
-        s.values.first
+      when SQL::AliasedExpression
+        s.aliaz
       when Symbol
         sch, table, aliaz = split_symbol(s)
         aliaz ? aliaz.to_sym : s
@@ -219,6 +238,7 @@ module Sequel
         s
       end
     end
+    alias first_source first_source_alias
 
     # Returns a copy of the dataset with the source changed.
     #
@@ -226,9 +246,31 @@ module Sequel
     #   dataset.from(:blah) # SQL: SELECT * FROM blah
     #   dataset.from(:blah, :foo) # SQL: SELECT * FROM blah, foo
     def from(*source)
-      clone(:from=>source.empty? ? nil : source)
+      table_alias_num = 0
+      sources = []
+      source.each do |s|
+        case s
+        when Hash
+          s.each{|k,v| sources << SQL::AliasedExpression.new(k,v)}
+        when Dataset
+          sources << SQL::AliasedExpression.new(s, dataset_alias(table_alias_num+=1))
+        when Symbol
+          sch, table, aliaz = split_symbol(s)
+          if aliaz
+            s = sch ? SQL::QualifiedIdentifier.new(sch.to_sym, table.to_sym) : SQL::Identifier.new(table.to_sym)
+            sources << SQL::AliasedExpression.new(s, aliaz.to_sym)
+          else
+            sources << s
+          end
+        else
+          sources << s
+        end
+      end
+      o = {:from=>sources.empty? ? nil : sources}
+      o[:num_dataset_sources] = table_alias_num if table_alias_num > 0
+      clone(o)
     end
-    
+
     # Returns a dataset selecting from the current dataset.
     #
     #   ds = DB[:items].order(:name)
@@ -237,8 +279,7 @@ module Sequel
     def from_self
       fs = {}
       @opts.keys.each{|k| fs[k] = nil} 
-      fs[:from] = [self]
-      clone(fs)
+      clone(fs).from(self)
     end
 
     # SQL fragment specifying an SQL function call
@@ -319,7 +360,7 @@ module Sequel
         if values.empty?
           insert_default_values_sql
         else
-          "#{insert_sql_base}#{from} VALUES #{literal(values)}"
+          "#{insert_sql_base}#{from} VALUES #{literal(values)}#{insert_sql_suffix}"
         end
       when Hash
         values = @opts[:defaults].merge(values) if @opts[:defaults]
@@ -332,20 +373,28 @@ module Sequel
             fl << literal(String === k ? k.to_sym : k)
             vl << literal(v)
           end
-          "#{insert_sql_base}#{from} (#{fl.join(COMMA_SEPARATOR)}) VALUES (#{vl.join(COMMA_SEPARATOR)})"
+          "#{insert_sql_base}#{from} (#{fl.join(COMMA_SEPARATOR)}) VALUES (#{vl.join(COMMA_SEPARATOR)})#{insert_sql_suffix}"
         end
       when Dataset
-        "#{insert_sql_base}#{from} #{literal(values)}"
+        "#{insert_sql_base}#{from} #{literal(values)}#{insert_sql_suffix}"
       end
     end
     
-    # Adds an INTERSECT clause using a second dataset object. If all is true 
-    # the clause used is INTERSECT ALL, which may return duplicate rows.
+    # Adds an INTERSECT clause using a second dataset object.
+    # An INTERSECT compound dataset returns all rows in both the current dataset
+    # and the given dataset.
+    # Raises an InvalidOperation if the operation is not supported.
+    # Options:
+    # * :all - Set to true to use INTERSECT ALL instead of INTERSECT, so duplicate rows can occur
+    # * :from_self - Set to false to not wrap the returned dataset in a from_self, use with care.
     #
     #   DB[:items].intersect(DB[:other_items]).sql
     #   #=> "SELECT * FROM items INTERSECT SELECT * FROM other_items"
-    def intersect(dataset, all = false)
-      compound_clone(:intersect, dataset, all)
+    def intersect(dataset, opts={})
+      opts = {:all=>opts} unless opts.is_a?(Hash)
+      raise(InvalidOperation, "INTERSECT not supported") unless supports_intersect_except?
+      raise(InvalidOperation, "INTERSECT ALL not supported") if opts[:all] && !supports_intersect_except_all?
+      compound_clone(:intersect, dataset, opts)
     end
 
     # Inverts the current filter
@@ -421,7 +470,7 @@ module Sequel
       if Dataset === table
         if table_alias.nil?
           table_alias_num = (@opts[:num_dataset_sources] || 0) + 1
-          table_alias = "t#{table_alias_num}"
+          table_alias = dataset_alias(table_alias_num)
         end
         table_name = table_alias
       else
@@ -435,7 +484,7 @@ module Sequel
         raise(Sequel::Error, "can't use a block if providing an array of symbols as expr") if block_given?
         SQL::JoinUsingClause.new(expr, type, table, table_alias)
       else
-        last_alias ||= @opts[:last_joined_table] || (first_source.is_a?(Dataset) ? 't1' : first_source)
+        last_alias ||= @opts[:last_joined_table] || first_source_alias
         if Sequel.condition_specifier?(expr)
           expr = expr.collect do |k, v|
             k = qualified_column_name(k, table_name) if k.is_a?(Symbol)
@@ -597,6 +646,11 @@ module Sequel
       [qcr.table, qcr.column].map{|x| [SQL::QualifiedIdentifier, SQL::Identifier, Symbol].any?{|c| x.is_a?(c)} ? literal(x) : quote_identifier(x)}.join('.')
     end
 
+    # Qualify to the given table, or first source if not table is given.
+    def qualify(table=first_source)
+      qualify_to(table)
+    end
+
     # Return a copy of the dataset with unqualified identifiers in the
     # SELECT, WHERE, GROUP, HAVING, and ORDER clauses qualified by the
     # given table. If no columns are currently selected, select all
@@ -730,13 +784,18 @@ module Sequel
       clone(:where => nil, :having => nil)
     end
 
-    # Adds a UNION clause using a second dataset object. If all is true the
-    # clause used is UNION ALL, which may return duplicate rows.
+    # Adds a UNION clause using a second dataset object.
+    # A UNION compound dataset returns all rows in either the current dataset
+    # or the given dataset.
+    # Options:
+    # * :all - Set to true to use UNION ALL instead of UNION, so duplicate rows can occur
+    # * :from_self - Set to false to not wrap the returned dataset in a from_self, use with care.
     #
     #   DB[:items].union(DB[:other_items]).sql
     #   #=> "SELECT * FROM items UNION SELECT * FROM other_items"
-    def union(dataset, all = false)
-      compound_clone(:union, dataset, all)
+    def union(dataset, opts={})
+      opts = {:all=>opts} unless opts.is_a?(Hash)
+      compound_clone(:union, dataset, opts)
     end
 
     # Returns a copy of the dataset with no order.
@@ -783,7 +842,7 @@ module Sequel
 
       sql
     end
-
+    
     # Add a condition to the WHERE clause.  See #filter for argument types.
     #
     #   dataset.group(:a).having(:a).filter(:b) # SELECT * FROM items GROUP BY a HAVING a AND b
@@ -792,6 +851,50 @@ module Sequel
       _filter(:where, *cond, &block)
     end
 
+    # The SQL fragment for the given window's options.
+    def window_sql(opts)
+      raise(Error, 'This dataset does not support window functions') unless supports_window_functions?
+      window = literal(opts[:window]) if opts[:window]
+      partition = "PARTITION BY #{expression_list(Array(opts[:partition]))}" if opts[:partition]
+      order = "ORDER BY #{expression_list(Array(opts[:order]))}" if opts[:order]
+      frame = case opts[:frame]
+        when nil
+          nil
+        when :all
+          "ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING"
+        when :rows
+          "ROWS UNBOUNDED PRECEDING"
+        else
+          raise Error, "invalid window frame clause, should be :all, :rows, or nil"
+      end
+      "(#{[window, partition, order, frame].compact.join(' ')})"
+    end
+
+    # The SQL fragment for the given window function's function and window.
+    def window_function_sql(function, window)
+      "#{literal(function)} OVER #{literal(window)}"
+    end
+    
+    # Add a simple common table expression (CTE) with the given name and a dataset that defines the CTE.
+    # A common table expression acts as an inline view for the query.
+    # Options:
+    # * :args - Specify the arguments/columns for the CTE, should be an array of symbols.
+    # * :recursive - Specify that this is a recursive CTE
+    def with(name, dataset, opts={})
+      raise(Error, 'This datatset does not support common table expressions') unless supports_cte?
+      clone(:with=>(@opts[:with]||[]) + [opts.merge(:name=>name, :dataset=>dataset)])
+    end
+
+    # Add a recursive common table expression (CTE) with the given name, a dataset that
+    # defines the nonrecursive part of the CTE, and a dataset that defines the recursive part
+    # of the CTE.  Options:
+    # * :args - Specify the arguments/columns for the CTE, should be an array of symbols.
+    # * :union_all - Set to false to use UNION instead of UNION ALL combining the nonrecursive and recursive parts.
+    def with_recursive(name, nonrecursive, recursive, opts={})
+      raise(Error, 'This datatset does not support common table expressions') unless supports_cte?
+      clone(:with=>(@opts[:with]||[]) + [opts.merge(:recursive=>true, :name=>name, :dataset=>nonrecursive.union(recursive, {:all=>opts[:union_all] != false, :from_self=>false}))])
+    end
+
     # Returns a copy of the dataset with the static SQL used.  This is useful if you want
     # to keep the same row_proc/graph, but change the SQL used to custom SQL.
     #
@@ -814,13 +917,6 @@ module Sequel
       (@opts[:limit] || @opts[:order]) ? from_self : self
     end
     
-    # Returns a table reference for use in the FROM clause.  Returns an SQL subquery
-    # frgament with an optional table alias.
-    def to_table_reference(table_alias=nil)
-      s = "(#{sql})"
-      table_alias ? as_sql(s, table_alias) : s
-    end
-
     private
 
     # Internal filter method so it works on either the having or where clauses.
@@ -830,6 +926,11 @@ module Sequel
       cond = SQL::BooleanExpression.new(:AND, @opts[clause], cond) if @opts[clause]
       clone(clause => cond)
     end
+    
+    # Do a simple join of the arguments (which should be strings or symbols) separated by commas
+    def argument_list(args)
+      args.join(COMMA_SEPARATOR)
+    end
 
     # SQL fragment for specifying an alias.  expression should already be literalized.
     def as_sql(expression, aliaz)
@@ -843,10 +944,16 @@ module Sequel
     end
     
     # Add the dataset to the list of compounds
-    def compound_clone(type, dataset, all)
-      compound_from_self.clone(:compounds=>Array(@opts[:compounds]).map{|x| x.dup} + [[type, dataset.compound_from_self, all]]).from_self
+    def compound_clone(type, dataset, opts)
+      ds = compound_from_self.clone(:compounds=>Array(@opts[:compounds]).map{|x| x.dup} + [[type, dataset.compound_from_self, opts[:all]]])
+      opts[:from_self] == false ? ds : ds.from_self
     end
 
+    # The alias to use for datasets, takes a number to make sure the name is unique.
+    def dataset_alias(number)
+      :"#{DATASET_ALIAS_BASE_NAME}#{number}"
+    end
+    
     # Converts an array of expressions into a comma separated string of
     # expressions.
     def expression_list(columns)
@@ -902,6 +1009,11 @@ module Sequel
       "#{insert_sql_base}#{source_list(@opts[:from])} DEFAULT VALUES"
     end
 
+    # SQL statement for end of an INSERT statement
+    def insert_sql_suffix
+      nil
+    end
+
     # Inverts the given order by breaking it into a list of column references
     # and inverting them.
     #
@@ -950,12 +1062,12 @@ module Sequel
 
     # SQL fragment for Date, using the ISO8601 format.
     def literal_date(v)
-      "'#{v}'"
+      requires_sql_standard_datetimes? ? v.strftime("DATE '%Y-%m-%d'") : "'#{v}'"
     end
 
     # SQL fragment for DateTime, using the ISO8601 format.
     def literal_datetime(v)
-      "'#{v}'"
+      requires_sql_standard_datetimes? ? v.strftime("TIMESTAMP '%Y-%m-%d %H:%M:%S'") : "'#{v}'"
     end
 
     # SQL fragment for SQL::Expression, result depends on the specific type of expression.
@@ -1009,7 +1121,7 @@ module Sequel
 
     # SQL fragment for Time, uses the ISO8601 format.
     def literal_time(v)
-      "'#{v.iso8601}'"
+      requires_sql_standard_datetimes? ? v.strftime("TIMESTAMP '%Y-%m-%d %H:%M:%S'") : "'#{v.iso8601}'"
     end
 
     # SQL fragment for true.
@@ -1074,6 +1186,15 @@ module Sequel
         SQL::SQLArray.new(qualified_expression(e.array, table))
       when SQL::Subscript 
         SQL::Subscript.new(qualified_expression(e.f, table), qualified_expression(e.sub, table))
+      when SQL::WindowFunction
+        SQL::WindowFunction.new(qualified_expression(e.function, table), qualified_expression(e.window, table))
+      when SQL::Window
+        o = e.opts.dup
+        o[:partition] = qualified_expression(o[:partition], table) if o[:partition]
+        o[:order] = qualified_expression(o[:order], table) if o[:order]
+        SQL::Window.new(o)
+      when SQL::PlaceholderLiteralString
+        SQL::PlaceholderLiteralString.new(e.str, qualified_expression(e.args, table), e.parens)
       else
         e
       end
@@ -1142,23 +1263,21 @@ module Sequel
     def select_where_sql(sql)
       sql << " WHERE #{literal(@opts[:where])}" if @opts[:where]
     end
+    
+    def select_with_sql(sql)
+      ws = opts[:with]
+      return if !ws || ws.empty?
+      sql.replace("#{select_with_sql_base}#{ws.map{|w| "#{w[:name]}#{"(#{argument_list(w[:args])})" if w[:args]} AS (#{subselect_sql(w[:dataset])})"}.join(COMMA_SEPARATOR)} #{sql}")
+    end
+    
+    def select_with_sql_base
+      SQL_WITH
+    end
 
     # Converts an array of source names into into a comma separated list.
     def source_list(source)
-      if source.nil? || source.empty?
-        raise Error, 'No source specified for query'
-      end
-      auto_alias_count = @opts[:num_dataset_sources] || 0
-      m = source.map do |s|
-        case s
-        when Dataset
-          auto_alias_count += 1
-          s.to_table_reference("t#{auto_alias_count}")
-        else
-          table_ref(s)
-        end
-      end
-      m.join(COMMA_SEPARATOR)
+      raise(Error, 'No source specified for query') if source.nil? || source.empty?
+      source.map{|s| table_ref(s)}.join(COMMA_SEPARATOR)
     end
     
     # Splits the symbol into three parts.  Each part will
@@ -1193,18 +1312,7 @@ module Sequel
 
     # SQL fragment specifying a table name.
     def table_ref(t)
-      case t
-      when Symbol
-        literal_symbol(t)
-      when Dataset
-        t.to_table_reference
-      when Hash
-        t.map{|k, v| as_sql(table_ref(k), v)}.join(COMMA_SEPARATOR)
-      when String
-        quote_identifier(t)
-      else
-        literal(t)
-      end
+      t.is_a?(String) ? quote_identifier(t) : literal(t)
     end
   end
 end

data/lib/sequel/extensions/migration.rb

@@ -131,27 +131,40 @@ module Sequel
   #
   #   Sequel::Migrator.apply(DB, '.', 5, 1)
   module Migrator
+    DEFAULT_SCHEMA_COLUMN = :version
+    DEFAULT_SCHEMA_TABLE = :schema_info
     MIGRATION_FILE_PATTERN = /\A\d+_.+\.rb\z/.freeze
     MIGRATION_SPLITTER = '_'.freeze
 
-    # Migrates the supplied database in the specified directory from the
-    # current version to the target version. If no current version is
-    # supplied, it is extracted from a schema_info table. The schema_info
-    # table is automatically created and maintained by the apply function.
+    # Wrapper for run, maintaining backwards API compatibility
     def self.apply(db, directory, target = nil, current = nil)
-      # determine current and target version and direction
-      current ||= get_current_migration_version(db)
-      target ||= latest_migration_version(directory)
-      raise Error, "No current version available" if current.nil?
-      raise Error, "No target version available" if target.nil?
+      run(db, directory, :target => target, :current => current)
+    end
+
+    # Migrates the supplied database using the migration files in the the specified directory. Options:
+    # * :column - The column in the :table argument storing the migration version (default: :version).
+    # * :current - The current version of the database.  If not given, it is retrieved from the database
+    #   using the :table and :column options.
+    # * :table - The table containing the schema version (default: :schema_info).
+    # * :target - The target version to which to migrate.  If not given, migrates to the maximum version.
+    #
+    # Examples: 
+    #   Sequel::Migrator.run(DB, "migrations")
+    #   Sequel::Migrator.run(DB, "migrations", :target=>15, :current=>10)
+    #   Sequel::Migrator.run(DB, "app1/migrations", :column=> :app2_version)
+    #   Sequel::Migrator.run(DB, "app2/migrations", :column => :app2_version, :table=>:schema_info2)
+    def self.run(db, directory, opts={})
+      raise(Error, "Must supply a valid migration path") unless directory and File.directory?(directory)
+      raise(Error, "No current version available") unless current = opts[:current] || get_current_migration_version(db, opts)
+      raise(Error, "No target version available") unless target  = opts[:target]  || latest_migration_version(directory)
 
       direction = current < target ? :up : :down
       
       classes = migration_classes(directory, target, current, direction)
-      
+
       db.transaction do
         classes.each {|c| c.apply(db, direction)}
-        set_current_migration_version(db, target)
+        set_current_migration_version(db, target, opts)
       end
       
       target
@@ -159,9 +172,8 @@ module Sequel
 
     # Gets the current migration version stored in the database. If no version
     # number is stored, 0 is returned.
-    def self.get_current_migration_version(db)
-      r = schema_info_dataset(db).first
-      r ? r[:version] : 0
+    def self.get_current_migration_version(db, opts={})
+      (schema_info_dataset(db, opts).first || {})[opts[:column] || DEFAULT_SCHEMA_COLUMN] || 0
     end
 
     # Returns the latest version available in the specified directory.
@@ -203,19 +215,23 @@ module Sequel
     
     # Returns the dataset for the schema_info table. If no such table
     # exists, it is automatically created.
-    def self.schema_info_dataset(db)
-      db.create_table(:schema_info) {integer :version} unless db.table_exists?(:schema_info)
-      db[:schema_info]
+    def self.schema_info_dataset(db, opts={})
+      column = opts[:column] || DEFAULT_SCHEMA_COLUMN
+      table  = opts[:table]  || DEFAULT_SCHEMA_TABLE
+      db.create_table?(table){Integer column}
+      db.alter_table(table){add_column column, Integer} unless db.from(table).columns.include?(column)
+      db.from(table)
     end
     
     # Sets the current migration  version stored in the database.
-    def self.set_current_migration_version(db, version)
-      dataset = schema_info_dataset(db)
-      dataset.send(dataset.first ? :update : :<<, :version => version)
+    def self.set_current_migration_version(db, version, opts={})
+      column = opts[:column] || DEFAULT_SCHEMA_COLUMN
+      dataset = schema_info_dataset(db, opts)
+      dataset.send(dataset.first ? :update : :insert, column => version)
     end
 
     # Return the integer migration version based on the filename.
-    def self.migration_version_from_file(filename)
+    def self.migration_version_from_file(filename) # :nodoc:
       filename.split(MIGRATION_SPLITTER, 2).first.to_i
     end
     private_class_method :migration_version_from_file