sequel 3.21.0 → 3.36.0

data/lib/sequel/dataset/query.rb

@@ -1,7 +1,7 @@
 module Sequel
   class Dataset
     # ---------------------
-    # :section: Methods that return modified datasets
+    # :section: 1 - Methods that return modified datasets
     # These methods all return modified copies of the receiver.
     # ---------------------
 
@@ -28,12 +28,14 @@ module Sequel
     JOIN_METHODS = (CONDITIONED_JOIN_TYPES + UNCONDITIONED_JOIN_TYPES).map{|x| "#{x}_join".to_sym} + [:join, :join_table]
     
     # Methods that return modified datasets
-    QUERY_METHODS = %w'add_graph_aliases and distinct except exclude
-    filter for_update from from_self graph grep group group_and_count group_by having intersect invert
-    limit lock_style naked or order order_append order_by order_more order_prepend paginate qualify query
-    reverse reverse_order select select_all select_append select_more server
-    set_defaults set_graph_aliases set_overrides unfiltered ungraphed ungrouped union
-    unlimited unordered where with with_recursive with_sql'.collect{|x| x.to_sym} + JOIN_METHODS
+    QUERY_METHODS = (<<-METHS).split.map{|x| x.to_sym} + JOIN_METHODS
+      add_graph_aliases and distinct except exclude exclude_having exclude_where
+      filter for_update from from_self graph grep group group_and_count group_by having intersect invert
+      limit lock_style naked or order order_append order_by order_more order_prepend paginate qualify query
+      reverse reverse_order select select_all select_append select_group select_more server
+      set_defaults set_graph_aliases set_overrides unfiltered ungraphed ungrouped union
+      unlimited unordered where with with_recursive with_sql
+    METHS
 
     # Adds an further filter to an existing filter using AND. If no filter 
     # exists an error is raised. This method is identical to #filter except
@@ -80,7 +82,7 @@ module Sequel
     # :from_self :: Set to false to not wrap the returned dataset in a from_self, use with care.
     #
     #   DB[:items].except(DB[:other_items])
-    #   # SELECT * FROM items EXCEPT SELECT * FROM other_items
+    #   # SELECT * FROM (SELECT * FROM items EXCEPT SELECT * FROM other_items) AS t1
     #
     #   DB[:items].except(DB[:other_items], :all=>true, :from_self=>false)
     #   # SELECT * FROM items EXCEPT ALL SELECT * FROM other_items
@@ -103,12 +105,29 @@ module Sequel
     #   DB[:items].exclude(:category => 'software', :id=>3)
     #   # SELECT * FROM items WHERE ((category != 'software') OR (id != 3))
     def exclude(*cond, &block)
-      clause = (@opts[:having] ? :having : :where)
-      cond = cond.first if cond.size == 1
-      cond = filter_expr(cond, &block)
-      cond = SQL::BooleanExpression.invert(cond)
-      cond = SQL::BooleanExpression.new(:AND, @opts[clause], cond) if @opts[clause]
-      clone(clause => cond)
+      _filter_or_exclude(true, @opts[:having] ? :having : :where, *cond, &block)
+    end
+
+    # Inverts the given conditions and adds them to the HAVING clause.
+    #
+    #   DB[:items].select_group(:name).exclude_having{count(name) < 2}
+    #   # SELECT name FROM items GROUP BY name HAVING (count(name) >= 2)
+    def exclude_having(*cond, &block)
+      _filter_or_exclude(true, :having, *cond, &block)
+    end
+
+    # Inverts the given conditions and adds them to the WHERE clause.
+    #
+    #   DB[:items].select_group(:name).exclude_where(:category => 'software')
+    #   # SELECT * FROM items WHERE (category != 'software')
+    #
+    #   DB[:items].select_group(:name).
+    #     exclude_having{count(name) < 2}.
+    #     exclude_where(:category => 'software')
+    #   # SELECT name FROM items WHERE (category != 'software')
+    #   # GROUP BY name HAVING (count(name) >= 2)
+    def exclude_where(*cond, &block)
+      _filter_or_exclude(true, :where, *cond, &block)
     end
 
     # Returns a copy of the dataset with the given conditions imposed upon it.  
@@ -145,7 +164,7 @@ module Sequel
     #   DB[:items].filter('price < ?', 100)
     #   # SELECT * FROM items WHERE price < 100
     #
-    #   DB[:items].filter([[:id, (1,2,3)], [:id, 0..10]])
+    #   DB[:items].filter([[:id, [1,2,3]], [:id, 0..10]])
     #   # SELECT * FROM items WHERE ((id IN (1, 2, 3)) AND ((id >= 0) AND (id <= 10)))
     #
     #   DB[:items].filter('price < 100')
@@ -184,16 +203,22 @@ module Sequel
     def from(*source)
       table_alias_num = 0
       sources = []
+      ctes = nil
       source.each do |s|
         case s
         when Hash
           s.each{|k,v| sources << SQL::AliasedExpression.new(k,v)}
         when Dataset
+          if hoist_cte?(s)
+            ctes ||= []
+            ctes += s.opts[:with]
+            s = s.clone(:with=>nil)
+          end
           sources << SQL::AliasedExpression.new(s, dataset_alias(table_alias_num+=1))
         when Symbol
           sch, table, aliaz = split_symbol(s)
           if aliaz
-            s = sch ? SQL::QualifiedIdentifier.new(sch.to_sym, table.to_sym) : SQL::Identifier.new(table.to_sym)
+            s = sch ? SQL::QualifiedIdentifier.new(sch, table) : SQL::Identifier.new(table)
             sources << SQL::AliasedExpression.new(s, aliaz.to_sym)
           else
             sources << s
@@ -203,6 +228,7 @@ module Sequel
         end
       end
       o = {:from=>sources.empty? ? nil : sources}
+      o[:with] = (opts[:with] || []) + ctes if ctes
       o[:num_dataset_sources] = table_alias_num if table_alias_num > 0
       clone(o)
     end
@@ -270,21 +296,25 @@ module Sequel
     end
 
     # Returns a copy of the dataset with the results grouped by the value of 
-    # the given columns.
+    # the given columns.  If a block is given, it is treated
+    # as a virtual row block, similar to +filter+.
     #
     #   DB[:items].group(:id) # SELECT * FROM items GROUP BY id
     #   DB[:items].group(:id, :name) # SELECT * FROM items GROUP BY id, name
-    def group(*columns)
+    #   DB[:items].group{[a, sum(b)]} # SELECT * FROM items GROUP BY a, sum(b)
+    def group(*columns, &block)
+      virtual_row_columns(columns, block)
       clone(:group => (columns.compact.empty? ? nil : columns))
     end
 
     # Alias of group
-    def group_by(*columns)
-      group(*columns)
+    def group_by(*columns, &block)
+      group(*columns, &block)
     end
     
     # Returns a dataset grouped by the given column with count by group.
     # Column aliases may be supplied, and will be included in the select clause.
+    # If a block is given, it is treated as a virtual row block, similar to +filter+.
     #
     # Examples:
     #
@@ -299,8 +329,24 @@ module Sequel
     #   DB[:items].group_and_count(:first_name___name).all
     #   # SELECT first_name AS name, count(*) AS count FROM items GROUP BY first_name
     #   # => [{:name=>'a', :count=>1}, ...]
-    def group_and_count(*columns)
-      group(*columns.map{|c| unaliased_identifier(c)}).select(*(columns + [COUNT_OF_ALL_AS_COUNT]))
+    #
+    #   DB[:items].group_and_count{substr(first_name, 1, 1).as(initial)}.all
+    #   # SELECT substr(first_name, 1, 1) AS initial, count(*) AS count FROM items GROUP BY substr(first_name, 1, 1)
+    #   # => [{:initial=>'a', :count=>1}, ...]
+    def group_and_count(*columns, &block)
+      select_group(*columns, &block).select_more(COUNT_OF_ALL_AS_COUNT)
+    end
+
+    # Adds the appropriate CUBE syntax to GROUP BY.
+    def group_cube
+      raise Error, "GROUP BY CUBE not supported on #{db.database_type}" unless supports_group_cube?
+      clone(:group_options=>:cube)
+    end
+
+    # Adds the appropriate ROLLUP syntax to GROUP BY.
+    def group_rollup
+      raise Error, "GROUP BY ROLLUP not supported on #{db.database_type}" unless supports_group_rollup?
+      clone(:group_options=>:rollup)
     end
 
     # Returns a copy of the dataset with the HAVING conditions changed. See #filter for argument types.
@@ -356,7 +402,11 @@ module Sequel
       inner_join(*args, &block)
     end
 
-    # Returns a joined dataset.  Uses the following arguments:
+    # Returns a joined dataset.  Not usually called directly, users should use the
+    # appropriate join method (e.g. join, left_join, natural_join, cross_join) which fills
+    # in the +type+ argument.
+    #
+    # Takes the following arguments:
     #
     # * type - The type of join to do (e.g. :inner)
     # * table - Depends on type:
@@ -386,7 +436,29 @@ module Sequel
     #   in which case it yields the table alias/name for the table currently being joined,
     #   the table alias/name for the last joined (or first table), and an array of previous
     #   SQL::JoinClause. Unlike +filter+, this block is not treated as a virtual row block.
+    #
+    # Examples:
+    #
+    #   DB[:a].join_table(:cross, :b)
+    #   # SELECT * FROM a CROSS JOIN b
+    #
+    #   DB[:a].join_table(:inner, DB[:b], :c=>d)
+    #   # SELECT * FROM a INNER JOIN (SELECT * FROM b) AS t1 ON (t1.c = a.d)
+    #
+    #   DB[:a].join_table(:left, :b___c, [:d])
+    #   # SELECT * FROM a LEFT JOIN b AS c USING (d)
+    #
+    #   DB[:a].natural_join(:b).join_table(:inner, :c) do |ta, jta, js|
+    #     (:d.qualify(ta) > :e.qualify(jta)) & {:f.qualify(ta)=>DB.from(js.first.table).select(:g)}
+    #   end
+    #   # SELECT * FROM a NATURAL JOIN b INNER JOIN c
+    #   #   ON ((c.d > b.e) AND (c.f IN (SELECT g FROM b)))
     def join_table(type, table, expr=nil, options={}, &block)
+      if hoist_cte?(table)
+        s, ds = hoist_cte(table)
+        return s.join_table(type, ds, expr, options, &block)
+      end
+
       using_join = expr.is_a?(Array) && !expr.empty? && expr.all?{|x| x.is_a?(Symbol)}
       if using_join && !supports_join_using?
         h = {}
@@ -431,6 +503,7 @@ module Sequel
             v = qualified_column_name(v, last_alias) if v.is_a?(Symbol)
             [k,v]
           end
+          expr = SQL::BooleanExpression.from_value_pairs(expr)
         end
         if block
           expr2 = yield(table_name, last_alias, @opts[:join] || [])
@@ -531,7 +604,7 @@ module Sequel
     #   DB[:items].order{sum(name).desc} # SELECT * FROM items ORDER BY sum(name) DESC
     #   DB[:items].order(nil) # SELECT * FROM items
     def order(*columns, &block)
-      columns += Array(Sequel.virtual_row(&block)) if block
+      virtual_row_columns(columns, block)
       clone(:order => (columns.compact.empty?) ? nil : columns)
     end
     
@@ -565,7 +638,7 @@ module Sequel
       @opts[:order] ? ds.order_more(*@opts[:order]) : ds
     end
     
-    # Qualify to the given table, or first source if not table is given.
+    # Qualify to the given table, or first source if no table is given.
     #
     #   DB[:items].filter(:id=>1).qualify
     #   # SELECT items.* FROM items WHERE (items.id = 1)
@@ -606,6 +679,18 @@ module Sequel
       qualify_to(first_source)
     end
     
+    # Modify the RETURNING clause, only supported on a few databases.  If returning
+    # is used, instead of insert returning the autogenerated primary key or
+    # update/delete returning the number of modified rows, results are
+    # returned using +fetch_rows+.
+    #
+    #   DB[:items].returning # RETURNING *
+    #   DB[:items].returning(nil) # RETURNING NULL
+    #   DB[:items].returning(:id, :name) # RETURNING id, name
+    def returning(*values)
+      clone(:returning=>values)
+    end
+
     # Returns a copy of the dataset with the order reversed. If no order is
     # given, the existing order is inverted.
     #
@@ -629,7 +714,7 @@ module Sequel
     #   DB[:items].select(:a, :b) # SELECT a, b FROM items
     #   DB[:items].select{[a, sum(b)]} # SELECT a, sum(b) FROM items
     def select(*columns, &block)
-      columns += Array(Sequel.virtual_row(&block)) if block
+      virtual_row_columns(columns, block)
       m = []
       columns.each do |i|
         i.is_a?(Hash) ? m.concat(i.map{|k, v| SQL::AliasedExpression.new(k,v)}) : m << i
@@ -637,11 +722,19 @@ module Sequel
       clone(:select => m)
     end
     
-    # Returns a copy of the dataset selecting the wildcard.
+    # Returns a copy of the dataset selecting the wildcard if no arguments
+    # are given.  If arguments are given, treat them as tables and select
+    # all columns (using the wildcard) from each table.
     #
     #   DB[:items].select(:a).select_all # SELECT * FROM items
-    def select_all
-      clone(:select => nil)
+    #   DB[:items].select_all(:items) # SELECT items.* FROM items
+    #   DB[:items].select_all(:items, :foo) # SELECT items.*, foo.* FROM items
+    def select_all(*tables)
+      if tables.empty?
+        clone(:select => nil)
+      else
+        select(*tables.map{|t| i, a = split_alias(t); a || i}.map{|t| SQL::ColumnAll.new(t)})
+      end
     end
     
     # Returns a copy of the dataset with the given columns added
@@ -653,10 +746,29 @@ module Sequel
     #   DB[:items].select_append(:b) # SELECT *, b FROM items
     def select_append(*columns, &block)
       cur_sel = @opts[:select]
-      cur_sel = [WILDCARD] if !cur_sel || cur_sel.empty?
+      if !cur_sel || cur_sel.empty?
+        unless supports_select_all_and_column?
+          return select_all(*(Array(@opts[:from]) + Array(@opts[:join]))).select_more(*columns, &block)
+        end
+        cur_sel = [WILDCARD]
+      end
       select(*(cur_sel + columns), &block)
     end
 
+    # Set both the select and group clauses with the given +columns+.
+    # Column aliases may be supplied, and will be included in the select clause.
+    # This also takes a virtual row block similar to +filter+.
+    #
+    #   DB[:items].select_group(:a, :b)
+    #   # SELECT a, b FROM items GROUP BY a, b
+    #
+    #   DB[:items].select_group(:c___a){f(c2)}
+    #   # SELECT c AS a, f(c2) FROM items GROUP BY c, f(c2)
+    def select_group(*columns, &block)
+      virtual_row_columns(columns, block)
+      select(*columns).group(*columns.map{|c| unaliased_identifier(c)})
+    end
+
     # Returns a copy of the dataset with the given columns added
     # to the existing selected columns. If no columns are currently selected
     # it will just select the columns given. 
@@ -670,7 +782,7 @@ module Sequel
     end
     
     # Set the server for this dataset to use.  Used to pick a specific database
-    # shard to run a query against, or to override the default (which is SELECT uses
+    # shard to run a query against, or to override the default (where SELECT uses
     # :read_only database and all other queries use the :default database).  This
     # method is always available but is only useful when database sharding is being
     # used.
@@ -702,6 +814,22 @@ module Sequel
       clone(:overrides=>hash.merge(@opts[:overrides]||{}))
     end
     
+    # Unbind bound variables from this dataset's filter and return an array of two
+    # objects.  The first object is a modified dataset where the filter has been
+    # replaced with one that uses bound variable placeholders.  The second object
+    # is the hash of unbound variables.  You can then prepare and execute (or just
+    # call) the dataset with the bound variables to get results.
+    #
+    #   ds, bv = DB[:items].filter(:a=>1).unbind
+    #   ds # SELECT * FROM items WHERE (a = $a)
+    #   bv #  {:a => 1}
+    #   ds.call(:select, bv)
+    def unbind
+      u = Unbinder.new
+      ds = clone(:where=>u.transform(opts[:where]), :join=>u.transform(opts[:join]))
+      [ds, u.binds]
+    end
+
     # Returns a copy of the dataset with no filters (HAVING or WHERE clause) applied.
     # 
     #   DB[:items].group(:a).having(:a=>1).where(:b).unfiltered
@@ -726,8 +854,8 @@ module Sequel
     # :all :: Set to true to use UNION ALL instead of UNION, so duplicate rows can occur
     # :from_self :: Set to false to not wrap the returned dataset in a from_self, use with care.
     #
-    #   DB[:items].union(DB[:other_items]).sql
-    #   #=> "SELECT * FROM items UNION SELECT * FROM other_items"
+    #   DB[:items].union(DB[:other_items])
+    #   # SELECT * FROM (SELECT * FROM items UNION SELECT * FROM other_items) AS t1
     #
     #   DB[:items].union(DB[:other_items], :all=>true, :from_self=>false)
     #   # SELECT * FROM items UNION ALL SELECT * FROM other_items
@@ -774,7 +902,12 @@ module Sequel
     #   # WITH items AS (SELECT * FROM syx WHERE (name LIKE 'A%')) SELECT * FROM items
     def with(name, dataset, opts={})
       raise(Error, 'This datatset does not support common table expressions') unless supports_cte?
-      clone(:with=>(@opts[:with]||[]) + [opts.merge(:name=>name, :dataset=>dataset)])
+      if hoist_cte?(dataset)
+        s, ds = hoist_cte(dataset)
+        s.with(name, ds, opts)
+      else
+        clone(:with=>(@opts[:with]||[]) + [opts.merge(:name=>name, :dataset=>dataset)])
+      end
     end
 
     # Add a recursive common table expression (CTE) with the given name, a dataset that
@@ -796,20 +929,50 @@ module Sequel
     #   # SELECT i AS id, pi AS parent_id FROM t
     def with_recursive(name, nonrecursive, recursive, opts={})
       raise(Error, 'This datatset does not support common table expressions') unless supports_cte?
-      clone(:with=>(@opts[:with]||[]) + [opts.merge(:recursive=>true, :name=>name, :dataset=>nonrecursive.union(recursive, {:all=>opts[:union_all] != false, :from_self=>false}))])
+      if hoist_cte?(nonrecursive)
+        s, ds = hoist_cte(nonrecursive)
+        s.with_recursive(name, ds, recursive, opts)
+      elsif hoist_cte?(recursive)
+        s, ds = hoist_cte(recursive)
+        s.with_recursive(name, nonrecursive, ds, opts)
+      else
+        clone(:with=>(@opts[:with]||[]) + [opts.merge(:recursive=>true, :name=>name, :dataset=>nonrecursive.union(recursive, {:all=>opts[:union_all] != false, :from_self=>false}))])
+      end
     end
     
     # Returns a copy of the dataset with the static SQL used.  This is useful if you want
     # to keep the same row_proc/graph, but change the SQL used to custom SQL.
     #
     #   DB[:items].with_sql('SELECT * FROM foo') # SELECT * FROM foo
+    #
+    # You can use placeholders in your SQL and provide arguments for those placeholders:
+    #
+    #   DB[:items].with_sql('SELECT ? FROM foo', 1) # SELECT 1 FROM foo
+    #
+    # You can also provide a method name and arguments to call to get the SQL:
+    #
+    #   DB[:items].with_sql(:insert_sql, :b=>1) # INSERT INTO items (b) VALUES (1)
     def with_sql(sql, *args)
-      sql = SQL::PlaceholderLiteralString.new(sql, args) unless args.empty?
+      if sql.is_a?(Symbol)
+        sql = send(sql, *args)
+      else
+        sql = SQL::PlaceholderLiteralString.new(sql, args) unless args.empty?
+      end
       clone(:sql=>sql)
     end
     
     protected
 
+    # Add the dataset to the list of compounds
+    def compound_clone(type, dataset, opts)
+      if hoist_cte?(dataset)
+        s, ds = hoist_cte(dataset)
+        return s.compound_clone(type, ds, opts)
+      end
+      ds = compound_from_self.clone(:compounds=>Array(@opts[:compounds]).map{|x| x.dup} + [[type, dataset.compound_from_self, opts[:all]]])
+      opts[:from_self] == false ? ds : ds.from_self(opts)
+    end
+
     # Return true if the dataset has a non-nil value for any key in opts.
     def options_overlap(opts)
       !(@opts.collect{|k,v| k unless v.nil?}.compact & opts).empty?
@@ -823,22 +986,22 @@ module Sequel
 
     private
 
-    # Internal filter method so it works on either the having or where clauses.
-    def _filter(clause, *cond, &block)
+    # Internal filter/exclude method so it works on either the having or where clauses.
+    def _filter_or_exclude(invert, clause, *cond, &block)
       cond = cond.first if cond.size == 1
       if cond.respond_to?(:empty?) && cond.empty? && !block
         clone
       else
         cond = filter_expr(cond, &block)
+        cond = SQL::BooleanExpression.invert(cond) if invert
         cond = SQL::BooleanExpression.new(:AND, @opts[clause], cond) if @opts[clause]
         clone(clause => cond)
       end
     end
-    
-    # Add the dataset to the list of compounds
-    def compound_clone(type, dataset, opts)
-      ds = compound_from_self.clone(:compounds=>Array(@opts[:compounds]).map{|x| x.dup} + [[type, dataset.compound_from_self, opts[:all]]])
-      opts[:from_self] == false ? ds : ds.from_self(opts)
+
+    # Internal filter method so it works on either the having or where clauses.
+    def _filter(clause, *cond, &block)
+      _filter_or_exclude(false, clause, *cond, &block)
     end
 
     # SQL expression object based on the expr type.  See +filter+.
@@ -867,7 +1030,13 @@ module Sequel
       when Symbol, SQL::Expression
         expr
       when TrueClass, FalseClass
-        SQL::BooleanExpression.new(:NOOP, expr)
+        if supports_where_true?
+          SQL::BooleanExpression.new(:NOOP, expr)
+        elsif expr
+          SQL::Constants::SQLTRUE
+        else
+          SQL::Constants::SQLFALSE
+        end
       when String
         LiteralString.new("(#{expr})")
       else
@@ -875,6 +1044,18 @@ module Sequel
       end
     end
     
+    # Return two datasets, the first a clone of the receiver with the WITH
+    # clause from the given dataset added to it, and the second a clone of
+    # the given dataset with the WITH clause removed.
+    def hoist_cte(ds)
+      [clone(:with => (opts[:with] || []) + ds.opts[:with]), ds.clone(:with => nil)]
+    end
+
+    # Whether CTEs need to be hoisted from the given ds into the current ds.
+    def hoist_cte?(ds)
+      ds.is_a?(Dataset) && ds.opts[:with] && !supports_cte_in_subqueries?
+    end
+
     # Inverts the given order by breaking it into a list of column references
     # and inverting them.
     #
@@ -892,5 +1073,17 @@ module Sequel
         end
       end
     end
+
+    # Return self if the dataset already has a server, or a cloned dataset with the
+    # default server otherwise.
+    def default_server
+      @opts[:server] ? self : clone(:server=>:default)
+    end
+
+    # Treat the +block+ as a virtual_row block if not +nil+ and
+    # add the resulting columns to the +columns+ array (modifies +columns+).
+    def virtual_row_columns(columns, block)
+      columns.concat(Array(Sequel.virtual_row(&block))) if block
+    end
   end
 end