sequel 3.21.0 → 3.41.0

data/lib/sequel/dataset/query.rb

@@ -1,10 +1,15 @@
 module Sequel
   class Dataset
     # ---------------------
-    # :section: Methods that return modified datasets
+    # :section: 1 - Methods that return modified datasets
     # These methods all return modified copies of the receiver.
     # ---------------------
 
+    # Hash of extension name symbols to callable objects to load the extension
+    # into the Dataset object (usually by extending it with a module defined
+    # in the extension).
+    EXTENSIONS = {}
+
     # The dataset options that require the removal of cached columns
     # if changed.
     COLUMN_CHANGE_OPTS = [:select, :sql, :from, :join].freeze
@@ -28,12 +33,35 @@ module Sequel
     JOIN_METHODS = (CONDITIONED_JOIN_TYPES + UNCONDITIONED_JOIN_TYPES).map{|x| "#{x}_join".to_sym} + [:join, :join_table]
     
     # Methods that return modified datasets
-    QUERY_METHODS = %w'add_graph_aliases and distinct except exclude
-    filter for_update from from_self graph grep group group_and_count group_by having intersect invert
-    limit lock_style naked or order order_append order_by order_more order_prepend paginate qualify query
-    reverse reverse_order select select_all select_append select_more server
-    set_defaults set_graph_aliases set_overrides unfiltered ungraphed ungrouped union
-    unlimited unordered where with with_recursive with_sql'.collect{|x| x.to_sym} + JOIN_METHODS
+    QUERY_METHODS = (<<-METHS).split.map{|x| x.to_sym} + JOIN_METHODS
+      add_graph_aliases and distinct except exclude exclude_having exclude_where
+      filter for_update from from_self graph grep group group_and_count group_by having intersect invert
+      limit lock_style naked or order order_append order_by order_more order_prepend paginate qualify query
+      reverse reverse_order select select_all select_append select_group select_more server
+      set_defaults set_graph_aliases set_overrides unfiltered ungraphed ungrouped union
+      unlimited unordered where with with_recursive with_sql
+    METHS
+
+    # Register an extension callback for Dataset objects.  ext should be the
+    # extension name symbol, and mod should either be a Module that the
+    # dataset is extended with, or a callable object called with the database
+    # object.  If mod is not provided, a block can be provided and is treated
+    # as the mod object.
+    #
+    # If mod is a module, this also registers a Database extension that will
+    # extend all of the database's datasets.
+    def self.register_extension(ext, mod=nil, &block)
+      if mod
+        raise(Error, "cannot provide both mod and block to Dataset.register_extension") if block
+        if mod.is_a?(Module)
+          block = proc{|ds| ds.extend(mod)}
+          Sequel::Database.register_extension(ext){|db| db.extend_datasets(mod)}
+        else
+          block = mod
+        end
+      end
+      Sequel.synchronize{EXTENSIONS[ext] = block}
+    end
 
     # Adds an further filter to an existing filter using AND. If no filter 
     # exists an error is raised. This method is identical to #filter except
@@ -80,7 +108,7 @@ module Sequel
     # :from_self :: Set to false to not wrap the returned dataset in a from_self, use with care.
     #
     #   DB[:items].except(DB[:other_items])
-    #   # SELECT * FROM items EXCEPT SELECT * FROM other_items
+    #   # SELECT * FROM (SELECT * FROM items EXCEPT SELECT * FROM other_items) AS t1
     #
     #   DB[:items].except(DB[:other_items], :all=>true, :from_self=>false)
     #   # SELECT * FROM items EXCEPT ALL SELECT * FROM other_items
@@ -103,12 +131,51 @@ module Sequel
     #   DB[:items].exclude(:category => 'software', :id=>3)
     #   # SELECT * FROM items WHERE ((category != 'software') OR (id != 3))
     def exclude(*cond, &block)
-      clause = (@opts[:having] ? :having : :where)
-      cond = cond.first if cond.size == 1
-      cond = filter_expr(cond, &block)
-      cond = SQL::BooleanExpression.invert(cond)
-      cond = SQL::BooleanExpression.new(:AND, @opts[clause], cond) if @opts[clause]
-      clone(clause => cond)
+      _filter_or_exclude(true, @opts[:having] ? :having : :where, *cond, &block)
+    end
+
+    # Inverts the given conditions and adds them to the HAVING clause.
+    #
+    #   DB[:items].select_group(:name).exclude_having{count(name) < 2}
+    #   # SELECT name FROM items GROUP BY name HAVING (count(name) >= 2)
+    def exclude_having(*cond, &block)
+      _filter_or_exclude(true, :having, *cond, &block)
+    end
+
+    # Inverts the given conditions and adds them to the WHERE clause.
+    #
+    #   DB[:items].select_group(:name).exclude_where(:category => 'software')
+    #   # SELECT * FROM items WHERE (category != 'software')
+    #
+    #   DB[:items].select_group(:name).
+    #     exclude_having{count(name) < 2}.
+    #     exclude_where(:category => 'software')
+    #   # SELECT name FROM items WHERE (category != 'software')
+    #   # GROUP BY name HAVING (count(name) >= 2)
+    def exclude_where(*cond, &block)
+      _filter_or_exclude(true, :where, *cond, &block)
+    end
+
+    # Return a clone of the dataset loaded with the extensions, see #extension!.
+    def extension(*exts)
+      clone.extension!(*exts)
+    end
+
+    # Load an extension into the receiver.  In addition to requiring the extension file, this
+    # also modifies the dataset to work with the extension (usually extending it with a
+    # module defined in the extension file).  If no related extension file exists or the
+    # extension does not have specific support for Database objects, an Error will be raised.
+    # Returns self.
+    def extension!(*exts)
+      Sequel.extension(*exts)
+      exts.each do |ext|
+        if pr = Sequel.synchronize{EXTENSIONS[ext]}
+          pr.call(self)
+        else
+          raise(Error, "Extension #{ext} does not have specific support handling individual datasets")
+        end
+      end
+      self
     end
 
     # Returns a copy of the dataset with the given conditions imposed upon it.  
@@ -145,7 +212,7 @@ module Sequel
     #   DB[:items].filter('price < ?', 100)
     #   # SELECT * FROM items WHERE price < 100
     #
-    #   DB[:items].filter([[:id, (1,2,3)], [:id, 0..10]])
+    #   DB[:items].filter([[:id, [1,2,3]], [:id, 0..10]])
     #   # SELECT * FROM items WHERE ((id IN (1, 2, 3)) AND ((id >= 0) AND (id <= 10)))
     #
     #   DB[:items].filter('price < 100')
@@ -184,16 +251,22 @@ module Sequel
     def from(*source)
       table_alias_num = 0
       sources = []
+      ctes = nil
       source.each do |s|
         case s
         when Hash
           s.each{|k,v| sources << SQL::AliasedExpression.new(k,v)}
         when Dataset
+          if hoist_cte?(s)
+            ctes ||= []
+            ctes += s.opts[:with]
+            s = s.clone(:with=>nil)
+          end
           sources << SQL::AliasedExpression.new(s, dataset_alias(table_alias_num+=1))
         when Symbol
           sch, table, aliaz = split_symbol(s)
           if aliaz
-            s = sch ? SQL::QualifiedIdentifier.new(sch.to_sym, table.to_sym) : SQL::Identifier.new(table.to_sym)
+            s = sch ? SQL::QualifiedIdentifier.new(sch, table) : SQL::Identifier.new(table)
             sources << SQL::AliasedExpression.new(s, aliaz.to_sym)
           else
             sources << s
@@ -203,6 +276,7 @@ module Sequel
         end
       end
       o = {:from=>sources.empty? ? nil : sources}
+      o[:with] = (opts[:with] || []) + ctes if ctes
       o[:num_dataset_sources] = table_alias_num if table_alias_num > 0
       clone(o)
     end
@@ -270,21 +344,25 @@ module Sequel
     end
 
     # Returns a copy of the dataset with the results grouped by the value of 
-    # the given columns.
+    # the given columns.  If a block is given, it is treated
+    # as a virtual row block, similar to +filter+.
     #
     #   DB[:items].group(:id) # SELECT * FROM items GROUP BY id
     #   DB[:items].group(:id, :name) # SELECT * FROM items GROUP BY id, name
-    def group(*columns)
+    #   DB[:items].group{[a, sum(b)]} # SELECT * FROM items GROUP BY a, sum(b)
+    def group(*columns, &block)
+      virtual_row_columns(columns, block)
       clone(:group => (columns.compact.empty? ? nil : columns))
     end
 
     # Alias of group
-    def group_by(*columns)
-      group(*columns)
+    def group_by(*columns, &block)
+      group(*columns, &block)
     end
     
     # Returns a dataset grouped by the given column with count by group.
     # Column aliases may be supplied, and will be included in the select clause.
+    # If a block is given, it is treated as a virtual row block, similar to +filter+.
     #
     # Examples:
     #
@@ -299,8 +377,24 @@ module Sequel
     #   DB[:items].group_and_count(:first_name___name).all
     #   # SELECT first_name AS name, count(*) AS count FROM items GROUP BY first_name
     #   # => [{:name=>'a', :count=>1}, ...]
-    def group_and_count(*columns)
-      group(*columns.map{|c| unaliased_identifier(c)}).select(*(columns + [COUNT_OF_ALL_AS_COUNT]))
+    #
+    #   DB[:items].group_and_count{substr(first_name, 1, 1).as(initial)}.all
+    #   # SELECT substr(first_name, 1, 1) AS initial, count(*) AS count FROM items GROUP BY substr(first_name, 1, 1)
+    #   # => [{:initial=>'a', :count=>1}, ...]
+    def group_and_count(*columns, &block)
+      select_group(*columns, &block).select_more(COUNT_OF_ALL_AS_COUNT)
+    end
+
+    # Adds the appropriate CUBE syntax to GROUP BY.
+    def group_cube
+      raise Error, "GROUP BY CUBE not supported on #{db.database_type}" unless supports_group_cube?
+      clone(:group_options=>:cube)
+    end
+
+    # Adds the appropriate ROLLUP syntax to GROUP BY.
+    def group_rollup
+      raise Error, "GROUP BY ROLLUP not supported on #{db.database_type}" unless supports_group_rollup?
+      clone(:group_options=>:rollup)
     end
 
     # Returns a copy of the dataset with the HAVING conditions changed. See #filter for argument types.
@@ -356,12 +450,15 @@ module Sequel
       inner_join(*args, &block)
     end
 
-    # Returns a joined dataset.  Uses the following arguments:
+    # Returns a joined dataset.  Not usually called directly, users should use the
+    # appropriate join method (e.g. join, left_join, natural_join, cross_join) which fills
+    # in the +type+ argument.
+    #
+    # Takes the following arguments:
     #
     # * type - The type of join to do (e.g. :inner)
     # * table - Depends on type:
     #   * Dataset - a subselect is performed with an alias of tN for some value of N
-    #   * Model (or anything responding to :table_name) - table.table_name
     #   * String, Symbol: table
     # * expr - specifies conditions, depends on type:
     #   * Hash, Array of two element arrays - Assumes key (1st arg) is column of joined table (unless already
@@ -382,11 +479,36 @@ module Sequel
     #     to the same table more than once.  No alias is used by default.
     #   * :implicit_qualifier - The name to use for qualifying implicit conditions.  By default,
     #     the last joined or primary table is used.
+    #   * :qualify - Can be set to false to not do any implicit qualification.  Can be set
+    #     to :deep to use the Qualifier AST Transformer, which will attempt to qualify
+    #     subexpressions of the expression tree.
     # * block - The block argument should only be given if a JOIN with an ON clause is used,
     #   in which case it yields the table alias/name for the table currently being joined,
     #   the table alias/name for the last joined (or first table), and an array of previous
     #   SQL::JoinClause. Unlike +filter+, this block is not treated as a virtual row block.
+    #
+    # Examples:
+    #
+    #   DB[:a].join_table(:cross, :b)
+    #   # SELECT * FROM a CROSS JOIN b
+    #
+    #   DB[:a].join_table(:inner, DB[:b], :c=>d)
+    #   # SELECT * FROM a INNER JOIN (SELECT * FROM b) AS t1 ON (t1.c = a.d)
+    #
+    #   DB[:a].join_table(:left, :b___c, [:d])
+    #   # SELECT * FROM a LEFT JOIN b AS c USING (d)
+    #
+    #   DB[:a].natural_join(:b).join_table(:inner, :c) do |ta, jta, js|
+    #     (Sequel.qualify(ta, :d) > Sequel.qualify(jta, :e)) & {Sequel.qualify(ta, :f)=>DB.from(js.first.table).select(:g)}
+    #   end
+    #   # SELECT * FROM a NATURAL JOIN b INNER JOIN c
+    #   #   ON ((c.d > b.e) AND (c.f IN (SELECT g FROM b)))
     def join_table(type, table, expr=nil, options={}, &block)
+      if hoist_cte?(table)
+        s, ds = hoist_cte(table)
+        return s.join_table(type, ds, expr, options, &block)
+      end
+
       using_join = expr.is_a?(Array) && !expr.empty? && expr.all?{|x| x.is_a?(Symbol)}
       if using_join && !supports_join_using?
         h = {}
@@ -398,6 +520,7 @@ module Sequel
       when Hash
         table_alias = options[:table_alias]
         last_alias = options[:implicit_qualifier]
+        qualify_type = options[:qualify]
       when Symbol, String, SQL::Identifier
         table_alias = options
         last_alias = nil 
@@ -412,7 +535,6 @@ module Sequel
         end
         table_name = table_alias
       else
-        table = table.table_name if table.respond_to?(:table_name)
         table, implicit_table_alias = split_alias(table)
         table_alias ||= implicit_table_alias
         table_name = table_alias || table
@@ -427,10 +549,19 @@ module Sequel
         last_alias ||= @opts[:last_joined_table] || first_source_alias
         if Sequel.condition_specifier?(expr)
           expr = expr.collect do |k, v|
-            k = qualified_column_name(k, table_name) if k.is_a?(Symbol)
-            v = qualified_column_name(v, last_alias) if v.is_a?(Symbol)
+            case qualify_type
+            when false
+              nil # Do no qualification
+            when :deep
+              k = Sequel::Qualifier.new(self, table_name).transform(k)
+              v = Sequel::Qualifier.new(self, last_alias).transform(v)
+            else
+              k = qualified_column_name(k, table_name) if k.is_a?(Symbol)
+              v = qualified_column_name(v, last_alias) if v.is_a?(Symbol)
+            end
             [k,v]
           end
+          expr = SQL::BooleanExpression.from_value_pairs(expr)
         end
         if block
           expr2 = yield(table_name, last_alias, @opts[:join] || [])
@@ -461,7 +592,7 @@ module Sequel
     #   DB[:items].limit(10...20) # SELECT * FROM items LIMIT 10 OFFSET 10
     #   DB[:items].limit(10..20) # SELECT * FROM items LIMIT 11 OFFSET 10
     #   DB[:items].limit(nil, 20) # SELECT * FROM items OFFSET 20
-    def limit(l, o = nil)
+    def limit(l, o = (no_offset = true; nil))
       return from_self.limit(l, o) if @opts[:sql]
 
       if Range === l
@@ -479,6 +610,8 @@ module Sequel
           raise(Error, 'Offsets must be greater than or equal to 0') unless o >= 0
         end
         opts[:offset] = o
+      elsif !no_offset
+        opts[:offset] = nil
       end
       clone(opts)
     end
@@ -524,14 +657,14 @@ module Sequel
     #
     #   DB[:items].order(:name) # SELECT * FROM items ORDER BY name
     #   DB[:items].order(:a, :b) # SELECT * FROM items ORDER BY a, b
-    #   DB[:items].order('a + b'.lit) # SELECT * FROM items ORDER BY a + b
+    #   DB[:items].order(Sequel.lit('a + b')) # SELECT * FROM items ORDER BY a + b
     #   DB[:items].order(:a + :b) # SELECT * FROM items ORDER BY (a + b)
-    #   DB[:items].order(:name.desc) # SELECT * FROM items ORDER BY name DESC
-    #   DB[:items].order(:name.asc(:nulls=>:last)) # SELECT * FROM items ORDER BY name ASC NULLS LAST
+    #   DB[:items].order(Sequel.desc(:name)) # SELECT * FROM items ORDER BY name DESC
+    #   DB[:items].order(Sequel.asc(:name, :nulls=>:last)) # SELECT * FROM items ORDER BY name ASC NULLS LAST
     #   DB[:items].order{sum(name).desc} # SELECT * FROM items ORDER BY sum(name) DESC
     #   DB[:items].order(nil) # SELECT * FROM items
     def order(*columns, &block)
-      columns += Array(Sequel.virtual_row(&block)) if block
+      virtual_row_columns(columns, block)
       clone(:order => (columns.compact.empty?) ? nil : columns)
     end
     
@@ -565,7 +698,7 @@ module Sequel
       @opts[:order] ? ds.order_more(*@opts[:order]) : ds
     end
     
-    # Qualify to the given table, or first source if not table is given.
+    # Qualify to the given table, or first source if no table is given.
     #
     #   DB[:items].filter(:id=>1).qualify
     #   # SELECT items.* FROM items WHERE (items.id = 1)
@@ -606,19 +739,33 @@ module Sequel
       qualify_to(first_source)
     end
     
+    # Modify the RETURNING clause, only supported on a few databases.  If returning
+    # is used, instead of insert returning the autogenerated primary key or
+    # update/delete returning the number of modified rows, results are
+    # returned using +fetch_rows+.
+    #
+    #   DB[:items].returning # RETURNING *
+    #   DB[:items].returning(nil) # RETURNING NULL
+    #   DB[:items].returning(:id, :name) # RETURNING id, name
+    def returning(*values)
+      clone(:returning=>values)
+    end
+
     # Returns a copy of the dataset with the order reversed. If no order is
     # given, the existing order is inverted.
     #
     #   DB[:items].reverse(:id) # SELECT * FROM items ORDER BY id DESC
+    #   DB[:items].reverse{foo(bar)} # SELECT * FROM items ORDER BY foo(bar) DESC
     #   DB[:items].order(:id).reverse # SELECT * FROM items ORDER BY id DESC
-    #   DB[:items].order(:id).reverse(:name.asc) # SELECT * FROM items ORDER BY name ASC
-    def reverse(*order)
+    #   DB[:items].order(:id).reverse(Sequel.desc(:name)) # SELECT * FROM items ORDER BY name ASC
+    def reverse(*order, &block)
+      virtual_row_columns(order, block)
       order(*invert_order(order.empty? ? @opts[:order] : order))
     end
 
     # Alias of +reverse+
-    def reverse_order(*order)
-      reverse(*order)
+    def reverse_order(*order, &block)
+      reverse(*order, &block)
     end
 
     # Returns a copy of the dataset with the columns selected changed
@@ -629,7 +776,7 @@ module Sequel
     #   DB[:items].select(:a, :b) # SELECT a, b FROM items
     #   DB[:items].select{[a, sum(b)]} # SELECT a, sum(b) FROM items
     def select(*columns, &block)
-      columns += Array(Sequel.virtual_row(&block)) if block
+      virtual_row_columns(columns, block)
       m = []
       columns.each do |i|
         i.is_a?(Hash) ? m.concat(i.map{|k, v| SQL::AliasedExpression.new(k,v)}) : m << i
@@ -637,11 +784,19 @@ module Sequel
       clone(:select => m)
     end
     
-    # Returns a copy of the dataset selecting the wildcard.
+    # Returns a copy of the dataset selecting the wildcard if no arguments
+    # are given.  If arguments are given, treat them as tables and select
+    # all columns (using the wildcard) from each table.
     #
     #   DB[:items].select(:a).select_all # SELECT * FROM items
-    def select_all
-      clone(:select => nil)
+    #   DB[:items].select_all(:items) # SELECT items.* FROM items
+    #   DB[:items].select_all(:items, :foo) # SELECT items.*, foo.* FROM items
+    def select_all(*tables)
+      if tables.empty?
+        clone(:select => nil)
+      else
+        select(*tables.map{|t| i, a = split_alias(t); a || i}.map{|t| SQL::ColumnAll.new(t)})
+      end
     end
     
     # Returns a copy of the dataset with the given columns added
@@ -653,10 +808,29 @@ module Sequel
     #   DB[:items].select_append(:b) # SELECT *, b FROM items
     def select_append(*columns, &block)
       cur_sel = @opts[:select]
-      cur_sel = [WILDCARD] if !cur_sel || cur_sel.empty?
+      if !cur_sel || cur_sel.empty?
+        unless supports_select_all_and_column?
+          return select_all(*(Array(@opts[:from]) + Array(@opts[:join]))).select_more(*columns, &block)
+        end
+        cur_sel = [WILDCARD]
+      end
       select(*(cur_sel + columns), &block)
     end
 
+    # Set both the select and group clauses with the given +columns+.
+    # Column aliases may be supplied, and will be included in the select clause.
+    # This also takes a virtual row block similar to +filter+.
+    #
+    #   DB[:items].select_group(:a, :b)
+    #   # SELECT a, b FROM items GROUP BY a, b
+    #
+    #   DB[:items].select_group(:c___a){f(c2)}
+    #   # SELECT c AS a, f(c2) FROM items GROUP BY c, f(c2)
+    def select_group(*columns, &block)
+      virtual_row_columns(columns, block)
+      select(*columns).group(*columns.map{|c| unaliased_identifier(c)})
+    end
+
     # Returns a copy of the dataset with the given columns added
     # to the existing selected columns. If no columns are currently selected
     # it will just select the columns given. 
@@ -670,7 +844,7 @@ module Sequel
     end
     
     # Set the server for this dataset to use.  Used to pick a specific database
-    # shard to run a query against, or to override the default (which is SELECT uses
+    # shard to run a query against, or to override the default (where SELECT uses
     # :read_only database and all other queries use the :default database).  This
     # method is always available but is only useful when database sharding is being
     # used.
@@ -702,6 +876,22 @@ module Sequel
       clone(:overrides=>hash.merge(@opts[:overrides]||{}))
     end
     
+    # Unbind bound variables from this dataset's filter and return an array of two
+    # objects.  The first object is a modified dataset where the filter has been
+    # replaced with one that uses bound variable placeholders.  The second object
+    # is the hash of unbound variables.  You can then prepare and execute (or just
+    # call) the dataset with the bound variables to get results.
+    #
+    #   ds, bv = DB[:items].filter(:a=>1).unbind
+    #   ds # SELECT * FROM items WHERE (a = $a)
+    #   bv #  {:a => 1}
+    #   ds.call(:select, bv)
+    def unbind
+      u = Unbinder.new
+      ds = clone(:where=>u.transform(opts[:where]), :join=>u.transform(opts[:join]))
+      [ds, u.binds]
+    end
+
     # Returns a copy of the dataset with no filters (HAVING or WHERE clause) applied.
     # 
     #   DB[:items].group(:a).having(:a=>1).where(:b).unfiltered
@@ -726,8 +916,8 @@ module Sequel
     # :all :: Set to true to use UNION ALL instead of UNION, so duplicate rows can occur
     # :from_self :: Set to false to not wrap the returned dataset in a from_self, use with care.
     #
-    #   DB[:items].union(DB[:other_items]).sql
-    #   #=> "SELECT * FROM items UNION SELECT * FROM other_items"
+    #   DB[:items].union(DB[:other_items])
+    #   # SELECT * FROM (SELECT * FROM items UNION SELECT * FROM other_items) AS t1
     #
     #   DB[:items].union(DB[:other_items], :all=>true, :from_self=>false)
     #   # SELECT * FROM items UNION ALL SELECT * FROM other_items
@@ -774,7 +964,12 @@ module Sequel
     #   # WITH items AS (SELECT * FROM syx WHERE (name LIKE 'A%')) SELECT * FROM items
     def with(name, dataset, opts={})
       raise(Error, 'This datatset does not support common table expressions') unless supports_cte?
-      clone(:with=>(@opts[:with]||[]) + [opts.merge(:name=>name, :dataset=>dataset)])
+      if hoist_cte?(dataset)
+        s, ds = hoist_cte(dataset)
+        s.with(name, ds, opts)
+      else
+        clone(:with=>(@opts[:with]||[]) + [opts.merge(:name=>name, :dataset=>dataset)])
+      end
     end
 
     # Add a recursive common table expression (CTE) with the given name, a dataset that
@@ -783,33 +978,62 @@ module Sequel
     # :args :: Specify the arguments/columns for the CTE, should be an array of symbols.
     # :union_all :: Set to false to use UNION instead of UNION ALL combining the nonrecursive and recursive parts.
     #
-    #   DB[:t].select(:i___id, :pi___parent_id).
-    #    with_recursive(:t,
-    #                   DB[:i1].filter(:parent_id=>nil),
-    #                   DB[:t].join(:t, :i=>:parent_id).select(:i1__id, :i1__parent_id),
-    #                   :args=>[:i, :pi])
-    #   # WITH RECURSIVE t(i, pi) AS (
-    #   #   SELECT * FROM i1 WHERE (parent_id IS NULL)
+    #   DB[:t].with_recursive(:t,
+    #     DB[:i1].select(:id, :parent_id).filter(:parent_id=>nil),
+    #     DB[:i1].join(:t, :id=>:parent_id).select(:i1__id, :i1__parent_id),
+    #     :args=>[:id, :parent_id])
+    #   
+    #   # WITH RECURSIVE "t"("id", "parent_id") AS (
+    #   #   SELECT "id", "parent_id" FROM "i1" WHERE ("parent_id" IS NULL)
     #   #   UNION ALL
-    #   #   SELECT i1.id, i1.parent_id FROM t INNER JOIN t ON (t.i = t.parent_id)
-    #   # )
-    #   # SELECT i AS id, pi AS parent_id FROM t
+    #   #   SELECT "i1"."id", "i1"."parent_id" FROM "i1" INNER JOIN "t" ON ("t"."id" = "i1"."parent_id")
+    #   # ) SELECT * FROM "t"
     def with_recursive(name, nonrecursive, recursive, opts={})
       raise(Error, 'This datatset does not support common table expressions') unless supports_cte?
-      clone(:with=>(@opts[:with]||[]) + [opts.merge(:recursive=>true, :name=>name, :dataset=>nonrecursive.union(recursive, {:all=>opts[:union_all] != false, :from_self=>false}))])
+      if hoist_cte?(nonrecursive)
+        s, ds = hoist_cte(nonrecursive)
+        s.with_recursive(name, ds, recursive, opts)
+      elsif hoist_cte?(recursive)
+        s, ds = hoist_cte(recursive)
+        s.with_recursive(name, nonrecursive, ds, opts)
+      else
+        clone(:with=>(@opts[:with]||[]) + [opts.merge(:recursive=>true, :name=>name, :dataset=>nonrecursive.union(recursive, {:all=>opts[:union_all] != false, :from_self=>false}))])
+      end
     end
     
     # Returns a copy of the dataset with the static SQL used.  This is useful if you want
     # to keep the same row_proc/graph, but change the SQL used to custom SQL.
     #
     #   DB[:items].with_sql('SELECT * FROM foo') # SELECT * FROM foo
+    #
+    # You can use placeholders in your SQL and provide arguments for those placeholders:
+    #
+    #   DB[:items].with_sql('SELECT ? FROM foo', 1) # SELECT 1 FROM foo
+    #
+    # You can also provide a method name and arguments to call to get the SQL:
+    #
+    #   DB[:items].with_sql(:insert_sql, :b=>1) # INSERT INTO items (b) VALUES (1)
     def with_sql(sql, *args)
-      sql = SQL::PlaceholderLiteralString.new(sql, args) unless args.empty?
+      if sql.is_a?(Symbol)
+        sql = send(sql, *args)
+      else
+        sql = SQL::PlaceholderLiteralString.new(sql, args) unless args.empty?
+      end
       clone(:sql=>sql)
     end
     
     protected
 
+    # Add the dataset to the list of compounds
+    def compound_clone(type, dataset, opts)
+      if hoist_cte?(dataset)
+        s, ds = hoist_cte(dataset)
+        return s.compound_clone(type, ds, opts)
+      end
+      ds = compound_from_self.clone(:compounds=>Array(@opts[:compounds]).map{|x| x.dup} + [[type, dataset.compound_from_self, opts[:all]]])
+      opts[:from_self] == false ? ds : ds.from_self(opts)
+    end
+
     # Return true if the dataset has a non-nil value for any key in opts.
     def options_overlap(opts)
       !(@opts.collect{|k,v| k unless v.nil?}.compact & opts).empty?
@@ -823,22 +1047,22 @@ module Sequel
 
     private
 
-    # Internal filter method so it works on either the having or where clauses.
-    def _filter(clause, *cond, &block)
+    # Internal filter/exclude method so it works on either the having or where clauses.
+    def _filter_or_exclude(invert, clause, *cond, &block)
       cond = cond.first if cond.size == 1
       if cond.respond_to?(:empty?) && cond.empty? && !block
         clone
       else
         cond = filter_expr(cond, &block)
+        cond = SQL::BooleanExpression.invert(cond) if invert
         cond = SQL::BooleanExpression.new(:AND, @opts[clause], cond) if @opts[clause]
         clone(clause => cond)
       end
     end
-    
-    # Add the dataset to the list of compounds
-    def compound_clone(type, dataset, opts)
-      ds = compound_from_self.clone(:compounds=>Array(@opts[:compounds]).map{|x| x.dup} + [[type, dataset.compound_from_self, opts[:all]]])
-      opts[:from_self] == false ? ds : ds.from_self(opts)
+
+    # Internal filter method so it works on either the having or where clauses.
+    def _filter(clause, *cond, &block)
+      _filter_or_exclude(false, clause, *cond, &block)
     end
 
     # SQL expression object based on the expr type.  See +filter+.
@@ -867,7 +1091,13 @@ module Sequel
       when Symbol, SQL::Expression
         expr
       when TrueClass, FalseClass
-        SQL::BooleanExpression.new(:NOOP, expr)
+        if supports_where_true?
+          SQL::BooleanExpression.new(:NOOP, expr)
+        elsif expr
+          SQL::Constants::SQLTRUE
+        else
+          SQL::Constants::SQLFALSE
+        end
       when String
         LiteralString.new("(#{expr})")
       else
@@ -875,11 +1105,23 @@ module Sequel
       end
     end
     
+    # Return two datasets, the first a clone of the receiver with the WITH
+    # clause from the given dataset added to it, and the second a clone of
+    # the given dataset with the WITH clause removed.
+    def hoist_cte(ds)
+      [clone(:with => (opts[:with] || []) + ds.opts[:with]), ds.clone(:with => nil)]
+    end
+
+    # Whether CTEs need to be hoisted from the given ds into the current ds.
+    def hoist_cte?(ds)
+      ds.is_a?(Dataset) && ds.opts[:with] && !supports_cte_in_subqueries?
+    end
+
     # Inverts the given order by breaking it into a list of column references
     # and inverting them.
     #
-    #   DB[:items].invert_order([:id.desc]]) #=> [:id]
-    #   DB[:items].invert_order(:category, :price.desc]) #=> [:category.desc, :price]
+    #   DB[:items].invert_order([Sequel.desc(:id)]]) #=> [Sequel.asc(:id)]
+    #   DB[:items].invert_order([:category, Sequel.desc(:price)]) #=> [Sequel.desc(:category), Sequel.asc(:price)]
     def invert_order(order)
       return nil unless order
       new_order = []
@@ -892,5 +1134,17 @@ module Sequel
         end
       end
     end
+
+    # Return self if the dataset already has a server, or a cloned dataset with the
+    # default server otherwise.
+    def default_server
+      @opts[:server] ? self : clone(:server=>:default)
+    end
+
+    # Treat the +block+ as a virtual_row block if not +nil+ and
+    # add the resulting columns to the +columns+ array (modifies +columns+).
+    def virtual_row_columns(columns, block)
+      columns.concat(Array(Sequel.virtual_row(&block))) if block
+    end
   end
 end