sequel 4.26.0 → 5.37.0

data/lib/sequel/dataset.rb

@@ -1,30 +1,38 @@
+# frozen-string-literal: true
+
 module Sequel
-  # A dataset represents an SQL query, or more generally, an abstract
-  # set of rows in the database.  Datasets
-  # can be used to create, retrieve, update and delete records.
+  # A dataset represents an SQL query. Datasets
+  # can be used to select, insert, update and delete records.
   # 
   # Query results are always retrieved on demand, so a dataset can be kept
   # around and reused indefinitely (datasets never cache results):
   #
-  #   my_posts = DB[:posts].filter(:author => 'david') # no records are retrieved
+  #   my_posts = DB[:posts].where(author: 'david') # no records are retrieved
   #   my_posts.all # records are retrieved
   #   my_posts.all # records are retrieved again
   #
-  # Most dataset methods return modified copies of the dataset (functional style), so you can
-  # reuse different datasets to access data:
+  # Datasets are frozen and use a functional style where modification methods
+  # return modified copies of the the dataset.  This allows you to reuse
+  # datasets:
   #
   #   posts = DB[:posts]
-  #   davids_posts = posts.filter(:author => 'david')
-  #   old_posts = posts.filter('stamp < ?', Date.today - 7)
-  #   davids_old_posts = davids_posts.filter('stamp < ?', Date.today - 7)
+  #   davids_posts = posts.where(author: 'david')
+  #   old_posts = posts.where{stamp < Date.today - 7}
+  #   davids_old_posts = davids_posts.where{stamp < Date.today - 7}
   #
-  # Datasets are Enumerable objects, so they can be manipulated using any
-  # of the Enumerable methods, such as map, inject, etc.
+  # Datasets are Enumerable objects, so they can be manipulated using many
+  # of the Enumerable methods, such as +map+ and +inject+.  Note that there are some methods
+  # that Dataset defines that override methods defined in Enumerable and result in different
+  # behavior, such as +select+ and +group_by+.
   #
   # For more information, see the {"Dataset Basics" guide}[rdoc-ref:doc/dataset_basics.rdoc].
   class Dataset
     OPTS = Sequel::OPTS
 
+    # Whether Dataset#freeze can actually freeze datasets.  True only on ruby 2.4+,
+    # as it requires clone(freeze: false)
+    TRUE_FREEZE = RUBY_VERSION >= '2.4'
+
     include Enumerable
     include SQL::AliasMethods
     include SQL::BooleanMethods
@@ -36,5 +44,13 @@ module Sequel
     include SQL::StringMethods
   end
   
-  require(%w"query actions features graph prepared_statements misc mutation sql placeholder_literalizer", 'dataset')
+  require_relative "dataset/query"
+  require_relative "dataset/actions"
+  require_relative "dataset/features"
+  require_relative "dataset/graph"
+  require_relative "dataset/prepared_statements"
+  require_relative "dataset/misc"
+  require_relative "dataset/sql"
+  require_relative "dataset/placeholder_literalizer"
+  require_relative "dataset/dataset_module"
 end

data/lib/sequel/dataset/actions.rb

@@ -1,3 +1,5 @@
+# frozen-string-literal: true
+
 module Sequel
   class Dataset
     # ---------------------
@@ -8,17 +10,21 @@ module Sequel
     # ---------------------
     
     # Action methods defined by Sequel that execute code on the database.
-    ACTION_METHODS = (<<-METHS).split.map(&:to_sym)
-      << [] all avg count columns columns! delete each
-      empty? fetch_rows first first! get import insert interval last
-      map max min multi_insert paged_each range select_hash select_hash_groups select_map select_order_map
-      single_record single_value sum to_hash to_hash_groups truncate update
+    ACTION_METHODS = (<<-METHS).split.map(&:to_sym).freeze
+      << [] all as_hash avg count columns columns! delete each
+      empty? fetch_rows first first! get import insert last
+      map max min multi_insert paged_each select_hash select_hash_groups select_map select_order_map
+      single_record single_record! single_value single_value! sum to_hash to_hash_groups truncate update
+      where_all where_each where_single_value
     METHS
 
+    # The clone options to use when retrieving columns for a dataset.
+    COLUMNS_CLONE_OPTIONS = {:distinct => nil, :limit => 1, :offset=>nil, :where=>nil, :having=>nil, :order=>nil, :row_proc=>nil, :graph=>nil, :eager_graph=>nil}.freeze
+
     # Inserts the given argument into the database.  Returns self so it
     # can be used safely when chaining:
     # 
-    #   DB[:items] << {:id=>0, :name=>'Zero'} << DB[:old_items].select(:id, name)
+    #   DB[:items] << {id: 0, name: 'Zero'} << DB[:old_items].select(:id, name)
     def <<(arg)
       insert(arg)
       self
@@ -26,10 +32,10 @@ module Sequel
     
     # Returns the first record matching the conditions. Examples:
     #
-    #   DB[:table][:id=>1] # SELECT * FROM table WHERE (id = 1) LIMIT 1
-    #   # => {:id=1}
+    #   DB[:table][id: 1] # SELECT * FROM table WHERE (id = 1) LIMIT 1
+    #   # => {:id=>1}
     def [](*conditions)
-      raise(Error, ARRAY_ACCESS_ERROR_MSG) if (conditions.length == 1 and conditions.first.is_a?(Integer)) or conditions.length == 0
+      raise(Error, 'You cannot call Dataset#[] with an integer or with no arguments') if (conditions.length == 1 and conditions.first.is_a?(Integer)) or conditions.length == 0
       first(*conditions)
     end
 
@@ -52,8 +58,9 @@ module Sequel
     #   # => 3
     #   DB[:table].avg{function(column)} # SELECT avg(function(column)) FROM table LIMIT 1
     #   # => 1
-    def avg(column=Sequel.virtual_row(&Proc.new))
-      aggregate_dataset.get{avg(column).as(:avg)}
+    def avg(arg=(no_arg = true), &block)
+      arg = Sequel.virtual_row(&block) if no_arg
+      _aggregate(:avg, arg)
     end
   
     # Returns the columns in the result set in order as an array of symbols.
@@ -66,11 +73,7 @@ module Sequel
     #   DB[:table].columns
     #   # => [:id, :name]
     def columns
-      return @columns if @columns
-      ds = unfiltered.unordered.naked.clone(:distinct => nil, :limit => 1, :offset=>nil)
-      ds.each{break}
-      @columns = ds.instance_variable_get(:@columns)
-      @columns || []
+      _columns || columns!
     end
         
     # Ignore any cached column information and perform a query to retrieve
@@ -79,10 +82,18 @@ module Sequel
     #   DB[:table].columns!
     #   # => [:id, :name]
     def columns!
-      @columns = nil
-      columns
+      ds = clone(COLUMNS_CLONE_OPTIONS)
+      ds.each{break}
+
+      if cols = ds.cache[:_columns]
+        self.columns = cols
+      else
+        []
+      end
     end
     
+    COUNT_SELECT = Sequel.function(:count).*.as(:count)
+
     # Returns the number of records in the dataset. If an argument is provided,
     # it is used as the argument to count.  If a block is provided, it is
     # treated as a virtual row, and the result is used as the argument to
@@ -95,22 +106,24 @@ module Sequel
     #   DB[:table].count{foo(column)} # SELECT count(foo(column)) AS count FROM table LIMIT 1
     #   # => 1
     def count(arg=(no_arg=true), &block)
-      if no_arg
+      if no_arg && !block
+        cached_dataset(:_count_ds) do
+          aggregate_dataset.select(COUNT_SELECT).single_value_ds
+        end.single_value!.to_i
+      else
         if block
-          arg = Sequel.virtual_row(&block)
-          aggregate_dataset.get{count(arg).as(:count)}
-        else
-          aggregate_dataset.get{count{}.*.as(:count)}.to_i
+          if no_arg
+            arg = Sequel.virtual_row(&block)
+          else
+            raise Error, 'cannot provide both argument and block to Dataset#count'
+          end
         end
-      elsif block
-        raise Error, 'cannot provide both argument and block to Dataset#count'
-      else
-        aggregate_dataset.get{count(arg).as(:count)}
+
+        _aggregate(:count, arg)
       end
     end
-    
-    # Deletes the records in the dataset.  The returned value should be 
-    # number of records deleted, but that is adapter dependent.
+
+    # Deletes the records in the dataset, returning the number of records deleted.
     #
     #   DB[:table].delete # DELETE * FROM table
     #   # => 3
@@ -133,27 +146,30 @@ module Sequel
     # running queries inside the block, you should use +all+ instead of +each+
     # for the outer queries, or use a separate thread or shard inside +each+.
     def each
-      if row_proc = @row_proc
-        fetch_rows(select_sql){|r| yield row_proc.call(r)}
+      if rp = row_proc
+        fetch_rows(select_sql){|r| yield rp.call(r)}
       else
         fetch_rows(select_sql){|r| yield r}
       end
       self
     end
     
+    EMPTY_SELECT = Sequel::SQL::AliasedExpression.new(1, :one)
+
     # Returns true if no records exist in the dataset, false otherwise
     #
     #   DB[:table].empty? # SELECT 1 AS one FROM table LIMIT 1
     #   # => false
     def empty?
-      ds = @opts[:order] ? unordered : self
-      ds.get(Sequel::SQL::AliasedExpression.new(1, :one)).nil?
+      cached_dataset(:_empty_ds) do
+        single_value_ds.unordered.select(EMPTY_SELECT)
+      end.single_value!.nil?
     end
 
+    # Returns the first matching record if no arguments are given.
     # If a integer argument is given, it is interpreted as a limit, and then returns all 
-    # matching records up to that limit.  If no argument is passed,
-    # it returns the first matching record.  If any other type of
-    # argument(s) is passed, it is given to filter and the
+    # matching records up to that limit.  If any other type of
+    # argument(s) is passed, it is treated as a filter and the
     # first matching record is returned.  If a block is given, it is used
     # to filter the dataset before returning anything.
     #
@@ -168,35 +184,65 @@ module Sequel
     #   DB[:table].first(2) # SELECT * FROM table LIMIT 2
     #   # => [{:id=>6}, {:id=>4}]
     #
-    #   DB[:table].first(:id=>2) # SELECT * FROM table WHERE (id = 2) LIMIT 1
+    #   DB[:table].first(id: 2) # SELECT * FROM table WHERE (id = 2) LIMIT 1
     #   # => {:id=>2}
     #
-    #   DB[:table].first("id = 3") # SELECT * FROM table WHERE (id = 3) LIMIT 1
+    #   DB[:table].first(Sequel.lit("id = 3")) # SELECT * FROM table WHERE (id = 3) LIMIT 1
     #   # => {:id=>3}
     #
-    #   DB[:table].first("id = ?", 4) # SELECT * FROM table WHERE (id = 4) LIMIT 1
+    #   DB[:table].first(Sequel.lit("id = ?", 4)) # SELECT * FROM table WHERE (id = 4) LIMIT 1
     #   # => {:id=>4}
     #
     #   DB[:table].first{id > 2} # SELECT * FROM table WHERE (id > 2) LIMIT 1
     #   # => {:id=>5}
     #
-    #   DB[:table].first("id > ?", 4){id < 6} # SELECT * FROM table WHERE ((id > 4) AND (id < 6)) LIMIT 1
+    #   DB[:table].first(Sequel.lit("id > ?", 4)){id < 6} # SELECT * FROM table WHERE ((id > 4) AND (id < 6)) LIMIT 1
     #   # => {:id=>5}
     #
     #   DB[:table].first(2){id < 2} # SELECT * FROM table WHERE (id < 2) LIMIT 2
     #   # => [{:id=>1}]
     def first(*args, &block)
-      ds = block ? filter(&block) : self
+      case args.length
+      when 0
+        unless block
+          return single_record
+        end
+      when 1
+        arg = args[0]
+        if arg.is_a?(Integer)
+          res = if block
+            if loader = cached_placeholder_literalizer(:_first_integer_cond_loader) do |pl|
+                where(pl.arg).limit(pl.arg)
+              end
 
-      if args.empty?
-        ds.single_record
-      else
-        args = (args.size == 1) ? args.first : args
-        if args.is_a?(Integer)
-          ds.limit(args).all
-        else
-          ds.filter(args).single_record
+              loader.all(filter_expr(&block), arg)
+            else
+              where(&block).limit(arg).all
+            end
+          else
+            if loader = cached_placeholder_literalizer(:_first_integer_loader) do |pl|
+               limit(pl.arg)
+              end
+
+              loader.all(arg)
+            else
+              limit(arg).all
+            end
+          end
+
+          return res
+        end
+        where_args = args
+        args = arg
+      end
+
+      if loader = cached_where_placeholder_literalizer(where_args||args, block, :_first_cond_loader) do |pl|
+          _single_record_ds.where(pl.arg)
         end
+
+        loader.first(filter_expr(args, &block))
+      else
+        _single_record_ds.where(args, &block).single_record!
       end
     end
 
@@ -227,15 +273,32 @@ module Sequel
     def get(column=(no_arg=true; nil), &block)
       ds = naked
       if block
-        raise(Error, ARG_BLOCK_ERROR_MSG) unless no_arg
+        raise(Error, 'Must call Dataset#get with an argument or a block, not both') unless no_arg
         ds = ds.select(&block)
         column = ds.opts[:select]
         column = nil if column.is_a?(Array) && column.length < 2
       else
-        ds = if column.is_a?(Array)
-          ds.select(*column)
+        case column
+        when Array
+          ds = ds.select(*column)
+        when LiteralString, Symbol, SQL::Identifier, SQL::QualifiedIdentifier, SQL::AliasedExpression
+          if loader = cached_placeholder_literalizer(:_get_loader) do |pl|
+              ds.single_value_ds.select(pl.arg)
+            end
+
+            return loader.get(column)
+          end
+
+          ds = ds.select(column)
         else
-          ds.select(auto_alias_expression(column))
+          if loader = cached_placeholder_literalizer(:_get_alias_loader) do |pl|
+              ds.single_value_ds.select(Sequel.as(pl.arg, :v))
+            end
+
+            return loader.get(column)
+          end
+
+          ds = ds.select(Sequel.as(column, :v))
         end
       end
 
@@ -270,6 +333,7 @@ module Sequel
     #                  after every 50 records.
     # :return :: When this is set to :primary_key, returns an array of
     #            autoincremented primary key values for the rows inserted.
+    #            This does not have an effect if +values+ is a Dataset.
     # :server :: Set the server/shard to use for the transaction and insert
     #            queries.
     # :slice :: Same as :commit_every, :commit_every takes precedence.
@@ -277,7 +341,7 @@ module Sequel
       return @db.transaction{insert(columns, values)} if values.is_a?(Dataset)
 
       return if values.empty?
-      raise(Error, IMPORT_ERROR_MSG) if columns.empty?
+      raise(Error, 'Using Sequel::Dataset#import with an empty column array is not allowed') if columns.empty?
       ds = opts[:server] ? server(opts[:server]) : self
       
       if slice_size = opts.fetch(:commit_every, opts.fetch(:slice, default_import_slice))
@@ -294,11 +358,12 @@ module Sequel
     end
 
     # Inserts values into the associated table.  The returned value is generally
-    # the value of the primary key for the inserted row, but that is adapter dependent.
+    # the value of the autoincremented primary key for the inserted row, assuming that
+    # a single row is inserted and the table has an autoincrementing primary key.
     #
     # +insert+ handles a number of different argument formats:
     # no arguments or single empty hash :: Uses DEFAULT VALUES
-    # single hash :: Most common format, treats keys as columns an values as values
+    # single hash :: Most common format, treats keys as columns and values as values
     # single array :: Treats entries as values, with no columns
     # two arrays :: Treats first array as columns, second array as values
     # single Dataset :: Treats as an insert based on a selection from the dataset given,
@@ -320,7 +385,7 @@ module Sequel
     #   DB[:items].insert([:a, :b], [1,2])
     #   # INSERT INTO items (a, b) VALUES (1, 2)
     #
-    #   DB[:items].insert(:a => 1, :b => 2)
+    #   DB[:items].insert(a: 1, b: 2)
     #   # INSERT INTO items (a, b) VALUES (1, 2)
     #
     #   DB[:items].insert(DB[:old_items])
@@ -337,17 +402,6 @@ module Sequel
       end
     end
     
-    # Returns the interval between minimum and maximum values for the given 
-    # column/expression. Uses a virtual row block if no argument is given.
-    #
-    #   DB[:table].interval(:id) # SELECT (max(id) - min(id)) FROM table LIMIT 1
-    #   # => 6
-    #   DB[:table].interval{function(column)} # SELECT (max(function(column)) - min(function(column))) FROM table LIMIT 1
-    #   # => 7
-    def interval(column=Sequel.virtual_row(&Proc.new))
-      aggregate_dataset.get{(max(column) - min(column)).as(:interval)}
-    end
-
     # Reverses the order and then runs #first with the given arguments and block.  Note that this
     # will not necessarily give you the last record in the dataset,
     # unless you have an unambiguous order.  If there is not
@@ -363,8 +417,8 @@ module Sequel
       reverse.first(*args, &block)
     end
     
-    # Maps column values for each record in the dataset (if a column name is
-    # given), or performs the stock mapping functionality of +Enumerable+ otherwise. 
+    # Maps column values for each record in the dataset (if an argument is given)
+    # or performs the stock mapping functionality of +Enumerable+ otherwise. 
     # Raises an +Error+ if both an argument and block are given.
     #
     #   DB[:table].map(:id) # SELECT * FROM table
@@ -379,7 +433,7 @@ module Sequel
     #   # => [[1, 'A'], [2, 'B'], [3, 'C'], ...]
     def map(column=nil, &block)
       if column
-        raise(Error, ARG_BLOCK_ERROR_MSG) if block
+        raise(Error, 'Must call Dataset#map with either an argument or a block, not both') if block
         return naked.map(column) if row_proc
         if column.is_a?(Array)
           super(){|r| r.values_at(*column)}
@@ -398,8 +452,9 @@ module Sequel
     #   # => 10
     #   DB[:table].max{function(column)} # SELECT max(function(column)) FROM table LIMIT 1
     #   # => 7
-    def max(column=Sequel.virtual_row(&Proc.new))
-      aggregate_dataset.get{max(column).as(:max)}
+    def max(arg=(no_arg = true), &block)
+      arg = Sequel.virtual_row(&block) if no_arg
+      _aggregate(:max, arg)
     end
 
     # Returns the minimum value for the given column/expression.
@@ -409,14 +464,15 @@ module Sequel
     #   # => 1
     #   DB[:table].min{function(column)} # SELECT min(function(column)) FROM table LIMIT 1
     #   # => 0
-    def min(column=Sequel.virtual_row(&Proc.new))
-      aggregate_dataset.get{min(column).as(:min)}
+    def min(arg=(no_arg = true), &block)
+      arg = Sequel.virtual_row(&block) if no_arg
+      _aggregate(:min, arg)
     end
 
     # This is a front end for import that allows you to submit an array of
     # hashes instead of arrays of columns and values:
     # 
-    #   DB[:table].multi_insert([{:x => 1}, {:x => 2}])
+    #   DB[:table].multi_insert([{x: 1}, {x: 2}])
     #   # INSERT INTO table (x) VALUES (1)
     #   # INSERT INTO table (x) VALUES (2)
     #
@@ -431,7 +487,7 @@ module Sequel
       import(columns, hashes.map{|h| columns.map{|c| h[c]}}, opts)
     end
 
-    # Yields each row in the dataset, but interally uses multiple queries as needed to
+    # Yields each row in the dataset, but internally uses multiple queries as needed to
     # process the entire result set without keeping all rows in the dataset in memory,
     # even if the underlying driver buffers all query results in memory.
     #
@@ -443,6 +499,10 @@ module Sequel
     # Sequel checks that the datasets using this method have an order, but it cannot
     # ensure that the order is unambiguous.
     #
+    # Note that this method is not safe to use on many adapters if you are
+    # running additional queries inside the provided block.  If you are
+    # running queries inside the block, use a separate thread or shard inside +paged_each+.
+    #
     # Options:
     # :rows_per_fetch :: The number of rows to fetch per query.  Defaults to 1000.
     # :strategy :: The strategy to use for paging of results.  By default this is :offset,
@@ -452,8 +512,8 @@ module Sequel
     #              selecting the columns you are ordering by, and none of the columns can contain
     #              NULLs.  Note that some Sequel adapters have optimized implementations that will
     #              use cursors or streaming regardless of the :strategy option used.
-    # :filter_values :: If the :strategy=>:filter option is used, this option should be a proc
-    #                   that accepts the last retreived row for the previous page and an array of
+    # :filter_values :: If the strategy: :filter option is used, this option should be a proc
+    #                   that accepts the last retrieved row for the previous page and an array of
     #                   ORDER BY expressions, and returns an array of values relating to those
     #                   expressions for the last retrieved row.  You will need to use this option
     #                   if your ORDER BY expressions are not simple columns, if they contain
@@ -472,13 +532,13 @@ module Sequel
     #   # SELECT * FROM table ORDER BY id LIMIT 100 OFFSET 100
     #   # ...
     #
-    #   DB[:table].order(:id).paged_each(:strategy=>:filter){|row| }
+    #   DB[:table].order(:id).paged_each(strategy: :filter){|row| }
     #   # SELECT * FROM table ORDER BY id LIMIT 1000
     #   # SELECT * FROM table WHERE id > 1001 ORDER BY id LIMIT 1000
     #   # ...
     #
-    #   DB[:table].order(:table__id).paged_each(:strategy=>:filter,
-    #     :filter_values=>proc{|row, exprs| [row[:id]]}){|row| }
+    #   DB[:table].order(:id).paged_each(strategy: :filter,
+    #     filter_values: lambda{|row, exprs| [row[:id]]}){|row| }
     #   # SELECT * FROM table ORDER BY id LIMIT 1000
     #   # SELECT * FROM table WHERE id > 1001 ORDER BY id LIMIT 1000
     #   # ...
@@ -542,55 +602,47 @@ module Sequel
       self
     end
 
-    # Returns a +Range+ instance made from the minimum and maximum values for the
-    # given column/expression.  Uses a virtual row block if no argument is given.
-    #
-    #   DB[:table].range(:id) # SELECT max(id) AS v1, min(id) AS v2 FROM table LIMIT 1
-    #   # => 1..10
-    #   DB[:table].interval{function(column)} # SELECT max(function(column)) AS v1, min(function(column)) AS v2 FROM table LIMIT 1
-    #   # => 0..7
-    def range(column=Sequel.virtual_row(&Proc.new))
-      if r = aggregate_dataset.select{[min(column).as(v1), max(column).as(v2)]}.first
-        (r[:v1]..r[:v2])
-      end
-    end
-    
     # Returns a hash with key_column values as keys and value_column values as
-    # values.  Similar to to_hash, but only selects the columns given.
+    # values.  Similar to as_hash, but only selects the columns given.  Like
+    # as_hash, it accepts an optional :hash parameter, into which entries will
+    # be merged. 
     #
-    #   DB[:table].select_hash(:id, :name) # SELECT id, name FROM table
+    #   DB[:table].select_hash(:id, :name)
+    #   # SELECT id, name FROM table
     #   # => {1=>'a', 2=>'b', ...}
     #
     # You can also provide an array of column names for either the key_column,
     # the value column, or both:
     #
-    #   DB[:table].select_hash([:id, :foo], [:name, :bar]) # SELECT * FROM table
-    #   # {[1, 3]=>['a', 'c'], [2, 4]=>['b', 'd'], ...}
+    #   DB[:table].select_hash([:id, :foo], [:name, :bar])
+    #   # SELECT id, foo, name, bar FROM table
+    #   # => {[1, 3]=>['a', 'c'], [2, 4]=>['b', 'd'], ...}
     #
     # When using this method, you must be sure that each expression has an alias
-    # that Sequel can determine.  Usually you can do this by calling the #as method
-    # on the expression and providing an alias.
-    def select_hash(key_column, value_column)
-      _select_hash(:to_hash, key_column, value_column)
+    # that Sequel can determine.
+    def select_hash(key_column, value_column, opts = OPTS)
+      _select_hash(:as_hash, key_column, value_column, opts)
     end
     
     # Returns a hash with key_column values as keys and an array of value_column values.
-    # Similar to to_hash_groups, but only selects the columns given.
+    # Similar to to_hash_groups, but only selects the columns given.  Like to_hash_groups,
+    # it accepts an optional :hash parameter, into which entries will be merged. 
     #
-    #   DB[:table].select_hash_groups(:name, :id) # SELECT id, name FROM table
+    #   DB[:table].select_hash_groups(:name, :id)
+    #   # SELECT id, name FROM table
     #   # => {'a'=>[1, 4, ...], 'b'=>[2, ...], ...}
     #
     # You can also provide an array of column names for either the key_column,
     # the value column, or both:
     #
-    #   DB[:table].select_hash_groups([:first, :middle], [:last, :id]) # SELECT * FROM table
-    #   # {['a', 'b']=>[['c', 1], ['d', 2], ...], ...}
+    #   DB[:table].select_hash_groups([:first, :middle], [:last, :id])
+    #   # SELECT first, middle, last, id FROM table
+    #   # => {['a', 'b']=>[['c', 1], ['d', 2], ...], ...}
     #
     # When using this method, you must be sure that each expression has an alias
-    # that Sequel can determine.  Usually you can do this by calling the #as method
-    # on the expression and providing an alias.
-    def select_hash_groups(key_column, value_column)
-      _select_hash(:to_hash_groups, key_column, value_column)
+    # that Sequel can determine.
+    def select_hash_groups(key_column, value_column, opts = OPTS)
+      _select_hash(:to_hash_groups, key_column, value_column, opts)
     end
 
     # Selects the column given (either as an argument or as a block), and
@@ -611,8 +663,7 @@ module Sequel
     #   # => [[1, 'A'], [2, 'B'], [3, 'C'], ...]
     #
     # If you provide an array of expressions, you must be sure that each entry
-    # in the array has an alias that Sequel can determine.  Usually you can do this
-    # by calling the #as method on the expression and providing an alias.
+    # in the array has an alias that Sequel can determine.
     def select_map(column=nil, &block)
       _select_map(column, false, &block)
     end
@@ -631,27 +682,56 @@ module Sequel
     #   # => [[1, 'A'], [2, 'B'], [3, 'C'], ...]
     #
     # If you provide an array of expressions, you must be sure that each entry
-    # in the array has an alias that Sequel can determine.  Usually you can do this
-    # by calling the #as method on the expression and providing an alias.
+    # in the array has an alias that Sequel can determine.
     def select_order_map(column=nil, &block)
       _select_map(column, true, &block)
     end
 
-    # Returns the first record in the dataset, or nil if the dataset
-    # has no records. Users should probably use +first+ instead of
-    # this method.
+    # Limits the dataset to one record, and returns the first record in the dataset,
+    # or nil if the dataset has no records. Users should probably use +first+ instead of
+    # this method. Example:
+    #
+    #   DB[:test].single_record # SELECT * FROM test LIMIT 1
+    #   # => {:column_name=>'value'}
     def single_record
-      clone(:limit=>1).each{|r| return r}
-      nil
+      _single_record_ds.single_record!
+    end
+
+    # Returns the first record in dataset, without limiting the dataset. Returns nil if
+    # the dataset has no records. Users should probably use +first+ instead of this method.
+    # This should only be used if you know the dataset is already limited to a single record.
+    # This method may be desirable to use for performance reasons, as it does not clone the
+    # receiver. Example:
+    #
+    #   DB[:test].single_record! # SELECT * FROM test
+    #   # => {:column_name=>'value'}
+    def single_record!
+      with_sql_first(select_sql)
     end
 
     # Returns the first value of the first record in the dataset.
     # Returns nil if dataset is empty.  Users should generally use
-    # +get+ instead of this method.
+    # +get+ instead of this method. Example:
+    #
+    #   DB[:test].single_value # SELECT * FROM test LIMIT 1
+    #   # => 'value'
     def single_value
-      if r = ungraphed.naked.single_record
-        r.values.first
+      single_value_ds.each do |r|
+        r.each{|_, v| return v}
       end
+      nil
+    end
+
+    # Returns the first value of the first record in the dataset, without limiting the dataset.
+    # Returns nil if the dataset is empty. Users should generally use +get+ instead of this
+    # method.  Should not be used on graphed datasets or datasets that have row_procs that
+    # don't return hashes.  This method may be desirable to use for performance reasons, as
+    # it does not clone the receiver.
+    #
+    #   DB[:test].single_value! # SELECT * FROM test
+    #   # => 'value'
+    def single_value!
+      with_sql_single_value(select_sql)
     end
     
     # Returns the sum for the given column/expression.
@@ -661,8 +741,9 @@ module Sequel
     #   # => 55
     #   DB[:table].sum{function(column)} # SELECT sum(function(column)) FROM table LIMIT 1
     #   # => 10
-    def sum(column=Sequel.virtual_row(&Proc.new))
-      aggregate_dataset.get{sum(column).as(:sum)}
+    def sum(arg=(no_arg = true), &block)
+      arg = Sequel.virtual_row(&block) if no_arg
+      _aggregate(:sum, arg)
     end
 
     # Returns a hash with one column used as key and another used as value.
@@ -670,45 +751,57 @@ module Sequel
     # will overwrite the value of the previous row(s). If the value_column
     # is not given or nil, uses the entire hash as the value.
     #
-    #   DB[:table].to_hash(:id, :name) # SELECT * FROM table
+    #   DB[:table].as_hash(:id, :name) # SELECT * FROM table
     #   # {1=>'Jim', 2=>'Bob', ...}
     #
-    #   DB[:table].to_hash(:id) # SELECT * FROM table
+    #   DB[:table].as_hash(:id) # SELECT * FROM table
     #   # {1=>{:id=>1, :name=>'Jim'}, 2=>{:id=>2, :name=>'Bob'}, ...}
     #
     # You can also provide an array of column names for either the key_column,
     # the value column, or both:
     #
-    #   DB[:table].to_hash([:id, :foo], [:name, :bar]) # SELECT * FROM table
+    #   DB[:table].as_hash([:id, :foo], [:name, :bar]) # SELECT * FROM table
     #   # {[1, 3]=>['Jim', 'bo'], [2, 4]=>['Bob', 'be'], ...}
     #
-    #   DB[:table].to_hash([:id, :name]) # SELECT * FROM table
-    #   # {[1, 'Jim']=>{:id=>1, :name=>'Jim'}, [2, 'Bob'=>{:id=>2, :name=>'Bob'}, ...}
-    def to_hash(key_column, value_column = nil)
-      h = {}
+    #   DB[:table].as_hash([:id, :name]) # SELECT * FROM table
+    #   # {[1, 'Jim']=>{:id=>1, :name=>'Jim'}, [2, 'Bob']=>{:id=>2, :name=>'Bob'}, ...}
+    #
+    # Options:
+    # :all :: Use all instead of each to retrieve the objects
+    # :hash :: The object into which the values will be placed.  If this is not
+    #          given, an empty hash is used.  This can be used to use a hash with
+    #          a default value or default proc.
+    def as_hash(key_column, value_column = nil, opts = OPTS)
+      h = opts[:hash] || {}
+      meth = opts[:all] ? :all : :each
       if value_column
-        return naked.to_hash(key_column, value_column) if row_proc
+        return naked.as_hash(key_column, value_column, opts) if row_proc
         if value_column.is_a?(Array)
           if key_column.is_a?(Array)
-            each{|r| h[r.values_at(*key_column)] = r.values_at(*value_column)}
+            public_send(meth){|r| h[r.values_at(*key_column)] = r.values_at(*value_column)}
           else
-            each{|r| h[r[key_column]] = r.values_at(*value_column)}
+            public_send(meth){|r| h[r[key_column]] = r.values_at(*value_column)}
           end
         else
           if key_column.is_a?(Array)
-            each{|r| h[r.values_at(*key_column)] = r[value_column]}
+            public_send(meth){|r| h[r.values_at(*key_column)] = r[value_column]}
           else
-            each{|r| h[r[key_column]] = r[value_column]}
+            public_send(meth){|r| h[r[key_column]] = r[value_column]}
           end
         end
       elsif key_column.is_a?(Array)
-        each{|r| h[key_column.map{|k| r[k]}] = r}
+        public_send(meth){|r| h[key_column.map{|k| r[k]}] = r}
       else
-        each{|r| h[r[key_column]] = r}
+        public_send(meth){|r| h[r[key_column]] = r}
       end
       h
     end
 
+    # Alias of as_hash for backwards compatibility.
+    def to_hash(*a)
+      as_hash(*a)
+    end
+
     # Returns a hash with one column used as key and the values being an
     # array of column values. If the value_column is not given or nil, uses
     # the entire hash as the value.
@@ -727,27 +820,34 @@ module Sequel
     #
     #   DB[:table].to_hash_groups([:first, :middle]) # SELECT * FROM table
     #   # {['Jim', 'Bob']=>[{:id=>1, :first=>'Jim', :middle=>'Bob', :last=>'Smith'}, ...], ...}
-    def to_hash_groups(key_column, value_column = nil)
-      h = {}
+    #
+    # Options:
+    # :all :: Use all instead of each to retrieve the objects
+    # :hash :: The object into which the values will be placed.  If this is not
+    #          given, an empty hash is used.  This can be used to use a hash with
+    #          a default value or default proc.
+    def to_hash_groups(key_column, value_column = nil, opts = OPTS)
+      h = opts[:hash] || {}
+      meth = opts[:all] ? :all : :each
       if value_column
-        return naked.to_hash_groups(key_column, value_column) if row_proc
+        return naked.to_hash_groups(key_column, value_column, opts) if row_proc
         if value_column.is_a?(Array)
           if key_column.is_a?(Array)
-            each{|r| (h[r.values_at(*key_column)] ||= []) << r.values_at(*value_column)}
+            public_send(meth){|r| (h[r.values_at(*key_column)] ||= []) << r.values_at(*value_column)}
           else
-            each{|r| (h[r[key_column]] ||= []) << r.values_at(*value_column)}
+            public_send(meth){|r| (h[r[key_column]] ||= []) << r.values_at(*value_column)}
           end
         else
           if key_column.is_a?(Array)
-            each{|r| (h[r.values_at(*key_column)] ||= []) << r[value_column]}
+            public_send(meth){|r| (h[r.values_at(*key_column)] ||= []) << r[value_column]}
           else
-            each{|r| (h[r[key_column]] ||= []) << r[value_column]}
+            public_send(meth){|r| (h[r[key_column]] ||= []) << r[value_column]}
           end
         end
       elsif key_column.is_a?(Array)
-        each{|r| (h[key_column.map{|k| r[k]}] ||= []) << r}
+        public_send(meth){|r| (h[key_column.map{|k| r[k]}] ||= []) << r}
       else
-        each{|r| (h[r[key_column]] ||= []) << r}
+        public_send(meth){|r| (h[r[key_column]] ||= []) << r}
       end
       h
     end
@@ -760,15 +860,14 @@ module Sequel
       execute_ddl(truncate_sql)
     end
 
-    # Updates values for the dataset.  The returned value is generally the
-    # number of rows updated, but that is adapter dependent. +values+ should
-    # a hash where the keys are columns to set and values are the values to
+    # Updates values for the dataset.  The returned value is the number of rows updated.
+    # +values+ should be a hash where the keys are columns to set and values are the values to
     # which to set the columns.
     #
-    #   DB[:table].update(:x=>nil) # UPDATE table SET x = NULL
+    #   DB[:table].update(x: nil) # UPDATE table SET x = NULL
     #   # => 10
     #
-    #   DB[:table].update(:x=>Sequel.expr(:x)+1, :y=>0) # UPDATE table SET x = (x + 1), y = 0
+    #   DB[:table].update(x: Sequel[:x]+1, y: 0) # UPDATE table SET x = (x + 1), y = 0
     #   # => 10
     def update(values=OPTS, &block)
       sql = update_sql(values)
@@ -779,6 +878,52 @@ module Sequel
       end
     end
 
+    # Return an array of all rows matching the given filter condition, also
+    # yielding each row to the given block.  Basically the same as where(cond).all(&block),
+    # except it can be optimized to not create an intermediate dataset.
+    #
+    #   DB[:table].where_all(id: [1,2,3])
+    #   # SELECT * FROM table WHERE (id IN (1, 2, 3))
+    def where_all(cond, &block)
+      if loader = _where_loader([cond], nil)
+        loader.all(filter_expr(cond), &block)
+      else
+        where(cond).all(&block)
+      end
+    end
+
+    # Iterate over all rows matching the given filter condition, 
+    # yielding each row to the given block.  Basically the same as where(cond).each(&block),
+    # except it can be optimized to not create an intermediate dataset.
+    #
+    #   DB[:table].where_each(id: [1,2,3]){|row| p row}
+    #   # SELECT * FROM table WHERE (id IN (1, 2, 3))
+    def where_each(cond, &block)
+      if loader = _where_loader([cond], nil)
+        loader.each(filter_expr(cond), &block)
+      else
+        where(cond).each(&block)
+      end
+    end
+
+    # Filter the datasets using the given filter condition, then return a single value.
+    # This assumes that the dataset has already been setup to limit the selection to
+    # a single column.  Basically the same as where(cond).single_value,
+    # except it can be optimized to not create an intermediate dataset.
+    #
+    #   DB[:table].select(:name).where_single_value(id: 1)
+    #   # SELECT name FROM table WHERE (id = 1) LIMIT 1
+    def where_single_value(cond)
+      if loader = cached_where_placeholder_literalizer([cond], nil, :_where_single_value_loader) do |pl|
+          single_value_ds.where(pl.arg)
+        end
+
+        loader.get(filter_expr(cond))
+      else
+        where(cond).single_value
+      end
+    end
+
     # Run the given SQL and return an array of all rows.  If a block is given,
     # each row is yielded to the block after all rows are loaded. See with_sql_each.
     def with_sql_all(sql, &block)
@@ -794,14 +939,11 @@ module Sequel
     alias with_sql_update with_sql_delete
 
     # Run the given SQL and yield each returned row to the block.
-    #
-    # This method should not be called on a shared dataset if the columns selected
-    # in the given SQL do not match the columns in the receiver.
     def with_sql_each(sql)
-      if row_proc = @row_proc
-        fetch_rows(sql){|r| yield row_proc.call(r)}
+      if rp = row_proc
+        _with_sql_dataset.fetch_rows(sql){|r| yield rp.call(r)}
       else
-        fetch_rows(sql){|r| yield r}
+        _with_sql_dataset.fetch_rows(sql){|r| yield r}
       end
       self
     end
@@ -818,7 +960,7 @@ module Sequel
     # only a single value.  See with_sql_each.
     def with_sql_single_value(sql)
       if r = with_sql_first(sql)
-        r.values.first
+        r.each{|_, v| return v}
       end
     end
 
@@ -834,7 +976,8 @@ module Sequel
     # separate insert commands for each row.  Otherwise, call #multi_insert_sql
     # and execute each statement it gives separately.
     def _import(columns, values, opts)
-      trans_opts = Hash[opts].merge!(:server=>@opts[:server])
+      trans_opts = Hash[opts]
+      trans_opts[:server] = @opts[:server]
       if opts[:return] == :primary_key
         @db.transaction(trans_opts){values.map{|v| insert(columns, v)}}
       else
@@ -850,9 +993,15 @@ module Sequel
   
     # Returns an array of the first value in each row.
     def _select_map_single
-      map{|r| r.values.first}
+      k = nil
+      map{|r| r[k||=r.keys.first]}
     end
   
+    # A dataset for returning single values from the current dataset.
+    def single_value_ds
+      clone(:limit=>1).ungraphed.naked
+    end
+    
     private
     
     # Internals of all and with_sql_all
@@ -864,10 +1013,21 @@ module Sequel
       a
     end
     
+    # Cached placeholder literalizer for methods that return values using aggregate functions.
+    def _aggregate(function, arg)
+      if loader = cached_placeholder_literalizer(:"_#{function}_loader") do |pl|
+            aggregate_dataset.limit(1).select(SQL::Function.new(function, pl.arg).as(function))
+          end
+        loader.get(arg)
+      else
+        aggregate_dataset.get(SQL::Function.new(function, arg).as(function))
+      end
+    end
+    
     # Internals of +select_hash+ and +select_hash_groups+
-    def _select_hash(meth, key_column, value_column)
+    def _select_hash(meth, key_column, value_column, opts=OPTS)
       select(*(key_column.is_a?(Array) ? key_column : [key_column]) + (value_column.is_a?(Array) ? value_column : [value_column])).
-        send(meth, hash_key_symbols(key_column), hash_key_symbols(value_column))
+        public_send(meth, hash_key_symbols(key_column), hash_key_symbols(value_column), opts)
     end
     
     # Internals of +select_map+ and +select_order_map+
@@ -884,6 +1044,18 @@ module Sequel
       end
     end
 
+    # A cached dataset for a single record for this dataset.
+    def _single_record_ds
+      cached_dataset(:_single_record_ds){clone(:limit=>1)}
+    end
+
+    # Loader used for where_all and where_each.
+    def _where_loader(where_args, where_block)
+      cached_where_placeholder_literalizer(where_args, where_block, :_where_loader) do |pl|
+        where(pl.arg)
+      end
+    end
+
     # Automatically alias the given expression if it does not have an identifiable alias.
     def auto_alias_expression(v)
       case v
@@ -902,7 +1074,7 @@ module Sequel
 
     # Set the server to use to :default unless it is already set in the passed opts
     def default_server_opts(opts)
-      if @db.sharded?
+      if @db.sharded? && !opts.has_key?(:server)
         opts = Hash[opts]
         opts[:server] = @opts[:server] || :default
       end
@@ -913,7 +1085,7 @@ module Sequel
     # :read_only server unless a specific server is set.
     def execute(sql, opts=OPTS, &block)
       db = @db
-      if db.sharded?
+      if db.sharded? && !opts.has_key?(:server)
         opts = Hash[opts]
         opts[:server] = @opts[:server] || (@opts[:lock] ? :default : :read_only)
         opts
@@ -1003,11 +1175,10 @@ module Sequel
       Sequel.|(*cond)
     end
 
-    # Modify the identifier returned from the database based on the
-    # identifier_output_method.
+    # Downcase identifiers by default when outputing them from the database.
     def output_identifier(v)
       v = 'untitled' if v == ''
-      (i = identifier_output_method) ? v.to_s.send(i).to_sym : v.to_sym
+      v.to_s.downcase.to_sym
     end
     
     # This is run inside .all, after all of the records have been loaded
@@ -1056,5 +1227,18 @@ module Sequel
         c
       end
     end
+
+    # Cached dataset to use for with_sql_#{all,each,first,single_value}.
+    # This is used so that the columns returned by the given SQL do not
+    # affect the receiver of the with_sql_* method.
+    def _with_sql_dataset
+      if @opts[:_with_sql_ds]
+        self
+      else
+        cached_dataset(:_with_sql_ds) do
+          clone(:_with_sql_ds=>true)
+        end
+      end
+    end
   end
 end