sequel 4.41.0 → 4.42.0

data/lib/sequel/dataset/query.rb

@@ -18,7 +18,7 @@ module Sequel
 
     # Which options don't affect the SQL generation.  Used by simple_select_all?
     # to determine if this is a simple SELECT * FROM table.
-    NON_SQL_OPTIONS = [:server, :graph, :eager, :eager_graph, :graph_aliases].freeze
+    NON_SQL_OPTIONS = [:server, :graph, :eager, :eager_graph, :graph_aliases, :row_proc, :quote_identifiers, :identifier_input_method, :identifier_output_method, :skip_symbol_cache, :model, :model_object, :association_reflection, :fetch, :numrows, :autoid].freeze
     
     # These symbols have _join methods created (e.g. inner_join) that
     # call join_table with the symbol, passing along the arguments and
@@ -69,19 +69,44 @@ module Sequel
       where(*cond, &block)
     end
     
-    # Returns a new clone of the dataset with the given options merged.
-    # If the options changed include options in COLUMN_CHANGE_OPTS, the cached
-    # columns are deleted.  This method should generally not be called
-    # directly by user code.
-    def clone(opts = nil)
-      c = super()
-      if opts
-        c.instance_variable_set(:@opts, Hash[@opts].merge!(opts))
-        c.instance_variable_set(:@columns, nil) if @columns && !opts.each_key{|o| break if COLUMN_CHANGE_OPTS.include?(o)}
-      else
-        c.instance_variable_set(:@opts, Hash[@opts])
+    # On Ruby 2.4+, use clone(:freeze=>false) to create clones, because
+    # we use true freezing in that case, and we need to modify the opts
+    # in the frozen copy.
+    #
+    # On Ruby <2.4, just use Object#clone directly, since we don't
+    # use true freezing as it isn't possible.
+    if TRUE_FREEZE
+      # Save original clone implementation, as some other methods need
+      # to call it internally.
+      alias _clone clone
+      private :_clone
+
+      # Returns a new clone of the dataset with the given options merged.
+      # If the options changed include options in COLUMN_CHANGE_OPTS, the cached
+      # columns are deleted.  This method should generally not be called
+      # directly by user code.
+      def clone(opts = OPTS)
+        c = super(:freeze=>false)
+        c.opts.merge!(opts)
+        unless opts.each_key{|o| break if COLUMN_CHANGE_OPTS.include?(o)}
+          c.clear_columns_cache
+        end
+        c.freeze if frozen?
+        c
       end
-      c
+    else
+      # :nocov:
+      # :nodoc:
+      def clone(opts = OPTS)
+        c = super()
+        c.opts.merge!(opts)
+        unless opts.each_key{|o| break if COLUMN_CHANGE_OPTS.include?(o)}
+          c.clear_columns_cache
+        end
+        c.freeze if frozen?
+        c
+      end
+      # :nocov:
     end
 
     # Returns a copy of the dataset with the SQL DISTINCT clause. The DISTINCT
@@ -97,7 +122,7 @@ module Sequel
     def distinct(*args, &block)
       virtual_row_columns(args, block)
       raise(InvalidOperation, "DISTINCT ON not supported") if !args.empty? && !supports_distinct_on?
-      clone(:distinct => args)
+      clone(:distinct => args.freeze)
     end
 
     # Adds an EXCEPT clause using a second dataset object.
@@ -131,6 +156,22 @@ module Sequel
     #   
     #   DB[:items].exclude(:category => 'software', :id=>3)
     #   # SELECT * FROM items WHERE ((category != 'software') OR (id != 3))
+    #
+    # Also note that SQL uses 3-valued boolean logic (+true+, +false+, +NULL+), so
+    # the inverse of a true condition is a false condition, and will still
+    # not match rows that were NULL originally.  If you take the earlier
+    # example:
+    #
+    #   DB[:items].exclude(:category => 'software')
+    #   # SELECT * FROM items WHERE (category != 'software')
+    #
+    # Note that this does not match rows where +category+ is +NULL+.  This
+    # is because +NULL+ is an unknown value, and you do not know whether
+    # or not the +NULL+ category is +software+.  You can explicitly
+    # specify how to handle +NULL+ values if you want:
+    #
+    #   DB[:items].exclude(Sequel.~(:category=>nil) & {:category => 'software'})
+    #   # SELECT * FROM items WHERE ((category IS NULL) OR (category != 'software'))
     def exclude(*cond, &block)
       _filter_or_exclude(true, :where, *cond, &block)
     end
@@ -139,6 +180,9 @@ module Sequel
     #
     #   DB[:items].select_group(:name).exclude_having{count(name) < 2}
     #   # SELECT name FROM items GROUP BY name HAVING (count(name) >= 2)
+    #
+    # See documentation for exclude for how inversion is handled in regards
+    # to SQL 3-valued boolean logic.
     def exclude_having(*cond, &block)
       _filter_or_exclude(true, :having, *cond, &block)
     end
@@ -148,13 +192,25 @@ module Sequel
       exclude(*cond, &block)
     end
 
-    # Return a clone of the dataset loaded with the given dataset extensions.
-    # If no related extension file exists or the extension does not have
-    # specific support for Dataset objects, an Error will be raised.
-    def extension(*exts)
-      c = clone
-      c.send(:_extension!, exts)
-      c
+    if TRUE_FREEZE
+      # Return a clone of the dataset loaded with the given dataset extensions.
+      # If no related extension file exists or the extension does not have
+      # specific support for Dataset objects, an Error will be raised.
+      def extension(*a)
+        c = _clone(:freeze=>false)
+        c.send(:_extension!, a)
+        c.freeze if frozen?
+        c
+      end
+    else
+      # :nocov:
+      # :nodoc:
+      def extension(*exts)
+        c = clone
+        c.send(:_extension!, exts)
+        c
+      end
+      # :nocov:
     end
 
     # Alias for where.
@@ -166,7 +222,7 @@ module Sequel
     #
     #   DB[:table].for_update # SELECT * FROM table FOR UPDATE
     def for_update
-      lock_style(:update)
+      cached_dataset(:_for_update_ds){lock_style(:update)}
     end
 
     # Returns a copy of the dataset with the source changed. If no
@@ -203,8 +259,8 @@ module Sequel
           s
         end
       end
-      o = {:from=>source.empty? ? nil : source}
-      o[:with] = (opts[:with] || []) + ctes if ctes
+      o = {:from=>source.empty? ? nil : source.freeze}
+      o[:with] = ((opts[:with] || []) + ctes).freeze if ctes
       o[:num_dataset_sources] = table_alias_num if table_alias_num > 0
       clone(o)
     end
@@ -288,7 +344,7 @@ module Sequel
     #   DB[:items].group{[a, sum(b)]} # SELECT * FROM items GROUP BY a, sum(b)
     def group(*columns, &block)
       virtual_row_columns(columns, block)
-      clone(:group => (columns.compact.empty? ? nil : columns))
+      clone(:group => (columns.compact.empty? ? nil : columns.freeze))
     end
 
     # Alias of group
@@ -318,7 +374,7 @@ module Sequel
     #   # SELECT substr(first_name, 1, 1) AS initial, count(*) AS count FROM items GROUP BY substr(first_name, 1, 1)
     #   # => [{:initial=>'a', :count=>1}, ...]
     def group_and_count(*columns, &block)
-      select_group(*columns, &block).select_more(COUNT_OF_ALL_AS_COUNT)
+      select_group(*columns, &block).select_append(COUNT_OF_ALL_AS_COUNT)
     end
 
     # Returns a copy of the dataset with the given columns added to the list of
@@ -389,15 +445,20 @@ module Sequel
     #
     #   DB[:items].where(:category => 'software', :id=>3).invert
     #   # SELECT * FROM items WHERE ((category != 'software') OR (id != 3))
+    #
+    # See documentation for exclude for how inversion is handled in regards
+    # to SQL 3-valued boolean logic.
     def invert
-      having, where = @opts.values_at(:having, :where)
-      if having.nil? && where.nil?
-        where(false)
-      else
-        o = {}
-        o[:having] = SQL::BooleanExpression.invert(having) if having
-        o[:where] = SQL::BooleanExpression.invert(where) if where
-        clone(o)
+      cached_dataset(:_invert_ds) do
+        having, where = @opts.values_at(:having, :where)
+        if having.nil? && where.nil?
+          where(false)
+        else
+          o = {}
+          o[:having] = SQL::BooleanExpression.invert(having) if having
+          o[:where] = SQL::BooleanExpression.invert(where) if where
+          clone(o)
+        end
       end
     end
 
@@ -535,7 +596,7 @@ module Sequel
         SQL::JoinOnClause.new(expr, type, table_expr)
       end
 
-      opts = {:join => (@opts[:join] || []) + [join]}
+      opts = {:join => ((@opts[:join] || []) + [join]).freeze}
       opts[:last_joined_table] = table_name unless options[:reset_implicit_qualifier] == false
       opts[:num_dataset_sources] = table_alias_num if table_alias_num
       clone(opts)
@@ -615,7 +676,7 @@ module Sequel
     #   ds.all # => [{2=>:id}]
     #   ds.naked.all # => [{:id=>2}]
     def naked
-      with_row_proc(nil)
+      cached_dataset(:_naked_ds){with_row_proc(nil)}
     end
 
     # Returns a copy of the dataset with a specified order. Can be safely combined with limit.
@@ -661,12 +722,17 @@ module Sequel
     #   DB[:items].order(nil) # SELECT * FROM items
     def order(*columns, &block)
       virtual_row_columns(columns, block)
-      clone(:order => (columns.compact.empty?) ? nil : columns)
+      clone(:order => (columns.compact.empty?) ? nil : columns.freeze)
     end
     
-    # Alias of order_more, for naming consistency with order_prepend.
+    # Returns a copy of the dataset with the order columns added
+    # to the end of the existing order.
+    #
+    #   DB[:items].order(:a).order(:b) # SELECT * FROM items ORDER BY b
+    #   DB[:items].order(:a).order_append(:b) # SELECT * FROM items ORDER BY a, b
     def order_append(*columns, &block)
-      order_more(*columns, &block)
+      columns = @opts[:order] + columns if @opts[:order]
+      order(*columns, &block)
     end
 
     # Alias of order
@@ -674,14 +740,9 @@ module Sequel
       order(*columns, &block)
     end
 
-    # Returns a copy of the dataset with the order columns added
-    # to the end of the existing order.
-    #
-    #   DB[:items].order(:a).order(:b) # SELECT * FROM items ORDER BY b
-    #   DB[:items].order(:a).order_more(:b) # SELECT * FROM items ORDER BY a, b
+    # Alias of order_append.
     def order_more(*columns, &block)
-      columns = @opts[:order] + columns if @opts[:order]
-      order(*columns, &block)
+      order_append(*columns, &block)
     end
     
     # Returns a copy of the dataset with the order columns added
@@ -691,7 +752,7 @@ module Sequel
     #   DB[:items].order(:a).order_prepend(:b) # SELECT * FROM items ORDER BY b, a
     def order_prepend(*columns, &block)
       ds = order(*columns, &block)
-      @opts[:order] ? ds.order_more(*@opts[:order]) : ds
+      @opts[:order] ? ds.order_append(*@opts[:order]) : ds
     end
     
     # Qualify to the given table, or first source if no table is given.
@@ -708,7 +769,7 @@ module Sequel
       (o.keys & QUALIFY_KEYS).each do |k|
         h[k] = qualified_expression(o[k], table)
       end
-      h[:select] = [SQL::ColumnAll.new(table)] if !o[:select] || o[:select].empty?
+      h[:select] = [SQL::ColumnAll.new(table)].freeze if !o[:select] || o[:select].empty?
       clone(h)
     end
 
@@ -722,7 +783,7 @@ module Sequel
     #   DB[:items].returning(:id, :name) # RETURNING id, name
     def returning(*values)
       raise Error, "RETURNING is not supported on #{db.database_type}" unless supports_returning?(:insert)
-      clone(:returning=>values)
+      clone(:returning=>values.freeze)
     end
 
     # Returns a copy of the dataset with the order reversed. If no order is
@@ -733,8 +794,12 @@ module Sequel
     #   DB[:items].order(:id).reverse # SELECT * FROM items ORDER BY id DESC
     #   DB[:items].order(:id).reverse(Sequel.desc(:name)) # SELECT * FROM items ORDER BY name ASC
     def reverse(*order, &block)
-      virtual_row_columns(order, block)
-      order(*invert_order(order.empty? ? @opts[:order] : order))
+      if order.empty? && !block
+        cached_dataset(:_reverse_ds){order(*invert_order(@opts[:order]))}
+      else
+        virtual_row_columns(order, block)
+        order(*invert_order(order.empty? ? @opts[:order] : order.freeze))
+      end
     end
 
     # Alias of +reverse+
@@ -751,7 +816,7 @@ module Sequel
     #   DB[:items].select{[a, sum(b)]} # SELECT a, sum(b) FROM items
     def select(*columns, &block)
       virtual_row_columns(columns, block)
-      clone(:select => columns)
+      clone(:select => columns.freeze)
     end
     
     # Returns a copy of the dataset selecting the wildcard if no arguments
@@ -765,7 +830,7 @@ module Sequel
       if tables.empty?
         clone(:select => nil)
       else
-        select(*tables.map{|t| i, a = split_alias(t); a || i}.map{|t| SQL::ColumnAll.new(t)})
+        select(*tables.map{|t| i, a = split_alias(t); a || i}.map!{|t| SQL::ColumnAll.new(t)}.freeze)
       end
     end
     
@@ -780,7 +845,7 @@ module Sequel
       cur_sel = @opts[:select]
       if !cur_sel || cur_sel.empty?
         unless supports_select_all_and_column?
-          return select_all(*(Array(@opts[:from]) + Array(@opts[:join]))).select_more(*columns, &block)
+          return select_all(*(Array(@opts[:from]) + Array(@opts[:join]))).select_append(*columns, &block)
         end
         cur_sel = [WILDCARD]
       end
@@ -832,8 +897,10 @@ module Sequel
 
     # Skip locked rows when returning results from this dataset.
     def skip_locked
-      raise(Error, 'This dataset does not support skipping locked rows') unless supports_skip_locked?
-      clone(:skip_locked=>true)
+      cached_dataset(:_skip_locked_ds) do
+        raise(Error, 'This dataset does not support skipping locked rows') unless supports_skip_locked?
+        clone(:skip_locked=>true)
+      end
     end
 
     # Unbind bound variables from this dataset's filter and return an array of two
@@ -857,7 +924,7 @@ module Sequel
     #   DB[:items].group(:a).having(:a=>1).where(:b).unfiltered
     #   # SELECT * FROM items GROUP BY a
     def unfiltered
-      clone(:where => nil, :having => nil)
+      cached_dataset(:_unfiltered_ds){clone(:where => nil, :having => nil)}
     end
 
     # Returns a copy of the dataset with no grouping (GROUP or HAVING clause) applied.
@@ -865,7 +932,7 @@ module Sequel
     #   DB[:items].group(:a).having(:a=>1).where(:b).ungrouped
     #   # SELECT * FROM items WHERE b
     def ungrouped
-      clone(:group => nil, :having => nil)
+      cached_dataset(:_ungrouped_ds){clone(:group => nil, :having => nil)}
     end
 
     # Adds a UNION clause using a second dataset object.
@@ -892,14 +959,14 @@ module Sequel
     # 
     #   DB[:items].limit(10, 20).unlimited # SELECT * FROM items
     def unlimited
-      clone(:limit=>nil, :offset=>nil)
+      cached_dataset(:_unlimited_ds){clone(:limit=>nil, :offset=>nil)}
     end
 
     # Returns a copy of the dataset with no order.
     # 
     #   DB[:items].order(:a).unordered # SELECT * FROM items
     def unordered
-      order(nil)
+      cached_dataset(:_unordered_ds){clone(:order=>nil)}
     end
     
     # Returns a copy of the dataset with the given WHERE conditions imposed upon it.  
@@ -970,7 +1037,7 @@ module Sequel
         s, ds = hoist_cte(dataset)
         s.with(name, ds, opts)
       else
-        clone(:with=>(@opts[:with]||[]) + [Hash[opts].merge!(:name=>name, :dataset=>dataset)])
+        clone(:with=>((@opts[:with]||[]) + [Hash[opts].merge!(:name=>name, :dataset=>dataset)]).freeze)
       end
     end
 
@@ -999,15 +1066,33 @@ module Sequel
         s, ds = hoist_cte(recursive)
         s.with_recursive(name, nonrecursive, ds, opts)
       else
-        clone(:with=>(@opts[:with]||[]) + [Hash[opts].merge!(:recursive=>true, :name=>name, :dataset=>nonrecursive.union(recursive, {:all=>opts[:union_all] != false, :from_self=>false}))])
+        clone(:with=>((@opts[:with]||[]) + [Hash[opts].merge!(:recursive=>true, :name=>name, :dataset=>nonrecursive.union(recursive, {:all=>opts[:union_all] != false, :from_self=>false}))]).freeze)
       end
     end
     
-    # Return a clone of the dataset extended with the given modules.
-    def with_extend(*mods)
-      c = clone
-      c.extend(*mods)
-      c
+    if TRUE_FREEZE
+      # Return a clone of the dataset extended with the given modules.
+      # Note that like Object#extend, when multiple modules are provided
+      # as arguments the cloned dataset is extended with the modules in reverse
+      # order.  If a block is provided, a module is created using the block and
+      # the clone is extended with that module after any modules given as arguments.
+      def with_extend(*mods, &block)
+        c = _clone(:freeze=>false)
+        c.extend(*mods) unless mods.empty?
+        c.extend(Module.new(&block)) if block
+        c.freeze if frozen?
+        c
+      end
+    else
+      # :nocov:
+      # :nodoc:
+      def with_extend(*mods, &block)
+        c = clone
+        c.extend(*mods) unless mods.empty?
+        c.extend(Module.new(&block)) if block
+        c
+      end
+      # :nocov:
     end
 
     # Returns a cloned dataset with the given row_proc.
@@ -1016,9 +1101,7 @@ module Sequel
     #   ds.all # => [{:id=>2}]
     #   ds.with_row_proc(proc(&:invert)).all # => [{2=>:id}]
     def with_row_proc(callable)
-      c = clone
-      c.instance_variable_set(:@row_proc, callable)
-      c
+      clone(:row_proc=>callable)
     end
 
     # Returns a copy of the dataset with the static SQL used.  This is useful if you want
@@ -1050,7 +1133,7 @@ module Sequel
         s, ds = hoist_cte(dataset)
         return s.compound_clone(type, ds, opts)
       end
-      ds = compound_from_self.clone(:compounds=>Array(@opts[:compounds]).map(&:dup) + [[type, dataset.compound_from_self, opts[:all]]])
+      ds = compound_from_self.clone(:compounds=>(Array(@opts[:compounds]).map(&:dup) + [[type, dataset.compound_from_self, opts[:all]].freeze]).freeze)
       opts[:from_self] == false ? ds : ds.from_self(opts)
     end
 
@@ -1152,7 +1235,7 @@ module Sequel
     # clause from the given dataset added to it, and the second a clone of
     # the given dataset with the WITH clause removed.
     def hoist_cte(ds)
-      [clone(:with => (opts[:with] || []) + ds.opts[:with]), ds.clone(:with => nil)]
+      [clone(:with => ((opts[:with] || []) + ds.opts[:with]).freeze), ds.clone(:with => nil)]
     end
 
     # Whether CTEs need to be hoisted from the given ds into the current ds.

data/lib/sequel/dataset/sql.rb

@@ -198,11 +198,13 @@ module Sequel
     #            being an array of symbol/strings for the appropriate branch.
     def self.def_sql_method(mod, type, clauses)
       priv = type == :update || type == :insert
+      cacheable = type == :select || type == :delete
 
       lines = []
       lines << 'private' if priv
       lines << "def #{'_' if priv}#{type}_sql"
       lines << 'if sql = opts[:sql]; return static_sql(sql) end' unless priv
+      lines << "if sql = cache_get(:_#{type}_sql); return sql end" if cacheable
       lines << 'check_modification_allowed!' if type == :delete
       lines << 'sql = @opts[:append_sql] || sql_string_origin'
 
@@ -216,6 +218,7 @@ module Sequel
         lines.concat(clause_methods(type, clauses).map{|x| "#{x}(sql)"})
       end
 
+      lines << "cache_set(:_#{type}_sql, sql) if cache_sql?" if cacheable
       lines << 'sql'
       lines << 'end'
 
@@ -522,6 +525,11 @@ module Sequel
     # Append literalization of delayed evaluation to SQL string,
     # causing the delayed evaluation proc to be evaluated.
     def delayed_evaluation_sql_append(sql, delay)
+      # Delayed evaluations are used specifically so the SQL
+      # can differ in subsequent calls, so we definitely don't
+      # want to cache the sql in this case.
+      disable_sql_caching!
+
       if recorder = @opts[:placeholder_literalizer]
         recorder.use(sql, lambda{delay.call(self)}, nil)
       else
@@ -901,6 +909,12 @@ module Sequel
       end
     end
     
+    # Only allow caching the select SQL if the dataset is frozen and hasn't
+    # specifically been marked as not allowing SQL caching.
+    def cache_sql?
+      frozen? && !@opts[:no_cache_sql] && !cache_get(:_no_cache_sql)
+    end
+
     # Raise an InvalidOperation exception if deletion is not allowed
     # for this dataset
     def check_modification_allowed!
@@ -993,6 +1007,11 @@ module Sequel
       end
     end
 
+    # Disable caching of SQL for the current dataset
+    def disable_sql_caching!
+      cache_set(:_no_cache_sql, true)
+    end
+
     # An SQL FROM clause to use in SELECT statements where the dataset has
     # no from tables.
     def empty_from_sql
@@ -1099,10 +1118,9 @@ module Sequel
       end
     end
 
-    # Modify the identifier returned from the database based on the
-    # identifier_output_method.
+    # Upcase identifiers by default when inputting them into the database.
     def input_identifier(v)
-      (i = identifier_input_method) ? v.to_s.send(i) : v.to_s
+      v.to_s.upcase
     end
 
     def insert_into_sql(sql)
@@ -1250,6 +1268,10 @@ module Sequel
     # calls +sql_literal+ if object responds to it, otherwise raises an error.
     # If a database specific type is allowed, this should be overriden in a subclass.
     def literal_other_append(sql, v)
+      # We can't be sure if v will always literalize to the same SQL, so
+      # don't cache SQL for a dataset that uses this.
+      disable_sql_caching!
+
       if v.respond_to?(:sql_literal_append)
         v.sql_literal_append(self, sql)
       elsif v.respond_to?(:sql_literal)
@@ -1496,12 +1518,7 @@ module Sequel
 
     # Whether the symbol cache should be skipped when literalizing the dataset
     def skip_symbol_cache?
-      @skip_symbol_cache
-    end
-
-    # Set the dataset to skip the symbol cache
-    def skip_symbol_cache!
-      @skip_symbol_cache = true
+      @opts[:skip_symbol_cache]
     end
 
     # Append literalization of array of sources/tables to SQL string, raising an Error if there