sequel 5.8.0 → 5.38.0

data/lib/sequel/dataset/query.rb

@@ -81,7 +81,9 @@ module Sequel
       # If the options changed include options in COLUMN_CHANGE_OPTS, the cached
       # columns are deleted.  This method should generally not be called
       # directly by user code.
-      def clone(opts = (return self; nil))
+      def clone(opts = nil || (return self))
+        # return self used above because clone is called by almost all
+        # other query methods, and it is the fastest approach
         c = super(:freeze=>false)
         c.opts.merge!(opts)
         unless opts.each_key{|o| break if COLUMN_CHANGE_OPTS.include?(o)}
@@ -113,6 +115,11 @@ module Sequel
     #  DB[:items].distinct # SQL: SELECT DISTINCT * FROM items
     #  DB[:items].order(:id).distinct(:id) # SQL: SELECT DISTINCT ON (id) * FROM items ORDER BY id
     #  DB[:items].order(:id).distinct{func(:id)} # SQL: SELECT DISTINCT ON (func(id)) * FROM items ORDER BY id
+    #
+    # There is support for emualting the DISTINCT ON support in MySQL, but it
+    # does not support the ORDER of the dataset, and also doesn't work in many
+    # cases if the ONLY_FULL_GROUP_BY sql_mode is used, which is the default on
+    # MySQL 5.7.5+.
     def distinct(*args, &block)
       virtual_row_columns(args, block)
       if args.empty?
@@ -284,7 +291,7 @@ module Sequel
         c
       end
 
-      cache ? cached_dataset(:_from_self_ds, &pr) : pr.call
+      opts.empty? ? cached_dataset(:_from_self_ds, &pr) : pr.call
     end
 
     # Match any of the columns to any of the patterns. The terms can be
@@ -323,16 +330,17 @@ module Sequel
     #   # SELECT * FROM a WHERE ((a LIKE '%foo%' ESCAPE '\') AND (b LIKE '%foo%' ESCAPE '\')
     #   #   AND (a LIKE '%bar%' ESCAPE '\') AND (b LIKE '%bar%' ESCAPE '\'))
     def grep(columns, patterns, opts=OPTS)
+      column_op = opts[:all_columns] ? :AND : :OR
       if opts[:all_patterns]
         conds = Array(patterns).map do |pat|
-          SQL::BooleanExpression.new(opts[:all_columns] ? :AND : :OR, *Array(columns).map{|c| SQL::StringExpression.like(c, pat, opts)})
+          SQL::BooleanExpression.new(column_op, *Array(columns).map{|c| SQL::StringExpression.like(c, pat, opts)})
         end
-        where(SQL::BooleanExpression.new(opts[:all_patterns] ? :AND : :OR, *conds))
+        where(SQL::BooleanExpression.new(:AND, *conds))
       else
         conds = Array(columns).map do |c|
           SQL::BooleanExpression.new(:OR, *Array(patterns).map{|pat| SQL::StringExpression.like(c, pat, opts)})
         end
-        where(SQL::BooleanExpression.new(opts[:all_columns] ? :AND : :OR, *conds))
+        where(SQL::BooleanExpression.new(column_op, *conds))
       end
     end
 
@@ -1039,12 +1047,26 @@ module Sequel
       add_filter(:where, cond, &block)
     end
     
+    # Return a clone of the dataset with an addition named window that can be
+    # referenced in window functions. See Sequel::SQL::Window for a list of
+    # options that can be passed in. Example:
+    #
+    #   DB[:items].window(:w, :partition=>:c1, :order=>:c2)
+    #   # SELECT * FROM items WINDOW w AS (PARTITION BY c1 ORDER BY c2)
+    def window(name, opts)
+      clone(:window=>((@opts[:window]||EMPTY_ARRAY) + [[name, SQL::Window.new(opts)].freeze]).freeze)
+    end
+
     # Add a common table expression (CTE) with the given name and a dataset that defines the CTE.
     # A common table expression acts as an inline view for the query.
     # Options:
     # :args :: Specify the arguments/columns for the CTE, should be an array of symbols.
     # :recursive :: Specify that this is a recursive CTE
     #
+    # PostgreSQL Specific Options:
+    # :materialized :: Set to false to force inlining of the CTE, or true to force not inlining
+    #                  the CTE (PostgreSQL 12+).
+    #
     #   DB[:items].with(:items, DB[:syx].where(Sequel[:name].like('A%')))
     #   # WITH items AS (SELECT * FROM syx WHERE (name LIKE 'A%' ESCAPE '\')) SELECT * FROM items
     def with(name, dataset, opts=OPTS)
@@ -1074,7 +1096,7 @@ module Sequel
     #   #   SELECT i1.id, i1.parent_id FROM i1 INNER JOIN t ON (t.id = i1.parent_id)
     #   # ) SELECT * FROM t
     def with_recursive(name, nonrecursive, recursive, opts=OPTS)
-      raise(Error, 'This datatset does not support common table expressions') unless supports_cte?
+      raise(Error, 'This dataset does not support common table expressions') unless supports_cte?
       if hoist_cte?(nonrecursive)
         s, ds = hoist_cte(nonrecursive)
         s.with_recursive(name, ds, recursive, opts)
@@ -1216,6 +1238,20 @@ module Sequel
       self
     end
 
+    # If invert is true, invert the condition.
+    def _invert_filter(cond, invert)
+      if invert
+        SQL::BooleanExpression.invert(cond)
+      else
+        cond
+      end
+    end
+
+    # Add the given filter condition. Arguments:
+    # clause :: Symbol or which SQL clause to effect, should be :where or :having
+    # cond :: The filter condition to add
+    # invert :: Whether the condition should be inverted (true or false)
+    # combine :: How to combine the condition with an existing condition, should be :AND or :OR
     def add_filter(clause, cond, invert=false, combine=:AND, &block)
       if cond == EMPTY_ARRAY && !block
         raise Error, "must provide an argument to a filtering method if not passing a block"
@@ -1235,8 +1271,7 @@ module Sequel
           cond = nil
         end
 
-        cond = filter_expr(cond, &block)
-        cond = SQL::BooleanExpression.invert(cond) if invert
+        cond = _invert_filter(filter_expr(cond, &block), invert)
         cond = SQL::BooleanExpression.new(combine, @opts[clause], cond) if @opts[clause]
 
         if cond.nil?

data/lib/sequel/dataset/sql.rb

@@ -28,7 +28,7 @@ module Sequel
 
       case values.size
       when 0
-        return insert_sql({})
+        return insert_sql(OPTS)
       when 1
         case vals = values[0]
         when Hash
@@ -389,8 +389,10 @@ module Sequel
         literal_append(sql, args[0])
         sql << ' ' << op.to_s << ' '
         literal_append(sql, args[1])
-        sql << " ESCAPE "
-        literal_append(sql, "\\")
+        if requires_like_escape?
+          sql << " ESCAPE "
+          literal_append(sql, "\\")
+        end
         sql << ')'
       when :ILIKE, :'NOT ILIKE'
         complex_expression_sql_append(sql, (op == :ILIKE ? :LIKE : :"NOT LIKE"), args.map{|v| Sequel.function(:UPPER, v)})
@@ -490,11 +492,24 @@ module Sequel
       end
 
       sql << '('
+      if filter = opts[:filter]
+        filter = filter_expr(filter, &opts[:filter_block])
+      end
       if opts[:*]
-        sql << '*'
+        if filter && !supports_filtered_aggregates?
+          literal_append(sql, Sequel.case({filter=>1}, nil))
+          filter = nil
+        else
+          sql <<  '*'
+        end
       else
         sql << "DISTINCT " if opts[:distinct]
-        expression_list_append(sql, f.args)
+        if filter && !supports_filtered_aggregates?
+          expression_list_append(sql, f.args.map{|arg| Sequel.case({filter=>arg}, nil)})
+          filter = nil
+        else
+          expression_list_append(sql, f.args)
+        end
         if order = opts[:order]
           sql << " ORDER BY "
           expression_list_append(sql, order)
@@ -508,9 +523,9 @@ module Sequel
         sql << ')'
       end
 
-      if filter = opts[:filter]
+      if filter
         sql << " FILTER (WHERE "
-        literal_append(sql, filter_expr(filter, &opts[:filter_block]))
+        literal_append(sql, filter)
         sql << ')'
       end
 
@@ -557,13 +572,30 @@ module Sequel
 
     # Append literalization of ordered expression to SQL string.
     def ordered_expression_sql_append(sql, oe)
+      if emulate = requires_emulating_nulls_first?
+        case oe.nulls
+        when :first
+          null_order = 0
+        when :last
+          null_order = 2
+        end
+
+        if null_order
+          literal_append(sql, Sequel.case({{oe.expression=>nil}=>null_order}, 1))
+          sql << ", "
+        end
+      end
+
       literal_append(sql, oe.expression)
       sql << (oe.descending ? ' DESC' : ' ASC')
-      case oe.nulls
-      when :first
-        sql << " NULLS FIRST"
-      when :last
-        sql << " NULLS LAST"
+
+      unless emulate
+        case oe.nulls
+        when :first
+          sql << " NULLS FIRST"
+        when :last
+          sql << " NULLS LAST"
+        end
       end
     end
 
@@ -577,7 +609,7 @@ module Sequel
           sql << str
         else
           re = /:(#{args.keys.map{|k| Regexp.escape(k.to_s)}.join('|')})\b/
-          loop do
+          while true
             previous, q, str = str.partition(re)
             sql << previous
             literal_append(sql, args[($1||q[1..-1].to_s).to_sym]) unless q.empty?
@@ -596,13 +628,13 @@ module Sequel
       else
         i = -1
         match_len = args.length - 1
-        loop do
+        while true
           previous, q, str = str.partition('?')
           sql << previous
           literal_append(sql, args.at(i+=1)) unless q.empty?
           if str.empty?
             unless i == match_len
-              raise Error, "Mismatched number of placeholders (#{i+1}) and placeholder arguments (#{args.length}) when using placeholder array"
+              raise Error, "Mismatched number of placeholders (#{i+1}) and placeholder arguments (#{args.length}) when using placeholder string"
             end
             break
           end
@@ -698,8 +730,19 @@ module Sequel
 
     # Append literalization of subscripts (SQL array accesses) to SQL string.
     def subscript_sql_append(sql, s)
+      case s.expression
+      when Symbol, SQL::Subscript, SQL::Identifier, SQL::QualifiedIdentifier
+        # nothing
+      else
+        wrap_expression = true
+        sql << '('
+      end
       literal_append(sql, s.expression)
-      sql << '['
+      if wrap_expression
+        sql << ')['
+      else
+        sql << '['
+      end
       sub = s.sub
       if sub.length == 1 && (range = sub.first).is_a?(Range)
         literal_append(sql, range.begin)
@@ -716,41 +759,86 @@ module Sequel
     # Append literalization of windows (for window functions) to SQL string.
     def window_sql_append(sql, opts)
       raise(Error, 'This dataset does not support window functions') unless supports_window_functions?
-      sql << '('
-      window, part, order, frame = opts.values_at(:window, :partition, :order, :frame)
       space = false
       space_s = ' '
-      if window
+
+      sql << '('
+
+      if window = opts[:window]
         literal_append(sql, window)
         space = true
       end
-      if part
+
+      if part = opts[:partition]
         sql << space_s if space
         sql << "PARTITION BY "
         expression_list_append(sql, Array(part))
         space = true
       end
-      if order
+
+      if order = opts[:order]
         sql << space_s if space
         sql << "ORDER BY "
         expression_list_append(sql, Array(order))
         space = true
       end
-      case frame
-        when nil
-          # nothing
-        when :all
-          sql << space_s if space
-          sql << "ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING"
-        when :rows
-          sql << space_s if space
-          sql << "ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW"
-        when String
-          sql << space_s if space
+
+      if frame = opts[:frame]
+        sql << space_s if space
+
+        if frame.is_a?(String)
           sql << frame
         else
-          raise Error, "invalid window frame clause, should be :all, :rows, a string, or nil"
+          case frame
+          when :all
+            frame_type = :rows
+            frame_start = :preceding
+            frame_end = :following
+          when :rows, :range, :groups
+            frame_type = frame
+            frame_start = :preceding
+            frame_end = :current
+          when Hash
+            frame_type = frame[:type]
+            unless frame_type == :rows || frame_type == :range || frame_type == :groups
+              raise Error, "invalid window :frame :type option: #{frame_type.inspect}"
+            end
+            unless frame_start = frame[:start]
+              raise Error, "invalid window :frame :start option: #{frame_start.inspect}"
+            end
+            frame_end = frame[:end]
+            frame_exclude = frame[:exclude]
+          else
+            raise Error, "invalid window :frame option: #{frame.inspect}"
+          end
+
+          sql << frame_type.to_s.upcase << " "
+          sql << 'BETWEEN ' if frame_end
+          window_frame_boundary_sql_append(sql, frame_start, :preceding)
+          if frame_end
+            sql << " AND "
+            window_frame_boundary_sql_append(sql, frame_end, :following)
+          end
+
+          if frame_exclude
+            sql << " EXCLUDE "
+
+            case frame_exclude
+            when :current
+              sql << "CURRENT ROW"
+            when :group
+              sql << "GROUP"
+            when :ties
+              sql << "TIES"
+            when :no_others
+              sql << "NO OTHERS"
+            else
+              raise Error, "invalid window :frame :exclude option: #{frame_exclude.inspect}"
+            end
+          end
+        end
       end
+
       sql << ')'
     end
 
@@ -1410,6 +1498,22 @@ module Sequel
     alias delete_where_sql select_where_sql
     alias update_where_sql select_where_sql
     
+    def select_window_sql(sql)
+      if ws = @opts[:window]
+        sql << " WINDOW "
+        c = false
+        co = ', '
+        as = ' AS '
+        ws.map do |name, window|
+          sql << co if c
+          literal_append(sql, name)
+          sql << as
+          literal_append(sql, window)
+          c ||= true
+        end
+      end
+    end
+
     def select_with_sql(sql)
       return unless supports_cte?
       ws = opts[:with]
@@ -1419,13 +1523,7 @@ module Sequel
       comma = ', '
       ws.each do |w|
         sql << comma if c
-        quote_identifier_append(sql, w[:name])
-        if args = w[:args]
-         sql << '('
-         identifier_list_append(sql, args)
-         sql << ')'
-        end
-        sql << ' AS '
+        select_with_sql_prefix(sql, w)
         literal_dataset_append(sql, w[:dataset])
         c ||= true
       end
@@ -1439,6 +1537,16 @@ module Sequel
       "WITH "
     end
 
+    def select_with_sql_prefix(sql, w)
+      quote_identifier_append(sql, w[:name])
+      if args = w[:args]
+       sql << '('
+       identifier_list_append(sql, args)
+       sql << ')'
+      end
+      sql << ' AS '
+    end
+
     # Whether the symbol cache should be skipped when literalizing the dataset
     def skip_symbol_cache?
       @opts[:skip_symbol_cache]
@@ -1490,7 +1598,17 @@ module Sequel
 
     # Append literalization of the subselect to SQL string.
     def subselect_sql_append(sql, ds)
-      ds.clone(:append_sql=>sql).sql
+      sds = subselect_sql_dataset(sql, ds)
+      sds.sql
+      unless sds.send(:cache_sql?)
+        # If subquery dataset does not allow caching SQL,
+        # then this dataset should not allow caching SQL.
+        disable_sql_caching!
+      end
+    end
+
+    def subselect_sql_dataset(sql, ds)
+      ds.clone(:append_sql=>sql)
     end
 
     # The number of decimal digits of precision to use in timestamps.
@@ -1533,5 +1651,35 @@ module Sequel
     def update_update_sql(sql)
       sql << 'UPDATE'
     end
+
+    def window_frame_boundary_sql_append(sql, boundary, direction)
+      case boundary
+      when :current
+       sql << "CURRENT ROW"
+      when :preceding
+        sql << "UNBOUNDED PRECEDING"
+      when :following
+        sql << "UNBOUNDED FOLLOWING"
+      else
+        if boundary.is_a?(Array)
+          offset, direction = boundary
+          unless boundary.length == 2 && (direction == :preceding || direction == :following)
+            raise Error, "invalid window :frame boundary (:start or :end) option: #{boundary.inspect}"
+          end
+        else
+          offset = boundary
+        end
+
+        case offset
+        when Numeric, String, SQL::Cast
+          # nothing
+        else
+          raise Error, "invalid window :frame boundary (:start or :end) option: #{boundary.inspect}"
+        end
+
+        literal_append(sql, offset)
+        sql << (direction == :preceding ? " PRECEDING" : " FOLLOWING")
+      end
+    end
   end
 end