sequel 5.95.0 → 5.99.0

data/lib/sequel/extensions/pg_auto_parameterize_duplicate_query_detection.rb

@@ -0,0 +1,191 @@
+# frozen-string-literal: true
+#
+# The pg_auto_parameterize_duplicate_query_detection extension builds on the
+# pg_auto_parameterize extension, adding support for detecting duplicate
+# queries inside a block that occur at the same location. This is designed
+# mostly to catch duplicate query issues (e.g. N+1 queries) during testing.
+#
+# To detect duplicate queries inside a block of code, wrap the code with
+# +detect_duplicate_queries+:
+#
+#   DB.detect_duplicate_queries{your_code}
+#
+# With this approach, if the test runs code where the same query is executed
+# more than once with the same call stack, a
+# Sequel::Postgres::AutoParameterizeDuplicateQueryDetection::DuplicateQueries
+# exception will be raised.
+#
+# You can pass the +:warn+ option to +detect_duplicate_queries+ to warn
+# instead of raising. Note that if the block passed to +detect_duplicate_queries+
+# raises, this extension will warn, and raise the original exception.
+#
+# For more control, you can pass the +:handler+ option to
+# +detect_duplicate_queries+. If the +:handler+ option is provided, this
+# extension will call the +:handler+ option with the hash of duplicate
+# query information, and will not raise or warn. This can be useful in
+# production environments, to record duplicate queries for later analysis.
+#
+# For accuracy, the entire call stack is always used as part of the hash key
+# to determine whether a query is duplicate. However, you can filter the
+# displayed backtrace by using the +:backtrace_filter+ option.
+#
+# +detect_duplicate_queries+ is concurrency aware, it uses the same approach
+# that Sequel's default connection pools use. So if you are running multiple
+# threads, +detect_duplicate_queries+ will only report duplicate queries for
+# the current thread (or fiber if the fiber_concurrency extension is used).
+#
+# When testing applications, it's probably best to use this to wrap the
+# application being tested. For example, testing with rack-test, if your app
+# is +App+, you would want to wrap it:
+#
+#   include Rack::Test::Methods
+#
+#   WrappedApp = lambda do |env|
+#     DB.detect_duplicate_queries{App.call(env)}
+#   end
+#
+#   def app
+#     WrappedApp
+#   end
+#
+# It is possible to use this to wrap each separate spec using an around hook,
+# but that can result in false positives when using libraries that have
+# implicit retrying (such as Capybara), as you can have the same call stack
+# for multiple requests.
+#
+# Related module: Sequel::Postgres::AutoParameterizeDuplicateQueryDetection
+
+module Sequel
+  module Postgres
+    # Enable detecting duplicate queries inside a block
+    module AutoParameterizeDuplicateQueryDetection
+      def self.extended(db)
+        db.instance_exec do
+          @duplicate_query_detection_contexts = {}
+          @duplicate_query_detection_mutex = Mutex.new
+        end
+      end
+
+      # Exception class raised when duplicate queries are detected.
+      class DuplicateQueries < Sequel::Error
+        # A hash of queries that were duplicate. Keys are arrays
+        # with 2 entries, the first being the query SQL, and the
+        # second being the related call stack (backtrace).
+        # The values are the number of query executions.
+        attr_reader :queries
+
+        def initialize(message, queries)
+          @queries = queries
+          super(message)
+        end
+      end
+
+      # Record each query executed so duplicates can be detected,
+      # if queries are being recorded.
+      def execute(sql, opts=OPTS, &block)
+        record, queries = duplicate_query_recorder_state
+
+        if record
+          queries[[sql.is_a?(Symbol) ? sql : sql.to_s, caller].freeze] += 1
+        end
+
+        super
+      end
+
+      # Ignore (do not record) queries inside given block. This can
+      # be useful in situations where you want to run your entire test suite
+      # with duplicate query detection, but you have duplicate queries in
+      # some parts of your application where it is not trivial to use a
+      # different approach. You can mark those specific sections with
+      # +ignore_duplicate_queries+, and still get duplicate query detection
+      # for the rest of the application.
+      def ignore_duplicate_queries(&block)
+        if state = duplicate_query_recorder_state
+          change_duplicate_query_recorder_state(state, false, &block)
+        else
+          # If we are not inside a detect_duplicate_queries block, there is
+          # no need to do anything, since we are not recording queries.
+          yield
+        end
+      end
+
+      # Run the duplicate query detector during the block.
+      # Options:
+      #
+      # :backtrace_filter :: Regexp used to filter the displayed backtrace.
+      # :handler :: If present, called with hash of duplicate query information,
+      #             instead of raising or warning.
+      # :warn :: Always warn instead of raising for duplicate queries.
+      #
+      # Note that if you nest calls to this method, only the top
+      # level call will respect the passed options.
+      def detect_duplicate_queries(opts=OPTS, &block)
+        current = Sequel.current
+        if state = duplicate_query_recorder_state(current)
+          return change_duplicate_query_recorder_state(state, true, &block)
+        end
+
+        @duplicate_query_detection_mutex.synchronize do
+          @duplicate_query_detection_contexts[current] = [true, Hash.new(0)]
+        end
+
+        begin
+          yield
+        rescue Exception => e
+          raise
+        ensure
+          _, queries = @duplicate_query_detection_mutex.synchronize do
+            @duplicate_query_detection_contexts.delete(current)
+          end
+          queries.delete_if{|_,v| v < 2}
+
+          unless queries.empty?
+            if handler = opts[:handler]
+              handler.call(queries)
+            else
+              backtrace_filter = opts[:backtrace_filter]
+              backtrace_filter_note = backtrace_filter ? " (filtered)" : ""
+              query_info = queries.map do |k,v|
+                backtrace = k[1]
+                backtrace = backtrace.grep(backtrace_filter) if backtrace_filter
+                "times:#{v}\nsql:#{k[0]}\nbacktrace#{backtrace_filter_note}:\n#{backtrace.join("\n")}\n"
+              end
+              message = "duplicate queries detected:\n\n#{query_info.join("\n")}"
+
+              if e || opts[:warn]
+                warn(message)
+              else
+                raise DuplicateQueries.new(message, queries)
+              end
+            end
+          end
+        end
+      end
+
+      private
+
+      # Get the query record state for the given context.
+      def duplicate_query_recorder_state(current=Sequel.current)
+        @duplicate_query_detection_mutex.synchronize{@duplicate_query_detection_contexts[current]}
+      end
+
+      # Temporarily change whether to record queries for the block, resetting the
+      # previous setting after the block exits.
+      def change_duplicate_query_recorder_state(state, setting)
+        prev = state[0]
+        state[0] = setting
+        
+        begin
+          yield
+        ensure
+          state[0] = prev
+        end
+      end
+    end
+  end
+
+  Database.register_extension(:pg_auto_parameterize_duplicate_query_detection) do |db|
+    db.extension(:pg_auto_parameterize)
+    db.extend(Postgres::AutoParameterizeDuplicateQueryDetection)
+  end
+end

data/lib/sequel/extensions/pg_auto_parameterize_in_array.rb

@@ -120,7 +120,7 @@ module Sequel
       # The bound variable type string to use for the bound variable array.
       # Returns nil if a bound variable should not be used for the array.
       def _bound_variable_type_for_array(r)
-        return unless Array === r && r.size >= pg_auto_parameterize_min_array_size
+        return unless (Array === r || Set === r) && r.size >= pg_auto_parameterize_min_array_size
         classes = r.map(&:class)
         classes.uniq!
         classes.delete(NilClass)
@@ -165,6 +165,7 @@ module Sequel
 
       # Convert RHS of IN/NOT IN operator to PGArray with given type.
       def _convert_array_to_pg_array_with_type(r, type)
+        r = r.to_a if Set === r
         Sequel.pg_array(r, type)
       end
     end

data/lib/sequel/extensions/pg_hstore_ops.rb

@@ -294,13 +294,18 @@ module Sequel
         SQL::Function.new(name, self, *args)
       end
 
-      # Wrap argument in a PGArray if it is an array
+      # Wrap argument in a PGArray if it is an array or a set
       def wrap_input_array(obj)
-        if obj.is_a?(Array) && Sequel.respond_to?(:pg_array) 
-          Sequel.pg_array(obj)
-        else
-          obj
+        if Sequel.respond_to?(:pg_array)
+          case obj
+          when Array
+            return Sequel.pg_array(obj)
+          when Set
+            return Sequel.pg_array(obj.to_a)
+          end
         end
+
+        obj
       end
 
       # Wrap argument in an Hstore if it is a hash

data/lib/sequel/extensions/pg_json_ops.rb

@@ -166,6 +166,11 @@
 #   #   "d" date EXISTS FALSE ON ERROR
 #   # ))
 #
+# On PostgreSQL 18+, strip_nulls can take an argument for whether to strip in arrays
+#
+#   j.strip_nulls(in_arrays: true)   # json_strip_nulls(json_column, true)
+#   j.strip_nulls(in_arrays: false)  # json_strip_nulls(json_column, false)
+#
 # If you are also using the pg_json extension, you should load it before
 # loading this extension.  Doing so will allow you to use the #op method on
 # JSONHash, JSONHarray, JSONBHash, and JSONBArray, allowing you to perform json/jsonb operations
@@ -380,11 +385,22 @@ module Sequel
         self.class.new(JSONQueryOp.new(self, path, opts))
       end
 
-      # Returns a json value stripped of all internal null values.
+      # Returns a json value stripped of all internal null values. Options:
+      #
+      # :in_arrays :: Whether to strip null values in JSON arrays
       #
-      #   json_op.strip_nulls # json_strip_nulls(json)
-      def strip_nulls
-        self.class.new(function(:strip_nulls))
+      #   json_op.strip_nulls                   # json_strip_nulls(json)
+      #   json_op.strip_nulls(in_arrays: true)  # json_strip_nulls(json, true)
+      #   json_op.strip_nulls(in_arrays: false) # json_strip_nulls(json, false)
+      def strip_nulls(opts=OPTS)
+        in_arrays = opts[:in_arrays]
+        f = if in_arrays.nil?
+          function(:strip_nulls)
+        else
+          function(:strip_nulls, in_arrays)
+        end
+
+        self.class.new(f)
       end
 
       # Returns json_table SQL function expression, querying JSON data and returning
@@ -825,13 +841,20 @@ module Sequel
         Sequel::SQL::BooleanExpression.new(:NOOP, Sequel::SQL::PlaceholderLiteralString.new(str, [value, other]))
       end
 
-      # Wrap argument in a PGArray if it is an array
+      # Wrap argument in a PGArray if it is an array or a set
       def wrap_input_array(obj)
-        if obj.is_a?(Array) && Sequel.respond_to?(:pg_array) 
-          Sequel.pg_array(obj)
-        else
-          obj
+        # :nocov:
+        if Sequel.respond_to?(:pg_array)
+        # :nocov:
+          case obj
+          when Array
+            return Sequel.pg_array(obj)
+          when Set
+            return Sequel.pg_array(obj.to_a)
+          end
         end
+
+        obj
       end
 
       # Wrap argument in a JSONBArray or JSONBHash if it is an array or hash.

data/lib/sequel/extensions/pg_multirange.rb

@@ -339,7 +339,7 @@ module Sequel
 
       # Allow automatic parameterization.
       def sequel_auto_param_type(ds)
-        "::#{db_type}"
+        "::#{db_type}" if all?{|range| range.is_a?(Range) || ds.send(:auto_param_type, range)}
       end
     end
   end

data/lib/sequel/extensions/pg_range.rb

@@ -421,6 +421,17 @@ module Sequel
         @exclude_end
       end
 
+      # Support a friendly output
+      def inspect
+        range = if empty?
+          "empty"
+        else
+          "#{@exclude_begin ? "(" : "["}#{@begin},#{@end}#{@exclude_end ? ")" : "]"}"
+        end
+
+        "#<#{self.class.name} #{range}#{"::#{@db_type}" if @db_type}>"
+      end
+
       # Append a literalize version of the receiver to the sql.
       def sql_literal_append(ds, sql)
         if (s = @db_type) && !empty?
@@ -481,9 +492,9 @@ module Sequel
         end
       end
 
-      # Allow automatic parameterization for ranges with types.
+      # Allow automatic parameterization for ranges with types, if both start .
       def sequel_auto_param_type(ds)
-        "::#{db_type}" if db_type
+        "::#{db_type}" if db_type && (!@begin || ds.send(:auto_param_type, @begin)) && (!@end || ds.send(:auto_param_type, @end))
       end
 
       private

data/lib/sequel/extensions/pg_row.rb

@@ -319,8 +319,8 @@ module Sequel
         end
 
         # Typecast the given object to the appropriate type using the
-        # typecaster.  Note that this does not conversion for the members
-        # of the composite type, since those conversion expect strings and
+        # typecaster.  Note that this does no conversion for the members
+        # of the composite type, since those conversions expect strings and
         # strings may not be provided.  
         def typecast(obj)
           case obj 

data/lib/sequel/extensions/set_literalizer.rb

@@ -1,56 +1,37 @@
 # frozen-string-literal: true
 #
-# The set_literalizer extension allows for using Set instances in many of the
-# same places that you would use Array instances:
-#
-#   DB[:table].where(column: Set.new([1, 2, 3]))
-#   # SELECT FROM table WHERE (column IN (1, 2, 3))
-#
-# To load the extension into all datasets created from a given Database:
-#
-#   DB.extension :set_literalizer
+# The set_literalizer extension should no longer be used, as Sequel
+# now supports Set values by default. For backwards compatibility
+# the set_literalizer extension will treat a set that contains only
+# 2 element arrays as a condition specifier (matching the behavior
+# for arrays where all elements are 2 element arrays). This is not
+# compatible with Sequel's current default behavior. If you are
+# relying on this behavior, it is recommended you convert the set
+# to an array.
 #
 # Related module: Sequel::Dataset::SetLiteralizer
 
-require 'set'
-
 module Sequel
+  # SEQUEL6: Remove
+  Sequel::Deprecation.deprecate("The set_literalizer extension", "Sequel now supports set literalization by default")
+
   class Dataset
     module SetLiteralizer
-      # Try to generate the same SQL for Set instances used in datasets
-      # that would be used for equivalent Array instances.
-      def complex_expression_sql_append(sql, op, args)
-        # Array instances are treated specially by
-        # Sequel::SQL::BooleanExpression.from_value_pairs. That cannot
-        # be modified by a dataset extension, so this tries to convert
-        # the complex expression values generated by default to what would
-        # be the complex expression values used for the equivalent array.
-        case op
-        when :'=', :'!='
-          if (set = args[1]).is_a?(Set)
-            op = op == :'=' ? :IN : :'NOT IN'
-            col = args[0]
-            array = set.to_a
-            if Sequel.condition_specifier?(array) && col.is_a?(Array)
-              array = Sequel.value_list(array)
-            end
-            args = [col, array]
-          end
-        end
-
-        super
-      end
-
       private
 
-      # Literalize Set instances by converting the set to array.
-      def literal_other_append(sql, v)
-        if Set === v
-          literal_append(sql, v.to_a) 
+      # Allow using sets as condition specifiers.
+      def filter_expr(expr = nil, &block)
+        if expr.is_a?(Set)
+          expr
         else
           super
         end
       end
+
+      # Literalize Set instances by converting the set to array.
+      def literal_set_append(sql, v)
+        literal_append(sql, v.to_a) 
+      end
     end
 
     register_extension(:set_literalizer, SetLiteralizer)

data/lib/sequel/extensions/split_array_nil.rb

@@ -25,6 +25,11 @@
 #   # with split_array_nils extension:
 #   #   SELECT * FROM table WHERE ((column NOT IN (1)) AND (column IS NOT NULL)))
 #
+# In addition to arrays, this splitting is also done for sets:
+#
+#   ds = DB[:table].where(column: Set[1, nil])
+#   # SELECT * FROM table WHERE ((column IN (1)) OR (column IS NULL)))
+#
 # To use this extension with a single dataset:
 #
 #   ds = ds.extension(:split_array_nil)
@@ -47,9 +52,14 @@ module Sequel
       case op
       when :IN, :"NOT IN"
         vals = args[1]
-        if vals.is_a?(Array) && vals.any?(&:nil?)
+        if (vals.is_a?(Array) || vals.is_a?(Set)) && vals.any?(&:nil?)
           cols = args[0]
-          vals = vals.compact
+          if vals.is_a?(Set)
+            vals = vals.dup
+            vals.delete(nil)
+          else
+            vals = vals.compact
+          end
           c = Sequel::SQL::BooleanExpression
           if op == :IN
             literal_append(sql, c.new(:OR, c.new(:IN, cols, vals), c.new(:IS, cols, nil)))

data/lib/sequel/extensions/to_dot.rb

@@ -79,6 +79,11 @@ module Sequel
         e.each_with_index do |val, j|
           v(val, j)
         end
+      when Set
+        dot "Set"
+        e.each_with_index do |val, j|
+          v(val, j)
+        end
       when Hash
         dot "Hash"
         e.each do |k, val|

data/lib/sequel/model/associations.rb

@@ -302,7 +302,7 @@ module Sequel
 
             if strategy == :window_function
               delete_rn = ds.row_number_column 
-              objects.each{|obj| obj.values.delete(delete_rn)}
+              objects.each{|obj| obj.remove_key!(delete_rn)}
             end
           elsif strategy == :union
             objects = []
@@ -1291,11 +1291,11 @@ module Sequel
           name = self[:name]
 
           self[:model].eager_load_results(self, eo) do |assoc_record|
-            assoc_record.values.delete(delete_rn) if delete_rn
+            assoc_record.remove_key!(delete_rn) if delete_rn
             hash_key = if uses_lcks
-              left_key_alias.map{|k| assoc_record.values.delete(k)}
+              left_key_alias.map{|k| assoc_record.remove_key!(k)}
             else
-              assoc_record.values.delete(left_key_alias)
+              assoc_record.remove_key!(left_key_alias)
             end
 
             objects = h[hash_key]
@@ -2387,7 +2387,7 @@ module Sequel
             delete_rn = opts.delete_row_number_column
 
             eager_load_results(opts, eo) do |assoc_record|
-              assoc_record.values.delete(delete_rn) if delete_rn
+              assoc_record.remove_key!(delete_rn) if delete_rn
               hash_key = uses_cks ? km.map{|k| assoc_record.get_column_value(k)} : assoc_record.get_column_value(km)
               objects = h[hash_key]
               if assign_singular
@@ -3063,7 +3063,7 @@ module Sequel
           if (((op == :'=' || op == :'!=') && r.is_a?(Sequel::Model)) ||
               (multiple = ((op == :IN || op == :'NOT IN') && ((is_ds = r.is_a?(Sequel::Dataset)) || (r.respond_to?(:all?) && r.all?{|x| x.is_a?(Sequel::Model)})))))
             l = args[0]
-            if ar = model.association_reflections[l]
+            if ar = model.association_reflection(l)
               raise Error, "filtering by associations is not allowed for #{ar.inspect}" if ar[:allow_filtering_by] == false
 
               if multiple
@@ -3544,7 +3544,7 @@ module Sequel
           else
             vals = Array(obj).reject{|o| !meths.all?{|m| o.get_column_value(m)}}
             return SQL::Constants::FALSE if vals.empty?
-            if obj.is_a?(Array)
+            if obj.is_a?(Array) || obj.is_a?(Set)
               if keys.length == 1
                 meth = meths.first
                 {keys.first=>vals.map{|o| o.get_column_value(meth)}}

data/lib/sequel/model/base.rb

@@ -2,13 +2,15 @@
 
 module Sequel
   class Model
+    # SEQUEL6: Remove Enumerable here, and send all Enumerable methods to dataset
+    # by default, with a plugin for the current behavior.
     extend Enumerable
     extend Inflections
 
     # Class methods for Sequel::Model that implement basic model functionality.
     #
     # * All of the following methods have class methods created that send the method
-    #   to the model's dataset: all, as_hash, avg, count, cross_join, distinct, each,
+    #   to the model's dataset: all, any?, as_hash, as_set, avg, count, cross_join, distinct, each,
     #   each_server, empty?, except, exclude, exclude_having, fetch_rows,
     #   filter, first, first!, for_update, from, from_self, full_join, full_outer_join,
     #   get, graph, grep, group, group_and_count, group_append, group_by, having, import,
@@ -17,7 +19,7 @@ module Sequel
     #   natural_join, natural_left_join, natural_right_join, offset, order, order_append, order_by,
     #   order_more, order_prepend, paged_each, qualify, reverse, reverse_order, right_join,
     #   right_outer_join, select, select_all, select_append, select_group, select_hash,
-    #   select_hash_groups, select_map, select_more, select_order_map, select_prepend, server,
+    #   select_hash_groups, select_map, select_more, select_order_map, select_prepend, select_set, server,
     #   single_record, single_record!, single_value, single_value!, sum, to_hash, to_hash_groups,
     #   truncate, unfiltered, ungraphed, ungrouped, union, unlimited, unordered, where, where_all,
     #   where_each, where_single_value, with, with_recursive, with_sql
@@ -695,7 +697,7 @@ module Sequel
       end
   
       # Add model methods that call dataset methods
-      Plugins.def_dataset_methods(self, (Dataset::ACTION_METHODS + Dataset::QUERY_METHODS + [:each_server]) - [:<<, :or, :[], :columns, :columns!, :delete, :update, :set_graph_aliases, :add_graph_aliases])
+      Plugins.def_dataset_methods(self, (Dataset::ACTION_METHODS + Dataset::QUERY_METHODS + [:any?, :each_server]) - [:<<, :or, :[], :columns, :columns!, :delete, :update, :set_graph_aliases, :add_graph_aliases])
   
       private
       
@@ -1358,17 +1360,16 @@ END
         @values.keys
       end
       
-      # Refresh this record using +for_update+ (by default, or the specified style when given)
+      # Refresh this record using +:update+ lock style (by default, or the specified style when given),
       # unless this is a new record.  Returns self. This can be used to make sure no other
-      # process is updating the record at the same time.
+      # process can modify the record during the transaction containing this call.  Using
+      # this method only makes sense inside transactions.
       #
       # If style is a string, it will be used directly. You should never pass a string
       # to this method that is derived from user input, as that can lead to
       # SQL injection.
       #
-      # A symbol may be used for database independent locking behavior, but
-      # all supported symbols have separate methods (e.g. for_update).
-      #
+      # A symbol may be used if the adapter supports that lock style.
       #
       #   a = Artist[1]
       #   Artist.db.transaction do
@@ -1378,7 +1379,7 @@ END
       #
       #   a = Artist[2]
       #   Artist.db.transaction do
-      #     a.lock!('FOR NO KEY UPDATE')
+      #     a.lock!(:no_key_update)
       #     a.update(name: 'B')
       #   end
       def lock!(style=:update)
@@ -1500,6 +1501,20 @@ END
         refresh
       end
   
+      # Remove a key from the instances values, and return the value
+      # of the key.
+      #
+      #   a = Album[1]
+      #   a.values
+      #   # => {id: 1, artist_id: 2}
+      #   a.remove_key!(:artist_id)
+      #   # => 2
+      #   a.values
+      #   # => {id: 1}
+      def remove_key!(key)
+        @values.delete(key)
+      end
+
       # Creates or updates the record, after making sure the record
       # is valid and before hooks execute successfully. Fails if:
       #
@@ -2329,5 +2344,17 @@ END
     # :nocov:
       singleton_class.send(:undef_method, :initialize_clone, :initialize_dup)
     end
+
+    # :nocov:
+    if defined?(Sequel::Postgres::SEQUEL_PG_VERSION_INTEGER) && Sequel::Postgres::SEQUEL_PG_VERSION_INTEGER >= 11800
+      # Automatically optimize model loading when sequel/core was loaded,
+      # then sequel/adapters/postgres (with sequel_pg), then sequel/model
+      begin
+        require "sequel_pg/model"
+      rescue LoadError
+        # nothing
+      end
+    end
+    # :nocov:
   end
 end