sequel 5.83.0 → 5.106.0

data/lib/sequel/extensions/pg_auto_parameterize_duplicate_query_detection.rb

@@ -0,0 +1,191 @@
+# frozen-string-literal: true
+#
+# The pg_auto_parameterize_duplicate_query_detection extension builds on the
+# pg_auto_parameterize extension, adding support for detecting duplicate
+# queries inside a block that occur at the same location. This is designed
+# mostly to catch duplicate query issues (e.g. N+1 queries) during testing.
+#
+# To detect duplicate queries inside a block of code, wrap the code with
+# +detect_duplicate_queries+:
+#
+#   DB.detect_duplicate_queries{your_code}
+#
+# With this approach, if the test runs code where the same query is executed
+# more than once with the same call stack, a
+# Sequel::Postgres::AutoParameterizeDuplicateQueryDetection::DuplicateQueries
+# exception will be raised.
+#
+# You can pass the +:warn+ option to +detect_duplicate_queries+ to warn
+# instead of raising. Note that if the block passed to +detect_duplicate_queries+
+# raises, this extension will warn, and raise the original exception.
+#
+# For more control, you can pass the +:handler+ option to
+# +detect_duplicate_queries+. If the +:handler+ option is provided, this
+# extension will call the +:handler+ option with the hash of duplicate
+# query information, and will not raise or warn. This can be useful in
+# production environments, to record duplicate queries for later analysis.
+#
+# For accuracy, the entire call stack is always used as part of the hash key
+# to determine whether a query is duplicate. However, you can filter the
+# displayed backtrace by using the +:backtrace_filter+ option.
+#
+# +detect_duplicate_queries+ is concurrency aware, it uses the same approach
+# that Sequel's default connection pools use. So if you are running multiple
+# threads, +detect_duplicate_queries+ will only report duplicate queries for
+# the current thread (or fiber if the fiber_concurrency extension is used).
+#
+# When testing applications, it's probably best to use this to wrap the
+# application being tested. For example, testing with rack-test, if your app
+# is +App+, you would want to wrap it:
+#
+#   include Rack::Test::Methods
+#
+#   WrappedApp = lambda do |env|
+#     DB.detect_duplicate_queries{App.call(env)}
+#   end
+#
+#   def app
+#     WrappedApp
+#   end
+#
+# It is possible to use this to wrap each separate spec using an around hook,
+# but that can result in false positives when using libraries that have
+# implicit retrying (such as Capybara), as you can have the same call stack
+# for multiple requests.
+#
+# Related module: Sequel::Postgres::AutoParameterizeDuplicateQueryDetection
+
+module Sequel
+  module Postgres
+    # Enable detecting duplicate queries inside a block
+    module AutoParameterizeDuplicateQueryDetection
+      def self.extended(db)
+        db.instance_exec do
+          @duplicate_query_detection_contexts = {}
+          @duplicate_query_detection_mutex = Mutex.new
+        end
+      end
+
+      # Exception class raised when duplicate queries are detected.
+      class DuplicateQueries < Sequel::Error
+        # A hash of queries that were duplicate. Keys are arrays
+        # with 2 entries, the first being the query SQL, and the
+        # second being the related call stack (backtrace).
+        # The values are the number of query executions.
+        attr_reader :queries
+
+        def initialize(message, queries)
+          @queries = queries
+          super(message)
+        end
+      end
+
+      # Record each query executed so duplicates can be detected,
+      # if queries are being recorded.
+      def execute(sql, opts=OPTS, &block)
+        record, queries = duplicate_query_recorder_state
+
+        if record
+          queries[[sql.is_a?(Symbol) ? sql : sql.to_s, caller].freeze] += 1
+        end
+
+        super
+      end
+
+      # Ignore (do not record) queries inside given block. This can
+      # be useful in situations where you want to run your entire test suite
+      # with duplicate query detection, but you have duplicate queries in
+      # some parts of your application where it is not trivial to use a
+      # different approach. You can mark those specific sections with
+      # +ignore_duplicate_queries+, and still get duplicate query detection
+      # for the rest of the application.
+      def ignore_duplicate_queries(&block)
+        if state = duplicate_query_recorder_state
+          change_duplicate_query_recorder_state(state, false, &block)
+        else
+          # If we are not inside a detect_duplicate_queries block, there is
+          # no need to do anything, since we are not recording queries.
+          yield
+        end
+      end
+
+      # Run the duplicate query detector during the block.
+      # Options:
+      #
+      # :backtrace_filter :: Regexp used to filter the displayed backtrace.
+      # :handler :: If present, called with hash of duplicate query information,
+      #             instead of raising or warning.
+      # :warn :: Always warn instead of raising for duplicate queries.
+      #
+      # Note that if you nest calls to this method, only the top
+      # level call will respect the passed options.
+      def detect_duplicate_queries(opts=OPTS, &block)
+        current = Sequel.current
+        if state = duplicate_query_recorder_state(current)
+          return change_duplicate_query_recorder_state(state, true, &block)
+        end
+
+        @duplicate_query_detection_mutex.synchronize do
+          @duplicate_query_detection_contexts[current] = [true, Hash.new(0)]
+        end
+
+        begin
+          yield
+        rescue Exception => e
+          raise
+        ensure
+          _, queries = @duplicate_query_detection_mutex.synchronize do
+            @duplicate_query_detection_contexts.delete(current)
+          end
+          queries.delete_if{|_,v| v < 2}
+
+          unless queries.empty?
+            if handler = opts[:handler]
+              handler.call(queries)
+            else
+              backtrace_filter = opts[:backtrace_filter]
+              backtrace_filter_note = backtrace_filter ? " (filtered)" : ""
+              query_info = queries.map do |k,v|
+                backtrace = k[1]
+                backtrace = backtrace.grep(backtrace_filter) if backtrace_filter
+                "times:#{v}\nsql:#{k[0]}\nbacktrace#{backtrace_filter_note}:\n#{backtrace.join("\n")}\n"
+              end
+              message = "duplicate queries detected:\n\n#{query_info.join("\n")}"
+
+              if e || opts[:warn]
+                warn(message)
+              else
+                raise DuplicateQueries.new(message, queries)
+              end
+            end
+          end
+        end
+      end
+
+      private
+
+      # Get the query record state for the given context.
+      def duplicate_query_recorder_state(current=Sequel.current)
+        @duplicate_query_detection_mutex.synchronize{@duplicate_query_detection_contexts[current]}
+      end
+
+      # Temporarily change whether to record queries for the block, resetting the
+      # previous setting after the block exits.
+      def change_duplicate_query_recorder_state(state, setting)
+        prev = state[0]
+        state[0] = setting
+        
+        begin
+          yield
+        ensure
+          state[0] = prev
+        end
+      end
+    end
+  end
+
+  Database.register_extension(:pg_auto_parameterize_duplicate_query_detection) do |db|
+    db.extension(:pg_auto_parameterize)
+    db.extend(Postgres::AutoParameterizeDuplicateQueryDetection)
+  end
+end

data/lib/sequel/extensions/pg_auto_parameterize_in_array.rb

@@ -21,15 +21,26 @@
 # DateTime :: timestamp (or timestamptz if pg_timestamptz extension is used)
 # Sequel::SQLTime :: time
 # Sequel::SQL::Blob :: bytea
+#
+# Arrays of string values are not automatically converted by default, because the Ruby
+# String class can represent a number of different database types.  To convert
+# arrays of Ruby strings to an untyped array (a query parameter with no explicit
+# type cast), set the +:treat_string_list_as_untyped_array+ Database option
+# before loading the extension.
 # 
-# String values are also supported using the +text+ type, but only if the 
-# +:treat_string_list_as_text_array+ Database option is used. This is because
-# treating strings as text can break programs, since the type for
-# literal strings in PostgreSQL is +unknown+, not +text+.
+# If you will only be using arrays of Ruby strings that represent the +text+ type,
+# you can use the +:treat_string_list_as_text_array+ Database option is used. This
+# can break programs, since the type for literal strings in PostgreSQL is +unknown+,
+# not +text+.
 #
-# The conversion is only done for single dimensional arrays that have more
-# than two elements, where all elements are of the same class (other than
-# nil values).
+# The conversion is only done for single dimensional arrays that have two or
+# more elements, where all elements are of the same class (other than
+# +nil+ values).  You can also do the conversion for arrays of 1 element by setting
+# <tt>pg_auto_parameterize_min_array_size: 1</tt> Database option.  This makes
+# finding cases that need special handling easier, but it doesn't match
+# how PostgreSQL internally converts the expression (PostgreSQL converts
+# <tt>IN (single_value)</tt> to <tt>= single_value</tt>, not
+# <tt>= ANY(ARRAY[single_value])</tt>).
 #
 # Related module: Sequel::Postgres::AutoParameterizeInArray
 
@@ -37,6 +48,47 @@ module Sequel
   module Postgres
     # Enable automatically parameterizing queries.
     module AutoParameterizeInArray
+      module TreatStringListAsUntypedArray
+        # Sentinal value to use as an auto param type to use auto parameterization
+        # of a string array without an explicit type cast.
+        NO_EXPLICIT_CAST = Object.new.freeze
+
+        # Wrapper for untyped PGArray values that will be parameterized directly
+        # into the query.  This should only be used in cases where you know the
+        # value should be added as a query parameter.
+        class ParameterizedUntypedPGArray < SQL::Wrapper
+          def to_s_append(ds, sql)
+            sql.add_arg(@value)
+          end
+        end
+
+        private
+
+        # Recognize NO_EXPLICIT_CAST sentinal value and use wrapped
+        # PGArray that will be parameterized into the query.
+        def _convert_array_to_pg_array_with_type(r, type)
+          if NO_EXPLICIT_CAST.equal?(type)
+            ParameterizedUntypedPGArray.new(Sequel.pg_array(r))
+          else
+            super
+          end
+        end
+
+        # Use a query parameter with no type cast for string arrays.
+        def _bound_variable_type_for_string_array(r)
+          NO_EXPLICIT_CAST
+        end
+      end
+
+      module TreatStringListAsTextArray
+        private
+
+        # Assume all string arrays used on RHS of IN/NOT IN are for type text[]
+        def _bound_variable_type_for_string_array(r)
+          "text"
+        end
+      end
+
       # Transform column IN (...) expressions into column = ANY($)
       # and column NOT IN (...) expressions into column != ALL($)
       # using an array bound variable for the ANY/ALL argument,
@@ -56,7 +108,7 @@ module Sequel
               op = :!=
               func = :ALL
             end
-            args = [l, Sequel.function(func, Sequel.pg_array(r, type))]
+            args = [l, Sequel.function(func, _convert_array_to_pg_array_with_type(r, type))]
           end
         end
 
@@ -68,7 +120,7 @@ module Sequel
       # The bound variable type string to use for the bound variable array.
       # Returns nil if a bound variable should not be used for the array.
       def _bound_variable_type_for_array(r)
-        return unless Array === r && r.size > 1
+        return unless (Array === r || Set === r) && r.size >= pg_auto_parameterize_min_array_size
         classes = r.map(&:class)
         classes.uniq!
         classes.delete(NilClass)
@@ -81,7 +133,7 @@ module Sequel
           # arrays natively (though the SQL used is different)
           "int8"
         elsif klass == String
-          "text" if db.typecast_value(:boolean, db.opts[:treat_string_list_as_text_array])
+          _bound_variable_type_for_string_array(r)
         elsif klass == BigDecimal
           "numeric"
         elsif klass == Date
@@ -100,11 +152,43 @@ module Sequel
           "bytea"
         end
       end
+
+      # Do not auto parameterize string arrays by default.
+      def _bound_variable_type_for_string_array(r)
+        nil
+      end
+
+      # The minimium size of array to auto parameterize.
+      def pg_auto_parameterize_min_array_size
+        2
+      end
+
+      # Convert RHS of IN/NOT IN operator to PGArray with given type.
+      def _convert_array_to_pg_array_with_type(r, type)
+        r = r.to_a if Set === r
+        Sequel.pg_array(r, type)
+      end
     end
   end
 
   Database.register_extension(:pg_auto_parameterize_in_array) do |db|
     db.extension(:pg_array, :pg_auto_parameterize)
     db.extend_datasets(Postgres::AutoParameterizeInArray)
+
+    if db.typecast_value(:boolean, db.opts[:treat_string_list_as_text_array])
+      db.extend_datasets(Postgres::AutoParameterizeInArray::TreatStringListAsTextArray)
+    elsif db.typecast_value(:boolean, db.opts[:treat_string_list_as_untyped_array])
+      db.extend_datasets(Postgres::AutoParameterizeInArray::TreatStringListAsUntypedArray)
+    end
+
+    if min_array_size = db.opts[:pg_auto_parameterize_min_array_size]
+      min_array_size = db.typecast_value(:integer, min_array_size)
+      mod = Module.new do
+        define_method(:pg_auto_parameterize_min_array_size){min_array_size}
+        private :pg_auto_parameterize_min_array_size
+      end
+      Sequel.set_temp_name(mod){"Sequel::Postgres::AutoParameterizeInArray::_MinArraySize#{min_array_size}"}
+      db.extend_datasets(mod)
+    end
   end
 end

data/lib/sequel/extensions/pg_enum.rb

@@ -127,7 +127,7 @@ module Sequel
 
       # Run the SQL on the database, reparsing the enum labels after it is run.
       def _process_enum_change_sql(sql)
-        run(sql)
+        run(sql.freeze)
         parse_enum_labels
         nil
       end
@@ -149,12 +149,12 @@ module Sequel
             from(:pg_type).
             where(:oid=>enum_labels.keys).
             exclude(:typarray=>0).
-            select_map([:typname, Sequel.cast(:typarray, Integer).as(:v)])
+            select_map([:typname, Sequel.cast(:typarray, Integer).as(:v), Sequel.cast(:oid, Integer).as(:sv)])
 
           existing_oids = conversion_procs.keys
-          array_types.each do |name, oid|
+          array_types.each do |name, oid, scalar_oid|
             next if existing_oids.include?(oid)
-            register_array_type(name, :oid=>oid)
+            register_array_type(name, :oid=>oid, :scalar_oid=>scalar_oid)
           end
         end
 

data/lib/sequel/extensions/pg_hstore.rb

@@ -136,7 +136,7 @@ module Sequel
       module DatabaseMethods
         def self.extended(db)
           db.instance_exec do
-            add_named_conversion_proc(:hstore, &HStore.method(:parse))
+            add_named_conversion_proc(:hstore){|v| HStore.parse(v)}
             @schema_type_classes[:hstore] = HStore
           end
         end

data/lib/sequel/extensions/pg_hstore_ops.rb

@@ -294,13 +294,18 @@ module Sequel
         SQL::Function.new(name, self, *args)
       end
 
-      # Wrap argument in a PGArray if it is an array
+      # Wrap argument in a PGArray if it is an array or a set
       def wrap_input_array(obj)
-        if obj.is_a?(Array) && Sequel.respond_to?(:pg_array) 
-          Sequel.pg_array(obj)
-        else
-          obj
+        if Sequel.respond_to?(:pg_array)
+          case obj
+          when Array
+            return Sequel.pg_array(obj)
+          when Set
+            return Sequel.pg_array(obj.to_a)
+          end
         end
+
+        obj
       end
 
       # Wrap argument in an Hstore if it is a hash