sequel 5.85.0 → 5.93.0

data/lib/sequel/dataset/sql.rb

@@ -685,10 +685,10 @@ module Sequel
     # being quoted, returns name as a string.  If identifiers are being quoted
     # quote the name with quoted_identifier.
     def quote_identifier_append(sql, name)
+      name = name.value if name.is_a?(SQL::Identifier)
       if name.is_a?(LiteralString)
         sql << name
       else
-        name = name.value if name.is_a?(SQL::Identifier)
         name = input_identifier(name)
         if quote_identifiers?
           quoted_identifier_append(sql, name)
@@ -700,11 +700,14 @@ module Sequel
 
     # Append literalization of identifier or unqualified identifier to SQL string.
     def quote_schema_table_append(sql, table)
-      schema, table = schema_and_table(table)
-      if schema
-        quote_identifier_append(sql, schema)
+      qualifiers = split_qualifiers(table)
+      table = qualifiers.pop
+
+      qualifiers.each do |q|
+        quote_identifier_append(sql, q)
         sql << '.'
       end
+
       quote_identifier_append(sql, table)
     end
 
@@ -1032,7 +1035,7 @@ module Sequel
       if column_aliases
         raise Error, "#{db.database_type} does not support derived column lists" unless supports_derived_column_lists?
         sql << '('
-        identifier_list_append(sql, column_aliases)
+        derived_column_list_sql_append(sql, column_aliases)
         sql << ')'
       end
     end
@@ -1165,6 +1168,11 @@ module Sequel
       end
     end
 
+    # Append the column aliases to the SQL.
+    def derived_column_list_sql_append(sql, column_aliases)
+      identifier_list_append(sql, column_aliases)
+    end
+    
     # Disable caching of SQL for the current dataset
     def disable_sql_caching!
       cache_set(:_no_cache_sql, true)
@@ -1425,10 +1433,6 @@ module Sequel
     # calls +sql_literal+ if object responds to it, otherwise raises an error.
     # If a database specific type is allowed, this should be overriden in a subclass.
     def literal_other_append(sql, v)
-      # We can't be sure if v will always literalize to the same SQL, so
-      # don't cache SQL for a dataset that uses this.
-      disable_sql_caching!
-
       if v.respond_to?(:sql_literal_append)
         v.sql_literal_append(self, sql)
       elsif v.respond_to?(:sql_literal)
@@ -1436,6 +1440,12 @@ module Sequel
       else
         raise Error, "can't express #{v.inspect} as a SQL literal"
       end
+
+      if !v.respond_to?(:sql_literal_allow_caching?) || !v.sql_literal_allow_caching?(self)
+        # We can't be sure if v will always literalize to the same SQL, so
+        # don't cache SQL for a dataset that uses this.
+        disable_sql_caching!
+      end
     end
 
     # SQL fragment for Sequel::SQLTime, containing just the time part

data/lib/sequel/extensions/connection_validator.rb

@@ -105,18 +105,23 @@ module Sequel
       1.times do
         if (conn = super) &&
            (timer = sync{@connection_timestamps.delete(conn)}) &&
-           Sequel.elapsed_seconds_since(timer) > @connection_validation_timeout &&
-           !db.valid_connection?(conn)
+           Sequel.elapsed_seconds_since(timer) > @connection_validation_timeout
 
-          case pool_type
-          when :sharded_threaded, :sharded_timed_queue
-            sync{@allocated[a.last].delete(Sequel.current)}
-          else
-            sync{@allocated.delete(Sequel.current)}
-          end
+          begin
+            valid = db.valid_connection?(conn)
+          ensure
+            unless valid
+              case pool_type
+              when :sharded_threaded, :sharded_timed_queue
+                sync{@allocated[a.last].delete(Sequel.current)}
+              else
+                sync{@allocated.delete(Sequel.current)}
+              end
 
-          disconnect_connection(conn)
-          redo
+              disconnect_connection(conn)
+              redo if valid == false
+            end
+          end
         end
       end
 

data/lib/sequel/extensions/migration.rb

@@ -223,7 +223,7 @@ module Sequel
       @actions << [:drop_join_table, *args]
     end
 
-    def create_table(name, opts=OPTS)
+    def create_table(name, opts=OPTS, &_)
       @actions << [:drop_table, name, opts]
     end
 
@@ -287,6 +287,10 @@ module Sequel
     def set_column_allow_null(name, allow_null=true)
       @actions << [:set_column_allow_null, name, !allow_null]
     end
+
+    def set_column_not_null(name)
+      @actions << [:set_column_allow_null, name]
+    end
   end
 
   # The preferred method for writing Sequel migrations, using a DSL:
@@ -371,7 +375,7 @@ module Sequel
   #
   # Part of the +migration+ extension.
   class Migrator
-    MIGRATION_FILE_PATTERN = /\A(\d+)_.+\.rb\z/i.freeze
+    MIGRATION_FILE_PATTERN = /\A(\d+)_(.+)\.rb\z/i.freeze
 
     # Mutex used around migration file loading
     MUTEX = Mutex.new
@@ -791,7 +795,23 @@ module Sequel
         next unless MIGRATION_FILE_PATTERN.match(file)
         files << File.join(directory, file)
       end
-      files.sort_by{|f| MIGRATION_FILE_PATTERN.match(File.basename(f))[1].to_i}
+      files.sort! do |a, b|
+        a_ver, a_name = split_migration_filename(a)
+        b_ver, b_name = split_migration_filename(b)
+        x = a_ver <=> b_ver
+        if x.zero?
+          x = a_name <=> b_name
+        end
+        x
+      end
+      files
+    end
+
+    # Return an integer and name (without extension) for the given path.
+    def split_migration_filename(path)
+      version, name = MIGRATION_FILE_PATTERN.match(File.basename(path)).captures
+      version = version.to_i
+      [version, name]
     end
     
     # Returns tuples of migration, filename, and direction

data/lib/sequel/extensions/null_dataset.rb

@@ -63,12 +63,12 @@ module Sequel
       end
 
       # Return self without sending a database query, never yielding.
-      def each
+      def each(&_)
         self
       end
 
       # Return nil without sending a database query, never yielding.
-      def fetch_rows(sql)
+      def fetch_rows(sql, &_)
         nil
       end
 

data/lib/sequel/extensions/pg_auto_parameterize.rb

@@ -394,7 +394,7 @@ module Sequel
         # there can be more than one parameter per column, so this doesn't prevent going
         # over the limit, though it does make it less likely.
         def default_import_slice
-          40
+          @opts[:no_auto_parameterize] ? super : 40
         end
 
         # Handle parameterization of multi_insert_sql
@@ -463,6 +463,11 @@ module Sequel
           @opts[:no_auto_parameterize] ? super : QueryString.new
         end
 
+        # A mutable string used as the prefix when explaining a query.
+        def explain_sql_string_origin(opts)
+          @opts[:no_auto_parameterize] ? super : (QueryString.new << super)
+        end
+
         # If subquery uses with_sql with a method name symbol, get the dataset
         # with_sql was called on, and use that as the subquery, recording the
         # arguments to with_sql that will be used to calculate the sql.

data/lib/sequel/extensions/pg_auto_parameterize_in_array.rb

@@ -21,15 +21,26 @@
 # DateTime :: timestamp (or timestamptz if pg_timestamptz extension is used)
 # Sequel::SQLTime :: time
 # Sequel::SQL::Blob :: bytea
+#
+# Arrays of string values are not automatically converted by default, because the Ruby
+# String class can represent a number of different database types.  To convert
+# arrays of Ruby strings to an untyped array (a query parameter with no explicit
+# type cast), set the +:treat_string_list_as_untyped_array+ Database option
+# before loading the extension.
 # 
-# String values are also supported using the +text+ type, but only if the 
-# +:treat_string_list_as_text_array+ Database option is used. This is because
-# treating strings as text can break programs, since the type for
-# literal strings in PostgreSQL is +unknown+, not +text+.
+# If you will only be using arrays of Ruby strings that represent the +text+ type,
+# you can use the +:treat_string_list_as_text_array+ Database option is used. This
+# can break programs, since the type for literal strings in PostgreSQL is +unknown+,
+# not +text+.
 #
-# The conversion is only done for single dimensional arrays that have more
-# than two elements, where all elements are of the same class (other than
-# nil values).
+# The conversion is only done for single dimensional arrays that have two or
+# more elements, where all elements are of the same class (other than
+# +nil+ values).  You can also do the conversion for arrays of 1 element by setting
+# <tt>pg_auto_parameterize_min_array_size: 1</tt> Database option.  This makes
+# finding cases that need special handling easier, but it doesn't match
+# how PostgreSQL internally converts the expression (PostgreSQL converts
+# <tt>IN (single_value)</tt> to <tt>= single_value</tt>, not
+# <tt>= ANY(ARRAY[single_value])</tt>).
 #
 # Related module: Sequel::Postgres::AutoParameterizeInArray
 
@@ -37,6 +48,47 @@ module Sequel
   module Postgres
     # Enable automatically parameterizing queries.
     module AutoParameterizeInArray
+      module TreatStringListAsUntypedArray
+        # Sentinal value to use as an auto param type to use auto parameterization
+        # of a string array without an explicit type cast.
+        NO_EXPLICIT_CAST = Object.new.freeze
+
+        # Wrapper for untyped PGArray values that will be parameterized directly
+        # into the query.  This should only be used in cases where you know the
+        # value should be added as a query parameter.
+        class ParameterizedUntypedPGArray < SQL::Wrapper
+          def to_s_append(ds, sql)
+            sql.add_arg(@value)
+          end
+        end
+
+        private
+
+        # Recognize NO_EXPLICIT_CAST sentinal value and use wrapped
+        # PGArray that will be parameterized into the query.
+        def _convert_array_to_pg_array_with_type(r, type)
+          if NO_EXPLICIT_CAST.equal?(type)
+            ParameterizedUntypedPGArray.new(Sequel.pg_array(r))
+          else
+            super
+          end
+        end
+
+        # Use a query parameter with no type cast for string arrays.
+        def _bound_variable_type_for_string_array(r)
+          NO_EXPLICIT_CAST
+        end
+      end
+
+      module TreatStringListAsTextArray
+        private
+
+        # Assume all string arrays used on RHS of IN/NOT IN are for type text[]
+        def _bound_variable_type_for_string_array(r)
+          "text"
+        end
+      end
+
       # Transform column IN (...) expressions into column = ANY($)
       # and column NOT IN (...) expressions into column != ALL($)
       # using an array bound variable for the ANY/ALL argument,
@@ -56,7 +108,7 @@ module Sequel
               op = :!=
               func = :ALL
             end
-            args = [l, Sequel.function(func, Sequel.pg_array(r, type))]
+            args = [l, Sequel.function(func, _convert_array_to_pg_array_with_type(r, type))]
           end
         end
 
@@ -68,7 +120,7 @@ module Sequel
       # The bound variable type string to use for the bound variable array.
       # Returns nil if a bound variable should not be used for the array.
       def _bound_variable_type_for_array(r)
-        return unless Array === r && r.size > 1
+        return unless Array === r && r.size >= pg_auto_parameterize_min_array_size
         classes = r.map(&:class)
         classes.uniq!
         classes.delete(NilClass)
@@ -81,7 +133,7 @@ module Sequel
           # arrays natively (though the SQL used is different)
           "int8"
         elsif klass == String
-          "text" if db.typecast_value(:boolean, db.opts[:treat_string_list_as_text_array])
+          _bound_variable_type_for_string_array(r)
         elsif klass == BigDecimal
           "numeric"
         elsif klass == Date
@@ -100,11 +152,42 @@ module Sequel
           "bytea"
         end
       end
+
+      # Do not auto parameterize string arrays by default.
+      def _bound_variable_type_for_string_array(r)
+        nil
+      end
+
+      # The minimium size of array to auto parameterize.
+      def pg_auto_parameterize_min_array_size
+        2
+      end
+
+      # Convert RHS of IN/NOT IN operator to PGArray with given type.
+      def _convert_array_to_pg_array_with_type(r, type)
+        Sequel.pg_array(r, type)
+      end
     end
   end
 
   Database.register_extension(:pg_auto_parameterize_in_array) do |db|
     db.extension(:pg_array, :pg_auto_parameterize)
     db.extend_datasets(Postgres::AutoParameterizeInArray)
+
+    if db.typecast_value(:boolean, db.opts[:treat_string_list_as_text_array])
+      db.extend_datasets(Postgres::AutoParameterizeInArray::TreatStringListAsTextArray)
+    elsif db.typecast_value(:boolean, db.opts[:treat_string_list_as_untyped_array])
+      db.extend_datasets(Postgres::AutoParameterizeInArray::TreatStringListAsUntypedArray)
+    end
+
+    if min_array_size = db.opts[:pg_auto_parameterize_min_array_size]
+      min_array_size = db.typecast_value(:integer, min_array_size)
+      mod = Module.new do
+        define_method(:pg_auto_parameterize_min_array_size){min_array_size}
+        private :pg_auto_parameterize_min_array_size
+      end
+      Sequel.set_temp_name(mod){"Sequel::Postgres::AutoParameterizeInArray::_MinArraySize#{min_array_size}"}
+      db.extend_datasets(mod)
+    end
   end
 end

data/lib/sequel/extensions/pg_enum.rb

@@ -149,12 +149,12 @@ module Sequel
             from(:pg_type).
             where(:oid=>enum_labels.keys).
             exclude(:typarray=>0).
-            select_map([:typname, Sequel.cast(:typarray, Integer).as(:v)])
+            select_map([:typname, Sequel.cast(:typarray, Integer).as(:v), Sequel.cast(:oid, Integer).as(:sv)])
 
           existing_oids = conversion_procs.keys
-          array_types.each do |name, oid|
+          array_types.each do |name, oid, scalar_oid|
             next if existing_oids.include?(oid)
-            register_array_type(name, :oid=>oid)
+            register_array_type(name, :oid=>oid, :scalar_oid=>scalar_oid)
           end
         end
 

data/lib/sequel/extensions/pg_row.rb

@@ -113,6 +113,7 @@ module Sequel
         # automatically casted to the database type when literalizing.
         def self.subclass(db_type)
           Class.new(self) do
+            Sequel.set_temp_name(self){"Sequel::Postgres::PGRow::ArrayRow::_Subclass(#{db_type})"}
             @db_type = db_type
           end
         end
@@ -170,6 +171,7 @@ module Sequel
         # type and columns.
         def self.subclass(db_type, columns)
           Class.new(self) do
+            Sequel.set_temp_name(self){"Sequel::Postgres::PGRow::HashRow::_Subclass(#{db_type})"}
             @db_type = db_type
             @columns = columns
           end
@@ -391,7 +393,7 @@ module Sequel
           db.instance_exec do
             @row_types = {}
             @row_schema_types = {}
-            extend(@row_type_method_module = Module.new)
+            extend(@row_type_method_module = Sequel.set_temp_name(Module.new){"Sequel::Postgres::PGRow::DatabaseMethods::_RowTypeMethodModule"})
             add_conversion_proc(2249, PGRow::Parser.new(:converter=>PGRow::ArrayRow))
             if respond_to?(:register_array_type)
               register_array_type('record', :oid=>2287, :scalar_oid=>2249)

data/lib/sequel/extensions/pg_schema_caching.rb

@@ -0,0 +1,90 @@
+# frozen-string-literal: true
+#
+# The pg_schema_caching extension builds on top of the schema_caching
+# extension, and allows it to handle custom PostgreSQL types. On
+# PostgreSQL, column schema hashes include an :oid entry for the OID
+# for the column's type.  For custom types, this OID is dependent on
+# the PostgreSQL database, so in most cases, test and development
+# versions of the same database, created with the same migrations,
+# will have different OIDs.
+#
+# To fix this case, the pg_schema_caching extension removes custom
+# OIDs from the schema cache when dumping the schema, replacing them
+# with a placeholder. When loading the cached schema, the Database
+# object makes a single query to get the OIDs for all custom types
+# used by the cached schema, and it updates all related column
+# schema hashes to set the correct :oid entry for the current
+# database.
+#
+# Related module: Sequel::Postgres::SchemaCaching
+
+require_relative "schema_caching"
+
+module Sequel
+  module Postgres
+    module SchemaCaching
+      include Sequel::SchemaCaching
+
+      private
+
+      # Load custom oids from database when loading schema cache file.
+      def load_schema_cache_file(file)
+        set_custom_oids_for_cached_schema(super)
+      end
+
+      # Find all column schema hashes that use custom types.
+      # Load the oids for custom types in a single query, and update
+      # each related column schema hash with the correct oid.
+      def set_custom_oids_for_cached_schema(schemas)
+        custom_oid_rows = {}
+
+        schemas.each_value do |cols|
+          cols.each do |_, h|
+            if h[:oid] == :custom
+              (custom_oid_rows[h[:db_type]] ||= []) << h
+            end
+          end
+        end
+
+        unless custom_oid_rows.empty?
+          from(:pg_type).where(:typname=>custom_oid_rows.keys).select_hash(:typname, :oid).each do |name, oid|
+            custom_oid_rows.delete(name).each do |row|
+              row[:oid] = oid
+            end
+          end
+        end
+
+        unless custom_oid_rows.empty?
+          warn "Could not load OIDs for the following custom types: #{custom_oid_rows.keys.sort.join(", ")}", uplevel: 3
+
+          schemas.keys.each do |k|
+            if schemas[k].any?{|_,h| h[:oid] == :custom}
+              # Remove schema entry for table, so it will be queried at runtime to get the correct oids
+              schemas.delete(k)
+            end
+          end
+        end
+
+        schemas
+      end
+
+      # Replace :oid entries for custom types with :custom.
+      def dumpable_schema_cache
+        sch = super
+
+        sch.each_value do |cols|
+          cols.each do |_, h|
+            if (oid = h[:oid]) && oid >= 10000
+              h[:oid] = :custom
+            end
+          end
+        end
+
+        sch
+      end
+    end
+  end
+
+  Database.register_extension(:pg_schema_caching, Postgres::SchemaCaching)
+end
+

data/lib/sequel/extensions/query_blocker.rb

@@ -0,0 +1,172 @@
+# frozen-string-literal: true
+#
+# The query_blocker extension adds Database#block_queries.
+# Inside the block passed to #block_queries, any attempts to
+# execute a query/statement on the database will raise a
+# Sequel::QueryBlocker::BlockedQuery exception.
+#
+#   DB.extension :query_blocker
+#   DB.block_queries do
+#     ds = DB[:table]          # No exception
+#     ds = ds.where(column: 1) # No exception
+#     ds.all                   # Exception raised
+#   end
+#
+# To handle concurrency, you can pass a :scope option:
+#
+#   # Current Thread
+#   DB.block_queries(scope: :thread){}
+#
+#   # Current Fiber
+#   DB.block_queries(scope: :fiber){}
+#
+#   # Specific Thread
+#   DB.block_queries(scope: Thread.current){}
+#
+#   # Specific Fiber
+#   DB.block_queries(scope: Fiber.current){}
+#
+# Database#block_queries is useful for blocking queries inside
+# the block.  However, there may be cases where you want to
+# allow queries in specific places inside a block_queries block.
+# You can use Database#allow_queries for that:
+#
+#   DB.block_queries do
+#     DB.allow_queries do
+#       DB[:table].all           # Query allowed
+#     end
+#
+#     DB[:table].all           # Exception raised
+#   end
+#
+# When mixing block_queries and allow_queries with scopes, the
+# narrowest scope has priority.  So if you are blocking with
+# :thread scope, and allowing with :fiber scope, queries in the
+# current fiber will be allowed, but queries in different fibers of
+# the current thread will be blocked.
+#
+# Note that this should catch all queries executed through the
+# Database instance.  Whether it catches queries executed directly
+# on a connection object depends on the adapter in use.
+#
+# Related module: Sequel::QueryBlocker
+
+require "fiber"
+
+#
+module Sequel
+  module QueryBlocker
+    # Exception class raised if there is an attempt to execute a 
+    # query/statement on the database inside a block passed to
+    # block_queries.
+    class BlockedQuery < Sequel::Error
+    end
+
+    def self.extended(db)
+      db.instance_exec do
+        @blocked_query_scopes ||= {}
+      end
+    end
+
+    # If checking a connection for validity, and a BlockedQuery exception is
+    # raised, treat it as a valid connection.  You cannot check whether the
+    # connection is valid without issuing a query, and if queries are blocked,
+    # you need to assume it is valid or assume it is not.  Since it most cases
+    # it will be valid, this assumes validity.
+    def valid_connection?(conn)
+      super
+    rescue BlockedQuery
+      true
+    end
+
+    # Check whether queries are blocked before executing them.
+    def log_connection_yield(sql, conn, args=nil)
+      # All database adapters should be calling this method around
+      # query execution (otherwise the queries would not get logged),
+      # ensuring the blocking is checked.  Any database adapter issuing
+      # a query without calling this method is considered buggy.
+      check_blocked_queries!
+      super
+    end
+
+    # Whether queries are currently blocked.
+    def block_queries?
+      b = @blocked_query_scopes
+      b.fetch(Fiber.current) do
+        b.fetch(Thread.current) do
+          b.fetch(:global, false)
+        end
+      end
+    end
+
+    # Allow queries inside the block.  Only useful if they are already blocked
+    # for the same scope. Useful for blocking queries generally, and only allowing
+    # them in specific places.  Takes the same :scope option as #block_queries.
+    def allow_queries(opts=OPTS, &block)
+      _allow_or_block_queries(false, opts, &block)
+    end
+
+    # Reject (raise an BlockedQuery exception) if there is an attempt to execute
+    # a query/statement inside the block.
+    #
+    # The :scope option indicates which queries are rejected inside the block:
+    #
+    # :global :: This is the default, and rejects all queries.
+    # :thread :: Reject all queries in the current thread.
+    # :fiber :: Reject all queries in the current fiber.
+    # Thread :: Reject all queries in the given thread.
+    # Fiber :: Reject all queries in the given fiber.
+    def block_queries(opts=OPTS, &block)
+      _allow_or_block_queries(true, opts, &block)
+    end
+    
+    private
+
+    # Internals of block_queries and allow_queries.
+    def _allow_or_block_queries(value, opts)
+      scope = query_blocker_scope(opts)
+      prev_value = nil
+      scopes = @blocked_query_scopes
+
+      begin
+        Sequel.synchronize do
+          prev_value = scopes[scope]
+          scopes[scope] = value
+        end
+
+        yield
+      ensure
+        Sequel.synchronize do
+          if prev_value.nil?
+            scopes.delete(scope)
+          else
+            scopes[scope] = prev_value
+          end
+        end
+      end
+    end
+
+    # The scope for the query block, either :global, or a Thread or Fiber instance.
+    def query_blocker_scope(opts)
+      case scope = opts[:scope]
+      when nil
+        :global
+      when :global, Thread, Fiber
+        scope
+      when :thread
+        Thread.current
+      when :fiber
+        Fiber.current
+      else
+        raise Sequel::Error, "invalid scope given to block_queries: #{scope.inspect}"
+      end
+    end
+
+    # Raise a BlockQuery exception if queries are currently blocked.
+    def check_blocked_queries!
+      raise BlockedQuery, "cannot execute query inside a block_queries block" if block_queries?
+    end
+  end
+
+  Database.register_extension(:query_blocker, QueryBlocker)
+end