sequel 4.26.0 → 5.37.0

data/lib/sequel/extensions/blank.rb

@@ -1,3 +1,5 @@
+# frozen-string-literal: true
+#
 # The blank extension adds the blank? method to all objects (e.g. Object#blank?).
 #
 # To load the extension:

data/lib/sequel/extensions/caller_logging.rb

@@ -0,0 +1,79 @@
+# frozen-string-literal: true
+#
+# The caller_logging extension includes caller information before
+# query logging, showing which code caused the query.  It skips
+# internal Sequel code, showing the first non-Sequel caller line.
+#
+#   DB.extension :caller_logging
+#   DB[:table].first
+#   # Logger:
+#   # (0.000041s) (source: /path/to/app/foo/t.rb:12 in `get_first`) SELECT * FROM table LIMIT 1
+#
+# You can further filter the caller lines by setting
+# <tt>Database#caller_logging_ignore</tt> to a regexp of additional
+# caller lines to ignore.  This is useful if you have specific
+# methods or internal extensions/plugins that you would also
+# like to ignore as they obscure the code actually making the
+# request.
+#
+#   DB.caller_logging_ignore = %r{/path/to/app/lib/plugins}
+#
+# You can also format the caller before it is placed in the logger,
+# using +caller_logging_formatter+:
+#
+#   DB.caller_logging_formatter = lambda do |caller|
+#     "(#{caller.sub(/\A\/path\/to\/app\//, '')})"
+#   end
+#   DB[:table].first
+#   # Logger:
+#   # (0.000041s) (foo/t.rb:12 in `get_first`) SELECT * FROM table LIMIT 1
+#
+# Related module: Sequel::CallerLogging
+
+require 'rbconfig'
+
+#
+module Sequel
+  module CallerLogging
+    SEQUEL_LIB_PATH = (File.expand_path('../../..', __FILE__) + '/').freeze
+
+    # A regexp of caller lines to ignore, in addition to internal Sequel and Ruby code.
+    attr_accessor :caller_logging_ignore
+
+    # A callable to format the external caller
+    attr_accessor :caller_logging_formatter
+
+    # Include caller information when logging query.
+    def log_connection_yield(sql, conn, args=nil)
+      if !@loggers.empty? && (external_caller = external_caller_for_log)
+        sql = "#{external_caller} #{sql}"
+      end
+      super
+    end
+
+    private
+
+    # The caller to log, ignoring internal Sequel and Ruby code, and user specified
+    # lines to ignore.
+    def external_caller_for_log
+      ignore = caller_logging_ignore
+      c = caller.find do |line|
+        !(line.start_with?(SEQUEL_LIB_PATH) ||
+          line.start_with?(RbConfig::CONFIG["rubylibdir"]) ||
+          (ignore && line =~ ignore))
+      end
+
+      if c
+        c = if formatter = caller_logging_formatter
+          formatter.call(c)
+        else
+          "(source: #{c})"
+        end
+      end
+
+      c
+    end
+  end
+
+  Database.register_extension(:caller_logging, CallerLogging)
+end

data/lib/sequel/extensions/columns_introspection.rb

@@ -1,3 +1,5 @@
+# frozen-string-literal: true
+#
 # The columns_introspection extension attempts to introspect the
 # selected columns for a dataset before issuing a query.  If it
 # thinks it can guess correctly at the columns the query will use,
@@ -14,6 +16,8 @@
 # To attempt to introspect columns for all datasets on a single database:
 #
 #   DB.extension(:columns_introspection)
+#
+# Related module: Sequel::ColumnsIntrospection
 
 #
 module Sequel
@@ -24,9 +28,11 @@ module Sequel
     # Symbols, SQL::Identifiers, SQL::QualifiedIdentifiers, and
     # SQL::AliasedExpressions.
     def columns
-      return @columns if @columns
+      if cols = _columns
+        return cols
+      end
       if (pcs = probable_columns) && pcs.all?
-        @columns = pcs
+        self.columns = pcs
       else
         super
       end
@@ -69,8 +75,7 @@ module Sequel
       when SQL::Identifier
         c.value.to_sym
       when SQL::QualifiedIdentifier
-        col = c.column
-        col.is_a?(SQL::Identifier) ? col.value.to_sym : col.to_sym
+        c.column.to_sym
       when SQL::AliasedExpression
         a = c.alias
         a.is_a?(SQL::Identifier) ? a.value.to_sym : a.to_sym

data/lib/sequel/extensions/connection_expiration.rb

@@ -0,0 +1,99 @@
+# frozen-string-literal: true
+#
+# The connection_expiration extension modifies a database's
+# connection pool to validate that connections checked out
+# from the pool are not expired, before yielding them for
+# use.  If it detects an expired connection, it removes it
+# from the pool and tries the next available connection,
+# creating a new connection if no available connection is
+# unexpired.  Example of use:
+#
+#   DB.extension(:connection_expiration)
+#
+# The default connection timeout is 14400 seconds (4 hours).
+# To override it:
+#
+#   DB.pool.connection_expiration_timeout = 3600 # 1 hour
+#
+# Note that this extension only affects the default threaded
+# and the sharded threaded connection pool.  The single
+# threaded and sharded single threaded connection pools are
+# not affected.  As the only reason to use the single threaded
+# pools is for speed, and this extension makes the connection
+# pool slower, there's not much point in modifying this
+# extension to work with the single threaded pools.  The
+# threaded pools work fine even in single threaded code, so if
+# you are currently using a single threaded pool and want to
+# use this extension, switch to using a threaded pool.
+#
+# Related module: Sequel::ConnectionExpiration
+
+#
+module Sequel
+  module ConnectionExpiration
+    class Retry < Error; end
+    Sequel::Deprecation.deprecate_constant(self, :Retry)
+
+    # The number of seconds that need to pass since
+    # connection creation before expiring a connection.
+    # Defaults to 14400 seconds (4 hours).
+    attr_accessor :connection_expiration_timeout
+
+    # The maximum number of seconds that will be added as a random delay to the expiration timeout
+    # Defaults to 0 seconds (no random delay).
+    attr_accessor :connection_expiration_random_delay
+
+    # Initialize the data structures used by this extension.
+    def self.extended(pool)
+      pool.instance_exec do
+        sync do
+          @connection_expiration_timestamps ||= {}
+          @connection_expiration_timeout ||= 14400
+          @connection_expiration_random_delay ||= 0
+        end
+      end
+    end
+
+    private
+
+    # Clean up expiration timestamps during disconnect.
+    def disconnect_connection(conn)
+      sync{@connection_expiration_timestamps.delete(conn)}
+      super
+    end
+
+    # Record the time the connection was created.
+    def make_new(*)
+      conn = super
+      @connection_expiration_timestamps[conn] = [Sequel.start_timer, @connection_expiration_timeout + (rand * @connection_expiration_random_delay)].freeze
+      conn
+    end
+
+    # When acquiring a connection, check if the connection is expired.
+    # If it is expired, disconnect the connection, and retry with a new
+    # connection.
+    def acquire(*a)
+      conn = nil
+      1.times do
+        if (conn = super) &&
+           (cet = sync{@connection_expiration_timestamps[conn]}) &&
+           Sequel.elapsed_seconds_since(cet[0]) > cet[1]
+
+          if pool_type == :sharded_threaded
+            sync{allocated(a.last).delete(Sequel.current)}
+          else
+            sync{@allocated.delete(Sequel.current)}
+          end
+
+          disconnect_connection(conn)
+          redo
+        end
+      end
+
+      conn
+    end
+  end
+
+  Database.register_extension(:connection_expiration){|db| db.pool.extend(ConnectionExpiration)}
+end
+

data/lib/sequel/extensions/connection_validator.rb

@@ -1,3 +1,5 @@
+# frozen-string-literal: true
+#
 # The connection_validator extension modifies a database's
 # connection pool to validate that connections checked out
 # from the pool are still valid, before yielding them for
@@ -42,11 +44,14 @@
 # threaded pools work fine even in single threaded code, so if
 # you are currently using a single threaded pool and want to
 # use this extension, switch to using a threaded pool.
+#
+# Related module: Sequel::ConnectionValidator
 
 #
 module Sequel
   module ConnectionValidator
     class Retry < Error; end
+    Sequel::Deprecation.deprecate_constant(self, :Retry)
 
     # The number of seconds that need to pass since
     # connection checkin before attempting to validate
@@ -56,9 +61,11 @@ module Sequel
 
     # Initialize the data structures used by this extension.
     def self.extended(pool)
-      pool.instance_eval do
-        @connection_timestamps ||= {}
-        @connection_validation_timeout = 3600
+      pool.instance_exec do
+        sync do
+          @connection_timestamps ||= {}
+          @connection_validation_timeout ||= 3600
+        end
       end
 
       # Make sure the valid connection SQL query is precached,
@@ -73,32 +80,38 @@ module Sequel
     # Record the time the connection was checked back into the pool.
     def checkin_connection(*)
       conn = super
-      @connection_timestamps[conn] = Time.now
+      @connection_timestamps[conn] = Sequel.start_timer
       conn
     end
 
+    # Clean up timestamps during disconnect.
+    def disconnect_connection(conn)
+      sync{@connection_timestamps.delete(conn)}
+      super
+    end
+
     # When acquiring a connection, if it has been
     # idle for longer than the connection validation timeout,
     # test the connection for validity.  If it is not valid,
     # disconnect the connection, and retry with a new connection.
     def acquire(*a)
-      begin
+      conn = nil
+
+      1.times do
         if (conn = super) &&
-           (t = sync{@connection_timestamps.delete(conn)}) &&
-           Time.now - t > @connection_validation_timeout &&
+           (timer = sync{@connection_timestamps.delete(conn)}) &&
+           Sequel.elapsed_seconds_since(timer) > @connection_validation_timeout &&
            !db.valid_connection?(conn)
 
           if pool_type == :sharded_threaded
-            sync{allocated(a.last).delete(Thread.current)}
+            sync{allocated(a.last).delete(Sequel.current)}
           else
-            sync{@allocated.delete(Thread.current)}
+            sync{@allocated.delete(Sequel.current)}
           end
 
-          db.disconnect_connection(conn)
-          raise Retry
+          disconnect_connection(conn)
+          redo
         end
-      rescue Retry
-        retry
       end
 
       conn

data/lib/sequel/extensions/constant_sql_override.rb

@@ -0,0 +1,65 @@
+# frozen-string-literal: true
+#
+# The constant_sql_override extension allows you to change the SQL
+# generated for Sequel constants.
+#
+# One possible use-case for this is to have Sequel::CURRENT_TIMESTAMP use UTC time when
+# you have Sequel.database_timezone = :utc, but the database uses localtime when
+# generating CURRENT_TIMESTAMP.
+#
+# You can set SQL overrides with Database#set_constant_sql:
+#
+#   DB.set_constant_sql(Sequel::CURRENT_TIMESTAMP, "CURRENT_TIMESTAMP AT TIME ZONE 'UTC'")
+#
+# Now, using Sequel::CURRENT_TIMESTAMP will use your override instead:
+#
+#   Album.where(released_at: Sequel::CURRENT_TIMESTAMP).sql
+#   # => SELECT "albums.*" FROM "albums" WHERE ("released_at" = CURRENT_TIMESTAMP AT TIME ZONE 'UTC')
+#
+# To use this extension, first load it into your Sequel::Database instance:
+#
+#   DB.extension :constant_sql_override
+#
+# Related module: Sequel::ConstantSqlOverride
+
+#
+module Sequel
+  module ConstantSqlOverride
+    module DatabaseMethods
+      # Create the initial empty hash of constant sql overrides.
+      def self.extended(db)
+        db.instance_exec do
+          @constant_sqls ||= {}
+          extend_datasets(DatasetMethods)
+        end
+      end
+
+      # Hash mapping constant symbols to SQL.  For internal use only.
+      attr_reader :constant_sqls # :nodoc:
+
+      # Set the SQL to use for the given Sequel::SQL::Constant
+      def set_constant_sql(constant, override)
+        @constant_sqls[constant.constant] = override
+      end
+
+      # Freeze the constant_sqls hash to prevent adding new overrides.
+      def freeze
+        @constant_sqls.freeze
+        super
+      end
+    end
+
+    module DatasetMethods
+      # Use overridden constant SQL
+      def constant_sql_append(sql, constant)
+        if constant_sql = db.constant_sqls[constant]
+          sql << constant_sql
+        else
+          super
+        end
+      end
+    end
+  end
+
+  Database.register_extension(:constant_sql_override, ConstantSqlOverride::DatabaseMethods)
+end

data/lib/sequel/extensions/constraint_validations.rb

@@ -1,3 +1,5 @@
+# frozen-string-literal: true
+#
 # The constraint_validations extension is designed to easily create database
 # constraints inside create_table and alter_table blocks.  It also adds
 # relevant metadata about the constraints to a separate table, which the
@@ -20,7 +22,7 @@
 # generally want to create it first, before creating any other application
 # tables.
 #
-# Because migrations instance_eval the up and down blocks on a database,
+# Because migrations instance_exec the up and down blocks on a database,
 # using this extension in a migration can be done via:
 #
 #   Sequel.migration do
@@ -41,7 +43,7 @@
 # similar to the validation_helpers model plugin API.  However,
 # instead of having separate validates_* methods, it just adds a validate
 # method that accepts a block to the schema generators.  Like the
-# create_table and alter_table blocks, this block is instance_evaled and
+# create_table and alter_table blocks, this block is instance_execed and
 # offers its own DSL. Example:
 #
 #   DB.create_table(:table) do
@@ -54,8 +56,8 @@
 #     end
 #   end
 #
-# instance_eval is used in this case because create_table and alter_table
-# already use instance_eval, so losing access to the surrounding receiver
+# instance_exec is used in this case because create_table and alter_table
+# already use instance_exec, so losing access to the surrounding receiver
 # is not an issue.
 #
 # Here's a breakdown of the constraints created for each constraint validation
@@ -76,6 +78,10 @@
 # includes [1, 2] :: CHECK column IN (1, 2)
 # includes 3..5 :: CHECK column >= 3 AND column <= 5
 # includes 3...5 :: CHECK column >= 3 AND column < 5
+# operator :>, 1 :: CHECK column > 1
+# operator :>=, 2 :: CHECK column >= 2
+# operator :<, "M" :: CHECK column < 'M'
+# operator :<=, 'K' :: CHECK column <= 'K'
 # unique :: UNIQUE (column)
 #
 # There are some additional API differences:
@@ -92,6 +98,8 @@
 #   patters are very simple, so many regexp patterns cannot be expressed by
 #   them, but only a couple databases (PostgreSQL and MySQL) support regexp
 #   patterns.
+# * The operator validation only supports >, >=, <, and <= operators, and the
+#   argument must be a string or an integer.
 # * When using the unique validation, column names cannot have embedded commas.
 #   For similar reasons, when using an includes validation with an array of
 #   strings, none of the strings in the array can have embedded commas.
@@ -121,12 +129,21 @@
 #   <tt>drop_constraint_validations_for(:table=>'table')</tt>, and then
 #   readd all constraints you want to use inside the alter table block,
 #   making no other changes inside the alter_table block.
+#
+# Dropping a table will automatically delete all constraint validations for
+# that table.  However, altering a table (e.g. to drop a column) will not
+# currently make any changes to the constraint validations metadata.
+#
+# Related module: Sequel::ConstraintValidations
 
 #
 module Sequel
   module ConstraintValidations
     # The default table name used for the validation metadata.
     DEFAULT_CONSTRAINT_VALIDATIONS_TABLE = :sequel_constraint_validations
+    OPERATORS = {:< => :lt, :<= => :lte, :> => :gt, :>= => :gte}.freeze
+    REVERSE_OPERATOR_MAP = {:str_lt => :<, :str_lte => :<=, :str_gt => :>, :str_gte => :>=,
+                            :int_lt => :<, :int_lte => :<=, :int_gt => :>, :int_gte => :>=}.freeze
 
     # Set the default validation metadata table name if it has not already
     # been set.
@@ -160,15 +177,32 @@ module Sequel
         END
       end
 
+      # Create operator validation.  The op should be either +:>+, +:>=+, +:<+, or +:<=+, and
+      # the arg should be either a string or an integer.
+      def operator(op, arg, columns, opts=OPTS)
+        raise Error, "invalid operator (#{op}) used when creating operator validation" unless suffix = OPERATORS[op]
+
+        prefix = case arg
+        when String
+          "str"
+        when Integer
+          "int"
+        else
+          raise Error, "invalid argument (#{arg.inspect}) used when creating operator validation"
+        end
+
+        @generator.validation({:type=>:"#{prefix}_#{suffix}", :columns=>Array(columns), :arg=>arg}.merge!(opts))
+      end
+
       # Given the name of a constraint, drop that constraint from the database,
       # and remove the related validation metadata.
       def drop(constraint)
         @generator.validation({:type=>:drop, :name=>constraint})
       end
 
-      # Alias of instance_eval for a nicer API.
+      # Alias of instance_exec for a nicer API.
       def process(&block)
-        instance_eval(&block)
+        instance_exec(&block)
       end
     end
 
@@ -230,10 +264,20 @@ module Sequel
       super do
         extend CreateTableGeneratorMethods
         @validations = []
-        instance_eval(&block) if block
+        instance_exec(&block) if block
       end
     end
 
+    # Drop all constraint validations for a table if dropping the table.
+    def drop_table(*names)
+      names.each do |name|
+        if !name.is_a?(Hash) && table_exists?(constraint_validations_table)
+          drop_constraint_validations_for(:table=>name)
+        end
+      end
+      super
+    end
+
     # Drop the constraint validations table.
     def drop_constraint_validations_table
       drop_table(constraint_validations_table)
@@ -278,7 +322,7 @@ module Sequel
       super do
         extend AlterTableGeneratorMethods
         @validations = []
-        instance_eval(&block) if block
+        instance_exec(&block) if block
       end
     end
 
@@ -313,9 +357,7 @@ module Sequel
     # This allows the code to handle schema qualified tables,
     # without quoting all table names.
     def constraint_validations_literal_table(table)
-      ds = dataset
-      ds.quote_identifiers = false
-      ds.literal(table)
+      dataset.with_quote_identifiers(false).literal(table)
     end
 
     # Before creating the table, add constraints for all of the
@@ -327,6 +369,18 @@ module Sequel
       super
     end
 
+    def constraint_validation_expression(cols, allow_nil)
+      exprs = cols.map do |c|
+        expr = yield c
+        if allow_nil
+          Sequel.|({c=>nil}, expr)
+        else
+          Sequel.&(Sequel.~(c=>nil), expr)
+        end
+      end
+      Sequel.&(*exprs)
+    end
+
     # For the given table, generator, and validations, add constraints
     # to the generator for each of the validations, as well as adding
     # validation metadata to the constraint validations table.
@@ -337,26 +391,44 @@ module Sequel
 
         case validation_type
         when :presence
-          string_check = columns.select{|c| generator_string_column?(generator, table, c)}.map{|c| [Sequel.trim(c), blank_string_value]}
-          generator_add_constraint_from_validation(generator, val, (Sequel.negate(string_check) unless string_check.empty?))
+          strings, non_strings = columns.partition{|c| generator_string_column?(generator, table, c)}
+          if !non_strings.empty? && !allow_nil
+            non_strings_expr = Sequel.&(*non_strings.map{|c| Sequel.~(c=>nil)})
+          end
+
+          unless strings.empty?
+            strings_expr = constraint_validation_expression(strings, allow_nil){|c| Sequel.~(Sequel.trim(c) => blank_string_value)}
+          end
+
+          expr = if non_strings_expr && strings_expr
+            Sequel.&(strings_expr, non_strings_expr)
+          else
+            strings_expr || non_strings_expr
+          end
+
+          if expr
+            generator.constraint(constraint, expr)
+          end
         when :exact_length
-          generator_add_constraint_from_validation(generator, val, Sequel.&(*columns.map{|c| {Sequel.char_length(c) => arg}}))
+          generator.constraint(constraint, constraint_validation_expression(columns, allow_nil){|c| {Sequel.char_length(c) => arg}})
         when :min_length
-          generator_add_constraint_from_validation(generator, val, Sequel.&(*columns.map{|c| Sequel.char_length(c) >= arg}))
+          generator.constraint(constraint, constraint_validation_expression(columns, allow_nil){|c| Sequel.char_length(c) >= arg})
         when :max_length
-          generator_add_constraint_from_validation(generator, val, Sequel.&(*columns.map{|c| Sequel.char_length(c) <= arg}))
+          generator.constraint(constraint, constraint_validation_expression(columns, allow_nil){|c| Sequel.char_length(c) <= arg})
+        when *REVERSE_OPERATOR_MAP.keys
+          generator.constraint(constraint, constraint_validation_expression(columns, allow_nil){|c| Sequel.identifier(c).public_send(REVERSE_OPERATOR_MAP[validation_type], arg)})
         when :length_range
           op = arg.exclude_end? ? :< : :<=
-          generator_add_constraint_from_validation(generator, val, Sequel.&(*columns.map{|c| (Sequel.char_length(c) >= arg.begin) & Sequel.char_length(c).send(op, arg.end)}))
+          generator.constraint(constraint, constraint_validation_expression(columns, allow_nil){|c| (Sequel.char_length(c) >= arg.begin) & Sequel.char_length(c).public_send(op, arg.end)})
           arg = "#{arg.begin}..#{'.' if arg.exclude_end?}#{arg.end}"
         when :format
-          generator_add_constraint_from_validation(generator, val, Sequel.&(*columns.map{|c| {c => arg}}))
+          generator.constraint(constraint, constraint_validation_expression(columns, allow_nil){|c| {c => arg}})
           if arg.casefold?
             validation_type = :iformat
           end
           arg = arg.source
         when :includes
-          generator_add_constraint_from_validation(generator, val, Sequel.&(*columns.map{|c| {c => arg}}))
+          generator.constraint(constraint, constraint_validation_expression(columns, allow_nil){|c| {c => arg}})
           if arg.is_a?(Range)
             if arg.begin.is_a?(Integer) && arg.end.is_a?(Integer)
               validation_type = :includes_int_range
@@ -377,7 +449,7 @@ module Sequel
             raise Error, "validates includes only supports arrays and ranges currently, cannot handle: #{arg.inspect}"
           end
         when :like, :ilike
-          generator_add_constraint_from_validation(generator, val, Sequel.&(*columns.map{|c| Sequel.send(validation_type, c, arg)}))
+          generator.constraint(constraint, constraint_validation_expression(columns, allow_nil){|c| Sequel.public_send(validation_type, c, arg)})
         when :unique
           generator.unique(columns, :name=>constraint)
           columns = [columns.join(',')]
@@ -401,30 +473,13 @@ module Sequel
         end
       end
 
-      ds = from(:sequel_constraint_validations)
+      ds = from(constraint_validations_table)
       unless drop_rows.empty?
         ds.where([:table, :constraint_name]=>drop_rows).delete
       end
       ds.multi_insert(rows.flatten)
     end
 
-    # Add the constraint to the generator, including a NOT NULL constraint
-    # for all columns unless the :allow_nil option is given.
-    def generator_add_constraint_from_validation(generator, val, cons)
-      if val[:allow_nil]
-        nil_cons = Sequel.expr(val[:columns].map{|c| [c, nil]})
-        cons = Sequel.|(nil_cons, cons) if cons
-      else
-        nil_cons = Sequel.negate(val[:columns].map{|c| [c, nil]})
-        cons = cons ? Sequel.&(nil_cons, cons) : nil_cons
-      end
-
-      if cons
-        generator.constraint(val[:name], cons)
-      end
-    end
-
-
     # Introspect the generator to determine if column
     # created is a string or not.
     def generator_string_column?(generator, table, c)