sequel 5.83.0 → 5.106.0

data/lib/sequel/dataset/deprecated_singleton_class_methods.rb

@@ -19,7 +19,7 @@ module Sequel
       def with_extend(*mods, &block)
         c = _clone(:freeze=>false)
         c.extend(*mods) unless mods.empty?
-        c.extend(DatasetModule.new(&block)) if block
+        c.extend(Sequel.set_temp_name(DatasetModule.new(&block)){"Sequel::Dataset::_DatasetModule(#{block.source_location.join(':')})"}) if block
         c.freeze
       end
 

data/lib/sequel/dataset/placeholder_literalizer.rb

@@ -117,6 +117,8 @@ module Sequel
           frags << final_sql
           prepared_sql << final_sql
 
+          frags.each(&:freeze)
+          frags.freeze
           [prepared_sql, frags]
         end
 
@@ -125,6 +127,7 @@ module Sequel
           @argn = -1
           @args = []
           ds = yield self, dataset
+          ds.opts[:sql].freeze
           sql = ds.clone(:placeholder_literalizer=>self).sql
 
           last_offset = 0

data/lib/sequel/dataset/prepared_statements.rb

@@ -20,7 +20,8 @@ module Sequel
     def self.prepared_statements_module(code, mods, meths=DEFAULT_PREPARED_STATEMENT_MODULE_METHODS, &block)
       code = PREPARED_STATEMENT_MODULE_CODE[code] || code
 
-      Module.new do
+      Module.new do 
+        Sequel.set_temp_name(self){"Sequel::Dataset::_PreparedStatementsModule(#{block.source_location.join(':') if block})"}
         Array(mods).each do |mod|
           include mod
         end
@@ -53,7 +54,7 @@ module Sequel
 
       # Set the bind arguments based on the hash and call super.
       def call(bind_vars=OPTS, &block)
-        sql = prepared_sql
+        sql = prepared_sql.freeze
         prepared_args.freeze
         ps = bind(bind_vars)
         ps.clone(:bind_arguments=>ps.map_to_prepared_args(ps.opts[:bind_vars]), :sql=>sql, :prepared_sql=>sql).run(&block)
@@ -90,8 +91,14 @@ module Sequel
         @opts[:log_sql]
       end
       
-      # The type of prepared statement, should be one of :select, :first,
-      # :insert, :update, :delete, or :single_value
+      # The type of SQL to generate for the prepared statement.  Generally
+      # the same as #prepared_type, but can be different.
+      def prepared_sql_type
+        @opts[:prepared_sql_type] || prepared_type
+      end
+      
+      # The type of prepared statement, which controls how the prepared statement
+      # handles results from the database.
       def prepared_type
         @opts[:prepared_type]
       end
@@ -140,7 +147,7 @@ module Sequel
       # Returns the SQL for the prepared statement, depending on
       # the type of the statement and the prepared_modify_values.
       def prepared_sql
-        case prepared_type
+        case prepared_sql_type
         when :select, :all, :each
           # Most common scenario, so listed first.
           select_sql
@@ -181,34 +188,30 @@ module Sequel
       
       # Run the method based on the type of prepared statement.
       def run(&block)
-        case prepared_type
+        case type = prepared_type
         when :select, :all
-          all(&block)
+          with_sql_all(prepared_sql, &block)
         when :each
-          each(&block)
-        when :insert_select
-          with_sql(prepared_sql).first
-        when :first
-          first
+          with_sql_each(prepared_sql, &block)
+        when :insert_select, :first
+          with_sql_first(prepared_sql)
         when :insert, :update, :delete
           if opts[:returning] && supports_returning?(prepared_type)
             returning_fetch_rows(prepared_sql)
-          elsif prepared_type == :delete
-            delete
+          elsif type == :delete
+            with_sql_delete(prepared_sql)
           else
-            public_send(prepared_type, *prepared_modify_values)
+            force_prepared_sql.public_send(type, *prepared_modify_values)
           end
-        when :insert_pk
-          fetch_rows(prepared_sql){|r| return r.values.first}
+        when :insert_pk, :single_value
+          with_sql_single_value(prepared_sql)
         when Array
           # :nocov:
-          case prepared_type[0]
+          case type[0]
           # :nocov:
           when :map, :as_hash, :to_hash, :to_hash_groups
-            public_send(*prepared_type, &block) 
+            force_prepared_sql.public_send(*type, &block) 
           end
-        when :single_value
-          single_value
         else
           raise Error, "unsupported prepared statement type used: #{prepared_type.inspect}"
         end
@@ -216,6 +219,16 @@ module Sequel
       
       private
       
+      # If the prepared_sql_type does not match the prepared statement, return a clone that
+      # with the prepared SQL, to ensure the prepared_sql_type is respected.
+      def force_prepared_sql
+        if prepared_sql_type != prepared_type
+          with_sql(prepared_sql.freeze)
+        else
+          self
+        end
+      end
+      
       # Returns the value of the prepared_args hash for the given key.
       def prepared_arg(k)
         @opts[:bind_vars][k]
@@ -274,7 +287,7 @@ module Sequel
 
       def run(&block)
         if @opts[:prepared_sql_frags]
-          sql = literal(Sequel::SQL::PlaceholderLiteralString.new(@opts[:prepared_sql_frags], @opts[:bind_arguments], false))
+          sql = literal(Sequel::SQL::PlaceholderLiteralString.new(@opts[:prepared_sql_frags], @opts[:bind_arguments], false)).freeze
           clone(:prepared_sql_frags=>nil, :sql=>sql, :prepared_sql=>sql).run(&block)
         else
           super
@@ -293,11 +306,12 @@ module Sequel
         prepared_sql, frags = Sequel::Dataset::PlaceholderLiteralizer::Recorder.new.send(:prepared_sql_and_frags, self, prepared_args) do |pl, ds|
           ds = ds.clone(:recorder=>pl)
 
-          case type
+          sql_type = prepared_sql_type || type
+          case sql_type
           when :first, :single_value
             ds.limit(1)
           when :update, :insert, :insert_select, :delete
-            ds.with_sql(:"#{type}_sql", *values)
+            ds.with_sql(:"#{sql_type}_sql", *values)
           when :insert_pk
             ds.with_sql(:insert_sql, *values)
           else
@@ -306,6 +320,9 @@ module Sequel
         end
 
         prepared_args.freeze
+        frags.freeze
+        frags.each(&:freeze)
+        prepared_sql.freeze
         clone(:prepared_sql_frags=>frags, :prepared_sql=>prepared_sql, :sql=>prepared_sql)
       end
 
@@ -343,13 +360,23 @@ module Sequel
       clone(:bind_vars=>bind_vars)
     end
     
-    # For the given type (:select, :first, :insert, :insert_select, :update, :delete, or :single_value),
-    # run the sql with the bind variables specified in the hash.  +values+ is a hash passed to
-    # insert or update (if one of those types is used), which may contain placeholders.
+    # For the given type, run the sql with the bind variables specified in the hash.
+    # +values+ is a hash passed to insert or update (if one of those types is used),
+    # which may contain placeholders.
+    #
+    # The following types are supported:
+    # 
+    # * :select, :all, :each, :first, :single_value, :insert, :insert_select, :insert_pk, :update, :delete
+    # * Array where first element is :map, :as_hash, :to_hash, :to_hash_groups (remaining elements
+    #   are passed to the related method)
     #
     #   DB[:table].where(id: :$id).call(:first, id: 1)
     #   # SELECT * FROM table WHERE id = ? LIMIT 1 -- (1)
     #   # => {:id=>1}
+    #
+    #   DB[:table].where(id: :$id).call(:update, {c: 1, id: 2}, col: :$c)
+    #   # UPDATE table WHERE id = ? SET col = ? -- (2, 1)
+    #   # => 1
     def call(type, bind_variables=OPTS, *values, &block)
       to_prepared_statement(type, values, :extend=>bound_variable_modules).call(bind_variables, &block)
     end
@@ -370,6 +397,14 @@ module Sequel
     #   # => {:id=>1, :name=>'Blah'}
     #
     #   DB.call(:select_by_name, name: 'Blah') # Same thing
+    #
+    # +values+ given are passed to +insert+ or +update+ if they are used:
+    #
+    #   ps = DB[:table].where(id: :$i).prepare(:update, :update_name, name: :$n)
+    #
+    #   ps.call(i: 1, n: 'Blah')
+    #   # UPDATE table WHERE id = ? SET name = ? -- (1, 'Blah')
+    #   # => 1
     def prepare(type, name, *values)
       ps = to_prepared_statement(type, values, :name=>name, :extend=>prepared_statement_modules, :no_delayed_evaluations=>true)
 
@@ -377,7 +412,7 @@ module Sequel
         ps = ps.with_extend(EmulatePreparedStatementMethods)
         ps.send(:emulated_prepared_statement, type, name, values)
       else
-        sql = ps.prepared_sql
+        sql = ps.prepared_sql.freeze
         ps.prepared_args.freeze
         ps.clone(:prepared_sql=>sql, :sql=>sql)
       end
@@ -386,6 +421,19 @@ module Sequel
       ps
     end
     
+    # Set the type of SQL to use for prepared statements based on this
+    # dataset.  Prepared statements default to using the same SQL type
+    # as the type that is passed to #prepare/#call, but there are cases
+    # where it is helpful to use a different SQL type.
+    #
+    # Available types are: :select, :first, :single_value, :update,
+    # :delete, :insert, :insert_select, :insert_pk
+    #
+    # Other types are treated as :select.
+    def prepare_sql_type(type)
+      clone(:prepared_sql_type => type)
+    end
+
     protected
     
     # Return a cloned copy of the current dataset extended with

data/lib/sequel/dataset/query.rb

@@ -204,7 +204,7 @@ module Sequel
       # If no related extension file exists or the extension does not have
       # specific support for Dataset objects, an error will be raised.
       def extension(*exts)
-        Sequel.extension(*exts)
+        exts.each{|ext| Sequel.extension(ext) unless Sequel.synchronize{EXTENSIONS[ext]}}
         mods = exts.map{|ext| Sequel.synchronize{EXTENSION_MODULES[ext]}}
         if mods.all?
           with_extend(*mods)
@@ -231,8 +231,7 @@ module Sequel
     #
     #   DB[:table].for_update # SELECT * FROM table FOR UPDATE
     def for_update
-       return self if opts[:lock] == :update
-      cached_dataset(:_for_update_ds){lock_style(:update)}
+      cached_lock_style_dataset(:_for_update_ds, :update)
     end
 
     # Returns a copy of the dataset with the source changed. If no
@@ -1238,9 +1237,9 @@ module Sequel
       # Note that like Object#extend, when multiple modules are provided
       # as arguments the subclass includes the modules in reverse order.
       def with_extend(*mods, &block)
-        c = Class.new(self.class)
+        c = Sequel.set_temp_name(Class.new(self.class)){"Sequel::Dataset::_Subclass"}
         c.include(*mods) unless mods.empty?
-        c.include(DatasetModule.new(&block)) if block
+        c.include(Sequel.set_temp_name(DatasetModule.new(&block)){"Sequel::Dataset::_DatasetModule(#{block.source_location[0,2].join(':')})"}) if block
         o = c.freeze.allocate
         o.instance_variable_set(:@db, @db)
         o.instance_variable_set(:@opts, @opts)
@@ -1302,7 +1301,7 @@ module Sequel
     # * truncate (if a TRUNCATE statement, with no arguments)
     def with_sql(sql, *args)
       if sql.is_a?(Symbol)
-        sql = public_send(sql, *args)
+        sql = public_send(sql, *args).freeze
       else
         sql = SQL::PlaceholderLiteralString.new(sql, args) unless args.empty?
       end
@@ -1359,9 +1358,13 @@ module Sequel
     unless TRUE_FREEZE
       # Load the extensions into the receiver, without checking if the receiver is frozen.
       def _extension!(exts)
-        Sequel.extension(*exts)
         exts.each do |ext|
-          if pr = Sequel.synchronize{EXTENSIONS[ext]}
+          unless pr = Sequel.synchronize{EXTENSIONS[ext]}
+            Sequel.extension(ext)
+            pr = Sequel.synchronize{EXTENSIONS[ext]}
+          end
+
+          if pr
             pr.call(self)
           else
             raise(Error, "Extension #{ext} does not have specific support handling individual datasets (try: Sequel.extension #{ext.inspect})")
@@ -1458,11 +1461,23 @@ module Sequel
       end
     end
 
+    # Internals of for_update and adapter-specific lock methods.
+    # Returns receiver if it already uses this lock style, and a cached
+    # dataset using the given key otherwise. The key could be derived from
+    # the style, but doing so would require allocation, so pass it in as
+    # an argument.
+    def cached_lock_style_dataset(key, style)
+      opts[:lock] == style ? self : cached_dataset(key){lock_style(style)}
+    end
+
     # The default :qualify option to use for join tables if one is not specified.
     def default_join_table_qualification
       :symbol
     end
-    
+
+    PAREN_WRAPPER = ["(".freeze, ")".freeze].freeze
+    private_constant :PAREN_WRAPPER
+
     # SQL expression object based on the expr type.  See +where+.
     def filter_expr(expr = nil, &block)
       expr = nil if expr == EMPTY_ARRAY
@@ -1483,8 +1498,8 @@ module Sequel
           raise Error, "Invalid filter expression: #{expr.inspect}"
         end
       when LiteralString
-        LiteralString.new("(#{expr})")
-      when Numeric, SQL::NumericExpression, SQL::StringExpression, Proc, String
+        SQL::PlaceholderLiteralString.new(PAREN_WRAPPER, [expr])
+      when Numeric, SQL::NumericExpression, SQL::StringExpression, Proc, String, Set
         raise Error, "Invalid filter expression: #{expr.inspect}"
       when TrueClass, FalseClass
         if supports_where_true?

data/lib/sequel/dataset/sql.rb

@@ -53,7 +53,7 @@ module Sequel
       when String
         case v
         when LiteralString
-          sql << v
+          literal_literal_string_append(sql, v)
         when SQL::Blob
           literal_blob_append(sql, v)
         else
@@ -85,6 +85,8 @@ module Sequel
         literal_date_append(sql, v)
       when Dataset
         literal_dataset_append(sql, v)
+      when Set
+        literal_set_append(sql, v)
       else
         literal_other_append(sql, v)
       end
@@ -375,9 +377,9 @@ module Sequel
         cols = args[0]
         vals = args[1]
         col_array = true if cols.is_a?(Array)
-        if vals.is_a?(Array)
+        if vals.is_a?(Array) || vals.is_a?(Set)
           val_array = true
-          empty_val_array = vals == []
+          empty_val_array = vals.empty?
         end
         if empty_val_array
           literal_append(sql, empty_array_value(op, cols))
@@ -560,6 +562,7 @@ module Sequel
       end
 
       if window = opts[:over]
+        sql << ' IGNORE NULLS' if window.opts[:ignore_nulls]
         sql << ' OVER '
         window_sql_append(sql, window.opts)
       end
@@ -636,12 +639,11 @@ module Sequel
         if args.empty?
           sql << str
         else
-          re = /:(#{args.keys.map{|k| Regexp.escape(k.to_s)}.join('|')})\b/
-          while true
+          re = /:(#{Regexp.union(args.keys.map(&:to_s))})\b/
+          until str.empty?
             previous, q, str = str.partition(re)
             sql << previous
-            literal_append(sql, args[($1||q[1..-1].to_s).to_sym]) unless q.empty?
-            break if str.empty?
+            literal_append(sql, args[$1.to_sym]) unless q.empty?
           end
         end
       elsif str.is_a?(Array)
@@ -685,10 +687,10 @@ module Sequel
     # being quoted, returns name as a string.  If identifiers are being quoted
     # quote the name with quoted_identifier.
     def quote_identifier_append(sql, name)
+      name = name.value if name.is_a?(SQL::Identifier)
       if name.is_a?(LiteralString)
         sql << name
       else
-        name = name.value if name.is_a?(SQL::Identifier)
         name = input_identifier(name)
         if quote_identifiers?
           quoted_identifier_append(sql, name)
@@ -700,11 +702,14 @@ module Sequel
 
     # Append literalization of identifier or unqualified identifier to SQL string.
     def quote_schema_table_append(sql, table)
-      schema, table = schema_and_table(table)
-      if schema
-        quote_identifier_append(sql, schema)
+      qualifiers = split_qualifiers(table)
+      table = qualifiers.pop
+
+      qualifiers.each do |q|
+        quote_identifier_append(sql, q)
         sql << '.'
       end
+
       quote_identifier_append(sql, table)
     end
 
@@ -1032,7 +1037,7 @@ module Sequel
       if column_aliases
         raise Error, "#{db.database_type} does not support derived column lists" unless supports_derived_column_lists?
         sql << '('
-        identifier_list_append(sql, column_aliases)
+        derived_column_list_sql_append(sql, column_aliases)
         sql << ')'
       end
     end
@@ -1165,6 +1170,11 @@ module Sequel
       end
     end
 
+    # Append the column aliases to the SQL.
+    def derived_column_list_sql_append(sql, column_aliases)
+      identifier_list_append(sql, column_aliases)
+    end
+    
     # Disable caching of SQL for the current dataset
     def disable_sql_caching!
       cache_set(:_no_cache_sql, true)
@@ -1331,7 +1341,7 @@ module Sequel
     # SQL fragment specifying a JOIN type, converts underscores to
     # spaces and upcases.
     def join_type_sql(join_type)
-      "#{join_type.to_s.gsub('_', ' ').upcase} JOIN"
+      "#{join_type.to_s.tr('_', ' ').upcase} JOIN"
     end
 
     # Append USING clause for JOIN USING
@@ -1415,6 +1425,11 @@ module Sequel
       v.to_s
     end
 
+    # Append string to SQL string.
+    def literal_literal_string_append(sql, v)
+      sql << v
+    end
+
     # SQL fragment for nil
     def literal_nil
       "NULL"
@@ -1425,10 +1440,6 @@ module Sequel
     # calls +sql_literal+ if object responds to it, otherwise raises an error.
     # If a database specific type is allowed, this should be overriden in a subclass.
     def literal_other_append(sql, v)
-      # We can't be sure if v will always literalize to the same SQL, so
-      # don't cache SQL for a dataset that uses this.
-      disable_sql_caching!
-
       if v.respond_to?(:sql_literal_append)
         v.sql_literal_append(self, sql)
       elsif v.respond_to?(:sql_literal)
@@ -1436,6 +1447,18 @@ module Sequel
       else
         raise Error, "can't express #{v.inspect} as a SQL literal"
       end
+
+      if !v.respond_to?(:sql_literal_allow_caching?) || !v.sql_literal_allow_caching?(self)
+        # We can't be sure if v will always literalize to the same SQL, so
+        # don't cache SQL for a dataset that uses this.
+        disable_sql_caching!
+      end
+    end
+
+    # Append a literalization of the set to SQL string.
+    # Treats as an expression as an SQL value list.
+    def literal_set_append(sql, v)
+      array_sql_append(sql, v)
     end
 
     # SQL fragment for Sequel::SQLTime, containing just the time part

data/lib/sequel/extensions/async_thread_pool.rb

@@ -357,7 +357,7 @@ module Sequel
           define_singleton_method(:async_job_class){proxy_klass}
 
           queue = @async_thread_queue = Queue.new
-          pool = @async_thread_pool = num_async_threads.times.map{JobProcessor.new(queue)}
+          pool = @async_thread_pool = Array.new(num_async_threads){JobProcessor.new(queue)}
           ObjectSpace.define_finalizer(db, JobProcessor.create_finalizer(queue, pool))
 
           extend_datasets(DatasetMethods)

data/lib/sequel/extensions/caller_logging.rb

@@ -60,9 +60,7 @@ module Sequel
     def external_caller_for_log
       ignore = caller_logging_ignore
       c = caller.find do |line|
-        !(line.start_with?(SEQUEL_LIB_PATH) ||
-          line.start_with?(RUBY_STDLIB) ||
-          line.start_with?(INTERNAL) ||
+        !(line.start_with?(SEQUEL_LIB_PATH, RUBY_STDLIB, INTERNAL) ||
           (ignore && line =~ ignore))
       end
 

data/lib/sequel/extensions/connection_checkout_event_callback.rb

@@ -0,0 +1,151 @@
+# frozen-string-literal: true
+#
+# The connection_checkout_event_callback extension modifies a database's
+# connection pool to allow for a checkout event callback. This callback is
+# called with the following arguments:
+#
+# :immediately_available :: Connection immediately available and returned
+# :not_immediately_available :: Connection not immediately available
+# :new_connection :: New connection created and returned
+# Float :: Number of seconds waiting to acquire a connection
+#
+# This is a low-level extension that allows for building telemetry
+# information. It doesn't implement any telemetry reporting itself. The
+# main reason for recording this information is to use it to determine the
+# appropriate size for the connection pool. Having too large a connection
+# pool can waste resources, while having too small a connection pool can
+# result in substantial time to check out a connection. In general, you
+# want to use as small a pool as possible while keeping the time to
+# checkout a connection low.
+#
+# To use the connection checkout event callback, you must first load the
+# extension:
+#
+#   DB.extension(:connection_checkout_event_callback)
+#
+# By default, an empty proc is used as the callback so that loading the
+# support doesn't break anything. If you are using the extension, you
+# should set the callback at some point during application startup:
+#
+#   DB.pool.on_checkout_event = proc do |event|
+#     # ...
+#   end
+#
+# When using the sharded connection pool, the callback is also
+# passed a second argument, the requested server shard (generally a
+# symbol), allowing for collection of per-shard telemetry:
+#
+#   DB.pool.on_checkout_event = proc do |event, server|
+#     # ...
+#   end
+#
+# Note that the callback may be called currently by multiple threads.
+# You should use some form of concurrency control inside the callback,
+# such as a mutex or queue.
+#
+# Below is a brief example of usage to determine the percentage of
+# connection requests where a connection was immediately available:
+#
+#   mutex = Mutex.new
+#   total = immediates = 0
+#
+#   DB.pool.on_checkout_event = proc do |event|
+#     case event
+#     when :immediately_available
+#       mutex.synchronize do
+#         total += 1
+#         immediates += 1
+#       end
+#     when :not_immediately_available
+#       mutex.synchronize do
+#         total += 1
+#       end
+#     end
+#   end
+#
+#   immediate_percentage = lambda do
+#     mutex.synchronize do
+#       100.0 * immediates / total
+#     end
+#   end
+#   
+# Note that this extension only works with the timed_queue
+# and sharded_timed_queue connection pools (the default
+# connection pools when using Ruby 3.2+).
+#
+# Related modules: Sequel::ConnectionCheckoutEventCallbacks::TimedQueue,
+# Sequel::ConnectionCheckoutEventCallbacks::ShardedTimedQueue
+
+#
+module Sequel
+  module ConnectionCheckoutEventCallbacks
+    module TimedQueue
+      # The callback that is called with connection checkout events.
+      attr_accessor :on_checkout_event
+
+      private
+
+      def available
+        conn = super
+        @on_checkout_event.call(conn ? :immediately_available : :not_immediately_available)
+        conn
+      end
+
+      def try_make_new
+        conn = super
+        @on_checkout_event.call(:new_connection) if conn
+        conn
+      end
+
+      def wait_until_available
+        timer = Sequel.start_timer
+        conn = super
+        @on_checkout_event.call(Sequel.elapsed_seconds_since(timer))
+        conn
+      end
+    end
+
+    module ShardedTimedQueue
+      # The callback that is called with connection checkout events.
+      attr_accessor :on_checkout_event
+
+      private
+
+      def available(queue, server)
+        conn = super
+        @on_checkout_event.call(conn ? :immediately_available : :not_immediately_available, server)
+        conn
+      end
+
+      def try_make_new(server)
+        conn = super
+        @on_checkout_event.call(:new_connection, server) if conn
+        conn
+      end
+
+      def wait_until_available(queue, server)
+        timer = Sequel.start_timer
+        conn = super
+        @on_checkout_event.call(Sequel.elapsed_seconds_since(timer), server)
+        conn
+      end
+    end
+  end
+
+  default_callback = proc{}
+
+  Database.register_extension(:connection_checkout_event_callback) do |db|
+    pool = db.pool
+
+    case pool.pool_type
+    when :timed_queue
+      db.pool.extend(ConnectionCheckoutEventCallbacks::TimedQueue)
+    when :sharded_timed_queue
+      db.pool.extend(ConnectionCheckoutEventCallbacks::ShardedTimedQueue)
+    else
+      raise Error, "the connection_checkout_event_callback extension is only supported when using a timed_queue connection pool"
+    end
+
+    pool.on_checkout_event ||= default_callback
+  end
+end