sequel 5.20.0 → 5.49.0

data/lib/sequel/plugins/composition.rb

@@ -143,10 +143,14 @@ module Sequel
                 compositions[name] = send(composer_meth)
               end
             end
-            define_method("#{name}=") do |v|
+            alias_method(name, name)
+
+            meth = :"#{name}="
+            define_method(meth) do |v|
               modified!
               compositions[name] = v
             end
+            alias_method(meth, meth)
           end
         end
 
@@ -167,8 +171,9 @@ module Sequel
 
         # Freeze compositions hash when freezing model instance.
         def freeze
-          compositions.freeze
+          compositions
           super
+          compositions.freeze
         end
 
         # For each composition, set the columns in the model class based

data/lib/sequel/plugins/concurrent_eager_loading.rb

@@ -0,0 +1,174 @@
+# frozen-string-literal: true
+
+module Sequel
+  extension 'async_thread_pool'
+
+  module Plugins
+    # The concurrent_eager_loading plugin allows for eager loading multiple associations
+    # concurrently in separate threads.  You must load the async_thread_pool Database
+    # extension into the Database object the model class uses in order for this plugin
+    # to work.
+    # 
+    # By default in Sequel, eager loading happens in a serial manner.  If you have code
+    # such as:
+    #
+    #   Album.eager(:artist, :genre, :tracks)
+    #
+    # Sequel will load the albums, then the artists for the albums, then
+    # the genres for the albums, then the tracks for the albums.
+    #
+    # With the concurrent_eager_loading plugin, you can use the +eager_load_concurrently+
+    # method to allow for concurrent eager loading:
+    #
+    #   Album.eager_load_concurrently.eager(:artist, :genre, :tracks)
+    #
+    # This will load the albums, first, since it needs to load the albums to know
+    # which artists, genres, and tracks to eagerly load. However, it will load the
+    # artists, genres, and tracks for the albums concurrently in separate threads.
+    # This can significantly improve performance, especially if there is significant
+    # latency between the application and the database. Note that using separate threads
+    # is only used in the case where there are multiple associations to eagerly load.
+    # With only a single association to eagerly load, there is no reason to use a
+    # separate thread, since it would not improve performance.
+    #
+    # If you want to make concurrent eager loading the default, you can load the
+    # plugin with the +:always+ option. In this case, all eager loads will be
+    # concurrent.  If you want to force a non-concurrent eager load, you can use
+    # +eager_load_serially+:
+    #
+    #   Album.eager_load_serially.eager(:artist, :genre, :tracks)
+    #
+    # Note that making concurrent eager loading the default is probably a bad idea
+    # if you are eager loading inside transactions and want the eager load to
+    # reflect changes made inside the transaction, unless you plan to use
+    # +eager_load_serially+ for such cases.  See the async_thread_pool
+    # Database extension documentation for more general caveats regarding its use.
+    #
+    # The default eager loaders for all of the association types that ship with Sequel
+    # support safe concurrent eager loading.  However, if you are specifying a custom
+    # +:eager_loader+ for an association, it may not work safely unless it it modified to
+    # support concurrent eager loading.  Taking this example from the
+    # {Advanced Associations guide}[rdoc-ref:doc/advanced_associations.rdoc]
+    #
+    #   Album.many_to_one :artist, :eager_loader=>(proc do |eo_opts|
+    #     eo_opts[:rows].each{|album| album.associations[:artist] = nil}
+    #     id_map = eo_opts[:id_map]
+    #     Artist.where(:id=>id_map.keys).all do |artist|
+    #       if albums = id_map[artist.id]
+    #         albums.each do |album|
+    #           album.associations[:artist] = artist
+    #         end
+    #       end
+    #     end
+    #   end)
+    #
+    # This would not support concurrent eager loading safely.  To support safe
+    # concurrent eager loading, you need to make sure you are not modifying
+    # the associations for objects concurrently by separate threads.  This is
+    # implemented using a mutex, which you can access via <tt>eo_opts[:mutex]</tt>.
+    # To keep things simple, you can use +Sequel.synchronize_with+ to only
+    # use this mutex if it is available.  You want to use the mutex around the
+    # code that initializes the associations (usually to +nil+ or <tt>[]</tt>),
+    # and also around the code that sets the associatied objects appropriately
+    # after they have been retreived.  You do not want to use the mutex around
+    # the code that loads the objects, since that will prevent concurrent loading.
+    # So after the changes, the custom eager loader would look like this:
+    #
+    #   Album.many_to_one :artist, :eager_loader=>(proc do |eo_opts|
+    #     Sequel.synchronize_with(eo[:mutex]) do
+    #       eo_opts[:rows].each{|album| album.associations[:artist] = nil}
+    #     end
+    #     id_map = eo_opts[:id_map]
+    #     rows = Artist.where(:id=>id_map.keys).all
+    #     Sequel.synchronize_with(eo[:mutex]) do
+    #       rows.each do |artist|
+    #         if albums = id_map[artist.id]
+    #           albums.each do |album|
+    #             album.associations[:artist] = artist
+    #           end
+    #         end
+    #       end
+    #     end
+    #   end)
+    #
+    # Usage:
+    #
+    #   # Make all model subclass datasets support concurrent eager loading
+    #   Sequel::Model.plugin :concurrent_eager_loading
+    #
+    #   # Make the Album class datasets support concurrent eager loading
+    #   Album.plugin :concurrent_eager_loading
+    #
+    #   # Make all model subclass datasets concurrently eager load by default
+    #   Sequel::Model.plugin :concurrent_eager_loading, always: true
+    module ConcurrentEagerLoading
+      def self.configure(mod, opts=OPTS)
+        if opts.has_key?(:always)
+          mod.instance_variable_set(:@always_eager_load_concurrently, opts[:always])
+        end
+      end
+
+      module ClassMethods
+        Plugins.inherited_instance_variables(self, :@always_eager_load_concurrently => nil)
+        Plugins.def_dataset_methods(self, [:eager_load_concurrently, :eager_load_serially])
+
+        # Whether datasets for this class should eager load concurrently by default.
+        def always_eager_load_concurrently?
+          @always_eager_load_concurrently
+        end
+      end
+
+      module DatasetMethods
+        # Return a cloned dataset that will eager load associated results concurrently
+        # using the async thread pool.
+        def eager_load_concurrently
+          cached_dataset(:_eager_load_concurrently) do
+            clone(:eager_load_concurrently=>true)
+          end
+        end
+
+        # Return a cloned dataset that will noteager load associated results concurrently
+        # using the async thread pool. Only useful if the current dataset has been marked
+        # as loading concurrently, or loading concurrently is the model's default behavior.
+        def eager_load_serially
+          cached_dataset(:_eager_load_serially) do
+            clone(:eager_load_concurrently=>false)
+          end
+        end
+
+        private
+
+        # Whether this particular dataset will eager load results concurrently.
+        def eager_load_concurrently?
+          v = @opts[:eager_load_concurrently]
+          v.nil? ? model.always_eager_load_concurrently? : v
+        end
+
+        # If performing eager loads concurrently, and at least 2 associations are being
+        # eagerly loaded, create a single mutex used for all eager loads.  After the
+        # eager loads have been performed, force loading of any async results, so that
+        # all eager loads will have been completed before this method returns.
+        def perform_eager_loads(eager_load_data)
+          return super if !eager_load_concurrently? || eager_load_data.length < 2
+
+          mutex = Mutex.new
+          eager_load_data.each_value do |eo|
+            eo[:mutex] = mutex
+          end
+
+          super.each do |v|
+            if Sequel::Database::AsyncThreadPool::BaseProxy === v
+              v.__value
+            end
+          end
+        end
+
+        # If performing eager loads concurrently, perform this eager load using the
+        # async thread pool.
+        def perform_eager_load(loader, eo)
+          eo[:mutex] ? db.send(:async_run){super} : super
+        end
+      end
+    end
+  end
+end

data/lib/sequel/plugins/constraint_validations.rb

@@ -220,7 +220,8 @@ module Sequel
               '%'
             when '%'
               '.*'
-            when '_'
+            else
+            #when '_'
               '.'
             end
           end

data/lib/sequel/plugins/csv_serializer.rb

@@ -65,8 +65,6 @@ module Sequel
     #   # Add CSV output capability to Album class instances
     #   Album.plugin :csv_serializer
     module CsvSerializer
-      CSV = Object.const_defined?(:CSV) ? ::CSV : ::FasterCSV
-
       # Set up the column readers to do deserialization and the column writers
       # to save the value in deserialized_values
       def self.configure(model, opts = OPTS)
@@ -75,13 +73,31 @@ module Sequel
         end
       end
 
+      # Avoid keyword argument separation warnings on Ruby 2.7, while still
+      # being compatible with 1.9.
+      if RUBY_VERSION >= "2.0"
+        instance_eval(<<-END, __FILE__, __LINE__+1)
+          def self.csv_call(*args, opts, &block)
+            CSV.send(*args, **opts, &block)
+          end
+        END
+      else
+        # :nocov:
+        # :nodoc:
+        def self.csv_call(*args, opts, &block)
+          CSV.send(*args, opts, &block)
+        end
+        # :nodoc:
+        # :nocov:
+      end
+
       module ClassMethods
         # The default opts to use when serializing model objects to CSV
         attr_reader :csv_serializer_opts
 
         # Attempt to parse an array of instances from the given CSV string
         def array_from_csv(csv, opts = OPTS)
-          CSV.parse(csv, process_csv_serializer_opts(opts)).map do |row|
+          CsvSerializer.csv_call(:parse, csv, process_csv_serializer_opts(opts)).map do |row|
             row = row.to_hash
             row.delete(nil)
             new(row)
@@ -108,7 +124,8 @@ module Sequel
           opts_cols = opts.delete(:columns)
           opts_include = opts.delete(:include)
           opts_except = opts.delete(:except)
-          opts[:headers] ||= Array(opts.delete(:only) || opts_cols || columns) + Array(opts_include) - Array(opts_except)
+          only = opts.delete(:only) 
+          opts[:headers] ||= Array(only || opts_cols || columns) + Array(opts_include) - Array(opts_except)
           opts
         end
 
@@ -130,7 +147,7 @@ module Sequel
         # :headers :: The headers to use for the CSV line. Use nil for a header
         #             to specify the column should be ignored.
         def from_csv(csv, opts = OPTS)
-          row = CSV.parse_line(csv, model.process_csv_serializer_opts(opts)).to_hash
+          row = CsvSerializer.csv_call(:parse_line, csv, model.process_csv_serializer_opts(opts)).to_hash
           row.delete(nil)
           set(row)
         end
@@ -146,9 +163,10 @@ module Sequel
         #             attributes to include in the CSV output.
         def to_csv(opts = OPTS)
           opts = model.process_csv_serializer_opts(opts)
+          headers = opts[:headers]
 
-          CSV.generate(opts) do |csv|
-            csv << opts[:headers].map{|k| public_send(k)}
+          CsvSerializer.csv_call(:generate, model.process_csv_serializer_opts(opts)) do |csv|
+            csv << headers.map{|k| public_send(k)}
           end
         end
       end
@@ -164,10 +182,11 @@ module Sequel
         def to_csv(opts = OPTS)
           opts = model.process_csv_serializer_opts({:columns=>columns}.merge!(opts))
           items = opts.delete(:array) || self
+          headers = opts[:headers]
 
-          CSV.generate(opts) do |csv|
+          CsvSerializer.csv_call(:generate, opts) do |csv|
             items.each do |object|
-              csv << opts[:headers].map{|header| object.public_send(header) }
+              csv << headers.map{|header| object.public_send(header)}
             end
           end
         end

data/lib/sequel/plugins/dataset_associations.rb

@@ -62,7 +62,10 @@ module Sequel
           ret = super
           r = association_reflection(name)
           meth = r.returns_array? ? name : pluralize(name).to_sym
-          dataset_module{define_method(meth){associated(name)}}
+          dataset_module do
+            define_method(meth){associated(name)}
+            alias_method(meth, meth)
+          end
           ret
         end
 

data/lib/sequel/plugins/dirty.rb

@@ -41,6 +41,15 @@ module Sequel
     #   artist.column_changes        # => {}
     #   artist.previous_changes      # => {:name=>['Foo', 'Bar']}
     #
+    #   artist.column_previously_was(:name)
+    #   # => 'Foo'
+    #   artist.column_previously_changed?(:name)
+    #   # => true
+    #   artist.column_previously_changed?(:name, from: 'Foo', to: 'Bar')
+    #   # => true
+    #   artist.column_previously_changed?(:name, from: 'Foo', to: 'Baz')
+    #   # => false
+    #
     # There is one caveat; when used with a column that also uses the
     # serialization plugin, setting the column back to its original value
     # after changing it is not correctly detected and will leave an entry
@@ -61,6 +70,19 @@ module Sequel
         # that were used in the update statement.
         attr_reader :previous_changes
 
+        # Reset the initial values after saving.
+        def after_save
+          super
+          reset_initial_values
+        end
+
+        # Save the current changes so they are available after updating.  This happens
+        # before after_save resets them.
+        def after_update
+          super
+          @previous_changes = column_changes
+        end
+
         # An array with the initial value and the current value
         # of the column, if the column has been changed.  If the
         # column has not been changed, returns nil.
@@ -92,6 +114,41 @@ module Sequel
           initial_values.has_key?(column)
         end
 
+        # Whether the column was previously changed.
+        # Options:
+        # :from :: If given, the previous initial value of the column must match this
+        # :to :: If given, the previous changed value of the column must match this
+        #
+        #   update(name: 'Current')
+        #   previous_changes                  # => {:name=>['Initial', 'Current']}
+        #   column_previously_changed?(:name) # => true
+        #   column_previously_changed?(:id)   # => false
+        #   column_previously_changed?(:name, from: 'Initial', to: 'Current') # => true
+        #   column_previously_changed?(:name, from: 'Foo', to: 'Current')     # => false
+        def column_previously_changed?(column, opts=OPTS)
+          return false unless (pc = @previous_changes) && (val = pc[column])
+
+          if opts.has_key?(:from)
+            return false unless val[0] == opts[:from]
+          end
+
+          if opts.has_key?(:to)
+            return false unless val[1] == opts[:to]
+          end
+
+          true
+        end
+
+        # The previous value of the column, which is the initial value of
+        # the column before the object was previously saved.
+        #
+        #   initial_value(:name) # => 'Initial'
+        #   update(name: 'Current')
+        #   column_previously_was(:name) # => 'Initial'
+        def column_previously_was(column)
+          (pc = @previous_changes) && (val = pc[column]) && val[0]
+        end
+
         # Freeze internal data structures
         def freeze
           initial_values.freeze
@@ -159,23 +216,10 @@ module Sequel
 
         private
 
-        # Reset the initial values when setting values.
-        def _refresh_set_values(hash)
-          reset_initial_values
-          super
-        end
-
-        # Reset the initial values after saving.
-        def after_save
-          super
-          reset_initial_values
-        end
-
-        # Save the current changes so they are available after updating.  This happens
-        # before after_save resets them.
-        def after_update
+        # Reset initial values when clearing changed columns
+        def _clear_changed_columns(reason)
+          reset_initial_values if reason == :initialize || reason == :refresh
           super
-          @previous_changes = column_changes
         end
 
         # When changing the column value, save the initial column value.  If the column
@@ -214,12 +258,6 @@ module Sequel
           self
         end
 
-        # Reset the initial values when initializing.
-        def initialize_set(h)
-          super
-          reset_initial_values
-        end
-
         # Array holding column symbols that were not present initially.  This is necessary
         # to differentiate between values that were not present and values that were
         # present but equal to nil.

data/lib/sequel/plugins/empty_failure_backtraces.rb

@@ -0,0 +1,38 @@
+# frozen-string-literal: true
+
+module Sequel
+  module Plugins
+    # The empty_failure_backtraces plugin uses empty backtraces when raising HookFailed and ValidationFailed
+    # exceptions.  This can be significantly faster, and if you are using these exceptions for
+    # flow control, you do not need the backtraces.  This plugin is about 10% faster on CRuby
+    # and 10-15x faster on JRuby 9.2.7.0+.  This does not have an effect on JRuby <9.2.7.0.
+    #
+    # Usage:
+    #
+    #   # Make all model subclass instances use empty backtraces for HookFailed
+    #   # and ValidationFailed exceptions (called before loading subclasses)
+    #   Sequel::Model.plugin :empty_failure_backtraces
+    #
+    #   # Make the Album class use empty backtraces for HookFailed and ValidationFailed exceptions
+    #   Album.plugin :empty_failure_backtraces
+    module EmptyFailureBacktraces
+      module InstanceMethods
+        private
+
+        # Use empty backtrace for HookFailed exceptions.
+        def hook_failed_error(msg)
+          e = super
+          e.set_backtrace([])
+          e
+        end
+
+        # Use empty backtrace for ValidationFailed exceptions.
+        def validation_failed_error
+          e = super
+          e.set_backtrace([])
+          e
+        end
+      end
+    end
+  end
+end

data/lib/sequel/plugins/forbid_lazy_load.rb

@@ -0,0 +1,216 @@
+# frozen-string-literal: true
+
+module Sequel
+  module Plugins
+    # The forbid_lazy_load plugin forbids lazy loading of associations
+    # for objects in cases where the object wasn't loaded with a
+    # method that only returns a single object.
+    #
+    # The main reason for doing this is it makes it easier to detect
+    # N+1 query issues. Note that Sequel also offers a
+    # tactical_eager_loading plugin which will automatically eagerly
+    # load associations for all objects retrived in the same query
+    # if any object would attempt to lazily load an association. That
+    # approach may be simpler if you are trying to prevent N+1 issues,
+    # though it does retain more objects in memory.
+    #
+    # This plugin offers multiple different ways to forbid lazy
+    # loading.  You can forbid lazy loading associations for individual
+    # model instances:
+    #
+    #   obj = Album[1]
+    #   obj.forbid_lazy_load
+    #   obj.artist # raises Sequel::Plugins::ForbidLazyLoad::Error
+    #
+    # +forbid_lazy_load+ is automatically called on instances if the
+    # instances are loaded via a method such as Dataset#all,
+    # Dataset#each, and other methods that load multiple instances
+    # at once.  These are the cases where lazily loading associations
+    # for such instances can cause N+1 issues.
+    #
+    #   Album.all.first.artist
+    #   objs.first.artist # raises Sequel::Plugins::ForbidLazyLoad::Error
+    #
+    #   Album.each do |obj|
+    #     obj.artist # raises Sequel::Plugins::ForbidLazyLoad::Error
+    #   end
+    #
+    #   Album[1].artist # no error
+    #
+    #   Album.first.artist # no error
+    #
+    # You can allow lazy loading associations for an instance that it
+    # was previously forbidden for:
+    # 
+    #   obj = Album.all.first
+    #   obj.allow_lazy_load
+    #   obj.artist # no error
+    #
+    # You can forbid lazy loading associations on a per-call basis,
+    # even if lazy loading of associations is allowed for the instance:
+    #
+    #   obj = Album[1]
+    #   obj.artist(forbid_lazy_load: true)
+    #   # raises Sequel::Plugins::ForbidLazyLoad::Error
+    #
+    # This also works for allowing lazy loading associations for a
+    # specific association load even if it is forbidden for the instance:
+    #  
+    #   obj = Album.all.first
+    #   obj.artist(forbid_lazy_load: false)
+    #   # nothing raised
+    #
+    # You can also forbid lazy loading on a per-association basis using the
+    # +:forbid_lazy_load+ association option with a +true+ value:
+    #
+    #   Album.many_to_one :artist, forbid_lazy_load: true
+    #   Album[1].artist # raises Sequel::Plugins::ForbidLazyLoad::Error
+    #
+    # However, you probably don't want to do this as it will forbid any
+    # lazy loading of the association, even if the loading could not
+    # result in an N+1 issue.
+    #
+    # On the flip side, you can allow lazy loading using the 
+    # +:forbid_lazy_load+ association option with a +false+ value:
+    #
+    #   Album.many_to_one :artist, forbid_lazy_load: false
+    #   Album.all.first.artist # no error
+    #
+    # One reason to do this is when using a plugin like static_cache
+    # on the associated model, where a query is not actually issued
+    # when doing a lazy association load.  To make that particular
+    # case easier, this plugin makes Model.finalize_associations
+    # automatically set the association option if the associated
+    # class uses the static_cache plugin.
+    #
+    # Note that even with this plugin, there can still be N+1 issues,
+    # such as:
+    #
+    #   Album.each do |obj| # 1 query for all albums
+    #     Artist[obj.artist_id] # 1 query per album for each artist
+    #   end
+    # 
+    # Usage:
+    #
+    #   # Make all model subclasses support forbidding lazy load
+    #   # (called before loading subclasses)
+    #   Sequel::Model.plugin :forbid_lazy_load
+    #
+    #   # Make the Album class support forbidding lazy load
+    #   Album.plugin :forbid_lazy_load
+    module ForbidLazyLoad
+      # Error raised when attempting to lazy load an association when
+      # lazy loading has been forbidden.
+      class Error < StandardError
+      end
+
+      module ClassMethods
+        Plugins.def_dataset_methods(self, :forbid_lazy_load)
+
+        # If the static_cache plugin is used by the associated class for
+        # an association, allow lazy loading that association, since the
+        # lazy association load will use a hash table lookup and not a query.
+        def allow_lazy_load_for_static_cache_associations
+          # :nocov:
+          if defined?(::Sequel::Plugins::StaticCache::ClassMethods)
+          # :nocov:
+            @association_reflections.each_value do |ref|
+              if ref.associated_class.is_a?(::Sequel::Plugins::StaticCache::ClassMethods)
+                ref[:forbid_lazy_load] = false
+              end
+            end
+          end
+        end
+
+        # Allow lazy loading for static cache associations before finalizing.
+        def finalize_associations
+          allow_lazy_load_for_static_cache_associations
+          super
+        end
+      end
+
+      module InstanceMethods
+        # Set this model instance to allow lazy loading of associations.
+        def allow_lazy_load
+          @forbid_lazy_load = false
+          self
+        end
+
+        # Set this model instance to not allow lazy loading of associations.
+        def forbid_lazy_load
+          @forbid_lazy_load = true
+          self
+        end
+
+        private
+
+        # Allow lazy loading for objects returned by singular associations.
+        def _load_associated_object(opts, dynamic_opts)
+          # The implementation that loads these associations does
+          # .all.first, which would result in the object returned being
+          # marked as forbidding lazy load.
+          obj = super
+          obj.allow_lazy_load if obj.is_a?(InstanceMethods)
+          obj
+        end
+
+        # Raise an Error if lazy loading has been forbidden for
+        # the instance, association, or call.
+        def _load_associated_objects(opts, dynamic_opts=OPTS)
+          case dynamic_opts[:forbid_lazy_load]
+          when false
+            # nothing
+          when nil
+            unless dynamic_opts[:reload]
+              case opts[:forbid_lazy_load]
+              when nil
+                raise Error, "lazy loading forbidden for this object (association: #{opts.inspect}, object: #{inspect})" if @forbid_lazy_load
+              when false
+                # nothing
+              else
+                raise Error, "lazy loading forbidden for this association (#{opts.inspect})"
+              end
+            end
+          else
+            raise Error, "lazy loading forbidden for this association method call (association: #{opts.inspect})"
+          end
+
+          super
+        end
+      end
+
+      module DatasetMethods
+        # Mark model instances retrieved in this call as forbidding lazy loading.
+        def each
+          if row_proc
+            super do |obj|
+              obj.forbid_lazy_load if obj.is_a?(InstanceMethods)
+              yield obj
+            end
+          else
+            super
+          end
+        end
+
+        # Mark model instances retrieved in this call as forbidding lazy loading.
+        def with_sql_each(sql)
+          if row_proc
+            super(sql) do |obj|
+              obj.forbid_lazy_load if obj.is_a?(InstanceMethods)
+              yield obj
+            end
+          else
+            super
+          end
+        end
+
+        # Mark model instances retrieved in this call as allowing lazy loading.
+        def with_sql_first(sql)
+          obj = super
+          obj.allow_lazy_load if obj.is_a?(InstanceMethods)
+          obj
+        end
+      end
+    end
+  end
+end