sequel 5.41.0 → 5.46.0

data/lib/sequel/plugins/composition.rb

@@ -171,8 +171,9 @@ module Sequel
 
         # Freeze compositions hash when freezing model instance.
         def freeze
-          compositions.freeze
+          compositions
           super
+          compositions.freeze
         end
 
         # For each composition, set the columns in the model class based

data/lib/sequel/plugins/concurrent_eager_loading.rb

@@ -0,0 +1,174 @@
+# frozen-string-literal: true
+
+module Sequel
+  extension 'async_thread_pool'
+
+  module Plugins
+    # The concurrent_eager_loading plugin allows for eager loading multiple associations
+    # concurrently in separate threads.  You must load the async_thread_pool Database
+    # extension into the Database object the model class uses in order for this plugin
+    # to work.
+    # 
+    # By default in Sequel, eager loading happens in a serial manner.  If you have code
+    # such as:
+    #
+    #   Album.eager(:artist, :genre, :tracks)
+    #
+    # Sequel will load the albums, then the artists for the albums, then
+    # the genres for the albums, then the tracks for the albums.
+    #
+    # With the concurrent_eager_loading plugin, you can use the +eager_load_concurrently+
+    # method to allow for concurrent eager loading:
+    #
+    #   Album.eager_load_concurrently.eager(:artist, :genre, :tracks)
+    #
+    # This will load the albums, first, since it needs to load the albums to know
+    # which artists, genres, and tracks to eagerly load. However, it will load the
+    # artists, genres, and tracks for the albums concurrently in separate threads.
+    # This can significantly improve performance, especially if there is significant
+    # latency between the application and the database. Note that using separate threads
+    # is only used in the case where there are multiple associations to eagerly load.
+    # With only a single association to eagerly load, there is no reason to use a
+    # separate thread, since it would not improve performance.
+    #
+    # If you want to make concurrent eager loading the default, you can load the
+    # plugin with the +:always+ option. In this case, all eager loads will be
+    # concurrent.  If you want to force a non-concurrent eager load, you can use
+    # +eager_load_serially+:
+    #
+    #   Album.eager_load_serially.eager(:artist, :genre, :tracks)
+    #
+    # Note that making concurrent eager loading the default is probably a bad idea
+    # if you are eager loading inside transactions and want the eager load to
+    # reflect changes made inside the transaction, unless you plan to use
+    # +eager_load_serially+ for such cases.  See the async_thread_pool
+    # Database extension documentation for more general caveats regarding its use.
+    #
+    # The default eager loaders for all of the association types that ship with Sequel
+    # support safe concurrent eager loading.  However, if you are specifying a custom
+    # +:eager_loader+ for an association, it may not work safely unless it it modified to
+    # support concurrent eager loading.  Taking this example from the
+    # {Advanced Associations guide}[rdoc-ref:doc/advanced_associations.rdoc]
+    #
+    #   Album.many_to_one :artist, :eager_loader=>(proc do |eo_opts|
+    #     eo_opts[:rows].each{|album| album.associations[:artist] = nil}
+    #     id_map = eo_opts[:id_map]
+    #     Artist.where(:id=>id_map.keys).all do |artist|
+    #       if albums = id_map[artist.id]
+    #         albums.each do |album|
+    #           album.associations[:artist] = artist
+    #         end
+    #       end
+    #     end
+    #   end)
+    #
+    # This would not support concurrent eager loading safely.  To support safe
+    # concurrent eager loading, you need to make sure you are not modifying
+    # the associations for objects concurrently by separate threads.  This is
+    # implemented using a mutex, which you can access via <tt>eo_opts[:mutex]</tt>.
+    # To keep things simple, you can use +Sequel.synchronize_with+ to only
+    # use this mutex if it is available.  You want to use the mutex around the
+    # code that initializes the associations (usually to +nil+ or <tt>[]</tt>),
+    # and also around the code that sets the associatied objects appropriately
+    # after they have been retreived.  You do not want to use the mutex around
+    # the code that loads the objects, since that will prevent concurrent loading.
+    # So after the changes, the custom eager loader would look like this:
+    #
+    #   Album.many_to_one :artist, :eager_loader=>(proc do |eo_opts|
+    #     Sequel.synchronize_with(eo[:mutex]) do
+    #       eo_opts[:rows].each{|album| album.associations[:artist] = nil}
+    #     end
+    #     id_map = eo_opts[:id_map]
+    #     rows = Artist.where(:id=>id_map.keys).all
+    #     Sequel.synchronize_with(eo[:mutex]) do
+    #       rows.each do |artist|
+    #         if albums = id_map[artist.id]
+    #           albums.each do |album|
+    #             album.associations[:artist] = artist
+    #           end
+    #         end
+    #       end
+    #     end
+    #   end)
+    #
+    # Usage:
+    #
+    #   # Make all model subclass datasets support concurrent eager loading
+    #   Sequel::Model.plugin :concurrent_eager_loading
+    #
+    #   # Make the Album class datasets support concurrent eager loading
+    #   Album.plugin :concurrent_eager_loading
+    #
+    #   # Make all model subclass datasets concurrently eager load by default
+    #   Sequel::Model.plugin :concurrent_eager_loading, always: true
+    module ConcurrentEagerLoading
+      def self.configure(mod, opts=OPTS)
+        if opts.has_key?(:always)
+          mod.instance_variable_set(:@always_eager_load_concurrently, opts[:always])
+        end
+      end
+
+      module ClassMethods
+        Plugins.inherited_instance_variables(self, :@always_eager_load_concurrently => nil)
+        Plugins.def_dataset_methods(self, [:eager_load_concurrently, :eager_load_serially])
+
+        # Whether datasets for this class should eager load concurrently by default.
+        def always_eager_load_concurrently?
+          @always_eager_load_concurrently
+        end
+      end
+
+      module DatasetMethods
+        # Return a cloned dataset that will eager load associated results concurrently
+        # using the async thread pool.
+        def eager_load_concurrently
+          cached_dataset(:_eager_load_concurrently) do
+            clone(:eager_load_concurrently=>true)
+          end
+        end
+
+        # Return a cloned dataset that will noteager load associated results concurrently
+        # using the async thread pool. Only useful if the current dataset has been marked
+        # as loading concurrently, or loading concurrently is the model's default behavior.
+        def eager_load_serially
+          cached_dataset(:_eager_load_serially) do
+            clone(:eager_load_concurrently=>false)
+          end
+        end
+
+        private
+
+        # Whether this particular dataset will eager load results concurrently.
+        def eager_load_concurrently?
+          v = @opts[:eager_load_concurrently]
+          v.nil? ? model.always_eager_load_concurrently? : v
+        end
+
+        # If performing eager loads concurrently, and at least 2 associations are being
+        # eagerly loaded, create a single mutex used for all eager loads.  After the
+        # eager loads have been performed, force loading of any async results, so that
+        # all eager loads will have been completed before this method returns.
+        def perform_eager_loads(eager_load_data)
+          return super if !eager_load_concurrently? || eager_load_data.length < 2
+
+          mutex = Mutex.new
+          eager_load_data.each_value do |eo|
+            eo[:mutex] = mutex
+          end
+
+          super.each do |v|
+            if Sequel::Database::AsyncThreadPool::BaseProxy === v
+              v.__value
+            end
+          end
+        end
+
+        # If performing eager loads concurrently, perform this eager load using the
+        # async thread pool.
+        def perform_eager_load(loader, eo)
+          eo[:mutex] ? db.send(:async_run){super} : super
+        end
+      end
+    end
+  end
+end

data/lib/sequel/plugins/json_serializer.rb

@@ -133,21 +133,39 @@ module Sequel
         end
       end
       
-      # Helper class used for making sure that cascading options
-      # for model associations works correctly.  Cascaded options
-      # work by creating instances of this class, which take a
-      # literal JSON string and have +to_json+ return it.
+      # SEQUEL6: Remove
+      # :nocov:
       class Literal
-        # Store the literal JSON to use
         def initialize(json)
           @json = json
         end
         
-        # Return the literal JSON to use
         def to_json(*a)
           @json
         end
       end
+      # :nocov:
+      Sequel::Deprecation.deprecate_constant(self, :Literal)
+
+      # Convert the given object to a JSON data structure using the given arguments.
+      def self.object_to_json_data(obj, *args, &block)
+        if obj.is_a?(Array)
+          obj.map{|x| object_to_json_data(x, *args, &block)}
+        else
+          if obj.respond_to?(:to_json_data)
+            obj.to_json_data(*args, &block)
+          else
+            begin
+              Sequel.parse_json(Sequel.object_to_json(obj, *args, &block))
+            # :nocov:
+            rescue Sequel.json_parser_error_class
+              # Support for old Ruby code that only supports parsing JSON object/array
+              Sequel.parse_json(Sequel.object_to_json([obj], *args, &block))[0]
+            # :nocov:
+            end
+          end
+        end
+      end
 
       module ClassMethods
         # The default opts to use when serializing model objects to JSON.
@@ -324,20 +342,7 @@ module Sequel
                 end
 
                 v = v.empty? ? [] : [v]
-
-                objs = public_send(k)
-
-                is_array = if r = model.association_reflection(k)
-                  r.returns_array?
-                else
-                  objs.is_a?(Array)
-                end
-                
-                h[key_name] = if is_array
-                  objs.map{|obj| Literal.new(Sequel.object_to_json(obj, *v))}
-                else
-                  Literal.new(Sequel.object_to_json(objs, *v))
-                end
+                h[key_name] = JsonSerializer.object_to_json_data(public_send(k), *v)
               end
             else
               Array(inc).each do |c|
@@ -347,7 +352,8 @@ module Sequel
                 else
                   key_name = c.to_s
                 end
-                h[key_name] = public_send(c)
+
+                h[key_name] = JsonSerializer.object_to_json_data(public_send(c))
               end
             end
           end
@@ -362,6 +368,15 @@ module Sequel
           h = yield h if block_given?
           Sequel.object_to_json(h, *a)
         end
+
+        # Convert the receiver to a JSON data structure using the given arguments.
+        def to_json_data(*args, &block)
+          if block
+            to_json(*args){|x| return block.call(x)}
+          else
+            to_json(*args){|x| return x}
+          end
+        end
       end
 
       module DatasetMethods
@@ -420,7 +435,7 @@ module Sequel
             else
               all
             end
-            array.map{|obj| Literal.new(Sequel.object_to_json(obj, opts, &opts[:instance_block]))}
+            JsonSerializer.object_to_json_data(array, opts, &opts[:instance_block])
           else
             all
           end

data/lib/sequel/plugins/nested_attributes.rb

@@ -108,9 +108,10 @@ module Sequel
         #            array of the allowable fields.
         # :limit :: For *_to_many associations, a limit on the number of records
         #           that will be processed, to prevent denial of service attacks.
-        # :reject_if :: A proc that is given each attribute hash before it is
+        # :reject_if :: A proc that is called with each attribute hash before it is
         #               passed to its associated object. If the proc returns a truthy
         #               value, the attribute hash is ignored.
+        # :reject_nil :: Ignore nil objects passed to nested attributes setter methods.
         # :remove :: Allow disassociation of nested records (can remove the associated
         #            object from the parent object, but not destroy the associated object).
         # :require_modification :: Whether to require modification of nested objects when
@@ -146,8 +147,9 @@ module Sequel
         def def_nested_attribute_method(reflection)
           @nested_attributes_module.class_eval do
             meth = :"#{reflection[:name]}_attributes="
+            assoc = reflection[:name]
             define_method(meth) do |v|
-              set_nested_attributes(reflection[:name], v)
+              set_nested_attributes(assoc, v)
             end
             alias_method meth, meth
           end
@@ -161,6 +163,7 @@ module Sequel
         def set_nested_attributes(assoc, obj, opts=OPTS)
           raise(Error, "no association named #{assoc} for #{model.inspect}") unless ref = model.association_reflection(assoc)
           raise(Error, "nested attributes are not enabled for association #{assoc} for #{model.inspect}") unless meta = ref[:nested_attributes]
+          return if obj.nil? && meta[:reject_nil]
           meta = meta.merge(opts)
           meta[:reflection] = ref
           if ref.returns_array?

data/lib/sequel/plugins/pg_array_associations.rb

@@ -384,26 +384,32 @@ module Sequel
           save_opts = {:validate=>opts[:validate]}
           save_opts[:raise_on_failure] = opts[:raise_on_save_failure] != false
 
-          opts[:adder] ||= proc do |o|
-            if array = o.get_column_value(key)
-              array << get_column_value(pk)
-            else
-              o.set_column_value("#{key}=", Sequel.pg_array([get_column_value(pk)], opts.array_type))
+          unless opts.has_key?(:adder)
+            opts[:adder] = proc do |o|
+              if array = o.get_column_value(key)
+                array << get_column_value(pk)
+              else
+                o.set_column_value("#{key}=", Sequel.pg_array([get_column_value(pk)], opts.array_type))
+              end
+              o.save(save_opts)
             end
-            o.save(save_opts)
           end
-  
-          opts[:remover] ||= proc do |o|
-            if (array = o.get_column_value(key)) && !array.empty?
-              array.delete(get_column_value(pk))
-              o.save(save_opts)
+    
+          unless opts.has_key?(:remover)
+            opts[:remover] = proc do |o|
+              if (array = o.get_column_value(key)) && !array.empty?
+                array.delete(get_column_value(pk))
+                o.save(save_opts)
+              end
             end
           end
 
-          opts[:clearer] ||= proc do
-            pk_value = get_column_value(pk)
-            db_type = opts.array_type
-            opts.associated_dataset.where(Sequel.pg_array_op(key).contains(Sequel.pg_array([pk_value], db_type))).update(key=>Sequel.function(:array_remove, key, Sequel.cast(pk_value, db_type)))
+          unless opts.has_key?(:clearer)
+            opts[:clearer] = proc do
+              pk_value = get_column_value(pk)
+              db_type = opts.array_type
+              opts.associated_dataset.where(Sequel.pg_array_op(key).contains(Sequel.pg_array([pk_value], db_type))).update(key=>Sequel.function(:array_remove, key, Sequel.cast(pk_value, db_type)))
+            end
           end
         end
 
@@ -426,10 +432,12 @@ module Sequel
             id_map = {}
             pkm = opts.primary_key_method
 
-            rows.each do |object|
-              if associated_pks = object.get_column_value(key)
-                associated_pks.each do |apk|
-                  (id_map[apk] ||= []) << object
+            Sequel.synchronize_with(eo[:mutex]) do
+              rows.each do |object|
+                if associated_pks = object.get_column_value(key)
+                  associated_pks.each do |apk|
+                    (id_map[apk] ||= []) << object
+                  end
                 end
               end
             end
@@ -484,30 +492,36 @@ module Sequel
             end
           end
 
-          opts[:adder] ||= proc do |o|
-            opk = o.get_column_value(opts.primary_key) 
-            if array = get_column_value(key)
-              modified!(key)
-              array << opk
-            else
-              set_column_value("#{key}=", Sequel.pg_array([opk], opts.array_type))
+          unless opts.has_key?(:adder)
+            opts[:adder] = proc do |o|
+              opk = o.get_column_value(opts.primary_key) 
+              if array = get_column_value(key)
+                modified!(key)
+                array << opk
+              else
+                set_column_value("#{key}=", Sequel.pg_array([opk], opts.array_type))
+              end
+              save_after_modify.call(self) if save_after_modify
             end
-            save_after_modify.call(self) if save_after_modify
           end
-  
-          opts[:remover] ||= proc do |o|
-            if (array = get_column_value(key)) && !array.empty?
-              modified!(key)
-              array.delete(o.get_column_value(opts.primary_key))
-              save_after_modify.call(self) if save_after_modify
+    
+          unless opts.has_key?(:remover)
+            opts[:remover] = proc do |o|
+              if (array = get_column_value(key)) && !array.empty?
+                modified!(key)
+                array.delete(o.get_column_value(opts.primary_key))
+                save_after_modify.call(self) if save_after_modify
+              end
             end
           end
 
-          opts[:clearer] ||= proc do
-            if (array = get_column_value(key)) && !array.empty?
-              modified!(key)
-              array.clear
-              save_after_modify.call(self) if save_after_modify
+          unless opts.has_key?(:clearer)
+            opts[:clearer] = proc do
+              if (array = get_column_value(key)) && !array.empty?
+                modified!(key)
+                array.clear
+                save_after_modify.call(self) if save_after_modify
+              end
             end
           end
         end

data/lib/sequel/plugins/rcte_tree.rb

@@ -170,11 +170,13 @@ module Sequel
           id_map = eo[:id_map]
           parent_map = {}
           children_map = {}
-          eo[:rows].each do |obj|
-            parent_map[prkey_conv[obj]] = obj
-            (children_map[key_conv[obj]] ||= []) << obj
-            obj.associations[ancestors] = []
-            obj.associations[parent] = nil
+          Sequel.synchronize_with(eo[:mutex]) do
+            eo[:rows].each do |obj|
+              parent_map[prkey_conv[obj]] = obj
+              (children_map[key_conv[obj]] ||= []) << obj
+              obj.associations[ancestors] = []
+              obj.associations[parent] = nil
+            end
           end
           r = model.association_reflection(ancestors)
           base_case = model.where(prkey=>id_map.keys).
@@ -207,10 +209,12 @@ module Sequel
               root.associations[ancestors] << obj
             end
           end
-          parent_map.each do |parent_id, obj|
-            if children = children_map[parent_id]
-              children.each do |child|
-                child.associations[parent] = obj
+          Sequel.synchronize_with(eo[:mutex]) do
+            parent_map.each do |parent_id, obj|
+              if children = children_map[parent_id]
+                children.each do |child|
+                  child.associations[parent] = obj
+                end
               end
             end
           end
@@ -268,10 +272,12 @@ module Sequel
           associations = eo[:associations]
           parent_map = {}
           children_map = {}
-          eo[:rows].each do |obj|
-            parent_map[prkey_conv[obj]] = obj
-            obj.associations[descendants] = []
-            obj.associations[childrena] = []
+          Sequel.synchronize_with(eo[:mutex]) do
+            eo[:rows].each do |obj|
+              parent_map[prkey_conv[obj]] = obj
+              obj.associations[descendants] = []
+              obj.associations[childrena] = []
+            end
           end
           r = model.association_reflection(descendants)
           base_case = model.where(key=>id_map.keys).
@@ -316,12 +322,14 @@ module Sequel
             
             (children_map[key_conv[obj]] ||= []) << obj
           end
-          children_map.each do |parent_id, objs|
-            objs = objs.uniq
-            parent_obj = parent_map[parent_id]
-            parent_obj.associations[childrena] = objs
-            objs.each do |obj|
-              obj.associations[parent] = parent_obj
+          Sequel.synchronize_with(eo[:mutex]) do
+            children_map.each do |parent_id, objs|
+              objs = objs.uniq
+              parent_obj = parent_map[parent_id]
+              parent_obj.associations[childrena] = objs
+              objs.each do |obj|
+                obj.associations[parent] = parent_obj
+              end
             end
           end
         end