sequel 5.83.0 → 5.106.0

data/lib/sequel/model/associations.rb

@@ -112,6 +112,8 @@ module Sequel
         # the dataset unmodified if no SQL limit strategy is needed.
         def apply_eager_graph_limit_strategy(strategy, ds)
           case strategy
+          when :lateral_subquery
+            apply_lateral_subquery_eager_graph_limit_strategy(ds)
           when :distinct_on
             apply_distinct_on_eager_limit_strategy(ds.order_prepend(*self[:order]))
           when :window_function
@@ -298,11 +300,16 @@ module Sequel
             # Correlated subqueries are not supported for regular eager loading
             strategy = :ruby if strategy == :correlated_subquery
             strategy = nil if strategy == :ruby && assign_singular?
-            objects = apply_eager_limit_strategy(ds, strategy, eager_limit).all
+
+            objects = if strategy == :lateral_subquery
+              apply_lateral_subquery_eager_limit_strategy(ds, ids, eager_limit).all
+            else
+              apply_eager_limit_strategy(ds, strategy, eager_limit).all
+            end
 
             if strategy == :window_function
               delete_rn = ds.row_number_column 
-              objects.each{|obj| obj.values.delete(delete_rn)}
+              objects.each{|obj| obj.remove_key!(delete_rn)}
             end
           elsif strategy == :union
             objects = []
@@ -320,7 +327,7 @@ module Sequel
                 end
                 loader.append_sql(sql, *k)
               end
-              objects.concat(ds.with_sql(sql).to_a)
+              objects.concat(ds.with_sql(sql.freeze).to_a)
             end
             ds = ds.eager(cascade) if cascade
             ds.send(:post_load, objects)
@@ -366,7 +373,12 @@ module Sequel
         # filtered.  Works by using a subquery to test that the objects passed
         # also meet the association filter criteria.
         def filter_by_associations_conditions_expression(obj)
-          ds = filter_by_associations_conditions_dataset.where(filter_by_associations_conditions_subquery_conditions(obj))
+          ds = if filter_by_associations_limit_strategy == :lateral_subquery
+            apply_lateral_subquery_filter_limit_strategy(associated_eager_dataset, obj)
+          else
+            filter_by_associations_conditions_dataset.where(filter_by_associations_conditions_subquery_conditions(obj))
+          end
+
           {filter_by_associations_conditions_key=>ds}
         end
 
@@ -414,6 +426,12 @@ module Sequel
           false
         end
 
+        # Hash value for the association reflection.  This is precomputed to avoid
+        # concurrency issues at runtime.
+        def hash
+          self[:_hash]
+        end
+
         # Initialize the associations cache for the current association for the given objects.
         def initialize_association_cache(objects)
           name = self[:name]
@@ -693,6 +711,9 @@ module Sequel
 
         # The predicate condition to use for the eager_loader.
         def eager_loading_predicate_condition(keys)
+          if transform = self[:eager_loading_predicate_transform]
+            keys = transform.call(keys, self)
+          end
           {predicate_key=>keys}
         end
 
@@ -745,6 +766,61 @@ module Sequel
           end
         end
 
+        def lateral_subquery_eager_limit_strategy_lateral_dataset(ds, limit_and_offset)
+          ds.
+            where(Array(filter_by_associations_conditions_key).zip(Array(filter_by_associations_conditions_associated_keys))).
+            limit(*limit_and_offset).
+            lateral
+        end
+
+        def apply_lateral_subquery_eager_limit_strategy(ds, ids, limit_and_offset)
+          table_name = self[:model].table_name
+          associated_table_name = associated_class.table_name
+
+          associated_class.
+            from(table_name).
+            select_all(associated_table_name).
+            join(lateral_subquery_eager_limit_strategy_lateral_dataset(ds, limit_and_offset).as(associated_table_name), true).
+            order(*self[:order])
+        end
+
+        # Return an expression to filter the filter by associations dataset to only
+        # rows related to given objects.
+        def _lateral_subquery_filter_limit_strategy_conditions(obj, key, value_method, value_column)
+          value = case obj
+          when Array
+            if key.is_a?(Array)
+              key_methods = Array(value_method)
+              obj.map{|o| key_methods.map{|meth| o.send(meth)}}
+            else
+              obj.map{|o| o.send(value_method)}
+            end
+          when Sequel::Dataset
+            obj.select(*Array(qualify(associated_class.table_name, value_column)))
+          else
+            if key.is_a?(Array)
+              return Array(key).zip(Array(value_method).map{|meth| obj.send(meth)})
+            else
+              obj.send(value_method)
+            end
+          end
+
+          {key => value}
+        end
+
+        def lateral_subquery_filter_limit_strategy_lateral_dataset(ds, obj)
+          lateral_subquery_filter_limit_strategy_filter_lateral_dataset(ds.
+            select(*qualify(associated_class.table_name, associated_class.primary_key)).
+            limit(*limit_and_offset).
+            lateral)
+        end
+
+        def apply_lateral_subquery_filter_limit_strategy(ds, obj)
+          lateral_subquery_filter_limit_strategy_filter_dataset(self[:model].
+            select(*lateral_subquery_filter_limit_strategy_lateral_dataset_select).
+            join(lateral_subquery_filter_limit_strategy_lateral_dataset(ds, obj).as(associated_class.table_name), filter_by_associations_conditions_subquery_conditions(obj)), obj)
+        end
+
         # Whether to limit the associated dataset to a single row.
         def limit_to_single_row?
           !returns_array?
@@ -759,7 +835,20 @@ module Sequel
         def placeholder_eager_loader
           cached_fetch(:placeholder_eager_loader) do
             eager_loading_dataset.placeholder_literalizer_loader do |pl, ds|
-              apply_eager_limit_strategy(ds.where(predicate_key=>pl.arg), eager_limit_strategy)
+              arg = pl.arg
+
+              if transform = self[:eager_loading_predicate_transform]
+                arg = arg.transform do |v|
+                  transform.call(v, self)
+                end
+              end
+
+              strategy = eager_limit_strategy
+              if strategy == :lateral_subquery
+                apply_lateral_subquery_eager_limit_strategy(ds, arg, limit_and_offset)
+              else
+                apply_eager_limit_strategy(ds.where(predicate_key=>arg), strategy)
+              end
             end
           end
         end
@@ -1001,7 +1090,7 @@ module Sequel
       class OneToManyAssociationReflection < AssociationReflection
         ASSOCIATION_TYPES[:one_to_many] = self
         
-        # Support a correlated subquery limit strategy when using eager_graph.
+        # Support a lateral_subquery and correlated_subquery limit strategy when using eager_graph.
         def apply_eager_graph_limit_strategy(strategy, ds)
           case strategy
           when :correlated_subquery
@@ -1101,6 +1190,54 @@ module Sequel
           ds.where(qualify(table_alias, primary_key)=>cs)
         end
 
+        # Use a LATERAL subquery to limit the dataset.  Note that this will not
+        # work correctly if the associated dataset uses qualified identifers in the WHERE clause,
+        # as they would reference the containing query instead of the subquery.
+        #
+        # This does not contain the conditions that are necessary to join to the
+        # query, since the necessary qualifier is not passed as an argument.
+        def apply_lateral_subquery_eager_graph_limit_strategy(ds)
+          table_name = ds.first_source_alias
+          qualifier = ds.opts[:eager_options][:implicit_qualifier]
+          graph_conditions = self[:_graph_conditions]
+
+          unless Sequel.condition_specifier?(graph_conditions)
+            raise Error, "lateral_subquery eager graph limit strategy only supported when graph conditions are a hash or array of pairs"
+          end
+
+          ds.
+            limit(*limit_and_offset).
+            order(*self[:order]).
+            where(graph_conditions.map{|k, v| [qualify(table_name, k), qualify(qualifier, v)]}).
+            lateral
+        end
+
+        # Avoid setting duplicate predicate condition when using the lateral subquery
+        # eager limit strategy.
+        def eager_loading_set_predicate_condition(ds, eo)
+          eager_limit_strategy == :lateral_subquery ? ds : super
+        end
+
+        def apply_lateral_subquery_eager_limit_strategy(ds, ids, limit_and_offset)
+          super.where(qualify(self[:model].table_name, self[:primary_key]) => ids)
+        end
+
+        def lateral_subquery_filter_limit_strategy_conditions(obj)
+          _lateral_subquery_filter_limit_strategy_conditions(obj, filter_by_associations_conditions_key, self[:key_method], self[:key])
+        end
+
+        def lateral_subquery_filter_limit_strategy_filter_lateral_dataset(ds)
+          ds.where(Array(filter_by_associations_conditions_key).zip(Array(filter_by_associations_conditions_associated_keys)))
+        end
+
+        def lateral_subquery_filter_limit_strategy_filter_dataset(ds, obj)
+          ds.where(lateral_subquery_filter_limit_strategy_conditions(obj))
+        end
+
+        def lateral_subquery_filter_limit_strategy_lateral_dataset_select
+          qualified_primary_key
+        end
+
         # Support correlated subquery strategy when filtering by limited associations.
         def apply_filter_by_associations_limit_strategy(ds)
           case filter_by_associations_limit_strategy
@@ -1274,11 +1411,11 @@ module Sequel
           name = self[:name]
 
           self[:model].eager_load_results(self, eo) do |assoc_record|
-            assoc_record.values.delete(delete_rn) if delete_rn
+            assoc_record.remove_key!(delete_rn) if delete_rn
             hash_key = if uses_lcks
-              left_key_alias.map{|k| assoc_record.values.delete(k)}
+              left_key_alias.map{|k| assoc_record.remove_key!(k)}
             else
-              assoc_record.values.delete(left_key_alias)
+              assoc_record.remove_key!(left_key_alias)
             end
 
             objects = h[hash_key]
@@ -1415,6 +1552,52 @@ module Sequel
           end
         end
 
+        def lateral_subquery_filter_limit_strategy_conditions(obj)
+          _lateral_subquery_filter_limit_strategy_conditions(obj, lateral_subquery_filter_limit_strategy_conditions_key, right_primary_key_method, right_primary_key)
+        end
+
+        def lateral_subquery_filter_limit_strategy_conditions_key
+          self[:right_key]
+        end
+
+        def lateral_subquery_filter_limit_strategy_filter_lateral_dataset(ds)
+          ds.where(Array(filter_by_associations_conditions_key).zip(Array(filter_by_associations_conditions_associated_keys)))
+        end
+
+        def lateral_subquery_filter_limit_strategy_filter_dataset(ds, obj)
+          ds.where(filter_by_associations_conditions_key => ds.db.from(self[:join_table]).
+            select(*self[:left_key]).
+            where(lateral_subquery_filter_limit_strategy_conditions(obj)))
+        end
+
+        def lateral_subquery_filter_limit_strategy_lateral_dataset_select
+          qualify(self[:model].table_name, self[:left_primary_key])
+        end
+
+        def lateral_subquery_eager_limit_strategy_lateral_dataset(ds, limit_and_offset)
+          ds = super
+          select = ds.opts[:select].dup
+          left_key_alias = self[:left_key_alias]
+          select.pop while (s = select.last).is_a?(Sequel::SQL::AliasedExpression) && (left_key_alias.is_a?(Array) ? left_key_alias.include?(s.alias) : s.alias == left_key_alias)
+          ds = ds.clone(:select=>select)
+        end
+
+        def apply_lateral_subquery_eager_limit_strategy(ds, ids, limit_and_offset)
+          table_name = self[:model].table_name
+          super.
+            select_all(associated_class.table_name).
+            select_append(*qualify(table_name, self[:left_primary_keys]).zip(Array(self[:left_key_alias])).map{|id, aliaz| Sequel.as(id, aliaz)}).
+            where(qualify(table_name, self[:left_primary_key]) => ids)
+        end
+
+        def apply_lateral_subquery_eager_graph_limit_strategy(ds)
+          ds.
+            limit(*limit_and_offset).
+            where(qualify(join_table_alias, self[:left_keys]).zip(qualify(ds.opts[:eager_options][:implicit_qualifier], self[:left_primary_key_columns]))).
+            order(*self[:order]).
+            lateral
+        end
+
         # Use the right_keys from the eager loading options if
         # using a separate query per table.
         def eager_loading_set_predicate_condition(ds, eo)
@@ -1707,6 +1890,9 @@ module Sequel
         #                  record should be populated.
         # :eager_loader_key :: A symbol for the key column to use to populate the key_hash
         #                      for the eager loader.  Can be set to nil to not populate the key_hash.
+        # :eager_loading_predicate_transform :: A callable object with which to transform the predicate key values used
+        #                                       when eager loading. Called with two arguments, the array of predicate key
+        #                                       values, and a the reflection for the association being eager loaded.
         # :extend :: A module or array of modules to extend the dataset with.
         # :filter_limit_strategy :: Determines the strategy used for enforcing limits and offsets when filtering by
         #                           limited associations.  Possible options are :window_function, :distinct_on, or
@@ -1769,6 +1955,9 @@ module Sequel
         #            Set to nil to not define a setter method for the association.
         # :subqueries_per_union :: The number of subqueries to use in each UNION query, for eager
         #                          loading limited associations using the default :union strategy.
+        # :use_placeholder_loader :: Whether to use a placeholder loader when eager loading the
+        #                            association.  Can be set to false to disable the use of a placeholder
+        #                            loader if one would be used by default.
         # :validate :: Set to false to not validate when implicitly saving any associated object.
         # === :many_to_one
         # :key :: foreign key in current model's table that references
@@ -1891,7 +2080,7 @@ module Sequel
             raise(Error, "cannot clone an association to an association of different type (association #{name} with type #{type} cloning #{opts[:clone]} with type #{cloned_assoc[:type]})")
           end
 
-          opts[:use_placeholder_loader] = !opts[:instance_specific] && !opts[:eager_graph]
+          opts[:use_placeholder_loader] = !opts[:instance_specific] && !opts[:eager_graph] unless opts.include?(:use_placeholder_loader)
           opts[:eager_block] = opts[:block] unless opts.include?(:eager_block)
           opts[:graph_join_type] ||= :left_outer
           opts[:order_eager_graph] = true unless opts.include?(:order_eager_graph)
@@ -1914,6 +2103,8 @@ module Sequel
           # Remove :class entry if it exists and is nil, to work with cached_fetch
           opts.delete(:class) unless opts[:class]
 
+          opts[:_hash] = [self, name].hash
+
           def_association(opts)
       
           orig_opts.delete(:clone)
@@ -1983,6 +2174,7 @@ module Sequel
           associate(:one_to_one, name, opts, &block)
         end
 
+        Plugins.model_instance_variables(self, :@associations, :@set_associated_object_if_same)
         Plugins.inherited_instance_variables(self, :@association_reflections=>:dup, :@autoreloading_associations=>:hash_dup, :@default_association_options=>:dup, :@default_association_type_options=>:hash_dup, :@cache_associations=>nil, :@default_eager_limit_strategy=>nil)
         Plugins.def_dataset_methods(self, [:eager, :eager_graph, :eager_graph_with_options, :association_join, :association_full_join, :association_inner_join, :association_left_join, :association_right_join])
         
@@ -2362,7 +2554,7 @@ module Sequel
             delete_rn = opts.delete_row_number_column
 
             eager_load_results(opts, eo) do |assoc_record|
-              assoc_record.values.delete(delete_rn) if delete_rn
+              assoc_record.remove_key!(delete_rn) if delete_rn
               hash_key = uses_cks ? km.map{|k| assoc_record.get_column_value(k)} : assoc_record.get_column_value(km)
               objects = h[hash_key]
               if assign_singular
@@ -2388,9 +2580,14 @@ module Sequel
           conditions = opts[:graph_conditions]
           opts[:cartesian_product_number] ||= one_to_one ? 0 : 1
           graph_block = opts[:graph_block]
+          graph_conditions = opts[:_graph_conditions] = use_only_conditions ? only_conditions : cks.zip(pkcs) + conditions
           opts[:eager_grapher] ||= proc do |eo|
             ds = eo[:self]
-            ds = ds.graph(opts.apply_eager_graph_limit_strategy(eo[:limit_strategy], eager_graph_dataset(opts, eo)), use_only_conditions ? only_conditions : cks.zip(pkcs) + conditions, eo.merge(:select=>select, :join_type=>eo[:join_type]||join_type, :qualify=>:deep), &graph_block)
+            graph_limit_strategy = eo[:limit_strategy]
+            egds = opts.apply_eager_graph_limit_strategy(graph_limit_strategy, eager_graph_dataset(opts, eo))
+            graph_conditions_true = true if graph_limit_strategy == :lateral_subquery
+
+            ds = ds.graph(egds, graph_conditions_true || graph_conditions, eo.merge(:select=>select, :join_type=>eo[:join_type]||join_type, :qualify=>:deep), &graph_block)
             # We only load reciprocals for one_to_many associations, as other reciprocals don't make sense
             ds.opts[:eager_graph][:reciprocals][eo[:table_alias]] = opts.reciprocal
             ds
@@ -2473,6 +2670,9 @@ module Sequel
         # Return dataset to graph into given the association reflection, applying the :callback option if set.
         def eager_graph_dataset(opts, eager_options)
           ds = opts.associated_class.dataset
+          if eager_options[:limit_strategy] == :lateral_subquery
+            ds = ds.clone(:eager_options=>eager_options)
+          end
           if opts[:graph_use_association_block] && (b = opts[:block])
             ds = b.call(ds)
           end
@@ -3038,7 +3238,7 @@ module Sequel
           if (((op == :'=' || op == :'!=') && r.is_a?(Sequel::Model)) ||
               (multiple = ((op == :IN || op == :'NOT IN') && ((is_ds = r.is_a?(Sequel::Dataset)) || (r.respond_to?(:all?) && r.all?{|x| x.is_a?(Sequel::Model)})))))
             l = args[0]
-            if ar = model.association_reflections[l]
+            if ar = model.association_reflection(l)
               raise Error, "filtering by associations is not allowed for #{ar.inspect}" if ar[:allow_filtering_by] == false
 
               if multiple
@@ -3519,7 +3719,7 @@ module Sequel
           else
             vals = Array(obj).reject{|o| !meths.all?{|m| o.get_column_value(m)}}
             return SQL::Constants::FALSE if vals.empty?
-            if obj.is_a?(Array)
+            if obj.is_a?(Array) || obj.is_a?(Set)
               if keys.length == 1
                 meth = meths.first
                 {keys.first=>vals.map{|o| o.get_column_value(meth)}}
@@ -3591,7 +3791,7 @@ module Sequel
 
         # Prepare a hash loaders and eager options which will be used to implement the eager loading.
         def prepare_eager_load(a, reflections, eager_assoc)
-          eager_load_data = {}
+          eager_load_data = {}.compare_by_identity
 
           # Key is foreign/primary key name symbol.
           # Value is hash with keys being foreign/primary key values (generally integers)