sequel 4.9.0 → 4.10.0

data/lib/sequel/extensions/columns_introspection.rb

@@ -49,7 +49,7 @@ module Sequel
           from.probable_columns
         when Symbol, SQL::Identifier, SQL::QualifiedIdentifier
           schemas = db.instance_variable_get(:@schemas)
-          if schemas && (sch = Sequel.synchronize{schemas[literal(from)]})
+          if schemas && (table = literal(from)) && (sch = Sequel.synchronize{schemas[table]})
             sch.map{|c,_| c}
           end
         end

data/lib/sequel/extensions/null_dataset.rb

@@ -2,10 +2,6 @@
 # returns a cloned dataset that will never issue a query to the
 # database.  It implements the null object pattern for datasets.
 #
-# To load the extension:
-#
-#   Sequel.extension :null_dataset
-#
 # The most common usage is probably in a method that must return
 # a dataset, where the method knows the dataset shouldn't return
 # anything.  With standard Sequel, you'd probably just add a
@@ -26,6 +22,14 @@
 # the same options to get the columns.
 #
 # This extension uses Object#extend at runtime, which can hurt performance.
+#
+# To add the nullify method to a single dataset:
+#
+#   ds = ds.extension(:null_dataset)
+#
+# To add the nullify method to all datasets on a single database:
+#
+#   DB.extension(:null_dataset)
 
 module Sequel
   class Dataset

data/lib/sequel/extensions/pg_array.rb

@@ -196,15 +196,14 @@ module Sequel
         def self.extended(db)
           db.instance_eval do
             @pg_array_schema_types ||= {}
+            procs = conversion_procs
+            procs[1115] = Creator.new("timestamp without time zone", procs[1114])
+            procs[1185] = Creator.new("timestamp with time zone", procs[1184])
             copy_conversion_procs([1009, 1007, 1016, 1231, 1022, 1000, 1001, 1182, 1183, 1270, 1005, 1028, 1021, 1014, 1015])
             [:string_array, :integer_array, :decimal_array, :float_array, :boolean_array, :blob_array, :date_array, :time_array, :datetime_array].each do |v|
               @schema_type_classes[v] = PGArray
             end
           end
-
-          procs = db.conversion_procs
-          procs[1115] = Creator.new("timestamp without time zone", procs[1114])
-          procs[1185] = Creator.new("timestamp with time zone", procs[1184])
         end
 
         # Handle arrays in bound variables
@@ -232,6 +231,7 @@ module Sequel
           end
           PGArray.register(db_type, opts, &block)
           @schema_type_classes[:"#{opts[:type_symbol] || db_type}_array"] = PGArray
+          conversion_procs_updated
         end
 
         # Return PGArray if this type matches any supported array type.

data/lib/sequel/extensions/pg_row.rb

@@ -499,6 +499,7 @@ module Sequel
             private meth
           end
 
+          conversion_procs_updated
           nil
         end
 

data/lib/sequel/model/associations.rb

@@ -54,9 +54,10 @@ module Sequel
         end
 
         # The dataset associated via this association, with the non-instance specific
-        # changes already applied.
+        # changes already applied.  This will be a joined dataset if the association
+        # requires joining tables.
         def associated_dataset
-          cached_fetch(:_dataset){apply_dataset_changes(associated_class.dataset.clone)}
+          cached_fetch(:_dataset){apply_dataset_changes(_associated_dataset)}
         end
 
         # Apply all non-instance specific changes to the given dataset and return it.
@@ -76,6 +77,16 @@ module Sequel
           ds
         end
 
+        # Apply all non-instance specific changes and the eager_block option to the given
+        # dataset and return it.
+        def apply_eager_dataset_changes(ds)
+          ds = apply_dataset_changes(ds)
+          if block = self[:eager_block]
+            ds = block.call(ds)
+          end
+          ds
+        end
+
         # Apply the eager graph limit strategy to the dataset to graph into the current dataset, or return
         # the dataset unmodified if no SQL limit strategy is needed.
         def apply_eager_graph_limit_strategy(strategy, ds)
@@ -91,8 +102,8 @@ module Sequel
 
         # Apply an eager limit strategy to the dataset, or return the dataset
         # unmodified if it doesn't need an eager limit strategy.
-        def apply_eager_limit_strategy(ds)
-          case eager_limit_strategy
+        def apply_eager_limit_strategy(ds, strategy=eager_limit_strategy)
+          case strategy
           when :distinct_on
             apply_distinct_on_eager_limit_strategy(ds)
           when :window_function
@@ -173,19 +184,31 @@ module Sequel
     
         # Return the symbol used for the row number column if the window function
         # eager limit strategy is being used, or nil otherwise.
-        def delete_row_number_column(ds)
+        def delete_row_number_column(ds=associated_dataset)
           if eager_limit_strategy == :window_function
             ds.row_number_column 
           end
         end
 
+        # Return an dataset that will load the appropriate associated objects for
+        # the given object using this association.
+        def association_dataset_for(object)
+          associated_dataset.where(predicate_keys.zip(predicate_key_values(object)))
+        end
+
+        ASSOCIATION_DATASET_PROC = proc{|r| r.association_dataset_for(self)}
+        # Proc used to create the association dataset method.
+        def association_dataset_proc
+          ASSOCIATION_DATASET_PROC
+        end
+
         # The eager_graph limit strategy to use for this dataset
         def eager_graph_limit_strategy(strategy)
           if self[:limit] || !returns_array?
             strategy = strategy[self[:name]] if strategy.is_a?(Hash)
             case strategy
             when true
-              true_eager_limit_strategy
+              true_eager_graph_limit_strategy
             when Symbol
               strategy
             else
@@ -210,6 +233,39 @@ module Sequel
           end
         end
 
+        # Eager load the associated objects using the hash of eager options,
+        # yielding each row to the block.
+        def eager_load_results(eo, &block)
+          rows = eo[:rows]
+          initialize_association_cache(rows) unless eo[:initialize_rows] == false
+          strategy = eager_limit_strategy
+          cascade = eo[:associations]
+
+          if eo[:eager_block] || eo[:loader] == false
+            strategy = true_eager_graph_limit_strategy if strategy == :union
+            objects = apply_eager_limit_strategy(eager_loading_dataset(eo), strategy).all
+            cascade = nil
+          elsif strategy == :union
+            objects = []
+            ds = associated_dataset
+            ds = self[:eager_block].call(ds) if self[:eager_block]
+            loader = union_eager_loader
+            joiner = " UNION ALL "
+            eo[:id_map].keys.each_slice(subqueries_per_union).each do |slice|
+              objects.concat(ds.with_sql(slice.map{|k| loader.sql(*k)}.join(joiner)).to_a)
+            end
+          else
+            objects = placeholder_eager_loader.all(eo[:id_map].keys)
+          end
+
+          if cascade && !(cascade = associated_dataset.send(:eager_options_for_associations, [cascade])).empty?
+            associated_eager_dataset.send(:eager_load, objects, cascade)
+          end
+
+          objects.each(&block)
+          apply_ruby_eager_limit_strategy(rows)
+        end
+
         # The key to use for the key hash when eager loading
         def eager_loader_key
           self[:eager_loader_key]
@@ -279,11 +335,28 @@ module Sequel
           false
         end
 
+        # A placeholder literalizer that can be used to lazily load the association. If
+        # one can't be used, returns nil.
+        def placeholder_loader
+          if use_placeholder_loader?
+            cached_fetch(:placeholder_loader) do
+              Sequel::Dataset::PlaceholderLiteralizer.loader(associated_dataset) do |pl, ds|
+                ds.where(*predicate_keys.map{|k| SQL::BooleanExpression.new(:'=', k, pl.arg)})
+              end
+            end
+          end
+        end
+
         # The keys to use for loading of the regular dataset, as an array.
         def predicate_keys
           cached_fetch(:predicate_keys){Array(predicate_key)}
         end
 
+        # The values that predicate_keys should match for objects to be associated.
+        def predicate_key_values(object)
+          predicate_key_methods.map{|k| object.send(k)}
+        end
+
         # Qualify +col+ with the given table name.  If +col+ is an array of columns,
         # return an array of qualified columns.  Only qualifies Symbols and SQL::Identifier
         # values, other values are not modified.
@@ -425,10 +498,16 @@ module Sequel
           end
         end
 
+        # The base dataset used for the association, before any order/conditions
+        # options have been applied.
+        def _associated_dataset
+          associated_class.dataset.clone
+        end
+
         # Apply a limit strategy to the given dataset so that filter by
         # associations works with a limited dataset.
         def apply_filter_by_associations_limit_strategy(ds)
-          case eager_limit_strategy
+          case filter_by_associations_limit_strategy
           when :distinct_on
             apply_filter_by_associations_distinct_on_limit_strategy(ds)
           when :window_function
@@ -443,30 +522,67 @@ module Sequel
         # each key is included.
         def apply_filter_by_associations_distinct_on_limit_strategy(ds)
           k = filter_by_associations_limit_key 
-          ds.where(k=>apply_distinct_on_eager_limit_strategy(filter_by_associations_limit_subquery.select(*k)))
+          ds.where(k=>apply_distinct_on_eager_limit_strategy(associated_eager_dataset.select(*k)))
         end
 
         # Apply a distinct on eager limit strategy using IN with a subquery
         # that uses a filter on the row_number window function to ensure
         # that only rows inside the limit are returned.
         def apply_filter_by_associations_window_function_limit_strategy(ds)
-          ds.where(filter_by_associations_limit_key=>apply_window_function_eager_limit_strategy(filter_by_associations_limit_subquery.select(*filter_by_associations_limit_alias_key)).select(*filter_by_associations_limit_aliases))
+          ds.where(filter_by_associations_limit_key=>apply_window_function_eager_limit_strategy(associated_eager_dataset.select(*filter_by_associations_limit_alias_key)).select(*filter_by_associations_limit_aliases))
         end
 
         # The associated_dataset with the eager_block callback already applied.
         def associated_eager_dataset
           cached_fetch(:associated_eager_dataset) do
-            ds = associated_dataset
-            ds = self[:eager_block].call(ds) if self[:eager_block]
+            ds = associated_dataset.unlimited
+            if block = self[:eager_block]
+              ds = block.call(ds)
+            end
             ds
           end
         end
 
+        # The dataset to use for eager loading associated objects for multiple current objects,
+        # given the hash passed to the eager loader.
+        def eager_loading_dataset(eo=OPTS)
+          ds = eo[:dataset] || associated_eager_dataset
+          if id_map = eo[:id_map]
+            ds = ds.where(eager_loading_predicate_condition(id_map.keys))
+          end
+          if associations = eo[:associations]
+            ds = ds.eager(associations)
+          end
+          if block = eo[:eager_block]
+            ds = block.call(ds)
+          end
+          if eager_loading_use_associated_key?
+            ds = ds.select_append(*associated_key_array)
+          end
+          if self[:eager_graph]
+            raise(Error, "cannot eagerly load a #{self[:type]} association that uses :eager_graph") if eager_loading_use_associated_key?
+            ds = ds.eager_graph(self[:eager_graph])
+          end
+          ds
+        end
+
         # The default eager limit strategy to use for this association
         def default_eager_limit_strategy
           self[:model].default_eager_limit_strategy || :ruby
         end
 
+        # The predicate condition to use for the eager_loader.
+        def eager_loading_predicate_condition(keys)
+          {predicate_key=>keys}
+        end
+
+        # Add conditions to the dataset to not include NULL values for
+        # the associated keys, and select those keys.
+        def filter_by_associations_add_conditions_dataset_filter(ds)
+          k = filter_by_associations_conditions_associated_keys
+          ds.select(*k).where(Sequel.negate(k.zip([])))
+        end
+
         # The conditions to add to the filter by associations conditions
         # subquery to restrict it to to the object(s) that was used as the
         # filter value.
@@ -487,16 +603,28 @@ module Sequel
         # values.
         def filter_by_associations_conditions_dataset
           cached_fetch(:filter_by_associations_conditions_dataset) do
-            ds = associated_eager_dataset.unordered.unlimited
+            ds = associated_eager_dataset.unordered
             ds = filter_by_associations_add_conditions_dataset_filter(ds)
             ds = apply_filter_by_associations_limit_strategy(ds)
             ds
           end
         end
 
-        # The base subquery to use when filtering by limited associations
-        def filter_by_associations_limit_subquery
-          associated_eager_dataset.unlimited
+        # The strategy to use to filter by a limited association
+        def filter_by_associations_limit_strategy
+          v = fetch(:filter_limit_strategy, self[:eager_limit_strategy])
+          if v || self[:limit] || !returns_array?
+            case v ||= self[:model].default_eager_limit_strategy
+            when :union, :ruby
+              # Can't use a union or ruby-based strategy for filtering by associations, switch to default eager graph limit
+              # strategy.
+              true_eager_graph_limit_strategy
+            when Symbol
+              v
+            when true
+              true_eager_graph_limit_strategy
+            end
+          end
         end
 
         # Whether to limit the associated dataset to a single row.
@@ -509,6 +637,15 @@ module Sequel
           limit_and_offset.last
         end
 
+        # A placeholder literalizer used to speed up eager loading.
+        def placeholder_eager_loader
+          cached_fetch(:placeholder_eager_loader) do
+            Sequel::Dataset::PlaceholderLiteralizer.loader(associated_dataset) do |pl, ds|
+              apply_eager_limit_strategy(eager_loading_dataset.where(predicate_key=>pl.arg), eager_limit_strategy)
+            end
+          end
+        end
+
         # Whether the given association reflection is possible reciprocal
         # association for the current association reflection.
         def reciprocal_association?(assoc_reflect)
@@ -518,21 +655,60 @@ module Sequel
             assoc_reflect[:block].nil?
         end
     
+        # The number of subqueries to use in each union query, used to eagerly load
+        # limited associations.  Defaults to 40, the optimal number depends on the
+        # latency between the database and the application.
+        def subqueries_per_union
+          self[:subqueries_per_union] || 40
+        end
+
         # If +s+ is an array, map +s+ over the block.  Otherwise, just call the
         # block with +s+.
         def transform(s)
           s.is_a?(Array) ? s.map(&Proc.new) : (yield s)
         end
 
-        # The eager limit strategy used when true is given as the value, choosing the
-        # best strategy based on what the database supports.
+        # What eager limit strategy should be used when true is given as the value,
+        # defaults to UNION as that is the fastest strategy if the appropriate keys are indexed.
         def true_eager_limit_strategy
+          if self[:eager_graph] || (offset && !associated_dataset.supports_offsets_in_correlated_subqueries?)
+            # An SQL-based approach won't work if you are also eager graphing,
+            # so use a ruby based approach in that case.
+            :ruby
+          else
+            :union 
+          end
+        end
+
+        # The eager_graph limit strategy used when true is given as the value, choosing the
+        # best strategy based on what the database supports.
+        def true_eager_graph_limit_strategy
           if associated_class.dataset.supports_window_functions?
             :window_function
           else
             :ruby
           end
         end
+
+        # A placeholder literalizer used to speed up the creation of union queries when eager
+        # loading a limited association.
+        def union_eager_loader
+          cached_fetch(:union_eager_loader) do
+            Sequel::Dataset::PlaceholderLiteralizer.loader(associated_dataset) do |pl, ds|
+              keys = predicate_keys
+              ds = ds.where(keys.map{pl.arg}.zip(keys))
+              if eager_loading_use_associated_key?
+                ds = ds.select_append(*associated_key_array)
+              end
+              ds.from_self
+            end
+          end
+        end
+
+        # Whether the placeholder loader can be used to load the association.
+        def use_placeholder_loader?
+          !self[:instance_specific] && !self[:eager_graph]
+        end
       end
     
       class ManyToOneAssociationReflection < AssociationReflection
@@ -571,6 +747,11 @@ module Sequel
           nil
         end
 
+        # many_to_one associations don't need a filter by associations limit strategy
+        def filter_by_associations_limit_strategy
+          nil
+        end
+
         # The expression to use on the left hand side of the IN lookup when eager loading
         def predicate_key
           cached_fetch(:predicate_key){qualified_primary_key}
@@ -623,9 +804,8 @@ module Sequel
     
         private
     
-        def filter_by_associations_add_conditions_dataset_filter(ds)
-          pk = qualify(associated_class.table_name, primary_keys)
-          ds.select(*pk).where(Sequel.negate(pk.zip([])))
+        def filter_by_associations_conditions_associated_keys
+          qualify(associated_class.table_name, primary_keys)
         end
 
         def filter_by_associations_conditions_key
@@ -638,6 +818,10 @@ module Sequel
           super && self[:key]
         end
         
+        def predicate_key_methods
+          self[:keys]
+        end
+    
         def reciprocal_association?(assoc_reflect)
           super && self[:keys] == assoc_reflect[:keys] && primary_key == assoc_reflect.primary_key
         end
@@ -652,6 +836,16 @@ module Sequel
       class OneToManyAssociationReflection < AssociationReflection
         ASSOCIATION_TYPES[:one_to_many] = self
         
+        # Support a correlated subquery limit strategy when using eager_graph.
+        def apply_eager_graph_limit_strategy(strategy, ds)
+          case strategy
+          when :correlated_subquery
+            apply_correlated_subquery_limit_strategy(ds)
+          else
+            super
+          end
+        end
+
         # The keys in the associated model's table related to this association
         def associated_object_keys
           self[:keys]
@@ -673,7 +867,7 @@ module Sequel
         def default_key
           :"#{underscore(demodulize(self[:model].name))}_id"
         end
-        
+
         # Handle silent failure of add/remove methods if raise_on_save_failure is false.
         def handle_silent_modification_failure?
           self[:raise_on_save_failure] == false
@@ -684,7 +878,7 @@ module Sequel
           cached_fetch(:predicate_key){qualify_assoc(self[:key])}
         end
         alias qualified_key predicate_key
-    
+
         # The column in the current table that the key in the associated table references.
         def primary_key
           self[:primary_key]
@@ -718,9 +912,35 @@ module Sequel
     
         private
     
-        def filter_by_associations_add_conditions_dataset_filter(ds)
-          k = qualify(associated_class.table_name, self[:keys])
-          ds.select(*k).where(Sequel.negate(k.zip([])))
+        # Use a correlated subquery to limit the dataset.  Note that this will not
+        # work correctly if the associated dataset uses qualified identifers in the WHERE clause,
+        # as they would reference the containing query instead of the subquery.
+        def apply_correlated_subquery_limit_strategy(ds)
+          table = ds.first_source_table
+          table_alias = ds.first_source_alias
+          primary_key = associated_class.primary_key
+          key = self[:key]
+          cs_alias = :t1
+          cs = associated_dataset.
+            from(Sequel.as(table, :t1)).
+            select(*qualify(cs_alias, primary_key)).
+            where(Array(qualify(cs_alias, key)).zip(Array(qualify(table_alias, key)))).
+            limit(*limit_and_offset)
+          ds.where(qualify(table_alias, primary_key)=>cs)
+        end
+
+        # Support correlated subquery strategy when filtering by limited associations.
+        def apply_filter_by_associations_limit_strategy(ds)
+          case filter_by_associations_limit_strategy
+          when :correlated_subquery
+            apply_correlated_subquery_limit_strategy(ds)
+          else
+            super
+          end
+        end
+
+        def filter_by_associations_conditions_associated_keys
+          qualify(associated_class.table_name, self[:keys])
         end
 
         def filter_by_associations_conditions_key
@@ -739,6 +959,10 @@ module Sequel
           qualify(associated_class.table_name, associated_class.primary_key)
         end
 
+        def predicate_key_methods
+          self[:primary_keys]
+        end
+    
         def reciprocal_association?(assoc_reflect)
           super && self[:keys] == assoc_reflect[:keys] && primary_key == assoc_reflect.primary_key
         end
@@ -747,6 +971,19 @@ module Sequel
         def reciprocal_type
           :many_to_one
         end
+
+        # Support automatic use of correlated subqueries if :ruby option is best available option,
+        # MySQL is not being used, and either the associated class has a non-composite primary key
+        # or the database supports multiple columns in IN.
+        def true_eager_graph_limit_strategy
+          r = super
+          ds = associated_dataset
+          if r == :ruby && ds.supports_limits_in_correlated_subqueries? && (Array(associated_class.primary_key).length == 1 || ds.supports_multiple_column_in?) && (!offset || ds.supports_offsets_in_correlated_subqueries?)
+            :correlated_subquery
+          else
+            r
+          end
+        end
       end
 
       # Methods that turn an association that returns multiple objects into an association that
@@ -762,7 +999,17 @@ module Sequel
         # Add conditions when filtering by singular associations with orders, since the
         # underlying relationship is probably not one-to-one.
         def filter_by_associations_add_conditions?
-          super || self[:order]
+          super || self[:order] || self[:eager_limit_strategy] || self[:filter_limit_strategy]
+        end
+
+        # Make sure singular associations always have 1 as the limit
+        def limit_and_offset
+          r = super
+          if r.first == 1
+            r
+          else
+            [1, r[1]]
+          end
         end
 
         # Singular associations always return a single object, not an array.
@@ -777,8 +1024,14 @@ module Sequel
           super if self[:order] || offset
         end
 
+        # Use a strategy for filtering by associations if there is an order or an offset,
+        # or a specific limiting strategy has been specified.
+        def filter_by_associations_limit_strategy
+          super if self[:order] || offset || self[:eager_limit_strategy] || self[:filter_limit_strategy]
+        end
+
         # Use the DISTINCT ON eager limit strategy for true if the database supports it.
-        def true_eager_limit_strategy
+        def true_eager_graph_limit_strategy
           if associated_class.dataset.supports_ordered_distinct_on? && !offset
             :distinct_on
           else
@@ -838,6 +1091,36 @@ module Sequel
           self[:uses_left_composite_keys] ? (0...self[:left_keys].length).map{|i| :"x_foreign_key_#{i}_x"} : :x_foreign_key_x
         end
       
+        # The default eager loader used if the user doesn't override it.  Extracted
+        # to a method so the code can be shared with the many_through_many plugin.
+        def default_eager_loader(eo)
+          h = eo[:id_map]
+          assign_singular = assign_singular?
+          delete_rn = delete_row_number_column
+          uses_lcks = self[:uses_left_composite_keys]
+          left_key_alias = self[:left_key_alias]
+          name = self[:name]
+
+          self[:model].eager_load_results(self, eo) do |assoc_record|
+            assoc_record.values.delete(delete_rn) if delete_rn
+            hash_key = if uses_lcks
+              left_key_alias.map{|k| assoc_record.values.delete(k)}
+            else
+              assoc_record.values.delete(left_key_alias)
+            end
+            next unless objects = h[hash_key]
+            if assign_singular
+              objects.each do |object| 
+                object.associations[name] ||= assoc_record
+              end
+            else
+              objects.each do |object|
+                object.associations[name].push(assoc_record)
+              end
+            end
+          end
+        end
+
         # Default name symbol for the join table.
         def default_join_table
           [self[:class_name], self[:model].name].map{|i| underscore(pluralize(demodulize(i)))}.sort.join('_').to_sym
@@ -928,11 +1211,12 @@ module Sequel
 
         private
 
-        def filter_by_associations_add_conditions_dataset_filter(ds)
-          k = qualify(join_table_alias, self[:left_keys])
-          ds.select(*k).
-            where(Sequel.negate(k.zip([]))).
-            inner_join(self[:join_table], Array(self[:right_keys]).zip(right_primary_keys), :qualify=>:deep)
+        def _associated_dataset
+          super.inner_join(self[:join_table], self[:right_keys].zip(right_primary_keys), :qualify=>:deep)
+        end
+
+        def filter_by_associations_conditions_associated_keys
+          qualify(join_table_alias, self[:left_keys])
         end
 
         def filter_by_associations_conditions_key
@@ -952,10 +1236,10 @@ module Sequel
           qualify(join_table_alias, self[:left_keys]) + Array(qualify(associated_class.table_name, associated_class.primary_key))
         end
 
-        def filter_by_associations_limit_subquery
-          super.inner_join(self[:join_table], Array(self[:right_keys]).zip(right_primary_keys))
+        def predicate_key_methods
+          self[:left_primary_keys]
         end
-
+    
         def reciprocal_association?(assoc_reflect)
           super && assoc_reflect[:left_keys] == self[:right_keys] &&
             assoc_reflect[:right_keys] == self[:left_keys] &&
@@ -1066,11 +1350,9 @@ module Sequel
           association_reflections.values
         end
         
-        # Given an association reflection and a dataset, apply the
-        # :select, :conditions, :order, :eager, :distinct, and :eager_block
-        # association options to the given dataset and return the dataset
-        # or a modified copy of it.
+        # REMOVE410
         def apply_association_dataset_opts(opts, ds)
+          Deprecation.deprecate("Model.apply_association_dataset_opts/Model.eager_loading_dataset", "Use AssociationReflection#apply_dataset_changes/Association#reflection#apply_eager_dataset_changes instead.")
           ds = ds.select(*opts.select) if opts.select
           if c = opts[:conditions]
             ds = (c.is_a?(Array) && !Sequel.condition_specifier?(c)) ? ds.where(*c) : ds.where(c)
@@ -1168,6 +1450,9 @@ module Sequel
         # :eager_loader_key :: A symbol for the key column to use to populate the key_hash
         #                      for the eager loader.  Can be set to nil to not populate the key_hash.
         # :extend :: A module or array of modules to extend the dataset with.
+        # :filter_limit_strategy :: Determines the strategy used for enforcing limits and offsets when filtering by
+        #                           limited associations.  Possible options are :window_function, :distinct_on, or
+        #                           :correlated_subquery depending on association type and database type.
         # :graph_alias_base :: The base name to use for the table alias when eager graphing.  Defaults to the name
         #                      of the association.  If the alias name has already been used in the query, Sequel will create
         #                      a unique alias by appending a numeric suffix (e.g. alias_0, alias_1, ...) until the alias is
@@ -1215,6 +1500,8 @@ module Sequel
         #            the same name in both the join table and the associated table.
         # :setter :: Proc used to define the private _*= method for doing the work to setup the assocation
         #            between the given object and the current object (*_to_one associations).
+        # :subqueries_per_union :: The number of subqueries to use in each UNION query, for eager
+        #                          loading limited associations using the default :union strategy.
         # :validate :: Set to false to not validate when implicitly saving any associated object.
         # === :many_to_one
         # :key :: foreign key in current model's table that references
@@ -1303,6 +1590,12 @@ module Sequel
 
           opts = orig_opts.merge(:type => type, :name => name, :cache=>{}, :model => self)
           opts[:block] = block if block
+          if block || orig_opts[:block] || orig_opts[:dataset]
+            # It's possible the association is instance specific, in that it depends on
+            # values other than the foreign key value.  This needs to be checked for
+            # in certain places to disable optimizations.
+            opts[:instance_specific] = true
+          end
           opts = assoc_class.new.merge!(opts)
 
           if opts[:clone] && !opts.cloneable?(cloned_assoc)
@@ -1327,8 +1620,9 @@ module Sequel
           
           # Remove :class entry if it exists and is nil, to work with cached_fetch
           opts.delete(:class) unless opts[:class]
-          
+
           send(:"def_#{type}", opts)
+          def_association_instance_methods(opts)
       
           orig_opts.delete(:clone)
           orig_opts.merge!(:class_name=>opts[:class_name], :class=>opts[:class], :block=>opts[:block])
@@ -1347,7 +1641,12 @@ module Sequel
           association_reflections.keys
         end
 
-        # Modify and return eager loading dataset based on association options.
+        # Eager load the association with the given eager loader options.
+        def eager_load_results(opts, eo, &block)
+          opts.eager_load_results(eo, &block)
+        end
+
+        # REMOVE410
         def eager_loading_dataset(opts, ds, select, associations, eager_options=OPTS)
           ds = apply_association_dataset_opts(opts, ds)
           ds = ds.select(*select) if select
@@ -1411,29 +1710,58 @@ module Sequel
           association_module_def(name, opts, &block)
           association_module(opts).send(:private, name)
         end
-      
-        # Add the add_ instance method 
+
+        # REMOVE410
         def def_add_method(opts)
+          Deprecation.deprecate("Model.def_add_method", "The Model.associate method now sets up the add method you if an :adder association reflection entry is present.")
           association_module_def(opts.add_method, opts){|o,*args| add_associated_object(opts, o, *args)}
         end
-      
-        # Adds the association dataset methods to the association methods module.
+
+        # REMOVE410
         def def_association_dataset_methods(opts)
+          Deprecation.deprecate("Model.def_association_dataset_methods", "The Model.associate method now sets up the association dataset methods.")
           association_module_def(opts.dataset_method, opts){_dataset(opts)}
           def_association_method(opts)
         end
-
+      
         # Adds the association method to the association methods module.
         def def_association_method(opts)
           association_module_def(opts.association_method, opts){|*dynamic_opts, &block| load_associated_objects(opts, dynamic_opts[0], &block)}
         end
       
+        # Define all of the association instance methods for this association.
+        def def_association_instance_methods(opts)
+          association_module_def(opts.dataset_method, opts){_dataset(opts)}
+          def_association_method(opts)
+
+          return if opts[:read_only]
+
+          if opts[:setter] && opts[:_setter]
+            # This is backwards due to backwards compatibility
+            association_module_private_def(opts._setter_method, opts, &opts[:setter])
+            association_module_def(opts.setter_method, opts, &opts[:_setter])
+          end
+
+          if adder = opts[:adder]
+            association_module_private_def(opts._add_method, opts, &adder)
+            association_module_def(opts.add_method, opts){|o,*args| add_associated_object(opts, o, *args)}
+          end
+
+          if remover = opts[:remover]
+            association_module_private_def(opts._remove_method, opts, &remover)
+            association_module_def(opts.remove_method, opts){|o,*args| remove_associated_object(opts, o, *args)}
+          end
+
+          if clearer = opts[:clearer]
+            association_module_private_def(opts._remove_all_method, opts, &clearer)
+            association_module_def(opts.remove_all_method, opts){|*args| remove_all_associated_objects(opts, *args)}
+          end
+        end
+        
         # Configures many_to_many and one_through_one association reflection and adds the related association methods
         def def_many_to_many(opts)
           one_through_one = opts[:type] == :one_through_one
           opts[:read_only] = true if one_through_one
-          name = opts[:name]
-          model = self
           left = (opts[:left_key] ||= opts.default_left_key)
           lcks = opts[:left_keys] = Array(left)
           right = (opts[:right_key] ||= opts.default_right_key)
@@ -1448,49 +1776,15 @@ module Sequel
             rcpks = Array(opts[:right_primary_key])
             raise(Error, "mismatched number of right keys: #{rcks.inspect} vs #{rcpks.inspect}") unless rcks.length == rcpks.length
           end
-          uses_lcks = opts[:uses_left_composite_keys] = lcks.length > 1
+          opts[:uses_left_composite_keys] = lcks.length > 1
           opts[:uses_right_composite_keys] = rcks.length > 1
           opts[:cartesian_product_number] ||= one_through_one ? 0 : 1
           join_table = (opts[:join_table] ||= opts.default_join_table)
-          left_key_alias = opts[:left_key_alias] ||= opts.default_associated_key_alias
-          graph_jt_conds = opts[:graph_join_table_conditions] = opts.fetch(:graph_join_table_conditions, []).to_a
+          opts[:left_key_alias] ||= opts.default_associated_key_alias
           opts[:graph_join_table_join_type] ||= opts[:graph_join_type]
           opts[:after_load].unshift(:array_uniq!) if opts[:uniq]
-          slice_range = opts.slice_range
-          opts[:dataset] ||= proc{opts.associated_dataset.inner_join(join_table, rcks.zip(opts.right_primary_keys) + opts.predicate_keys.zip(lcpks.map{|k| send(k)}), :qualify=>:deep)}
-
-          opts[:eager_loader] ||= proc do |eo|
-            h = eo[:id_map]
-            rows = eo[:rows]
-            r = rcks.zip(opts.right_primary_keys)
-            l = [[opts.predicate_key, h.keys]]
-
-            ds = model.eager_loading_dataset(opts, opts.associated_class.inner_join(join_table, r + l, :qualify=>:deep), nil, eo[:associations], eo)
-            ds = opts.apply_eager_limit_strategy(ds)
-            opts.initialize_association_cache(rows)
-
-            assign_singular = opts.assign_singular?
-            delete_rn = opts.delete_row_number_column(ds)
-            ds.all do |assoc_record|
-              assoc_record.values.delete(delete_rn) if delete_rn
-              hash_key = if uses_lcks
-                left_key_alias.map{|k| assoc_record.values.delete(k)}
-              else
-                assoc_record.values.delete(left_key_alias)
-              end
-              next unless objects = h[hash_key]
-              if assign_singular
-                objects.each do |object| 
-                  object.associations[name] ||= assoc_record
-                end
-              else
-                objects.each do |object|
-                  object.associations[name].push(assoc_record)
-                end
-              end
-            end
-            opts.apply_ruby_eager_limit_strategy(rows)
-          end
+          opts[:dataset] ||= opts.association_dataset_proc
+          opts[:eager_loader] ||= opts.method(:default_eager_loader)
           
           join_type = opts[:graph_join_type]
           select = opts[:graph_select]
@@ -1498,6 +1792,7 @@ module Sequel
           only_conditions = opts[:graph_only_conditions]
           conditions = opts[:graph_conditions]
           graph_block = opts[:graph_block]
+          graph_jt_conds = opts[:graph_join_table_conditions] = opts.fetch(:graph_join_table_conditions, []).to_a
           use_jt_only_conditions = opts.include?(:graph_join_table_only_conditions)
           jt_only_conditions = opts[:graph_join_table_only_conditions]
           jt_join_type = opts[:graph_join_table_join_type]
@@ -1520,36 +1815,27 @@ module Sequel
             end
           end
       
-          def_association_dataset_methods(opts)
-      
           return if opts[:read_only] || one_through_one
       
-          adder = opts[:adder] || proc do |o|
+          opts[:adder] ||= proc do |o|
             h = {}
             lcks.zip(lcpks).each{|k, pk| h[k] = send(pk)}
             rcks.zip(opts.right_primary_key_methods).each{|k, pk| h[k] = o.send(pk)}
             _join_table_dataset(opts).insert(h)
           end
-          association_module_private_def(opts._add_method, opts, &adder) 
 
-          remover = opts[:remover] || proc do |o|
+          opts[:remover] ||= proc do |o|
             _join_table_dataset(opts).where(lcks.zip(lcpks.map{|k| send(k)}) + rcks.zip(opts.right_primary_key_methods.map{|k| o.send(k)})).delete
           end
-          association_module_private_def(opts._remove_method, opts, &remover)
 
-          clearer = opts[:clearer] || proc do
+          opts[:clearer] ||= proc do
             _join_table_dataset(opts).where(lcks.zip(lcpks.map{|k| send(k)})).delete
           end
-          association_module_private_def(opts._remove_all_method, opts, &clearer)
-      
-          def_add_method(opts)
-          def_remove_methods(opts)
         end
-        
+
         # Configures many_to_one association reflection and adds the related association methods
         def def_many_to_one(opts)
           name = opts[:name]
-          model = self
           opts[:key] = opts.default_key unless opts.has_key?(:key)
           key = opts[:key]
           opts[:eager_loader_key] = key unless opts.has_key?(:eager_loader_key)
@@ -1574,21 +1860,14 @@ module Sequel
             (auto_assocs[k] ||= []) << name
           end
 
-          opts[:dataset] ||= proc do
-            opts.associated_dataset.where(opts.predicate_keys.zip(cks.map{|k| send(k)}))
-          end
+          opts[:dataset] ||= opts.association_dataset_proc
           opts[:eager_loader] ||= proc do |eo|
             h = eo[:id_map]
-            keys = h.keys
+            pk_meths = opts.primary_key_methods
 
-            opts.initialize_association_cache(eo[:rows])
-
-            # Skip eager loading if no objects have a foreign key for this association
-            unless keys.empty?
-              klass = opts.associated_class
-              model.eager_loading_dataset(opts, klass.where(opts.predicate_key=>keys), nil, eo[:associations], eo).all do |assoc_record|
-                hash_key = uses_cks ? opts.primary_key_methods.map{|k| assoc_record.send(k)} : assoc_record.send(opts.primary_key_method)
-                next unless objects = h[hash_key]
+            eager_load_results(opts, eo) do |assoc_record|
+              hash_key = uses_cks ? pk_meths.map{|k| assoc_record.send(k)} : assoc_record.send(opts.primary_key_method)
+              if objects = h[hash_key]
                 objects.each{|object| object.associations[name] = assoc_record}
               end
             end
@@ -1606,20 +1885,16 @@ module Sequel
             ds.graph(eager_graph_dataset(opts, eo), use_only_conditions ? only_conditions : opts.primary_keys.zip(graph_cks) + conditions, eo.merge(:select=>select, :join_type=>eo[:join_type]||join_type, :qualify=>:deep, :from_self_alias=>eo[:from_self_alias]), &graph_block)
           end
       
-          def_association_dataset_methods(opts)
-          
           return if opts[:read_only]
       
-          setter = opts[:setter] || proc{|o| cks.zip(opts.primary_key_methods).each{|k, pk| send(:"#{k}=", (o.send(pk) if o))}}
-          association_module_private_def(opts._setter_method, opts, &setter)
-          association_module_def(opts.setter_method, opts){|o| set_associated_object(opts, o)}
+          opts[:setter] ||= proc{|o| cks.zip(opts.primary_key_methods).each{|k, pk| send(:"#{k}=", (o.send(pk) if o))}}
+          opts[:_setter] = proc{|o| set_associated_object(opts, o)}
         end
         
         # Configures one_to_many and one_to_one association reflections and adds the related association methods
         def def_one_to_many(opts)
           one_to_one = opts[:type] == :one_to_one
           name = opts[:name]
-          model = self
           key = (opts[:key] ||= opts.default_key)
           km = opts[:key_method] ||= opts[:key]
           cks = opts[:keys] = Array(key)
@@ -1631,23 +1906,14 @@ module Sequel
           pkcs = opts[:primary_key_columns] ||= Array(pkc)
           raise(Error, "mismatched number of keys: #{cks.inspect} vs #{cpks.inspect}") unless cks.length == cpks.length
           uses_cks = opts[:uses_composite_keys] = cks.length > 1
-          slice_range = opts.slice_range
-          opts[:dataset] ||= proc do
-            opts.associated_dataset.where(opts.predicate_keys.zip(cpks.map{|k| send(k)}))
-          end
+          opts[:dataset] ||= opts.association_dataset_proc
           opts[:eager_loader] ||= proc do |eo|
             h = eo[:id_map]
-            rows = eo[:rows]
             reciprocal = opts.reciprocal
-            klass = opts.associated_class
-
-            ds = model.eager_loading_dataset(opts, klass.where(opts.predicate_key=>h.keys), nil, eo[:associations], eo)
-            ds = opts.apply_eager_limit_strategy(ds)
-            opts.initialize_association_cache(rows)
-
             assign_singular = opts.assign_singular?
-            delete_rn = opts.delete_row_number_column(ds)
-            ds.all do |assoc_record|
+            delete_rn = opts.delete_row_number_column
+
+            eager_load_results(opts, eo) do |assoc_record|
               assoc_record.values.delete(delete_rn) if delete_rn
               hash_key = uses_cks ? km.map{|k| assoc_record.send(k)} : assoc_record.send(km)
               next unless objects = h[hash_key]
@@ -1665,7 +1931,6 @@ module Sequel
                 end
               end
             end
-            opts.apply_ruby_eager_limit_strategy(rows)
           end
           
           join_type = opts[:graph_join_type]
@@ -1683,50 +1948,40 @@ module Sequel
             ds
           end
       
-          def_association_dataset_methods(opts)
-          
+          return if opts[:read_only]
+
+          save_opts = {:validate=>opts[:validate]}
           ck_nil_hash ={}
           cks.each{|k| ck_nil_hash[k] = nil}
 
-          unless opts[:read_only]
-            save_opts = {:validate=>opts[:validate]}
-
-            if one_to_one
-              setter = opts[:setter] || proc do |o|
-                up_ds = _apply_association_options(opts, opts.associated_dataset.where(cks.zip(cpks.map{|k| send(k)})))
-                if o
-                  up_ds = up_ds.exclude(o.pk_hash) unless o.new?
-                  cks.zip(cpks).each{|k, pk| o.send(:"#{k}=", send(pk))}
-                end
-                checked_transaction do
-                  up_ds.update(ck_nil_hash)
-                  o.save(save_opts) || raise(Sequel::Error, "invalid associated object, cannot save") if o
-                end
-              end
-              association_module_private_def(opts._setter_method, opts, &setter)
-              association_module_def(opts.setter_method, opts){|o| set_one_to_one_associated_object(opts, o)}
-            else 
-              save_opts[:raise_on_failure] = opts[:raise_on_save_failure] != false
-
-              adder = opts[:adder] || proc do |o|
+          if one_to_one
+            opts[:setter] ||= proc do |o|
+              up_ds = _apply_association_options(opts, opts.associated_dataset.where(cks.zip(cpks.map{|k| send(k)})))
+              if o
+                up_ds = up_ds.exclude(o.pk_hash) unless o.new?
                 cks.zip(cpks).each{|k, pk| o.send(:"#{k}=", send(pk))}
-                o.save(save_opts)
               end
-              association_module_private_def(opts._add_method, opts, &adder)
-      
-              remover = opts[:remover] || proc do |o|
-                cks.each{|k| o.send(:"#{k}=", nil)}
-                o.save(save_opts)
+              checked_transaction do
+                up_ds.update(ck_nil_hash)
+                o.save(save_opts) || raise(Sequel::Error, "invalid associated object, cannot save") if o
               end
-              association_module_private_def(opts._remove_method, opts, &remover)
+            end
+            opts[:_setter] = proc{|o| set_one_to_one_associated_object(opts, o)}
+          else 
+            save_opts[:raise_on_failure] = opts[:raise_on_save_failure] != false
 
-              clearer = opts[:clearer] || proc do
-                _apply_association_options(opts, opts.associated_dataset.where(cks.zip(cpks.map{|k| send(k)}))).update(ck_nil_hash)
-              end
-              association_module_private_def(opts._remove_all_method, opts, &clearer)
+            opts[:adder] ||= proc do |o|
+              cks.zip(cpks).each{|k, pk| o.send(:"#{k}=", send(pk))}
+              o.save(save_opts)
+            end
+    
+            opts[:remover] ||= proc do |o|
+              cks.each{|k| o.send(:"#{k}=", nil)}
+              o.save(save_opts)
+            end
 
-              def_add_method(opts)
-              def_remove_methods(opts)
+            opts[:clearer] ||= proc do
+              _apply_association_options(opts, opts.associated_dataset.where(cks.zip(cpks.map{|k| send(k)}))).update(ck_nil_hash)
             end
           end
         end
@@ -1741,12 +1996,13 @@ module Sequel
           def_one_to_many(opts)
         end
         
-        # Add the remove_ and remove_all instance methods
+        # REMOVE410
         def def_remove_methods(opts)
+          Deprecation.deprecate("Model.def_remove_methods", "The Model.associate method now sets up the remove/remove_all methods for you if a :remover or :clearer association reflection entry is present.")
           association_module_def(opts.remove_method, opts){|o,*args| remove_associated_object(opts, o, *args)}
           association_module_def(opts.remove_all_method, opts){|*args| remove_all_associated_objects(opts, *args)}
         end
-
+          
         # Return dataset to graph into given the association reflection, applying the :callback option if set.
         def eager_graph_dataset(opts, eager_options)
           ds = opts.associated_class.dataset
@@ -1796,6 +2052,13 @@ module Sequel
           ds
         end
         
+        # A placeholder literalizer that can be used to load the association, or nil to not use one.
+        def _associated_object_loader(opts, dynamic_opts)
+          if !dynamic_opts[:callback] && (loader = opts.placeholder_loader)
+            loader
+          end
+        end
+
         # Return an association dataset for the given association reflection
         def _dataset(opts)
           raise(Sequel::Error, "model object #{inspect} does not have a primary key") if opts.dataset_need_primary_key? && !pk
@@ -1819,10 +2082,19 @@ module Sequel
           _load_associated_object_array(opts, dynamic_opts).first
         end
 
+        # Return the associated single object using a primary key lookup on the associated class.
+        def _load_associated_object_via_primary_key(opts)
+          opts.associated_class.send(:primary_key_lookup, ((fk = opts[:key]).is_a?(Array) ? fk.map{|c| send(c)} : send(fk)))
+        end
+
         # Load the associated objects for the given association reflection and dynamic options
         # as an array.
         def _load_associated_object_array(opts, dynamic_opts)
-          _associated_dataset(opts, dynamic_opts).all
+          if loader = _associated_object_loader(opts, dynamic_opts)
+            loader.all(*opts.predicate_key_values(self))
+          else
+            _associated_dataset(opts, dynamic_opts).all
+          end
         end
 
         # Return the associated objects from the dataset, without association callbacks, reciprocals, and caching.
@@ -1832,7 +2104,7 @@ module Sequel
             if opts.returns_array?
               _load_associated_object_array(opts, dynamic_opts)
             elsif load_with_primary_key_lookup?(opts, dynamic_opts)
-              opts.associated_class.send(:primary_key_lookup, ((fk = opts[:key]).is_a?(Array) ? fk.map{|c| send(c)} : send(fk)))
+              _load_associated_object_via_primary_key(opts)
             else
               _load_associated_object(opts, dynamic_opts)
             end
@@ -2196,23 +2468,12 @@ END
         # Each association's order, if defined, is respected.
         # If the association uses a block or has an :eager_block argument, it is used.
         def eager(*associations)
-          opt = @opts[:eager]
-          opt = opt ? opt.dup : {}
-          associations.flatten.each do |association|
-            case association
-            when Symbol
-              check_association(model, association)
-              opt[association] = nil
-            when Hash
-              association.keys.each{|assoc| check_association(model, assoc)}
-              opt.merge!(association)
-            else
-              raise(Sequel::Error, 'Associations must be in the form of a symbol or hash')
-            end
-          end
-          clone(:eager=>opt)
+          opts = @opts[:eager]
+          association_opts = eager_options_for_associations(associations)
+          opts = opts ? opts.merge(association_opts) : association_opts
+          clone(:eager=>opts)
         end
-      
+
         # The secondary eager loading method.  Loads all associations in a single query. This
         # method should only be used if you need to filter or order based on columns in associated tables.
         #
@@ -2247,6 +2508,7 @@ END
         #                    Appropriate :limit_strategy values are:
         #                    true :: Pick the most appropriate based on what the database supports
         #                    :distinct_on :: Force use of DISTINCT ON stategy (*_one associations only)
+        #                    :correlated_subquery :: Force use of correlated subquery strategy (one_to_* associations only)
         #                    :window_function :: Force use of window function strategy
         #                    :ruby :: Don't modify the SQL, implement limits/offsets with array slicing
         #
@@ -2321,7 +2583,6 @@ END
               associations = assoc.is_a?(Array) ? assoc : [assoc]
             end
           end
-          orig_ds = ds
           local_opts = ds.opts[:eager_graph][:local]
           limit_strategy = r.eager_graph_limit_strategy(local_opts[:limit_strategy])
           ds = loader.call(:self=>ds, :table_alias=>assoc_table_alias, :implicit_qualifier=>(ta == ds.opts[:eager_graph][:master]) ? first_source : qualifier_from_alias_symbol(ta, first_source), :callback=>callback, :join_type=>local_opts[:join_type], :limit_strategy=>limit_strategy, :from_self_alias=>ds.opts[:eager_graph][:master])
@@ -2391,6 +2652,25 @@ END
           end
         end
 
+        # Process the array of associations arguments (Symbols, Arrays, and Hashes),
+        # and return a hash of options suitable for cascading.
+        def eager_options_for_associations(associations)
+          opts = {}
+          associations.flatten.each do |association|
+            case association
+            when Symbol
+              check_association(model, association)
+              opts[association] = nil
+            when Hash
+              association.keys.each{|assoc| check_association(model, assoc)}
+              opts.merge!(association)
+            else
+              raise(Sequel::Error, 'Associations must be in the form of a symbol or hash')
+            end
+          end
+          opts
+        end
+      
         # Return an expression for filtering by the given association reflection and associated object.
         def association_filter_expression(op, ref, obj)
           meth = :"#{ref[:type]}_association_filter_expression"