sequel 3.21.0 → 3.39.0

data/lib/sequel/plugins/instance_filters.rb

@@ -68,6 +68,16 @@ module Sequel
       
         private
         
+        # If there are any instance filters, make sure not to use the
+        # instance delete optimization.
+        def _delete_without_checking
+          if @instance_filters && !@instance_filters.empty?
+            _delete_dataset.delete 
+          else
+            super
+          end
+        end
+
         # Lazily initialize the instance filter array.
         def instance_filters
           @instance_filters ||= []

data/lib/sequel/plugins/instance_hooks.rb

@@ -29,7 +29,7 @@ module Sequel
         BEFORE_HOOKS = Sequel::Model::BEFORE_HOOKS
         AFTER_HOOKS = Sequel::Model::AFTER_HOOKS - [:after_initialize]
         HOOKS = BEFORE_HOOKS + AFTER_HOOKS
-        HOOKS.each{|h| class_eval("def #{h}_hook(&block); add_instance_hook(:#{h}, &block) end", __FILE__, __LINE__)}
+        HOOKS.each{|h| class_eval("def #{h}_hook(&block); add_instance_hook(:#{h}, &block); self end", __FILE__, __LINE__)}
         
         BEFORE_HOOKS.each{|h| class_eval("def #{h}; run_before_instance_hooks(:#{h}) == false ? false : super end", __FILE__, __LINE__)}
         AFTER_HOOKS.each{|h| class_eval(<<-END, __FILE__, __LINE__ + 1)}

data/lib/sequel/plugins/json_serializer.rb

@@ -56,6 +56,11 @@ module Sequel
     #   Album.to_json
     #   Album.filter(:artist_id=>1).to_json(:include=>:tags)
     #
+    # If you have an existing array of model instances you want to convert to
+    # JSON, you can call the class to_json method with the :array option:
+    #
+    #   Album.to_json(:array=>[Album[1], Album[2]])
+    #
     # Usage:
     #
     #   # Add JSON output capability to all model subclass instances (called before loading subclasses)
@@ -134,8 +139,13 @@ module Sequel
       module InstanceMethods
         # Parse the provided JSON, which should return a hash,
         # and call +set+ with that hash.
-        def from_json(json)
-          set(JSON.parse(json))
+        def from_json(json, opts={})
+          h = JSON.parse(json)
+          if fields = opts[:fields]
+            set_fields(h, fields, opts)
+          else
+            set(h)
+          end
         end
 
         # Return a string in JSON format.  Accepts the following
@@ -149,7 +159,7 @@ module Sequel
         #             to include in the JSON output.  Using a nested
         #             hash, you can pass options to associations
         #             to affect the JSON used for associated objects.
-        # :naked :: Not to add the JSON.create_id key to the JSON
+        # :naked :: Not to add the JSON.create_id (json_class) key to the JSON
         #           output hash, so when the JSON is parsed, it
         #           will yield a hash instead of a model object.
         # :only :: Symbol or Array of Symbols of columns to only
@@ -195,7 +205,21 @@ module Sequel
       module DatasetMethods
         # Return a JSON string representing an array of all objects in
         # this dataset.  Takes the same options as the the instance
-        # method, and passes them to every instance.
+        # method, and passes them to every instance.  Additionally,
+        # respects the following options:
+        #
+        # :array :: An array of instances.  If this is not provided,
+        #           calls #all on the receiver to get the array.
+        # :root :: If set to :collection, only wraps the collection
+        #          in a root object.  If set to :instance, only wraps
+        #          the instances in a root object.  If set to :both,
+        #          wraps both the collection and instances in a root
+        #          object.  Unfortunately, for backwards compatibility,
+        #          if this option is true and doesn't match one of those
+        #          symbols, it defaults to both.  That may change in a
+        #          future version, so for forwards compatibility, you
+        #          should pick a specific symbol for your desired
+        #          behavior.
         def to_json(*a)
           if opts = a.first.is_a?(Hash)
             opts = model.json_serializer_opts.merge(a.first)
@@ -203,8 +227,36 @@ module Sequel
           else
             opts = model.json_serializer_opts
           end
-          res = row_proc ? all.map{|obj| Literal.new(obj.to_json(opts))} : all
-          opts[:root] ? {model.send(:pluralize, model.send(:underscore, model.to_s)) => res}.to_json(*a) : res.to_json(*a)
+
+          collection_root = case opts[:root]
+          when nil, false, :instance
+            false
+          when :collection
+            opts = opts.dup
+            opts.delete(:root)
+            opts[:naked] = true unless opts.has_key?(:naked)
+            true
+          else
+            true
+          end
+
+          res = if row_proc 
+            array = if opts[:array]
+              opts = opts.dup
+              opts.delete(:array)
+            else
+              all
+            end
+            array.map{|obj| Literal.new(obj.to_json(opts))}
+           else
+            all
+          end
+
+          if collection_root
+            {model.send(:pluralize, model.send(:underscore, model.to_s)) => res}.to_json(*a)
+          else
+            res.to_json(*a)
+          end
         end
       end
     end

data/lib/sequel/plugins/list.rb

@@ -39,9 +39,11 @@ module Sequel
     #   Item.plugin :list, :scope=>:user_id
     # 
     # Note that using this plugin modifies the order of the model's dataset to
-    # sort by the position and scope fields.
+    # sort by the position and scope fields.  Also note that this plugin is subject to
+    # race conditions, and is not safe when concurrent modifications are made
+    # to the same list.
     #
-    # Also note that unlike ruby arrays, the list plugin assumes that the
+    # Additionally, note that unlike ruby arrays, the list plugin assumes that the
     # first entry in the list has position 1, not position 0.
     #
     # Copyright (c) 2007-2010 Sharon Rosner, Wayne E. Seguin, Aman Gupta, Adrian Madrid, Jeremy Evans
@@ -90,6 +92,15 @@ module Sequel
           list_dataset.first(position_field => p)
         end
 
+        # Set the value of the position_field to the maximum value plus 1 unless the
+        # position field already has a value.
+        def before_create
+          unless send(position_field)
+            send("#{position_field}=", list_dataset.max(position_field).to_i+1)
+          end
+          super
+        end
+
         # Find the last position in the list containing this instance.
         def last_position
           list_dataset.max(position_field).to_i

data/lib/sequel/plugins/many_through_many.rb

@@ -47,20 +47,18 @@ module Sequel
       class ManyThroughManyAssociationReflection < Sequel::Model::Associations::ManyToManyAssociationReflection
         Sequel::Model::Associations::ASSOCIATION_TYPES[:many_through_many] = self
 
-        # The table containing the column to use for the associated key when eagerly loading
-        def associated_key_table
-          self[:associated_key_table] = self[:final_reverse_edge][:alias]
-        end
-        
         # The default associated key alias(es) to use when eager loading
         # associations via eager.
         def default_associated_key_alias
           self[:uses_left_composite_keys] ? (0...self[:through].first[:left].length).map{|i| :"x_foreign_key_#{i}_x"} : :x_foreign_key_x
         end
 
-        # The list of joins to use when eager graphing
-        def edges
-          self[:edges] || calculate_edges || self[:edges]
+        %w'associated_key_table predicate_key edges final_edge final_reverse_edge reverse_edges'.each do |meth|
+          class_eval(<<-END, __FILE__, __LINE__+1)
+            def #{meth}
+              cached_fetch(:#{meth}){calculate_edges[:#{meth}]}
+            end
+          END
         end
 
         # Many through many associations don't have a reciprocal
@@ -68,11 +66,6 @@ module Sequel
           nil
         end
 
-        # The list of joins to use when lazy loading or eager loading
-        def reverse_edges
-          self[:reverse_edges] || calculate_edges || self[:reverse_edges]
-        end
-
         private
 
         # Make sure to use unique table aliases when lazy loading or eager loading
@@ -94,7 +87,7 @@ module Sequel
 
         # Transform the :through option into a list of edges and reverse edges to use to join tables when loading the association.
         def calculate_edges
-          es = [{:left_table=>self[:model].table_name, :left_key=>self[:left_primary_key]}]
+          es = [{:left_table=>self[:model].table_name, :left_key=>self[:left_primary_key_column]}]
           self[:through].each do |t|
             es.last.merge!(:right_key=>t[:left], :right_table=>t[:table], :join_type=>t[:join_type]||self[:graph_join_type], :conditions=>(t[:conditions]||[]).to_a, :block=>t[:block])
             es.last[:only_conditions] = t[:only_conditions] if t.include?(:only_conditions)
@@ -109,37 +102,38 @@ module Sequel
           reverse_edges = es.reverse.map{|e| {:table=>e[:left_table], :left=>e[:left_key], :right=>e[:right_key]}}
           reverse_edges.pop
           calculate_reverse_edge_aliases(reverse_edges)
-          self[:final_edge] = edges.pop
-          self[:final_reverse_edge] = reverse_edges.pop
-          self[:edges] = edges
-          self[:reverse_edges] = reverse_edges
-          nil
+          final_reverse_edge = reverse_edges.pop
+          final_reverse_alias = final_reverse_edge[:alias]
+
+          h = {:final_edge=>edges.pop,
+               :final_reverse_edge=>final_reverse_edge,
+               :edges=>edges,
+               :reverse_edges=>reverse_edges,
+               :predicate_key=>qualify(final_reverse_alias, edges.first[:right]),
+               :associated_key_table=>final_reverse_edge[:alias],
+          }
+          h.each{|k, v| cached_set(k, v)}
+          h
         end
       end
+
       module ClassMethods
         # Create a many_through_many association.  Arguments:
         # * name - Same as associate, the name of the association.
         # * through - The tables and keys to join between the current table and the associated table.
         #   Must be an array, with elements that are either 3 element arrays, or hashes with keys :table, :left, and :right.
         #   The required entries in the array/hash are:
-        #   * :table (first array element) - The name of the table to join.
-        #   * :left (middle array element) - The key joining the table to the previous table. Can use an
-        #     array of symbols for a composite key association.
-        #   * :right (last array element) - The key joining the table to the next table. Can use an
-        #     array of symbols for a composite key association.
+        #   :table (first array element) :: The name of the table to join.
+        #   :left (middle array element) :: The key joining the table to the previous table. Can use an
+        #                                   array of symbols for a composite key association.
+        #   :right (last array element) :: The key joining the table to the next table. Can use an
+        #                                  array of symbols for a composite key association.
         #   If a hash is provided, the following keys are respected when using eager_graph:
-        #   * :block - A proc to use as the block argument to join.
-        #   * :conditions - Extra conditions to add to the JOIN ON clause.  Must be a hash or array of two pairs.
-        #   * :join_type - The join type to use for the join, defaults to :left_outer.
-        #   * :only_conditions - Conditions to use for the join instead of the ones specified by the keys.
-        # * opts - The options for the associaion.  Takes the same options as associate, and supports these additional options:
-        #   * :left_primary_key - column in current table that the first :left option in
-        #     through points to, as a symbol. Defaults to primary key of current table. Can use an
-        #     array of symbols for a composite key association.
-        #   * :right_primary_key - column in associated table that the final :right option in 
-        #     through points to, as a symbol. Defaults to primary key of the associated table.  Can use an
-        #     array of symbols for a composite key association.
-        #   * :uniq - Adds a after_load callback that makes the array of objects unique.
+        #   :block :: A proc to use as the block argument to join.
+        #   :conditions :: Extra conditions to add to the JOIN ON clause.  Must be a hash or array of two pairs.
+        #   :join_type :: The join type to use for the join, defaults to :left_outer.
+        #   :only_conditions :: Conditions to use for the join instead of the ones specified by the keys.
+        # * opts - The options for the associaion.  Takes the same options as many_to_many.
         def many_through_many(name, through, opts={}, &block)
           associate(:many_through_many, name, opts.merge(through.is_a?(Hash) ? through : {:through=>through}), &block)
         end
@@ -170,24 +164,41 @@ module Sequel
           uses_lcks = opts[:uses_left_composite_keys] = left_key.is_a?(Array)
           left_keys = Array(left_key)
           left_pk = (opts[:left_primary_key] ||= self.primary_key)
+          opts[:eager_loader_key] = left_pk unless opts.has_key?(:eager_loader_key)
           left_pks = opts[:left_primary_keys] = Array(left_pk)
+          lpkc = opts[:left_primary_key_column] ||= left_pk
+          lpkcs = opts[:left_primary_key_columns] ||= Array(lpkc)
           opts[:dataset] ||= lambda do
             ds = opts.associated_class
-            opts.reverse_edges.each{|t| ds = ds.join(t[:table], Array(t[:left]).zip(Array(t[:right])), :table_alias=>t[:alias])}
-            ft = opts[:final_reverse_edge]
-            ds.join(ft[:table],  Array(ft[:left]).zip(Array(ft[:right])) + left_keys.zip(left_pks.map{|k| send(k)}), :table_alias=>ft[:alias])
+            opts.reverse_edges.each{|t| ds = ds.join(t[:table], Array(t[:left]).zip(Array(t[:right])), :table_alias=>t[:alias], :qualify=>:deep)}
+            ft = opts.final_reverse_edge
+            ds.join(ft[:table],  Array(ft[:left]).zip(Array(ft[:right])) + opts.predicate_keys.zip(left_pks.map{|k| send(k)}), :table_alias=>ft[:alias], :qualify=>:deep)
           end
 
           left_key_alias = opts[:left_key_alias] ||= opts.default_associated_key_alias
           opts[:eager_loader] ||= lambda do |eo|
-            h = eo[:key_hash][left_pk]
-            eo[:rows].each{|object| object.associations[name] = []}
+            h = eo[:id_map]
+            rows = eo[:rows]
+            rows.each{|object| object.associations[name] = []}
             ds = opts.associated_class 
-            opts.reverse_edges.each{|t| ds = ds.join(t[:table], Array(t[:left]).zip(Array(t[:right])), :table_alias=>t[:alias])}
-            ft = opts[:final_reverse_edge]
-            conds = uses_lcks ? [[left_keys.map{|k| SQL::QualifiedIdentifier.new(ft[:table], k)}, h.keys]] : [[left_key, h.keys]]
-            ds = ds.join(ft[:table], Array(ft[:left]).zip(Array(ft[:right])) + conds, :table_alias=>ft[:alias])
-            model.eager_loading_dataset(opts, ds, Array(opts.select), eo[:associations], eo).all do |assoc_record|
+            opts.reverse_edges.each{|t| ds = ds.join(t[:table], Array(t[:left]).zip(Array(t[:right])), :table_alias=>t[:alias], :qualify=>:deep)}
+            ft = opts.final_reverse_edge
+            ds = ds.join(ft[:table], Array(ft[:left]).zip(Array(ft[:right])) + [[opts.predicate_key, h.keys]], :table_alias=>ft[:alias], :qualify=>:deep)
+            ds = model.eager_loading_dataset(opts, ds, nil, eo[:associations], eo)
+            case opts.eager_limit_strategy
+            when :window_function
+              delete_rn = true
+              rn = ds.row_number_column
+              ds = apply_window_function_eager_limit_strategy(ds, opts)
+            when :correlated_subquery
+              ds = apply_correlated_subquery_eager_limit_strategy(ds, opts) do |xds|
+                dsa = ds.send(:dataset_alias, 2)
+                opts.reverse_edges.each{|t| xds = xds.join(t[:table], Array(t[:left]).zip(Array(t[:right])), :table_alias=>t[:alias])}
+                xds.join(ft[:table], Array(ft[:left]).zip(Array(ft[:right])) + left_keys.map{|k| [k, SQL::QualifiedIdentifier.new(ft[:table], k)]}, :table_alias=>dsa, :qualify=>:deep)
+              end
+            end
+            ds.all do |assoc_record|
+              assoc_record.values.delete(rn) if delete_rn
               hash_key = if uses_lcks
                 left_key_alias.map{|k| assoc_record.values.delete(k)}
               else
@@ -196,6 +207,10 @@ module Sequel
               next unless objects = h[hash_key]
               objects.each{|object| object.associations[name].push(assoc_record)}
             end
+            if opts.eager_limit_strategy == :ruby
+              limit, offset = opts.limit_and_offset
+              rows.each{|o| o.associations[name] = o.associations[name].slice(offset||0, limit) || []}
+            end
           end
 
           join_type = opts[:graph_join_type]
@@ -204,19 +219,56 @@ module Sequel
           only_conditions = opts[:graph_only_conditions]
           use_only_conditions = opts.include?(:graph_only_conditions)
           conditions = opts[:graph_conditions]
-          opts[:eager_grapher] ||= proc do |ds, assoc_alias, table_alias|
-            iq = table_alias
+          opts[:eager_grapher] ||= proc do |eo|
+            ds = eo[:self]
+            iq = eo[:implicit_qualifier]
             opts.edges.each do |t|
-              ds = ds.graph(t[:table], t.fetch(:only_conditions, (Array(t[:right]).zip(Array(t[:left])) + t[:conditions])), :select=>false, :table_alias=>ds.unused_table_alias(t[:table]), :join_type=>t[:join_type], :implicit_qualifier=>iq, &t[:block])
+              ds = ds.graph(t[:table], t.fetch(:only_conditions, (Array(t[:right]).zip(Array(t[:left])) + t[:conditions])), :select=>false, :table_alias=>ds.unused_table_alias(t[:table]), :join_type=>t[:join_type], :qualify=>:deep, :implicit_qualifier=>iq, &t[:block])
               iq = nil
             end
-            fe = opts[:final_edge]
-            ds.graph(opts.associated_class, use_only_conditions ? only_conditions : (Array(opts.right_primary_key).zip(Array(fe[:left])) + conditions), :select=>select, :table_alias=>assoc_alias, :join_type=>join_type, &graph_block)
+            fe = opts.final_edge
+            ds.graph(opts.associated_class, use_only_conditions ? only_conditions : (Array(opts.right_primary_key).zip(Array(fe[:left])) + conditions), :select=>select, :table_alias=>eo[:table_alias], :qualify=>:deep, :join_type=>join_type, &graph_block)
           end
 
           def_association_dataset_methods(opts)
         end
       end
+
+      module DatasetMethods
+        private
+
+        # Use a subquery to filter rows to those related to the given associated object
+        def many_through_many_association_filter_expression(op, ref, obj)
+          lpks = ref[:left_primary_key_columns]
+          lpks = lpks.first if lpks.length == 1
+          lpks = ref.qualify(model.table_name, lpks)
+          edges = ref.edges
+          first, rest = edges.first, edges[1..-1]
+          last = edges.last
+          ds = model.db[first[:table]].select(*Array(ref.qualify(first[:table], first[:right])))
+          rest.each{|e| ds = ds.join(e[:table], e.fetch(:only_conditions, (Array(e[:right]).zip(Array(e[:left])) + e[:conditions])), :table_alias=>ds.unused_table_alias(e[:table]), :qualify=>:deep, &e[:block])}
+          last_alias = if rest.empty?
+            first[:table]
+          else
+            last_join = ds.opts[:join].last
+            last_join.table_alias || last_join.table
+          end
+
+          meths = if obj.is_a?(Sequel::Dataset)
+            ref.qualify(obj.model.table_name, ref.right_primary_keys)
+          else
+            ref.right_primary_key_methods
+          end
+
+          exp = association_filter_key_expression(ref.qualify(last_alias, Array(ref.final_edge[:left])), meths, obj)
+          if exp == SQL::Constants::FALSE
+            association_filter_handle_inversion(op, exp, Array(lpks))
+          else
+            ds = ds.where(exp).exclude(SQL::BooleanExpression.from_value_pairs(ds.opts[:select].zip([]), :OR))
+            association_filter_handle_inversion(op, SQL::BooleanExpression.from_value_pairs(lpks=>ds), Array(lpks))
+          end
+        end
+      end
     end
   end
 end

data/lib/sequel/plugins/many_to_one_pk_lookup.rb

@@ -0,0 +1,71 @@
+module Sequel
+  module Plugins
+    # This is a fairly simple plugin that modifies the internal association loading logic
+    # for many_to_one associations to use a simple primary key lookup on the associated
+    # class, which is generally faster as it uses mostly static SQL.  Additional, if the
+    # associated class is caching primary key lookups, you get the benefit of a cached
+    # lookup.
+    #
+    # This plugin is generally not as fast as the prepared_statements_associations plugin
+    # in the case where the model is not caching primary key lookups, however, it is
+    # probably significantly faster if the model is caching primary key lookups.  If
+    # the prepared_statements_associations plugin has been loaded first, this
+    # plugin will only use the primary key lookup code if the associated model is
+    # caching primary key lookups.
+    #
+    # This plugin attempts to determine cases where the primary key lookup would have
+    # different results than the regular lookup, and use the regular lookup in that case,
+    # but it cannot handle all situations correctly, which is why it is not Sequel's
+    # default behavior.
+    #
+    # You can disable primary key lookups on a per association basis with this
+    # plugin using the :many_to_one_pk_lookup=>false association option.
+    # 
+    # Usage:
+    #
+    #   # Make all model subclass instances use primary key lookups for many_to_one
+    #   # association loading
+    #   Sequel::Model.plugin :many_to_one_pk_lookup
+    #
+    #   # Do so for just the album class.
+    #   Album.plugin :many_to_one_pk_lookup
+    module ManyToOnePkLookup
+      module InstanceMethods
+        private
+
+        # If the current association is a fairly simple many_to_one association, use
+        # a simple primary key lookup on the associated model, which can benefit from
+        # caching if the associated model is using caching.
+        def _load_associated_object(opts, dynamic_opts)
+          klass = opts.associated_class
+          cache_lookup = opts.send(:cached_fetch, :many_to_one_pk_lookup) do 
+            opts[:type] == :many_to_one &&
+              opts[:key] &&
+              opts.primary_key == klass.primary_key
+          end
+          if cache_lookup &&
+            !dynamic_opts[:callback] &&
+            (o = klass.send(:primary_key_lookup, ((fk = opts[:key]).is_a?(Array) ? fk.map{|c| send(c)} : send(fk))))
+            o
+          else
+            super
+          end
+        end
+
+        # Deal with the situation where the prepared_statements_associations plugin is
+        # loaded first, by using a primary key lookup for many_to_one associations if
+        # the associated class is using caching, and using the default code otherwise.
+        # This is done because the prepared_statements_associations code is probably faster
+        # than the primary key lookup this plugin uses if the model is not caching lookups,
+        # but probably slower if the model is caching lookups.
+        def _load_associated_objects(opts, dynamic_opts={})
+          if opts.can_have_associated_objects?(self) && opts[:type] == :many_to_one && opts.associated_class.respond_to?(:cache_get_pk)
+            _load_associated_object(opts, dynamic_opts)
+          else
+            super
+          end
+        end
+      end
+    end
+  end
+end