sequel 3.21.0 → 3.41.0

data/lib/sequel/plugins/serialization.rb

@@ -11,8 +11,17 @@ module Sequel
     # set the @deserialized_values entry.  This plugin adds a before_save hook
     # that serializes all @deserialized_values to @values.
     #
-    # You can use either marshal, yaml, or json as the serialization format.
-    # If you use yaml or json, you should require them by yourself.
+    # You can specify the serialization format as a pair of serializer/deserializer
+    # callable objects.  You can also specify the serialization format as a single
+    # symbol, if such a symbol has a registered serializer/deserializer pair in the
+    # plugin.  By default, the plugin registers the :marshal, :yaml, and :json
+    # serialization formats.  To register your own serialization formats, use
+    # Sequel::Plugins::Serialization.register_format.
+    # If you use yaml or json format, you need to require the libraries, Sequel
+    # does not do the requiring for you.
+    #
+    # You can specify the columns to serialize when loading the plugin, or later
+    # using the serialize_attributes class method.
     #
     # Because of how this plugin works, it must be used inside each model class
     # that needs serialization, after any set_dataset method calls in that class.
@@ -22,56 +31,96 @@ module Sequel
     # == Example
     #
     #   require 'sequel'
+    #   # Require json, as the plugin doesn't require it for you.
     #   require 'json'
+    #
+    #   # Register custom serializer/deserializer pair
+    #   Sequel::Plugins::Serialization.register_format(:reverse,
+    #     lambda{|v| v.reverse},
+    #     lambda{|v| v.reverse})
+    #
     #   class User < Sequel::Model
+    #     # Built-in format support when loading the plugin
     #     plugin :serialization, :json, :permissions
-    #     # or
+    #
+    #     # Built-in format support after loading the plugin using serialize_attributes
     #     plugin :serialization
-    #     serialize_attributes :marshal, :permissions, :attributes
+    #     serialize_attributes :marshal, :permissions
+    #
+    #     # Use custom registered serialization format just like built-in format
+    #     serialize_attributes :reverse, :password
+    #
+    #     # Use a custom serializer/deserializer pair without registering
+    #     serialize_attributes [lambda{|v| v.reverse}, lambda{|v| v.reverse}], :password
     #   end
     #   user = User.create
     #   user.permissions = { :global => 'read-only' }
     #   user.save
     module Serialization
+      # The default serializers supported by the serialization module.
+      # Use register_format to add serializers to this hash.
+      REGISTERED_FORMATS = {}
+
       # Set up the column readers to do deserialization and the column writers
       # to save the value in deserialized_values.
       def self.apply(model, *args)
-        model.instance_eval{@serialization_map = {}}
+        model.instance_eval do
+          @deserialization_map = {}
+          @serialization_map = {}
+        end
       end
       
+      # Automatically call serialize_attributes with the format and columns unless
+      # no columns were provided.
       def self.configure(model, format=nil, *columns)
         model.serialize_attributes(format, *columns) unless columns.empty?
       end
 
+      # Register a serializer/deserializer pair with a format symbol, to allow
+      # models to pick this format by name.  Both serializer and deserializer
+      # should be callable objects.
+      def self.register_format(format, serializer, deserializer)
+        REGISTERED_FORMATS[format] = [serializer, deserializer]
+      end
+      register_format(:marshal, lambda{|v| [Marshal.dump(v)].pack('m')}, lambda{|v| Marshal.load(v.unpack('m')[0]) rescue Marshal.load(v)})
+      register_format(:yaml, lambda{|v| v.to_yaml}, lambda{|v| YAML.load(v)})
+      register_format(:json, lambda{|v| v.to_json}, lambda{|v| JSON.parse(v)})
+
       module ClassMethods
-        # A map of the serialized columns for this model.  Keys are column
-        # symbols, values are serialization formats (:marshal, :yaml, or :json).
+        # A hash with column name symbols and callable values, with the value
+        # called to deserialize the column.
+        attr_reader :deserialization_map
+
+        # A hash with column name symbols and callable values, with the value
+        # called to serialize the column.
         attr_reader :serialization_map
 
         # Module to store the serialized column accessor methods, so they can
         # call be overridden and call super to get the serialization behavior
         attr_accessor :serialization_module
 
-        # Copy the serialization format and columns to serialize into the subclass.
+        # Copy the serialization_map and deserialization map into the subclass.
         def inherited(subclass)
           super
           sm = serialization_map.dup
-          subclass.instance_eval{@serialization_map = sm}
-        end
-        
-        # The first value in the serialization map.  This is only for
-        # backwards compatibility, use serialization_map in new code.
-        def serialization_format
-          serialization_map.values.first
+          dsm = deserialization_map.dup
+          subclass.instance_eval do
+            @deserialization_map = dsm
+            @serialization_map = sm
+          end
         end
         
         # Create instance level reader that deserializes column values on request,
-        # and instance level writer that stores new deserialized value in deserialized
-        # columns
+        # and instance level writer that stores new deserialized values.
         def serialize_attributes(format, *columns)
-          raise(Error, "Unsupported serialization format (#{format}), should be :marshal, :yaml, or :json") unless [:marshal, :yaml, :json].include?(format)
+          if format.is_a?(Symbol)
+            unless format = REGISTERED_FORMATS[format]
+              raise(Error, "Unsupported serialization format: #{format} (valid formats: #{REGISTERED_FORMATS.keys.map{|k| k.inspect}.join})")
+            end
+          end
+          serializer, deserializer = format
           raise(Error, "No columns given.  The serialization plugin requires you specify which columns to serialize") if columns.empty?
-          define_serialized_attribute_accessor(format, *columns)
+          define_serialized_attribute_accessor(serializer, deserializer, *columns)
         end
         
         # The columns that will be serialized.  This is only for
@@ -83,12 +132,13 @@ module Sequel
         private
 
         # Add serializated attribute acessor methods to the serialization_module
-        def define_serialized_attribute_accessor(format, *columns)
+        def define_serialized_attribute_accessor(serializer, deserializer, *columns)
           m = self
           include(self.serialization_module ||= Module.new) unless serialization_module
           serialization_module.class_eval do
             columns.each do |column|
-              m.serialization_map[column] = format
+              m.serialization_map[column] = serializer
+              m.deserialization_map[column] = deserializer
               define_method(column) do 
                 if deserialized_values.has_key?(column)
                   deserialized_values[column]
@@ -110,52 +160,51 @@ module Sequel
         attr_reader :deserialized_values
 
         # Set @deserialized_values to the empty hash
-        def initialize(*args, &block)
+        def initialize_set(values)
           @deserialized_values = {}
           super
         end
 
-        # Serialize all deserialized values
+        # Serialize deserialized values before saving
         def before_save
-          deserialized_values.each{|k,v| @values[k] = serialize_value(k, v)}
+          serialize_deserialized_values
           super
         end
         
-        # Empty the deserialized values when refreshing.
-        def refresh
-          @deserialized_values = {}
+        # Initialization the deserialized values for objects retrieved from the database.
+        def set_values(*)
+          @deserialized_values ||= {}
           super
         end
 
         private
 
-        # Deserialize the column from either marshal or yaml format
+        # Empty the deserialized values when refreshing.
+        def _refresh(*)
+          @deserialized_values = {}
+          super
+        end
+
+        # Deserialize the column value.  Called when the model column accessor is called to
+        # return a deserialized value.
         def deserialize_value(column, v)
-          return v if v.nil?
-          case model.serialization_map[column] 
-          when :marshal
-            Marshal.load(v.unpack('m')[0]) rescue Marshal.load(v)
-          when :yaml
-            YAML.load v if v
-          when :json
-            JSON.parse v if v
-          else
-            raise Error, "Bad serialization format (#{model.serialization_map[column].inspect}) for column #{column.inspect}"
+          unless v.nil?
+            raise Sequel::Error, "no entry in deserialization_map for #{column.inspect}" unless callable = model.deserialization_map[column]
+            callable.call(v)
           end
         end
 
-        # Serialize the column to either marshal or yaml format
+        # Serialize all deserialized values
+        def serialize_deserialized_values
+          deserialized_values.each{|k,v| @values[k] = serialize_value(k, v)}
+        end
+
+        # Serialize the column value.  Called before saving to ensure the serialized value
+        # is saved in the database.
         def serialize_value(column, v)
-          return v if v.nil?
-          case model.serialization_map[column] 
-          when :marshal
-            [Marshal.dump(v)].pack('m')
-          when :yaml
-            v.to_yaml
-          when :json
-            v.to_json
-          else
-            raise Error, "Bad serialization format (#{model.serialization_map[column].inspect}) for column #{column.inspect}"
+          unless v.nil?
+            raise Sequel::Error, "no entry in serialization_map for #{column.inspect}" unless callable = model.serialization_map[column]
+            callable.call(v)
           end
         end
       end

data/lib/sequel/plugins/serialization_modification_detection.rb

@@ -0,0 +1,63 @@
+module Sequel
+  module Plugins
+    # This plugin extends the serialization plugin and enables it to detect
+    # changes in serialized values by checking whether the current
+    # deserialized value is the same as the original deserialized value.
+    # The serialization plugin does not do such checks by default, as they
+    # often aren't needed and can hurt performance.
+    #
+    # Note that for this plugin to work correctly, the values you are
+    # serializing must roundtrip correctly (i.e. deserialize(serialize(value))
+    # should equal value).  This is true in most cases, but not in all.  For
+    # example, ruby symbols round trip through yaml, but not json (as they get
+    # turned into strings in json).
+    #
+    # == Example
+    #
+    #   require 'sequel'
+    #   require 'json'
+    #   class User < Sequel::Model
+    #     plugin :serialization, :json, :permissions
+    #     plugin :serialization_modification_detection
+    #   end
+    #   user = User.create(:permissions => {})
+    #   user.permissions[:global] = 'read-only'
+    #   user.save_changes
+    module SerializationModificationDetection
+      # Load the serialization plugin automatically.
+      def self.apply(model)
+        model.plugin :serialization
+      end
+      
+      module InstanceMethods
+        # Clear the cache of original deserialized values after saving so that it doesn't
+        # show the column is modified after saving.
+        def after_save
+          super
+          @original_deserialized_values = {}
+        end
+
+        # Detect which serialized columns have changed.
+        def changed_columns
+          cc = super
+          deserialized_values.each{|c, v| cc << c if !cc.include?(c) && original_deserialized_value(c) != v} 
+          cc
+        end
+
+        private
+
+        # For new objects, serialize any existing deserialized values so that changes can
+        # be detected.
+        def initialize_set(values)
+          super
+          serialize_deserialized_values
+        end
+
+        # Return the original deserialized value of the column, caching it to improve performance.
+        def original_deserialized_value(column)
+          (@original_deserialized_values ||= {})[column] ||= deserialize_value(column, self[column])
+        end
+      end
+    end
+  end
+end

data/lib/sequel/plugins/sharding.rb

@@ -1,13 +1,14 @@
 module Sequel
   module Plugins
-    # The sharding plugin makes it easy to use Sequel's sharding features
-    # with models.  It lets you create model objects on specific shards,
-    # and any models retrieved from specific shards are automatically
-    # saved back to those shards.  It also works with associations,
-    # so that model objects retrieved from specific shards will only
-    # return associated objects from that shard, and using the
-    # add/remove/remove_all association methods will only affect
-    # that shard.
+    # The sharding plugin augments Sequel's default model sharding support
+    # in the following ways:
+    #
+    # 1) It automatically sets model instances to be saved back to the
+    #    shard they were retreived from.
+    # 2) It makes model associations use the same shard as the model
+    #    object.
+    # 3) It adds a slightly nicer API for creating model instances on
+    #    specific shards.
     # 
     # Usage:
     #
@@ -40,15 +41,21 @@ module Sequel
         def new_using_server(s, values={}, &block)
           new(values, &block).set_server(s)
         end
-      end
 
-      module InstanceMethods
-        # Set the shard that this object is tied to.  Returns self.
-        def set_server(s)
-          @server = s
-          self
+        private
+
+        # Set the server for each graphed dataset to the current server
+        # unless the graphed dataset already has a server set.
+        def eager_graph_dataset(opts, eager_options)
+          ds = super
+          if s = eager_options[:self].opts[:server]
+            ds = ds.server(s) unless ds.opts[:server]
+          end
+          ds
         end
+      end
 
+      module InstanceMethods
         # Set the server that this object is tied to, unless it has
         # already been set.  Returns self.
         def set_server?(s)
@@ -56,11 +63,6 @@ module Sequel
           self
         end
 
-        # Ensure that the instance dataset is tied to the correct shard.
-        def this
-          use_server(super)
-        end
-
         private
 
         # Ensure that association datasets are tied to the correct shard.
@@ -68,21 +70,11 @@ module Sequel
           use_server(super)
         end
 
-        # Ensure that the object is inserted into the correct shard.
-        def _insert_dataset
-          use_server(super)
-        end
-
         # Ensure that the join table for many_to_many associations uses the correct shard.
         def _join_table_dataset(opts)
           use_server(super)
         end
 
-        # Make sure to use the correct shard when using a transaction
-        def checked_transaction(opts={}, &block)
-          super(@server ? {:server=>@server}.merge(opts) : opts, &block)
-        end
-
         # If creating the object by doing <tt>add_association</tt> for a
         # +many_to_many+ association, make sure the associated object is created on the
         # current object's shard, unless the passed object already has an assigned shard.
@@ -90,11 +82,6 @@ module Sequel
           o.set_server?(@server) if o.respond_to?(:set_server?)
           super
         end
-
-        # Set the given dataset to use the current object's shard.
-        def use_server(ds)
-          @server ? ds.server(@server) : ds
-        end
       end
 
       module DatasetMethods
@@ -109,26 +96,6 @@ module Sequel
           end
           ds
         end
-
-        private
-
-        # Set the shard of all retrieved objects to the shard of
-        # the table's dataset, or the current shard if the table's
-        # dataset does not have a shard.
-        def graph_each
-          ta = @opts[:graph][:table_aliases]
-          s = @opts[:server]
-          super do |r|
-            r.each do |k, v|
-              if ds = ta[k]
-                dss = ds.opts[:server]
-              end
-              vs = dss || s
-              v.set_server(vs) if vs && v.respond_to?(:set_server)
-            end
-            yield r
-          end
-        end
       end
     end
   end

data/lib/sequel/plugins/single_table_inheritance.rb

@@ -35,6 +35,17 @@ module Sequel
     #   Employee.plugin :single_table_inheritance, :type,
     #     :model_map=>{'line staff'=>:Staff, 'supervisor'=>:Manager}
     #
+    #   # By default the plugin sets the respective column value
+    #   # when a new instance is created.
+    #   Staff.create.type == 'line staff'
+    #   Manager.create.type == 'supervisor'
+    #
+    #   # You can customize this behavior with the :key_chooser option.
+    #   # This is most useful when using a non-bijective mapping.
+    #   Employee.plugin :single_table_inheritance, :type,
+    #     :model_map=>{'line staff'=>:Staff, 'supervisor'=>:Manager},
+    #     :key_chooser=>proc{|instance| instance.model.sti_key_map[instance.model.to_s].first || 'stranger' }
+    #
     #   # Using custom procs, with :model_map taking column values
     #   # and yielding either a class, string, symbol, or nil, 
     #   # and :key_map taking a class object and returning the column
@@ -43,6 +54,15 @@ module Sequel
     #     :model_map=>proc{|v| v.reverse},
     #     :key_map=>proc{|klass| klass.name.reverse}
     #
+    #   # You can use the same class for multiple values.
+    #   # This is mainly useful when the sti_key column contains multiple values
+    #   # which are different but do not require different code.
+    #   Employee.plugin :single_table_inheritance, :type,
+    #     :model_map=>{'staff' => "Staff",
+    #                  'manager => "Manager",
+    #                  'overpayed staff => "Staff",
+    #                  'underpayed staff' => "Staff"}
+    #
     # One minor issue to note is that if you specify the <tt>:key_map</tt>
     # option as a hash, instead of having it inferred from the <tt>:model_map</tt>,
     # you should only use class name strings as keys, you should not use symbols
@@ -57,20 +77,38 @@ module Sequel
           @sti_model_map = opts[:model_map] || lambda{|v| v if v && v != ''}
           @sti_key_map = if km = opts[:key_map]
             if km.is_a?(Hash)
-              h = Hash.new{|h,k| h[k.to_s] unless k.is_a?(String)}
-              h.merge!(km)
+              h = Hash.new do |h,k| 
+                unless k.is_a?(String)
+                  h[k.to_s]
+                else
+                  []
+                end
+              end
+              km.each do |k,v|
+                h[k.to_s] = [ ] unless h.key?(k.to_s)
+                h[k.to_s].push( *Array(v) )
+              end
+              h
             else
               km
             end
           elsif sti_model_map.is_a?(Hash)
-            h = Hash.new{|h,k| h[k.to_s] unless k.is_a?(String)}
+            h = Hash.new do |h,k| 
+              unless k.is_a?(String)
+                h[k.to_s]
+              else
+                []
+              end
+            end
             sti_model_map.each do |k,v|
-              h[v.to_s] = k
+              h[v.to_s] = [ ] unless h.key?(v.to_s)
+              h[v.to_s] << k
             end
             h
           else
             lambda{|klass| klass.name.to_s}
           end
+          @sti_key_chooser = opts[:key_chooser] || lambda{|inst| Array(inst.model.sti_key_map[inst.model]).last }
           dataset.row_proc = lambda{|r| model.sti_load(r)}
         end
       end
@@ -97,6 +135,10 @@ module Sequel
         # the value of the sti_key column to the appropriate class to use.
         attr_reader :sti_model_map
 
+        # A proc which returns the value to use for new instances.
+        # This defaults to a lookup in the key map.
+        attr_reader :sti_key_chooser
+
         # Copy the necessary attributes to the subclasses, and filter the
         # subclass's dataset based on the sti_kep_map entry for the class.
         def inherited(subclass)
@@ -105,31 +147,34 @@ module Sequel
           sd = sti_dataset
           skm = sti_key_map
           smm = sti_model_map
-          key = skm[subclass] 
+          skc = sti_key_chooser
+          key = Array(skm[subclass]).dup
           sti_subclass_added(key)
-          ska = [key]
-          subclass.set_dataset(sd.filter(SQL::QualifiedIdentifier.new(table_name, sk)=>ska), :inherited=>true)
+          rp = dataset.row_proc
+          subclass.set_dataset(sd.filter(SQL::QualifiedIdentifier.new(table_name, sk)=>key), :inherited=>true)
           subclass.instance_eval do
+            dataset.row_proc = rp
             @sti_key = sk
-            @sti_key_array = ska
+            @sti_key_array = key
             @sti_dataset = sd
             @sti_key_map = skm
             @sti_model_map = smm
-            @simple_table = nil
+            @sti_key_chooser = skc
+            self.simple_table = nil
           end
         end
 
         # Return an instance of the class specified by sti_key,
         # used by the row_proc.
         def sti_load(r)
-          sti_class(sti_model_map[r[sti_key]]).load(r)
+          sti_class(sti_model_map[r[sti_key]]).call(r)
         end
 
         # Make sure that all subclasses of the parent class correctly include 
         # keys for all of their descendant classes.
         def sti_subclass_added(key)
           if sti_key_array
-            sti_key_array << key
+            Sequel.synchronize{sti_key_array.push(*Array(key))}
             superclass.sti_subclass_added(key)
           end
         end
@@ -157,7 +202,7 @@ module Sequel
       module InstanceMethods
         # Set the sti_key column based on the sti_key_map.
         def before_create
-          send("#{model.sti_key}=", model.sti_key_map[model]) unless self[model.sti_key]
+          send("#{model.sti_key}=", model.sti_key_chooser.call(self)) unless self[model.sti_key]
           super
         end
       end

data/lib/sequel/plugins/static_cache.rb

@@ -0,0 +1,99 @@
+module Sequel
+  module Plugins
+    # The static_cache plugin is designed for models that are not modified at all
+    # in production use cases, or at least where modifications to them would usually
+    # coincide with an application restart.  When loaded into a model class, it 
+    # retrieves all rows in the database and staticly caches a ruby array and hash
+    # keyed on primary key containing all of the model instances.  All of these instances
+    # are frozen so they won't be modified unexpectedly.
+    #
+    # The caches this plugin creates are used for the following things:
+    #
+    # * Primary key lookups (e.g. Model[1])
+    # * Model.all calls
+    # * Model.each calls
+    # * Model.map calls without an argument
+    # * Model.to_hash calls without an argument
+    #
+    # Usage:
+    #
+    #   # Cache the AlbumType class staticly
+    #   AlbumType.plugin :static_cache
+    module StaticCache
+      # Populate the static caches when loading the plugin.
+      def self.configure(model)
+        model.send(:load_cache)
+      end
+
+      module ClassMethods
+        # A frozen ruby hash holding all of the model's frozen instances, keyed by frozen primary key.
+        attr_reader :cache
+
+        # An array of all of the model's frozen instances, without issuing a database
+        # query.
+        def all
+          @all.dup
+        end
+
+        # Return the frozen object with the given pk, or nil if no such object exists
+        # in the cache, without issuing a database query.
+        def cache_get_pk(pk)
+          cache[pk]
+        end
+
+        # Yield each of the model's frozen instances to the block, without issuing a database
+        # query.
+        def each(&block)
+          @all.each(&block)
+        end
+
+        # If no arguments are given, yield each of the model's frozen instances to the block.
+        # and return a new array, without issuing a database query.  If any arguments are
+        # given, use the default Sequel behavior.
+        def map(*a)
+          if a.empty?
+            @all.map(&(Proc.new if block_given?))
+          else
+            super
+          end
+        end
+
+        # Reload the cache when the dataset changes.
+        def set_dataset(*)
+          s = super
+          load_cache
+          s
+        end
+
+        # If no arguments are given, yield an identity map for the model with frozen primary keys
+        # and instances, without issuing a database query. If any arguments are
+        # given, use the default Sequel behavior.
+        def to_hash(*a)
+          if a.empty?
+            cache.dup
+          else
+            super
+          end
+        end
+
+        private
+
+        # Return the frozen object with the given pk, or nil if no such object exists
+        # in the cache, without issuing a database query.
+        def primary_key_lookup(pk)
+          cache[pk]
+        end
+
+        # Reload the cache for this model by retrieving all of the instances in the dataset
+        # freezing them, and populating the cached array and hash.
+        def load_cache
+          a = dataset.all
+          h = {}
+          a.each{|o| h[o.pk.freeze] = o.freeze}
+          @all = a.freeze
+          @cache = h.freeze
+        end
+      end
+    end
+  end
+end