sequel 3.21.0 → 3.39.0

data/lib/sequel/plugins/prepared_statements_associations.rb

@@ -0,0 +1,87 @@
+module Sequel
+  module Plugins
+    # The prepared_statements_associations plugin modifies the regular association
+    # load method to use a cached prepared statement to load the associations.
+    # It will not work on all associations, but it should skip the use of prepared
+    # statements for associations where it will not work, assuming you load the
+    # plugin before defining the associations.
+    #
+    # Usage:
+    #
+    #   # Make all model subclasses more safe when using prepared statements (called before loading subclasses)
+    #   Sequel::Model.plugin :prepared_statements_associations
+    #
+    #   # Make the Album class more safe when using prepared statements
+    #   Album.plugin :prepared_statements_associations
+    module PreparedStatementsAssociations
+      # Synchronize access to the integer sequence so that no two calls get the same integer.
+      MUTEX = Mutex.new
+
+      i = 0
+      # This plugin names prepared statements uniquely using an integer sequence, this
+      # lambda returns the next integer to use.
+      NEXT = lambda{MUTEX.synchronize{i += 1}}
+
+      module ClassMethods
+        # Disable prepared statement use if a block is given, or the :dataset or :conditions
+        # options are used, or you are cloning an association.
+        def associate(type, name, opts = {}, &block)
+          if block || opts[:dataset] || opts[:conditions] || (opts[:clone] && association_reflection(opts[:clone])[:prepared_statement] == false)
+            opts = opts.merge(:prepared_statement=>false)
+          end
+          super(type, name, opts, &block)
+        end
+      end
+
+      module InstanceMethods
+        private
+
+        # Return a bound variable hash that maps the keys in +ks+ (qualified by the +table+)
+        # to the values of the results of sending the methods in +vs+.
+        def association_bound_variable_hash(table, ks, vs)
+          Hash[*ks.zip(vs).map{|k, v| [:"#{table}.#{k}", send(v)]}.flatten]
+        end
+
+        # Given an association reflection, return a bound variable hash for the given
+        # association for this instance's values.
+        def association_bound_variables(opts)
+          case opts[:type]
+          when :many_to_one
+            association_bound_variable_hash(opts.associated_class.table_name, opts.primary_keys, opts[:keys])
+          when :one_to_many
+            association_bound_variable_hash(opts.associated_class.table_name, opts[:keys], opts[:primary_keys])
+          when :many_to_many
+            association_bound_variable_hash(opts.join_table_alias, opts[:left_keys], opts[:left_primary_keys])
+          when :many_through_many
+            association_bound_variable_hash(opts.final_reverse_edge[:alias], Array(opts[:left_key]), opts[:left_primary_keys])
+          end
+        end
+
+        # Given an association reflection, return and cache a prepared statement for this association such
+        # that, given appropriate bound variables, the prepared statement will work correctly for any
+        # instance.
+        def association_prepared_statement(opts)
+          opts.send(:cached_fetch, :prepared_statement) do
+            ps = _associated_dataset(opts, {}).unbind.first.prepare(opts.returns_array? ? :select : :first, :"smpsap_#{NEXT.call}")
+            ps.log_sql = true
+            ps
+          end
+        end
+
+        # If a prepared statement can be used to load the associated objects, execute it to retrieve them.  Otherwise,
+        # fall back to the default implementation.
+        def _load_associated_objects(opts, dynamic_opts={})
+          if !opts.can_have_associated_objects?(self) || dynamic_opts[:callback] || (ps = opts[:prepared_statement]) == false
+            super
+          else 
+            if bv = association_bound_variables(opts)
+              (ps || association_prepared_statement(opts)).call(bv)
+            else
+              super
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sequel/plugins/prepared_statements_safe.rb

@@ -0,0 +1,82 @@
+module Sequel
+  module Plugins
+    # The prepared_statements_safe plugin modifies the model to reduce the number of
+    # prepared statements that can be created, by setting as many columns as possible
+    # before creating, and by changing +save_changes+ to save all columns instead of
+    # just the changed ones.
+    #
+    # This plugin depends on the +prepared_statements+ plugin.
+    # 
+    # Usage:
+    #
+    #   # Make all model subclasses more safe when using prepared statements (called before loading subclasses)
+    #   Sequel::Model.plugin :prepared_statements_safe
+    #
+    #   # Make the Album class more safe when using prepared statements
+    #   Album.plugin :prepared_statements_safe
+    module PreparedStatementsSafe
+      # Depend on the prepared_statements plugin
+      def self.apply(model)
+        model.plugin(:prepared_statements)
+      end
+
+      # Set the column defaults to use when creating on the model.
+      def self.configure(model)
+        model.send(:set_prepared_statements_column_defaults)
+      end
+
+      module ClassMethods
+        # A hash with column symbol keys and default values.  Instance's
+        # values are merged into this hash before creating to reduce the
+        # number of free columns (columns that may or may not be present
+        # in the INSERT statement), as the number of prepared statements
+        # that can be created is 2^N (where N is the number of free columns).
+        attr_reader :prepared_statements_column_defaults
+        
+        def inherited(subclass)
+          super
+          subclass.instance_variable_set(:@prepared_statements_column_defaults, @prepared_statements_column_defaults) if @prepared_statements_column_defaults && !subclass.prepared_statements_column_defaults
+        end
+
+        # Set the column defaults to use when creating on the subclass.
+        def set_dataset(*)
+          x = super
+          set_prepared_statements_column_defaults
+          x
+        end
+
+        private
+
+        # Set the column defaults based on the database schema.  All columns
+        # are set to a default value unless they are a primary key column or
+        # they don't have a parseable default.
+        def set_prepared_statements_column_defaults
+          if db_schema
+            h = {}
+            db_schema.each do |k, v|
+              h[k] = v[:ruby_default] if (v[:ruby_default] || !v[:default]) && !v[:primary_key]
+            end
+            @prepared_statements_column_defaults = h
+          end
+        end
+      end
+
+      module InstanceMethods
+        # Merge the current values into the default values to reduce the number
+        # of free columns.
+        def before_create
+          if v = model.prepared_statements_column_defaults
+            set_values(v.merge(values))
+          end
+          super
+        end
+
+        # Always do a full save of all columns to reduce the number of prepared
+        # statements that can be used.
+        def save_changes(opts={})
+          save(opts) || false if modified?
+        end
+      end
+    end
+  end
+end

data/lib/sequel/plugins/prepared_statements_with_pk.rb

@@ -0,0 +1,59 @@
+module Sequel
+  module Plugins
+    # The prepared_statements_with_pk plugin allows Dataset#with_pk for model datasets
+    # to use prepared statements by extract the values of previously bound variables
+    # using <tt>Dataset#unbind</tt>, and attempting to use a prepared statement if the
+    # variables can be unbound correctly. See  +Unbinder+ for details about what types of
+    # dataset filters can be unbound correctly.
+    #
+    # This plugin depends on the +prepared_statements+ plugin and should be considered unsafe.
+    # Unbinding dataset values cannot be done correctly in all cases, and use of this plugin
+    # in cases where not there are variables that are not unbound can lead to an denial of
+    # service attack by allocating an arbitrary number of prepared statements.  You have been
+    # warned.
+    #
+    # Usage:
+    #
+    #   # Make all model subclasses use prepared statements for Dataset#with_pk (called before loading subclasses)
+    #   Sequel::Model.plugin :prepared_statements_with_pk
+    #
+    #   # Make the Album class use prepared statements for Dataset#with_pk
+    #   Album.plugin :prepared_statements_with_pk
+    module PreparedStatementsWithPk
+      # Depend on the prepared_statements plugin
+      def self.apply(model)
+        model.plugin(:prepared_statements)
+      end
+
+      module ClassMethods
+        private
+
+        # Return a prepared statement that can be used to lookup a row given a dataset for the row matching
+        # the primary key.
+        def prepared_lookup_dataset(ds)
+          cached_prepared_statement(:lookup_sql, ds.sql){prepare_statement(ds.filter(prepared_statement_key_array(primary_key).map{|k, v| [SQL::QualifiedIdentifier.new(ds.model.table_name, k), v]}), :first)}
+        end
+      end
+
+      module DatasetMethods
+        # Use a prepared statement to find a row with the matching primary key
+        # inside this dataset.
+        def with_pk(pk)
+          begin
+            ds, bv = unbind
+          rescue UnbindDuplicate
+            super
+          else
+            begin
+              bv = bv.merge!(model.primary_key_hash(pk)){|k, v1, v2| ((v1 == v2) ? v1 : raise(UnbindDuplicate))}
+            rescue UnbindDuplicate
+              super
+            else
+              model.send(:prepared_lookup_dataset, ds).call(bv)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sequel/plugins/rcte_tree.rb

@@ -109,7 +109,15 @@ module Sequel
         ka = opts[:key_alias] ||= :x_root_x
         t = opts[:cte_name] ||= :t
         opts[:reciprocal] = nil
-        c_all = SQL::ColumnAll.new(model.table_name)
+        c_all = if model.dataset.recursive_cte_requires_column_aliases?
+          # Work around Oracle/ruby-oci8 bug that returns integers as BigDecimals in recursive queries.
+          conv_bd = model.db.database_type == :oracle
+          col_aliases = model.dataset.columns
+          model_table = model.table_name
+          col_aliases.map{|c| SQL::QualifiedIdentifier.new(model_table, c)}
+        else
+          [SQL::ColumnAll.new(model.table_name)]
+        end
         
         a = opts.merge(opts.fetch(:ancestors, {}))
         ancestors = a.fetch(:name, :ancestors)
@@ -125,9 +133,9 @@ module Sequel
           end
           table_alias = model.dataset.schema_and_table(model.table_name)[1].to_sym
           model.from(t => table_alias).
-           with_recursive(t, base_ds.select_all,
-            recursive_ds.
-            select(c_all))
+           with_recursive(t, col_aliases ? base_ds.select(*col_aliases) : base_ds.select_all,
+            recursive_ds.select(*c_all),
+            :args=>col_aliases)
         end
         aal = Array(a[:after_load])
         aal << proc do |m, ancs|
@@ -152,7 +160,7 @@ module Sequel
         end
         a[:after_load] ||= aal
         a[:eager_loader] ||= proc do |eo|
-          id_map = eo[:key_hash][key]
+          id_map = eo[:id_map]
           parent_map = {}
           children_map = {}
           eo[:rows].each do |obj|
@@ -163,9 +171,9 @@ module Sequel
           end
           r = model.association_reflection(ancestors)
           base_case = model.filter(prkey=>id_map.keys).
-           select(SQL::AliasedExpression.new(prkey, ka), c_all)
+           select(SQL::AliasedExpression.new(prkey, ka), *c_all)
           recursive_case = model.join(t, key=>prkey).
-           select(SQL::QualifiedIdentifier.new(t, ka), c_all)
+           select(SQL::QualifiedIdentifier.new(t, ka), *c_all)
           if c = r[:conditions]
             (base_case, recursive_case) = [base_case, recursive_case].collect do |ds|
               (c.is_a?(Array) && !Sequel.condition_specifier?(c)) ? ds.filter(*c) : ds.filter(c)
@@ -175,7 +183,8 @@ module Sequel
           elds = model.eager_loading_dataset(r,
            model.from(t => table_alias).
             with_recursive(t, base_case,
-             recursive_case),
+             recursive_case,
+             :args=>(([ka] + col_aliases) if col_aliases)),
            r.select,
            eo[:associations], eo)
           elds = elds.select_append(ka) unless elds.opts[:select] == nil
@@ -192,7 +201,9 @@ module Sequel
               (children_map[obj[key]] ||= []) << obj
             end
             
-            if roots = id_map[obj.values.delete(ka)]
+            kv = obj.values.delete(ka)
+            kv = kv.to_i if conv_bd && kv.is_a?(BigDecimal)
+            if roots = id_map[kv]
               roots.each do |root|
                 root.associations[ancestors] << obj
               end
@@ -222,9 +233,9 @@ module Sequel
           end
           table_alias = model.dataset.schema_and_table(model.table_name)[1].to_sym
           model.from(t => table_alias).
-           with_recursive(t, base_ds.select_all,
-            recursive_ds.
-            select(SQL::ColumnAll.new(model.table_name)))
+           with_recursive(t, col_aliases ? base_ds.select(*col_aliases) : base_ds.select_all,
+            recursive_ds.select(*c_all),
+            :args=>col_aliases)
           end
         dal = Array(d[:after_load])
         dal << proc do |m, descs|
@@ -248,7 +259,7 @@ module Sequel
         end
         d[:after_load] = dal
         d[:eager_loader] ||= proc do |eo|
-          id_map = eo[:key_hash][prkey]
+          id_map = eo[:id_map]
           associations = eo[:associations]
           parent_map = {}
           children_map = {}
@@ -259,9 +270,9 @@ module Sequel
           end
           r = model.association_reflection(descendants)
           base_case = model.filter(key=>id_map.keys).
-           select(SQL::AliasedExpression.new(key, ka), c_all)
+           select(SQL::AliasedExpression.new(key, ka), *c_all)
           recursive_case = model.join(t, prkey=>key).
-           select(SQL::QualifiedIdentifier.new(t, ka), c_all)
+           select(SQL::QualifiedIdentifier.new(t, ka), *c_all)
           if c = r[:conditions]
             (base_case, recursive_case) = [base_case, recursive_case].collect do |ds|
               (c.is_a?(Array) && !Sequel.condition_specifier?(c)) ? ds.filter(*c) : ds.filter(c)
@@ -276,7 +287,8 @@ module Sequel
           end
           table_alias = model.dataset.schema_and_table(model.table_name)[1].to_sym
           elds = model.eager_loading_dataset(r,
-           model.from(t => table_alias).with_recursive(t, base_case, recursive_case),
+           model.from(t => table_alias).with_recursive(t, base_case, recursive_case,
+            :args=>(([ka] + col_aliases + (level ? [la] : [])) if col_aliases)),
            r.select,
            associations, eo)
           elds = elds.select_append(ka) unless elds.opts[:select] == nil
@@ -296,7 +308,9 @@ module Sequel
               parent_map[opk] = obj
             end
             
-            if root = id_map[obj.values.delete(ka)].first
+            kv = obj.values.delete(ka)
+            kv = kv.to_i if conv_bd && kv.is_a?(BigDecimal)
+            if root = id_map[kv].first
               root.associations[descendants] << obj
             end
             

data/lib/sequel/plugins/schema.rb

@@ -29,7 +29,7 @@ module Sequel
         # Drops the table if it exists and then runs create_table.  Should probably
         # not be used except in testing.
         def create_table!(*args, &block)
-          drop_table rescue nil
+          drop_table?
           create_table(*args, &block)
         end
 
@@ -38,11 +38,16 @@ module Sequel
           create_table(*args, &block) unless table_exists?
         end
         
-        # Drops table.
+        # Drops table.  If the table doesn't exist, this will probably raise an error.
         def drop_table
           db.drop_table(table_name)
         end
     
+        # Drops table if it already exists, do nothing if it doesn't exist.
+        def drop_table?
+          db.drop_table?(table_name)
+        end
+    
         # Returns table schema created with set_schema for direct descendant of Model.
         # Does not retreive schema information from the database, see db_schema if you
         # want that.
@@ -61,7 +66,7 @@ module Sequel
         # test code or simple examples.
         def set_schema(name = nil, &block)
           set_dataset(db[name]) if name
-          @schema = Sequel::Schema::Generator.new(db, &block)
+          @schema = db.create_table_generator(&block)
           set_primary_key(@schema.primary_key_name) if @schema.primary_key_name
         end
         

data/lib/sequel/plugins/serialization.rb

@@ -11,8 +11,17 @@ module Sequel
     # set the @deserialized_values entry.  This plugin adds a before_save hook
     # that serializes all @deserialized_values to @values.
     #
-    # You can use either marshal, yaml, or json as the serialization format.
-    # If you use yaml or json, you should require them by yourself.
+    # You can specify the serialization format as a pair of serializer/deserializer
+    # callable objects.  You can also specify the serialization format as a single
+    # symbol, if such a symbol has a registered serializer/deserializer pair in the
+    # plugin.  By default, the plugin registers the :marshal, :yaml, and :json
+    # serialization formats.  To register your own serialization formats, use
+    # Sequel::Plugins::Serialization.register_format.
+    # If you use yaml or json format, you need to require the libraries, Sequel
+    # does not do the requiring for you.
+    #
+    # You can specify the columns to serialize when loading the plugin, or later
+    # using the serialize_attributes class method.
     #
     # Because of how this plugin works, it must be used inside each model class
     # that needs serialization, after any set_dataset method calls in that class.
@@ -22,56 +31,96 @@ module Sequel
     # == Example
     #
     #   require 'sequel'
+    #   # Require json, as the plugin doesn't require it for you.
     #   require 'json'
+    #
+    #   # Register custom serializer/deserializer pair
+    #   Sequel::Plugins::Serialization.register_format(:reverse,
+    #     lambda{|v| v.reverse},
+    #     lambda{|v| v.reverse})
+    #
     #   class User < Sequel::Model
+    #     # Built-in format support when loading the plugin
     #     plugin :serialization, :json, :permissions
-    #     # or
+    #
+    #     # Built-in format support after loading the plugin using serialize_attributes
     #     plugin :serialization
-    #     serialize_attributes :marshal, :permissions, :attributes
+    #     serialize_attributes :marshal, :permissions
+    #
+    #     # Use custom registered serialization format just like built-in format
+    #     serialize_attributes :reverse, :password
+    #
+    #     # Use a custom serializer/deserializer pair without registering
+    #     serialize_attributes [lambda{|v| v.reverse}, lambda{|v| v.reverse}], :password
     #   end
     #   user = User.create
     #   user.permissions = { :global => 'read-only' }
     #   user.save
     module Serialization
+      # The default serializers supported by the serialization module.
+      # Use register_format to add serializers to this hash.
+      REGISTERED_FORMATS = {}
+
       # Set up the column readers to do deserialization and the column writers
       # to save the value in deserialized_values.
       def self.apply(model, *args)
-        model.instance_eval{@serialization_map = {}}
+        model.instance_eval do
+          @deserialization_map = {}
+          @serialization_map = {}
+        end
       end
       
+      # Automatically call serialize_attributes with the format and columns unless
+      # no columns were provided.
       def self.configure(model, format=nil, *columns)
         model.serialize_attributes(format, *columns) unless columns.empty?
       end
 
+      # Register a serializer/deserializer pair with a format symbol, to allow
+      # models to pick this format by name.  Both serializer and deserializer
+      # should be callable objects.
+      def self.register_format(format, serializer, deserializer)
+        REGISTERED_FORMATS[format] = [serializer, deserializer]
+      end
+      register_format(:marshal, lambda{|v| [Marshal.dump(v)].pack('m')}, lambda{|v| Marshal.load(v.unpack('m')[0]) rescue Marshal.load(v)})
+      register_format(:yaml, lambda{|v| v.to_yaml}, lambda{|v| YAML.load(v)})
+      register_format(:json, lambda{|v| v.to_json}, lambda{|v| JSON.parse(v)})
+
       module ClassMethods
-        # A map of the serialized columns for this model.  Keys are column
-        # symbols, values are serialization formats (:marshal, :yaml, or :json).
+        # A hash with column name symbols and callable values, with the value
+        # called to deserialize the column.
+        attr_reader :deserialization_map
+
+        # A hash with column name symbols and callable values, with the value
+        # called to serialize the column.
         attr_reader :serialization_map
 
         # Module to store the serialized column accessor methods, so they can
         # call be overridden and call super to get the serialization behavior
         attr_accessor :serialization_module
 
-        # Copy the serialization format and columns to serialize into the subclass.
+        # Copy the serialization_map and deserialization map into the subclass.
         def inherited(subclass)
           super
           sm = serialization_map.dup
-          subclass.instance_eval{@serialization_map = sm}
-        end
-        
-        # The first value in the serialization map.  This is only for
-        # backwards compatibility, use serialization_map in new code.
-        def serialization_format
-          serialization_map.values.first
+          dsm = deserialization_map.dup
+          subclass.instance_eval do
+            @deserialization_map = dsm
+            @serialization_map = sm
+          end
         end
         
         # Create instance level reader that deserializes column values on request,
-        # and instance level writer that stores new deserialized value in deserialized
-        # columns
+        # and instance level writer that stores new deserialized values.
         def serialize_attributes(format, *columns)
-          raise(Error, "Unsupported serialization format (#{format}), should be :marshal, :yaml, or :json") unless [:marshal, :yaml, :json].include?(format)
+          if format.is_a?(Symbol)
+            unless format = REGISTERED_FORMATS[format]
+              raise(Error, "Unsupported serialization format: #{format} (valid formats: #{REGISTERED_FORMATS.keys.map{|k| k.inspect}.join})")
+            end
+          end
+          serializer, deserializer = format
           raise(Error, "No columns given.  The serialization plugin requires you specify which columns to serialize") if columns.empty?
-          define_serialized_attribute_accessor(format, *columns)
+          define_serialized_attribute_accessor(serializer, deserializer, *columns)
         end
         
         # The columns that will be serialized.  This is only for
@@ -83,12 +132,13 @@ module Sequel
         private
 
         # Add serializated attribute acessor methods to the serialization_module
-        def define_serialized_attribute_accessor(format, *columns)
+        def define_serialized_attribute_accessor(serializer, deserializer, *columns)
           m = self
           include(self.serialization_module ||= Module.new) unless serialization_module
           serialization_module.class_eval do
             columns.each do |column|
-              m.serialization_map[column] = format
+              m.serialization_map[column] = serializer
+              m.deserialization_map[column] = deserializer
               define_method(column) do 
                 if deserialized_values.has_key?(column)
                   deserialized_values[column]
@@ -110,52 +160,51 @@ module Sequel
         attr_reader :deserialized_values
 
         # Set @deserialized_values to the empty hash
-        def initialize(*args, &block)
+        def initialize_set(values)
           @deserialized_values = {}
           super
         end
 
-        # Serialize all deserialized values
+        # Serialize deserialized values before saving
         def before_save
-          deserialized_values.each{|k,v| @values[k] = serialize_value(k, v)}
+          serialize_deserialized_values
           super
         end
         
-        # Empty the deserialized values when refreshing.
-        def refresh
-          @deserialized_values = {}
+        # Initialization the deserialized values for objects retrieved from the database.
+        def set_values(*)
+          @deserialized_values ||= {}
           super
         end
 
         private
 
-        # Deserialize the column from either marshal or yaml format
+        # Empty the deserialized values when refreshing.
+        def _refresh(*)
+          @deserialized_values = {}
+          super
+        end
+
+        # Deserialize the column value.  Called when the model column accessor is called to
+        # return a deserialized value.
         def deserialize_value(column, v)
-          return v if v.nil?
-          case model.serialization_map[column] 
-          when :marshal
-            Marshal.load(v.unpack('m')[0]) rescue Marshal.load(v)
-          when :yaml
-            YAML.load v if v
-          when :json
-            JSON.parse v if v
-          else
-            raise Error, "Bad serialization format (#{model.serialization_map[column].inspect}) for column #{column.inspect}"
+          unless v.nil?
+            raise Sequel::Error, "no entry in deserialization_map for #{column.inspect}" unless callable = model.deserialization_map[column]
+            callable.call(v)
           end
         end
 
-        # Serialize the column to either marshal or yaml format
+        # Serialize all deserialized values
+        def serialize_deserialized_values
+          deserialized_values.each{|k,v| @values[k] = serialize_value(k, v)}
+        end
+
+        # Serialize the column value.  Called before saving to ensure the serialized value
+        # is saved in the database.
         def serialize_value(column, v)
-          return v if v.nil?
-          case model.serialization_map[column] 
-          when :marshal
-            [Marshal.dump(v)].pack('m')
-          when :yaml
-            v.to_yaml
-          when :json
-            v.to_json
-          else
-            raise Error, "Bad serialization format (#{model.serialization_map[column].inspect}) for column #{column.inspect}"
+          unless v.nil?
+            raise Sequel::Error, "no entry in serialization_map for #{column.inspect}" unless callable = model.serialization_map[column]
+            callable.call(v)
           end
         end
       end