sequel 3.21.0 → 3.41.0

data/lib/sequel/plugins/prepared_statements.rb

@@ -0,0 +1,168 @@
+module Sequel
+  module Plugins
+    # The prepared_statements plugin modifies the model to use prepared statements for
+    # instance level deletes and saves, as well as class level lookups by
+    # primary key.
+    #
+    # Note that this plugin is unsafe in some circumstances, as it can allow up to
+    # 2^N prepared statements to be created for each type of insert and update query, where
+    # N is the number of colums in the table. It is recommended that you use the
+    # +prepared_statements_safe+ plugin in addition to this plugin to reduce the number
+    # of prepared statements that can be created, unless you tightly control how your
+    # model instances are saved.
+    # 
+    # This plugin does not work correctly with the instance filters plugin
+    # or the update_primary_key plugin.
+    # 
+    # Usage:
+    #
+    #   # Make all model subclasses use prepared statements  (called before loading subclasses)
+    #   Sequel::Model.plugin :prepared_statements
+    #
+    #   # Make the Album class use prepared statements
+    #   Album.plugin :prepared_statements
+    module PreparedStatements
+      # Synchronize access to the integer sequence so that no two calls get the same integer.
+      MUTEX = Mutex.new
+      
+      i = 0
+      # This plugin names prepared statements uniquely using an integer sequence, this
+      # lambda returns the next integer to use.
+      NEXT = lambda{MUTEX.synchronize{i += 1}}
+
+      # Setup the datastructure used to hold the prepared statements in the model.
+      def self.apply(model)
+        model.instance_variable_set(:@prepared_statements, :insert=>{}, :insert_select=>{}, :update=>{}, :lookup_sql=>{}, :fixed=>{})
+      end
+
+      module ClassMethods
+        # Setup the datastructure used to hold the prepared statements in the subclass.
+        def inherited(subclass)
+          super
+          subclass.instance_variable_set(:@prepared_statements, :insert=>{}, :insert_select=>{}, :update=>{}, :lookup_sql=>{}, :fixed=>{})
+        end
+
+        private
+
+        # Create a prepared statement based on the given dataset with a unique name for the given
+        # type of query and values.
+        def prepare_statement(ds, type, vals={})
+          ps = ds.prepare(type, :"smpsp_#{NEXT.call}", vals)
+          ps.log_sql = true
+          ps
+        end
+
+        # Return a sorted array of columns for use as a hash key.
+        def prepared_columns(cols)
+          RUBY_VERSION >= '1.9' ? cols.sort : cols.sort_by{|c| c.to_s}
+        end
+
+        # Return a prepared statement that can be used to delete a row from this model's dataset.
+        def prepared_delete
+          cached_prepared_statement(:fixed, :delete){prepare_statement(filter(prepared_statement_key_array(primary_key)), :delete)}
+        end
+
+        # Return a prepared statement that can be used to insert a row using the given columns.
+        def prepared_insert(cols)
+          cached_prepared_statement(:insert, prepared_columns(cols)){prepare_statement(dataset, :insert, prepared_statement_key_hash(cols))}
+        end
+
+        # Return a prepared statement that can be used to insert a row using the given columns
+        # and return that column values for the row created.
+        def prepared_insert_select(cols)
+          if dataset.supports_insert_select?
+            cached_prepared_statement(:insert_select, prepared_columns(cols)){prepare_statement(naked.clone(:server=>dataset.opts.fetch(:server, :default)), :insert_select, prepared_statement_key_hash(cols))}
+          end
+        end
+
+        # Return a prepared statement that can be used to lookup a row solely based on the primary key.
+        def prepared_lookup
+          cached_prepared_statement(:fixed, :lookup){prepare_statement(filter(prepared_statement_key_array(primary_key)), :first)}
+        end
+
+        # Return a prepared statement that can be used to refresh a row to get new column values after insertion.
+        def prepared_refresh
+          cached_prepared_statement(:fixed, :refresh){prepare_statement(naked.clone(:server=>dataset.opts.fetch(:server, :default)).filter(prepared_statement_key_array(primary_key)), :first)}
+        end
+
+        # Return an array of two element arrays with the column symbol as the first entry and the
+        # placeholder symbol as the second entry.
+        def prepared_statement_key_array(keys)
+          if dataset.requires_placeholder_type_specifiers?
+            sch = db_schema
+            Array(keys).map do |k|
+              if (s = sch[k]) && (t = s[:type])
+                [k, :"$#{k}__#{t}"]
+              else
+                [k, :"$#{k}"]
+              end
+            end
+          else
+            Array(keys).map{|k| [k, :"$#{k}"]}
+          end
+        end
+
+        # Return a hash mapping column symbols to placeholder symbols.
+        def prepared_statement_key_hash(keys)
+          Hash[*(prepared_statement_key_array(keys).flatten)]
+        end
+
+        # Return a prepared statement that can be used to update row using the given columns.
+        def prepared_update(cols)
+          cached_prepared_statement(:update, prepared_columns(cols)){prepare_statement(filter(prepared_statement_key_array(primary_key)), :update, prepared_statement_key_hash(cols))}
+        end
+
+        # Use a prepared statement to query the database for the row matching the given primary key.
+        def primary_key_lookup(pk)
+          prepared_lookup.call(primary_key_hash(pk))
+        end
+
+        private
+
+        # If a prepared statement has already been cached for the given type and subtype,
+        # return it.  Otherwise, yield to the block to get the prepared statement, and cache it.
+        def cached_prepared_statement(type, subtype)
+          h = @prepared_statements[type]
+          Sequel.synchronize do
+            if v = h[subtype]
+              return v end
+          end
+          ps = yield
+          Sequel.synchronize{h[subtype] = ps}
+        end
+      end
+
+      module InstanceMethods
+        private
+
+        # Use a prepared statement to delete the row.
+        def _delete_without_checking
+          model.send(:prepared_delete).call(pk_hash)
+        end
+
+        # Use a prepared statement to insert the values into the model's dataset.
+        def _insert_raw(ds)
+          model.send(:prepared_insert, @values.keys).call(@values)
+        end
+
+        # Use a prepared statement to insert the values into the model's dataset
+        # and return the new column values.
+        def _insert_select_raw(ds)
+          if ps = model.send(:prepared_insert_select, @values.keys)
+            ps.call(@values)
+          end
+        end
+
+        # Use a prepared statement to refresh this model's column values.
+        def _refresh_get(ds)
+          model.send(:prepared_refresh).call(pk_hash)
+        end
+
+        # Use a prepared statement to update this model's columns in the database.
+        def _update_without_checking(columns)
+          model.send(:prepared_update, columns.keys).call(columns.merge(pk_hash))
+        end
+      end
+    end
+  end
+end

data/lib/sequel/plugins/prepared_statements_associations.rb

@@ -0,0 +1,87 @@
+module Sequel
+  module Plugins
+    # The prepared_statements_associations plugin modifies the regular association
+    # load method to use a cached prepared statement to load the associations.
+    # It will not work on all associations, but it should skip the use of prepared
+    # statements for associations where it will not work, assuming you load the
+    # plugin before defining the associations.
+    #
+    # Usage:
+    #
+    #   # Make all model subclasses more safe when using prepared statements (called before loading subclasses)
+    #   Sequel::Model.plugin :prepared_statements_associations
+    #
+    #   # Make the Album class more safe when using prepared statements
+    #   Album.plugin :prepared_statements_associations
+    module PreparedStatementsAssociations
+      # Synchronize access to the integer sequence so that no two calls get the same integer.
+      MUTEX = Mutex.new
+
+      i = 0
+      # This plugin names prepared statements uniquely using an integer sequence, this
+      # lambda returns the next integer to use.
+      NEXT = lambda{MUTEX.synchronize{i += 1}}
+
+      module ClassMethods
+        # Disable prepared statement use if a block is given, or the :dataset or :conditions
+        # options are used, or you are cloning an association.
+        def associate(type, name, opts = {}, &block)
+          if block || opts[:dataset] || opts[:conditions] || (opts[:clone] && association_reflection(opts[:clone])[:prepared_statement] == false)
+            opts = opts.merge(:prepared_statement=>false)
+          end
+          super(type, name, opts, &block)
+        end
+      end
+
+      module InstanceMethods
+        private
+
+        # Return a bound variable hash that maps the keys in +ks+ (qualified by the +table+)
+        # to the values of the results of sending the methods in +vs+.
+        def association_bound_variable_hash(table, ks, vs)
+          Hash[*ks.zip(vs).map{|k, v| [:"#{table}.#{k}", send(v)]}.flatten]
+        end
+
+        # Given an association reflection, return a bound variable hash for the given
+        # association for this instance's values.
+        def association_bound_variables(opts)
+          case opts[:type]
+          when :many_to_one
+            association_bound_variable_hash(opts.associated_class.table_name, opts.primary_keys, opts[:keys])
+          when :one_to_many
+            association_bound_variable_hash(opts.associated_class.table_name, opts[:keys], opts[:primary_keys])
+          when :many_to_many
+            association_bound_variable_hash(opts.join_table_alias, opts[:left_keys], opts[:left_primary_keys])
+          when :many_through_many
+            association_bound_variable_hash(opts.final_reverse_edge[:alias], Array(opts[:left_key]), opts[:left_primary_keys])
+          end
+        end
+
+        # Given an association reflection, return and cache a prepared statement for this association such
+        # that, given appropriate bound variables, the prepared statement will work correctly for any
+        # instance.
+        def association_prepared_statement(opts)
+          opts.send(:cached_fetch, :prepared_statement) do
+            ps = _associated_dataset(opts, {}).unbind.first.prepare(opts.returns_array? ? :select : :first, :"smpsap_#{NEXT.call}")
+            ps.log_sql = true
+            ps
+          end
+        end
+
+        # If a prepared statement can be used to load the associated objects, execute it to retrieve them.  Otherwise,
+        # fall back to the default implementation.
+        def _load_associated_objects(opts, dynamic_opts={})
+          if !opts.can_have_associated_objects?(self) || dynamic_opts[:callback] || (ps = opts[:prepared_statement]) == false
+            super
+          else 
+            if bv = association_bound_variables(opts)
+              (ps || association_prepared_statement(opts)).call(bv)
+            else
+              super
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sequel/plugins/prepared_statements_safe.rb

@@ -0,0 +1,82 @@
+module Sequel
+  module Plugins
+    # The prepared_statements_safe plugin modifies the model to reduce the number of
+    # prepared statements that can be created, by setting as many columns as possible
+    # before creating, and by changing +save_changes+ to save all columns instead of
+    # just the changed ones.
+    #
+    # This plugin depends on the +prepared_statements+ plugin.
+    # 
+    # Usage:
+    #
+    #   # Make all model subclasses more safe when using prepared statements (called before loading subclasses)
+    #   Sequel::Model.plugin :prepared_statements_safe
+    #
+    #   # Make the Album class more safe when using prepared statements
+    #   Album.plugin :prepared_statements_safe
+    module PreparedStatementsSafe
+      # Depend on the prepared_statements plugin
+      def self.apply(model)
+        model.plugin(:prepared_statements)
+      end
+
+      # Set the column defaults to use when creating on the model.
+      def self.configure(model)
+        model.send(:set_prepared_statements_column_defaults)
+      end
+
+      module ClassMethods
+        # A hash with column symbol keys and default values.  Instance's
+        # values are merged into this hash before creating to reduce the
+        # number of free columns (columns that may or may not be present
+        # in the INSERT statement), as the number of prepared statements
+        # that can be created is 2^N (where N is the number of free columns).
+        attr_reader :prepared_statements_column_defaults
+        
+        def inherited(subclass)
+          super
+          subclass.instance_variable_set(:@prepared_statements_column_defaults, @prepared_statements_column_defaults) if @prepared_statements_column_defaults && !subclass.prepared_statements_column_defaults
+        end
+
+        # Set the column defaults to use when creating on the subclass.
+        def set_dataset(*)
+          x = super
+          set_prepared_statements_column_defaults
+          x
+        end
+
+        private
+
+        # Set the column defaults based on the database schema.  All columns
+        # are set to a default value unless they are a primary key column or
+        # they don't have a parseable default.
+        def set_prepared_statements_column_defaults
+          if db_schema
+            h = {}
+            db_schema.each do |k, v|
+              h[k] = v[:ruby_default] if (v[:ruby_default] || !v[:default]) && !v[:primary_key]
+            end
+            @prepared_statements_column_defaults = h
+          end
+        end
+      end
+
+      module InstanceMethods
+        # Merge the current values into the default values to reduce the number
+        # of free columns.
+        def before_create
+          if v = model.prepared_statements_column_defaults
+            set_values(v.merge(values))
+          end
+          super
+        end
+
+        # Always do a full save of all columns to reduce the number of prepared
+        # statements that can be used.
+        def save_changes(opts={})
+          save(opts) || false if modified?
+        end
+      end
+    end
+  end
+end

data/lib/sequel/plugins/prepared_statements_with_pk.rb

@@ -0,0 +1,59 @@
+module Sequel
+  module Plugins
+    # The prepared_statements_with_pk plugin allows Dataset#with_pk for model datasets
+    # to use prepared statements by extract the values of previously bound variables
+    # using <tt>Dataset#unbind</tt>, and attempting to use a prepared statement if the
+    # variables can be unbound correctly. See  +Unbinder+ for details about what types of
+    # dataset filters can be unbound correctly.
+    #
+    # This plugin depends on the +prepared_statements+ plugin and should be considered unsafe.
+    # Unbinding dataset values cannot be done correctly in all cases, and use of this plugin
+    # in cases where not there are variables that are not unbound can lead to an denial of
+    # service attack by allocating an arbitrary number of prepared statements.  You have been
+    # warned.
+    #
+    # Usage:
+    #
+    #   # Make all model subclasses use prepared statements for Dataset#with_pk (called before loading subclasses)
+    #   Sequel::Model.plugin :prepared_statements_with_pk
+    #
+    #   # Make the Album class use prepared statements for Dataset#with_pk
+    #   Album.plugin :prepared_statements_with_pk
+    module PreparedStatementsWithPk
+      # Depend on the prepared_statements plugin
+      def self.apply(model)
+        model.plugin(:prepared_statements)
+      end
+
+      module ClassMethods
+        private
+
+        # Return a prepared statement that can be used to lookup a row given a dataset for the row matching
+        # the primary key.
+        def prepared_lookup_dataset(ds)
+          cached_prepared_statement(:lookup_sql, ds.sql){prepare_statement(ds.filter(prepared_statement_key_array(primary_key).map{|k, v| [SQL::QualifiedIdentifier.new(ds.model.table_name, k), v]}), :first)}
+        end
+      end
+
+      module DatasetMethods
+        # Use a prepared statement to find a row with the matching primary key
+        # inside this dataset.
+        def with_pk(pk)
+          begin
+            ds, bv = unbind
+          rescue UnbindDuplicate
+            super
+          else
+            begin
+              bv = bv.merge!(model.primary_key_hash(pk)){|k, v1, v2| ((v1 == v2) ? v1 : raise(UnbindDuplicate))}
+            rescue UnbindDuplicate
+              super
+            else
+              model.send(:prepared_lookup_dataset, ds).call(bv)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sequel/plugins/rcte_tree.rb

@@ -109,7 +109,15 @@ module Sequel
         ka = opts[:key_alias] ||= :x_root_x
         t = opts[:cte_name] ||= :t
         opts[:reciprocal] = nil
-        c_all = SQL::ColumnAll.new(model.table_name)
+        c_all = if model.dataset.recursive_cte_requires_column_aliases?
+          # Work around Oracle/ruby-oci8 bug that returns integers as BigDecimals in recursive queries.
+          conv_bd = model.db.database_type == :oracle
+          col_aliases = model.dataset.columns
+          model_table = model.table_name
+          col_aliases.map{|c| SQL::QualifiedIdentifier.new(model_table, c)}
+        else
+          [SQL::ColumnAll.new(model.table_name)]
+        end
         
         a = opts.merge(opts.fetch(:ancestors, {}))
         ancestors = a.fetch(:name, :ancestors)
@@ -125,9 +133,9 @@ module Sequel
           end
           table_alias = model.dataset.schema_and_table(model.table_name)[1].to_sym
           model.from(t => table_alias).
-           with_recursive(t, base_ds.select_all,
-            recursive_ds.
-            select(c_all))
+           with_recursive(t, col_aliases ? base_ds.select(*col_aliases) : base_ds.select_all,
+            recursive_ds.select(*c_all),
+            :args=>col_aliases)
         end
         aal = Array(a[:after_load])
         aal << proc do |m, ancs|
@@ -152,7 +160,7 @@ module Sequel
         end
         a[:after_load] ||= aal
         a[:eager_loader] ||= proc do |eo|
-          id_map = eo[:key_hash][key]
+          id_map = eo[:id_map]
           parent_map = {}
           children_map = {}
           eo[:rows].each do |obj|
@@ -163,9 +171,9 @@ module Sequel
           end
           r = model.association_reflection(ancestors)
           base_case = model.filter(prkey=>id_map.keys).
-           select(SQL::AliasedExpression.new(prkey, ka), c_all)
+           select(SQL::AliasedExpression.new(prkey, ka), *c_all)
           recursive_case = model.join(t, key=>prkey).
-           select(SQL::QualifiedIdentifier.new(t, ka), c_all)
+           select(SQL::QualifiedIdentifier.new(t, ka), *c_all)
           if c = r[:conditions]
             (base_case, recursive_case) = [base_case, recursive_case].collect do |ds|
               (c.is_a?(Array) && !Sequel.condition_specifier?(c)) ? ds.filter(*c) : ds.filter(c)
@@ -175,7 +183,8 @@ module Sequel
           elds = model.eager_loading_dataset(r,
            model.from(t => table_alias).
             with_recursive(t, base_case,
-             recursive_case),
+             recursive_case,
+             :args=>(([ka] + col_aliases) if col_aliases)),
            r.select,
            eo[:associations], eo)
           elds = elds.select_append(ka) unless elds.opts[:select] == nil
@@ -192,7 +201,9 @@ module Sequel
               (children_map[obj[key]] ||= []) << obj
             end
             
-            if roots = id_map[obj.values.delete(ka)]
+            kv = obj.values.delete(ka)
+            kv = kv.to_i if conv_bd && kv.is_a?(BigDecimal)
+            if roots = id_map[kv]
               roots.each do |root|
                 root.associations[ancestors] << obj
               end
@@ -222,9 +233,9 @@ module Sequel
           end
           table_alias = model.dataset.schema_and_table(model.table_name)[1].to_sym
           model.from(t => table_alias).
-           with_recursive(t, base_ds.select_all,
-            recursive_ds.
-            select(SQL::ColumnAll.new(model.table_name)))
+           with_recursive(t, col_aliases ? base_ds.select(*col_aliases) : base_ds.select_all,
+            recursive_ds.select(*c_all),
+            :args=>col_aliases)
           end
         dal = Array(d[:after_load])
         dal << proc do |m, descs|
@@ -248,7 +259,7 @@ module Sequel
         end
         d[:after_load] = dal
         d[:eager_loader] ||= proc do |eo|
-          id_map = eo[:key_hash][prkey]
+          id_map = eo[:id_map]
           associations = eo[:associations]
           parent_map = {}
           children_map = {}
@@ -259,9 +270,9 @@ module Sequel
           end
           r = model.association_reflection(descendants)
           base_case = model.filter(key=>id_map.keys).
-           select(SQL::AliasedExpression.new(key, ka), c_all)
+           select(SQL::AliasedExpression.new(key, ka), *c_all)
           recursive_case = model.join(t, prkey=>key).
-           select(SQL::QualifiedIdentifier.new(t, ka), c_all)
+           select(SQL::QualifiedIdentifier.new(t, ka), *c_all)
           if c = r[:conditions]
             (base_case, recursive_case) = [base_case, recursive_case].collect do |ds|
               (c.is_a?(Array) && !Sequel.condition_specifier?(c)) ? ds.filter(*c) : ds.filter(c)
@@ -276,7 +287,8 @@ module Sequel
           end
           table_alias = model.dataset.schema_and_table(model.table_name)[1].to_sym
           elds = model.eager_loading_dataset(r,
-           model.from(t => table_alias).with_recursive(t, base_case, recursive_case),
+           model.from(t => table_alias).with_recursive(t, base_case, recursive_case,
+            :args=>(([ka] + col_aliases + (level ? [la] : [])) if col_aliases)),
            r.select,
            associations, eo)
           elds = elds.select_append(ka) unless elds.opts[:select] == nil
@@ -296,7 +308,9 @@ module Sequel
               parent_map[opk] = obj
             end
             
-            if root = id_map[obj.values.delete(ka)].first
+            kv = obj.values.delete(ka)
+            kv = kv.to_i if conv_bd && kv.is_a?(BigDecimal)
+            if root = id_map[kv].first
               root.associations[descendants] << obj
             end
             

data/lib/sequel/plugins/schema.rb

@@ -29,7 +29,7 @@ module Sequel
         # Drops the table if it exists and then runs create_table.  Should probably
         # not be used except in testing.
         def create_table!(*args, &block)
-          drop_table rescue nil
+          drop_table?
           create_table(*args, &block)
         end
 
@@ -38,11 +38,16 @@ module Sequel
           create_table(*args, &block) unless table_exists?
         end
         
-        # Drops table.
+        # Drops table.  If the table doesn't exist, this will probably raise an error.
         def drop_table
           db.drop_table(table_name)
         end
     
+        # Drops table if it already exists, do nothing if it doesn't exist.
+        def drop_table?
+          db.drop_table?(table_name)
+        end
+    
         # Returns table schema created with set_schema for direct descendant of Model.
         # Does not retreive schema information from the database, see db_schema if you
         # want that.
@@ -61,7 +66,7 @@ module Sequel
         # test code or simple examples.
         def set_schema(name = nil, &block)
           set_dataset(db[name]) if name
-          @schema = Sequel::Schema::Generator.new(db, &block)
+          @schema = db.create_table_generator(&block)
           set_primary_key(@schema.primary_key_name) if @schema.primary_key_name
         end