sequel 5.68.0 → 5.77.0

data/lib/sequel/model/exceptions.rb

@@ -24,11 +24,23 @@ module Sequel
   UndefinedAssociation = Class.new(Error)
   ).name
 
-  (
   # Raised when a mass assignment method is called in strict mode with either a restricted column
   # or a column without a setter method.
-  MassAssignmentRestriction = Class.new(Error)
-  ).name
+  class MassAssignmentRestriction < Error
+    # The Sequel::Model object related to this exception.
+    attr_reader :model
+
+    # The column related to this exception, as a string.
+    attr_reader :column
+
+    # Create an instance of this class with the model and column set.
+    def self.create(msg, model, column)
+      e = new("#{msg} for class #{model.class.inspect}")
+      e.instance_variable_set(:@model, model)
+      e.instance_variable_set(:@column, column)
+      e
+    end
+  end
   
   # Exception class raised when +raise_on_save_failure+ is set and validation fails
   class ValidationFailed < Error

data/lib/sequel/plugins/column_encryption.rb

@@ -31,7 +31,6 @@ rescue RuntimeError, OpenSSL::Cipher::CipherError
   # :nocov:
 end
 
-require 'base64'
 require 'securerandom'
 
 module Sequel
@@ -326,7 +325,7 @@ module Sequel
     #   DB.alter_table(:ce_test) do
     #     c = Sequel[:encrypted_column_name]
     #     add_constraint(:enc_base64) do
-    #       octet_length(decode(regexp_replace(regexp_replace(c, '_', '/', 'g'), '-', '+', 'g'), 'base64')) >= 65}
+    #       octet_length(decode(regexp_replace(regexp_replace(c, '_', '/', 'g'), '-', '+', 'g'), 'base64')) >= 65
     #     end
     #   end
     #
@@ -375,7 +374,7 @@ module Sequel
         # Decrypt using any supported format and any available key.
         def decrypt(data)
           begin
-            data = Base64.urlsafe_decode64(data)
+            data = urlsafe_decode64(data)
           rescue ArgumentError
             raise Error, "Unable to decode encrypted column: invalid base64"
           end
@@ -448,7 +447,7 @@ module Sequel
         # The prefix string of columns for the given search type and the first configured encryption key.
         # Used to find values that do not use this prefix in order to perform reencryption.
         def current_key_prefix(search_type)
-          Base64.urlsafe_encode64("#{search_type.chr}\0#{@key_id.chr}")
+          urlsafe_encode64("#{search_type.chr}\0#{@key_id.chr}")
         end
 
         # The prefix values to search for the given data (an array of strings), assuming the column uses
@@ -472,11 +471,33 @@ module Sequel
 
         private
 
+        if RUBY_VERSION >= '2.4'
+          def decode64(str)
+            str.unpack1("m0")
+          end
+        # :nocov:
+        else
+          def decode64(str)
+            str.unpack("m0")[0]
+          end
+        # :nocov:
+        end
+
+        def urlsafe_encode64(bin)
+          str = [bin].pack("m0")
+          str.tr!("+/", "-_")
+          str
+        end
+
+        def urlsafe_decode64(str)
+          decode64(str.tr("-_", "+/"))
+        end
+
         # An array of strings, one for each configured encryption key, to find encypted values matching
         # the given data and search format.
         def _search_prefixes(data, search_type)
           @key_map.map do |key_id, (key, _)|
-            Base64.urlsafe_encode64(_search_prefix(data, search_type, key_id, key))
+            urlsafe_encode64(_search_prefix(data, search_type, key_id, key))
           end
         end
 
@@ -509,7 +530,7 @@ module Sequel
           cipher_text << cipher.update(data) if data_size > 0
           cipher_text << cipher.final
 
-          Base64.urlsafe_encode64("#{prefix}#{random_data}#{cipher_iv}#{cipher.auth_tag}#{cipher_text}")
+          urlsafe_encode64("#{prefix}#{random_data}#{cipher_iv}#{cipher.auth_tag}#{cipher_text}")
         end
       end
 

data/lib/sequel/plugins/defaults_setter.rb

@@ -1,5 +1,7 @@
 # frozen-string-literal: true
 
+require 'delegate'
+
 module Sequel
   module Plugins
     # The defaults_setter plugin makes the column getter methods return the default
@@ -106,6 +108,20 @@ module Sequel
             lambda{Date.today}
           when Sequel::CURRENT_TIMESTAMP
             lambda{dataset.current_datetime}
+          when Hash, Array
+            v = Marshal.dump(v).freeze
+            lambda{Marshal.load(v)}
+          when Delegator
+            # DelegateClass returns an anonymous case, which cannot be marshalled, so marshal the
+            # underlying object and create a new instance of the class with the unmarshalled object.
+            klass = v.class
+            case o = v.__getobj__
+            when Hash, Array
+              v = Marshal.dump(o).freeze
+              lambda{klass.new(Marshal.load(v))}
+            else
+              v
+            end
           else
             v
           end

data/lib/sequel/plugins/list.rb

@@ -185,10 +185,13 @@ module Sequel
         end
 
         # Set the value of the position_field to the maximum value plus 1 unless the
-        # position field already has a value.
+        # position field already has a value. If the list is empty, the position will
+        # be set to the model's +top_of_list+ value.
         def before_validation
           unless get_column_value(position_field)
-            set_column_value("#{position_field}=", list_dataset.max(position_field).to_i+1)
+            current_max = list_dataset.max(position_field)
+            value = current_max.nil? ? model.top_of_list : current_max.to_i + 1
+            set_column_value("#{position_field}=", value)
           end
           super
         end

data/lib/sequel/plugins/mssql_optimistic_locking.rb

@@ -26,57 +26,27 @@ module Sequel
     module MssqlOptimisticLocking
       # Load the instance_filters plugin into the model.
       def self.apply(model, opts=OPTS)
-        model.plugin :instance_filters
+        model.plugin(:optimistic_locking_base)
       end
 
-      # Set the lock_column to the :lock_column option (default: :timestamp)
+      # Set the lock column
       def self.configure(model, opts=OPTS)
-        model.lock_column = opts[:lock_column] || :timestamp
+        model.lock_column = opts[:lock_column] || model.lock_column || :timestamp
       end
-
-      module ClassMethods
-        # The timestamp/rowversion column containing the version for the current row.
-        attr_accessor :lock_column
-
-        Plugins.inherited_instance_variables(self, :@lock_column=>nil)
-      end
-
+      
       module InstanceMethods
-        # Add the lock column instance filter to the object before destroying it.
-        def before_destroy
-          lock_column_instance_filter
-          super
-        end
-        
-        # Add the lock column instance filter to the object before updating it.
-        def before_update
-          lock_column_instance_filter
-          super
-        end
-        
         private
         
-        # Add the lock column instance filter to the object.
-        def lock_column_instance_filter
-          lc = model.lock_column
-          instance_filter(lc=>Sequel.blob(get_column_value(lc)))
-        end
-
-        # Clear the instance filters when refreshing, so that attempting to
-        # refresh after a failed save removes the previous lock column filter
-        # (the new one will be added before updating).
-        def _refresh(ds)
-          clear_instance_filters
-          super
+        # Make the instance filter value a blob.
+        def lock_column_instance_filter_value
+          Sequel.blob(super)
         end
 
         # Remove the lock column from the columns to update.
         # SQL Server automatically updates the lock column value, and does not like
         # it to be assigned.
         def _save_update_all_columns_hash
-          v = @values.dup
-          cc = changed_columns
-          Array(primary_key).each{|x| v.delete(x) unless cc.include?(x)}
+          v = super
           v.delete(model.lock_column)
           v
         end

data/lib/sequel/plugins/optimistic_locking.rb

@@ -12,64 +12,31 @@ module Sequel
     #   p1 = Person[1]
     #   p2 = Person[1]
     #   p1.update(name: 'Jim') # works
-    #   p2.update(name: 'Bob') # raises Sequel::Plugins::OptimisticLocking::Error
+    #   p2.update(name: 'Bob') # raises Sequel::NoExistingObject
     #
     # In order for this plugin to work, you need to make sure that the database
-    # table has a +lock_version+ column (or other column you name via the lock_column
-    # class level accessor) that defaults to 0.
+    # table has a +lock_version+ column that defaults to 0. To change the column
+    # used, provide a +:lock_column+ option when loading the plugin:
+    #
+    #     plugin :optimistic_locking, lock_column: :version
     #
     # This plugin relies on the instance_filters plugin.
     module OptimisticLocking
       # Exception class raised when trying to update or destroy a stale object.
       Error = Sequel::NoExistingObject
       
-      # Load the instance_filters plugin into the model.
       def self.apply(model, opts=OPTS)
-        model.plugin :instance_filters
+        model.plugin(:optimistic_locking_base)
       end
 
-      # Set the lock_column to the :lock_column option, or :lock_version if
-      # that option is not given.
+      # Set the lock column
       def self.configure(model, opts=OPTS)
-        model.lock_column = opts[:lock_column] || :lock_version
+        model.lock_column = opts[:lock_column] || model.lock_column || :lock_version
       end
-      
-      module ClassMethods
-        # The column holding the version of the lock
-        attr_accessor :lock_column
-        
-        Plugins.inherited_instance_variables(self, :@lock_column=>nil)
-      end
-    
+
       module InstanceMethods
-        # Add the lock column instance filter to the object before destroying it.
-        def before_destroy
-          lock_column_instance_filter
-          super
-        end
-        
-        # Add the lock column instance filter to the object before updating it.
-        def before_update
-          lock_column_instance_filter
-          super
-        end
-        
         private
         
-        # Add the lock column instance filter to the object.
-        def lock_column_instance_filter
-          lc = model.lock_column
-          instance_filter(lc=>get_column_value(lc))
-        end
-
-        # Clear the instance filters when refreshing, so that attempting to
-        # refresh after a failed save removes the previous lock column filter
-        # (the new one will be added before updating).
-        def _refresh(ds)
-          clear_instance_filters
-          super
-        end
-        
         # Only update the row if it has the same lock version, and increment the
         # lock version.
         def _update_columns(columns)

data/lib/sequel/plugins/optimistic_locking_base.rb

@@ -0,0 +1,55 @@
+# frozen-string-literal: true
+
+module Sequel
+  module Plugins
+    # Base for other optimistic locking plugins
+    module OptimisticLockingBase
+      # Load the instance_filters plugin into the model.
+      def self.apply(model)
+        model.plugin :instance_filters
+      end
+
+      module ClassMethods
+        # The column holding the version of the lock
+        attr_accessor :lock_column
+        
+        Plugins.inherited_instance_variables(self, :@lock_column=>nil)
+      end
+    
+      module InstanceMethods
+        # Add the lock column instance filter to the object before destroying it.
+        def before_destroy
+          lock_column_instance_filter
+          super
+        end
+        
+        # Add the lock column instance filter to the object before updating it.
+        def before_update
+          lock_column_instance_filter
+          super
+        end
+        
+        private
+        
+        # Add the lock column instance filter to the object.
+        def lock_column_instance_filter
+          instance_filter(model.lock_column=>lock_column_instance_filter_value)
+        end
+
+        # Use the current value of the lock column
+        def lock_column_instance_filter_value
+          public_send(model.lock_column)
+        end
+
+        # Clear the instance filters when refreshing, so that attempting to
+        # refresh after a failed save removes the previous lock column filter
+        # (the new one will be added before updating).
+        def _refresh(ds)
+          clear_instance_filters
+          super
+        end
+      end
+    end
+  end
+end
+

data/lib/sequel/plugins/paged_operations.rb

@@ -0,0 +1,181 @@
+# frozen-string-literal: true
+
+module Sequel
+  module Plugins
+    # The paged_operations plugin adds +paged_update+ and
+    # +paged_delete+ dataset methods.  These behave similarly to
+    # the default +update+ and +delete+ dataset methods, except
+    # that the update or deletion is done in potentially multiple
+    # queries (by default, affecting 1000 rows per query).
+    # For a large table, this prevents the change from
+    # locking the table for a long period of time.
+    #
+    # Because the point of this is to prevent locking tables for
+    # long periods of time, the separate queries are not contained
+    # in a transaction, which means if a later query fails,
+    # earlier queries will still be committed.  You could prevent
+    # this by using a transaction manually, but that defeats the
+    # purpose of using these methods.
+    #
+    # Examples:
+    #
+    #   Album.where{name <= 'M'}.paged_update(updated_at: Sequel::CURRENT_TIMESTAMP)
+    #   # SELECT id FROM albums WHERE (name <= 'M') ORDER BY id LIMIT 1 OFFSET 1000
+    #   # UPDATE albums SET updated_at = CURRENT_TIMESTAMP WHERE ((name <= 'M') AND ("id" < 1002))
+    #   # SELECT id FROM albums WHERE ((name <= 'M') AND (id >= 1002)) ORDER BY id LIMIT 1 OFFSET 1000
+    #   # UPDATE albums SET updated_at = CURRENT_TIMESTAMP WHERE ((name <= 'M') AND ("id" < 2002) AND (id >= 1002))
+    #   # ...
+    #   # SELECT id FROM albums WHERE ((name <= 'M') AND (id >= 10002)) ORDER BY id LIMIT 1 OFFSET 1000
+    #   # UPDATE albums SET updated_at = CURRENT_TIMESTAMP WHERE ((name <= 'M') AND (id >= 10002))
+    #
+    #   Album.where{name > 'M'}.paged_delete
+    #   # SELECT id FROM albums WHERE (name > 'M') ORDER BY id LIMIT 1 OFFSET 1000
+    #   # DELETE FROM albums WHERE ((name > 'M') AND (id < 1002))
+    #   # SELECT id FROM albums WHERE (name > 'M') ORDER BY id LIMIT 1 OFFSET 1000
+    #   # DELETE FROM albums WHERE ((name > 'M') AND (id < 2002))
+    #   # ...
+    #   # SELECT id FROM albums WHERE (name > 'M') ORDER BY id LIMIT 1 OFFSET 1000
+    #   # DELETE FROM albums WHERE (name > 'M')
+    #
+    # The plugin also adds a +paged_datasets+ method that will yield
+    # separate datasets limited in size that in total handle all
+    # rows in the receiver:
+    #
+    #   Album.where{name > 'M'}.paged_datasets{|ds| puts ds.sql}
+    #   # Runs: SELECT id FROM albums WHERE (name <= 'M') ORDER BY id LIMIT 1 OFFSET 1000
+    #   # Prints: SELECT * FROM albums WHERE ((name <= 'M') AND ("id" < 1002))
+    #   # Runs: SELECT id FROM albums WHERE ((name <= 'M') AND (id >= 1002)) ORDER BY id LIMIT 1 OFFSET 1000
+    #   # Prints: SELECT * FROM albums WHERE ((name <= 'M') AND ("id" < 2002) AND (id >= 1002))
+    #   # ...
+    #   # Runs: SELECT id FROM albums WHERE ((name <= 'M') AND (id >= 10002)) ORDER BY id LIMIT 1 OFFSET 1000
+    #   # Prints: SELECT * FROM albums WHERE ((name <= 'M') AND (id >= 10002))
+    #
+    # To set the number of rows per page, pass a :rows_per_page option:
+    #
+    #   Album.where{name <= 'M'}.paged_update({x: Sequel[:x] + 1}, rows_per_page: 4)
+    #   # SELECT id FROM albums WHERE (name <= 'M') ORDER BY id LIMIT 1 OFFSET 4
+    #   # UPDATE albums SET x = x + 1 WHERE ((name <= 'M') AND ("id" < 5))
+    #   # SELECT id FROM albums WHERE ((name <= 'M') AND (id >= 5)) ORDER BY id LIMIT 1 OFFSET 4
+    #   # UPDATE albums SET x = x + 1 WHERE ((name <= 'M') AND ("id" < 9) AND (id >= 5))
+    #   # ...
+    #   # SELECT id FROM albums WHERE ((name <= 'M') AND (id >= 12345)) ORDER BY id LIMIT 1 OFFSET 4
+    #   # UPDATE albums SET x = x + 1 WHERE ((name <= 'M') AND (id >= 12345))
+    # 
+    # You should avoid using +paged_update+ or +paged_datasets+
+    # with updates that modify the primary key, as such usage is
+    # not supported by this plugin.
+    #
+    # This plugin only supports models with scalar primary keys.
+    #
+    # Usage:
+    #
+    #   # Make all model subclasses support paged update/delete/datasets
+    #   # (called before loading subclasses)
+    #   Sequel::Model.plugin :paged_operations
+    #
+    #   # Make the Album class support paged update/delete/datasets
+    #   Album.plugin :paged_operations
+    module PagedOperations
+      module ClassMethods
+        Plugins.def_dataset_methods(self, [:paged_datasets, :paged_delete, :paged_update])
+      end
+
+      module DatasetMethods
+        # Yield datasets for subsets of the receiver that are limited
+        # to no more than 1000 rows (you can configure the number of
+        # rows using +:rows_per_page+).
+        #
+        # Options:
+        # :rows_per_page :: The maximum number of rows in each yielded dataset
+        #                   (unless concurrent modifications are made to the table).
+        def paged_datasets(opts=OPTS)
+          unless defined?(yield)
+            return enum_for(:paged_datasets, opts)
+          end
+
+          pk = _paged_operations_pk(:paged_update)
+          base_offset_ds = offset_ds = _paged_operations_offset_ds(opts)
+          first = nil
+
+          while last = offset_ds.get(pk)
+            ds = where(pk < last)
+            ds = ds.where(pk >= first) if first
+            yield ds
+            first = last
+            offset_ds = base_offset_ds.where(pk >= first)
+          end
+
+          ds = self
+          ds = ds.where(pk >= first) if first
+          yield ds
+          nil
+        end
+
+        # Delete all rows of the dataset using using multiple queries so that
+        # no more than 1000 rows are deleted at a time (you can configure the
+        # number of rows using +:rows_per_page+).
+        #
+        # Options:
+        # :rows_per_page :: The maximum number of rows affected by each DELETE query
+        #                   (unless concurrent modifications are made to the table).
+        def paged_delete(opts=OPTS)
+          if (db.database_type == :oracle && !supports_fetch_next_rows?) || (db.database_type == :mssql && !is_2012_or_later?)
+            raise Error, "paged_delete is not supported on MSSQL/Oracle when using emulated offsets"
+          end
+          pk = _paged_operations_pk(:paged_delete)
+          rows_deleted = 0
+          offset_ds = _paged_operations_offset_ds(opts)
+          while last = offset_ds.get(pk)
+            rows_deleted += where(pk < last).delete
+          end
+          rows_deleted + delete
+        end
+
+        # Update all rows of the dataset using using multiple queries so that
+        # no more than 1000 rows are updated at a time (you can configure the
+        # number of rows using +:rows_per_page+). All arguments are
+        # passed to Dataset#update.
+        #
+        # Options:
+        # :rows_per_page :: The maximum number of rows affected by each UPDATE query
+        #                   (unless concurrent modifications are made to the table).
+        def paged_update(values, opts=OPTS)
+          rows_updated = 0
+          paged_datasets(opts) do |ds|
+            rows_updated += ds.update(values)
+          end
+          rows_updated
+        end
+
+        private
+
+        # Run some basic checks common to paged_{datasets,delete,update}
+        # and return the primary key to operate on as a Sequel::Identifier.
+        def _paged_operations_pk(meth)
+          raise Error, "cannot use #{meth} if dataset has a limit or offset" if @opts[:limit] || @opts[:offset]
+          if db.database_type == :db2 && db.offset_strategy == :emulate
+            raise Error, "the paged_operations plugin is not supported on DB2 when using emulated offsets, set the :offset_strategy Database option to 'limit_offset' or 'offset_fetch'"
+          end
+
+          case pk = model.primary_key
+          when Symbol
+            Sequel.identifier(pk)
+          when Array
+            raise Error, "cannot use #{meth} on a model with a composite primary key"
+          else
+            raise Error, "cannot use #{meth} on a model without a primary key"
+          end
+        end
+
+        # The dataset that will be used by paged_{datasets,delete,update}
+        # to get the upper limit for the next query.
+        def _paged_operations_offset_ds(opts)
+          if rows_per_page = opts[:rows_per_page]
+            raise Error, ":rows_per_page option must be at least 1" unless rows_per_page >= 1
+          end
+          _force_primary_key_order.offset(rows_per_page || 1000)
+        end
+      end
+    end
+  end
+end

data/lib/sequel/plugins/pg_auto_constraint_validations.rb

@@ -133,7 +133,11 @@ module Sequel
         # Dump the in-memory cached metadata to the cache file.
         def dump_pg_auto_constraint_validations_cache
           raise Error, "No pg_auto_constraint_validations setup" unless file = @pg_auto_constraint_validations_cache_file
-          File.open(file, 'wb'){|f| f.write(Marshal.dump(@pg_auto_constraint_validations_cache))}
+          pg_auto_constraint_validations_cache = {}
+          @pg_auto_constraint_validations_cache.sort.each do |k, v|
+            pg_auto_constraint_validations_cache[k] = v
+          end
+          File.open(file, 'wb'){|f| f.write(Marshal.dump(pg_auto_constraint_validations_cache))}
           nil
         end
 

data/lib/sequel/plugins/pg_xmin_optimistic_locking.rb

@@ -0,0 +1,109 @@
+# frozen-string-literal: true
+
+module Sequel
+  module Plugins
+    # This plugin implements optimistic locking mechanism on PostgreSQL based
+    # on the xmin of the row. The xmin system column is automatically set to
+    # the current transaction id whenever the row is inserted or updated:
+    # 
+    #   class Person < Sequel::Model
+    #     plugin :pg_xmin_optimistic_locking
+    #   end
+    #   p1 = Person[1]
+    #   p2 = Person[1]
+    #   p1.update(name: 'Jim') # works
+    #   p2.update(name: 'Bob') # raises Sequel::NoExistingObject
+    #
+    # The advantage of pg_xmin_optimistic_locking plugin compared to the
+    # regular optimistic_locking plugin as that it does not require any
+    # additional columns setup on the model.  This allows it to be loaded
+    # in the base model and have all subclasses automatically use
+    # optimistic locking.  The disadvantage is that testing can be
+    # more difficult if you are modifying the underlying row between
+    # when a model is retrieved and when it is saved.
+    #
+    # This plugin may not work with the class_table_inheritance plugin.
+    #
+    # This plugin relies on the instance_filters plugin.
+    module PgXminOptimisticLocking
+      WILDCARD = LiteralString.new('*').freeze
+      
+      # Define the xmin column accessor
+      def self.apply(model)
+        model.instance_exec do
+          plugin(:optimistic_locking_base)
+          @lock_column = :xmin
+          def_column_accessor(:xmin)
+        end
+      end
+
+      # Update the dataset to append the xmin column if it is usable
+      # and there is a dataset for the model.
+      def self.configure(model)
+        model.instance_exec do
+          set_dataset(@dataset) if @dataset
+        end
+      end
+
+      module ClassMethods
+        private
+
+        # Ensure the dataset selects the xmin column if doing so 
+        def convert_input_dataset(ds)
+          append_xmin_column_if_usable(super)
+        end
+
+        # If the xmin column is not already selected, and selecting it does not
+        # raise an error, append it to the selections.
+        def append_xmin_column_if_usable(ds)
+          select = ds.opts[:select]
+
+          unless select && select.include?(:xmin)
+            xmin_ds = ds.select_append(:xmin)
+            begin
+              columns = xmin_ds.columns!
+            rescue Sequel::DatabaseConnectionError, Sequel::DatabaseDisconnectError
+              raise
+            rescue Sequel::DatabaseError
+              # ignore, could be view, subquery, table returning function, etc.
+            else
+              ds = xmin_ds if columns.include?(:xmin)
+            end
+          end
+
+          ds
+        end
+      end
+
+      module InstanceMethods
+        private
+
+        # Only set the lock column instance filter if there is an xmin value. 
+        def lock_column_instance_filter
+          super if @values[:xmin]
+        end
+
+        # Include xmin value when inserting initial row
+        def _insert_dataset
+          super.returning(WILDCARD, :xmin)
+        end
+        
+        # Remove the xmin from the columns to update.
+        # PostgreSQL automatically updates the xmin value, and it cannot be assigned.
+        def _save_update_all_columns_hash
+          v = super
+          v.delete(:xmin)
+          v
+        end
+
+        # Add an RETURNING clause to fetch the updated xmin when updating the row.
+        def _update_without_checking(columns)
+          ds = _update_dataset
+          rows = ds.clone(ds.send(:default_server_opts, :sql=>ds.returning(:xmin).update_sql(columns))).all
+          values[:xmin] = rows.first[:xmin] unless rows.empty?
+          rows.length
+        end
+      end
+    end
+  end
+end