sequel 5.72.0 → 5.74.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9e135f19f0f4fb1b78f2c3d0c542ca3db937fa6e1e4960d11120e36a0f6ca314
-  data.tar.gz: f1550faa63cfda46f2fc2c4142e009e19357ce961f8e63542874cabdefbaf597
+  metadata.gz: 16c90ef17199e4e48f39edee069981ba0030cf63dc481b136f637b1fe069743e
+  data.tar.gz: 52063f32827a04c33173867207290da294878d898d25b80c37666cf66a54b780
 SHA512:
-  metadata.gz: '093c3ca50f23e97bc3a10b1b0b2a4dea3f79e9cc85bff92801676709fad00e36dd8a84156a5179758bf091fe9e1486d65fc3de300a2b8038f181f36d16481918'
-  data.tar.gz: db0f0c8c81e1fa33b80dbc2e45d477997392965c5aca8a32fc92125843cff4b49045d3d83bd182c9b8bdf980336e1dc97c47a1a32e186e6ef01e5b7d98bbd0a5
+  metadata.gz: d808534a13ae702a884524ceae4adcdb8eff9d46681a62d5b2c0d926b46279743350520248311c41f8a385c7fc66d17cbc44c7f1af52f1cffd356fca498d6bb2
+  data.tar.gz: 309d0f0c0a7c47a4f1acc90107443e7fd2aa51cb09038082354e822711b11986ae271bb784e710336cebafc83b8b5a99adf088bfd19e2353728d25de86f942c4

data/CHANGELOG

@@ -1,3 +1,31 @@
+=== 5.74.0 (2023-11-01)
+
+* Make generated columns show up in Database#schema when using SQLite 3.37+ (jeremyevans) (#2087)
+
+* Add revert method for Sequel.migration blocks, to revert changes inside the block on up, and apply the changes on down (jeremyevans)
+
+* Re-add is_json and is_not_json methods to the pg_json_ops extension, as the support was re-added in PostgreSQL 16 (jeremyevans)
+
+* Avoid infinite loop when handling exceptions with a cause loop in jdbc adapter (jeremyevans)
+
+=== 5.73.0 (2023-10-01)
+
+* Handle disconnect errors in ibmdb and jdbc/db2 adapters (jeremyevans) (#2083)
+
+* Support skipping transactions in Dataset#{import,paged_each} using :skip_transaction option (jeremyevans)
+
+* Add Database#transaction :skip_transaction option to skip creating a transaction or savepoint (jeremyevans)
+
+* Stop using a transaction for a single query if calling Dataset#import with a dataset (jeremyevans)
+
+* Add paged_operations plugin for paged deletes and updates and other custom operations (jeremyevans) (#2080)
+
+* Support to_tsquery: :websearch option to Dataset#full_text_search on PostgreSQL 11+ (jeremyevans) (#2075)
+
+* Add MassAssignmentRestriction#model and #column for getting the model instance and related column for mass assignment errors (artofhuman, jeremyevans) (#2079)
+
+* Stop using base64 library in column_encryption plugin (jeremyevans)
+
 === 5.72.0 (2023-09-01)
 
 * Sort caches before marshalling when using schema_caching, index_caching, static_cache_cache, and pg_auto_constraint_validations (jeremyevans)

data/README.rdoc

@@ -927,7 +927,7 @@ Sequel fully supports the currently supported versions of Ruby (MRI) and JRuby.
 support unsupported versions of Ruby or JRuby, but such support may be dropped in any
 minor version if keeping it becomes a support issue.  The minimum Ruby version
 required to run the current version of Sequel is 1.9.2, and the minimum JRuby version is
-9.0.0.0.
+9.2.0.0 (due to the bigdecimal dependency).
 
 == Maintainer
 

data/doc/migration.rdoc

@@ -90,6 +90,20 @@ the following methods:
 
 If you use any other methods, you should create your own +down+ block.
 
+To revert a migration created with +change+, you can copy the migration to a new file, and
+replace +change+ with +revert+. For example, if you no longer need the artists table, you
+can use the following migration.  This will drop the artists table when migrating up, and
+recreate it when migrating down:
+
+  Sequel.migration do
+    revert do
+      create_table(:artists) do
+        primary_key :id
+        String :name, null: false
+      end
+    end
+  end
+
 In normal usage, when Sequel's migrator runs, it runs the +up+ blocks for all
 migrations that have not yet been applied.  However, you can use the <tt>-M</tt>
 switch to specify the version to which to migrate, and if it is lower than the

data/doc/release_notes/5.73.0.txt

@@ -0,0 +1,66 @@
+= New Features
+
+* A paged_operations plugin has been added, which adds support for
+  paged_datasets, paged_update, and paged_delete dataset methods.
+  This methods are designed to be used on large datasets, to split 
+  a large query into separate smaller queries, to avoid locking the
+  related database table for a long period of time.
+  paged_update and paged_delete operate the same as update and delete,
+  returning the number of rows updated or deleted. paged_datasets yields
+  one or more datasets representing subsets of the receiver, with the
+  union of all of those datasets comprising all records in the receiver:
+
+    Album.plugin :paged_operations
+
+    Album.where{name > 'M'}.paged_datasets{|ds| puts ds.sql}
+    # Runs: SELECT id FROM albums WHERE (name <= 'M') ORDER BY id LIMIT 1 OFFSET 1000
+    # Prints: SELECT * FROM albums WHERE ((name <= 'M') AND ("id" < 1002))
+    # Runs: SELECT id FROM albums WHERE ((name <= 'M') AND (id >= 1002)) ORDER BY id LIMIT 1 OFFSET 1000
+    # Prints: SELECT * FROM albums WHERE ((name <= 'M') AND ("id" < 2002) AND (id >= 1002))
+    # ...
+    # Runs: SELECT id FROM albums WHERE ((name <= 'M') AND (id >= 10002)) ORDER BY id LIMIT 1 OFFSET 1000
+    # Prints: SELECT * FROM albums WHERE ((name <= 'M') AND (id >= 10002))
+
+    Album.where{name <= 'M'}.paged_update(:updated_at=>Sequel::CURRENT_TIMESTAMP)
+    # SELECT id FROM albums WHERE (name <= 'M') ORDER BY id LIMIT 1 OFFSET 1000
+    # UPDATE albums SET updated_at = CURRENT_TIMESTAMP WHERE ((name <= 'M') AND ("id" < 1002))
+    # SELECT id FROM albums WHERE ((name <= 'M') AND (id >= 1002)) ORDER BY id LIMIT 1 OFFSET 1000
+    # UPDATE albums SET updated_at = CURRENT_TIMESTAMP WHERE ((name <= 'M') AND ("id" < 2002) AND (id >= 1002))
+    # ...
+    # SELECT id FROM albums WHERE ((name <= 'M') AND (id >= 10002)) ORDER BY id LIMIT 1 OFFSET 1000
+    # UPDATE albums SET updated_at = CURRENT_TIMESTAMP WHERE ((name <= 'M') AND (id >= 10002))
+
+    Album.where{name > 'M'}.paged_delete
+    # SELECT id FROM albums WHERE (name > 'M') ORDER BY id LIMIT 1 OFFSET 1000
+    # DELETE FROM albums WHERE ((name > 'M') AND (id < 1002))
+    # SELECT id FROM albums WHERE (name > 'M') ORDER BY id LIMIT 1 OFFSET 1000
+    # DELETE FROM albums WHERE ((name > 'M') AND (id < 2002))
+    # ...
+    # SELECT id FROM albums WHERE (name > 'M') ORDER BY id LIMIT 1 OFFSET 1000
+    # DELETE FROM albums WHERE (name > 'M')
+
+* A Dataset#transaction :skip_transaction option is now support to
+  checkout a connection from the pool without opening a transaction.  This
+  makes it easier to handle cases where a transaction may or not be used
+  based on configuration/options.  Dataset#import and Dataset#paged_each
+  now both support the :skip_transaction option to skip transactions.
+
+* Dataset#full_text_search now supports the to_tsquery: :websearch option
+  on PostgreSQL 11+, to use the websearch_to_tsquery database function.
+
+* The Sequel::MassAssignmentRestriction exception now supports model
+  and column methods to get provide additional information about the
+  exception.  Additionally, the exception message now includes information
+  about the model class.
+
+= Other Improvements
+
+* The ibmdb and jdbc/db2 adapter now both handle disconnect errors
+  correctly, removing the related connection from the pool.
+
+* Dataset#import no longer uses an explicit transaction if given a dataset
+  value, as in that case, only a single query is used.
+
+* The column_encryption plugin no longer uses the base64 library.  The
+  base64 library is moving from the standard library to a bundled gem
+  in Ruby 3.4, and this avoids having a dependency on it.

data/doc/release_notes/5.74.0.txt

@@ -0,0 +1,45 @@
+= New Features
+
+* Sequel.migration blocks now support a revert method, which reverts
+  the changes in the block on up, and applies them on down.  So if
+  you have a migration such as:
+
+    Sequel.migration do
+      change do
+        create_table :table do
+          # ...
+        end
+      end
+    end
+
+  and you later want to add a migration that drops the table, you
+  can use:
+
+    Sequel.migration do
+      revert do
+        create_table :table do
+          # ...
+        end
+      end
+    end
+
+  This will drop the table when migrating up, and create a table 
+  with the given schema when migrating down.
+
+* is_json and is_not_json methods have been added to the pg_json_ops
+  extension, for the IS [NOT] JSON operator supported in PostgreSQL
+  16+.  These were previously added in Sequel 5.59.0, and removed
+  in Sequel 5.61.0 as support was removed in PostgreSQL 15 beta 4.
+  PostgreSQL 16 shipped with support for them, so support has been
+  recommitted to Sequel.
+
+= Other Improvements
+
+* SQLite generated columns now show up in Database#schema when using
+  SQLite 3.37+.
+
+* Sequel now attempts to avoid an infinite loop in pathlogical cases
+  in the jdbc adapter, where the exception cause chain has a loop.
+  Additionally, if an exception is already recognized as a disconnect,
+  or an exception already responds to a getSQLState method, Sequel no
+  longer looks at the causes of the exception.

data/lib/sequel/adapters/ibmdb.rb

@@ -301,7 +301,7 @@ module Sequel
       end
 
       def database_exception_sqlstate(exception, opts)
-        exception.sqlstate
+        exception.sqlstate if exception.respond_to?(:sqlstate)
       end
 
       def dataset_class_default

data/lib/sequel/adapters/jdbc/sqlanywhere.rb

@@ -36,6 +36,10 @@ module Sequel
 
         private
 
+        def database_exception_use_sqlstates?
+          false
+        end
+
         # Use @@IDENTITY to get the last inserted id
         def last_insert_id(conn, opts=OPTS)
           statement(conn) do |stmt|

data/lib/sequel/adapters/jdbc/sqlserver.rb

@@ -79,6 +79,10 @@ module Sequel
           super.with_extend(MetadataDatasetMethods)
         end
 
+        def database_exception_use_sqlstates?
+          false
+        end
+
         def disconnect_error?(exception, opts)
           super || (exception.message =~ /connection is closed/)
         end

data/lib/sequel/adapters/jdbc.rb

@@ -396,11 +396,16 @@ module Sequel
 
       def database_exception_sqlstate(exception, opts)
         if database_exception_use_sqlstates?
-          while exception.respond_to?(:cause)
-            exception = exception.cause
-            return exception.getSQLState if exception.respond_to?(:getSQLState)
-          end
+          _database_exception_sqlstate(exception, opts)
         end
+      end
+
+      def _database_exception_sqlstate(exception, opts)
+        16.times do
+          return exception.getSQLState if exception.respond_to?(:getSQLState)
+          break unless exception.respond_to?(:cause) && (exception = exception.cause)
+        end
+
         nil
       end
 
@@ -415,8 +420,7 @@ module Sequel
 
       # Raise a disconnect error if the SQL state of the cause of the exception indicates so.
       def disconnect_error?(exception, opts)
-        cause = exception.respond_to?(:cause) ? exception.cause : exception
-        super || (cause.respond_to?(:getSQLState) && cause.getSQLState =~ /^08/)
+        super || (_database_exception_sqlstate(exception, opts) =~ /^08/)
       end
 
       # Execute the prepared statement.  If the provided name is a

data/lib/sequel/adapters/shared/db2.rb

@@ -215,6 +215,18 @@ module Sequel
         DATABASE_ERROR_REGEXPS
       end
 
+      DISCONNECT_SQL_STATES = %w'40003 08001 08003'.freeze
+      def disconnect_error?(exception, opts)
+        sqlstate = database_exception_sqlstate(exception, opts)
+
+        case sqlstate
+        when *DISCONNECT_SQL_STATES
+          true
+        else
+          super
+        end
+      end
+
       # DB2 has issues with quoted identifiers, so
       # turn off database quoting by default.
       def quote_identifiers_default

data/lib/sequel/adapters/shared/postgres.rb

@@ -1798,7 +1798,7 @@ module Sequel
       # :phrase :: Similar to :plain, but also adding an ILIKE filter to ensure that
       #            returned rows also include the exact phrase used.
       # :rank :: Set to true to order by the rank, so that closer matches are returned first.
-      # :to_tsquery :: Can be set to :plain or :phrase to specify the function to use to
+      # :to_tsquery :: Can be set to :plain, :phrase, or :websearch to specify the function to use to
       #                convert the terms to a ts_query.
       # :tsquery :: Specifies the terms argument is already a valid SQL expression returning a
       #             tsquery, and can be used directly in the query.
@@ -1818,6 +1818,8 @@ module Sequel
           query_func = case to_tsquery = opts[:to_tsquery]
           when :phrase, :plain
             :"#{to_tsquery}to_tsquery"
+          when :websearch
+            :"websearch_to_tsquery"
           else
             (opts[:phrase] || opts[:plain]) ? :plainto_tsquery : :to_tsquery
           end

data/lib/sequel/adapters/shared/sqlite.rb

@@ -504,7 +504,6 @@ module Sequel
             # table_xinfo PRAGMA used, remove hidden columns
             # that are not generated columns
             if row[:generated] = (row.delete(:hidden) != 0)
-              next unless row[:type].end_with?(' GENERATED ALWAYS')
               row[:type] = row[:type].sub(' GENERATED ALWAYS', '')
             end
           end

data/lib/sequel/database/schema_methods.rb

@@ -712,8 +712,9 @@ module Sequel
       e = options[:ignore_index_errors] || options[:if_not_exists]
       generator.indexes.each do |index|
         begin
-          pr = proc{index_sql_list(name, [index]).each{|sql| execute_ddl(sql)}}
-          supports_transactional_ddl? ? transaction(:savepoint=>:only, &pr) : pr.call
+          transaction(:savepoint=>:only, :skip_transaction=>supports_transactional_ddl? == false) do
+            index_sql_list(name, [index]).each{|sql| execute_ddl(sql)}
+          end
         rescue Error
           raise unless e
         end

data/lib/sequel/database/transactions.rb

@@ -166,6 +166,8 @@ module Sequel
     #               uses :auto_savepoint, you can set this to false to not use a savepoint.
     #               If the value given for this option is :only, it will only create a
     #               savepoint if it is inside a transaction.
+    # :skip_transaction :: If set, do not actually open a transaction or savepoint,
+    #                      just checkout a connection and yield it.
     #
     # PostgreSQL specific options:
     #
@@ -193,6 +195,10 @@ module Sequel
         end
       else
         synchronize(opts[:server]) do |conn|
+          if opts[:skip_transaction]
+            return yield(conn)
+          end
+
           if opts[:savepoint] == :only
             if supports_savepoints?
               if _trans(conn)

data/lib/sequel/dataset/actions.rb

@@ -356,9 +356,11 @@ module Sequel
     #            This does not have an effect if +values+ is a Dataset.
     # :server :: Set the server/shard to use for the transaction and insert
     #            queries.
+    # :skip_transaction :: Do not use a transaction even when using multiple
+    #                      INSERT queries.
     # :slice :: Same as :commit_every, :commit_every takes precedence.
     def import(columns, values, opts=OPTS)
-      return @db.transaction{insert(columns, values)} if values.is_a?(Dataset)
+      return insert(columns, values) if values.is_a?(Dataset)
 
       return if values.empty?
       raise(Error, 'Using Sequel::Dataset#import with an empty column array is not allowed') if columns.empty?
@@ -588,6 +590,8 @@ module Sequel
     #                   if your ORDER BY expressions are not simple columns, if they contain
     #                   qualified identifiers that would be ambiguous unqualified, if they contain
     #                   any identifiers that are aliased in SELECT, and potentially other cases.
+    # :skip_transaction :: Do not use a transaction. This can be useful if you want to prevent
+    #                      a lock on the database table, at the expense of consistency.
     #
     # Examples:
     #
@@ -1111,11 +1115,9 @@ module Sequel
     # are provided. When only a single value or statement is provided, then yield
     # without using a transaction.
     def _import_transaction(values, trans_opts, &block)
-      if values.length > 1
-        @db.transaction(trans_opts, &block)
-      else
-        yield
-      end
+      # OK to mutate trans_opts as it is generated by _import
+      trans_opts[:skip_transaction] = true if values.length <= 1
+      @db.transaction(trans_opts, &block)
     end
   
     # Internals of +select_hash+ and +select_hash_groups+

data/lib/sequel/extensions/migration.rb

@@ -159,6 +159,19 @@ module Sequel
       migration.up = block
       migration.down = MigrationReverser.new.reverse(&block)
     end
+
+    # Creates a revert migration.  This is the same as creating
+    # the same block with +down+, but it also calls the block and attempts
+    # to create a +up+ block that will reverse the changes made by
+    # the block.  This is designed to revert the changes in the
+    # provided block.
+    #
+    # There are no guarantees that this will work perfectly
+    # in all cases, but it works for some simple cases.
+    def revert(&block)
+      migration.down = block
+      migration.up = MigrationReverser.new.reverse(&block)
+    end
   end
 
   # Handles the reversing of reversible migrations.  Basically records
@@ -482,11 +495,7 @@ module Sequel
         @use_transactions
       end
 
-      if use_trans
-        db.transaction(&block)
-      else
-        yield
-      end
+      db.transaction(:skip_transaction=>use_trans == false, &block)
     end
 
     # Load the migration file, raising an exception if the file does not define

data/lib/sequel/extensions/pg_json_ops.rb

@@ -123,6 +123,15 @@
 #   c = Sequel.pg_jsonb_op(:c)
 #   DB[:t].update(c['key1'] => 1.to_json, c['key2'] => "a".to_json)
 #
+# On PostgreSQL 16+, the <tt>IS [NOT] JSON</tt> operator is supported:
+#
+#   j.is_json                              # j IS JSON
+#   j.is_json(type: :object)               # j IS JSON OBJECT
+#   j.is_json(type: :object, unique: true) # j IS JSON OBJECT WITH UNIQUE
+#   j.is_not_json                          # j IS NOT JSON
+#   j.is_not_json(type: :array)                # j IS NOT JSON ARRAY
+#   j.is_not_json(unique: true)                # j IS NOT JSON WITH UNIQUE
+#
 # If you are also using the pg_json extension, you should load it before
 # loading this extension.  Doing so will allow you to use the #op method on
 # JSONHash, JSONHarray, JSONBHash, and JSONBArray, allowing you to perform json/jsonb operations
@@ -151,6 +160,18 @@ module Sequel
       GET_PATH = ["(".freeze, " #> ".freeze, ")".freeze].freeze
       GET_PATH_TEXT = ["(".freeze, " #>> ".freeze, ")".freeze].freeze
 
+      IS_JSON = ["(".freeze, " IS JSON".freeze, "".freeze, ")".freeze].freeze
+      IS_NOT_JSON = ["(".freeze, " IS NOT JSON".freeze, "".freeze, ")".freeze].freeze
+      EMPTY_STRING = Sequel::LiteralString.new('').freeze
+      WITH_UNIQUE = Sequel::LiteralString.new(' WITH UNIQUE').freeze
+      IS_JSON_MAP = {
+        nil => EMPTY_STRING,
+        :value => Sequel::LiteralString.new(' VALUE').freeze,
+        :scalar => Sequel::LiteralString.new(' SCALAR').freeze,
+        :object => Sequel::LiteralString.new(' OBJECT').freeze,
+        :array => Sequel::LiteralString.new(' ARRAY').freeze
+      }.freeze
+
       # Get JSON array element or object field as json.  If an array is given,
       # gets the object at the specified path.
       #
@@ -233,6 +254,30 @@ module Sequel
         end
       end
 
+      # Return whether the json object can be parsed as JSON.
+      #
+      # Options:
+      # :type :: Check whether the json object can be parsed as a specific type
+      #          of JSON (:value, :scalar, :object, :array).
+      # :unique :: Check JSON objects for unique keys.
+      #
+      #   json_op.is_json                 # json IS JSON
+      #   json_op.is_json(type: :object)  # json IS JSON OBJECT
+      #   json_op.is_json(unique: true)   # json IS JSON WITH UNIQUE
+      def is_json(opts=OPTS)
+        _is_json(IS_JSON, opts)
+      end
+
+      # Return whether the json object cannot be parsed as JSON. The opposite
+      # of #is_json. See #is_json for options.
+      #
+      #   json_op.is_not_json                 # json IS NOT JSON
+      #   json_op.is_not_json(type: :object)  # json IS NOT JSON OBJECT
+      #   json_op.is_not_json(unique: true)   # json IS NOT JSON WITH UNIQUE
+      def is_not_json(opts=OPTS)
+        _is_json(IS_NOT_JSON, opts)
+      end
+
       # Returns a set of keys AS text in the json object.
       #
       #   json_op.keys # json_object_keys(json)
@@ -286,6 +331,13 @@ module Sequel
 
       private
 
+      # Internals of IS [NOT] JSON support
+      def _is_json(lit_array, opts)
+        raise Error, "invalid is_json :type option: #{opts[:type].inspect}" unless type = IS_JSON_MAP[opts[:type]]
+        unique = opts[:unique] ? WITH_UNIQUE : EMPTY_STRING
+        Sequel::SQL::BooleanExpression.new(:NOOP, Sequel::SQL::PlaceholderLiteralString.new(lit_array, [self, type, unique]))
+      end
+
       # Return a placeholder literal with the given str and args, wrapped
       # in an JSONOp or JSONBOp, used by operators that return json or jsonb.
       def json_op(str, args)

data/lib/sequel/model/base.rb

@@ -1945,8 +1945,10 @@ module Sequel
       end
       
       # If transactions should be used, wrap the yield in a transaction block.
-      def checked_transaction(opts=OPTS)
-        use_transaction?(opts) ? db.transaction({:server=>this_server}.merge!(opts)){yield} : yield
+      def checked_transaction(opts=OPTS, &block)
+        h = {:server=>this_server}.merge!(opts)
+        h[:skip_transaction] = true unless use_transaction?(opts)
+        db.transaction(h, &block)
       end
 
       # Change the value of the column to given value, recording the change.
@@ -2031,19 +2033,20 @@ module Sequel
         meths = setter_methods(type)
         strict = strict_param_setting
         hash.each do |k,v|
+          k = k.to_s
           m = "#{k}="
           if meths.include?(m)
             set_column_value(m, v)
           elsif strict
             # Avoid using respond_to? or creating symbols from user input
             if public_methods.map(&:to_s).include?(m)
-              if Array(model.primary_key).map(&:to_s).member?(k.to_s) && model.restrict_primary_key?
-                raise MassAssignmentRestriction, "#{k} is a restricted primary key"
+              if Array(model.primary_key).map(&:to_s).member?(k) && model.restrict_primary_key?
+                raise MassAssignmentRestriction.create("#{k} is a restricted primary key", self, k)
               else
-                raise MassAssignmentRestriction, "#{k} is a restricted column"
+                raise MassAssignmentRestriction.create("#{k} is a restricted column", self, k)
               end
             else
-              raise MassAssignmentRestriction, "method #{m} doesn't exist"
+              raise MassAssignmentRestriction.create("method #{m} doesn't exist", self, k)
             end
           end
         end
@@ -2147,8 +2150,9 @@ module Sequel
       #   # DELETE FROM artists WHERE (id = 2)
       #   # ...
       def destroy
-        pr = proc{all(&:destroy).length}
-        model.use_transactions ? @db.transaction(:server=>opts[:server], &pr) : pr.call
+        @db.transaction(:server=>opts[:server], :skip_transaction=>model.use_transactions == false) do
+          all(&:destroy).length
+        end
       end
 
       # If there is no order already defined on this dataset, order it by
@@ -2228,11 +2232,17 @@ module Sequel
 
       private
 
+      # Return the dataset ordered by the model's primary key.  This should not
+      # be used if the model does not have a primary key.
+      def _force_primary_key_order
+        cached_dataset(:_pk_order_ds){order(*model.primary_key)}
+      end
+
       # If the dataset is not already ordered, and the model has a primary key,
       # return a clone ordered by the primary key.
       def _primary_key_order
-        if @opts[:order].nil? && model && (pk = model.primary_key)
-          cached_dataset(:_pk_order_ds){order(*pk)}
+        if @opts[:order].nil? && model && model.primary_key
+          _force_primary_key_order
         end
       end
 

data/lib/sequel/model/exceptions.rb

@@ -24,11 +24,23 @@ module Sequel
   UndefinedAssociation = Class.new(Error)
   ).name
 
-  (
   # Raised when a mass assignment method is called in strict mode with either a restricted column
   # or a column without a setter method.
-  MassAssignmentRestriction = Class.new(Error)
-  ).name
+  class MassAssignmentRestriction < Error
+    # The Sequel::Model object related to this exception.
+    attr_reader :model
+
+    # The column related to this exception, as a string.
+    attr_reader :column
+
+    # Create an instance of this class with the model and column set.
+    def self.create(msg, model, column)
+      e = new("#{msg} for class #{model.class.inspect}")
+      e.instance_variable_set(:@model, model)
+      e.instance_variable_set(:@column, column)
+      e
+    end
+  end
   
   # Exception class raised when +raise_on_save_failure+ is set and validation fails
   class ValidationFailed < Error

data/lib/sequel/plugins/column_encryption.rb

@@ -31,7 +31,6 @@ rescue RuntimeError, OpenSSL::Cipher::CipherError
   # :nocov:
 end
 
-require 'base64'
 require 'securerandom'
 
 module Sequel
@@ -375,7 +374,7 @@ module Sequel
         # Decrypt using any supported format and any available key.
         def decrypt(data)
           begin
-            data = Base64.urlsafe_decode64(data)
+            data = urlsafe_decode64(data)
           rescue ArgumentError
             raise Error, "Unable to decode encrypted column: invalid base64"
           end
@@ -448,7 +447,7 @@ module Sequel
         # The prefix string of columns for the given search type and the first configured encryption key.
         # Used to find values that do not use this prefix in order to perform reencryption.
         def current_key_prefix(search_type)
-          Base64.urlsafe_encode64("#{search_type.chr}\0#{@key_id.chr}")
+          urlsafe_encode64("#{search_type.chr}\0#{@key_id.chr}")
         end
 
         # The prefix values to search for the given data (an array of strings), assuming the column uses
@@ -472,11 +471,33 @@ module Sequel
 
         private
 
+        if RUBY_VERSION >= '2.4'
+          def decode64(str)
+            str.unpack1("m0")
+          end
+        # :nocov:
+        else
+          def decode64(str)
+            str.unpack("m0")[0]
+          end
+        # :nocov:
+        end
+
+        def urlsafe_encode64(bin)
+          str = [bin].pack("m0")
+          str.tr!("+/", "-_")
+          str
+        end
+
+        def urlsafe_decode64(str)
+          decode64(str.tr("-_", "+/"))
+        end
+
         # An array of strings, one for each configured encryption key, to find encypted values matching
         # the given data and search format.
         def _search_prefixes(data, search_type)
           @key_map.map do |key_id, (key, _)|
-            Base64.urlsafe_encode64(_search_prefix(data, search_type, key_id, key))
+            urlsafe_encode64(_search_prefix(data, search_type, key_id, key))
           end
         end
 
@@ -509,7 +530,7 @@ module Sequel
           cipher_text << cipher.update(data) if data_size > 0
           cipher_text << cipher.final
 
-          Base64.urlsafe_encode64("#{prefix}#{random_data}#{cipher_iv}#{cipher.auth_tag}#{cipher_text}")
+          urlsafe_encode64("#{prefix}#{random_data}#{cipher_iv}#{cipher.auth_tag}#{cipher_text}")
         end
       end
 

data/lib/sequel/plugins/paged_operations.rb

@@ -0,0 +1,181 @@
+# frozen-string-literal: true
+
+module Sequel
+  module Plugins
+    # The paged_operations plugin adds +paged_update+ and
+    # +paged_delete+ dataset methods.  These behave similarly to
+    # the default +update+ and +delete+ dataset methods, except
+    # that the update or deletion is done in potentially multiple
+    # queries (by default, affecting 1000 rows per query).
+    # For a large table, this prevents the change from
+    # locking the table for a long period of time.
+    #
+    # Because the point of this is to prevent locking tables for
+    # long periods of time, the separate queries are not contained
+    # in a transaction, which means if a later query fails,
+    # earlier queries will still be committed.  You could prevent
+    # this by using a transaction manually, but that defeats the
+    # purpose of using these methods.
+    #
+    # Examples:
+    #
+    #   Album.where{name <= 'M'}.paged_update(updated_at: Sequel::CURRENT_TIMESTAMP)
+    #   # SELECT id FROM albums WHERE (name <= 'M') ORDER BY id LIMIT 1 OFFSET 1000
+    #   # UPDATE albums SET updated_at = CURRENT_TIMESTAMP WHERE ((name <= 'M') AND ("id" < 1002))
+    #   # SELECT id FROM albums WHERE ((name <= 'M') AND (id >= 1002)) ORDER BY id LIMIT 1 OFFSET 1000
+    #   # UPDATE albums SET updated_at = CURRENT_TIMESTAMP WHERE ((name <= 'M') AND ("id" < 2002) AND (id >= 1002))
+    #   # ...
+    #   # SELECT id FROM albums WHERE ((name <= 'M') AND (id >= 10002)) ORDER BY id LIMIT 1 OFFSET 1000
+    #   # UPDATE albums SET updated_at = CURRENT_TIMESTAMP WHERE ((name <= 'M') AND (id >= 10002))
+    #
+    #   Album.where{name > 'M'}.paged_delete
+    #   # SELECT id FROM albums WHERE (name > 'M') ORDER BY id LIMIT 1 OFFSET 1000
+    #   # DELETE FROM albums WHERE ((name > 'M') AND (id < 1002))
+    #   # SELECT id FROM albums WHERE (name > 'M') ORDER BY id LIMIT 1 OFFSET 1000
+    #   # DELETE FROM albums WHERE ((name > 'M') AND (id < 2002))
+    #   # ...
+    #   # SELECT id FROM albums WHERE (name > 'M') ORDER BY id LIMIT 1 OFFSET 1000
+    #   # DELETE FROM albums WHERE (name > 'M')
+    #
+    # The plugin also adds a +paged_datasets+ method that will yield
+    # separate datasets limited in size that in total handle all
+    # rows in the receiver:
+    #
+    #   Album.where{name > 'M'}.paged_datasets{|ds| puts ds.sql}
+    #   # Runs: SELECT id FROM albums WHERE (name <= 'M') ORDER BY id LIMIT 1 OFFSET 1000
+    #   # Prints: SELECT * FROM albums WHERE ((name <= 'M') AND ("id" < 1002))
+    #   # Runs: SELECT id FROM albums WHERE ((name <= 'M') AND (id >= 1002)) ORDER BY id LIMIT 1 OFFSET 1000
+    #   # Prints: SELECT * FROM albums WHERE ((name <= 'M') AND ("id" < 2002) AND (id >= 1002))
+    #   # ...
+    #   # Runs: SELECT id FROM albums WHERE ((name <= 'M') AND (id >= 10002)) ORDER BY id LIMIT 1 OFFSET 1000
+    #   # Prints: SELECT * FROM albums WHERE ((name <= 'M') AND (id >= 10002))
+    #
+    # To set the number of rows per page, pass a :rows_per_page option:
+    #
+    #   Album.where{name <= 'M'}.paged_update({x: Sequel[:x] + 1}, rows_per_page: 4)
+    #   # SELECT id FROM albums WHERE (name <= 'M') ORDER BY id LIMIT 1 OFFSET 4
+    #   # UPDATE albums SET x = x + 1 WHERE ((name <= 'M') AND ("id" < 5))
+    #   # SELECT id FROM albums WHERE ((name <= 'M') AND (id >= 5)) ORDER BY id LIMIT 1 OFFSET 4
+    #   # UPDATE albums SET x = x + 1 WHERE ((name <= 'M') AND ("id" < 9) AND (id >= 5))
+    #   # ...
+    #   # SELECT id FROM albums WHERE ((name <= 'M') AND (id >= 12345)) ORDER BY id LIMIT 1 OFFSET 4
+    #   # UPDATE albums SET x = x + 1 WHERE ((name <= 'M') AND (id >= 12345))
+    # 
+    # You should avoid using +paged_update+ or +paged_datasets+
+    # with updates that modify the primary key, as such usage is
+    # not supported by this plugin.
+    #
+    # This plugin only supports models with scalar primary keys.
+    #
+    # Usage:
+    #
+    #   # Make all model subclasses support paged update/delete/datasets
+    #   # (called before loading subclasses)
+    #   Sequel::Model.plugin :paged_operations
+    #
+    #   # Make the Album class support paged update/delete/datasets
+    #   Album.plugin :paged_operations
+    module PagedOperations
+      module ClassMethods
+        Plugins.def_dataset_methods(self, [:paged_datasets, :paged_delete, :paged_update])
+      end
+
+      module DatasetMethods
+        # Yield datasets for subsets of the receiver that are limited
+        # to no more than 1000 rows (you can configure the number of
+        # rows using +:rows_per_page+).
+        #
+        # Options:
+        # :rows_per_page :: The maximum number of rows in each yielded dataset
+        #                   (unless concurrent modifications are made to the table).
+        def paged_datasets(opts=OPTS)
+          unless defined?(yield)
+            return enum_for(:paged_datasets, opts)
+          end
+
+          pk = _paged_operations_pk(:paged_update)
+          base_offset_ds = offset_ds = _paged_operations_offset_ds(opts)
+          first = nil
+
+          while last = offset_ds.get(pk)
+            ds = where(pk < last)
+            ds = ds.where(pk >= first) if first
+            yield ds
+            first = last
+            offset_ds = base_offset_ds.where(pk >= first)
+          end
+
+          ds = self
+          ds = ds.where(pk >= first) if first
+          yield ds
+          nil
+        end
+
+        # Delete all rows of the dataset using using multiple queries so that
+        # no more than 1000 rows are deleted at a time (you can configure the
+        # number of rows using +:rows_per_page+).
+        #
+        # Options:
+        # :rows_per_page :: The maximum number of rows affected by each DELETE query
+        #                   (unless concurrent modifications are made to the table).
+        def paged_delete(opts=OPTS)
+          if (db.database_type == :oracle && !supports_fetch_next_rows?) || (db.database_type == :mssql && !is_2012_or_later?)
+            raise Error, "paged_delete is not supported on MSSQL/Oracle when using emulated offsets"
+          end
+          pk = _paged_operations_pk(:paged_delete)
+          rows_deleted = 0
+          offset_ds = _paged_operations_offset_ds(opts)
+          while last = offset_ds.get(pk)
+            rows_deleted += where(pk < last).delete
+          end
+          rows_deleted + delete
+        end
+
+        # Update all rows of the dataset using using multiple queries so that
+        # no more than 1000 rows are updated at a time (you can configure the
+        # number of rows using +:rows_per_page+). All arguments are
+        # passed to Dataset#update.
+        #
+        # Options:
+        # :rows_per_page :: The maximum number of rows affected by each UPDATE query
+        #                   (unless concurrent modifications are made to the table).
+        def paged_update(values, opts=OPTS)
+          rows_updated = 0
+          paged_datasets(opts) do |ds|
+            rows_updated += ds.update(values)
+          end
+          rows_updated
+        end
+
+        private
+
+        # Run some basic checks common to paged_{datasets,delete,update}
+        # and return the primary key to operate on as a Sequel::Identifier.
+        def _paged_operations_pk(meth)
+          raise Error, "cannot use #{meth} if dataset has a limit or offset" if @opts[:limit] || @opts[:offset]
+          if db.database_type == :db2 && db.offset_strategy == :emulate
+            raise Error, "the paged_operations plugin is not supported on DB2 when using emulated offsets, set the :offset_strategy Database option to 'limit_offset' or 'offset_fetch'"
+          end
+
+          case pk = model.primary_key
+          when Symbol
+            Sequel.identifier(pk)
+          when Array
+            raise Error, "cannot use #{meth} on a model with a composite primary key"
+          else
+            raise Error, "cannot use #{meth} on a model without a primary key"
+          end
+        end
+
+        # The dataset that will be used by paged_{datasets,delete,update}
+        # to get the upper limit for the next query.
+        def _paged_operations_offset_ds(opts)
+          if rows_per_page = opts[:rows_per_page]
+            raise Error, ":rows_per_page option must be at least 1" unless rows_per_page >= 1
+          end
+          _force_primary_key_order.offset(rows_per_page || 1000)
+        end
+      end
+    end
+  end
+end

data/lib/sequel/version.rb

@@ -6,7 +6,7 @@ module Sequel
 
   # The minor version of Sequel.  Bumped for every non-patch level
   # release, generally around once a month.
-  MINOR = 72
+  MINOR = 74
 
   # The tiny version of Sequel.  Usually 0, only bumped for bugfix
   # releases that fix regressions from previous versions.

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sequel
 version: !ruby/object:Gem::Version
-  version: 5.72.0
+  version: 5.74.0
 platform: ruby
 authors:
 - Jeremy Evans
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-09-01 00:00:00.000000000 Z
+date: 2023-11-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bigdecimal
@@ -219,6 +219,8 @@ extra_rdoc_files:
 - doc/release_notes/5.70.0.txt
 - doc/release_notes/5.71.0.txt
 - doc/release_notes/5.72.0.txt
+- doc/release_notes/5.73.0.txt
+- doc/release_notes/5.74.0.txt
 - doc/release_notes/5.8.0.txt
 - doc/release_notes/5.9.0.txt
 files:
@@ -319,6 +321,8 @@ files:
 - doc/release_notes/5.70.0.txt
 - doc/release_notes/5.71.0.txt
 - doc/release_notes/5.72.0.txt
+- doc/release_notes/5.73.0.txt
+- doc/release_notes/5.74.0.txt
 - doc/release_notes/5.8.0.txt
 - doc/release_notes/5.9.0.txt
 - doc/schema_modification.rdoc
@@ -572,6 +576,7 @@ files:
 - lib/sequel/plugins/nested_attributes.rb
 - lib/sequel/plugins/optimistic_locking.rb
 - lib/sequel/plugins/optimistic_locking_base.rb
+- lib/sequel/plugins/paged_operations.rb
 - lib/sequel/plugins/pg_array_associations.rb
 - lib/sequel/plugins/pg_auto_constraint_validations.rb
 - lib/sequel/plugins/pg_row.rb