sequel 5.41.0 → 5.42.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b0886ae2f4e4f4490b56ced6137e2d849cac2458119d6e67d9a756328336ece6
-  data.tar.gz: 9c4946c8ca82b2b1c99ee9065bef0aa81175128564cd7ab0c0405bd4e682b339
+  metadata.gz: 41e987d42df8d1b3ab41dd6555b5e338b94ba0e2fcd11fa3db7f9b955920d533
+  data.tar.gz: fdcaed20caa20bbaffe5854cbcd6836787e92a2bbadb89373337602926d476ca
 SHA512:
-  metadata.gz: aa61ea3a0cb3d0e3ab021d27589e10250ed9d9d5a0b4bf314729d55661624161cabffe9c87278761890a900949756e185ce1f263bf742d9b317f2621a1f75c09
-  data.tar.gz: a8253ea1f6e77b592eb40de7b2544b7b34ff8cd02a79e4dfe5bb26b796dd80d49f052e307590af71fb67b292bc9b9c1def8f400eea0977fdb0d85928a6e9f181
+  metadata.gz: c9a3927fe546ee7f91497e5bd50e9fa9241dd46aa7d3b3c331d10f8ac369eedd204a488207c509e909d9eb5d7deeeb13ab20888568f7e91f53fba526323df566
+  data.tar.gz: a9108c8ce0d78d93c94f0a947171687022dd20ee1305633533340f1cb1a71d012248f9a9bb72eb7af28cc258240ec2e8f090683d1ad7a68c33f69d261c684ea0

data/CHANGELOG

@@ -1,3 +1,19 @@
+=== 5.42.0 (2021-03-01)
+
+* Make the ado timestamp conversion proc a normal conversion proc that can be overridden similar to other conversion procs (jeremyevans)
+
+* Add :reject_nil option to the nested_attributes method, to ignore calls where nil is passed as the associated object data (jeremyevans)
+
+* Add async_thread_pool plugin for easier async usage with model classes and support for async destroy, with_pk, and with_pk! methods (jeremyevans)
+
+* Add async_thread_pool Database extension for executing queries asynchronously using a thread pool (jeremyevans)
+
+* Fix possible thread safety issue in Database#extension that could allow Module#extended to be called twice with the same Database instance (jeremyevans)
+
+* Support cases where validations make modifications beyond setting errors in Model#freeze (jeremyevans)
+
+* Add Model#to_json_data to the json_serializer plugin, returning a JSON data structure (jeremyevans)
+
 === 5.41.0 (2021-02-01)
 
 * Have explicit :text option for a String column take priority over :size option on PostgreSQL (jeremyevans) (#1750)

data/doc/release_notes/5.42.0.txt

@@ -0,0 +1,136 @@
+= New Features
+
+* An async_thread_pool Database extension has been added, which
+  executes queries and processes results using a separate thread
+  pool.  This allows you do do things like:
+
+    foos = DB[:foos].async.all
+    bars = DB[:bars].async.select_map(:name)
+    foo_bars = DB[:foo_bars].async.each{|x| p x}
+
+  and have the three method calls (all, select_map, and each)
+  execute concurrently.  On Ruby implementations without a global
+  VM lock, such as JRuby, it will allow for parallel execution of
+  the method calls.  On CRuby, the main benefit will be for cases
+  where query execution takes a long time or there is significant
+  latency between the application and the database.
+  
+  When you call a method on foos, bars, or foo_bars, if the thread
+  pool hasn't finished processing the method, the calling code will
+  block until the method call has finished.
+
+  By default, for consistency, calling code will not preempt the
+  async thread pool.  For example, if you do:
+
+    DB[:foos].async.all.size
+
+  The calling code will always wait for the async thread pool to
+  run the all method, and then the calling code will call size on
+  the result.  This ensures that async queries will not use the
+  same connection as the the calling thread, even if calling thread
+  has a connection checked out.
+
+  In some cases, such as when the async thread pool is very busy,
+  preemption is desired for performance reasons.  If you set the
+  :preempt_async_thread Database option before loading the
+  async_thread_pool extension, preemption will be allowed.  With
+  preemption allowed, if the async thread pool has not started the
+  processing of the method at the time the calling code needs the
+  results of the method, the calling code will preempt the async
+  thread pool, and run the method on the current thread.
+
+  By default, the async thread pool uses the same number of threads as
+  the Database objects :max_connections attribute (the default for
+  that is 4).  You can modify the number of async threads by setting
+  the :num_async_threads Database option before loading the Database
+  async_thread_pool extension.
+
+  Most Dataset methods that execute queries on the database and return
+  results will operate asynchronously if the the dataset is set to be
+  asynchronous via the Dataset#async method.  This includes most
+  methods available due to the inclusion in Enumerable, even if not
+  defined by Dataset itself.
+
+  There are multiple caveats when using the async_thread_pool
+  extension:
+  
+  * Asynchronous behavior is harder to understand and harder to
+    debug.  It would be wise to only use this support in cases where
+    it provides is significant performance benefit.
+
+  * Dataset methods executed asynchronously will use a separate
+    database connection than the calling thread, so they will not
+    respect transactions in the calling thread, or other cases where
+    the calling thread checks out a connection directly using
+    Database#synchronize.  They will also not respect the use of
+    Database#with_server (from the server_block extension) in the
+    calling thread.
+
+  * Dataset methods executed asynchronously should never ignore their
+    return value.  Code such as:
+
+      DB[:table].async.insert(1)
+
+    is probablematic because without storing the return value, you
+    have no way to block until the insert has been completed.
+
+  * The returned object for Dataset methods executed asynchronously is
+    a proxy object (promise).  So you should never do:
+
+      row = DB[:table].async.first
+      # ...
+      if row
+      end
+
+      # or:
+
+      bool = DB[:table].async.get(:boolean_column)
+      # ...
+      if bool
+      end
+
+    because the if branches will always be taken as row and bool will
+    never be nil or false.  If you want to get the underlying value,
+    call itself on the proxy object (or __value if using Ruby <2.2).
+
+    For the same reason, you should not use the proxy objects directly
+    in case expressions or as arguments to Class#===.  Use itself or
+    __value in those cases.
+
+  * Dataset methods executed asynchronously that include blocks have the
+    block executed asynchronously as well, assuming that the method
+    calls the block.  Because these blocks are executed in a separate
+    thread, you cannot use control flow modifiers such as break or
+    return in them.
+
+* An async_thread_pool model plugin has been added.  This requires the
+  async_thread_pool extension has been loaded into the model's Database
+  object, and allows you to call Model.async instead of
+  Model.dataset.async.  It also adds async support to the destroy,
+  with_pk, and with_pk! model dataset methods.
+
+* Model#to_json_data has been added to the json_serializer plugin, for
+  returning a hash of data that can be converted to JSON, instead of
+  a JSON string.
+
+* A :reject_nil option has been added to the nested_attributes method
+  in the nested_attributes plugin.  This will ignore calls to the
+  nested attributes setter method where nil is passed as the setter
+  method argument.
+
+= Other Improvements
+
+* Model#freeze now works in case where model validation modifies the
+  object beyond adding errors.
+
+* Model#freeze in the composition, serialization, and
+  serialization_modification_detection plugins now works in cases
+  where validation would end up loading the composed or
+  serialized values.
+
+* Database#extension now avoids a possible thread safety issue that
+  could result in the extension being loaded into the Database twice.
+
+* The ado adapter now supports overriding the timestamp conversion
+  proc.  Previously, unlike other conversion procs, the timestamp
+  conversion proc was hard coded and could not be overridden.

data/doc/testing.rdoc

@@ -157,6 +157,8 @@ The SEQUEL_INTEGRATION_URL environment variable specifies the Database connectio
 
 === Other
 
+SEQUEL_ASYNC_THREAD_POOL :: Use the async_thread_pool extension when running the specs
+SEQUEL_ASYNC_THREAD_POOL_PREEMPT :: Use the async_thread_pool extension when running the specs, with the :preempt_async_thread option
 SEQUEL_COLUMNS_INTROSPECTION :: Use the columns_introspection extension when running the specs
 SEQUEL_CONNECTION_VALIDATOR :: Use the connection validator extension when running the specs
 SEQUEL_DUPLICATE_COLUMNS_HANDLER :: Use the duplicate columns handler extension with value given when running the specs

data/lib/sequel/adapters/ado.rb

@@ -195,10 +195,25 @@ module Sequel
         end
 
         @conversion_procs = CONVERSION_PROCS.dup
+        @conversion_procs[AdDBTimeStamp] = method(:adb_timestamp_to_application_timestamp)
 
         super
       end
 
+      def adb_timestamp_to_application_timestamp(v)
+        # This hard codes a timestamp_precision of 6 when converting.
+        # That is the default timestamp_precision, but the ado/mssql adapter uses a timestamp_precision
+        # of 3.  However, timestamps returned by ado/mssql have nsec values that end up rounding to a
+        # the same value as if a timestamp_precision of 3 was hard coded (either xxx999yzz, where y is
+        # 5-9 or xxx000yzz where y is 0-4).
+        #
+        # ADO subadapters should override this they would like a different timestamp precision and the
+        # this code does not work for them (for example, if they provide full nsec precision).
+        #
+        # Note that fractional second handling for WIN32OLE objects is not correct on ruby <2.2
+        to_application_timestamp([v.year, v.month, v.day, v.hour, v.min, v.sec, (v.nsec/1000.0).round * 1000])
+      end
+
       def dataset_class_default
         Dataset
       end
@@ -233,23 +248,8 @@ module Sequel
           cols = []
           conversion_procs = db.conversion_procs
 
-          ts_cp = nil
           recordset.Fields.each do |field|
-            type = field.Type
-            cp = if type == AdDBTimeStamp
-              ts_cp ||= begin
-                nsec_div = 1000000000.0/(10**(timestamp_precision))
-                nsec_mul = 10**(timestamp_precision+3)
-                meth = db.method(:to_application_timestamp)
-                lambda do |v|
-                  # Fractional second handling is not correct on ruby <2.2
-                  meth.call([v.year, v.month, v.day, v.hour, v.min, v.sec, (v.nsec/nsec_div).round * nsec_mul])
-                end
-              end
-            else
-              conversion_procs[type]
-            end
-            cols << [output_identifier(field.Name), cp]
+            cols << [output_identifier(field.Name), conversion_procs[field.Type]]
           end
 
           self.columns = cols.map(&:first)

data/lib/sequel/database/misc.rb

@@ -213,8 +213,7 @@ module Sequel
       Sequel.extension(*exts)
       exts.each do |ext|
         if pr = Sequel.synchronize{EXTENSIONS[ext]}
-          unless Sequel.synchronize{@loaded_extensions.include?(ext)}
-            Sequel.synchronize{@loaded_extensions << ext}
+          if Sequel.synchronize{@loaded_extensions.include?(ext) ? false : (@loaded_extensions << ext)}
             pr.call(self)
           end
         else

data/lib/sequel/database/schema_generator.rb

@@ -159,7 +159,7 @@ module Sequel
         nil
       end
       
-      # Adds a named constraint (or unnamed if name is nil),
+      # Adds a named CHECK constraint (or unnamed if name is nil),
       # with the given block or args. To provide options for the constraint, pass
       # a hash as the first argument.
       #
@@ -167,6 +167,15 @@ module Sequel
       #   # CONSTRAINT blah CHECK num >= 1 AND num <= 5
       #   constraint({name: :blah, deferrable: true}, num: 1..5)
       #   # CONSTRAINT blah CHECK num >= 1 AND num <= 5 DEFERRABLE INITIALLY DEFERRED
+      #
+      # If the first argument is a hash, the following options are supported:
+      #
+      # Options:
+      # :name :: The name of the CHECK constraint
+      # :deferrable :: Whether the CHECK constraint should be marked DEFERRABLE.
+      #
+      # PostgreSQL specific options:
+      # :not_valid :: Whether the CHECK constraint should be marked NOT VALID.
       def constraint(name, *args, &block)
         opts = name.is_a?(Hash) ? name : {:name=>name}
         constraints << opts.merge(:type=>:check, :check=>block || args)

data/lib/sequel/database/schema_methods.rb

@@ -262,6 +262,10 @@ module Sequel
     #   # SELECT * FROM items WHERE foo
     #   # WITH CHECK OPTION
     #
+    #   DB.create_view(:bar_items, DB[:items].select(:foo), columns: [:bar])
+    #   # CREATE VIEW bar_items (bar) AS
+    #   # SELECT foo FROM items
+    #
     # Options:
     # :columns :: The column names to use for the view.  If not given,
     #             automatically determined based on the input dataset.

data/lib/sequel/extensions/async_thread_pool.rb

@@ -0,0 +1,438 @@
+# frozen-string-literal: true
+#
+# The async_thread_pool extension adds support for running database
+# queries in a separate threads using a thread pool. With the following
+# code
+#
+#   DB.extension :async_thread_pool
+#   foos = DB[:foos].async.where{:name=>'A'..'M'}.all
+#   bar_names = DB[:bar].async.select_order_map(:name)
+#   baz_1 = DB[:bazes].async.first(:id=>1)
+#
+# All 3 queries will be run in separate threads.  +foos+, +bar_names+
+# and +baz_1+ will be proxy objects.  Calling a method on the proxy
+# object will wait for the query to be run, and will return the result
+# of calling that method on the result of the query method. For example,
+# if you run:
+#
+#   foos = DB[:foos].async.where{:name=>'A'..'M'}.all
+#   bar_names = DB[:bars].async.select_order_map(:name)
+#   baz_1 = DB[:bazes].async.first(:id=>1)
+#   sleep(1)
+#   foos.size
+#   bar_names.first
+#   baz_1.name
+#
+# These three queries will generally be run concurrently in separate
+# threads.  If you instead run:
+#   
+#   DB[:foos].async.where{:name=>'A'..'M'}.all.size
+#   DB[:bars].async.select_order_map(:name).first
+#   DB[:bazes].async.first(:id=>1).name
+#
+# Then will run each query sequentially, since you need the result of
+# one query before running the next query.  The queries will still be
+# run in separate threads (by default).
+#
+# What is run in the separate thread is the entire method call that
+# returns results.  So with the original example:
+#
+#   foos = DB[:foos].async.where{:name=>'A'..'M'}.all
+#   bar_names = DB[:bars].async.select_order_map(:name)
+#   baz_1 = DB[:bazes].async.first(:id=>1)
+#
+# The +all+, <tt>select_order_map(:name)</tt>, and <tt>first(:id=>1)</tt>
+# calls are run in separate threads.  If a block is passed to a method
+# such as +all+ or +each+, the block is also run in that thread.  If you
+# have code such as:
+#
+#   h = {}
+#   DB[:foos].async.each{|row| h[row[:id]] = row}
+#   bar_names = DB[:bars].async.select_order_map(:name)
+#   p h
+#
+# You may end up with it printing an empty hash or partial hash, because the
+# async +each+ call will not have run or finished running.  Since the
+# <tt>p h</tt> code relies on a side-effect of the +each+ block and not the
+# return value of the +each+ call, it will not wait for the loading.
+#
+# You should avoid using +async+ for any queries where you are ignoring the
+# return value, as otherwise you have no way to wait for the query to be run.
+#
+# Datasets that use async will use async threads to load data for the majority
+# of methods that can return data.  However, dataset methods that return
+# enumerators will not use an async thread (e.g. calling # Dataset#map
+# without a block or arguments does not use an async thread or return a
+# proxy object).
+#
+# Because async methods (including their blocks) run in a separate thread, you
+# should not use control flow modifiers such as +return+ or +break+ in async
+# queries.  Doing so will result in a error.
+#
+# Because async results are returned as proxy objects, it's a bad idea
+# to use them in a boolean setting:
+#
+#   result = DB[:foo].async.get(:boolean_column)
+#   # or:
+#   result = DB[:foo].async.first
+#
+#   # ...
+#   if result 
+#     # will always execute this banch, since result is a proxy object
+#   end
+#
+# In this case, you can call the +__value+ method to return the actual
+# result:
+#
+#   if result.__value
+#     # will not execute this branch if the dataset method returned nil or false
+#   end
+#
+# Similarly, because a proxy object is used, you should be careful using the
+# result in a case statement or an argument to <tt>Class#===</tt>:
+#
+#   # ...
+#   case result
+#   when Hash, true, false
+#     # will never take this branch, since result is a proxy object
+#   end
+#
+# Similar to usage in an +if+ statement, you should use +__value+:
+#
+#   case result.__value
+#   when Hash, true, false
+#     # will never take this branch, since result is a proxy object
+#   end
+#
+# On Ruby 2.2+, you can use +itself+ instead of +__value+.  It's preferable to
+# use +itself+ if you can, as that will allow code to work with both proxy
+# objects and regular objects.
+#
+# Because separate threads and connections are used for async queries,
+# they do not use any state on the current connection/thread. So if
+# you do:
+#
+#   DB.transaction{DB[:table].async.all}
+#
+# Be aware that the transaction runs on one connection, and the SELECT
+# query on a different connection.  If you use currently using
+# transactional testing (running each test inside a transaction/savepoint),
+# and want to start using this extension, you should first switch to
+# non-transactional testing of the code that will use the async thread
+# pool before using this extension, as otherwise the use of
+# <tt>Dataset#async</tt> will likely break your tests.
+#
+# If you are using Database#synchronize to checkout a connection, the
+# same issue applies, where the async query runs on a different
+# connection:
+#
+#   DB.synchronize{DB[:table].async.all}
+# 
+# Similarly, if you are using the server_block extension, any async
+# queries inside with_server blocks will not use the server specified:
+#
+#   DB.with_server(:shard1) do
+#     DB[:a].all # Uses shard1
+#     DB[:a].async.all # Uses default shard
+#   end
+#
+# You need to manually specify the shard for any dataset using an async
+# query:
+# 
+#   DB.with_server(:shard1) do
+#     DB[:a].all # Uses shard1
+#     DB[:a].async.server(:shard1).all # Uses shard1
+#   end
+#
+# When the async_thread_pool extension, the size of the async thread pool
+# can be set by using the +:num_async_threads+ Database option, which must
+# be set before loading the async_thread_pool extension.  This defaults
+# to the size of the Database object's connection pool.
+#
+# By default, for consistent behavior, the async_thread_pool extension
+# will always run the query in a separate thread. However, in some cases,
+# such as when the async thread pool is busy and the results of a query
+# are needed right away, it can improve performance to allow preemption,
+# so that the query will run in the current thread instead of waiting
+# for an async thread to become available.  With the following code:
+#
+#   foos = DB[:foos].async.where{:name=>'A'..'M'}.all
+#   bar_names = DB[:bar].async.select_order_map(:name)
+#   if foos.length > 4
+#     baz_1 = DB[:bazes].async.first(:id=>1)
+#   end
+# 
+# Whether you need the +baz_1+ variable depends on the value of foos.
+# If the async thread pool is busy, and by the time the +foos.length+
+# call is made, the async thread pool has not started the processing
+# to get the +foos+ value, it can improve performance to start that
+# processing in the current thread, since it is needed immediately to
+# determine whether to schedule query to get the +baz_1+ variable.
+# The default is to not allow preemption, because if the current
+# thread is used, it may have already checked out a connection that
+# could be used, and that connection could be inside a transaction or
+# have some other manner of connection-specific state applied to it.
+# If you want to allow preemption, you can set the
+# +:preempt_async_thread+ Database option before loading the
+# async_thread_pool extension.
+#
+# Related module: Sequel::Database::AsyncThreadPool::DatasetMethods
+
+
+# 
+module Sequel
+  module Database::AsyncThreadPool
+    # JobProcessor is a wrapper around a single thread, that will
+    # process a queue of jobs until it is shut down.
+    class JobProcessor # :nodoc:
+      def self.create_finalizer(queue, pool)
+        proc{run_finalizer(queue, pool)}
+      end
+
+      def self.run_finalizer(queue, pool)
+        # Push a nil for each thread using the queue, signalling
+        # that thread to close.
+        pool.each{queue.push(nil)}
+
+        # Join each of the closed threads.
+        pool.each(&:join)
+
+        # Clear the thread pool.  Probably not necessary, but this allows
+        # for a simple way to check whether this finalizer has been run.
+        pool.clear
+
+        nil
+      end
+      private_class_method :run_finalizer
+
+      def initialize(queue)
+        @thread = ::Thread.new do
+          while proxy = queue.pop
+            proxy.__send__(:__run)
+          end
+        end
+      end
+
+      # Join the thread, should only be called by the related finalizer.
+      def join
+        @thread.join
+      end
+    end
+
+    # Wrapper for exception instances raised by async jobs.  The
+    # wrapped exception will be raised by the code getting the value
+    # of the job.
+    WrappedException = Struct.new(:exception)
+
+    # Base proxy object class for jobs processed by async threads and
+    # the returned result.
+    class BaseProxy < BasicObject
+      # Store a block that returns the result when called.
+      def initialize(&block)
+        ::Kernel.raise Error, "must provide block for an async job" unless block
+        @block = block
+      end
+
+      # Pass all method calls to the returned result.
+      def method_missing(*args, &block)
+        __value.public_send(*args, &block)
+      end
+      # :nocov:
+      ruby2_keywords(:method_missing) if respond_to?(:ruby2_keywords, true)
+      # :nocov:
+
+      # Delegate respond_to? calls to the returned result.
+      def respond_to_missing?(*args)
+        __value.respond_to?(*args)
+      end
+
+      # Override some methods defined by default so they apply to the
+      # returned result and not the current object.
+      [:!, :==, :!=, :instance_eval, :instance_exec].each do |method|
+        define_method(method) do |*args, &block|
+          __value.public_send(method, *args, &block)
+        end
+      end
+
+      # Wait for the value to be loaded if it hasn't already been loaded.
+      # If the code to load the return value raised an exception that was
+      # wrapped, reraise the exception.
+      def __value
+        unless defined?(@value)
+          __get_value
+        end
+
+        if @value.is_a?(WrappedException)
+          ::Kernel.raise @value
+        end
+
+        @value
+      end
+
+      private
+
+      # Run the block and return the block value.  If the block call raises
+      # an exception, wrap the exception.
+      def __run_block
+        # This may not catch concurrent calls (unless surrounded by a mutex), but
+        # it's not worth trying to protect against that.  It's enough to just check for
+        # multiple non-concurrent calls.
+        ::Kernel.raise Error, "Cannot run async block multiple times" unless block = @block
+
+        @block = nil
+
+        begin
+          block.call
+        rescue ::Exception => e
+          WrappedException.new(e)
+        end
+      end
+    end
+
+    # Default object class for async job/proxy result.  This uses a queue for
+    # synchronization.  The JobProcessor will push a result until the queue,
+    # and the code to get the value will pop the result from that queue (and
+    # repush the result to handle thread safety).
+    class Proxy < BaseProxy
+      def initialize
+        super
+        @queue = ::Queue.new
+      end
+
+      private
+
+      def __run
+        @queue.push(__run_block)
+      end
+
+      def __get_value
+        @value = @queue.pop
+
+        # Handle thread-safety by repushing the popped value, so that
+        # concurrent calls will receive the same value
+        @queue.push(@value)
+      end
+    end
+
+    # Object class for async job/proxy result when the :preempt_async_thread
+    # Database option is used.  Uses a mutex for synchronization, and either
+    # the JobProcessor or the calling thread can run code to get the value.
+    class PreemptableProxy < BaseProxy
+      def initialize
+        super
+        @mutex = ::Mutex.new
+      end
+
+      private
+
+      def __get_value
+        @mutex.synchronize do
+          unless defined?(@value)
+            @value = __run_block
+          end
+        end
+      end
+      alias __run __get_value
+    end
+
+    module DatabaseMethods
+      def self.extended(db)
+        db.instance_exec do
+          unless pool.pool_type == :threaded || pool.pool_type == :sharded_threaded
+            raise Error, "can only load async_thread_pool extension if using threaded or sharded_threaded connection pool"
+          end
+
+          num_async_threads = opts[:num_async_threads] ? typecast_value_integer(opts[:num_async_threads]) : (Integer(opts[:max_connections] || 4))
+          raise Error, "must have positive number for num_async_threads" if num_async_threads <= 0
+
+          proxy_klass = typecast_value_boolean(opts[:preempt_async_thread]) ? PreemptableProxy : Proxy
+          define_singleton_method(:async_job_class){proxy_klass}
+
+          queue = @async_thread_queue = Queue.new
+          pool = @async_thread_pool = num_async_threads.times.map{JobProcessor.new(queue)}
+          ObjectSpace.define_finalizer(db, JobProcessor.create_finalizer(queue, pool))
+
+          extend_datasets(DatasetMethods)
+        end
+      end
+
+      private
+
+      # Wrap the block in a job/proxy object and schedule it to run using the async thread pool.
+      def async_run(&block)
+        proxy = async_job_class.new(&block)
+        @async_thread_queue.push(proxy)
+        proxy
+      end
+    end
+
+    ASYNC_METHODS = ([:all?, :any?, :drop, :entries, :grep_v, :include?, :inject, :member?, :minmax, :none?, :one?, :reduce, :sort, :take, :tally, :to_a, :to_h, :uniq, :zip] & Enumerable.instance_methods) + (Dataset::ACTION_METHODS - [:map, :paged_each])
+    ASYNC_BLOCK_METHODS = ([:collect, :collect_concat, :detect,  :drop_while, :each_cons, :each_entry, :each_slice, :each_with_index, :each_with_object, :filter_map, :find, :find_all, :find_index, :flat_map, :max_by, :min_by, :minmax_by, :partition, :reject, :reverse_each, :sort_by, :take_while] & Enumerable.instance_methods) + [:paged_each]
+    ASYNC_ARGS_OR_BLOCK_METHODS = [:map]
+
+    module DatasetMethods
+      # Define an method in the given module that will run the given method using an async thread
+      # if the current dataset is async.
+      def self.define_async_method(mod, method)
+        mod.send(:define_method, method) do |*args, &block|
+          if @opts[:async]
+            ds = sync
+            db.send(:async_run){ds.send(method, *args, &block)}
+          else
+            super(*args, &block)
+          end
+        end
+      end
+
+      # Define an method in the given module that will run the given method using an async thread
+      # if the current dataset is async and a block is provided.
+      def self.define_async_block_method(mod, method)
+        mod.send(:define_method, method) do |*args, &block|
+          if block && @opts[:async]
+            ds = sync
+            db.send(:async_run){ds.send(method, *args, &block)}
+          else
+            super(*args, &block)
+          end
+        end
+      end
+
+      # Define an method in the given module that will run the given method using an async thread
+      # if the current dataset is async and arguments or a block is provided.
+      def self.define_async_args_or_block_method(mod, method)
+        mod.send(:define_method, method) do |*args, &block|
+          if (block || !args.empty?) && @opts[:async]
+            ds = sync
+            db.send(:async_run){ds.send(method, *args, &block)}
+          else
+            super(*args, &block)
+          end
+        end
+      end
+
+      # Override all of the methods that return results to do the processing in an async thread
+      # if they have been marked to run async and should run async (i.e. they don't return an
+      # Enumerator).
+      ASYNC_METHODS.each{|m| define_async_method(self, m)}
+      ASYNC_BLOCK_METHODS.each{|m| define_async_block_method(self, m)}
+      ASYNC_ARGS_OR_BLOCK_METHODS.each{|m| define_async_args_or_block_method(self, m)}
+
+      # Return a cloned dataset that will load results using the async thread pool.
+      def async
+        cached_dataset(:_async) do
+          clone(:async=>true)
+        end
+      end
+
+      # Return a cloned dataset that will not load results using the async thread pool.
+      # Only used if the current dataset has been marked as using the async thread pool.
+      def sync
+        cached_dataset(:_sync) do
+          clone(:async=>false)
+        end
+      end
+    end
+  end
+
+  Database.register_extension(:async_thread_pool, Database::AsyncThreadPool::DatabaseMethods)
+end

data/lib/sequel/model/base.rb

@@ -1260,12 +1260,12 @@ module Sequel
       # Once an object is frozen, you cannot modify it's values, changed_columns,
       # errors, or dataset.
       def freeze
-        values.freeze
-        _changed_columns.freeze
         unless errors.frozen?
           validate
           errors.freeze
         end
+        values.freeze
+        _changed_columns.freeze
         this if !new? && model.primary_key
         super
       end

data/lib/sequel/plugins/async_thread_pool.rb

@@ -0,0 +1,39 @@
+# frozen-string-literal: true
+
+module Sequel
+  extension 'async_thread_pool'
+
+  module Plugins
+    # The async_thread_pool plugin makes it slightly easier to use the async_thread_pool
+    # Dataset extension with models.  It makes Model.async return an async dataset for the
+    # model, and support async behavior for #destroy, #with_pk, and #with_pk! for model
+    # datasets:
+    #
+    #   # Will load the artist with primary key 1 asynchronously
+    #   artist = Artist.async.with_pk(1)
+    #
+    # You must load the async_thread_pool Database extension into the Database object the
+    # model class uses in order for async behavior to work.
+    #
+    # Usage:
+    #
+    #   # Make all model subclass datasets support support async class methods and additional
+    #   # async dataset methods
+    #   Sequel::Model.plugin :async_thread_pool
+    #
+    #   # Make the Album class support async class method and additional async dataset methods
+    #   Album.plugin :async_thread_pool
+    module AsyncThreadPool
+      module ClassMethods
+        Plugins.def_dataset_methods(self, :async)
+      end
+
+      module DatasetMethods
+        [:destroy, :with_pk, :with_pk!].each do |meth|
+          ::Sequel::Database::AsyncThreadPool::DatasetMethods.define_async_method(self, meth)
+        end
+      end
+    end
+  end
+end
+

data/lib/sequel/plugins/composition.rb

@@ -171,8 +171,9 @@ module Sequel
 
         # Freeze compositions hash when freezing model instance.
         def freeze
-          compositions.freeze
+          compositions
           super
+          compositions.freeze
         end
 
         # For each composition, set the columns in the model class based

data/lib/sequel/plugins/json_serializer.rb

@@ -133,21 +133,39 @@ module Sequel
         end
       end
       
-      # Helper class used for making sure that cascading options
-      # for model associations works correctly.  Cascaded options
-      # work by creating instances of this class, which take a
-      # literal JSON string and have +to_json+ return it.
+      # SEQUEL6: Remove
+      # :nocov:
       class Literal
-        # Store the literal JSON to use
         def initialize(json)
           @json = json
         end
         
-        # Return the literal JSON to use
         def to_json(*a)
           @json
         end
       end
+      # :nocov:
+      Sequel::Deprecation.deprecate_constant(self, :Literal)
+
+      # Convert the given object to a JSON data structure using the given arguments.
+      def self.object_to_json_data(obj, *args, &block)
+        if obj.is_a?(Array)
+          obj.map{|x| object_to_json_data(x, *args, &block)}
+        else
+          if obj.respond_to?(:to_json_data)
+            obj.to_json_data(*args, &block)
+          else
+            begin
+              Sequel.parse_json(Sequel.object_to_json(obj, *args, &block))
+            # :nocov:
+            rescue Sequel.json_parser_error_class
+              # Support for old Ruby code that only supports parsing JSON object/array
+              Sequel.parse_json(Sequel.object_to_json([obj], *args, &block))[0]
+            # :nocov:
+            end
+          end
+        end
+      end
 
       module ClassMethods
         # The default opts to use when serializing model objects to JSON.
@@ -324,20 +342,7 @@ module Sequel
                 end
 
                 v = v.empty? ? [] : [v]
-
-                objs = public_send(k)
-
-                is_array = if r = model.association_reflection(k)
-                  r.returns_array?
-                else
-                  objs.is_a?(Array)
-                end
-                
-                h[key_name] = if is_array
-                  objs.map{|obj| Literal.new(Sequel.object_to_json(obj, *v))}
-                else
-                  Literal.new(Sequel.object_to_json(objs, *v))
-                end
+                h[key_name] = JsonSerializer.object_to_json_data(public_send(k), *v)
               end
             else
               Array(inc).each do |c|
@@ -347,7 +352,8 @@ module Sequel
                 else
                   key_name = c.to_s
                 end
-                h[key_name] = public_send(c)
+
+                h[key_name] = JsonSerializer.object_to_json_data(public_send(c))
               end
             end
           end
@@ -362,6 +368,15 @@ module Sequel
           h = yield h if block_given?
           Sequel.object_to_json(h, *a)
         end
+
+        # Convert the receiver to a JSON data structure using the given arguments.
+        def to_json_data(*args, &block)
+          if block
+            to_json(*args){|x| return block.call(x)}
+          else
+            to_json(*args){|x| return x}
+          end
+        end
       end
 
       module DatasetMethods
@@ -420,7 +435,7 @@ module Sequel
             else
               all
             end
-            array.map{|obj| Literal.new(Sequel.object_to_json(obj, opts, &opts[:instance_block]))}
+            JsonSerializer.object_to_json_data(array, opts, &opts[:instance_block])
           else
             all
           end

data/lib/sequel/plugins/nested_attributes.rb

@@ -108,9 +108,10 @@ module Sequel
         #            array of the allowable fields.
         # :limit :: For *_to_many associations, a limit on the number of records
         #           that will be processed, to prevent denial of service attacks.
-        # :reject_if :: A proc that is given each attribute hash before it is
+        # :reject_if :: A proc that is called with each attribute hash before it is
         #               passed to its associated object. If the proc returns a truthy
         #               value, the attribute hash is ignored.
+        # :reject_nil :: Ignore nil objects passed to nested attributes setter methods.
         # :remove :: Allow disassociation of nested records (can remove the associated
         #            object from the parent object, but not destroy the associated object).
         # :require_modification :: Whether to require modification of nested objects when
@@ -146,8 +147,9 @@ module Sequel
         def def_nested_attribute_method(reflection)
           @nested_attributes_module.class_eval do
             meth = :"#{reflection[:name]}_attributes="
+            assoc = reflection[:name]
             define_method(meth) do |v|
-              set_nested_attributes(reflection[:name], v)
+              set_nested_attributes(assoc, v)
             end
             alias_method meth, meth
           end
@@ -161,6 +163,7 @@ module Sequel
         def set_nested_attributes(assoc, obj, opts=OPTS)
           raise(Error, "no association named #{assoc} for #{model.inspect}") unless ref = model.association_reflection(assoc)
           raise(Error, "nested attributes are not enabled for association #{assoc} for #{model.inspect}") unless meta = ref[:nested_attributes]
+          return if obj.nil? && meta[:reject_nil]
           meta = meta.merge(opts)
           meta[:reflection] = ref
           if ref.returns_array?

data/lib/sequel/plugins/serialization.rb

@@ -177,8 +177,9 @@ module Sequel
 
         # Freeze the deserialized values
         def freeze
-          deserialized_values.freeze
+          deserialized_values
           super
+          deserialized_values.freeze
         end
 
         # Serialize deserialized values before saving

data/lib/sequel/plugins/serialization_modification_detection.rb

@@ -50,8 +50,8 @@ module Sequel
         # Freeze the original deserialized values when freezing the instance.
         def freeze
           @original_deserialized_values ||= {}
-          @original_deserialized_values.freeze
           super
+          @original_deserialized_values.freeze
         end
 
         private

data/lib/sequel/version.rb

@@ -6,7 +6,7 @@ module Sequel
 
   # The minor version of Sequel.  Bumped for every non-patch level
   # release, generally around once a month.
-  MINOR = 41
+  MINOR = 42
 
   # The tiny version of Sequel.  Usually 0, only bumped for bugfix
   # releases that fix regressions from previous versions.

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sequel
 version: !ruby/object:Gem::Version
-  version: 5.41.0
+  version: 5.42.0
 platform: ruby
 authors:
 - Jeremy Evans
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-02-01 00:00:00.000000000 Z
+date: 2021-03-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: minitest
@@ -185,6 +185,7 @@ extra_rdoc_files:
 - doc/release_notes/5.4.0.txt
 - doc/release_notes/5.40.0.txt
 - doc/release_notes/5.41.0.txt
+- doc/release_notes/5.42.0.txt
 - doc/release_notes/5.5.0.txt
 - doc/release_notes/5.6.0.txt
 - doc/release_notes/5.7.0.txt
@@ -254,6 +255,7 @@ files:
 - doc/release_notes/5.4.0.txt
 - doc/release_notes/5.40.0.txt
 - doc/release_notes/5.41.0.txt
+- doc/release_notes/5.42.0.txt
 - doc/release_notes/5.5.0.txt
 - doc/release_notes/5.6.0.txt
 - doc/release_notes/5.7.0.txt
@@ -352,6 +354,7 @@ files:
 - lib/sequel/extensions/_pretty_table.rb
 - lib/sequel/extensions/any_not_empty.rb
 - lib/sequel/extensions/arbitrary_servers.rb
+- lib/sequel/extensions/async_thread_pool.rb
 - lib/sequel/extensions/auto_literal_strings.rb
 - lib/sequel/extensions/blank.rb
 - lib/sequel/extensions/caller_logging.rb
@@ -447,6 +450,7 @@ files:
 - lib/sequel/plugins/association_multi_add_remove.rb
 - lib/sequel/plugins/association_pks.rb
 - lib/sequel/plugins/association_proxies.rb
+- lib/sequel/plugins/async_thread_pool.rb
 - lib/sequel/plugins/auto_validations.rb
 - lib/sequel/plugins/before_after_save.rb
 - lib/sequel/plugins/blacklist_security.rb