sequel 5.39.0 → 5.44.0

data/lib/sequel/adapters/jdbc.rb

@@ -71,11 +71,11 @@ module Sequel
     class TypeConvertor
       CONVERTORS = convertors = {}
       %w'Boolean Float Double Int Long Short'.each do |meth|
-        x = convertors[meth.to_sym] = Object.new
+        x = x = convertors[meth.to_sym] = Object.new
         class_eval("def x.call(r, i) v = r.get#{meth}(i); v unless r.wasNull end", __FILE__, __LINE__)
       end
       %w'Object Array String Time Date Timestamp BigDecimal Blob Bytes Clob'.each do |meth|
-        x = convertors[meth.to_sym] = Object.new
+        x = x = convertors[meth.to_sym] = Object.new
         class_eval("def x.call(r, i) r.get#{meth}(i) end", __FILE__, __LINE__)
       end
       x = convertors[:RubyTime] = Object.new

data/lib/sequel/adapters/shared/postgres.rb

@@ -1500,9 +1500,11 @@ module Sequel
       # disallowed or there is a size specified, use the varchar type.
       # Otherwise use the text type.
       def type_literal_generic_string(column)
-        if column[:fixed]
+        if column[:text]
+          :text
+        elsif column[:fixed]
           "char(#{column[:size]||255})"
-        elsif column[:text] == false or column[:size]
+        elsif column[:text] == false || column[:size]
           "varchar(#{column[:size]||255})"
         else
           :text

data/lib/sequel/adapters/shared/sqlite.rb

@@ -424,10 +424,10 @@ module Sequel
         skip_indexes = []
         indexes(table, :only_autocreated=>true).each do |name, h|
           skip_indexes << name
-          if h[:unique]
+          if h[:unique] && !opts[:no_unique]
             if h[:columns].length == 1
               unique_columns.concat(h[:columns])
-            elsif h[:columns].map(&:to_s) != pks && !opts[:no_unique]
+            elsif h[:columns].map(&:to_s) != pks
               constraints << {:type=>:unique, :columns=>h[:columns]}
             end
           end
@@ -561,7 +561,7 @@ module Sequel
       Dataset.def_sql_method(self, :delete, [['if db.sqlite_version >= 30803', %w'with delete from where'], ["else", %w'delete from where']])
       Dataset.def_sql_method(self, :insert, [['if db.sqlite_version >= 30803', %w'with insert conflict into columns values on_conflict'], ["else", %w'insert conflict into columns values']])
       Dataset.def_sql_method(self, :select, [['if opts[:values]', %w'with values compounds'], ['else', %w'with select distinct columns from join where group having window compounds order limit lock']])
-      Dataset.def_sql_method(self, :update, [['if db.sqlite_version >= 30803', %w'with update table set where'], ["else", %w'update table set where']])
+      Dataset.def_sql_method(self, :update, [['if db.sqlite_version >= 33300', %w'with update table set from where'], ['elsif db.sqlite_version >= 30803', %w'with update table set where'], ["else", %w'update table set where']])
 
       def cast_sql_append(sql, expr, type)
         if type == Time or type == DateTime
@@ -753,6 +753,11 @@ module Sequel
         false
       end
 
+      # SQLite does not support deleting from a joined dataset
+      def supports_deleting_joins?
+        false
+      end
+
       # SQLite does not support INTERSECT ALL or EXCEPT ALL
       def supports_intersect_except_all?
         false
@@ -763,6 +768,11 @@ module Sequel
         false
       end
       
+      # SQLite 3.33.0 supports modifying joined datasets
+      def supports_modifying_joins?
+        db.sqlite_version >= 33300
+      end
+
       # SQLite does not support multiple columns for the IN/NOT IN operators
       def supports_multiple_column_in?
         false
@@ -825,6 +835,13 @@ module Sequel
         end
       end
 
+      # Raise an InvalidOperation exception if insert is not allowed for this dataset.
+      def check_insert_allowed!
+        raise(InvalidOperation, "Grouped datasets cannot be modified") if opts[:group]
+        raise(InvalidOperation, "Joined datasets cannot be modified") if joined_dataset?
+      end
+      alias check_delete_allowed! check_insert_allowed!
+
       # SQLite supports a maximum of 500 rows in a VALUES clause.
       def default_import_slice
         500
@@ -944,6 +961,23 @@ module Sequel
       def _truncate_sql(table)
         "DELETE FROM #{table}"
       end
+
+      # Use FROM to specify additional tables in an update query
+      def update_from_sql(sql)
+        if(from = @opts[:from][1..-1]).empty?
+          raise(Error, 'Need multiple FROM tables if updating/deleting a dataset with JOINs') if @opts[:join]
+        else
+          sql << ' FROM '
+          source_list_append(sql, from)
+          select_join_sql(sql)
+        end
+      end
+
+      # Only include the primary table in the main update clause
+      def update_table_sql(sql)
+        sql << ' '
+        source_list_append(sql, @opts[:from][0..0])
+      end
     end
   end
 end

data/lib/sequel/core.rb

@@ -176,6 +176,17 @@ module Sequel
       JSON.parse(json, :create_additions=>false)
     end
 
+    # If a mutex is given, synchronize access using it.  If nil is given, just
+    # yield to the block.  This is designed for cases where a mutex may or may
+    # not be provided.
+    def synchronize_with(mutex)
+      if mutex
+        mutex.synchronize{yield}
+      else
+        yield
+      end
+    end
+
     # Convert each item in the array to the correct type, handling multi-dimensional
     # arrays.  For each element in the array or subarrays, call the converter,
     # unless the value is nil.

data/lib/sequel/database/misc.rb

@@ -213,8 +213,7 @@ module Sequel
       Sequel.extension(*exts)
       exts.each do |ext|
         if pr = Sequel.synchronize{EXTENSIONS[ext]}
-          unless Sequel.synchronize{@loaded_extensions.include?(ext)}
-            Sequel.synchronize{@loaded_extensions << ext}
+          if Sequel.synchronize{@loaded_extensions.include?(ext) ? false : (@loaded_extensions << ext)}
             pr.call(self)
           end
         else

data/lib/sequel/database/schema_generator.rb

@@ -159,7 +159,7 @@ module Sequel
         nil
       end
       
-      # Adds a named constraint (or unnamed if name is nil),
+      # Adds a named CHECK constraint (or unnamed if name is nil),
       # with the given block or args. To provide options for the constraint, pass
       # a hash as the first argument.
       #
@@ -167,6 +167,15 @@ module Sequel
       #   # CONSTRAINT blah CHECK num >= 1 AND num <= 5
       #   constraint({name: :blah, deferrable: true}, num: 1..5)
       #   # CONSTRAINT blah CHECK num >= 1 AND num <= 5 DEFERRABLE INITIALLY DEFERRED
+      #
+      # If the first argument is a hash, the following options are supported:
+      #
+      # Options:
+      # :name :: The name of the CHECK constraint
+      # :deferrable :: Whether the CHECK constraint should be marked DEFERRABLE.
+      #
+      # PostgreSQL specific options:
+      # :not_valid :: Whether the CHECK constraint should be marked NOT VALID.
       def constraint(name, *args, &block)
         opts = name.is_a?(Hash) ? name : {:name=>name}
         constraints << opts.merge(:type=>:check, :check=>block || args)
@@ -205,14 +214,12 @@ module Sequel
       end
 
       # Add a full text index on the given columns.
+      # See #index for additional options.
       #
       # PostgreSQL specific options:
       # :index_type :: Can be set to :gist to use a GIST index instead of the
       #                default GIN index.
       # :language :: Set a language to use for the index (default: simple).
-      #
-      # Microsoft SQL Server specific options:
-      # :key_index :: The KEY INDEX to use for the full text index.
       def full_text_index(columns, opts = OPTS)
         index(columns, opts.merge(:type => :full_text))
       end
@@ -222,35 +229,43 @@ module Sequel
         columns.any?{|c| c[:name] == name}
       end
       
-      # Add an index on the given column(s) with the given options.
+      # Add an index on the given column(s) with the given options. Examples:
+      #
+      #   index :name
+      #   # CREATE INDEX table_name_index ON table (name)
+      #
+      #   index [:artist_id, :name]
+      #   # CREATE INDEX table_artist_id_name_index ON table (artist_id, name)
+      #
+      #   index [:artist_id, :name], name: :foo
+      #   # CREATE INDEX foo ON table (artist_id, name)
+      #
       # General options:
       #
+      # :include :: Include additional column values in the index, without
+      #             actually indexing on those values (only supported by
+      #             some databases).
       # :name :: The name to use for the index. If not given, a default name
       #          based on the table and columns is used.
-      # :type :: The type of index to use (only supported by some databases)
+      # :type :: The type of index to use (only supported by some databases,
+      #          :full_text and :spatial values are handled specially).
       # :unique :: Make the index unique, so duplicate values are not allowed.
-      # :where :: Create a partial index (only supported by some databases)
+      # :where :: A filter expression, used to create a partial index (only
+      #           supported by some databases).
       #
       # PostgreSQL specific options:
       #
       # :concurrently :: Create the index concurrently, so it doesn't block
       #                  operations on the table while the index is being
       #                  built.
-      # :opclass :: Use a specific operator class in the index.
-      # :include :: Include additional column values in the index, without
-      #             actually indexing on those values (PostgreSQL 11+).
+      # :if_not_exists :: Only create the index if an index of the same name doesn't already exist.
+      # :opclass :: Set an opclass to use for all columns (per-column opclasses require
+      #             custom SQL).
       # :tablespace :: Specify tablespace for index.
       #
       # Microsoft SQL Server specific options:
       #
-      # :include :: Include additional column values in the index, without
-      #             actually indexing on those values.
-      #
-      #   index :name
-      #   # CREATE INDEX table_name_index ON table (name)
-      #
-      #   index [:artist_id, :name]
-      #   # CREATE INDEX table_artist_id_name_index ON table (artist_id, name)
+      # :key_index :: Sets the KEY INDEX to the given value.
       def index(columns, opts = OPTS)
         indexes << {:columns => Array(columns)}.merge!(opts)
         nil
@@ -316,6 +331,7 @@ module Sequel
       end
       
       # Add a spatial index on the given columns.
+      # See #index for additional options.
       def spatial_index(columns, opts = OPTS)
         index(columns, opts.merge(:type => :spatial))
       end
@@ -442,7 +458,7 @@ module Sequel
       end
       
       # Add a full text index on the given columns.
-      # See CreateTableGenerator#index for available options.
+      # See CreateTableGenerator#full_text_index for available options.
       def add_full_text_index(columns, opts = OPTS)
         add_index(columns, {:type=>:full_text}.merge!(opts))
       end
@@ -451,34 +467,6 @@ module Sequel
       # CreateTableGenerator#index for available options.
       #
       #   add_index(:artist_id) # CREATE INDEX table_artist_id_index ON table (artist_id)
-      #
-      # Options:
-      #
-      # :name :: Give a specific name for the index. Highly recommended if you plan on
-      #          dropping the index later.
-      # :where :: A filter expression, used to setup a partial index (if supported).
-      # :unique :: Create a unique index.
-      #
-      # PostgreSQL specific options:
-      #
-      # :concurrently :: Create the index concurrently, so it doesn't require an exclusive lock
-      #                  on the table.
-      # :index_type :: The underlying index type to use for a full_text index, gin by default).
-      # :language :: The language to use for a full text index (simple by default).
-      # :opclass :: Set an opclass to use for all columns (per-column opclasses require
-      #             custom SQL).
-      # :type :: Set the index type (e.g. full_text, spatial, hash, gin, gist, btree).
-      # :if_not_exists :: Only create the index if an index of the same name doesn't already exists
-      #
-      # MySQL specific options:
-      #
-      # :type :: Set the index type, with full_text and spatial indexes handled specially.
-      #
-      # Microsoft SQL Server specific options:
-      #
-      # :include :: Includes additional columns in the index.
-      # :key_index :: Sets the KEY INDEX to the given value.
-      # :type :: clustered uses a clustered index, full_text uses a full text index.
       def add_index(columns, opts = OPTS)
         @operations << {:op => :add_index, :columns => Array(columns)}.merge!(opts)
         nil

data/lib/sequel/database/schema_methods.rb

@@ -262,6 +262,10 @@ module Sequel
     #   # SELECT * FROM items WHERE foo
     #   # WITH CHECK OPTION
     #
+    #   DB.create_view(:bar_items, DB[:items].select(:foo), columns: [:bar])
+    #   # CREATE VIEW bar_items (bar) AS
+    #   # SELECT foo FROM items
+    #
     # Options:
     # :columns :: The column names to use for the view.  If not given,
     #             automatically determined based on the input dataset.

data/lib/sequel/dataset/features.rb

@@ -51,6 +51,11 @@ module Sequel
       false
     end
 
+    # Whether deleting from joined datasets is supported, false by default.
+    def supports_deleting_joins?
+      supports_modifying_joins?
+    end
+    
     # Whether the database supports derived column lists (e.g.
     # "table_expr AS table_alias(column_alias1, column_alias2, ...)"), true by
     # default.
@@ -178,6 +183,11 @@ module Sequel
       true
     end
     
+    # Whether updating joined datasets is supported, false by default.
+    def supports_updating_joins?
+      supports_modifying_joins?
+    end
+    
     # Whether the dataset supports the WINDOW clause to define windows used by multiple
     # window functions, false by default.
     def supports_window_clause?

data/lib/sequel/dataset/prepared_statements.rb

@@ -201,7 +201,9 @@ module Sequel
         when :insert_pk
           fetch_rows(prepared_sql){|r| return r.values.first}
         when Array
+          # :nocov:
           case prepared_type[0]
+          # :nocov:
           when :map, :as_hash, :to_hash, :to_hash_groups
             public_send(*prepared_type, &block) 
           end

data/lib/sequel/dataset/sql.rb

@@ -22,7 +22,7 @@ module Sequel
     def insert_sql(*values)
       return static_sql(@opts[:sql]) if @opts[:sql]
 
-      check_modification_allowed!
+      check_insert_allowed!
 
       columns = []
 
@@ -172,7 +172,7 @@ module Sequel
     # than one table.
     def update_sql(values = OPTS)
       return static_sql(opts[:sql]) if opts[:sql]
-      check_modification_allowed!
+      check_update_allowed!
       check_not_limited!(:update)
 
       case values
@@ -215,7 +215,7 @@ module Sequel
       lines << "def #{'_' if priv}#{type}_sql"
       lines << 'if sql = opts[:sql]; return static_sql(sql) end' unless priv
       lines << "if sql = cache_get(:_#{type}_sql); return sql end" if cacheable
-      lines << 'check_modification_allowed!' << 'check_not_limited!(:delete)' if type == :delete
+      lines << 'check_delete_allowed!' << 'check_not_limited!(:delete)' if type == :delete
       lines << 'sql = @opts[:append_sql] || sql_string_origin'
 
       if clauses.all?{|c| c.is_a?(Array)}
@@ -918,10 +918,35 @@ module Sequel
       !@opts[:no_cache_sql] && !cache_get(:_no_cache_sql)
     end
 
-    # Raise an InvalidOperation exception if deletion is not allowed for this dataset.
+    # Raise an InvalidOperation exception if modification is not allowed for this dataset.
+    # Check whether it is allowed to insert into this dataset.
+    # Only for backwards compatibility with older external adapters.
     def check_modification_allowed!
+      # SEQUEL6: Remove
+      Sequel::Deprecation.deprecate("Dataset#check_modification_allowed!", "Use check_{insert,delete,update,truncation}_allowed! instead")
+      _check_modification_allowed!(supports_modifying_joins?)
+    end
+
+    # Check whether it is allowed to insert into this dataset.
+    def check_insert_allowed!
+      _check_modification_allowed!(false)
+    end
+    alias check_truncation_allowed! check_insert_allowed!
+
+    # Check whether it is allowed to delete from this dataset.
+    def check_delete_allowed!
+      _check_modification_allowed!(supports_deleting_joins?)
+    end
+
+    # Check whether it is allowed to update this dataset.
+    def check_update_allowed!
+      _check_modification_allowed!(supports_updating_joins?)
+    end
+
+    # Internals of the check_*_allowed! methods
+    def _check_modification_allowed!(modifying_joins_supported)
       raise(InvalidOperation, "Grouped datasets cannot be modified") if opts[:group]
-      raise(InvalidOperation, "Joined datasets cannot be modified") if !supports_modifying_joins? && joined_dataset?
+      raise(InvalidOperation, "Joined datasets cannot be modified") if !modifying_joins_supported && joined_dataset?
     end
 
     # Raise error if the dataset uses limits or offsets.
@@ -930,11 +955,6 @@ module Sequel
       raise InvalidOperation, "Dataset##{type} not supported on datasets with limits or offsets" if opts[:limit] || opts[:offset]
     end
 
-    # Alias of check_modification_allowed!
-    def check_truncation_allowed!
-      check_modification_allowed!
-    end
-
     # Append column list to SQL string.
     # If the column list is empty, a wildcard (*) is appended.
     def column_list_append(sql, columns)
@@ -971,7 +991,9 @@ module Sequel
     # operators unsupported by some databases. Used by adapters for databases
     # that don't support the operators natively.
     def complex_expression_emulate_append(sql, op, args)
+      # :nocov:
       case op
+      # :nocov:
       when :%
         complex_expression_arg_pairs_append(sql, args){|a, b| Sequel.function(:MOD, a, b)}
       when :>>

data/lib/sequel/extensions/async_thread_pool.rb

@@ -0,0 +1,438 @@
+# frozen-string-literal: true
+#
+# The async_thread_pool extension adds support for running database
+# queries in a separate threads using a thread pool. With the following
+# code
+#
+#   DB.extension :async_thread_pool
+#   foos = DB[:foos].async.where{:name=>'A'..'M'}.all
+#   bar_names = DB[:bar].async.select_order_map(:name)
+#   baz_1 = DB[:bazes].async.first(:id=>1)
+#
+# All 3 queries will be run in separate threads.  +foos+, +bar_names+
+# and +baz_1+ will be proxy objects.  Calling a method on the proxy
+# object will wait for the query to be run, and will return the result
+# of calling that method on the result of the query method. For example,
+# if you run:
+#
+#   foos = DB[:foos].async.where{:name=>'A'..'M'}.all
+#   bar_names = DB[:bars].async.select_order_map(:name)
+#   baz_1 = DB[:bazes].async.first(:id=>1)
+#   sleep(1)
+#   foos.size
+#   bar_names.first
+#   baz_1.name
+#
+# These three queries will generally be run concurrently in separate
+# threads.  If you instead run:
+#   
+#   DB[:foos].async.where{:name=>'A'..'M'}.all.size
+#   DB[:bars].async.select_order_map(:name).first
+#   DB[:bazes].async.first(:id=>1).name
+#
+# Then will run each query sequentially, since you need the result of
+# one query before running the next query.  The queries will still be
+# run in separate threads (by default).
+#
+# What is run in the separate thread is the entire method call that
+# returns results.  So with the original example:
+#
+#   foos = DB[:foos].async.where{:name=>'A'..'M'}.all
+#   bar_names = DB[:bars].async.select_order_map(:name)
+#   baz_1 = DB[:bazes].async.first(:id=>1)
+#
+# The +all+, <tt>select_order_map(:name)</tt>, and <tt>first(:id=>1)</tt>
+# calls are run in separate threads.  If a block is passed to a method
+# such as +all+ or +each+, the block is also run in that thread.  If you
+# have code such as:
+#
+#   h = {}
+#   DB[:foos].async.each{|row| h[row[:id]] = row}
+#   bar_names = DB[:bars].async.select_order_map(:name)
+#   p h
+#
+# You may end up with it printing an empty hash or partial hash, because the
+# async +each+ call will not have run or finished running.  Since the
+# <tt>p h</tt> code relies on a side-effect of the +each+ block and not the
+# return value of the +each+ call, it will not wait for the loading.
+#
+# You should avoid using +async+ for any queries where you are ignoring the
+# return value, as otherwise you have no way to wait for the query to be run.
+#
+# Datasets that use async will use async threads to load data for the majority
+# of methods that can return data.  However, dataset methods that return
+# enumerators will not use an async thread (e.g. calling # Dataset#map
+# without a block or arguments does not use an async thread or return a
+# proxy object).
+#
+# Because async methods (including their blocks) run in a separate thread, you
+# should not use control flow modifiers such as +return+ or +break+ in async
+# queries.  Doing so will result in a error.
+#
+# Because async results are returned as proxy objects, it's a bad idea
+# to use them in a boolean setting:
+#
+#   result = DB[:foo].async.get(:boolean_column)
+#   # or:
+#   result = DB[:foo].async.first
+#
+#   # ...
+#   if result 
+#     # will always execute this banch, since result is a proxy object
+#   end
+#
+# In this case, you can call the +__value+ method to return the actual
+# result:
+#
+#   if result.__value
+#     # will not execute this branch if the dataset method returned nil or false
+#   end
+#
+# Similarly, because a proxy object is used, you should be careful using the
+# result in a case statement or an argument to <tt>Class#===</tt>:
+#
+#   # ...
+#   case result
+#   when Hash, true, false
+#     # will never take this branch, since result is a proxy object
+#   end
+#
+# Similar to usage in an +if+ statement, you should use +__value+:
+#
+#   case result.__value
+#   when Hash, true, false
+#     # will never take this branch, since result is a proxy object
+#   end
+#
+# On Ruby 2.2+, you can use +itself+ instead of +__value+.  It's preferable to
+# use +itself+ if you can, as that will allow code to work with both proxy
+# objects and regular objects.
+#
+# Because separate threads and connections are used for async queries,
+# they do not use any state on the current connection/thread. So if
+# you do:
+#
+#   DB.transaction{DB[:table].async.all}
+#
+# Be aware that the transaction runs on one connection, and the SELECT
+# query on a different connection.  If you use currently using
+# transactional testing (running each test inside a transaction/savepoint),
+# and want to start using this extension, you should first switch to
+# non-transactional testing of the code that will use the async thread
+# pool before using this extension, as otherwise the use of
+# <tt>Dataset#async</tt> will likely break your tests.
+#
+# If you are using Database#synchronize to checkout a connection, the
+# same issue applies, where the async query runs on a different
+# connection:
+#
+#   DB.synchronize{DB[:table].async.all}
+# 
+# Similarly, if you are using the server_block extension, any async
+# queries inside with_server blocks will not use the server specified:
+#
+#   DB.with_server(:shard1) do
+#     DB[:a].all # Uses shard1
+#     DB[:a].async.all # Uses default shard
+#   end
+#
+# You need to manually specify the shard for any dataset using an async
+# query:
+# 
+#   DB.with_server(:shard1) do
+#     DB[:a].all # Uses shard1
+#     DB[:a].async.server(:shard1).all # Uses shard1
+#   end
+#
+# When the async_thread_pool extension, the size of the async thread pool
+# can be set by using the +:num_async_threads+ Database option, which must
+# be set before loading the async_thread_pool extension.  This defaults
+# to the size of the Database object's connection pool.
+#
+# By default, for consistent behavior, the async_thread_pool extension
+# will always run the query in a separate thread. However, in some cases,
+# such as when the async thread pool is busy and the results of a query
+# are needed right away, it can improve performance to allow preemption,
+# so that the query will run in the current thread instead of waiting
+# for an async thread to become available.  With the following code:
+#
+#   foos = DB[:foos].async.where{:name=>'A'..'M'}.all
+#   bar_names = DB[:bar].async.select_order_map(:name)
+#   if foos.length > 4
+#     baz_1 = DB[:bazes].async.first(:id=>1)
+#   end
+# 
+# Whether you need the +baz_1+ variable depends on the value of foos.
+# If the async thread pool is busy, and by the time the +foos.length+
+# call is made, the async thread pool has not started the processing
+# to get the +foos+ value, it can improve performance to start that
+# processing in the current thread, since it is needed immediately to
+# determine whether to schedule query to get the +baz_1+ variable.
+# The default is to not allow preemption, because if the current
+# thread is used, it may have already checked out a connection that
+# could be used, and that connection could be inside a transaction or
+# have some other manner of connection-specific state applied to it.
+# If you want to allow preemption, you can set the
+# +:preempt_async_thread+ Database option before loading the
+# async_thread_pool extension.
+#
+# Related module: Sequel::Database::AsyncThreadPool::DatasetMethods
+
+
+# 
+module Sequel
+  module Database::AsyncThreadPool
+    # JobProcessor is a wrapper around a single thread, that will
+    # process a queue of jobs until it is shut down.
+    class JobProcessor # :nodoc:
+      def self.create_finalizer(queue, pool)
+        proc{run_finalizer(queue, pool)}
+      end
+
+      def self.run_finalizer(queue, pool)
+        # Push a nil for each thread using the queue, signalling
+        # that thread to close.
+        pool.each{queue.push(nil)}
+
+        # Join each of the closed threads.
+        pool.each(&:join)
+
+        # Clear the thread pool.  Probably not necessary, but this allows
+        # for a simple way to check whether this finalizer has been run.
+        pool.clear
+
+        nil
+      end
+      private_class_method :run_finalizer
+
+      def initialize(queue)
+        @thread = ::Thread.new do
+          while proxy = queue.pop
+            proxy.__send__(:__run)
+          end
+        end
+      end
+
+      # Join the thread, should only be called by the related finalizer.
+      def join
+        @thread.join
+      end
+    end
+
+    # Wrapper for exception instances raised by async jobs.  The
+    # wrapped exception will be raised by the code getting the value
+    # of the job.
+    WrappedException = Struct.new(:exception)
+
+    # Base proxy object class for jobs processed by async threads and
+    # the returned result.
+    class BaseProxy < BasicObject
+      # Store a block that returns the result when called.
+      def initialize(&block)
+        ::Kernel.raise Error, "must provide block for an async job" unless block
+        @block = block
+      end
+
+      # Pass all method calls to the returned result.
+      def method_missing(*args, &block)
+        __value.public_send(*args, &block)
+      end
+      # :nocov:
+      ruby2_keywords(:method_missing) if respond_to?(:ruby2_keywords, true)
+      # :nocov:
+
+      # Delegate respond_to? calls to the returned result.
+      def respond_to_missing?(*args)
+        __value.respond_to?(*args)
+      end
+
+      # Override some methods defined by default so they apply to the
+      # returned result and not the current object.
+      [:!, :==, :!=, :instance_eval, :instance_exec].each do |method|
+        define_method(method) do |*args, &block|
+          __value.public_send(method, *args, &block)
+        end
+      end
+
+      # Wait for the value to be loaded if it hasn't already been loaded.
+      # If the code to load the return value raised an exception that was
+      # wrapped, reraise the exception.
+      def __value
+        unless defined?(@value)
+          __get_value
+        end
+
+        if @value.is_a?(WrappedException)
+          ::Kernel.raise @value
+        end
+
+        @value
+      end
+
+      private
+
+      # Run the block and return the block value.  If the block call raises
+      # an exception, wrap the exception.
+      def __run_block
+        # This may not catch concurrent calls (unless surrounded by a mutex), but
+        # it's not worth trying to protect against that.  It's enough to just check for
+        # multiple non-concurrent calls.
+        ::Kernel.raise Error, "Cannot run async block multiple times" unless block = @block
+
+        @block = nil
+
+        begin
+          block.call
+        rescue ::Exception => e
+          WrappedException.new(e)
+        end
+      end
+    end
+
+    # Default object class for async job/proxy result.  This uses a queue for
+    # synchronization.  The JobProcessor will push a result until the queue,
+    # and the code to get the value will pop the result from that queue (and
+    # repush the result to handle thread safety).
+    class Proxy < BaseProxy
+      def initialize
+        super
+        @queue = ::Queue.new
+      end
+
+      private
+
+      def __run
+        @queue.push(__run_block)
+      end
+
+      def __get_value
+        @value = @queue.pop
+
+        # Handle thread-safety by repushing the popped value, so that
+        # concurrent calls will receive the same value
+        @queue.push(@value)
+      end
+    end
+
+    # Object class for async job/proxy result when the :preempt_async_thread
+    # Database option is used.  Uses a mutex for synchronization, and either
+    # the JobProcessor or the calling thread can run code to get the value.
+    class PreemptableProxy < BaseProxy
+      def initialize
+        super
+        @mutex = ::Mutex.new
+      end
+
+      private
+
+      def __get_value
+        @mutex.synchronize do
+          unless defined?(@value)
+            @value = __run_block
+          end
+        end
+      end
+      alias __run __get_value
+    end
+
+    module DatabaseMethods
+      def self.extended(db)
+        db.instance_exec do
+          unless pool.pool_type == :threaded || pool.pool_type == :sharded_threaded
+            raise Error, "can only load async_thread_pool extension if using threaded or sharded_threaded connection pool"
+          end
+
+          num_async_threads = opts[:num_async_threads] ? typecast_value_integer(opts[:num_async_threads]) : (Integer(opts[:max_connections] || 4))
+          raise Error, "must have positive number for num_async_threads" if num_async_threads <= 0
+
+          proxy_klass = typecast_value_boolean(opts[:preempt_async_thread]) ? PreemptableProxy : Proxy
+          define_singleton_method(:async_job_class){proxy_klass}
+
+          queue = @async_thread_queue = Queue.new
+          pool = @async_thread_pool = num_async_threads.times.map{JobProcessor.new(queue)}
+          ObjectSpace.define_finalizer(db, JobProcessor.create_finalizer(queue, pool))
+
+          extend_datasets(DatasetMethods)
+        end
+      end
+
+      private
+
+      # Wrap the block in a job/proxy object and schedule it to run using the async thread pool.
+      def async_run(&block)
+        proxy = async_job_class.new(&block)
+        @async_thread_queue.push(proxy)
+        proxy
+      end
+    end
+
+    ASYNC_METHODS = ([:all?, :any?, :drop, :entries, :grep_v, :include?, :inject, :member?, :minmax, :none?, :one?, :reduce, :sort, :take, :tally, :to_a, :to_h, :uniq, :zip] & Enumerable.instance_methods) + (Dataset::ACTION_METHODS - [:map, :paged_each])
+    ASYNC_BLOCK_METHODS = ([:collect, :collect_concat, :detect,  :drop_while, :each_cons, :each_entry, :each_slice, :each_with_index, :each_with_object, :filter_map, :find, :find_all, :find_index, :flat_map, :max_by, :min_by, :minmax_by, :partition, :reject, :reverse_each, :sort_by, :take_while] & Enumerable.instance_methods) + [:paged_each]
+    ASYNC_ARGS_OR_BLOCK_METHODS = [:map]
+
+    module DatasetMethods
+      # Define an method in the given module that will run the given method using an async thread
+      # if the current dataset is async.
+      def self.define_async_method(mod, method)
+        mod.send(:define_method, method) do |*args, &block|
+          if @opts[:async]
+            ds = sync
+            db.send(:async_run){ds.send(method, *args, &block)}
+          else
+            super(*args, &block)
+          end
+        end
+      end
+
+      # Define an method in the given module that will run the given method using an async thread
+      # if the current dataset is async and a block is provided.
+      def self.define_async_block_method(mod, method)
+        mod.send(:define_method, method) do |*args, &block|
+          if block && @opts[:async]
+            ds = sync
+            db.send(:async_run){ds.send(method, *args, &block)}
+          else
+            super(*args, &block)
+          end
+        end
+      end
+
+      # Define an method in the given module that will run the given method using an async thread
+      # if the current dataset is async and arguments or a block is provided.
+      def self.define_async_args_or_block_method(mod, method)
+        mod.send(:define_method, method) do |*args, &block|
+          if (block || !args.empty?) && @opts[:async]
+            ds = sync
+            db.send(:async_run){ds.send(method, *args, &block)}
+          else
+            super(*args, &block)
+          end
+        end
+      end
+
+      # Override all of the methods that return results to do the processing in an async thread
+      # if they have been marked to run async and should run async (i.e. they don't return an
+      # Enumerator).
+      ASYNC_METHODS.each{|m| define_async_method(self, m)}
+      ASYNC_BLOCK_METHODS.each{|m| define_async_block_method(self, m)}
+      ASYNC_ARGS_OR_BLOCK_METHODS.each{|m| define_async_args_or_block_method(self, m)}
+
+      # Return a cloned dataset that will load results using the async thread pool.
+      def async
+        cached_dataset(:_async) do
+          clone(:async=>true)
+        end
+      end
+
+      # Return a cloned dataset that will not load results using the async thread pool.
+      # Only used if the current dataset has been marked as using the async thread pool.
+      def sync
+        cached_dataset(:_sync) do
+          clone(:async=>false)
+        end
+      end
+    end
+  end
+
+  Database.register_extension(:async_thread_pool, Database::AsyncThreadPool::DatabaseMethods)
+end