sequel 5.77.0 → 5.81.0

data/lib/sequel/dataset/query.rb

@@ -43,7 +43,7 @@ module Sequel
       add_graph_aliases distinct except exclude exclude_having
       filter for_update from from_self graph grep group group_and_count group_append group_by having intersect invert
       limit lock_style naked offset or order order_append order_by order_more order_prepend qualify
-      reverse reverse_order select select_all select_append select_group select_more server
+      reverse reverse_order select select_all select_append select_group select_more select_prepend server
       set_graph_aliases unfiltered ungraphed ungrouped union
       unlimited unordered where with with_recursive with_sql
     METHS
@@ -129,6 +129,7 @@ module Sequel
     def distinct(*args, &block)
       virtual_row_columns(args, block)
       if args.empty?
+        return self if opts[:distinct] == EMPTY_ARRAY
         cached_dataset(:_distinct_ds){clone(:distinct => EMPTY_ARRAY)}
       else
         raise(InvalidOperation, "DISTINCT ON not supported") unless supports_distinct_on?
@@ -230,6 +231,7 @@ module Sequel
     #
     #   DB[:table].for_update # SELECT * FROM table FOR UPDATE
     def for_update
+       return self if opts[:lock] == :update
       cached_dataset(:_for_update_ds){lock_style(:update)}
     end
 
@@ -641,6 +643,7 @@ module Sequel
     #   DB.from(:a, DB[:b].where(Sequel[:a][:c]=>Sequel[:b][:d]).lateral)
     #   # SELECT * FROM a, LATERAL (SELECT * FROM b WHERE (a.c = b.d))
     def lateral
+      return self if opts[:lateral]
       cached_dataset(:_lateral_ds){clone(:lateral=>true)}
     end
 
@@ -744,6 +747,7 @@ module Sequel
     #   ds.all # => [{2=>:id}]
     #   ds.naked.all # => [{:id=>2}]
     def naked
+      return self unless opts[:row_proc]
       cached_dataset(:_naked_ds){with_row_proc(nil)}
     end
 
@@ -753,6 +757,7 @@ module Sequel
     #   DB[:items].for_update.nowait
     #   # SELECT * FROM items FOR UPDATE NOWAIT
     def nowait
+      return self if opts[:nowait]
       cached_dataset(:_nowait_ds) do
         raise(Error, 'This dataset does not support raises errors instead of waiting for locked rows') unless supports_nowait?
         clone(:nowait=>true)
@@ -878,6 +883,7 @@ module Sequel
     #   end
     def returning(*values)
       if values.empty?
+        return self if opts[:returning] == EMPTY_ARRAY
         cached_dataset(:_returning_ds) do
           raise Error, "RETURNING is not supported on #{db.database_type}" unless supports_returning?(:insert)
           clone(:returning=>EMPTY_ARRAY)
@@ -930,6 +936,7 @@ module Sequel
     #   DB[:items].select_all(:items, :foo) # SELECT items.*, foo.* FROM items
     def select_all(*tables)
       if tables.empty?
+        return self unless opts[:select]
         cached_dataset(:_select_all_ds){clone(:select => nil)}
       else
         select(*tables.map{|t| i, a = split_alias(t); a || i}.map!{|t| SQL::ColumnAll.new(t)}.freeze)
@@ -944,14 +951,8 @@ module Sequel
     #   DB[:items].select(:a).select_append(:b) # SELECT a, b FROM items
     #   DB[:items].select_append(:b) # SELECT *, b FROM items
     def select_append(*columns, &block)
-      cur_sel = @opts[:select]
-      if !cur_sel || cur_sel.empty?
-        unless supports_select_all_and_column?
-          return select_all(*(Array(@opts[:from]) + Array(@opts[:join]))).select_append(*columns, &block)
-        end
-        cur_sel = [WILDCARD]
-      end
-      select(*(cur_sel + columns), &block)
+      virtual_row_columns(columns, block)
+      select(*(_current_select(true) + columns))
     end
 
     # Set both the select and group clauses with the given +columns+.
@@ -973,6 +974,18 @@ module Sequel
       select_append(*columns, &block)
     end
     
+    # Returns a copy of the dataset with the given columns added
+    # to the existing selected columns.  If no columns are currently selected,
+    # it will select the columns given in addition to *.
+    #
+    #   DB[:items].select(:a).select(:b) # SELECT b FROM items
+    #   DB[:items].select(:a).select_prepend(:b) # SELECT b, a FROM items
+    #   DB[:items].select_prepend(:b) # SELECT b, * FROM items
+    def select_prepend(*columns, &block)
+      virtual_row_columns(columns, block)
+      select(*(columns + _current_select(false)))
+    end
+
     # Set the server for this dataset to use.  Used to pick a specific database
     # shard to run a query against, or to override the default (where SELECT uses
     # :read_only database and all other queries use the :default database).  This
@@ -999,6 +1012,7 @@ module Sequel
 
     # Specify that the check for limits/offsets when updating/deleting be skipped for the dataset.
     def skip_limit_check
+      return self if opts[:skip_limit_check]
       cached_dataset(:_skip_limit_check_ds) do
         clone(:skip_limit_check=>true)
       end
@@ -1006,6 +1020,7 @@ module Sequel
 
     # Skip locked rows when returning results from this dataset.
     def skip_locked
+      return self if opts[:skip_locked]
       cached_dataset(:_skip_locked_ds) do
         raise(Error, 'This dataset does not support skipping locked rows') unless supports_skip_locked?
         clone(:skip_locked=>true)
@@ -1017,6 +1032,7 @@ module Sequel
     #   DB[:items].group(:a).having(a: 1).where(:b).unfiltered
     #   # SELECT * FROM items GROUP BY a
     def unfiltered
+      return self unless opts[:where] || opts[:having]
       cached_dataset(:_unfiltered_ds){clone(:where => nil, :having => nil)}
     end
 
@@ -1025,6 +1041,7 @@ module Sequel
     #   DB[:items].group(:a).having(a: 1).where(:b).ungrouped
     #   # SELECT * FROM items WHERE b
     def ungrouped
+      return self unless opts[:group] || opts[:having]
       cached_dataset(:_ungrouped_ds){clone(:group => nil, :having => nil)}
     end
 
@@ -1052,6 +1069,7 @@ module Sequel
     # 
     #   DB[:items].limit(10, 20).unlimited # SELECT * FROM items
     def unlimited
+      return self unless opts[:limit] || opts[:offset]
       cached_dataset(:_unlimited_ds){clone(:limit=>nil, :offset=>nil)}
     end
 
@@ -1059,6 +1077,7 @@ module Sequel
     # 
     #   DB[:items].order(:a).unordered # SELECT * FROM items
     def unordered
+      return self unless opts[:order]
       cached_dataset(:_unordered_ds){clone(:order=>nil)}
     end
     
@@ -1353,6 +1372,36 @@ module Sequel
     end
     # :nocov:
 
+    # A frozen array for the currently selected columns.
+    def _current_select(allow_plain_wildcard)
+      cur_sel = @opts[:select]
+
+      if !cur_sel || cur_sel.empty?
+        cur_sel = if allow_plain_wildcard && supports_select_all_and_column?
+          [WILDCARD].freeze
+        else
+          _current_select_column_all
+        end
+      elsif !allow_plain_wildcard && cur_sel.include?(WILDCARD)
+        cur_sel = cur_sel.dup
+        index = cur_sel.index(WILDCARD)
+        cur_sel.delete(WILDCARD)
+        _current_select_column_all.each_with_index do |ca, i|
+          cur_sel.insert(index+i, ca)
+        end
+        cur_sel.freeze
+      end
+
+      cur_sel
+    end
+
+    # An array of SQL::ColumnAll objects for all FROM and JOIN tables.  Used for select_append
+    # and select_prepend.
+    def _current_select_column_all
+      tables = Array(@opts[:from]) + Array(@opts[:join])
+      tables.map{|t| i, a = split_alias(t); a || i}.map!{|t| SQL::ColumnAll.new(t)}.freeze
+    end
+
     # If invert is true, invert the condition.
     def _invert_filter(cond, invert)
       if invert

data/lib/sequel/exceptions.rb

@@ -18,6 +18,11 @@ module Sequel
       end
     end
   end  
+
+  (
+  # Error raised when there is a failed attempt to acquire an advisory lock.
+  AdvisoryLockError = Class.new(Error)
+  ).name
     
   (
   # Error raised when the adapter requested doesn't exist or can't be loaded.

data/lib/sequel/extensions/async_thread_pool.rb

@@ -176,6 +176,13 @@
 # +:preempt_async_thread+ Database option before loading the
 # async_thread_pool extension.
 #
+# Note that the async_thread_pool extension creates the thread pool
+# when it is loaded into the Database.  If you fork after loading
+# the extension, the extension will not work, as fork does not
+# copy the thread pools.  If you are using a forking webserver
+# (or any other system that forks worker processes), load this
+# extension in each child process, do not load it before forking.
+#
 # Related module: Sequel::Database::AsyncThreadPool::DatasetMethods
 
 

data/lib/sequel/extensions/caller_logging.rb

@@ -36,6 +36,8 @@ require 'rbconfig'
 module Sequel
   module CallerLogging
     SEQUEL_LIB_PATH = (File.expand_path('../../..', __FILE__) + '/').freeze
+    RUBY_STDLIB = RbConfig::CONFIG["rubylibdir"]
+    INTERNAL = '<internal'
 
     # A regexp of caller lines to ignore, in addition to internal Sequel and Ruby code.
     attr_accessor :caller_logging_ignore
@@ -59,7 +61,8 @@ module Sequel
       ignore = caller_logging_ignore
       c = caller.find do |line|
         !(line.start_with?(SEQUEL_LIB_PATH) ||
-          line.start_with?(RbConfig::CONFIG["rubylibdir"]) ||
+          line.start_with?(RUBY_STDLIB) ||
+          line.start_with?(INTERNAL) ||
           (ignore && line =~ ignore))
       end
 

data/lib/sequel/extensions/migration.rb

@@ -403,6 +403,11 @@ module Sequel
       migrator_class(directory).new(db, directory, opts).is_current?
     end
 
+    # Lock ID to use for advisory locks when running migrations
+    # "sequel-migration".codepoints.reduce(:*) % (2**63)
+    MIGRATION_ADVISORY_LOCK_ID = 4966325471869609408
+    private_constant :MIGRATION_ADVISORY_LOCK_ID
+
     # Migrates the supplied database using the migration files in the specified directory. Options:
     # :allow_missing_migration_files :: Don't raise an error if there are missing migration files.
     #                                   It is very risky to use this option, since it can result in
@@ -416,6 +421,8 @@ module Sequel
     # :table :: The table containing the schema version (default: :schema_info for integer migrations and
     #           :schema_migrations for timestamped migrations).
     # :target :: The target version to which to migrate.  If not given, migrates to the maximum version.
+    # :use_advisory_lock :: Use advisory locks in migrations (only use this if Sequel supports advisory
+    #                       locks for the database).
     #
     # Examples: 
     #   Sequel::Migrator.run(DB, "migrations")
@@ -423,7 +430,11 @@ module Sequel
     #   Sequel::Migrator.run(DB, "app1/migrations", column: :app2_version)
     #   Sequel::Migrator.run(DB, "app2/migrations", column: :app2_version, table: :schema_info2)
     def self.run(db, directory, opts=OPTS)
-      migrator_class(directory).new(db, directory, opts).run
+      if opts[:use_advisory_lock]
+        db.with_advisory_lock(MIGRATION_ADVISORY_LOCK_ID){run(db, directory, opts.merge(:use_advisory_lock=>false))}
+      else
+        migrator_class(directory).new(db, directory, opts).run
+      end
     end
 
     # Choose the Migrator subclass to use.  Uses the TimestampMigrator

data/lib/sequel/extensions/provenance.rb

@@ -0,0 +1,110 @@
+# frozen-string-literal: true
+#
+# The provenance dataset extension tracks the locations of all
+# dataset clones that resulted in the current dataset, and includes
+# the information as a comment in the dataset's SQL.  This makes it
+# possible to see how a query was built, which can aid debugging.
+# Example:
+#
+#   DB[:table].
+#     select(:a).
+#     where{b > 10}.
+#     order(:c).
+#     limit(10)
+#   # SQL:
+#   # SELECT a FROM table WHERE (b > 10) ORDER BY c LIMIT 10 --
+#   #  -- Dataset Provenance
+#   #  -- Keys:[:from] Source:(eval at bin/sequel:257):2:in `<main>'
+#   #  -- Keys:[:select] Source:(eval at bin/sequel:257):3:in `<main>'
+#   #  -- Keys:[:where] Source:(eval at bin/sequel:257):4:in `<main>'
+#   #  -- Keys:[:order] Source:(eval at bin/sequel:257):5:in `<main>'
+#   #  -- Keys:[:limit] Source:(eval at bin/sequel:257):6:in `<main>'
+#
+# With the above example, the source is fairly obvious and not helpful,
+# but in real applications, where datasets can be built from multiple
+# files, seeing where each dataset clone was made can be helpful.
+#
+# The Source listed will skip locations in the Ruby standard library
+# as well as Sequel itself.  Other locations can be skipped by
+# providing a Database :provenance_caller_ignore Regexp option:
+#
+#   DB.opts[:provenance_caller_ignore] = /\/gems\/library_name-/
+#
+# Related module: Sequel::Dataset::Provenance
+
+#
+module Sequel
+  class Dataset
+    module Provenance
+      SEQUEL_LIB_PATH = (File.expand_path('../../..', __FILE__) + '/').freeze
+      RUBY_STDLIB = RbConfig::CONFIG["rubylibdir"]
+      INTERNAL = '<internal'
+
+      if TRUE_FREEZE
+        # Include provenance information when cloning datasets.
+        def clone(opts = nil || (return self))
+          super(provenance_opts(opts))
+        end
+      else
+        # :nocov:
+        def clone(opts = OPTS) # :nodoc:
+          super(provenance_opts(opts))
+        end
+        # :nocov:
+      end
+
+      %w'select insert update delete'.each do |type|
+        # Include the provenance information as a comment when preparing dataset SQL
+        define_method(:"#{type}_sql") do |*a|
+          sql = super(*a)
+
+          if provenance = @opts[:provenance]
+            comment = provenance.map do |hash|
+              " -- Keys:#{hash[:keys].inspect} Source:#{hash[:source]}".to_s.gsub(/\s+/, ' ')
+            end
+            comment << ""
+            comment.unshift " -- Dataset Provenance"
+            comment.unshift " -- "
+            comment = comment.join("\n")
+
+            if sql.frozen?
+              sql += comment
+              sql.freeze
+            elsif @opts[:append_sql] || @opts[:placeholder_literalizer]
+              sql << comment
+            else
+              sql += comment
+            end
+          end
+
+          sql
+        end
+      end
+
+      private
+
+      # Return a copy of opts with provenance information added.
+      def provenance_opts(opts)
+        provenance = {source: provenance_source, keys: opts.keys.freeze}.freeze
+        opts = opts.dup
+        opts[:provenance] = ((@opts[:provenance] || EMPTY_ARRAY).dup << provenance).freeze
+        opts
+      end
+
+      # Return the caller line for the provenance change. This skips
+      # Sequel itself and the standard library.  Additional locations
+      # can be skipped using the :provenance_caller_ignore Dataset option.
+      def provenance_source
+        ignore = db.opts[:provenance_caller_ignore]
+        caller.find do |line|
+          !(line.start_with?(SEQUEL_LIB_PATH) ||
+            line.start_with?(RUBY_STDLIB) ||
+            line.start_with?(INTERNAL) ||
+            (ignore && line =~ ignore))
+        end
+      end
+    end
+
+    register_extension(:provenance, Provenance)
+  end
+end

data/lib/sequel/extensions/sqlite_json_ops.rb

@@ -2,27 +2,34 @@
 #
 # The sqlite_json_ops extension adds support to Sequel's DSL to make
 # it easier to call SQLite JSON functions and operators (added
-# first in SQLite 3.38.0).
+# first in SQLite 3.38.0).  It also supports the SQLite JSONB functions
+# added in SQLite 3.45.0.
 #
 # To load the extension:
 #
 #   Sequel.extension :sqlite_json_ops
 #
-# This extension works by calling methods on Sequel::SQLite::JSONOp objects,
-# which you can create via Sequel.sqlite_json_op:
+# This extension works by calling methods on Sequel::SQLite::JSONOp and
+# Sequel::SQLite::JSONBOp objects, which you can create using
+# Sequel.sqlite_json_op and Sequel.sqlite_jsonb_op:
 #
 #   j = Sequel.sqlite_json_op(:json_column)
+#   jb = Sequel.sqlite_jsonb_op(:jsonb_column)
 #
-# Also, on most Sequel expression objects, you can call the sqlite_json_op method
-# to create a Sequel::SQLite::JSONOp object:
+# Also, on most Sequel expression objects, you can call the sqlite_json_op or
+# sqlite_jsonb_op method to create a Sequel::SQLite::JSONOp or 
+# Sequel::SQLite::JSONBOp object:
 #
 #   j = Sequel[:json_column].sqlite_json_op
+#   jb = Sequel[:jsonb_column].sqlite_jsonb_op
 #
 # If you have loaded the {core_extensions extension}[rdoc-ref:doc/core_extensions.rdoc],
 # or you have loaded the core_refinements extension
 # and have activated refinements for the file, you can also use Symbol#sqlite_json_op:
+# or Symbol#sqlite_jsonb_op:
 #
 #   j = :json_column.sqlite_json_op
+#   jb = :json_column.sqlite_jsonb_op
 #
 # The following methods are available for Sequel::SQLite::JSONOp instances:
 #
@@ -30,11 +37,13 @@
 #   j.get(1)                 # (json_column ->> 1)
 #   j.get_text(1)            # (json_column -> 1)
 #   j.extract('$.a')         # json_extract(json_column, '$.a')
+#   jb.extract('$.a')        # jsonb_extract(jsonb_column, '$.a')
 #
 #   j.array_length           # json_array_length(json_column)
 #   j.type                   # json_type(json_column)
 #   j.valid                  # json_valid(json_column)
-#   j.json                   # json(json_column)
+#   jb.json                  # json(jsonb_column)
+#   j.jsonb                  # jsonb(json_column)
 #
 #   j.insert('$.a', 1)       # json_insert(json_column, '$.a', 1)
 #   j.set('$.a', 1)          # json_set(json_column, '$.a', 1)
@@ -42,22 +51,30 @@
 #   j.remove('$.a')          # json_remove(json_column, '$.a')
 #   j.patch('{"a":2}')       # json_patch(json_column, '{"a":2}')
 #
+#   jb.insert('$.a', 1)      # jsonb_insert(jsonb_column, '$.a', 1)
+#   jb.set('$.a', 1)         # jsonb_set(jsonb_column, '$.a', 1)
+#   jb.replace('$.a', 1)     # jsonb_replace(jsonb_column, '$.a', 1)
+#   jb.remove('$.a')         # jsonb_remove(jsonb_column, '$.a')
+#   jb.patch('{"a":2}')      # jsonb_patch(jsonb_column, '{"a":2}')
+#
 #   j.each                   # json_each(json_column)
 #   j.tree                   # json_tree(json_column)
 #
-# Related modules: Sequel::SQLite::JSONOp
+# Related modules: Sequel::SQLite::JSONBaseOp, Sequel::SQLite::JSONOp,
+# Sequel::SQLite::JSONBOp
 
 #
 module Sequel
   module SQLite
-    # The JSONOp class is a simple container for a single object that
-    # defines methods that yield Sequel expression objects representing
-    # SQLite json operators and functions.
+    # JSONBaseOp is an abstract base wrapper class for a object that
+    # defines methods that return Sequel expression objects representing
+    # SQLite json operators and functions. It is subclassed by both
+    # JSONOp and JSONBOp for json and jsonb specific behavior.
     #
     # In the method documentation examples, assume that:
     #
     #   json_op = Sequel.sqlite_json_op(:json)
-    class JSONOp < Sequel::SQL::Wrapper
+    class JSONBaseOp < Sequel::SQL::Wrapper
       GET = ["(".freeze, " ->> ".freeze, ")".freeze].freeze
       private_constant :GET
 
@@ -82,7 +99,7 @@ module Sequel
       #   json_op.array_length         # json_array_length(json)
       #   json_op.array_length('$[1]') # json_array_length(json, '$[1]')
       def array_length(*args)
-        Sequel::SQL::NumericExpression.new(:NOOP, function(:array_length, *args))
+        Sequel::SQL::NumericExpression.new(:NOOP, SQL::Function.new(:json_array_length, self, *args))
       end
 
       # Returns an expression for a set of information extracted from the top-level
@@ -92,7 +109,7 @@ module Sequel
       #   json_op.each        # json_each(json)
       #   json_op.each('$.a') # json_each(json, '$.a')
       def each(*args)
-        function(:each, *args)
+        SQL::Function.new(:json_each, self, *args)
       end
 
       # Returns an expression for the JSON array element or object field at the specified
@@ -129,10 +146,17 @@ module Sequel
       #
       #   json_op.json   # json(json)
       def json
-        self.class.new(SQL::Function.new(:json, self))
+        JSONOp.new(SQL::Function.new(:json, self))
       end
       alias minify json
 
+      # Returns the JSONB format of the JSON.
+      #
+      #   json_op.jsonb   # jsonb(json)
+      def jsonb
+        JSONBOp.new(SQL::Function.new(:jsonb, self))
+      end
+
       # Returns an expression for updating the JSON object using the RFC 7396 MergePatch algorithm
       #
       #   json_op.patch('{"a": 1, "b": null}') # json_patch(json, '{"a": 1, "b": null}')
@@ -172,7 +196,7 @@ module Sequel
       #   json_op.tree        # json_tree(json)
       #   json_op.tree('$.a') # json_tree(json, '$.a')
       def tree(*args)
-        function(:tree, *args)
+        SQL::Function.new(:json_tree, self, *args)
       end
 
       # Returns an expression for the type of the JSON value or the JSON value at the given path.
@@ -180,13 +204,13 @@ module Sequel
       #   json_op.type         # json_type(json)
       #   json_op.type('$[1]') # json_type(json, '$[1]')
       def type(*args)
-        Sequel::SQL::StringExpression.new(:NOOP, function(:type, *args))
+        Sequel::SQL::StringExpression.new(:NOOP, SQL::Function.new(:json_type, self, *args))
       end
       alias typeof type
 
       # Returns a boolean expression for whether the JSON is valid or not.
       def valid
-        Sequel::SQL::BooleanExpression.new(:NOOP, function(:valid))
+        Sequel::SQL::BooleanExpression.new(:NOOP, SQL::Function.new(:json_valid, self))
       end
 
       private
@@ -198,7 +222,7 @@ module Sequel
 
       # Internals of the methods that return functions prefixed with +json_+.
       def function(name, *args)
-        SQL::Function.new("json_#{name}", self, *args)
+        SQL::Function.new("#{function_prefix}_#{name}", self, *args)
       end
 
       # Internals of the methods that return functions prefixed with +json_+, that
@@ -208,12 +232,36 @@ module Sequel
       end
     end
 
+    # JSONOp is used for SQLite json-specific functions and operators.
+    class JSONOp < JSONBaseOp
+      private
+
+      def function_prefix
+        "json"
+      end
+    end
+
+    # JSONOp is used for SQLite jsonb-specific functions and operators.
+    class JSONBOp < JSONBaseOp
+      private
+
+      def function_prefix
+        "jsonb"
+      end
+    end
+
     module JSONOpMethods
       # Wrap the receiver in an JSONOp so you can easily use the SQLite
       # json functions and operators with it.
       def sqlite_json_op
         JSONOp.new(self)
       end
+
+      # Wrap the receiver in an JSONBOp so you can easily use the SQLite
+      # jsonb functions and operators with it.
+      def sqlite_jsonb_op
+        JSONBOp.new(self)
+      end
     end
   end
 
@@ -227,6 +275,16 @@ module Sequel
         SQLite::JSONOp.new(v)
       end
     end
+
+    # Return the object wrapped in an SQLite::JSONBOp.
+    def sqlite_jsonb_op(v)
+      case v
+      when SQLite::JSONBOp
+        v
+      else
+        SQLite::JSONBOp.new(v)
+      end
+    end
   end
 
   class SQL::GenericExpression