sequel 5.7.1 → 5.8.0

data/lib/sequel/adapters/shared/sqlanywhere.rb

@@ -232,7 +232,7 @@ module Sequel
     end
 
     module DatasetMethods
-      Dataset.def_sql_method(self, :insert, %w'with insert into columns values')
+      Dataset.def_sql_method(self, :insert, %w'insert into columns values')
       Dataset.def_sql_method(self, :select, %w'with select distinct limit columns into from join where group having compounds order lock')
 
       # Whether to convert smallint to boolean arguments for this dataset.
@@ -247,7 +247,7 @@ module Sequel
       end
 
       def supports_cte?(type=:select)
-        type == :select || type == :insert
+        type == :select
       end
 
       # SQLAnywhere supports GROUPING SETS

data/lib/sequel/adapters/shared/sqlite.rb

@@ -179,7 +179,12 @@ module Sequel
         fks = fetch("PRAGMA foreign_keys")
         run "PRAGMA foreign_keys = 0" if fks
         transaction do 
-          if ops.length > 1 && ops.all?{|op| op[:op] == :add_constraint}
+          if ops.length > 1 && ops.all?{|op| op[:op] == :add_constraint || op[:op] == :set_column_null}
+            null_ops, ops = ops.partition{|op| op[:op] == :set_column_null}
+
+            # Apply NULL/NOT NULL ops first, since those should be purely idependent of the constraints.
+            null_ops.each{|op| alter_table_sql_list(table, [op]).flatten.each{|sql| execute_ddl(sql)}}
+
             # If you are just doing constraints, apply all of them at the same time,
             # as otherwise all but the last one get lost.
             alter_table_sql_list(table, [{:op=>:add_constraints, :ops=>ops}]).flatten.each{|sql| execute_ddl(sql)}

data/lib/sequel/dataset/features.rb

@@ -115,6 +115,11 @@ module Sequel
       true
     end
     
+    # Whether the dataset supports skipping raising an error instead of waiting for locked rows when returning data, false by default.
+    def supports_nowait?
+      false
+    end
+
     # Whether modifying joined datasets is supported, false by default.
     def supports_modifying_joins?
       false

data/lib/sequel/dataset/query.rb

@@ -88,7 +88,6 @@ module Sequel
           c.clear_columns_cache
         end
         c.freeze
-        c
       end
     else
       # :nocov:
@@ -116,8 +115,12 @@ module Sequel
     #  DB[:items].order(:id).distinct{func(:id)} # SQL: SELECT DISTINCT ON (func(id)) * FROM items ORDER BY id
     def distinct(*args, &block)
       virtual_row_columns(args, block)
-      raise(InvalidOperation, "DISTINCT ON not supported") if !args.empty? && !supports_distinct_on?
-      clone(:distinct => args.freeze)
+      if args.empty?
+        cached_dataset(:_distinct_ds){clone(:distinct => EMPTY_ARRAY)}
+      else
+        raise(InvalidOperation, "DISTINCT ON not supported") unless supports_distinct_on?
+        clone(:distinct => args.freeze)
+      end
     end
 
     # Adds an EXCEPT clause using a second dataset object.
@@ -190,7 +193,6 @@ module Sequel
         c = _clone(:freeze=>false)
         c.send(:_extension!, a)
         c.freeze
-        c
       end
     else
       # :nocov:
@@ -274,11 +276,15 @@ module Sequel
     def from_self(opts=OPTS)
       fs = {}
       @opts.keys.each{|k| fs[k] = nil unless non_sql_option?(k)}
-      c = clone(fs).from(opts[:alias] ? as(opts[:alias], opts[:column_aliases]) : self)
-      if cols = _columns
-        c.send(:columns=, cols)
+      pr = proc do
+        c = clone(fs).from(opts[:alias] ? as(opts[:alias], opts[:column_aliases]) : self)
+        if cols = _columns
+          c.send(:columns=, cols)
+        end
+        c
       end
-      c
+
+      cache ? cached_dataset(:_from_self_ds, &pr) : pr.call
     end
 
     # Match any of the columns to any of the patterns. The terms can be
@@ -616,7 +622,7 @@ module Sequel
     #   DB.from(:a, DB[:b].where(Sequel[:a][:c]=>Sequel[:b][:d]).lateral)
     #   # SELECT * FROM a, LATERAL (SELECT * FROM b WHERE (a.c = b.d))
     def lateral
-      clone(:lateral=>true)
+      cached_dataset(:_lateral_ds){clone(:lateral=>true)}
     end
 
     # If given an integer, the dataset will contain only the first l results.
@@ -672,6 +678,18 @@ module Sequel
       cached_dataset(:_naked_ds){with_row_proc(nil)}
     end
 
+    # Returns a copy of the dataset that will raise a DatabaseLockTimeout instead
+    # of waiting for rows that are locked by another transaction
+    #
+    #   DB[:items].for_update.nowait
+    #   # SELECT * FROM items FOR UPDATE NOWAIT
+    def nowait
+      cached_dataset(:_nowait_ds) do
+        raise(Error, 'This dataset does not support raises errors instead of waiting for locked rows') unless supports_nowait?
+        clone(:nowait=>true)
+      end
+    end
+
     # Returns a copy of the dataset with a specified order. Can be safely combined with limit.
     # If you call limit with an offset, it will override override the offset if you've called
     # offset first.
@@ -755,15 +773,20 @@ module Sequel
     #
     #   DB[:items].where(id: 1).qualify(:i)
     #   # SELECT i.* FROM items WHERE (i.id = 1)
-    def qualify(table=first_source)
+    def qualify(table=(cache=true; first_source))
       o = @opts
       return self if o[:sql]
-      h = {}
-      (o.keys & QUALIFY_KEYS).each do |k|
-        h[k] = qualified_expression(o[k], table)
+
+      pr = proc do
+        h = {}
+        (o.keys & QUALIFY_KEYS).each do |k|
+          h[k] = qualified_expression(o[k], table)
+        end
+        h[:select] = [SQL::ColumnAll.new(table)].freeze if !o[:select] || o[:select].empty?
+        clone(h)
       end
-      h[:select] = [SQL::ColumnAll.new(table)].freeze if !o[:select] || o[:select].empty?
-      clone(h)
+
+      cache ? cached_dataset(:_qualify_ds, &pr) : pr.call
     end
 
     # Modify the RETURNING clause, only supported on a few databases.  If returning
@@ -785,8 +808,15 @@ module Sequel
     #     # hash for each row deleted, with values for all columns
     #   end
     def returning(*values)
-      raise Error, "RETURNING is not supported on #{db.database_type}" unless supports_returning?(:insert)
-      clone(:returning=>values.freeze)
+      if values.empty?
+        cached_dataset(:_returning_ds) do
+          raise Error, "RETURNING is not supported on #{db.database_type}" unless supports_returning?(:insert)
+          clone(:returning=>EMPTY_ARRAY)
+        end
+      else
+        raise Error, "RETURNING is not supported on #{db.database_type}" unless supports_returning?(:insert)
+        clone(:returning=>values.freeze)
+      end
     end
 
     # Returns a copy of the dataset with the order reversed. If no order is
@@ -831,7 +861,7 @@ module Sequel
     #   DB[:items].select_all(:items, :foo) # SELECT items.*, foo.* FROM items
     def select_all(*tables)
       if tables.empty?
-        clone(:select => nil)
+        cached_dataset(:_select_all_ds){clone(:select => nil)}
       else
         select(*tables.map{|t| i, a = split_alias(t); a || i}.map!{|t| SQL::ColumnAll.new(t)}.freeze)
       end
@@ -1067,7 +1097,6 @@ module Sequel
         c.extend(*mods) unless mods.empty?
         c.extend(DatasetModule.new(&block)) if block
         c.freeze
-        c
       end
     else
       # :nocov:

data/lib/sequel/exceptions.rb

@@ -73,6 +73,13 @@ module Sequel
   SerializationFailure = Class.new(DatabaseError)
   ).name
 
+  (
+  # Error raised when Sequel determines the database could not acquire a necessary lock
+  # before timing out. Use of Dataset#nowait can often cause this exception when
+  # retrieving rows.
+  DatabaseLockTimeout = Class.new(DatabaseError)
+  ).name
+
   (
   # Error raised on an invalid operation, such as trying to update or delete
   # a joined or grouped dataset when the database does not support that.

data/lib/sequel/extensions/connection_expiration.rb

@@ -38,12 +38,17 @@ module Sequel
     # Defaults to 14400 seconds (4 hours).
     attr_accessor :connection_expiration_timeout
 
+    # The maximum number of seconds that will be added as a random delay to the expiration timeout
+    # Defaults to 0 seconds (no random delay).
+    attr_accessor :connection_expiration_random_delay
+
     # Initialize the data structures used by this extension.
     def self.extended(pool)
       pool.instance_exec do
         sync do
           @connection_expiration_timestamps ||= {}
           @connection_expiration_timeout ||= 14400
+          @connection_expiration_random_delay ||= 0
         end
       end
     end
@@ -59,7 +64,7 @@ module Sequel
     # Record the time the connection was created.
     def make_new(*)
       conn = super
-      @connection_expiration_timestamps[conn] = Sequel.start_timer
+      @connection_expiration_timestamps[conn] = [Sequel.start_timer, @connection_expiration_timeout + (rand * @connection_expiration_random_delay)].freeze
       conn
     end
 
@@ -69,8 +74,8 @@ module Sequel
     def acquire(*a)
       begin
         if (conn = super) &&
-           (timer = sync{@connection_expiration_timestamps[conn]}) &&
-           Sequel.elapsed_seconds_since(timer) > @connection_expiration_timeout
+           (cet = sync{@connection_expiration_timestamps[conn]}) &&
+           Sequel.elapsed_seconds_since(cet[0]) > cet[1]
 
           if pool_type == :sharded_threaded
             sync{allocated(a.last).delete(Thread.current)}

data/lib/sequel/extensions/pg_enum.rb

@@ -13,6 +13,10 @@
 #
 #   DB.add_enum_value(:enum_type_name, 'value4')
 #
+# If you want to rename an enum type, you can use rename_enum:
+#
+#   DB.rename_enum(:enum_type_name, :enum_type_another_name)
+#
 # If you want to drop an enum type, you can use drop_enum:
 #
 #   DB.drop_enum(:enum_type_name)
@@ -63,7 +67,10 @@ module Sequel
       # Parse the available enum values when loading this extension into
       # your database.
       def self.extended(db)
-        db.send(:parse_enum_labels)
+        db.instance_exec do
+          @enum_labels = {}
+          parse_enum_labels
+        end
       end
 
       # Run the SQL to add the given value to the existing enum type.
@@ -92,6 +99,15 @@ module Sequel
         nil
       end
 
+      # Run the SQL to rename the enum type with the given name
+      # to the another given name.
+      def rename_enum(enum, new_name)
+        sql = "ALTER TYPE #{quote_schema_table(enum)} RENAME TO #{quote_schema_table(new_name)}"
+        run sql
+        parse_enum_labels
+        nil
+      end
+
       # Run the SQL to drop the enum type with the given name.
       # Options:
       # :if_exists :: Do not raise an error if the enum type does not exist
@@ -109,15 +125,15 @@ module Sequel
       # the pg_type table to get names and array oids for
       # enums.
       def parse_enum_labels
-        @enum_labels = metadata_dataset.from(:pg_enum).
+        enum_labels = metadata_dataset.from(:pg_enum).
           order(:enumtypid, :enumsortorder).
           select_hash_groups(Sequel.cast(:enumtypid, Integer).as(:v), :enumlabel).freeze
-        @enum_labels.each_value(&:freeze)
+        enum_labels.each_value(&:freeze)
 
         if respond_to?(:register_array_type)
           array_types = metadata_dataset.
             from(:pg_type).
-            where(:oid=>@enum_labels.keys).
+            where(:oid=>enum_labels.keys).
             exclude(:typarray=>0).
             select_map([:typname, Sequel.cast(:typarray, Integer).as(:v)])
 
@@ -127,13 +143,16 @@ module Sequel
             register_array_type(name, :oid=>oid)
           end
         end
+
+        Sequel.synchronize{@enum_labels.replace(enum_labels)}
       end
 
       # For schema entries that are enums, set the type to
       # :enum and add a :enum_values entry with the enum values.
       def schema_post_process(_)
         super.each do |_, s|
-          if values = @enum_labels[s[:oid]]
+          oid = s[:oid]
+          if values = Sequel.synchronize{@enum_labels[oid]}
             s[:type] = :enum
             s[:enum_values] = values
           end
@@ -154,6 +173,10 @@ module Sequel
       def create_enum(name, _)
         @actions << [:drop_enum, name]
       end
+
+      def rename_enum(old_name, new_name)
+        @actions << [:rename_enum, new_name, old_name]
+      end
     end
   end
 

data/lib/sequel/plugins/association_proxies.rb

@@ -61,11 +61,23 @@ module Sequel
       # associated objects and call the method on the associated object array.
       # Calling any other method will call that method on the association's dataset.
       class AssociationProxy < BasicObject
-        array = []
+        array = [].freeze
 
-        # Default proc used to determine whether to sent the method to the dataset.
-        # If the array would respond to it, sends it to the array instead of the dataset.
-        DEFAULT_PROXY_TO_DATASET = proc{|opts| !array.respond_to?(opts[:method])}
+        if RUBY_VERSION < '2.6'
+          # Default proc used to determine whether to send the method to the dataset.
+          # If the array would respond to it, sends it to the array instead of the dataset.
+          DEFAULT_PROXY_TO_DATASET = proc do |opts|
+            array_method = array.respond_to?(opts[:method])
+            if !array_method && opts[:method] == :filter
+              Sequel::Deprecation.deprecate "The behavior of the #filter method for association proxies will change in Ruby 2.6. Switch from using #filter to using #where to conserve current behavior."
+            end
+            !array_method
+          end
+        else
+          # :nocov:
+          DEFAULT_PROXY_TO_DATASET = proc{|opts| !array.respond_to?(opts[:method])}
+          # :nocov:
+        end
 
         # Set the association reflection to use, and whether the association should be
         # reloaded if an array method is called.

data/lib/sequel/plugins/error_splitter.rb

@@ -32,25 +32,30 @@ module Sequel
     #   Album.plugin :error_splitter
     module ErrorSplitter
       module InstanceMethods
-        # If the model instance is not valid, go through all of the errors entries.  For
-        # any that apply to multiple columns, remove them and add separate error entries,
-        # one per column.
+        private
+
+        # If the model instance is not valid, split the errors before returning.
         def _valid?(opts)
           v = super
           unless v
-            errors.keys.select{|k| k.is_a?(Array)}.each do |ks|
-              msgs = errors.delete(ks)
-              ks.each do |k|
-                msgs.each do |msg|
-                  errors.add(k, msg)
-                end
+            split_validation_errors(errors)
+          end
+          v
+        end
+
+        # Go through all of the errors entries.  For any that apply to multiple columns,
+        # remove them and add separate error entries, one per column.
+        def split_validation_errors(errors)
+          errors.keys.select{|k| k.is_a?(Array)}.each do |ks|
+            msgs = errors.delete(ks)
+            ks.each do |k|
+              msgs.each do |msg|
+                errors.add(k, msg)
               end
             end
           end
-          v
         end
       end
     end
   end
 end
-

data/lib/sequel/plugins/pg_auto_constraint_validations.rb

@@ -0,0 +1,260 @@
+# frozen-string-literal: true
+
+module Sequel
+  module Plugins
+    # The pg_auto_constraint_validations plugin automatically converts some constraint
+    # violation exceptions that are raised by INSERT/UPDATE queries into validation
+    # failures.  This can allow for using the same error handling code for both
+    # regular validation errors (checked before attempting the INSERT/UPDATE), and
+    # constraint violations (raised during the INSERT/UPDATE).
+    #
+    # This handles the following constraint violations:
+    #
+    # * NOT NULL
+    # * CHECK
+    # * UNIQUE (except expression/functional indexes)
+    # * FOREIGN KEY (both referencing and referenced by)
+    #
+    # If the plugin cannot convert the constraint violation error to a validation
+    # error, it just reraises the initial exception, so this should not cause
+    # problems if the plugin doesn't know how to convert the exception.
+    #
+    # This plugin is not intended as a replacement for other validations,
+    # it is intended as a last resort.  The purpose of validations is to provide nice
+    # error messages for the user, and the error messages generated by this plugin are
+    # fairly generic.  The error messages can be customized using the :messages plugin
+    # option, but there is only a single message used per constraint type.
+    #
+    # This plugin only works on the postgres adapter when using the pg 0.16+ driver,
+    # PostgreSQL 9.3+ server, and PostgreSQL 9.3+ client library (libpq). In other cases
+    # it will be a no-op.
+    #
+    # Example:
+    #
+    #   album = Album.new(:artist_id=>1) # Assume no such artist exists
+    #   begin
+    #     album.save
+    #   rescue Sequel::ValidationFailed
+    #     album.errors.on(:artist_id) # ['is invalid']
+    #   end
+    # 
+    # Usage:
+    #
+    #   # Make all model subclasses automatically convert constraint violations
+    #   # to validation failures (called before loading subclasses)
+    #   Sequel::Model.plugin :pg_auto_constraint_validations
+    #
+    #   # Make the Album class automatically convert constraint violations
+    #   # to validation failures
+    #   Album.plugin :pg_auto_constraint_validations
+    module PgAutoConstraintValidations
+      (
+      # The default error messages for each constraint violation type.
+      DEFAULT_ERROR_MESSAGES = {
+        :not_null=>"is not present",
+        :check=>"is invalid",
+        :unique=>'is already taken',
+        :foreign_key=>'is invalid',
+        :referenced_by=>'cannot be changed currently'
+      }.freeze).each_value(&:freeze)
+
+      # Setup the constraint violation metadata.  Options:
+      # :messages :: Override the default error messages for each constraint
+      #              violation type (:not_null, :check, :unique, :foreign_key, :referenced_by)
+      def self.configure(model, opts=OPTS)
+        model.instance_exec do
+          setup_pg_auto_constraint_validations
+          @pg_auto_constraint_validations_messages = (@pg_auto_constraint_validations_messages || DEFAULT_ERROR_MESSAGES).merge(opts[:messages] || {}).freeze
+        end
+      end
+
+      module ClassMethods
+        # Hash of metadata checked when an instance attempts to convert a constraint
+        # violation into a validation failure.
+        attr_reader :pg_auto_constraint_validations
+
+        # Hash of error messages keyed by constraint type symbol to use in the
+        # generated validation failures.
+        attr_reader :pg_auto_constraint_validations_messages
+
+        Plugins.inherited_instance_variables(self, :@pg_auto_constraint_validations=>nil, :@pg_auto_constraint_validations_messages=>nil)
+        Plugins.after_set_dataset(self, :setup_pg_auto_constraint_validations)
+
+        private
+
+        # Get the list of constraints, unique indexes, foreign keys in the current
+        # table, and keys in the current table referenced by foreign keys in other
+        # tables.  Store this information so that if a constraint violation occurs,
+        # all necessary metadata is already available in the model, so a query is
+        # not required at runtime.  This is both for performance and because in
+        # general after the constraint violation failure you will be inside a
+        # failed transaction and not able to execute queries.
+        def setup_pg_auto_constraint_validations
+          return unless @dataset
+
+          case @dataset.first_source_table
+          when Symbol, String, SQL::Identifier, SQL::QualifiedIdentifier
+           convert_errors = db.respond_to?(:error_info)
+          end
+
+          unless convert_errors
+            # Might be a table returning function or subquery, skip handling those.
+            # Might have db not support error_info, skip handling that. 
+            @pg_auto_constraint_validations = nil
+            return
+          end
+
+          checks = {}
+          indexes = {}
+          foreign_keys = {}
+          referenced_by = {}
+
+          db.check_constraints(table_name).each do |k, v|
+            checks[k] = v[:columns].dup.freeze
+          end
+          db.indexes(table_name, :include_partial=>true).each do |k, v|
+            if v[:unique]
+              indexes[k] = v[:columns].dup.freeze
+            end
+          end
+          db.foreign_key_list(table_name, :schema=>false).each do |fk|
+            foreign_keys[fk[:name]] = fk[:columns].dup.freeze
+          end
+          db.foreign_key_list(table_name, :reverse=>true, :schema=>false).each do |fk|
+            referenced_by[[fk[:schema], fk[:table], fk[:name]].freeze] = fk[:key].dup.freeze
+          end
+
+          schema, table = db[:pg_class].
+            join(:pg_namespace, :oid=>:relnamespace, db.send(:regclass_oid, table_name)=>:oid).
+            get([:nspname, :relname])
+
+          (@pg_auto_constraint_validations = {
+            :schema=>schema,
+            :table=>table,
+            :check=>checks,
+            :unique=>indexes,
+            :foreign_key=>foreign_keys,
+            :referenced_by=>referenced_by
+          }.freeze).each_value(&:freeze)
+        end
+      end
+
+      module InstanceMethods
+        private
+
+        # Yield to the given block, and if a Sequel::ConstraintViolation is raised, try
+        # to convert it to a Sequel::ValidationFailed error using the PostgreSQL error
+        # metadata.
+        def check_pg_constraint_error(ds)
+          yield
+        rescue Sequel::ConstraintViolation => e
+          begin
+            unless cv_info = model.pg_auto_constraint_validations
+              # Necessary metadata does not exist, just reraise the exception.
+              raise e
+            end
+
+            info = ds.db.error_info(e)
+            m = ds.method(:output_identifier)
+            schema = info[:schema]
+            table = info[:table]
+            if constraint = info[:constraint]
+              constraint = m.call(constraint)
+            end
+            messages = model.pg_auto_constraint_validations_messages
+
+            case e
+            when Sequel::NotNullConstraintViolation
+              if column = info[:column]
+                add_pg_constraint_validation_error([m.call(column)], messages[:not_null])
+              end
+            when Sequel::CheckConstraintViolation
+              if columns = cv_info[:check][constraint]
+                add_pg_constraint_validation_error(columns, messages[:check])
+              end
+            when Sequel::UniqueConstraintViolation
+              if columns = cv_info[:unique][constraint]
+                add_pg_constraint_validation_error(columns, messages[:unique])
+              end
+            when Sequel::ForeignKeyConstraintViolation
+              message_primary = info[:message_primary]
+              if message_primary.start_with?('update')
+                # This constraint violation is different from the others, because the constraint
+                # referenced is a constraint for a different table, not for this table.  This
+                # happens when another table references the current table, and the referenced
+                # column in the current update is modified such that referential integrity
+                # would be broken.  Use the reverse foreign key information to figure out
+                # which column is affected in that case.
+                skip_schema_table_check = true
+                if columns = cv_info[:referenced_by][[m.call(schema), m.call(table), constraint]]
+                  add_pg_constraint_validation_error(columns, messages[:referenced_by])
+                end
+              elsif message_primary.start_with?('insert')
+                if columns = cv_info[:foreign_key][constraint]
+                  add_pg_constraint_validation_error(columns, messages[:foreign_key])
+                end
+              end
+            end
+          rescue => e2
+            # If there is an error trying to conver the constraint violation
+            # into a validation failure, it's best to just raise the constraint
+            # violation.  This can make debugging the above block of code more
+            # difficult.
+            raise e
+          else
+            unless skip_schema_table_check
+              # The constraint violation could be caused by a trigger modifying 
+              # a different table.  Check that the error schema and table
+              # match the model's schema and table, or clear the validation error
+              # that was set above.
+              if schema != cv_info[:schema] || table != cv_info[:table]
+                errors.clear
+              end
+            end
+
+            if errors.empty?
+              # If we weren't able to parse the constraint violation metadata and
+              # convert it to an appropriate validation failure, or the schema/table
+              # didn't match, then raise the constraint violation.
+              raise e
+            end
+
+            # Integrate with error_splitter plugin to split any multi-column errors
+            # and add them as separate single column errors
+            if respond_to?(:split_validation_errors, true)
+              split_validation_errors(errors)
+            end
+
+            vf = ValidationFailed.new(self)
+            vf.set_backtrace(e.backtrace)
+            vf.wrapped_exception = e
+            raise vf
+          end
+        end
+
+        # If there is a single column instead of an array of columns, add the error
+        # for the column, otherwise add the error for the array of columns.
+        def add_pg_constraint_validation_error(column, message)
+          column = column.first if column.length == 1 
+          errors.add(column, message)
+        end
+
+        # Convert PostgreSQL constraint errors when inserting.
+        def _insert_raw(ds)
+          check_pg_constraint_error(ds){super}
+        end
+
+        # Convert PostgreSQL constraint errors when inserting.
+        def _insert_select_raw(ds)
+          check_pg_constraint_error(ds){super}
+        end
+
+        # Convert PostgreSQL constraint errors when updating.
+        def _update_without_checking(_)
+          check_pg_constraint_error(_update_dataset){super}
+        end
+      end
+    end
+  end
+end
+