sequel 5.8.0 → 5.38.0

data/lib/sequel/adapters/postgres.rb

@@ -15,6 +15,12 @@ begin
   end
 
   Sequel::Postgres::USES_PG = true
+  if defined?(PG::TypeMapByClass)
+    type_map = Sequel::Postgres::PG_QUERY_TYPE_MAP = PG::TypeMapByClass.new
+    type_map[Integer] = PG::TextEncoder::Integer.new
+    type_map[FalseClass] = type_map[TrueClass] = PG::TextEncoder::Boolean.new
+    type_map[Float] = PG::TextEncoder::Float.new
+  end
 rescue LoadError => e 
   begin
     require 'postgres-pr/postgres-compat'
@@ -64,6 +70,10 @@ module Sequel
         # string names of the server side prepared statement, and values
         # are SQL strings.
         attr_reader :prepared_statements
+
+        unless public_method_defined?(:async_exec_params)
+          alias async_exec_params async_exec
+        end
       else
         # Make postgres-pr look like pg
         CONNECTION_OK = -1
@@ -143,7 +153,7 @@ module Sequel
 
       # Return the PGResult containing the query results.
       def execute_query(sql, args)
-        @db.log_connection_yield(sql, self, args){args ? async_exec(sql, args) : async_exec(sql)}
+        @db.log_connection_yield(sql, self, args){args ? async_exec_params(sql, args) : async_exec(sql)}
       end
     end
     
@@ -167,6 +177,15 @@ module Sequel
         end
       end
 
+      # Call a procedure with the given name and arguments.  Returns a hash if the procedure
+      # returns a value, and nil otherwise.  Example:
+      #
+      #   DB.call_procedure(:foo, 1, 2)
+      #   # CALL foo(1, 2)
+      def call_procedure(name, *args)
+        dataset.send(:call_procedure, name, args)
+      end
+
       # Connects to the database.  In addition to the standard database
       # options, using the :encoding or :charset option changes the
       # client encoding for the connection, :connect_timeout is a
@@ -179,7 +198,7 @@ module Sequel
         if USES_PG
           connection_params = {
             :host => opts[:host],
-            :port => opts[:port] || 5432,
+            :port => opts[:port],
             :dbname => opts[:database],
             :user => opts[:user],
             :password => opts[:password],
@@ -188,7 +207,7 @@ module Sequel
             :sslrootcert => opts[:sslrootcert]
           }.delete_if { |key, value| blank_object?(value) }
           connection_params.merge!(opts[:driver_options]) if opts[:driver_options]
-          conn = Adapter.connect(connection_params)
+          conn = Adapter.connect(opts[:conn_str] || connection_params)
 
           conn.instance_variable_set(:@prepared_statements, {})
 
@@ -211,6 +230,9 @@ module Sequel
         end
 
         conn.instance_variable_set(:@db, self)
+        if USES_PG && conn.respond_to?(:type_map_for_queries=) && defined?(PG_QUERY_TYPE_MAP)
+          conn.type_map_for_queries = PG_QUERY_TYPE_MAP
+        end
 
         if encoding = opts[:encoding] || opts[:charset]
           if conn.respond_to?(:set_client_encoding)
@@ -271,7 +293,7 @@ module Sequel
         def error_info(e)
           e = e.wrapped_exception if e.is_a?(DatabaseError)
           r = e.result
-          h = {
+          {
             :schema => r.error_field(::PG::PG_DIAG_SCHEMA_NAME),
             :table => r.error_field(::PG::PG_DIAG_TABLE_NAME),
             :column => r.error_field(::PG::PG_DIAG_COLUMN_NAME),
@@ -449,7 +471,7 @@ module Sequel
                   raise Error, 'calling #listen with :loop requires a block' unless block
                   loop_call = l.respond_to?(:call)
                   catch(:stop) do
-                    loop do
+                    while true
                       t = timeout_block ? [timeout_block.call] : []
                       conn.wait_for_notify(*t, &block)
                       l.call(conn) if loop_call
@@ -490,7 +512,7 @@ module Sequel
         @use_iso_date_format = typecast_value_boolean(@opts.fetch(:use_iso_date_format, true))
         initialize_postgres_adapter
         add_conversion_proc(17, method(:unescape_bytea)) if USES_PG
-        add_conversion_proc(1082, TYPE_TRANSLATOR.method(:date)) if @use_iso_date_format
+        add_conversion_proc(1082, TYPE_TRANSLATOR_DATE) if @use_iso_date_format
         self.convert_infinite_timestamps = @opts[:convert_infinite_timestamps]
       end
 
@@ -594,6 +616,9 @@ module Sequel
       
       # Use a cursor for paging.
       def paged_each(opts=OPTS, &block)
+        unless block_given?
+          return enum_for(:paged_each, opts)
+        end
         use_cursor(opts).each(&block)
       end
 
@@ -663,11 +688,6 @@ module Sequel
             end
             LiteralString.new("#{prepared_arg_placeholder}#{i}")
           end
-
-          # Always assume a prepared argument.
-          def prepared_arg?(k)
-           true
-          end
         end
 
         BindArgumentMethods = prepared_statements_module(:bind, [ArgumentMapper], %w'execute execute_dui')
@@ -692,6 +712,15 @@ module Sequel
       
       private
       
+      # Generate and execute a procedure call.
+      def call_procedure(name, args)
+        sql = String.new
+        sql << "CALL "
+        identifier_append(sql, name)
+        literal_append(sql, args)
+        with_sql_first(sql)
+      end
+
       # Use a cursor to fetch groups of records at a time, yielding them to the block.
       def cursor_fetch_rows(sql)
         server_opts = {:server=>@opts[:server] || :read_only}
@@ -712,7 +741,7 @@ module Sequel
               yield_hash_rows(res, cols){|h| yield h}
               return if res.ntuples < rows_per_fetch
             end
-            loop do
+            while true
               execute(fetch_sql) do |res|
                 yield_hash_rows(res, cols){|h| yield h}
                 return if res.ntuples < rows_per_fetch
@@ -737,9 +766,9 @@ module Sequel
         cols = []
         procs = db.conversion_procs
         res.nfields.times do |fieldnum|
-          cols << [fieldnum, procs[res.ftype(fieldnum)], output_identifier(res.fname(fieldnum))]
+          cols << [procs[res.ftype(fieldnum)], output_identifier(res.fname(fieldnum))]
         end
-        self.columns = cols.map{|c| c[2]}
+        self.columns = cols.map{|c| c[1]}
         cols
       end
       
@@ -756,13 +785,20 @@ module Sequel
       # For each row in the result set, yield a hash with column name symbol
       # keys and typecasted values.
       def yield_hash_rows(res, cols)
-        res.ntuples.times do |recnum|
+        ntuples = res.ntuples
+        recnum = 0
+        while recnum < ntuples
+          fieldnum = 0
+          nfields = cols.length
           converted_rec = {}
-          cols.each do |fieldnum, type_proc, fieldsym|
+          while fieldnum < nfields
+            type_proc, fieldsym = cols[fieldnum]
             value = res.getvalue(recnum, fieldnum)
             converted_rec[fieldsym] = (value && type_proc) ? type_proc.call(value) : value
+            fieldnum += 1 
           end
           yield converted_rec
+          recnum += 1
         end
       end
     end

data/lib/sequel/adapters/shared/access.rb

@@ -109,12 +109,6 @@ module Sequel
           complex_expression_sql_append(sql, :LIKE, args)
         when :'NOT ILIKE'
           complex_expression_sql_append(sql, :'NOT LIKE', args)
-        when :LIKE, :'NOT LIKE'
-          sql << '('
-          literal_append(sql, args[0])
-          sql << ' ' << op.to_s << ' '
-          literal_append(sql, args[1])
-          sql << ')'
         when :'!='
           sql << '('
           literal_append(sql, args[0])
@@ -177,6 +171,12 @@ module Sequel
         clone(:into => table)
       end
 
+      # Access uses [] for quoting identifiers, and can't handle
+      # ] inside identifiers.
+      def quoted_identifier_append(sql, v)
+        sql << '[' << v.to_s << ']'
+      end
+
       # Access does not support derived column lists.
       def supports_derived_column_lists?
         false
@@ -240,6 +240,16 @@ module Sequel
         end
       end
 
+      # Access does not natively support NULLS FIRST/LAST.
+      def requires_emulating_nulls_first?
+        true
+      end
+
+      # Access doesn't support ESCAPE for LIKE.
+      def requires_like_escape?
+        false
+      end
+
       # Access requires parentheses when joining more than one table
       def select_from_sql(sql)
         if f = @opts[:from]
@@ -275,12 +285,6 @@ module Sequel
           literal_append(sql, l)
         end
       end
-
-      # Access uses [] for quoting identifiers, and can't handle
-      # ] inside identifiers.
-      def quoted_identifier_append(sql, v)
-        sql << '[' << v.to_s << ']'
-      end
     end
   end
 end

data/lib/sequel/adapters/shared/db2.rb

@@ -418,6 +418,11 @@ module Sequel
         false
       end
 
+      # At least some versions of DB do not support NULLS FIRST/LAST.
+      def requires_emulating_nulls_first?
+        true
+      end
+
       # Modify the sql to limit the number of rows returned.
       # Uses :offset_strategy Database option to determine how to format the
       # limit and offset.

data/lib/sequel/adapters/shared/mssql.rb

@@ -40,6 +40,12 @@ module Sequel
       # :numrows :: The number of rows affected by the stored procedure
       # output params :: Values for any output paramters, using the name given for the output parameter
       #
+      # Because Sequel datasets only support a single result set per query, and retrieving 
+      # the result code and number of rows requires a query, this does not support
+      # stored procedures which also return result sets.  To handle such stored procedures,
+      # you should drop down to the connection/driver level by using Sequel::Database#synchronize
+      # to get access to the underlying connection object.
+      #
       # Examples:
       #
       #     DB.call_mssql_sproc(:SequelTest, {args: ['input arg', :output]})
@@ -273,7 +279,7 @@ module Sequel
             end
           end
           sqls << "ALTER TABLE #{quote_schema_table(table)} ALTER COLUMN #{column_definition_sql(op)}"
-          sqls << alter_table_sql(table, op.merge(:op=>:set_column_default, :default=>default)) if default
+          sqls << alter_table_sql(table, op.merge(:op=>:set_column_default, :default=>default, :skip_drop_default=>true)) if default
           sqls
         when :set_column_null
           sch = schema(table).find{|k,v| k.to_s == op[:name].to_s}.last
@@ -284,7 +290,9 @@ module Sequel
           end
           "ALTER TABLE #{quote_schema_table(table)} ALTER COLUMN #{quote_identifier(op[:name])} #{type_literal(:type=>type)} #{'NOT ' unless op[:null]}NULL"
         when :set_column_default
-          "ALTER TABLE #{quote_schema_table(table)} ADD CONSTRAINT #{quote_identifier("sequel_#{table}_#{op[:name]}_def")} DEFAULT #{literal(op[:default])} FOR #{quote_identifier(op[:name])}"
+          sqls = []
+          add_drop_default_constraint_sql(sqls, table, op[:name]) unless op[:skip_drop_default]
+          sqls << "ALTER TABLE #{quote_schema_table(table)} ADD CONSTRAINT #{quote_identifier("sequel_#{table}_#{op[:name]}_def")} DEFAULT #{literal(op[:default])} FOR #{quote_identifier(op[:name])}"
         else
           super(table, op)
         end
@@ -421,20 +429,19 @@ module Sequel
         m = output_identifier_meth(opts[:dataset])
         m2 = input_identifier_meth(opts[:dataset])
         tn = m2.call(table_name.to_s)
-        table_id = get(Sequel.function(:object_id, tn))
         info_sch_sch = opts[:information_schema_schema]
         inf_sch_qual = lambda{|s| info_sch_sch ? Sequel.qualify(info_sch_sch, s) : Sequel[s]}
-        sys_qual = lambda{|s| info_sch_sch ? Sequel.qualify(info_sch_sch, Sequel.qualify(Sequel.lit(''), s)) : Sequel[s]}
+        table_id = metadata_dataset.from(inf_sch_qual.call(Sequel[:sys][:objects])).where(:name => tn).select_map(:object_id).first
 
-        identity_cols = metadata_dataset.from(Sequel.lit('[sys].[columns]')).
+        identity_cols = metadata_dataset.from(inf_sch_qual.call(Sequel[:sys][:columns])).
           where(:object_id=>table_id, :is_identity=>true).
           select_map(:name)
 
-        pk_index_id = metadata_dataset.from(sys_qual.call(Sequel.lit('sysindexes'))).
+        pk_index_id = metadata_dataset.from(inf_sch_qual.call(Sequel[:sys][:sysindexes])).
           where(:id=>table_id, :indid=>1..254){{(status & 2048)=>2048}}.
           get(:indid)
-        pk_cols = metadata_dataset.from(sys_qual.call(Sequel.lit('sysindexkeys')).as(:sik)).
-          join(sys_qual.call(Sequel.lit('syscolumns')).as(:sc), :id=>:id, :colid=>:colid).
+        pk_cols = metadata_dataset.from(inf_sch_qual.call(Sequel[:sys][:sysindexkeys]).as(:sik)).
+          join(inf_sch_qual.call(Sequel[:sys][:syscolumns]).as(:sc), :id=>:id, :colid=>:colid).
           where{{sik[:id]=>table_id, sik[:indid]=>pk_index_id}}.
           select_order_map{sc[:name]}
 
@@ -652,11 +659,11 @@ module Sequel
       # case by default, and that also adds a default order, so it's better to just
       # avoid the subquery.
       def select_sql
-        if @opts[:offset] && !@opts[:order] && is_2012_or_later?
-          order(1).select_sql
-        else
-          super
+        if @opts[:offset]
+          raise(Error, "Using with_ties is not supported with an offset on Microsoft SQL Server") if @opts[:limit_with_ties]
+          return order(1).select_sql if is_2012_or_later? && !@opts[:order]
         end
+        super
       end
 
       # The version of the database server.
@@ -748,6 +755,12 @@ module Sequel
         false
       end
       
+      # Use WITH TIES when limiting the result set to also include additional
+      # rows matching the last row.
+      def with_ties
+        clone(:limit_with_ties=>true)
+      end
+
       protected
       
       # If returned primary keys are requested, use OUTPUT unless already set on the
@@ -759,19 +772,15 @@ module Sequel
           output(nil, [SQL::QualifiedIdentifier.new(:inserted, first_primary_key)])._import(columns, values, opts)
         elsif @opts[:output]
           statements = multi_insert_sql(columns, values)
+          ds = naked
           @db.transaction(opts.merge(:server=>@opts[:server])) do
-            statements.map{|st| with_sql(st)}
+            statements.map{|st| ds.with_sql(st)}
           end.first.map{|v| v.length == 1 ? v.values.first : v}
         else
           super
         end
       end
 
-      # MSSQL does not allow ordering in sub-clauses unless TOP (limit) is specified
-      def aggregate_dataset
-        (options_overlap(Sequel::Dataset::COUNT_FROM_SELF_OPTS) && !options_overlap([:limit])) ? unordered.from_self : super
-      end
-
       # If the dataset using a order without a limit or offset or custom SQL, 
       # remove the order.  Compounds on Microsoft SQL Server have undefined
       # order unless the result is specifically ordered.  Applying the current
@@ -791,6 +800,11 @@ module Sequel
 
       private
 
+      # MSSQL does not allow ordering in sub-clauses unless TOP (limit) is specified
+      def aggregate_dataset
+        (options_overlap(Sequel::Dataset::COUNT_FROM_SELF_OPTS) && !options_overlap([:limit])) ? unordered.from_self : super
+      end
+
       # Allow update and delete for unordered, limited datasets only.
       def check_not_limited!(type)
         return if @opts[:skip_limit_check] && type != :truncate
@@ -951,6 +965,10 @@ module Sequel
           sql << " TOP "
           literal_append(sql, l)
         end
+
+        if @opts[:limit_with_ties]
+          sql << " WITH TIES"
+        end
       end
 
       def update_limit_sql(sql)
@@ -1046,6 +1064,11 @@ module Sequel
         end
       end
 
+      # MSSQL does not natively support NULLS FIRST/LAST.
+      def requires_emulating_nulls_first?
+        true
+      end
+
       # MSSQL supports 100-nsec precision for time columns, but ruby by
       # default only supports usec precision.
       def sqltime_precision

data/lib/sequel/adapters/shared/mysql.rb

@@ -19,7 +19,7 @@ module Sequel
       include Sequel::Database::SplitAlterTable
 
       CAST_TYPES = {String=>:CHAR, Integer=>:SIGNED, Time=>:DATETIME, DateTime=>:DATETIME, Numeric=>:DECIMAL, BigDecimal=>:DECIMAL, File=>:BINARY}.freeze
-      COLUMN_DEFINITION_ORDER = [:collate, :null, :default, :unique, :primary_key, :auto_increment, :references].freeze
+      COLUMN_DEFINITION_ORDER = [:generated, :collate, :null, :default, :unique, :primary_key, :auto_increment, :references].freeze
       
       # Set the default charset used for CREATE TABLE.  You can pass the
       # :charset option to create_table to override this setting.
@@ -137,6 +137,11 @@ module Sequel
         true
       end
       
+      # Generated columns are supported in MariaDB 5.2.0+ and MySQL 5.7.6+.
+      def supports_generated_columns?
+        server_version >= (mariadb? ? 50200 : 50706)
+      end
+
       # MySQL 5+ supports prepared transactions (two-phase commit) using XA
       def supports_prepared_transactions?
         server_version >= 50000
@@ -227,7 +232,18 @@ module Sequel
       alias alter_table_rename_column_sql alter_table_change_column_sql
       alias alter_table_set_column_type_sql alter_table_change_column_sql
       alias alter_table_set_column_null_sql alter_table_change_column_sql
-      alias alter_table_set_column_default_sql alter_table_change_column_sql
+
+      def alter_table_set_column_default_sql(table, op)
+        return super unless op[:default].nil?
+
+        opts = schema(table).find{|x| x[0] == op[:name]}
+
+        if opts && opts[1][:allow_null] == false
+          "ALTER COLUMN #{quote_identifier(op[:name])} DROP DEFAULT"
+        else
+          super
+        end
+      end
 
       def alter_table_add_constraint_sql(table, op)
         if op[:type] == :foreign_key
@@ -331,6 +347,23 @@ module Sequel
         end
       end
 
+      # Add generation clause SQL fragment to column creation SQL.
+      def column_definition_generated_sql(sql, column)
+        if (generated_expression = column[:generated_always_as])
+          sql << " GENERATED ALWAYS AS (#{literal(generated_expression)})"
+          case (type = column[:generated_type])
+          when nil
+            # none, database default
+          when :virtual
+            sql << " VIRTUAL"
+          when :stored
+            sql << (mariadb? ? " PERSISTENT" : " STORED")
+          else
+            raise Error, "unsupported :generated_type option: #{type.inspect}"
+          end
+        end
+      end
+
       def column_definition_order
         COLUMN_DEFINITION_ORDER
       end
@@ -408,7 +441,7 @@ module Sequel
         /cannot be null/ => NotNullConstraintViolation,
         /Deadlock found when trying to get lock; try restarting transaction/ => SerializationFailure,
         /CONSTRAINT .+ failed for/ => CheckConstraintViolation,
-        /\AStatement aborted because lock\(s\) could not be acquired immediately and NOWAIT is set\./ => DatabaseLockTimeout,
+        /\A(Statement aborted because lock\(s\) could not be acquired immediately and NOWAIT is set\.|Lock wait timeout exceeded; try restarting transaction)/ => DatabaseLockTimeout,
       }.freeze
       def database_error_regexps
         DATABASE_ERROR_REGEXPS
@@ -473,7 +506,11 @@ module Sequel
         metadata_dataset.with_sql("DESCRIBE ?", table).map do |row|
           extra = row.delete(:Extra)
           if row[:primary_key] = row.delete(:Key) == 'PRI'
-            row[:auto_increment] = !!(extra.to_s =~ /auto_increment/io)
+            row[:auto_increment] = !!(extra.to_s =~ /auto_increment/i)
+          end
+          if supports_generated_columns?
+            # Extra field contains VIRTUAL or PERSISTENT for generated columns
+            row[:generated] = !!(extra.to_s =~ /VIRTUAL|STORED|PERSISTENT/i)
           end
           row[:allow_null] = row.delete(:Null) == 'YES'
           row[:default] = row.delete(:Default)
@@ -565,7 +602,7 @@ module Sequel
 
       Dataset.def_sql_method(self, :delete, %w'with delete from where order limit')
       Dataset.def_sql_method(self, :insert, %w'insert ignore into columns values on_duplicate_key_update')
-      Dataset.def_sql_method(self, :select, %w'with select distinct calc_found_rows columns from join where group having compounds order limit lock')
+      Dataset.def_sql_method(self, :select, %w'with select distinct calc_found_rows columns from join where group having window compounds order limit lock')
       Dataset.def_sql_method(self, :update, %w'with update ignore table set where order limit')
 
       include Sequel::Dataset::Replace
@@ -654,7 +691,7 @@ module Sequel
       end
       
       # Return the results of an EXPLAIN query as a string. Options:
-      # :extended :: Use EXPLAIN EXPTENDED instead of EXPLAIN if true.
+      # :extended :: Use EXPLAIN EXTENDED instead of EXPLAIN if true.
       def explain(opts=OPTS)
         # Load the PrettyTable class, needed for explain output
         Sequel.extension(:_pretty_table) unless defined?(Sequel::PrettyTable)
@@ -680,15 +717,6 @@ module Sequel
         SQL::PlaceholderLiteralString.new((opts[:boolean] ? MATCH_AGAINST_BOOLEAN : MATCH_AGAINST), [Array(cols), terms])
       end
 
-      # Transforms :straight to STRAIGHT_JOIN.
-      def join_type_sql(join_type)
-        if join_type == :straight
-          'STRAIGHT_JOIN'
-        else
-          super
-        end
-      end
-      
       # Sets up the insert methods to use INSERT IGNORE.
       # Useful if you have a unique key and want to just skip
       # inserting rows that violate the unique key restriction.
@@ -767,9 +795,9 @@ module Sequel
         true
       end
 
-      # MySQL does not support INTERSECT or EXCEPT
+      # MariaDB 10.3+ supports INTERSECT or EXCEPT
       def supports_intersect_except?
-        false
+        db.mariadb? && db.server_version >= 100300
       end
       
       # MySQL does not support limits in correlated subqueries (or any subqueries that use IN).
@@ -782,9 +810,9 @@ module Sequel
         true
       end
 
-      # MySQL 8+ supports NOWAIT.
+      # MySQL 8+ and MariaDB 10.3+ support NOWAIT.
       def supports_nowait?
-        !db.mariadb? && db.server_version >= 80000
+        db.server_version >= (db.mariadb? ? 100300 : 80000)
       end
 
       # MySQL's DISTINCT ON emulation using GROUP BY does not respect the
@@ -809,6 +837,11 @@ module Sequel
         db.supports_timestamp_usecs?
       end
 
+      # MySQL 8+ supports WINDOW clause.
+      def supports_window_clause?
+        !db.mariadb? && db.server_version >= 80000
+      end
+
       # MariaDB 10.2+ and MySQL 8+ support window functions
       def supports_window_functions?
         db.server_version >= (db.mariadb? ? 100200 : 80000)
@@ -914,6 +947,15 @@ module Sequel
         end
       end
 
+      # Transforms :straight to STRAIGHT_JOIN.
+      def join_type_sql(join_type)
+        if join_type == :straight
+          'STRAIGHT_JOIN'
+        else
+          super
+        end
+      end
+      
       # MySQL allows a LIMIT in DELETE and UPDATE statements.
       def limit_sql(sql)
         if l = @opts[:limit]
@@ -966,6 +1008,11 @@ module Sequel
         super || key == :insert_ignore || key == :update_ignore || key == :on_duplicate_key_update
       end
 
+      # MySQL does not natively support NULLS FIRST/LAST.
+      def requires_emulating_nulls_first?
+        true
+      end
+
       def select_only_offset_sql(sql)
         sql << " LIMIT "
         literal_append(sql, @opts[:offset])