sequel 3.38.0 → 3.39.0

data/lib/sequel/adapters/mysql.rb

@@ -300,7 +300,7 @@ module Sequel
             # Pretend tinyint is another integer type if its length is not 1, to
             # avoid casting to boolean if Sequel::MySQL.convert_tinyint_to_bool
             # is set.
-            type_proc = f.type == 1 && f.length != 1 ? cps[2] : cps[f.type]
+            type_proc = f.type == 1 && cast_tinyint_integer?(f) ? cps[2] : cps[f.type]
             [output_identifier(f.name), type_proc, i+=1]
           end
           @columns = cols.map{|c| c.first}
@@ -338,6 +338,13 @@ module Sequel
       end
       
       private
+
+      # Whether a tinyint field should be casted as an integer.  By default,
+      # casts to integer if the field length is not 1.  Can be overwritten
+      # to make tinyint casting dataset dependent.
+      def cast_tinyint_integer?(field)
+        field.length != 1
+      end
       
       # Set the :type option to :select if it hasn't been set.
       def execute(sql, opts={}, &block)

data/lib/sequel/adapters/mysql2.rb

@@ -87,7 +87,7 @@ module Sequel
       # yield the connection if a block is given.
       def _execute(conn, sql, opts)
         begin
-          r = log_yield((log_sql = opts[:log_sql]) ? sql + log_sql : sql){conn.query(sql, :database_timezone => timezone, :application_timezone => Sequel.application_timezone, :cast_booleans => convert_tinyint_to_bool)}
+          r = log_yield((log_sql = opts[:log_sql]) ? sql + log_sql : sql){conn.query(sql, :database_timezone => timezone, :application_timezone => Sequel.application_timezone)}
           if opts[:type] == :select
             yield r if r
           elsif block_given?
@@ -150,7 +150,7 @@ module Sequel
             cols = r.fields
             @columns = cols2 = cols.map{|c| output_identifier(c.to_s)}
             cs = cols.zip(cols2)
-            r.each do |row|
+            r.each(:cast_booleans=>convert_tinyint_to_bool?) do |row|
               h = {}
               cs.each do |a, b|
                 h[b] = row[a]
@@ -159,7 +159,7 @@ module Sequel
             end
           else
             @columns = r.fields
-            r.each{|h| yield h}
+            r.each(:cast_booleans=>convert_tinyint_to_bool?){|h| yield h}
           end
         end
         self
@@ -167,6 +167,13 @@ module Sequel
 
       private
 
+      # Whether to cast tinyint(1) columns to integer instead of boolean.
+      # By default, uses the opposite of the database's convert_tinyint_to_bool
+      # setting.  Exists for compatibility with the mysql adapter.
+      def convert_tinyint_to_bool?
+        @db.convert_tinyint_to_bool
+      end
+
       # Set the :type option to :select if it hasn't been set.
       def execute(sql, opts={}, &block)
         super(sql, {:type=>:select}.merge(opts), &block)

data/lib/sequel/adapters/postgres.rb

@@ -133,7 +133,7 @@ module Sequel
         begin
           block_given? ? yield(q) : q.cmd_tuples
         ensure
-          q.clear if q
+          q.clear if q && q.respond_to?(:clear)
         end
       end
 
@@ -235,13 +235,11 @@ module Sequel
       end
 
       if SEQUEL_POSTGRES_USES_PG
-        # +copy_table+ uses PostgreSQL's +COPY+ SQL statement to return formatted
+        # +copy_table+ uses PostgreSQL's +COPY TO STDOUT+ SQL statement to return formatted
         # results directly to the caller.  This method is only supported if pg is the
         # underlying ruby driver.  This method should only be called if you want
-        # results returned to the client.  If you are using +COPY FROM+ or +COPY TO+
-        # with a filename, you should just use +run+ instead of this method.  This
-        # method does not currently support +COPY FROM STDIN+, but that may be supported
-        # in the future.
+        # results returned to the client.  If you are using +COPY TO+
+        # with a filename, you should just use +run+ instead of this method.
         #
         # The table argument supports the following types:
         #
@@ -297,6 +295,67 @@ module Sequel
           end 
         end
 
+        # +copy_into+ uses PostgreSQL's +COPY FROM STDIN+ SQL statement to do very fast inserts 
+        # into a table using input preformatting in either CSV or PostgreSQL text format.
+        # This method is only supported if pg is the underlying ruby driver.  This method should only be called if you want
+        # results returned to the client.  If you are using +COPY FROM+
+        # with a filename, you should just use +run+ instead of this method.
+        #
+        # The following options are respected:
+        #
+        # :columns :: The columns to insert into, with the same order as the columns in the
+        #             input data.  If this isn't given, uses all columns in the table.
+        # :data :: The data to copy to PostgreSQL, which should already be in CSV or PostgreSQL
+        #          text format.  This can be either a string, or any object that responds to
+        #          each and yields string.
+        # :format :: The format to use.  text is the default, so this should be :csv or :binary.
+        # :options :: An options SQL string to use, which should contain comma separated options.
+        # :server :: The server on which to run the query.
+        #
+        # If a block is provided and :data option is not, this will yield to the block repeatedly.
+        # The block should return a string, or nil to signal that it is finished.
+        def copy_into(table, opts={})
+          sql = "COPY #{literal(table)}"
+          if cols = opts[:columns]
+            sql << literal(Array(cols))
+          end
+          sql << " FROM STDIN"
+          if opts[:options] || opts[:format]
+            sql << " ("
+            sql << "FORMAT #{opts[:format]}" if opts[:format]
+            sql << "#{', ' if opts[:format]}#{opts[:options]}" if opts[:options]
+            sql << ')'
+          end
+          
+          data = opts[:data]
+          data = Array(data) if data.is_a?(String)
+
+          if block_given? && data
+            raise Error, "Cannot provide both a :data option and a block to copy_into"
+          elsif !block_given? && !data
+            raise Error, "Must provide either a :data option or a block to copy_into"
+          end
+
+          synchronize(opts[:server]) do |conn| 
+            conn.execute(sql)
+            begin
+              if block_given?
+                while buf = yield
+                  conn.put_copy_data(buf)
+                end
+              else
+                data.each{|buf| conn.put_copy_data(buf)}
+              end
+            rescue Exception => e
+              conn.put_copy_end("ruby exception occurred while copying data into PostgreSQL")
+              raise
+            ensure
+              conn.put_copy_end unless e
+              conn.get_result
+            end
+          end 
+        end
+
         # Listens on the given channel (or multiple channels if channel is an array), waiting for notifications.
         # After a notification is received, or the timeout has passed, stops listening to the channel. Options:
         #
@@ -373,6 +432,11 @@ module Sequel
         end
       end
 
+      # Execute the prepared statement name with the given arguments on the connection.
+      def _execute_prepared_statement(conn, ps_name, args, opts)
+        conn.exec_prepared(ps_name, args)
+      end
+
       # Convert exceptions raised from the block into DatabaseErrors.
       def check_database_errors
         begin
@@ -426,11 +490,11 @@ module Sequel
           log_sql << ")"
         end
 
-        q = conn.check_disconnect_errors{log_yield(log_sql, args){conn.exec_prepared(ps_name, args)}}
+        q = conn.check_disconnect_errors{log_yield(log_sql, args){_execute_prepared_statement(conn, ps_name, args, opts)}}
         begin
           block_given? ? yield(q) : q.cmd_tuples
         ensure
-          q.clear
+          q.clear if q && q.respond_to?(:clear)
         end
       end
 

data/lib/sequel/adapters/shared/db2.rb

@@ -197,6 +197,7 @@ module Sequel
       PAREN_CLOSE = Dataset::PAREN_CLOSE
       PAREN_OPEN = Dataset::PAREN_OPEN
       BITWISE_METHOD_MAP = {:& =>:BITAND, :| => :BITOR, :^ => :BITXOR, :'B~'=>:BITNOT}
+      EMULATED_FUNCTION_MAP = {:char_length=>'length'.freeze}
       BOOL_TRUE = '1'.freeze
       BOOL_FALSE = '0'.freeze
       CAST_STRING_OPEN = "RTRIM(CHAR(".freeze

data/lib/sequel/adapters/shared/mssql.rb

@@ -12,6 +12,7 @@ module Sequel
       SQL_ROLLBACK = "IF @@TRANCOUNT > 0 ROLLBACK TRANSACTION".freeze
       SQL_ROLLBACK_TO_SAVEPOINT = 'IF @@TRANCOUNT > 0 ROLLBACK TRANSACTION autopoint_%d'.freeze
       SQL_SAVEPOINT = 'SAVE TRANSACTION autopoint_%d'.freeze
+      MSSQL_DEFAULT_RE = /\A(?:\(N?('.*')\)|\(\((-?\d+(?:\.\d+)?)\)\))\z/
       
       # Whether to use N'' to quote strings, which allows unicode characters inside the
       # strings.  True by default for compatibility, can be set to false for a possible
@@ -109,6 +110,39 @@ module Sequel
         AUTO_INCREMENT
       end
       
+      # Preprocess the array of operations.  If it looks like some operations depend
+      # on results of earlier operations and may require reloading the schema to
+      # work correctly, split those operations into separate lists, and between each
+      # list, remove the cached schema so that the later operations deal with the
+      # then current table schema.
+      def apply_alter_table(name, ops)
+        modified_columns = []
+        op_groups = [[]]
+        ops.each do |op|
+          case op[:op]
+          when :add_column, :set_column_type, :set_column_null
+            if modified_columns.include?(op[:name])
+              op_groups << []
+            else
+              modified_columns << op[:name]
+            end
+          when :rename_column
+            if modified_columns.include?(op[:name]) || modified_columns.include?(op[:new_name])
+              op_groups << []
+            end
+            modified_columns << op[:name] unless modified_columns.include?(op[:name])
+            modified_columns << op[:new_name] unless modified_columns.include?(op[:new_name])
+          end
+          op_groups.last << op
+        end
+
+        op_groups.each do |ops|
+          next if ops.empty?
+          alter_table_sql_list(name, ops).each{|sql| execute_ddl(sql)}
+          remove_cached_schema(name)
+        end
+      end
+    
       # MSSQL specific syntax for altering tables.
       def alter_table_sql(table, op)
         case op[:op]
@@ -158,6 +192,14 @@ module Sequel
         SQL_BEGIN
       end
       
+      # Handle MSSQL specific default format.
+      def column_schema_normalize_default(default, type)
+        if m = MSSQL_DEFAULT_RE.match(default)
+          default = m[1] || m[2]
+        end
+        super(default, type)
+      end
+
       # Commit the active transaction on the connection, does not commit/release
       # savepoints.
       def commit_transaction(conn, opts={})
@@ -421,6 +463,21 @@ module Sequel
         mutation_method(:disable_insert_output)
       end
 
+      # There is no function on Microsoft SQL Server that does character length
+      # and respects trailing spaces (datalength respects trailing spaces, but
+      # counts bytes instead of characters).  Use a hack to work around the
+      # trailing spaces issue.
+      def emulated_function_sql_append(sql, f)
+        case f.f
+        when :char_length
+          literal_append(sql, SQL::Function.new(:len, Sequel.join([f.args.first, 'x'])) - 1)
+        when :trim
+          literal_append(sql, SQL::Function.new(:ltrim, SQL::Function.new(:rtrim, f.args.first)))
+        else
+          super
+        end
+      end
+      
       # MSSQL uses the CONTAINS keyword for full text search
       def full_text_search(cols, terms, opts = {})
         terms = "\"#{terms.join('" OR "')}\"" if terms.is_a?(Array)

data/lib/sequel/adapters/shared/mysql.rb

@@ -33,6 +33,7 @@ module Sequel
       CAST_TYPES = {String=>:CHAR, Integer=>:SIGNED, Time=>:DATETIME, DateTime=>:DATETIME, Numeric=>:DECIMAL, BigDecimal=>:DECIMAL, File=>:BINARY}
       COLUMN_DEFINITION_ORDER = [:collate, :null, :default, :unique, :primary_key, :auto_increment, :references]
       PRIMARY = 'PRIMARY'.freeze
+      MYSQL_TIMESTAMP_RE = /\ACURRENT_(?:DATE|TIMESTAMP)?\z/
       
       # MySQL's cast rules are restrictive in that you can't just cast to any possible
       # database type.
@@ -110,8 +111,10 @@ module Sequel
 
       # Get version of MySQL server, used for determined capabilities.
       def server_version
-        m = /(\d+)\.(\d+)\.(\d+)/.match(get(SQL::Function.new(:version)))
-        @server_version ||= (m[1].to_i * 10000) + (m[2].to_i * 100) + m[3].to_i
+        @server_version ||= begin
+          m = /(\d+)\.(\d+)\.(\d+)/.match(get(SQL::Function.new(:version)))
+          (m[1].to_i * 10000) + (m[2].to_i * 100) + m[3].to_i
+        end
       end
       
       # MySQL supports CREATE TABLE IF NOT EXISTS syntax.
@@ -167,18 +170,50 @@ module Sequel
       
       private
       
-      # Use MySQL specific syntax for rename column, set column type, and
-      # drop index cases.
-      def alter_table_sql(table, op)
+      # Preprocess the array of operations.  If it looks like some operations depend
+      # on results of earlier operations and may require reloading the schema to
+      # work correctly, split those operations into separate lists, and between each
+      # list, remove the cached schema so that the later operations deal with the
+      # then current table schema.
+      def apply_alter_table(name, ops)
+        modified_columns = []
+        op_groups = [[]]
+        ops.each do |op|
+          case op[:op]
+          when :add_column, :set_column_type, :set_column_null, :set_column_default
+            if modified_columns.include?(op[:name])
+              op_groups << []
+            else
+              modified_columns << op[:name]
+            end
+          when :rename_column
+            if modified_columns.include?(op[:name]) || modified_columns.include?(op[:new_name])
+              op_groups << []
+            end
+            modified_columns << op[:name] unless modified_columns.include?(op[:name])
+            modified_columns << op[:new_name] unless modified_columns.include?(op[:new_name])
+          end
+          op_groups.last << op
+        end
+
+        op_groups.each do |ops|
+          next if ops.empty?
+          alter_table_sql_list(name, ops).each{|sql| execute_ddl(sql)}
+          remove_cached_schema(name)
+        end
+      end
+    
+      # Use MySQL specific syntax for some alter table operations.
+      def alter_table_op_sql(table, op)
         case op[:op]
         when :add_column
           if related = op.delete(:table)
-            sql = super(table, op)
+            sql = super
             op[:table] = related
             op[:key] ||= primary_key_from_schema(related)
-            [sql, "ALTER TABLE #{quote_schema_table(table)} ADD FOREIGN KEY (#{quote_identifier(op[:name])})#{column_references_sql(op)}"]
+            sql << ", ADD FOREIGN KEY (#{quote_identifier(op[:name])})#{column_references_sql(op)}"
           else
-            super(table, op)
+            super
           end
         when :rename_column, :set_column_type, :set_column_null, :set_column_default
           o = op[:op]
@@ -189,31 +224,52 @@ module Sequel
           opts[:null] = o == :set_column_null ? op[:null] : opts[:allow_null]
           opts[:default] = o == :set_column_default ? op[:default] : opts[:ruby_default]
           opts.delete(:default) if opts[:default] == nil
-          "ALTER TABLE #{quote_schema_table(table)} CHANGE COLUMN #{quote_identifier(op[:name])} #{column_definition_sql(op.merge(opts))}"
-        when :drop_index
-          "#{drop_index_sql(table, op)} ON #{quote_schema_table(table)}"
+          "CHANGE COLUMN #{quote_identifier(op[:name])} #{column_definition_sql(op.merge(opts))}"
         when :drop_constraint
           type = case op[:type]
           when :primary_key
-            return "ALTER TABLE #{quote_schema_table(table)} DROP PRIMARY KEY"
+            "DROP PRIMARY KEY"
           when :foreign_key
-            'FOREIGN KEY'
+            "DROP FOREIGN KEY #{quote_identifier(op[:name])}"
           when :unique
-            'INDEX'
-          else
-            raise(Error, "must specify constraint type via :type=>(:foreign_key|:primary_key|:unique) when dropping constraints on MySQL")
+            "DROP INDEX #{quote_identifier(op[:name])}"
           end
-          "ALTER TABLE #{quote_schema_table(table)} DROP #{type} #{quote_identifier(op[:name])}"
         when :add_constraint
           if op[:type] == :foreign_key
             op[:key] ||= primary_key_from_schema(op[:table])
           end
-          super(table, op)
+          super
         else
-          super(table, op)
+          super
         end
       end
 
+      # MySQL server requires table names when dropping indexes.
+      def alter_table_sql(table, op)
+        case op[:op]
+        when :drop_index
+          "#{drop_index_sql(table, op)} ON #{quote_schema_table(table)}"
+        else
+          super
+        end
+      end
+
+      # Handle MySQL specific default format.
+      def column_schema_normalize_default(default, type)
+        if column_schema_default_string_type?(type)
+          return if [:date, :datetime, :time].include?(type) && MYSQL_TIMESTAMP_RE.match(default)
+          default = "'#{default.gsub("'", "''").gsub('\\', '\\\\')}'"
+        end
+        super(default, type)
+      end
+
+      # Don't allow combining adding foreign key operations with other
+      # operations, since in some cases adding a foreign key constraint in
+      # the same query as other operations results in MySQL error 150.
+      def combinable_alter_table_op?(op)
+        super && !(op[:op] == :add_constraint && op[:type] == :foreign_key)
+      end
+
       # The SQL queries to execute on initial connection
       def mysql_connection_setting_sqls
         sqls = []
@@ -369,6 +425,16 @@ module Sequel
         end
       end
 
+      # Recognize MySQL set type.
+      def schema_column_type(db_type)
+        case db_type
+        when /\Aset/io
+          :set
+        else
+          super
+        end
+      end
+
       # Use the MySQL specific DESCRIBE syntax to get a table description.
       def schema_parse_table(table_name, opts)
         m = output_identifier_meth(opts[:dataset])
@@ -387,6 +453,11 @@ module Sequel
         end
       end
 
+      # MySQL can combine multiple alter table ops into a single query.
+      def supports_combining_alter_table_ops?
+        true
+      end
+
       # Respect the :size option if given to produce
       # tinyblob, mediumblob, and longblob if :tiny,
       # :medium, or :long is given.
@@ -667,6 +738,11 @@ module Sequel
         false
       end
     
+      # MySQL supports pattern matching via regular expressions
+      def supports_regexp?
+        true
+      end
+
       # MySQL does support fractional timestamps in literal timestamps, but it
       # ignores them.  Also, using them seems to cause problems on 1.9.  Since
       # they are ignored anyway, not using them is probably best.