departure-next 6.7.1.pre.pre

data/lib/active_record/connection_adapters/percona_adapter.rb

@@ -0,0 +1,187 @@
+require 'active_record/connection_adapters/abstract_mysql_adapter'
+require 'active_record/connection_adapters/statement_pool'
+require 'active_record/connection_adapters/mysql2_adapter'
+require 'active_record/connection_adapters/patch_connection_handling'
+require 'active_support/core_ext/string/filters'
+require 'departure'
+require 'forwardable'
+require_relative './patch_connection_handling'
+
+module ActiveRecord
+  module ConnectionAdapters
+    class DepartureAdapter < AbstractMysqlAdapter
+      TYPE_MAP = Type::TypeMap.new.tap { |m| initialize_type_map(m) } if defined?(initialize_type_map)
+
+      class Column < ActiveRecord::ConnectionAdapters::MySQL::Column
+        def adapter
+          DepartureAdapter
+        end
+      end
+
+      class SchemaCreation < ActiveRecord::ConnectionAdapters::MySQL::SchemaCreation
+        def visit_DropForeignKey(name) # rubocop:disable Style/MethodName
+          fk_name =
+            if name =~ /^__(.+)/
+              Regexp.last_match(1)
+            else
+              "_#{name}"
+            end
+
+          "DROP FOREIGN KEY #{fk_name}"
+        end
+      end
+
+      extend Forwardable
+
+      unless method_defined?(:change_column_for_alter)
+        include ForAlterStatements
+      end
+
+      ADAPTER_NAME = 'Percona'.freeze
+
+      def_delegators :mysql_adapter, :each_hash, :set_field_encoding
+
+      def initialize(connection, _logger, connection_options, _config)
+        @mysql_adapter = connection_options[:mysql_adapter]
+        super
+        @prepared_statements = false
+      end
+
+      def write_query?(sql) # :nodoc:
+        !ActiveRecord::ConnectionAdapters::AbstractAdapter.build_read_query_regexp(
+          :desc, :describe, :set, :show, :use
+        ).match?(sql)
+      end
+
+      def exec_delete(sql, name, binds)
+        execute(to_sql(sql, binds), name)
+        mysql_adapter.raw_connection.affected_rows
+      end
+      alias exec_update exec_delete
+
+      def exec_insert(sql, name, binds, pk = nil, sequence_name = nil, returning: nil) # rubocop:disable Lint/UnusedMethodArgument, Metrics/LineLength, Metrics/ParameterLists
+        execute(to_sql(sql, binds), name)
+      end
+
+      def internal_exec_query(sql, name = 'SQL', _binds = [], **_kwargs) # :nodoc:
+        result = execute(sql, name)
+        fields = result.fields if defined?(result.fields)
+        ActiveRecord::Result.new(fields, result.to_a)
+      end
+      alias exec_query internal_exec_query
+
+      # Executes a SELECT query and returns an array of rows. Each row is an
+      # array of field values.
+
+      def select_rows(arel, name = nil, binds = [])
+        select_all(arel, name, binds).rows
+      end
+
+      # Executes a SELECT query and returns an array of record hashes with the
+      # column names as keys and column values as values.
+      def select(sql, name = nil, binds = [], **kwargs)
+        exec_query(sql, name, binds, **kwargs)
+      end
+
+      # Returns true, as this adapter supports migrations
+      def supports_migrations?
+        true
+      end
+
+      # rubocop:disable Metrics/ParameterLists
+      def new_column(field, default, type_metadata, null, table_name, default_function, collation, comment)
+        Column.new(field, default, type_metadata, null, table_name, default_function, collation, comment)
+      end
+      # rubocop:enable Metrics/ParameterLists
+
+      # Adds a new index to the table
+      #
+      # @param table_name [String, Symbol]
+      # @param column_name [String, Symbol]
+      # @param options [Hash] optional
+      def add_index(table_name, column_name, options = {})
+        if ActiveRecord::VERSION::STRING >= '6.1'
+          index_definition, = add_index_options(table_name, column_name, **options)
+          execute <<-SQL.squish
+            ALTER TABLE #{quote_table_name(index_definition.table)}
+              ADD #{schema_creation.accept(index_definition)}
+          SQL
+        else
+          index_name, index_type, index_columns, index_options = add_index_options(table_name, column_name, **options)
+          execute <<-SQL.squish
+            ALTER TABLE #{quote_table_name(table_name)}
+              ADD #{index_type} INDEX
+              #{quote_column_name(index_name)} (#{index_columns})#{index_options}
+          SQL
+        end
+      end
+
+      # Remove the given index from the table.
+      #
+      # @param table_name [String, Symbol]
+      # @param options [Hash] optional
+      def remove_index(table_name, column_name = nil, **options)
+        if ActiveRecord::VERSION::STRING >= '6.1'
+          return if options[:if_exists] && !index_exists?(table_name, column_name, **options)
+          index_name = index_name_for_remove(table_name, column_name, options)
+        else
+          index_name = index_name_for_remove(table_name, options)
+        end
+
+        execute "ALTER TABLE #{quote_table_name(table_name)} DROP INDEX #{quote_column_name(index_name)}"
+      end
+
+      def schema_creation
+        SchemaCreation.new(self)
+      end
+
+      def change_table(table_name, _options = {})
+        recorder = ActiveRecord::Migration::CommandRecorder.new(self)
+        yield update_table_definition(table_name, recorder)
+        bulk_change_table(table_name, recorder.commands)
+      end
+
+      # Returns the MySQL error number from the exception. The
+      # AbstractMysqlAdapter requires it to be implemented
+      def error_number(_exception); end
+
+      def full_version
+        if ActiveRecord::VERSION::MAJOR < 6
+          get_full_version
+        else
+          schema_cache.database_version.full_version_string
+        end
+      end
+
+      # This is a method defined in Rails 6.0, and we have no control over the
+      # naming of this method.
+      def get_full_version # rubocop:disable Style/AccessorMethodName
+        mysql_adapter.raw_connection.server_info[:version]
+      end
+
+      def last_inserted_id(result)
+        mysql_adapter.send(:last_inserted_id, result)
+      end
+
+      private
+
+      attr_reader :mysql_adapter
+
+      if ActiveRecord.version >= Gem::Version.create('7.1.0')
+        def raw_execute(sql, name, async: false, allow_retry: false, materialize_transactions: true)
+          log(sql, name, async: async) do
+            with_raw_connection(allow_retry: allow_retry, materialize_transactions: materialize_transactions) do |conn|
+              sync_timezone_changes(conn)
+              result = conn.query(sql)
+              verified!
+              handle_warnings(sql)
+              result
+            end
+          end
+        end
+      end
+
+      def reconnect; end
+    end
+  end
+end

data/lib/active_record/connection_adapters/rails_7_2_departure_adapter.rb

@@ -0,0 +1,218 @@
+require 'active_record/connection_adapters/abstract_mysql_adapter'
+require 'active_record/connection_adapters/statement_pool'
+require 'active_record/connection_adapters/mysql2_adapter'
+require 'active_record/connection_adapters/patch_connection_handling'
+require 'active_support/core_ext/string/filters'
+require 'departure'
+require 'forwardable'
+
+module ActiveRecord
+  module ConnectionAdapters
+    class Rails72DepartureAdapter < AbstractMysqlAdapter
+      TYPE_MAP = Type::TypeMap.new.tap { |m| initialize_type_map(m) } if defined?(initialize_type_map)
+
+      class Column < ActiveRecord::ConnectionAdapters::MySQL::Column
+        def adapter
+          Rails72DepartureAdapter
+        end
+      end
+
+      class SchemaCreation < ActiveRecord::ConnectionAdapters::MySQL::SchemaCreation
+        def visit_DropForeignKey(name) # rubocop:disable Style/MethodName
+          fk_name =
+            if name =~ /^__(.+)/
+              Regexp.last_match(1)
+            else
+              "_#{name}"
+            end
+
+          "DROP FOREIGN KEY #{fk_name}"
+        end
+      end
+
+      extend Forwardable
+
+      include ForAlterStatements unless method_defined?(:change_column_for_alter)
+
+      ADAPTER_NAME = 'Percona'.freeze
+
+      def self.new_client(config)
+        connection_details = Departure::ConnectionDetails.new(config)
+        verbose = ActiveRecord::Migration.verbose
+        sanitizers = [
+          Departure::LogSanitizers::PasswordSanitizer.new(connection_details)
+        ]
+        percona_logger = Departure::LoggerFactory.build(sanitizers: sanitizers, verbose: verbose)
+        cli_generator = Departure::CliGenerator.new(connection_details)
+
+        mysql_adapter = ActiveRecord::ConnectionAdapters::Mysql2Adapter.new(config.merge(adapter: 'mysql2'))
+
+        Departure::Runner.new(
+          percona_logger,
+          cli_generator,
+          mysql_adapter
+        )
+      end
+
+      def initialize(config)
+        super
+
+        @config[:flags] ||= 0
+
+        if @config[:flags].is_a? Array
+          @config[:flags].push 'FOUND_ROWS'
+        else
+          @config[:flags] |= ::Mysql2::Client::FOUND_ROWS
+        end
+
+        @prepared_statements = false
+      end
+
+      def write_query?(sql) # :nodoc:
+        !ActiveRecord::ConnectionAdapters::AbstractAdapter.build_read_query_regexp(
+          :desc, :describe, :set, :show, :use
+        ).match?(sql)
+      end
+
+      def exec_delete(sql, name, binds)
+        execute(to_sql(sql, binds), name)
+
+        @raw_connection.affected_rows
+      end
+      alias exec_update exec_delete
+
+      def exec_insert(sql, name, binds, pky = nil, sequence_name = nil, returning: nil) # rubocop:disable Lint/UnusedMethodArgument, Metrics/Metrics/ParameterLists
+        execute(to_sql(sql, binds), name)
+      end
+
+      def internal_exec_query(sql, name = 'SQL', _binds = [], **_kwargs) # :nodoc:
+        result = execute(sql, name)
+        fields = result.fields if defined?(result.fields)
+        ActiveRecord::Result.new(fields || [], result.to_a)
+      end
+      alias exec_query internal_exec_query
+
+      # Executes a SELECT query and returns an array of rows. Each row is an
+      # array of field values.
+
+      def select_rows(arel, name = nil, binds = [])
+        select_all(arel, name, binds).rows
+      end
+
+      # Executes a SELECT query and returns an array of record hashes with the
+      # column names as keys and column values as values.
+      def select(sql, name = nil, binds = [], **kwargs)
+        exec_query(sql, name, binds, **kwargs)
+      end
+
+      # Returns true, as this adapter supports migrations
+      def supports_migrations?
+        true
+      end
+
+      # rubocop:disable Metrics/ParameterLists
+      def new_column(field, default, type_metadata, null, table_name, default_function, collation, comment)
+        Column.new(field, default, type_metadata, null, table_name, default_function, collation, comment)
+      end
+      # rubocop:enable Metrics/ParameterLists
+
+      # Adds a new index to the table
+      #
+      # @param table_name [String, Symbol]
+      # @param column_name [String, Symbol]
+      # @param options [Hash] optional
+      def add_index(table_name, column_name, options = {})
+        index_definition, = add_index_options(table_name, column_name, **options)
+        execute <<-SQL.squish
+          ALTER TABLE #{quote_table_name(index_definition.table)}
+            ADD #{schema_creation.accept(index_definition)}
+        SQL
+      end
+
+      # Remove the given index from the table.
+      #
+      # @param table_name [String, Symbol]
+      # @param options [Hash] optional
+      def remove_index(table_name, column_name = nil, **options)
+        return if options[:if_exists] && !index_exists?(table_name, column_name, **options)
+
+        index_name = index_name_for_remove(table_name, column_name, options)
+
+        execute "ALTER TABLE #{quote_table_name(table_name)} DROP INDEX #{quote_column_name(index_name)}"
+      end
+
+      def schema_creation
+        SchemaCreation.new(self)
+      end
+
+      def change_table(table_name, _options = {})
+        recorder = ActiveRecord::Migration::CommandRecorder.new(self)
+        yield update_table_definition(table_name, recorder)
+        bulk_change_table(table_name, recorder.commands)
+      end
+
+      def full_version
+        get_full_version
+      end
+
+      def get_full_version # rubocop:disable Style/AccessorMethodName
+        return @get_full_version if defined? @get_full_version
+
+        with_raw_connection do |conn|
+          @get_full_version = conn.database_adapter.get_database_version.full_version_string
+        end
+      end
+
+      def last_inserted_id(result)
+        @raw_connection.database_adapter.send(:last_inserted_id, result)
+      end
+
+      private
+
+      attr_reader :mysql_adapter
+
+      def each_hash(result, &block) # :nodoc:
+        @raw_connection.database_adapter.each_hash(result, &block)
+      end
+
+      # Must return the MySQL error number from the exception, if the exception has an
+      # error number.
+      def error_number(exception)
+        @raw_connection.database_adapter.error_number(exception)
+      end
+
+      def raw_execute(sql, name, async: false, allow_retry: false, materialize_transactions: true)
+        log(sql, name, async: async) do |notification_payload|
+          with_raw_connection(allow_retry: allow_retry, materialize_transactions: materialize_transactions) do |conn|
+            sync_timezone_changes(conn)
+            result = conn.query(sql)
+            # conn.abandon_results!
+            verified! if allow_retry
+            handle_warnings(sql)
+            if result.is_a? Process::Status
+              notification_payload[:exit_code] = result.exitstatus
+              notification_payload[:exit_pid] = result.pid
+            elsif result.respond_to?(:size)
+              notification_payload[:row_count] = result.size
+            end
+            result
+          end
+        end
+      end
+
+      def connect
+        @raw_connection = self.class.new_client(@config)
+      rescue ConnectionNotEstablished => e
+        raise e.set_pool(@pool)
+      end
+
+      def reconnect
+        @lock.synchronize do
+          @raw_connection&.close
+          @raw_connection = nil
+          connect
+        end
+      end
+    end
+  end
+end

data/lib/departure/alter_argument.rb

@@ -0,0 +1,49 @@
+module Departure
+  class InvalidAlterStatement < StandardError; end
+
+  # Represents the '--alter' argument of Percona's pt-online-schema-change
+  # See https://www.percona.com/doc/percona-toolkit/2.0/pt-online-schema-change.html
+  class AlterArgument
+    ALTER_TABLE_REGEX = /\AALTER TABLE [^\s]*[\n]* /
+
+    attr_reader :table_name
+
+    # Constructor
+    #
+    # @param statement [String]
+    # @raise [InvalidAlterStatement] if the statement is not an ALTER TABLE
+    def initialize(statement)
+      @statement = statement
+
+      match = statement.match(ALTER_TABLE_REGEX)
+      raise InvalidAlterStatement unless match
+      # Separates the ALTER TABLE from the table_name
+      #
+      # Removes the grave marks, if they are there, so we can get the table_name
+      @table_name = String(match)
+                      .split(' ')[2]
+                      .delete('`')
+    end
+
+    # Returns the '--alter' pt-online-schema-change argument as a string. See
+    # https://www.percona.com/doc/percona-toolkit/2.0/pt-online-schema-change.html
+    def to_s
+      "--alter \"#{parsed_statement}\""
+    end
+
+    private
+
+    attr_reader :statement
+
+    # Removes the 'ALTER TABLE' portion of the SQL statement
+    #
+    # @return [String]
+    def parsed_statement
+      @parsed_statement ||= statement
+        .gsub(ALTER_TABLE_REGEX, '')
+        .gsub('`', '\\\`')
+        .gsub(/\\n/, '')
+        .gsub('"', '\\\"')
+    end
+  end
+end

data/lib/departure/cli_generator.rb

@@ -0,0 +1,84 @@
+require 'departure/dsn'
+require 'departure/option'
+require 'departure/alter_argument'
+require 'departure/connection_details'
+require 'departure/user_options'
+
+module Departure
+  # Generates the equivalent Percona's pt-online-schema-change command to the
+  # given SQL statement
+  #
+  # --no-check-alter is used to allow running CHANGE COLUMN statements. For more details, check:
+  # www.percona.com/doc/percona-toolkit/2.2/pt-online-schema-change.html#cmdoption-pt-online-schema-change--[no]check-alter # rubocop:disable Metrics/LineLength
+  #
+  class CliGenerator
+    COMMAND_NAME = 'pt-online-schema-change'.freeze
+    DEFAULT_OPTIONS = Set.new(
+      [
+        Option.new('execute'),
+        Option.new('statistics'),
+        Option.new('alter-foreign-keys-method', 'auto'),
+        Option.new('no-check-alter')
+      ]
+    ).freeze
+
+    # TODO: Better doc.
+    #
+    # Constructor. Specify any arguments to pass to pt-online-schema-change
+    # passing the PERCONA_ARGS env var when executing the migration
+    #
+    # @param connection_data [Hash]
+    def initialize(connection_details)
+      @connection_details = connection_details
+    end
+
+    # Generates the percona command. Fills all the connection credentials from
+    # the current AR connection, but that can be amended via ENV-vars:
+    # PERCONA_DB_HOST, PERCONA_DB_USER, PERCONA_DB_PASSWORD, PERCONA_DB_NAME
+    # Table name can't not be amended, it populates automatically from the
+    # migration data
+    #
+    # @param table_name [String]
+    # @param statement [String] MySQL statement
+    # @return [String]
+    def generate(table_name, statement)
+      alter_argument = AlterArgument.new(statement)
+      dsn = DSN.new(connection_details.database, table_name)
+
+      "#{command} #{all_options} #{dsn} #{alter_argument}"
+    end
+
+    # Generates the percona command for a raw MySQL statement. Fills all the
+    # connection credentials from the current AR connection, but that can
+    # amended via ENV-vars: PERCONA_DB_HOST, PERCONA_DB_USER,
+    # PERCONA_DB_PASSWORD, PERCONA_DB_NAME Table name can't not be amended, it
+    # populates automatically from the migration data
+    #
+    # @param statement [String] MySQL statement
+    # @return [String]
+    def parse_statement(statement)
+      alter_argument = AlterArgument.new(statement)
+      dsn = DSN.new(connection_details.database, alter_argument.table_name)
+
+      "#{command} #{all_options} #{dsn} #{alter_argument}"
+    end
+
+    private
+
+    attr_reader :connection_details
+
+    def command
+      "#{COMMAND_NAME} #{connection_details}"
+    end
+
+    # Returns all the arguments to execute pt-online-schema-change with
+    #
+    # @return [String]
+    def all_options
+      env_variable_options = UserOptions.new
+      global_configuration_options = UserOptions.new(Departure.configuration.global_percona_args)
+      options = env_variable_options.merge(global_configuration_options).merge(DEFAULT_OPTIONS)
+      options.to_a.join(' ')
+    end
+  end
+end

data/lib/departure/command.rb

@@ -0,0 +1,105 @@
+module Departure
+  # Executes the given command returning it's status and errors
+  class Command
+    COMMAND_NOT_FOUND = 127
+
+    # Constructor
+    #
+    # @param command_line [String]
+    # @param error_log_path [String]
+    # @param logger [#write_no_newline]
+    def initialize(command_line, error_log_path, logger, redirect_stderr)
+      @command_line = command_line
+      @error_log_path = error_log_path
+      @logger = logger
+      @redirect_stderr = redirect_stderr
+    end
+
+    # Executes the command returning its status. It also prints its stdout to
+    # the logger and its stderr to the file specified in error_log_path.
+    #
+    # @raise [NoStatusError] if the spawned process' status can't be retrieved
+    # @raise [SignalError] if the spawned process received a signal
+    # @raise [CommandNotFoundError] if pt-online-schema-change can't be found
+    #
+    # @return [Process::Status]
+    def run
+      log_started
+
+      run_in_process
+
+      log_finished
+
+      validate_status!
+      status
+    end
+
+    private
+
+    attr_reader :command_line, :error_log_path, :logger, :status, :redirect_stderr
+
+    # Runs the command in a separate process, capturing its stdout and
+    # execution status
+    def run_in_process
+      Open3.popen3(full_command) do |_stdin, stdout, _stderr, waith_thr|
+        begin
+          loop do
+            IO.select([stdout])
+            data = stdout.read_nonblock(8192)
+            logger.write_no_newline(data)
+          end
+        rescue EOFError # rubocop:disable Lint/HandleExceptions
+          # noop
+        ensure
+          @status = waith_thr.value
+        end
+      end
+    end
+
+    # Builds the actual command including stderr redirection to the specified
+    # log file or stdout
+    #
+    # @return [String]
+    def full_command
+      if redirect_stderr
+        "#{command_line} 2> #{error_log_path}"
+      else
+        "#{command_line} 2>&1"
+      end
+    end
+
+    # Validates the status of the execution
+    #
+    # @raise [NoStatusError] if the spawned process' status can't be retrieved
+    # @raise [SignalError] if the spawned process received a signal
+    # @raise [CommandNotFoundError] if pt-online-schema-change can't be found
+    def validate_status!
+      raise SignalError.new(status) if status.signaled? # rubocop:disable Style/RaiseArgs
+      raise CommandNotFoundError if status.exitstatus == COMMAND_NOT_FOUND
+      raise Error, error_message unless status.success?
+    end
+
+    # Returns the error message that appeared in the process' stderr
+    #
+    # @return [String]
+    def error_message
+      if redirect_stderr
+        File.read(error_log_path)
+      else
+        ''
+      end
+    end
+
+    # Logs when the execution started
+    def log_started
+      logger.write("\n")
+      logger.say("Running #{command_line}\n\n", true)
+    end
+
+    # Prints a line break to keep the logs separate from the execution time
+    # print by the migration
+    def log_finished
+      logger.write("\n")
+    end
+  end
+end

data/lib/departure/configuration.rb

@@ -0,0 +1,21 @@
+module Departure
+  class Configuration
+    attr_accessor :tmp_path, :global_percona_args, :enabled_by_default, :redirect_stderr
+
+    def initialize
+      @tmp_path = '.'.freeze
+      @error_log_filename = 'departure_error.log'.freeze
+      @global_percona_args = nil
+      @enabled_by_default = true
+      @redirect_stderr = true
+    end
+
+    def error_log_path
+      File.join(tmp_path, error_log_filename)
+    end
+
+    private
+
+    attr_reader :error_log_filename
+  end
+end

data/lib/departure/connection_base.rb

@@ -0,0 +1,11 @@
+module Departure
+  class ConnectionBase < ActiveRecord::Base
+    def self.establish_connection(config = nil)
+      super.tap do
+        ActiveRecord::Base.connection_specification_name = connection_specification_name
+      end
+    end
+  end
+
+  class OriginalAdapterConnection < ConnectionBase; end
+end