arrival 0.1.0

data/lib/active_record/connection_adapters/for_alter.rb

@@ -0,0 +1,91 @@
+require 'active_record/connection_adapters/mysql/schema_statements'
+
+module ForAlterStatements
+  class << self
+    def included(_)
+      STDERR.puts 'Including for_alter statements'
+    end
+  end
+
+  def bulk_change_table(table_name, operations) #:nodoc:
+    sqls = operations.flat_map do |command, args|
+      table = args.shift
+      arguments = args
+
+      method = :"#{command}_for_alter"
+
+      raise "Unknown method called : #{method}(#{arguments.inspect})" unless respond_to?(method, true)
+      public_send(method, table, *arguments)
+    end.join(', ')
+
+    execute("ALTER TABLE #{quote_table_name(table_name)} #{sqls}")
+  end
+
+  def change_column_for_alter(table_name, column_name, type, options = {})
+    column = column_for(table_name, column_name)
+    type ||= column.sql_type
+
+    options = {
+      default: column.default,
+      null: column.null,
+      comment: column.comment
+    }.merge(options)
+
+    td = create_table_definition(table_name)
+    cd = td.new_column_definition(column.name, type, options)
+    schema_creation.accept(ActiveRecord::ConnectionAdapters::ChangeColumnDefinition.new(cd, column.name))
+  end
+
+  def rename_column_for_alter(table_name, column_name, new_column_name)
+    column  = column_for(table_name, column_name)
+    options = {
+      default: column.default,
+      null: column.null,
+      auto_increment: column.auto_increment?
+    }
+
+    columns_sql = "SHOW COLUMNS FROM #{quote_table_name(table_name)} LIKE #{quote(column_name)}"
+    current_type = exec_query(columns_sql, 'SCHEMA').first['Type']
+    td = create_table_definition(table_name)
+    cd = td.new_column_definition(new_column_name, current_type, options)
+    schema_creation.accept(ActiveRecord::ConnectionAdapters::ChangeColumnDefinition.new(cd, column.name))
+  end
+
+  def add_index_for_alter(table_name, column_name, options = {})
+    index_name, index_type, index_columns, _,
+      index_algorithm, index_using = add_index_options(table_name, column_name, options)
+
+    index_algorithm[0, 0] = ', ' if index_algorithm.present?
+    "ADD #{index_type} INDEX #{quote_column_name(index_name)} #{index_using} (#{index_columns})#{index_algorithm}"
+  end
+
+  def remove_index_for_alter(table_name, options = {})
+    index_name = index_name_for_remove(table_name, options)
+    "DROP INDEX #{quote_column_name(index_name)}"
+  end
+
+  def add_timestamps_for_alter(table_name, options = {})
+    [
+      add_column_for_alter(table_name, :created_at, :datetime, options),
+      add_column_for_alter(table_name, :updated_at, :datetime, options)
+    ]
+  end
+
+  def remove_timestamps_for_alter(table_name, _options = {})
+    [remove_column_for_alter(table_name, :updated_at), remove_column_for_alter(table_name, :created_at)]
+  end
+
+  def add_column_for_alter(table_name, column_name, type, options = {})
+    td = create_table_definition(table_name)
+    cd = td.new_column_definition(column_name, type, options)
+    schema_creation.accept(ActiveRecord::ConnectionAdapters::AddColumnDefinition.new(cd))
+  end
+
+  def remove_column_for_alter(_table_name, column_name, _type = nil, _options = {})
+    "DROP COLUMN #{quote_column_name(column_name)}"
+  end
+
+  def remove_columns_for_alter(table_name, *column_names)
+    column_names.map { |column_name| remove_column_for_alter(table_name, column_name) }
+  end
+end

data/lib/active_record/connection_adapters/percona_adapter.rb

@@ -0,0 +1,158 @@
+require 'active_record/connection_adapters/abstract_mysql_adapter'
+require 'active_record/connection_adapters/statement_pool'
+require 'active_record/connection_adapters/mysql2_adapter'
+require 'arrival'
+require 'forwardable'
+
+module ActiveRecord
+  module ConnectionHandling
+    # Establishes a connection to the database that's used by all Active
+    # Record objects.
+    def percona_connection(config)
+      config[:username] = 'root' if config[:username].nil?
+      mysql2_connection = mysql2_connection(config)
+
+      connection_details =Arrival::ConnectionDetails.new(config)
+      verbose = ActiveRecord::Migration.verbose
+      sanitizers = [
+        Arrival::LogSanitizers::PasswordSanitizer.new(connection_details)
+      ]
+      percona_logger = Arrival::LoggerFactory.build(sanitizers: sanitizers, verbose: verbose)
+      cli_generator = Arrival::CliGenerator.new(connection_details)
+
+      runner = Arrival::Runner.new(
+        percona_logger,
+        cli_generator,
+        mysql2_connection
+      )
+
+      connection_options = { mysql_adapter: mysql2_connection }
+
+      ConnectionAdapters::ArrivalAdapter.new(
+        runner,
+        logger,
+        connection_options,
+        config
+      )
+    end
+  end
+
+  module ConnectionAdapters
+    class ArrivalAdapter < AbstractMysqlAdapter
+      class Column < ActiveRecord::ConnectionAdapters::MySQL::Column
+        def adapter
+          ArrivalAdapter
+        end
+      end
+
+      class SchemaCreation < ActiveRecord::ConnectionAdapters::MySQL::SchemaCreation
+        def visit_DropForeignKey(name) # rubocop:disable Naming/MethodName
+          fk_name =
+            if name =~ /^__(.+)/
+              Regexp.last_match(1)
+            else
+              "_#{name}"
+            end
+
+          "DROP FOREIGN KEY #{fk_name}"
+        end
+      end
+
+      extend Forwardable
+
+      unless method_defined?(:change_column_for_alter)
+        include ForAlterStatements
+      end
+
+      ADAPTER_NAME = 'Percona'.freeze
+
+      def_delegators :mysql_adapter, :last_inserted_id, :each_hash, :set_field_encoding
+
+      def initialize(connection, _logger, connection_options, _config)
+        @mysql_adapter = connection_options[:mysql_adapter]
+        super
+        @prepared_statements = false
+      end
+
+      def exec_delete(sql, name, binds)
+        execute(to_sql(sql, binds), name)
+        @connection.affected_rows
+      end
+      alias exec_update exec_delete
+
+      def exec_insert(sql, name, binds, pk = nil, sequence_name = nil) # rubocop:disable Lint/UnusedMethodArgument, Metrics/LineLength
+        execute(to_sql(sql, binds), name)
+      end
+
+      def exec_query(sql, name = 'SQL', _binds = [])
+        result = execute(sql, name)
+        ActiveRecord::Result.new(result.fields, result.to_a)
+      end
+
+      # Executes a SELECT query and returns an array of rows. Each row is an
+      # array of field values.
+
+      def select_rows(arel, name = nil, binds = [])
+        select_all(arel, name, binds).rows
+      end
+
+      # Executes a SELECT query and returns an array of record hashes with the
+      # column names as keys and column values as values.
+      def select(sql, name = nil, binds = [])
+        exec_query(sql, name, binds)
+      end
+
+      # Returns true, as this adapter supports migrations
+      def supports_migrations?
+        true
+      end
+
+      # rubocop:disable Metrics/ParameterLists
+      def new_column(field, default, type_metadata, null, table_name, default_function, collation, comment)
+        Column.new(field, default, type_metadata, null, table_name, default_function, collation, comment)
+      end
+      # rubocop:enable Metrics/ParameterLists
+
+      # Adds a new index to the table
+      #
+      # @param table_name [String, Symbol]
+      # @param column_name [String, Symbol]
+      # @param options [Hash] optional
+      def add_index(table_name, column_name, options = {})
+        index_name, index_type, index_columns, index_options = add_index_options(table_name, column_name, options)
+        execute "ALTER TABLE #{quote_table_name(table_name)} ADD #{index_type} INDEX #{quote_column_name(index_name)} (#{index_columns})#{index_options}" # rubocop:disable Metrics/LineLength
+      end
+
+      # Remove the given index from the table.
+      #
+      # @param table_name [String, Symbol]
+      # @param options [Hash] optional
+      def remove_index(table_name, options = {})
+        index_name = index_name_for_remove(table_name, options)
+        execute "ALTER TABLE #{quote_table_name(table_name)} DROP INDEX #{quote_column_name(index_name)}"
+      end
+
+      def schema_creation
+        SchemaCreation.new(self)
+      end
+
+      def change_table(table_name, _options = {})
+        recorder = ActiveRecord::Migration::CommandRecorder.new(self)
+        yield update_table_definition(table_name, recorder)
+        bulk_change_table(table_name, recorder.commands)
+      end
+
+      # Returns the MySQL error number from the exception. The
+      # AbstractMysqlAdapter requires it to be implemented
+      def error_number(_exception); end
+
+      def full_version
+        mysql_adapter.raw_connection.server_info[:version]
+      end
+
+      private
+
+      attr_reader :mysql_adapter
+    end
+  end
+end

data/lib/arrival.rb

@@ -0,0 +1,68 @@
+require 'active_record'
+require 'active_support/all'
+
+require 'active_record/connection_adapters/for_alter'
+
+require 'arrival/version'
+require 'arrival/log_sanitizers/password_sanitizer'
+require 'arrival/runner'
+require 'arrival/cli_generator'
+require 'arrival/logger'
+require 'arrival/null_logger'
+require 'arrival/logger_factory'
+require 'arrival/configuration'
+require 'arrival/errors'
+require 'arrival/command'
+
+require 'arrival/railtie' if defined?(Rails)
+
+# We need the OS not to buffer the IO to see pt-osc's output while migrating
+$stdout.sync = true
+
+module Arrival
+  class << self
+    attr_accessor :configuration
+  end
+
+  def self.configure
+    self.configuration ||= Configuration.new
+    yield(configuration)
+  end
+
+  # Hooks Percona Migrator into Rails migrations by replacing the configured
+  # database adapter
+  def self.load
+    ActiveRecord::Migration.class_eval do
+      alias_method :original_migrate, :migrate
+
+      # Replaces the current connection adapter with the PerconaAdapter and
+      # patches LHM, then it continues with the regular migration process.
+      #
+      # @param direction [Symbol] :up or :down
+      def migrate(direction)
+        reconnect_with_percona
+        include_foreigner if defined?(Foreigner)
+
+        ::Lhm.migration = self
+        original_migrate(direction)
+      end
+
+      # Includes the Foreigner's Mysql2Adapter implemention in
+      # ArrivalAdapter to support foreign keys
+      def include_foreigner
+        Foreigner::Adapter.safe_include(
+          :ArrivalAdapter,
+          Foreigner::ConnectionAdapters::Mysql2Adapter
+        )
+      end
+
+      # Make all connections in the connection pool to use PerconaAdapter
+      # instead of the current adapter.
+      def reconnect_with_percona
+        connection_config = ActiveRecord::Base
+          .connection_config.merge(adapter: 'percona')
+        ActiveRecord::Base.establish_connection(connection_config)
+      end
+    end
+  end
+end

data/lib/arrival/alter_argument.rb

@@ -0,0 +1,43 @@
+module Arrival
+  class InvalidAlterStatement < StandardError; end
+
+  # Represents the '--alter' argument of Percona's pt-online-schema-change
+  # See https://www.percona.com/doc/percona-toolkit/2.0/pt-online-schema-change.html
+  class AlterArgument
+    ALTER_TABLE_REGEX = /\AALTER TABLE `(\w+)` /
+
+    attr_reader :table_name
+
+    # Constructor
+    #
+    # @param statement [String]
+    # @raise [InvalidAlterStatement] if the statement is not an ALTER TABLE
+    def initialize(statement)
+      @statement = statement
+
+      match = statement.match(ALTER_TABLE_REGEX)
+      raise InvalidAlterStatement unless match
+
+      @table_name = match.captures[0]
+    end
+
+    # Returns the '--alter' pt-online-schema-change argument as a string. See
+    # https://www.percona.com/doc/percona-toolkit/2.0/pt-online-schema-change.html
+    def to_s
+      "--alter \"#{parsed_statement}\""
+    end
+
+    private
+
+    attr_reader :statement
+
+    # Removes the 'ALTER TABLE' portion of the SQL statement
+    #
+    # @return [String]
+    def parsed_statement
+      @parsed_statement ||= statement
+        .gsub(ALTER_TABLE_REGEX, '')
+        .gsub('`', '\\\`')
+    end
+  end
+end

data/lib/arrival/cli_generator.rb

@@ -0,0 +1,85 @@
+require 'arrival/dsn'
+require 'arrival/option'
+require 'arrival/alter_argument'
+require 'arrival/connection_details'
+require 'arrival/user_options'
+
+module Arrival
+  # Generates the equivalent Percona's pt-online-schema-change command to the
+  # given SQL statement
+  #
+  # --no-check-alter is used to allow running CHANGE COLUMN statements. For more details, check:
+  # www.percona.com/doc/percona-toolkit/2.2/pt-online-schema-change.html#cmdoption-pt-online-schema-change--[no]check-alter # rubocop:disable Metrics/LineLength
+  #
+  class CliGenerator
+    # COMMAND_NAME = 'pt-online-schema-change'.freeze
+    COMMAND_NAME = 'gh-ost'.freeze
+    DEFAULT_OPTIONS = Set.new(
+      [
+        Option.new('execute'),
+        Option.new('statistics'),
+        Option.new('alter-foreign-keys-method', 'auto'),
+        Option.new('no-check-alter')
+      ]
+    ).freeze
+
+    # TODO: Better doc.
+    #
+    # Constructor. Specify any arguments to pass to pt-online-schema-change
+    # passing the PERCONA_ARGS env var when executing the migration
+    #
+    # @param connection_data [Hash]
+    def initialize(connection_details)
+      @connection_details = connection_details
+    end
+
+    # Generates the percona command. Fills all the connection credentials from
+    # the current AR connection, but that can be amended via ENV-vars:
+    # PERCONA_DB_HOST, PERCONA_DB_USER, PERCONA_DB_PASSWORD, PERCONA_DB_NAME
+    # Table name can't not be amended, it populates automatically from the
+    # migration data
+    #
+    # @param table_name [String]
+    # @param statement [String] MySQL statement
+    # @return [String]
+    def generate(table_name, statement)
+      alter_argument = AlterArgument.new(statement)
+      dsn = DSN.new(connection_details.database, table_name)
+
+      "#{command} #{all_options} #{dsn} #{alter_argument}"
+    end
+
+    # Generates the percona command for a raw MySQL statement. Fills all the
+    # connection credentials from the current AR connection, but that can
+    # amended via ENV-vars: PERCONA_DB_HOST, PERCONA_DB_USER,
+    # PERCONA_DB_PASSWORD, PERCONA_DB_NAME Table name can't not be amended, it
+    # populates automatically from the migration data
+    #
+    # @param statement [String] MySQL statement
+    # @return [String]
+    def parse_statement(statement)
+      alter_argument = AlterArgument.new(statement)
+      dsn = DSN.new(connection_details.database, alter_argument.table_name)
+
+      "#{command} #{all_options} #{dsn} #{alter_argument}"
+    end
+
+    private
+
+    attr_reader :connection_details
+
+    def command
+      "#{COMMAND_NAME} #{connection_details}"
+    end
+
+    # Returns all the arguments to execute pt-online-schema-change with
+    #
+    # @return [String]
+    def all_options
+      env_variable_options = UserOptions.new
+      global_configuration_options = UserOptions.new(Arrival.configuration.global_percona_args)
+      options = env_variable_options.merge(global_configuration_options).merge(DEFAULT_OPTIONS)
+      options.to_a.join(' ')
+    end
+  end
+end