departure 1.0.0

data/lib/active_record/connection_adapters/percona_adapter.rb

@@ -0,0 +1,121 @@
+require 'active_record/connection_adapters/abstract_mysql_adapter'
+require 'active_record/connection_adapters/statement_pool'
+require 'active_record/connection_adapters/mysql2_adapter'
+require 'departure'
+require 'forwardable'
+
+module ActiveRecord
+  class Base
+    # Establishes a connection to the database that's used by all Active
+    # Record objects.
+    def self.percona_connection(config)
+      mysql2_connection = mysql2_connection(config)
+
+      verbose = ActiveRecord::Migration.verbose
+      percona_logger = Departure::LoggerFactory.build(verbose: verbose)
+      cli_generator = Departure::CliGenerator.new(config)
+
+      runner = Departure::Runner.new(
+        percona_logger,
+        cli_generator,
+        mysql2_connection
+      )
+
+      connection_options = { mysql_adapter: mysql2_connection }
+
+      ConnectionAdapters::DepartureAdapter.new(
+        runner,
+        logger,
+        connection_options,
+        config
+      )
+    end
+  end
+
+  module ConnectionAdapters
+    class DepartureAdapter < AbstractMysqlAdapter
+
+      class Column < AbstractMysqlAdapter::Column
+        def adapter
+          DepartureAdapter
+        end
+      end
+
+      extend Forwardable
+
+      ADAPTER_NAME = 'Percona'.freeze
+
+      def_delegators :mysql_adapter, :last_inserted_id, :each_hash
+
+      def initialize(connection, _logger, connection_options, _config)
+        super
+        @visitor = BindSubstitution.new(self)
+        @mysql_adapter = connection_options[:mysql_adapter]
+      end
+
+      def exec_delete(sql, name, binds)
+        execute(to_sql(sql, binds), name)
+        @connection.affected_rows
+      end
+      alias :exec_update :exec_delete
+
+      def exec_insert(sql, name, binds)
+        execute(to_sql(sql, binds), name)
+      end
+
+      def exec_query(sql, name = 'SQL', _binds = [])
+        result = execute(sql, name)
+        ActiveRecord::Result.new(result.fields, result.to_a)
+      end
+
+      # Executes a SELECT query and returns an array of rows. Each row is an
+      # array of field values.
+      def select_rows(sql, name = nil)
+        execute(sql, name).to_a
+      end
+
+      # Executes a SELECT query and returns an array of record hashes with the
+      # column names as keys and column values as values.
+      def select(sql, name = nil, binds = [])
+        exec_query(sql, name, binds).to_a
+      end
+
+      # Returns true, as this adapter supports migrations
+      def supports_migrations?
+        true
+      end
+
+      def new_column(field, default, type, null, collation)
+        Column.new(field, default, type, null, collation)
+      end
+
+      # Adds a new index to the table
+      #
+      # @param table_name [String, Symbol]
+      # @param column_name [String, Symbol]
+      # @param options [Hash] optional
+      def add_index(table_name, column_name, options = {})
+        index_name, index_type, index_columns, index_options = add_index_options(table_name, column_name, options)
+        execute "ALTER TABLE #{quote_table_name(table_name)} ADD #{index_type} INDEX #{quote_column_name(index_name)} (#{index_columns})#{index_options}"
+      end
+
+      # Remove the given index from the table.
+      #
+      # @param table_name [String, Symbol]
+      # @param options [Hash] optional
+      def remove_index(table_name, options = {})
+        index_name = index_name_for_remove(table_name, options)
+        execute "ALTER TABLE #{quote_table_name(table_name)} DROP INDEX #{quote_column_name(index_name)}"
+      end
+
+      # Returns the MySQL error number from the exception. The
+      # AbstractMysqlAdapter requires it to be implemented
+      def error_number(_exception)
+      end
+
+      private
+
+      attr_reader :mysql_adapter
+    end
+  end
+end

data/lib/departure.rb

@@ -0,0 +1,65 @@
+require 'active_record'
+require 'active_support/all'
+
+require 'departure/version'
+require 'departure/runner'
+require 'departure/cli_generator'
+require 'departure/logger'
+require 'departure/null_logger'
+require 'departure/logger_factory'
+require 'departure/configuration'
+require 'departure/errors'
+
+require 'departure/railtie' if defined?(Rails)
+
+# We need the OS not to buffer the IO to see pt-osc's output while migrating
+$stdout.sync = true
+
+module Departure
+  class << self
+    attr_accessor :configuration
+  end
+
+  def self.configure
+    self.configuration ||= Configuration.new
+    yield(configuration)
+  end
+
+  # Hooks Percona Migrator into Rails migrations by replacing the configured
+  # database adapter
+  def self.load
+    ActiveRecord::Migration.class_eval do
+      alias_method :original_migrate, :migrate
+
+      # Replaces the current connection adapter with the PerconaAdapter and
+      # patches LHM, then it continues with the regular migration process.
+      #
+      # @param direction [Symbol] :up or :down
+      def migrate(direction)
+        reconnect_with_percona
+        include_foreigner if defined?(Foreigner)
+
+        ::Lhm.migration = self
+        original_migrate(direction)
+      end
+
+      # Includes the Foreigner's Mysql2Adapter implemention in
+      # DepartureAdapter to support foreign keys
+      def include_foreigner
+        Foreigner::Adapter.safe_include(
+          :DepartureAdapter,
+          Foreigner::ConnectionAdapters::Mysql2Adapter
+        )
+      end
+
+      # Make all connections in the connection pool to use PerconaAdapter
+      # instead of the current adapter.
+      def reconnect_with_percona
+        connection_config = ActiveRecord::Base.connection_config
+        ActiveRecord::Base.establish_connection(
+          connection_config.merge(adapter: 'percona')
+        )
+      end
+    end
+  end
+end

data/lib/departure/alter_argument.rb

@@ -0,0 +1,43 @@
+module Departure
+  class InvalidAlterStatement < StandardError; end
+
+  # Represents the '--alter' argument of Percona's pt-online-schema-change
+  # See https://www.percona.com/doc/percona-toolkit/2.0/pt-online-schema-change.html
+  class AlterArgument
+    ALTER_TABLE_REGEX = /\AALTER TABLE `(\w+)` /
+
+    attr_reader :table_name
+
+    # Constructor
+    #
+    # @param statement [String]
+    # @raise [InvalidAlterStatement] if the statement is not an ALTER TABLE
+    def initialize(statement)
+      @statement = statement
+
+      match = statement.match(ALTER_TABLE_REGEX)
+      raise InvalidAlterStatement unless match
+
+      @table_name = match.captures[0]
+    end
+
+    # Returns the '--alter' pt-online-schema-change argument as a string. See
+    # https://www.percona.com/doc/percona-toolkit/2.0/pt-online-schema-change.html
+    def to_s
+      "--alter \"#{parsed_statement}\""
+    end
+
+    private
+
+    attr_reader :statement
+
+    # Removes the 'ALTER TABLE' portion of the SQL statement
+    #
+    # @return [String]
+    def parsed_statement
+      @parsed_statement ||= statement
+        .gsub(ALTER_TABLE_REGEX, '')
+        .gsub('`','\\\`')
+    end
+  end
+end

data/lib/departure/cli_generator.rb

@@ -0,0 +1,134 @@
+require 'departure/alter_argument'
+
+module Departure
+
+  # Represents the 'DSN' argument of Percona's pt-online-schema-change
+  # See https://www.percona.com/doc/percona-toolkit/2.0/pt-online-schema-change.html#dsn-options
+  class DSN
+
+    # Constructor
+    #
+    # @param database [String, Symbol]
+    # @param table_name [String, Symbol]
+    def initialize(database, table_name)
+      @database = database
+      @table_name = table_name
+    end
+
+    # Returns the pt-online-schema-change DSN string. See
+    # https://www.percona.com/doc/percona-toolkit/2.0/pt-online-schema-change.html#dsn-options
+    def to_s
+      "D=#{database},t=#{table_name}"
+    end
+
+    private
+
+    attr_reader :table_name, :database
+  end
+
+  # Generates the equivalent Percona's pt-online-schema-change command to the
+  # given SQL statement
+  class CliGenerator # Command
+    BASE_COMMAND = 'pt-online-schema-change'
+    BASE_OPTIONS = %w(
+      --execute
+      --statistics
+      --recursion-method=none
+      --alter-foreign-keys-method=auto
+    )
+
+    # Constructor
+    #
+    # @param connection_data [Hash]
+    def initialize(connection_data)
+      @connection_data = connection_data
+      init_base_command
+      add_connection_details
+    end
+
+    # Generates the percona command. Fills all the connection credentials from
+    # the current AR connection, but that can be amended via ENV-vars:
+    # PERCONA_DB_HOST, PERCONA_DB_USER, PERCONA_DB_PASSWORD, PERCONA_DB_NAME
+    # Table name can't not be amended, it populates automatically from the
+    # migration data
+    #
+    # @param table_name [String]
+    # @param statement [String] MySQL statement
+    # @return [String]
+    def generate(table_name, statement)
+      alter_argument = AlterArgument.new(statement)
+      dsn = DSN.new(database, table_name)
+
+      "#{self} #{dsn} #{alter_argument}"
+    end
+
+    # Generates the percona command for a raw MySQL statement. Fills all the
+    # connection credentials from the current AR connection, but that can
+    # amended via ENV-vars: PERCONA_DB_HOST, PERCONA_DB_USER,
+    # PERCONA_DB_PASSWORD, PERCONA_DB_NAME Table name can't not be amended, it
+    # populates automatically from the migration data
+    #
+    # @param statement [String] MySQL statement
+    # @return [String]
+    def parse_statement(statement)
+      alter_argument = AlterArgument.new(statement)
+      dsn = DSN.new(database, alter_argument.table_name)
+
+      "#{self} #{dsn} #{alter_argument}"
+    end
+
+    private
+
+    attr_reader :connection_data
+
+    # Sets up the command with its options
+    def init_base_command
+      @command = [BASE_COMMAND, BASE_OPTIONS.join(' ')]
+    end
+
+    # Adds the host, user and password, if present, to the command
+    def add_connection_details
+      @command.push("-h #{host}")
+      @command.push("-u #{user}")
+      @command.push("-p #{password}") if password.present?
+    end
+
+    # Returns the command as a string that can be executed in a shell
+    #
+    # @return [String]
+    def to_s
+      @command.join(' ')
+    end
+
+    # Returns the database host name, defaulting to localhost. If PERCONA_DB_HOST
+    # is passed its value will be used instead
+    #
+    # @return [String]
+    def host
+      ENV['PERCONA_DB_HOST'] || connection_data[:host] || 'localhost'
+    end
+
+    # Returns the database user. If PERCONA_DB_USER is passed its value will be
+    # used instead
+    #
+    # @return [String]
+    def user
+      ENV['PERCONA_DB_USER'] || connection_data[:username]
+    end
+
+    # Returns the database user's password. If PERCONA_DB_PASSWORD is passed its
+    # value will be used instead
+    #
+    # @return [String]
+    def password
+      ENV['PERCONA_DB_PASSWORD'] || connection_data[:password]
+    end
+
+    # TODO: Doesn't the abstract adapter already handle this somehow?
+    # Returns the database name. If PERCONA_DB_NAME is passed its value will be
+    # used instead
+    def database
+      ENV['PERCONA_DB_NAME'] || connection_data[:database]
+    end
+  end
+end

data/lib/departure/configuration.rb

@@ -0,0 +1,18 @@
+module Departure
+  class Configuration
+    attr_accessor :tmp_path
+
+    def initialize
+      @tmp_path = '.'.freeze
+      @error_log_filename = 'departure_error.log'.freeze
+    end
+
+    def error_log_path
+      File.join(tmp_path, error_log_filename)
+    end
+
+    private
+
+    attr_reader :error_log_filename
+  end
+end

data/lib/departure/errors.rb

@@ -0,0 +1,35 @@
+module Departure
+  class Error < StandardError; end
+
+  # Used when for whatever reason we couldn't get the spawned process'
+  # status.
+  class NoStatusError < Error
+    def message
+      'Status could not be retrieved'.freeze
+    end
+  end
+
+  # Used when the spawned process failed by receiving a signal.
+  # pt-online-schema-change returns "SIGSEGV (signal 11)" on failures.
+  class SignalError < Error
+    attr_reader :status
+
+    # Constructor
+    #
+    # @param status [Process::Status]
+    def initialize(status)
+      super
+      @status = status
+    end
+
+    def message
+      status.to_s
+    end
+  end
+
+  class CommandNotFoundError < Error
+    def message
+      'Please install pt-online-schema-change. Check: https://www.percona.com/doc/percona-toolkit for further details'
+    end
+  end
+end

data/lib/departure/logger.rb

@@ -0,0 +1,31 @@
+module Departure
+  # Copies the ActiveRecord::Migration #say and #write plus a new
+  # #write_no_newline to log the migration's status. It's not possible to reuse
+  # the from ActiveRecord::Migration because the migration's instance can't be
+  # seen from the connection adapter.
+  class Logger
+
+    # Outputs the message through the stdout, following the
+    # ActiveRecord::Migration log format
+    #
+    # @param message [String]
+    # @param subitem [Boolean] whether to show message as a nested log item
+    def say(message, subitem = false)
+      write "#{subitem ? "   ->" : "--"} #{message}"
+    end
+
+    # Outputs the text through the stdout adding a new line at the end
+    #
+    # @param text [String]
+    def write(text = '')
+      puts(text)
+    end
+
+    # Outputs the text through the stdout without adding a new line at the end
+    #
+    # @param text [String]
+    def write_no_newline(text)
+      print(text)
+    end
+  end
+end