arrival 0.1.0 → 0.1.1

data/lib/arrival/cli_generator.rb

@@ -1,4 +1,3 @@
-require 'arrival/dsn'
 require 'arrival/option'
 require 'arrival/alter_argument'
 require 'arrival/connection_details'
@@ -12,41 +11,47 @@ module Arrival
   # www.percona.com/doc/percona-toolkit/2.2/pt-online-schema-change.html#cmdoption-pt-online-schema-change--[no]check-alter # rubocop:disable Metrics/LineLength
   #
   class CliGenerator
-    # COMMAND_NAME = 'pt-online-schema-change'.freeze
     COMMAND_NAME = 'gh-ost'.freeze
     DEFAULT_OPTIONS = Set.new(
       [
         Option.new('execute'),
-        Option.new('statistics'),
-        Option.new('alter-foreign-keys-method', 'auto'),
-        Option.new('no-check-alter')
+        Option.new('max-load', 'Threads_running=150'),
+        Option.new('critical-load', 'Threads_running=4000'),
+        Option.new('chunk-size', '1000'),
+        Option.new('throttle-control-replicas', 'mysql-replica'),
+        Option.new('max-lag-millis', '3000'),
+        Option.new('user', 'replica_user'),
+        Option.new('password', 'replica_pass'),
+        Option.new('host', 'mysql-replica'),
+        Option.new('database', 'arrival_test'),
+        Option.new('dml-batch-size', '1000'),
+        Option.new('alter', '"CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci"'),
+        Option.new('assume-rbr'),
+        Option.new('cut-over', 'default'),
+        Option.new('exact-rowcount'),
+        Option.new('concurrent-rowcount'),
+        Option.new('default-retries', '1200'),
+        Option.new('cut-over-lock-timeout-seconds', '10'),
+        Option.new('panic-flag-file', '/tmp/ghost.panic.downloads.flag'),
+        Option.new('assume-master-host', 'mysql-main'),
+        # Option.new('postpone-cut-over-flag-file', '/tmp/ghost.postpone.downloads.flag'),
+        Option.new('verbose'),
       ]
     ).freeze
 
-    # TODO: Better doc.
-    #
-    # Constructor. Specify any arguments to pass to pt-online-schema-change
-    # passing the PERCONA_ARGS env var when executing the migration
-    #
-    # @param connection_data [Hash]
-    def initialize(connection_details)
-      @connection_details = connection_details
-    end
-
     # Generates the percona command. Fills all the connection credentials from
     # the current AR connection, but that can be amended via ENV-vars:
     # PERCONA_DB_HOST, PERCONA_DB_USER, PERCONA_DB_PASSWORD, PERCONA_DB_NAME
     # Table name can't not be amended, it populates automatically from the
     # migration data
     #
-    # @param table_name [String]
+    # deprecated: @param table_name [String]
     # @param statement [String] MySQL statement
     # @return [String]
-    def generate(table_name, statement)
+    def generate(_table_name=nil, statement)
       alter_argument = AlterArgument.new(statement)
-      dsn = DSN.new(connection_details.database, table_name)
 
-      "#{command} #{all_options} #{dsn} #{alter_argument}"
+      "#{command} #{all_options} #{alter_argument} #{Option.new("table", alter_argument.table_name)}"
     end
 
     # Generates the percona command for a raw MySQL statement. Fills all the
@@ -59,9 +64,9 @@ module Arrival
     # @return [String]
     def parse_statement(statement)
       alter_argument = AlterArgument.new(statement)
-      dsn = DSN.new(connection_details.database, alter_argument.table_name)
+      # dsn = DSN.new(connection_details.database, alter_argument.table_name)
 
-      "#{command} #{all_options} #{dsn} #{alter_argument}"
+      "#{command} #{all_options} #{alter_argument} #{Option.new("table", alter_argument.table_name)}"
     end
 
     private
@@ -77,7 +82,7 @@ module Arrival
     # @return [String]
     def all_options
       env_variable_options = UserOptions.new
-      global_configuration_options = UserOptions.new(Arrival.configuration.global_percona_args)
+      global_configuration_options = UserOptions.new(Arrival.configuration.global_arrival_args)
       options = env_variable_options.merge(global_configuration_options).merge(DEFAULT_OPTIONS)
       options.to_a.join(' ')
     end

data/lib/arrival/command.rb

@@ -37,22 +37,25 @@ module Arrival
 
     attr_reader :command_line, :error_log_path, :logger, :status
 
+    # TODO: Update comment
     # Runs the command in a separate process, capturing its stdout and
     # execution status
     def run_in_process
-      Open3.popen3(full_command) do |_stdin, stdout, _stderr, waith_thr|
-        begin
-          loop do
-            IO.select([stdout])
-            data = stdout.read_nonblock(8192)
-            logger.write_no_newline(data)
-          end
-        rescue EOFError # rubocop:disable Lint/HandleExceptions
-          # noop
-        ensure
-          @status = waith_thr.value
-        end
-      end
+      stdout = system(full_command)
+      @exit_code = $?
+      # Open3.capture2(full_command) do |_stdin, stdout, waith_thr|
+      #   begin
+      #     loop do
+      #       IO.select([stdout])
+      #       data = stdout.read_nonblock(8192)
+      #       logger.write_no_newline(data)
+      #     end
+      #   rescue EOFError # rubocop:disable Lint/HandleExceptions
+      #     # noop
+      #   ensure
+      #     @status = waith_thr.value
+      #   end
+      # end
     end
 
     # Builds the actual command including stderr redirection to the specified
@@ -69,9 +72,10 @@ module Arrival
     # @raise [SignalError] if the spawned process received a signal
     # @raise [CommandNotFoundError] if pt-online-schema-change can't be found
     def validate_status!
-      raise SignalError.new(status) if status.signaled? # rubocop:disable Style/RaiseArgs
-      raise CommandNotFoundError if status.exitstatus == COMMAND_NOT_FOUND
-      raise Error, error_message unless status.success?
+      # binding.pry
+      # raise SignalError.new(status) if status.signaled? # rubocop:disable Style/RaiseArgs
+      # raise CommandNotFoundError if status.exitstatus == COMMAND_NOT_FOUND
+      # raise Error, error_message unless status.success?
     end
 
     # Returns the error message that appeared in the process' stderr

data/lib/arrival/configuration.rb

@@ -1,11 +1,11 @@
 module Arrival
   class Configuration
-    attr_accessor :tmp_path, :global_percona_args
+    attr_accessor :tmp_path, :global_arrival_args
 
     def initialize
       @tmp_path = '.'.freeze
-      @error_log_filename = 'arrival_error.log'.freeze
-      @global_percona_args = nil
+      @error_log_filename = 'departure_error.log'.freeze
+      @global_arrival_args = nil
     end
 
     def error_log_path

data/lib/arrival/railtie.rb

@@ -1,28 +1,19 @@
-require 'arrival'
-require 'lhm' # It's our own Lhm adapter, not the gem
-require 'rails'
+# require 'activerecord/railtie'
 
 module Arrival
-  class Railtie < Rails::Railtie
-    railtie_name :arrival
-
-    # It drops all previous database connections and reconnects using this
-    # PerconaAdapter. By doing this, all later ActiveRecord methods called in
-    # the migration will use this adapter instead of Mysql2Adapter.
-    #
-    # It also patches ActiveRecord's #migrate method so that it patches LHM
-    # first. This will make migrations written with LHM to go through the
-    # regular Rails Migration DSL.
-    initializer 'arrival.configure_rails_initialization' do
-      ActiveSupport.on_load(:active_record) do
-        Arrival.load
-      end
-    end
-
-    initializer 'arrival.configure' do |app|
-      Arrival.configure do |config|
-        config.tmp_path = app.paths['tmp'].first
-      end
+  ActiveSupport.on_load(:active_record) do
+    Arrival.load
+    Arrival.configure do |config|
+      # config.tmp_path = app.paths['tmp'].first
     end
   end
+  # class Railtie < ActiveRecord::Railtie
+  #   railtie_name :arrival
+
+  #   initializer 'departure.configure_rails_initialization' do
+  #     ActiveSupport.on_load(:active_record) do
+  #       Departure.load
+  #     end
+  #   end
+  # end
 end

data/lib/arrival/version.rb

@@ -1,3 +1,3 @@
 module Arrival
-  VERSION = '0.1.0'.freeze
+  VERSION = '0.1.1'
 end

data/replica/mysql_replica.env

@@ -1,7 +1,7 @@
 # Note, by default mysql root does not have a password. You need to restart a server to bring MYSQL_ROOT_PASSWORD working. Use "docker-compose restart" command.
 MYSQL_ROOT_PASSWORD=111
 MYSQL_PORT=3306
-MYSQL_USER=mydb_slave_user
-MYSQL_PASSWORD=mydb_slave_pwd
+MYSQL_USER=replica_user
+MYSQL_PASSWORD=replica_pass
 MYSQL_DATABASE=mydb
 MYSQL_LOWER_CASE_TABLE_NAMES=0

data/setup_replication.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+
+docker-compose down
+rm -rf ./main/data/*
+rm -rf ./replica/data/*
+docker-compose build
+docker-compose up -d
+
+until docker-compose exec mysql-main sh -c 'export MYSQL_PWD=111; mysql -u root -e ";"'
+do
+    echo "Waiting for mysql-main database connection..."
+    sleep 4
+done
+
+priv_stmt='GRANT REPLICATION SLAVE ON *.* TO "replica_user"@"%" IDENTIFIED BY "replica_pass"; FLUSH PRIVILEGES;'
+docker exec mysql-main sh -c "export MYSQL_PWD=111; mysql -u root -e '$priv_stmt'"
+
+until docker-compose exec mysql-replica sh -c 'export MYSQL_PWD=111; mysql -u root -e ";"'
+do
+    echo "Waiting for mysql-replica database connection..."
+    sleep 4
+done
+
+docker-ip() {
+    docker inspect --format '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$@"
+}
+
+usage_statement='GRANT ALL PRIVILEGES ON *.* TO "replica_user"@"%"; FLUSH PRIVILEGES'
+docker exec mysql-main sh -c "export MYSQL_PWD=111; mysql -u root -e '$usage_statement'"
+
+
+MS_STATUS=`docker exec mysql-main sh -c 'export MYSQL_PWD=111; mysql -u root -e "SHOW MASTER STATUS"'`
+CURRENT_LOG=`echo $MS_STATUS | awk '{print $6}'`
+CURRENT_POS=`echo $MS_STATUS | awk '{print $7}'`
+
+# CHANGE MASTER TO MASTER_HOST='172.22.0.2',MASTER_USER='replica_user',MASTER_PASSWORD='mydb_slave_pwd',MASTER_LOG_FILE='$CURRENT_LOG',MASTER_LOG_POS=$CURRENT_POS; START SLAVE;
+
+start_slave_stmt="CHANGE MASTER TO MASTER_HOST='$(docker-ip mysql-main)',MASTER_USER='replica_user',MASTER_PASSWORD='replica_pass',MASTER_LOG_FILE='$CURRENT_LOG',MASTER_LOG_POS=$CURRENT_POS; START SLAVE;"
+start_slave_cmd='export MYSQL_PWD=111; mysql -u root -e "'
+start_slave_cmd+="$start_slave_stmt"
+start_slave_cmd+='"'
+docker exec mysql-replica sh -c "$start_slave_cmd"
+
+
+
+docker exec mysql-replica sh -c "export MYSQL_PWD=111; mysql -u root -e 'SHOW SLAVE STATUS \G'"

data/test_database.rb

@@ -6,26 +6,24 @@ require 'active_record/connection_adapters/mysql2_adapter'
 # the tests.
 #
 class TestDatabase
-  # Constructor
-  #
-  # @param config [Hash]
-  def initialize(config)
-    @config = config
-    @database = config['database']
+  attr_reader :env_config
+
+  def initialize(_opts = {})
+    @database = env_config["database"]
   end
 
   # Creates the test database, the schema_migrations and the comments tables.
   # It drops any of them if they already exist
   def setup
     setup_test_database
-    drop_and_create_schema_migrations_table
+    # drop_and_create_schema_migrations_table
   end
 
   # Creates the #{@database} database and the comments table in it.
   # Before, it drops both if they already exist
   def setup_test_database
     drop_and_create_test_database
-    drop_and_create_comments_table
+    # drop_and_create_comments_table
   end
 
   # Creates the ActiveRecord's schema_migrations table required for
@@ -40,6 +38,10 @@ class TestDatabase
     run_commands(sql)
   end
 
+  def env_config
+    config.dup[ENV['RAKE_ENV'] || "development"]
+  end
+
   private
 
   attr_reader :config, :database
@@ -64,17 +66,18 @@ class TestDatabase
   end
 
   def run_commands(sql)
-    conn.execute('START TRANSACTION')
-    sql.each { |str| conn.execute(str) }
-    conn.execute('COMMIT')
+    connection.execute('START TRANSACTION')
+    sql.each { |str| connection.execute(str) }
+    connection.execute('COMMIT')
   end
 
-  def conn
-    @conn ||= ActiveRecord::Base.mysql2_connection(
-      host: @config['hostname'],
-      username: @config['username'],
-      password: @config['password'],
-      reconnect: true
+  def connection
+    @connection ||= ActiveRecord::Base.mysql2_connection(
+      env_config.reject { |k, _v| k == "database" }
     )
   end
+
+  def config
+    @config ||= YAML.safe_load(File.read('db/config.yml')).freeze
+  end
 end

metadata

@@ -1,29 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: arrival
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - Arno Fleming
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-11-25 00:00:00.000000000 Z
+date: 2019-12-04 00:00:00.000000000 Z
 dependencies:
-- !ruby/object:Gem::Dependency
-  name: railties
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 5.2.0
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 5.2.0
 - !ruby/object:Gem::Dependency
   name: activerecord
   requirement: !ruby/object:Gem::Requirement
@@ -47,7 +33,7 @@ dependencies:
         version: 0.4.0
     - - "<="
       - !ruby/object:Gem::Version
-        version: 0.5.2
+        version: 0.5.3
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -57,7 +43,7 @@ dependencies:
         version: 0.4.0
     - - "<="
       - !ruby/object:Gem::Version
-        version: 0.5.2
+        version: 0.5.3
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -108,32 +94,18 @@ dependencies:
         version: '1.2'
 - !ruby/object:Gem::Dependency
   name: pry
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: climate_control
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.0.3
+        version: 0.12.2
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.0.3
+        version: 0.12.2
 description: Execute your ActiveRecord migrations with Github's `gh-ost`, a triggerless
   online schema changer
 email:
@@ -142,9 +114,7 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- ".codeclimate.yml"
-- ".github/ISSUE_TEMPLATE/bug_report.md"
-- ".github/ISSUE_TEMPLATE/feature_request.md"
+- ".dockerignore"
 - ".gitignore"
 - ".rspec"
 - ".rubocop.yml"
@@ -152,6 +122,7 @@ files:
 - CHANGELOG.md
 - CODE_OF_CONDUCT.md
 - Dockerfile
+- Dockerfile2.2
 - Gemfile
 - LICENSE.txt
 - README.md
@@ -162,37 +133,32 @@ files:
 - bin/console
 - bin/rspec
 - bin/setup
-- config.yml.erb
+- config.yml
 - configuration.rb
+- db/config.yml
+- departure_error.log
 - docker-compose-inspiration.yml
 - docker-compose.yml
+- lib/active_record/connection_adapters/arrival_adapter.rb
 - lib/active_record/connection_adapters/for_alter.rb
-- lib/active_record/connection_adapters/percona_adapter.rb
 - lib/arrival.rb
 - lib/arrival/alter_argument.rb
 - lib/arrival/cli_generator.rb
 - lib/arrival/command.rb
 - lib/arrival/configuration.rb
 - lib/arrival/connection_details.rb
-- lib/arrival/dsn.rb
 - lib/arrival/errors.rb
-- lib/arrival/log_sanitizers/password_sanitizer.rb
 - lib/arrival/logger.rb
-- lib/arrival/logger_factory.rb
-- lib/arrival/null_logger.rb
 - lib/arrival/option.rb
 - lib/arrival/railtie.rb
 - lib/arrival/runner.rb
 - lib/arrival/user_options.rb
 - lib/arrival/version.rb
-- lib/lhm.rb
-- lib/lhm/adapter.rb
-- lib/lhm/column_with_sql.rb
-- lib/lhm/column_with_type.rb
 - main/conf/mysql.conf.cnf
 - main/mysql_main.env
 - replica/conf/mysql.conf.cnf
 - replica/mysql_replica.env
+- setup_replication.sh
 - test_database.rb
 homepage: https://github.com/wetransfer/arrival
 licenses: