arrival 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c332dc7b7c51a5b6618708a6eb86048150299eb763768f14f0e16c1c3ddd4edb
-  data.tar.gz: cd107ee392065839636c80f6c03884d5e137d1d974f7af98a3354f1d939c946f
+  metadata.gz: d1e2f10a31f7bc2d3087e7a8861df36aaf746b602c2e9819c2b9474ba0265ebf
+  data.tar.gz: c9bd276b6c165ca87d3e716c257736ae0d4777a312389f7372671a7ef97c90be
 SHA512:
-  metadata.gz: 810977503202add1a717f2ef5a63eebaeac5fadda9cbe46a7f1bc0817e42f28957b425dd4bf4ca5e182ec6e92c009ace7b188780efc85bf28c7cc6e1fd1c3e23
-  data.tar.gz: 64cc923ed13acfad1f6259be3b268e15f69501a4beed5d59c737c978b1f29da6a6a0d8f32fd213a6611db10451670cf4ff6aeab68f36d8b7aededc4918a0a50b
+  metadata.gz: 8079d723c39aa6d60fa24d8824dc8c1a6cfbc3ee614b00f78807aa0694043fdcb89e23432dd74c899a1e875edd30294c3ff23f23bf27a2c6761d9f55a02f11f3
+  data.tar.gz: b5771ab12e86691db61a22592855b0eec127a577c034cbe61badda2665975d87dec16083cca0b241bb6eeeec1127ca8def9e9725a1ce46f87f5b3bd15fa690c9

data/.dockerignore

@@ -0,0 +1,3 @@
+main/
+replica/
+arrival*.gem

data/Dockerfile

@@ -1,22 +1,27 @@
-FROM ruby:2.6
-ENV GHOST_VERSION=1.0.48
+FROM ruby:2.6.5
 
+ENV GHOST_VERSION=v1.0.48 \
+    GHOST_TIMESTAMP=20190214020851
 # Install apt based dependencies required to run Rails as
 # well as RubyGems. As the Ruby image itself is based on a
 # Debian image, we use apt-get to install those.
 RUN apt-get update && \
   apt-get install -y \
-    build-essential \
-    percona-toolkit && \
+    mariadb-client && \
+#    build-essential \
+#     percona-toolkit && \
   apt-get clean && \
   apt-get remove
 
-# Download gh-ost and install the binary
 WORKDIR /usr/bin
-RUN wget https://github.com/github/gh-ost/releases/download/v$GHOST_VERSION/gh-ost-binary-linux-20190214020851.tar.gz && \
-  tar -xvf gh-ost-binary-linux-20190214020851.tar.gz
+RUN wget --no-verbose --tries=5 \
+    https://github.com/github/gh-ost/releases/download/$GHOST_VERSION/gh-ost-binary-linux-$GHOST_TIMESTAMP.tar.gz && \
+  tar -xvf gh-ost-binary-linux-$GHOST_TIMESTAMP.tar.gz
 
 RUN mkdir -p /app /app/lib/arrival
+# Configure the main working directory. This is the base
+# directory used in any further RUN, COPY, and ENTRYPOINT
+# commands.
 WORKDIR /app
 
 # Copy the Gemfile as well as the Gemfile.lock and install
@@ -25,10 +30,7 @@ WORKDIR /app
 # are made.
 COPY arrival.gemspec Gemfile ./
 COPY lib/arrival/version.rb ./lib/arrival/
-
-RUN gem update --system && \
-  gem install bundler -v 1.17.3 && \
-  bundle install --jobs $(nproc) --retry 5
+RUN gem install bundler -v 1.17.3 && bundle install --jobs $(nproc) --retry 5
 
 # Copy the main application.
 COPY . ./

data/Dockerfile2.2

@@ -0,0 +1,39 @@
+FROM ruby:2.2
+ENV GHOST_VERSION=1.0.48
+
+# Install apt based dependencies required to run Rails as
+# well as RubyGems. As the Ruby image itself is based on a
+# Debian image, we use apt-get to install those.
+RUN apt-get update && \
+  apt-get install -y \
+    build-essential \
+    percona-toolkit && \
+  apt-get clean && \
+  apt-get remove
+
+# Configure the main working directory. This is the base
+# directory used in any further RUN, COPY, and ENTRYPOINT
+# commands.
+RUN mkdir -p /app /app/lib/arrival
+WORKDIR /usr/bin
+RUN wget https://github.com/github/gh-ost/releases/download/v$GHOST_VERSION/gh-ost-binary-linux-20190214020851.tar.gz
+WORKDIR /app
+
+# Copy the Gemfile as well as the Gemfile.lock and install
+# the RubyGems. This is a separate step so the dependencies
+# will be cached unless changes to one of those two files
+# are made.
+COPY arrival.gemspec Gemfile ./
+COPY lib/arrival/version.rb ./lib/arrival/
+
+RUN gem update --system && \
+  gem install bundler -v 1.17.3 && \
+  bundle install --jobs $(nproc) --retry 5
+
+# Copy the main application.
+COPY . ./
+
+# The main command to run when the container starts. Also
+# tell the Rails dev server to bind to all interfaces by
+# default.
+#CMD ["bundle", "exec", "rails", "server", "-b", "0.0.0.0"]

data/Rakefile

@@ -1,7 +1,6 @@
 require 'bundler/gem_tasks'
 require 'rspec/core/rake_task'
 
-require './configuration'
 require './test_database'
 
 RSpec::Core::RakeTask.new(:spec)
@@ -9,9 +8,9 @@ RSpec::Core::RakeTask.new(:spec)
 task default: :spec
 
 namespace :db do
-  desc 'Create the test database'
+  desc 'Create the database'
   task :create do
-    config = Configuration.new
-    TestDatabase.new(config).setup_test_database
+    TestDatabase.new.setup_test_database
+    puts "Database is created"
   end
 end

data/arrival.gemspec

@@ -17,13 +17,12 @@ Gem::Specification.new do |spec|
   spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
   spec.require_paths = ['lib']
 
-  spec.add_runtime_dependency 'railties', '~> 5.2.0'
   spec.add_runtime_dependency 'activerecord', '~> 5.2.0'
-  spec.add_runtime_dependency 'mysql2', '>= 0.4.0', '<= 0.5.2'
+  spec.add_runtime_dependency 'mysql2', '>= 0.4.0', '<= 0.5.3'
 
   spec.add_development_dependency 'rake', '~> 10.0'
   spec.add_development_dependency 'rspec', '~> 3.4', '>= 3.4.0'
   spec.add_development_dependency 'rspec-its', '~> 1.2'
-  spec.add_development_dependency 'pry' #,'~> 8.2', '>= 8.2.1'
-  spec.add_development_dependency 'climate_control', '~> 0.0.3'
+  spec.add_development_dependency 'pry' ,'~> 0.12.2'
+  # spec.add_development_dependency 'simplecov' #,'~> 8.2', '>= 8.2.1'
 end

data/bin/console

@@ -1,7 +1,7 @@
 #!/usr/bin/env ruby
 
 require 'bundler/setup'
-require 'percona_migrator'
+require 'arrival'
 
 # You can add fixtures and/or initialization code here to make experimenting
 # with your gem easier. You can also use a different console, if you like.

data/config.yml

@@ -0,0 +1,4 @@
+username: 'replica_user'
+password: 'replica_pass'
+database: 'arrival_test'
+hostname: 'mysql-replica'

data/configuration.rb

@@ -1,13 +1,10 @@
 require 'yaml'
-require 'erb'
 
 class Configuration
-  CONFIG_PATH = 'config.yml.erb'.freeze
-
   attr_reader :config
 
   def initialize
-    @config = YAML.load(ERB.new(File.read(CONFIG_PATH)).result).freeze
+    @config = YAML.safe_load(File.read('config.yml')).freeze
   end
 
   def [](key)

data/db/config.yml

@@ -0,0 +1,15 @@
+development:
+  adapter: mysql2
+  pool: 5
+  host: "mysql-main"
+  username: "root"
+  password: "111"
+  database: arrival_development
+
+test:
+  adapter: mysql2
+  pool: 5
+  host: "mysql-main"
+  username: "root"
+  password: "111"
+  database: arrival_test

data/departure_error.log

@@ -0,0 +1,80 @@
+2019-12-04 08:29:48 INFO starting gh-ost 1.0.48
+2019-12-04 08:29:48 INFO Migrating `arrival_test`.`comments`
+2019-12-04 08:29:48 INFO connection validated on mysql-replica:3306
+2019-12-04 08:29:48 INFO User has ALL privileges
+2019-12-04 08:29:48 INFO binary logs validated on mysql-replica:3306
+2019-12-04 08:29:48 INFO Inspector initiated on 62e8fa90771a:3306, version 5.7.25-log
+2019-12-04 08:29:48 INFO Table found. Engine=InnoDB
+2019-12-04 08:29:48 INFO Estimated number of rows via EXPLAIN: 1
+2019-12-04 08:29:48 INFO Master forced to be mysql-main:3306
+2019-12-04 08:29:48 INFO log_slave_updates validated on mysql-replica:3306
+2019-12-04 08:29:48 INFO connection validated on mysql-replica:3306
+2019-12-04 08:29:48 INFO Connecting binlog streamer at mysql-bin.000003:513602
+2019-12-04 08:29:48 INFO rotate to next log from mysql-bin.000003:0 to mysql-bin.000003
+2019-12-04 08:29:48 INFO connection validated on mysql-main:3306
+2019-12-04 08:29:49 INFO connection validated on mysql-main:3306
+2019-12-04 08:29:49 INFO will use time_zone='SYSTEM' on applier
+2019-12-04 08:29:49 INFO Examining table structure on applier
+2019-12-04 08:29:49 INFO Applier initiated on fc06e3c916b4:3306, version 5.7.25-log
+2019-12-04 08:29:49 INFO Dropping table `arrival_test`.`_comments_ghc`
+2019-12-04 08:29:49 INFO Table dropped
+2019-12-04 08:29:49 INFO Creating changelog table `arrival_test`.`_comments_ghc`
+2019-12-04 08:29:49 INFO Changelog table created
+2019-12-04 08:29:49 INFO Creating ghost table `arrival_test`.`_comments_gho`
+2019-12-04 08:29:49 INFO Ghost table created
+2019-12-04 08:29:49 INFO Altering ghost table `arrival_test`.`_comments_gho`
+2019-12-04 08:29:49 INFO Ghost table altered
+2019-12-04 08:29:49 INFO Waiting for ghost table to be migrated. Current lag is 0s
+2019-12-04 08:29:49 INFO Intercepted changelog state GhostTableMigrated
+2019-12-04 08:29:49 INFO Handled changelog state GhostTableMigrated
+2019-12-04 08:29:49 INFO Chosen shared unique key is PRIMARY
+2019-12-04 08:29:49 INFO Shared columns are id
+2019-12-04 08:29:49 INFO Listening on unix socket file: /tmp/gh-ost.arrival_test.comments.sock
+2019-12-04 08:29:49 INFO As instructed, counting rows in the background; meanwhile I will use an estimated count, and will update it later on
+2019-12-04 08:29:49 INFO As instructed, I'm issuing a SELECT COUNT(*) on the table. This may take a while
+2019-12-04 08:29:49 INFO Migration min values: [<nil>]
+2019-12-04 08:29:49 INFO Exact number of rows via COUNT: 0
+2019-12-04 08:29:49 INFO Migration max values: [<nil>]
+2019-12-04 08:29:49 INFO Waiting for first throttle metrics to be collected
+2019-12-04 08:29:49 INFO First throttle metrics collected
+2019-12-04 08:29:49 INFO Row copy complete
+2019-12-04 08:29:49 INFO Grabbing voluntary lock: gh-ost.430.lock
+2019-12-04 08:29:49 INFO Setting LOCK timeout as 20 seconds
+2019-12-04 08:29:49 INFO Looking for magic cut-over table
+2019-12-04 08:29:49 INFO Creating magic cut-over table `arrival_test`.`_comments_del`
+2019-12-04 08:29:49 INFO Magic cut-over table created
+2019-12-04 08:29:49 INFO Locking `arrival_test`.`comments`, `arrival_test`.`_comments_del`
+2019-12-04 08:29:49 INFO Tables locked
+2019-12-04 08:29:49 INFO Session locking original & magic tables is 430
+2019-12-04 08:29:49 INFO Writing changelog state: AllEventsUpToLockProcessed:1575448189399341800
+2019-12-04 08:29:49 INFO Waiting for events up to lock
+2019-12-04 08:29:49 INFO Intercepted changelog state AllEventsUpToLockProcessed
+2019-12-04 08:29:49 INFO Handled changelog state AllEventsUpToLockProcessed
+2019-12-04 08:29:50 INFO Waiting for events up to lock: got AllEventsUpToLockProcessed:1575448189399341800
+2019-12-04 08:29:50 INFO Done waiting for events up to lock; duration=920.8407ms
+2019-12-04 08:29:50 ERROR Error 1213: Deadlock found when trying to get lock; try restarting transaction
+2019-12-04 08:29:50 INFO Setting RENAME timeout as 10 seconds
+2019-12-04 08:29:50 INFO Session renaming tables is 432
+2019-12-04 08:29:50 INFO Issuing and expecting this to block: rename /* gh-ost */ table `arrival_test`.`comments` to `arrival_test`.`_comments_del`, `arrival_test`.`_comments_gho` to `arrival_test`.`comments`
+2019-12-04 08:29:50 INFO Found atomic RENAME to be blocking, as expected. Double checking the lock is still in place (though I don't strictly have to)
+2019-12-04 08:29:50 INFO Checking session lock: gh-ost.430.lock
+2019-12-04 08:29:50 INFO Connection holding lock on original table still exists
+2019-12-04 08:29:50 INFO Will now proceed to drop magic table and unlock tables
+2019-12-04 08:29:50 INFO Dropping magic cut-over table
+2019-12-04 08:29:50 INFO Releasing lock from `arrival_test`.`comments`, `arrival_test`.`_comments_del`
+2019-12-04 08:29:50 INFO Tables unlocked
+2019-12-04 08:29:50 INFO Tables renamed
+2019-12-04 08:29:50 INFO Lock & rename duration: 1.0548297s. During this time, queries on `comments` were blocked
+2019-12-04 08:29:50 INFO Looking for magic cut-over table
+2019-12-04 08:29:50 INFO Closed streamer connection. err=<nil>
+2019-12-04 08:29:50 INFO Dropping table `arrival_test`.`_comments_ghc`
+2019-12-04 08:29:50 INFO Table dropped
+2019-12-04 08:29:50 INFO Am not dropping old table because I want this operation to be as live as possible. If you insist I should do it, please add `--ok-to-drop-table` next time. But I prefer you do not. To drop the old table, issue:
+2019-12-04 08:29:50 INFO -- drop table `arrival_test`.`_comments_del`
+2019-12-04 08:29:50 INFO Done migrating `arrival_test`.`comments`
+2019-12-04 08:29:50 INFO Removing socket file: /tmp/gh-ost.arrival_test.comments.sock
+2019-12-04 08:29:50 INFO Tearing down inspector
+2019-12-04 08:29:50 INFO Tearing down applier
+2019-12-04 08:29:50 INFO Tearing down streamer
+2019-12-04 08:29:50 INFO Tearing down throttler
+2019-12-04 08:29:50 ERROR Error 1146: Table 'arrival_test._comments_ghc' doesn't exist

data/docker-compose.yml

@@ -1,10 +1,10 @@
 version: '3'
 services:
-  mysql_main:
+  mysql-main:
     image: mysql:5.7
     env_file:
       - ./main/mysql_main.env
-    container_name: "mysql_main"
+    container_name: "mysql-main"
     restart: "no"
     # ports:
     #   - 4406:3306
@@ -12,16 +12,16 @@ services:
       - ./main/conf/mysql.conf.cnf:/etc/mysql/conf.d/mysql.conf.cnf
       - ./main/data:/var/lib/mysql
 
-  mysql_replica:
+  mysql-replica:
     image: mysql:5.7
     env_file:
       - ./replica/mysql_replica.env
-    container_name: "mysql_replica"
+    container_name: "mysql-replica"
     restart: "no"
     # ports:
     #   - 5506:3306
     depends_on:
-      - mysql_main
+      - mysql-main
     volumes:
       - ./replica/conf/mysql.conf.cnf:/etc/mysql/conf.d/mysql.conf.cnf
       - ./replica/data:/var/lib/mysql
@@ -29,8 +29,10 @@ services:
   arrival:
     build: .
     depends_on:
-      - mysql_main
-      - mysql_replica
+      - mysql-main
+      - mysql-replica
+    volumes:
+      - ./:/app/
     # tty: true
     # stdin_open: true
     # environment:

data/lib/active_record/connection_adapters/{percona_adapter.rb → arrival_adapter.rb}

@@ -1,32 +1,29 @@
-require 'active_record/connection_adapters/abstract_mysql_adapter'
-require 'active_record/connection_adapters/statement_pool'
-require 'active_record/connection_adapters/mysql2_adapter'
-require 'arrival'
-require 'forwardable'
-
 module ActiveRecord
   module ConnectionHandling
     # Establishes a connection to the database that's used by all Active
     # Record objects.
-    def percona_connection(config)
+    def arrival_connection(config)
       config[:username] = 'root' if config[:username].nil?
-      mysql2_connection = mysql2_connection(config)
+      mysql_connection = mysql2_connection(config)
 
-      connection_details =Arrival::ConnectionDetails.new(config)
-      verbose = ActiveRecord::Migration.verbose
+      connection_details = Arrival::ConnectionDetails.new(config)
+      ActiveRecord::Migration.verbose
       sanitizers = [
-        Arrival::LogSanitizers::PasswordSanitizer.new(connection_details)
+        # Arrival::LogSanitizers::PasswordSanitizer.new(connection_details)
       ]
-      percona_logger = Arrival::LoggerFactory.build(sanitizers: sanitizers, verbose: verbose)
-      cli_generator = Arrival::CliGenerator.new(connection_details)
+      arrival_logger = Arrival::Logger.new(sanitizers)
+
+      # arrival_logger =
+      cli_generator = Arrival::CliGenerator.new
+      # cli_generator =
 
       runner = Arrival::Runner.new(
-        percona_logger,
+        arrival_logger,
         cli_generator,
-        mysql2_connection
+        mysql_connection
       )
 
-      connection_options = { mysql_adapter: mysql2_connection }
+      connection_options = { mysql_adapter: mysql_connection }
 
       ConnectionAdapters::ArrivalAdapter.new(
         runner,
@@ -36,7 +33,6 @@ module ActiveRecord
       )
     end
   end
-
   module ConnectionAdapters
     class ArrivalAdapter < AbstractMysqlAdapter
       class Column < ActiveRecord::ConnectionAdapters::MySQL::Column
@@ -44,7 +40,6 @@ module ActiveRecord
           ArrivalAdapter
         end
       end
-
       class SchemaCreation < ActiveRecord::ConnectionAdapters::MySQL::SchemaCreation
         def visit_DropForeignKey(name) # rubocop:disable Naming/MethodName
           fk_name =
@@ -64,7 +59,7 @@ module ActiveRecord
         include ForAlterStatements
       end
 
-      ADAPTER_NAME = 'Percona'.freeze
+      ADAPTER_NAME = 'Arrival'.freeze
 
       def_delegators :mysql_adapter, :last_inserted_id, :each_hash, :set_field_encoding
 

data/lib/arrival.rb

@@ -1,23 +1,29 @@
 require 'active_record'
-require 'active_support/all'
+# require 'active_support/all'
 
-require 'active_record/connection_adapters/for_alter'
+# require 'active_record/connection_adapters/for_alter'
 
 require 'arrival/version'
-require 'arrival/log_sanitizers/password_sanitizer'
+require 'arrival/railtie'
+# require 'arrival/runner'
+require 'active_record/connection_adapters/for_alter'
+# require 'arrival/cli_generator'
+# require 'arrival/configuration'
+
+# require 'arrival/log_sanitizers/password_sanitizer'
 require 'arrival/runner'
 require 'arrival/cli_generator'
 require 'arrival/logger'
-require 'arrival/null_logger'
-require 'arrival/logger_factory'
+# require 'arrival/null_logger'
+# require 'arrival/logger_factory'
 require 'arrival/configuration'
 require 'arrival/errors'
 require 'arrival/command'
 
-require 'arrival/railtie' if defined?(Rails)
+# require 'arrival/railtie' if defined?(Rails)
 
-# We need the OS not to buffer the IO to see pt-osc's output while migrating
-$stdout.sync = true
+# # We need the OS not to buffer the IO to see pt-osc's output while migrating
+# $stdout.sync = true
 
 module Arrival
   class << self
@@ -29,7 +35,7 @@ module Arrival
     yield(configuration)
   end
 
-  # Hooks Percona Migrator into Rails migrations by replacing the configured
+  # Hooks Arrival into Rails migrations by replacing the configured
   # database adapter
   def self.load
     ActiveRecord::Migration.class_eval do
@@ -40,28 +46,28 @@ module Arrival
       #
       # @param direction [Symbol] :up or :down
       def migrate(direction)
-        reconnect_with_percona
-        include_foreigner if defined?(Foreigner)
+        reconnect_with_arrival
+        # include_foreigner if defined?(Foreigner)
 
-        ::Lhm.migration = self
+        # ::Lhm.migration = self
         original_migrate(direction)
       end
 
       # Includes the Foreigner's Mysql2Adapter implemention in
       # ArrivalAdapter to support foreign keys
-      def include_foreigner
-        Foreigner::Adapter.safe_include(
-          :ArrivalAdapter,
-          Foreigner::ConnectionAdapters::Mysql2Adapter
-        )
-      end
+      # def include_foreigner
+      #   Foreigner::Adapter.safe_include(
+      #     :ArrivalAdapter,
+      #     Foreigner::ConnectionAdapters::Mysql2Adapter
+      #   )
+      # end
 
       # Make all connections in the connection pool to use PerconaAdapter
       # instead of the current adapter.
-      def reconnect_with_percona
-        connection_config = ActiveRecord::Base
-          .connection_config.merge(adapter: 'percona')
-        ActiveRecord::Base.establish_connection(connection_config)
+      def reconnect_with_arrival
+        ActiveRecord::Base.establish_connection(
+          TestDatabase.new.env_config.merge("adapter" => "arrival")
+        )
       end
     end
   end