arrival 0.1.0 → 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,4 +1,3 @@
1
- require 'arrival/dsn'
2
1
  require 'arrival/option'
3
2
  require 'arrival/alter_argument'
4
3
  require 'arrival/connection_details'
@@ -12,41 +11,47 @@ module Arrival
12
11
  # www.percona.com/doc/percona-toolkit/2.2/pt-online-schema-change.html#cmdoption-pt-online-schema-change--[no]check-alter # rubocop:disable Metrics/LineLength
13
12
  #
14
13
  class CliGenerator
15
- # COMMAND_NAME = 'pt-online-schema-change'.freeze
16
14
  COMMAND_NAME = 'gh-ost'.freeze
17
15
  DEFAULT_OPTIONS = Set.new(
18
16
  [
19
17
  Option.new('execute'),
20
- Option.new('statistics'),
21
- Option.new('alter-foreign-keys-method', 'auto'),
22
- Option.new('no-check-alter')
18
+ Option.new('max-load', 'Threads_running=150'),
19
+ Option.new('critical-load', 'Threads_running=4000'),
20
+ Option.new('chunk-size', '1000'),
21
+ Option.new('throttle-control-replicas', 'mysql-replica'),
22
+ Option.new('max-lag-millis', '3000'),
23
+ Option.new('user', 'replica_user'),
24
+ Option.new('password', 'replica_pass'),
25
+ Option.new('host', 'mysql-replica'),
26
+ Option.new('database', 'arrival_test'),
27
+ Option.new('dml-batch-size', '1000'),
28
+ Option.new('alter', '"CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci"'),
29
+ Option.new('assume-rbr'),
30
+ Option.new('cut-over', 'default'),
31
+ Option.new('exact-rowcount'),
32
+ Option.new('concurrent-rowcount'),
33
+ Option.new('default-retries', '1200'),
34
+ Option.new('cut-over-lock-timeout-seconds', '10'),
35
+ Option.new('panic-flag-file', '/tmp/ghost.panic.downloads.flag'),
36
+ Option.new('assume-master-host', 'mysql-main'),
37
+ # Option.new('postpone-cut-over-flag-file', '/tmp/ghost.postpone.downloads.flag'),
38
+ Option.new('verbose'),
23
39
  ]
24
40
  ).freeze
25
41
 
26
- # TODO: Better doc.
27
- #
28
- # Constructor. Specify any arguments to pass to pt-online-schema-change
29
- # passing the PERCONA_ARGS env var when executing the migration
30
- #
31
- # @param connection_data [Hash]
32
- def initialize(connection_details)
33
- @connection_details = connection_details
34
- end
35
-
36
42
  # Generates the percona command. Fills all the connection credentials from
37
43
  # the current AR connection, but that can be amended via ENV-vars:
38
44
  # PERCONA_DB_HOST, PERCONA_DB_USER, PERCONA_DB_PASSWORD, PERCONA_DB_NAME
39
45
  # Table name can't not be amended, it populates automatically from the
40
46
  # migration data
41
47
  #
42
- # @param table_name [String]
48
+ # deprecated: @param table_name [String]
43
49
  # @param statement [String] MySQL statement
44
50
  # @return [String]
45
- def generate(table_name, statement)
51
+ def generate(_table_name=nil, statement)
46
52
  alter_argument = AlterArgument.new(statement)
47
- dsn = DSN.new(connection_details.database, table_name)
48
53
 
49
- "#{command} #{all_options} #{dsn} #{alter_argument}"
54
+ "#{command} #{all_options} #{alter_argument} #{Option.new("table", alter_argument.table_name)}"
50
55
  end
51
56
 
52
57
  # Generates the percona command for a raw MySQL statement. Fills all the
@@ -59,9 +64,9 @@ module Arrival
59
64
  # @return [String]
60
65
  def parse_statement(statement)
61
66
  alter_argument = AlterArgument.new(statement)
62
- dsn = DSN.new(connection_details.database, alter_argument.table_name)
67
+ # dsn = DSN.new(connection_details.database, alter_argument.table_name)
63
68
 
64
- "#{command} #{all_options} #{dsn} #{alter_argument}"
69
+ "#{command} #{all_options} #{alter_argument} #{Option.new("table", alter_argument.table_name)}"
65
70
  end
66
71
 
67
72
  private
@@ -77,7 +82,7 @@ module Arrival
77
82
  # @return [String]
78
83
  def all_options
79
84
  env_variable_options = UserOptions.new
80
- global_configuration_options = UserOptions.new(Arrival.configuration.global_percona_args)
85
+ global_configuration_options = UserOptions.new(Arrival.configuration.global_arrival_args)
81
86
  options = env_variable_options.merge(global_configuration_options).merge(DEFAULT_OPTIONS)
82
87
  options.to_a.join(' ')
83
88
  end
@@ -37,22 +37,25 @@ module Arrival
37
37
 
38
38
  attr_reader :command_line, :error_log_path, :logger, :status
39
39
 
40
+ # TODO: Update comment
40
41
  # Runs the command in a separate process, capturing its stdout and
41
42
  # execution status
42
43
  def run_in_process
43
- Open3.popen3(full_command) do |_stdin, stdout, _stderr, waith_thr|
44
- begin
45
- loop do
46
- IO.select([stdout])
47
- data = stdout.read_nonblock(8192)
48
- logger.write_no_newline(data)
49
- end
50
- rescue EOFError # rubocop:disable Lint/HandleExceptions
51
- # noop
52
- ensure
53
- @status = waith_thr.value
54
- end
55
- end
44
+ stdout = system(full_command)
45
+ @exit_code = $?
46
+ # Open3.capture2(full_command) do |_stdin, stdout, waith_thr|
47
+ # begin
48
+ # loop do
49
+ # IO.select([stdout])
50
+ # data = stdout.read_nonblock(8192)
51
+ # logger.write_no_newline(data)
52
+ # end
53
+ # rescue EOFError # rubocop:disable Lint/HandleExceptions
54
+ # # noop
55
+ # ensure
56
+ # @status = waith_thr.value
57
+ # end
58
+ # end
56
59
  end
57
60
 
58
61
  # Builds the actual command including stderr redirection to the specified
@@ -69,9 +72,10 @@ module Arrival
69
72
  # @raise [SignalError] if the spawned process received a signal
70
73
  # @raise [CommandNotFoundError] if pt-online-schema-change can't be found
71
74
  def validate_status!
72
- raise SignalError.new(status) if status.signaled? # rubocop:disable Style/RaiseArgs
73
- raise CommandNotFoundError if status.exitstatus == COMMAND_NOT_FOUND
74
- raise Error, error_message unless status.success?
75
+ # binding.pry
76
+ # raise SignalError.new(status) if status.signaled? # rubocop:disable Style/RaiseArgs
77
+ # raise CommandNotFoundError if status.exitstatus == COMMAND_NOT_FOUND
78
+ # raise Error, error_message unless status.success?
75
79
  end
76
80
 
77
81
  # Returns the error message that appeared in the process' stderr
@@ -1,11 +1,11 @@
1
1
  module Arrival
2
2
  class Configuration
3
- attr_accessor :tmp_path, :global_percona_args
3
+ attr_accessor :tmp_path, :global_arrival_args
4
4
 
5
5
  def initialize
6
6
  @tmp_path = '.'.freeze
7
- @error_log_filename = 'arrival_error.log'.freeze
8
- @global_percona_args = nil
7
+ @error_log_filename = 'departure_error.log'.freeze
8
+ @global_arrival_args = nil
9
9
  end
10
10
 
11
11
  def error_log_path
@@ -1,28 +1,19 @@
1
- require 'arrival'
2
- require 'lhm' # It's our own Lhm adapter, not the gem
3
- require 'rails'
1
+ # require 'activerecord/railtie'
4
2
 
5
3
  module Arrival
6
- class Railtie < Rails::Railtie
7
- railtie_name :arrival
8
-
9
- # It drops all previous database connections and reconnects using this
10
- # PerconaAdapter. By doing this, all later ActiveRecord methods called in
11
- # the migration will use this adapter instead of Mysql2Adapter.
12
- #
13
- # It also patches ActiveRecord's #migrate method so that it patches LHM
14
- # first. This will make migrations written with LHM to go through the
15
- # regular Rails Migration DSL.
16
- initializer 'arrival.configure_rails_initialization' do
17
- ActiveSupport.on_load(:active_record) do
18
- Arrival.load
19
- end
20
- end
21
-
22
- initializer 'arrival.configure' do |app|
23
- Arrival.configure do |config|
24
- config.tmp_path = app.paths['tmp'].first
25
- end
4
+ ActiveSupport.on_load(:active_record) do
5
+ Arrival.load
6
+ Arrival.configure do |config|
7
+ # config.tmp_path = app.paths['tmp'].first
26
8
  end
27
9
  end
10
+ # class Railtie < ActiveRecord::Railtie
11
+ # railtie_name :arrival
12
+
13
+ # initializer 'departure.configure_rails_initialization' do
14
+ # ActiveSupport.on_load(:active_record) do
15
+ # Departure.load
16
+ # end
17
+ # end
18
+ # end
28
19
  end
@@ -1,3 +1,3 @@
1
1
  module Arrival
2
- VERSION = '0.1.0'.freeze
2
+ VERSION = '0.1.1'
3
3
  end
@@ -1,7 +1,7 @@
1
1
  # Note, by default mysql root does not have a password. You need to restart a server to bring MYSQL_ROOT_PASSWORD working. Use "docker-compose restart" command.
2
2
  MYSQL_ROOT_PASSWORD=111
3
3
  MYSQL_PORT=3306
4
- MYSQL_USER=mydb_slave_user
5
- MYSQL_PASSWORD=mydb_slave_pwd
4
+ MYSQL_USER=replica_user
5
+ MYSQL_PASSWORD=replica_pass
6
6
  MYSQL_DATABASE=mydb
7
7
  MYSQL_LOWER_CASE_TABLE_NAMES=0
@@ -0,0 +1,46 @@
1
+ #!/bin/bash
2
+
3
+ docker-compose down
4
+ rm -rf ./main/data/*
5
+ rm -rf ./replica/data/*
6
+ docker-compose build
7
+ docker-compose up -d
8
+
9
+ until docker-compose exec mysql-main sh -c 'export MYSQL_PWD=111; mysql -u root -e ";"'
10
+ do
11
+ echo "Waiting for mysql-main database connection..."
12
+ sleep 4
13
+ done
14
+
15
+ priv_stmt='GRANT REPLICATION SLAVE ON *.* TO "replica_user"@"%" IDENTIFIED BY "replica_pass"; FLUSH PRIVILEGES;'
16
+ docker exec mysql-main sh -c "export MYSQL_PWD=111; mysql -u root -e '$priv_stmt'"
17
+
18
+ until docker-compose exec mysql-replica sh -c 'export MYSQL_PWD=111; mysql -u root -e ";"'
19
+ do
20
+ echo "Waiting for mysql-replica database connection..."
21
+ sleep 4
22
+ done
23
+
24
+ docker-ip() {
25
+ docker inspect --format '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$@"
26
+ }
27
+
28
+ usage_statement='GRANT ALL PRIVILEGES ON *.* TO "replica_user"@"%"; FLUSH PRIVILEGES'
29
+ docker exec mysql-main sh -c "export MYSQL_PWD=111; mysql -u root -e '$usage_statement'"
30
+
31
+
32
+ MS_STATUS=`docker exec mysql-main sh -c 'export MYSQL_PWD=111; mysql -u root -e "SHOW MASTER STATUS"'`
33
+ CURRENT_LOG=`echo $MS_STATUS | awk '{print $6}'`
34
+ CURRENT_POS=`echo $MS_STATUS | awk '{print $7}'`
35
+
36
+ # CHANGE MASTER TO MASTER_HOST='172.22.0.2',MASTER_USER='replica_user',MASTER_PASSWORD='mydb_slave_pwd',MASTER_LOG_FILE='$CURRENT_LOG',MASTER_LOG_POS=$CURRENT_POS; START SLAVE;
37
+
38
+ start_slave_stmt="CHANGE MASTER TO MASTER_HOST='$(docker-ip mysql-main)',MASTER_USER='replica_user',MASTER_PASSWORD='replica_pass',MASTER_LOG_FILE='$CURRENT_LOG',MASTER_LOG_POS=$CURRENT_POS; START SLAVE;"
39
+ start_slave_cmd='export MYSQL_PWD=111; mysql -u root -e "'
40
+ start_slave_cmd+="$start_slave_stmt"
41
+ start_slave_cmd+='"'
42
+ docker exec mysql-replica sh -c "$start_slave_cmd"
43
+
44
+
45
+
46
+ docker exec mysql-replica sh -c "export MYSQL_PWD=111; mysql -u root -e 'SHOW SLAVE STATUS \G'"
@@ -6,26 +6,24 @@ require 'active_record/connection_adapters/mysql2_adapter'
6
6
  # the tests.
7
7
  #
8
8
  class TestDatabase
9
- # Constructor
10
- #
11
- # @param config [Hash]
12
- def initialize(config)
13
- @config = config
14
- @database = config['database']
9
+ attr_reader :env_config
10
+
11
+ def initialize(_opts = {})
12
+ @database = env_config["database"]
15
13
  end
16
14
 
17
15
  # Creates the test database, the schema_migrations and the comments tables.
18
16
  # It drops any of them if they already exist
19
17
  def setup
20
18
  setup_test_database
21
- drop_and_create_schema_migrations_table
19
+ # drop_and_create_schema_migrations_table
22
20
  end
23
21
 
24
22
  # Creates the #{@database} database and the comments table in it.
25
23
  # Before, it drops both if they already exist
26
24
  def setup_test_database
27
25
  drop_and_create_test_database
28
- drop_and_create_comments_table
26
+ # drop_and_create_comments_table
29
27
  end
30
28
 
31
29
  # Creates the ActiveRecord's schema_migrations table required for
@@ -40,6 +38,10 @@ class TestDatabase
40
38
  run_commands(sql)
41
39
  end
42
40
 
41
+ def env_config
42
+ config.dup[ENV['RAKE_ENV'] || "development"]
43
+ end
44
+
43
45
  private
44
46
 
45
47
  attr_reader :config, :database
@@ -64,17 +66,18 @@ class TestDatabase
64
66
  end
65
67
 
66
68
  def run_commands(sql)
67
- conn.execute('START TRANSACTION')
68
- sql.each { |str| conn.execute(str) }
69
- conn.execute('COMMIT')
69
+ connection.execute('START TRANSACTION')
70
+ sql.each { |str| connection.execute(str) }
71
+ connection.execute('COMMIT')
70
72
  end
71
73
 
72
- def conn
73
- @conn ||= ActiveRecord::Base.mysql2_connection(
74
- host: @config['hostname'],
75
- username: @config['username'],
76
- password: @config['password'],
77
- reconnect: true
74
+ def connection
75
+ @connection ||= ActiveRecord::Base.mysql2_connection(
76
+ env_config.reject { |k, _v| k == "database" }
78
77
  )
79
78
  end
79
+
80
+ def config
81
+ @config ||= YAML.safe_load(File.read('db/config.yml')).freeze
82
+ end
80
83
  end
metadata CHANGED
@@ -1,29 +1,15 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: arrival
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.0
4
+ version: 0.1.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Arno Fleming
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2019-11-25 00:00:00.000000000 Z
11
+ date: 2019-12-04 00:00:00.000000000 Z
12
12
  dependencies:
13
- - !ruby/object:Gem::Dependency
14
- name: railties
15
- requirement: !ruby/object:Gem::Requirement
16
- requirements:
17
- - - "~>"
18
- - !ruby/object:Gem::Version
19
- version: 5.2.0
20
- type: :runtime
21
- prerelease: false
22
- version_requirements: !ruby/object:Gem::Requirement
23
- requirements:
24
- - - "~>"
25
- - !ruby/object:Gem::Version
26
- version: 5.2.0
27
13
  - !ruby/object:Gem::Dependency
28
14
  name: activerecord
29
15
  requirement: !ruby/object:Gem::Requirement
@@ -47,7 +33,7 @@ dependencies:
47
33
  version: 0.4.0
48
34
  - - "<="
49
35
  - !ruby/object:Gem::Version
50
- version: 0.5.2
36
+ version: 0.5.3
51
37
  type: :runtime
52
38
  prerelease: false
53
39
  version_requirements: !ruby/object:Gem::Requirement
@@ -57,7 +43,7 @@ dependencies:
57
43
  version: 0.4.0
58
44
  - - "<="
59
45
  - !ruby/object:Gem::Version
60
- version: 0.5.2
46
+ version: 0.5.3
61
47
  - !ruby/object:Gem::Dependency
62
48
  name: rake
63
49
  requirement: !ruby/object:Gem::Requirement
@@ -108,32 +94,18 @@ dependencies:
108
94
  version: '1.2'
109
95
  - !ruby/object:Gem::Dependency
110
96
  name: pry
111
- requirement: !ruby/object:Gem::Requirement
112
- requirements:
113
- - - ">="
114
- - !ruby/object:Gem::Version
115
- version: '0'
116
- type: :development
117
- prerelease: false
118
- version_requirements: !ruby/object:Gem::Requirement
119
- requirements:
120
- - - ">="
121
- - !ruby/object:Gem::Version
122
- version: '0'
123
- - !ruby/object:Gem::Dependency
124
- name: climate_control
125
97
  requirement: !ruby/object:Gem::Requirement
126
98
  requirements:
127
99
  - - "~>"
128
100
  - !ruby/object:Gem::Version
129
- version: 0.0.3
101
+ version: 0.12.2
130
102
  type: :development
131
103
  prerelease: false
132
104
  version_requirements: !ruby/object:Gem::Requirement
133
105
  requirements:
134
106
  - - "~>"
135
107
  - !ruby/object:Gem::Version
136
- version: 0.0.3
108
+ version: 0.12.2
137
109
  description: Execute your ActiveRecord migrations with Github's `gh-ost`, a triggerless
138
110
  online schema changer
139
111
  email:
@@ -142,9 +114,7 @@ executables: []
142
114
  extensions: []
143
115
  extra_rdoc_files: []
144
116
  files:
145
- - ".codeclimate.yml"
146
- - ".github/ISSUE_TEMPLATE/bug_report.md"
147
- - ".github/ISSUE_TEMPLATE/feature_request.md"
117
+ - ".dockerignore"
148
118
  - ".gitignore"
149
119
  - ".rspec"
150
120
  - ".rubocop.yml"
@@ -152,6 +122,7 @@ files:
152
122
  - CHANGELOG.md
153
123
  - CODE_OF_CONDUCT.md
154
124
  - Dockerfile
125
+ - Dockerfile2.2
155
126
  - Gemfile
156
127
  - LICENSE.txt
157
128
  - README.md
@@ -162,37 +133,32 @@ files:
162
133
  - bin/console
163
134
  - bin/rspec
164
135
  - bin/setup
165
- - config.yml.erb
136
+ - config.yml
166
137
  - configuration.rb
138
+ - db/config.yml
139
+ - departure_error.log
167
140
  - docker-compose-inspiration.yml
168
141
  - docker-compose.yml
142
+ - lib/active_record/connection_adapters/arrival_adapter.rb
169
143
  - lib/active_record/connection_adapters/for_alter.rb
170
- - lib/active_record/connection_adapters/percona_adapter.rb
171
144
  - lib/arrival.rb
172
145
  - lib/arrival/alter_argument.rb
173
146
  - lib/arrival/cli_generator.rb
174
147
  - lib/arrival/command.rb
175
148
  - lib/arrival/configuration.rb
176
149
  - lib/arrival/connection_details.rb
177
- - lib/arrival/dsn.rb
178
150
  - lib/arrival/errors.rb
179
- - lib/arrival/log_sanitizers/password_sanitizer.rb
180
151
  - lib/arrival/logger.rb
181
- - lib/arrival/logger_factory.rb
182
- - lib/arrival/null_logger.rb
183
152
  - lib/arrival/option.rb
184
153
  - lib/arrival/railtie.rb
185
154
  - lib/arrival/runner.rb
186
155
  - lib/arrival/user_options.rb
187
156
  - lib/arrival/version.rb
188
- - lib/lhm.rb
189
- - lib/lhm/adapter.rb
190
- - lib/lhm/column_with_sql.rb
191
- - lib/lhm/column_with_type.rb
192
157
  - main/conf/mysql.conf.cnf
193
158
  - main/mysql_main.env
194
159
  - replica/conf/mysql.conf.cnf
195
160
  - replica/mysql_replica.env
161
+ - setup_replication.sh
196
162
  - test_database.rb
197
163
  homepage: https://github.com/wetransfer/arrival
198
164
  licenses: