arrival 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. checksums.yaml +7 -0
  2. data/.codeclimate.yml +8 -0
  3. data/.github/ISSUE_TEMPLATE/bug_report.md +31 -0
  4. data/.github/ISSUE_TEMPLATE/feature_request.md +20 -0
  5. data/.gitignore +13 -0
  6. data/.rspec +2 -0
  7. data/.rubocop.yml +59 -0
  8. data/.travis.yml +20 -0
  9. data/CHANGELOG.md +184 -0
  10. data/CODE_OF_CONDUCT.md +50 -0
  11. data/Dockerfile +39 -0
  12. data/Gemfile +6 -0
  13. data/LICENSE.txt +22 -0
  14. data/README.md +209 -0
  15. data/RELEASING.md +17 -0
  16. data/Rakefile +17 -0
  17. data/arrival.gemspec +29 -0
  18. data/bin/console +14 -0
  19. data/bin/rspec +16 -0
  20. data/bin/setup +7 -0
  21. data/config.yml.erb +4 -0
  22. data/configuration.rb +16 -0
  23. data/docker-compose-inspiration.yml +44 -0
  24. data/docker-compose.yml +40 -0
  25. data/lib/active_record/connection_adapters/for_alter.rb +91 -0
  26. data/lib/active_record/connection_adapters/percona_adapter.rb +158 -0
  27. data/lib/arrival.rb +68 -0
  28. data/lib/arrival/alter_argument.rb +43 -0
  29. data/lib/arrival/cli_generator.rb +85 -0
  30. data/lib/arrival/command.rb +96 -0
  31. data/lib/arrival/configuration.rb +19 -0
  32. data/lib/arrival/connection_details.rb +96 -0
  33. data/lib/arrival/dsn.rb +24 -0
  34. data/lib/arrival/errors.rb +39 -0
  35. data/lib/arrival/log_sanitizers/password_sanitizer.rb +22 -0
  36. data/lib/arrival/logger.rb +42 -0
  37. data/lib/arrival/logger_factory.rb +16 -0
  38. data/lib/arrival/null_logger.rb +15 -0
  39. data/lib/arrival/option.rb +75 -0
  40. data/lib/arrival/railtie.rb +28 -0
  41. data/lib/arrival/runner.rb +62 -0
  42. data/lib/arrival/user_options.rb +44 -0
  43. data/lib/arrival/version.rb +3 -0
  44. data/lib/lhm.rb +23 -0
  45. data/lib/lhm/adapter.rb +107 -0
  46. data/lib/lhm/column_with_sql.rb +96 -0
  47. data/lib/lhm/column_with_type.rb +29 -0
  48. data/main/conf/mysql.conf.cnf +9 -0
  49. data/main/mysql_main.env +7 -0
  50. data/replica/conf/mysql.conf.cnf +10 -0
  51. data/replica/mysql_replica.env +7 -0
  52. data/test_database.rb +80 -0
  53. metadata +220 -0
data/Gemfile ADDED
@@ -0,0 +1,6 @@
1
+ source 'https://rubygems.org'
2
+
3
+ gemspec
4
+
5
+ gem 'codeclimate-test-reporter', '~> 1.0.3', group: :test, require: nil
6
+ gem 'rubocop', '~> 0.49.1', require: false
@@ -0,0 +1,22 @@
1
+ The MIT License (MIT)
2
+
3
+ Copyright for portions of Arrival are held by Redbooth, Inc., 2015-2017. All
4
+ other copyright for Arrival are held by Pau Pérez, 2017.
5
+
6
+ Permission is hereby granted, free of charge, to any person obtaining a copy
7
+ of this software and associated documentation files (the "Software"), to deal
8
+ in the Software without restriction, including without limitation the rights
9
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10
+ copies of the Software, and to permit persons to whom the Software is
11
+ furnished to do so, subject to the following conditions:
12
+
13
+ The above copyright notice and this permission notice shall be included in
14
+ all copies or substantial portions of the Software.
15
+
16
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
22
+ THE SOFTWARE.
@@ -0,0 +1,209 @@
1
+ # Arrival [![Build Status](https://travis-ci.com/WeTransfer/arrival.svg?token=Mq6F1pVeWbb6XsB9HonL&branch=main)](https://travis-ci.com/WeTransfer/arrival)
2
+
3
+ Arrival is an **ActiveRecord connection adapter** that allows running
4
+ **MySQL online and non-blocking DDL** through `ActiveRecord::Migration` without needing
5
+ to use a different DSL other than Rails' migrations DSL.
6
+
7
+ It uses `pt-online-schema-change` command-line tool of
8
+ [Percona
9
+ Toolkit](https://www.percona.com/doc/percona-toolkit/2.0/pt-online-schema-change.html)
10
+ which runs MySQL alter table statements without downtime.
11
+
12
+ ## Rename from "Percona Migrator"
13
+
14
+ This project was formerly known as "Percona Migrator", but this incurs in an
15
+ infringement of Percona's trade mark policy and thus has to be renamed. Said
16
+ name is likely to cause confusion as to the source of the wrapper.
17
+
18
+ The next major versions will use "Arrival" as gem name.
19
+
20
+ ## Installation
21
+
22
+ Arrival relies on `pt-online-schema-change` from [Percona
23
+ Toolkit](https://www.percona.com/doc/percona-toolkit/2.0/pt-online-schema-change.html)
24
+
25
+ ### Mac
26
+
27
+ `brew install percona-toolkit`
28
+
29
+ If when running a migration you see an error like:
30
+
31
+ ```
32
+ PerconaMigrator::Error: Cannot connect to MySQL: Cannot connect to MySQL because
33
+ the Perl DBI module is not installed or not found.
34
+ ```
35
+
36
+ You also need to install the DBI and DBD::MySQL modules from `cpan`.
37
+
38
+ ```
39
+ $ sudo cpan
40
+ cpan> install DBI
41
+ cpan> install DBD::mysql
42
+ ```
43
+
44
+ ### Linux
45
+
46
+ #### Ubuntu/Debian based
47
+
48
+ `apt-get install percona-toolkit`
49
+
50
+ #### Arch Linux
51
+
52
+ `pacman -S percona-toolkit perl-dbd-mysql`
53
+
54
+ #### Other distros
55
+
56
+ For other Linux distributions check out the [Percona Toolkit download
57
+ page](https://www.percona.com/downloads/percona-toolkit/) to find the package
58
+ that fits your distribution.
59
+
60
+ You can also get it from [Percona's apt repository](https://www.percona.com/doc/percona-xtradb-cluster/5.5/installation/apt_repo.html)
61
+
62
+ Once installed, add this line to your application's Gemfile:
63
+
64
+ ```ruby
65
+ gem 'arrival'
66
+ ```
67
+
68
+ And then execute:
69
+
70
+ $ bundle
71
+
72
+ Or install it yourself as:
73
+
74
+ $ gem install arrival
75
+
76
+ ## Usage
77
+
78
+ Once you added it to your app's Gemfile, you can create and run Rails migrations
79
+ as usual.
80
+
81
+ All the `ALTER TABLE` statements will be executed with
82
+ `pt-online-schema-change`, which will provide additional output to the
83
+ migration.
84
+
85
+ ### pt-online-schema-change arguments
86
+
87
+ #### with environment variable
88
+
89
+ You can specify any `pt-online-schema-change` arguments when running the
90
+ migration. All what you pass in the PERCONA_ARGS env var, will be bypassed to the
91
+ binary, overwriting any default values. Note the format is the same as in
92
+ `pt-online-schema-change`. Check the full list in [Percona Toolkit
93
+ documentation](https://www.percona.com/doc/percona-toolkit/2.2/pt-online-schema-change.html#options)
94
+
95
+ ```ruby
96
+ $ PERCONA_ARGS='--chunk-time=1' bundle exec rake db:migrate:up VERSION=xxx
97
+ ```
98
+
99
+ or even mulitple arguments
100
+
101
+ ```ruby
102
+ $ PERCONA_ARGS='--chunk-time=1 --critical-load=55' bundle exec rake db:migrate:up VERSION=xxx
103
+ ```
104
+
105
+ Use caution when using PERCONA_ARGS with `db:migrate`, as your args will be applied
106
+ to every call that Arrival makes to pt-osc.
107
+
108
+ #### with global configuration
109
+
110
+ You can specify any `pt-online-schema-change` arguments in global gem configuration
111
+ using `global_percona_args` option.
112
+
113
+ ```ruby
114
+ Arrival.configure do |config|
115
+ config.global_percona_args = '--chunk-time=1 --critical-load=55'
116
+ end
117
+ ```
118
+
119
+ Unlike using `PERCONA_ARGS`, options provided with global configuration will be applied
120
+ every time sql command is executed via `pt-online-schema-change`.
121
+
122
+ Arguments provided in global configuration can be overwritten with `PERCONA_ARGS` env variable.
123
+
124
+ We recommend using this option with caution and only when you understand the consequences.
125
+
126
+ ### LHM support
127
+
128
+ If you moved to Soundcloud's [Lhm](https://github.com/soundcloud/lhm) already,
129
+ we got you covered. Arrival overrides Lhm's DSL so that all the alter
130
+ statements also go through `pt-online-schema-change` as well.
131
+
132
+ You can keep your Lhm migrations and start using Rails migration's DSL back
133
+ again in your next migration.
134
+
135
+ ## Configuration
136
+
137
+ You can override any of the default values from an initializer:
138
+
139
+ ```ruby
140
+ Arrival.configure do |config|
141
+ config.tmp_path = '/tmp/'
142
+ end
143
+ ```
144
+
145
+ It's strongly recommended to name it after this gems name, such as
146
+ `config/initializers/arrival.rb`
147
+
148
+ ## How it works
149
+
150
+ When booting your Rails app, Arrival extends the
151
+ `ActiveRecord::Migration#migrate` method to reset the connection and reestablish
152
+ it using the `ArrivalAdapter` instead of the one you defined in your
153
+ `config/database.yml`.
154
+
155
+ Then, when any migration DSL methods such as `add_column` or `create_table` are
156
+ executed, they all go to the
157
+ [ArrivalAdapter](https://github.com/arrivalrb/arrival/blob/master/lib/active_record/connection_adapters/arrival_adapter.rb).
158
+ There, the methods that require `ALTER TABLE` SQL statements, like `add_column`,
159
+ are overriden to get executed with
160
+ [Arrival::Runner](https://github.com/arrivalrb/arrival/blob/master/lib/arrival/runner.rb),
161
+ which deals with the `pt-online-schema-change` binary. All the others, like
162
+ `create_table`, are delegated to the ActiveRecord's built in Mysql2Adapter and
163
+ so they follow the regular path.
164
+
165
+ [Arrival::Runner](https://github.com/arrivalrb/arrival/blob/master/lib/arrival/runner.rb)
166
+ spawns a new process that runs the `pt-online-schema-change` binary present in
167
+ the system, with the appropriate arguments for the generated SQL.
168
+
169
+ When any errors occur, an `ActiveRecord::StatementInvalid` exception is
170
+ raised and the migration is aborted, as all other ActiveRecord connection
171
+ adapters.
172
+
173
+ ## Trouleshooting
174
+
175
+ ### Error creating new table: DBD::mysql::db do failed: Can't write; duplicate key in table (TABLE_NAME)
176
+ There is a [known bug](https://bugs.launchpad.net/percona-toolkit/+bug/1498128) in percona-toolkit version 2.2.15
177
+ that prevents schema changes when a table has constraints. You should upgrade to a version later than 2.2.17 to fix the issue.
178
+
179
+ ## Development
180
+
181
+ After checking out the repo, run `bin/setup` to install dependencies. Then, run
182
+ `rake spec` to run the tests. You can also run `bin/console` for an interactive
183
+ prompt that will allow you to experiment.
184
+
185
+ To install this gem onto your local machine, run `bundle exec rake install`. To
186
+ release a new version, update the version number in `version.rb`, and then run
187
+ `bundle exec rake release`, which will create a git tag for the version, push
188
+ git commits and tags, and push the `.gem` file to
189
+ [rubygems.org](https://rubygems.org).
190
+
191
+ ## Contributing
192
+
193
+ Bug reports and pull requests are welcome on GitHub at
194
+ https://github.com/arrivalrb/arrival. They need to be opened against
195
+ `master` or `v3.2` only if the changes fix a bug in Rails 3.2 apps.
196
+
197
+ Please note that this project is released with a Contributor Code of Conduct. By
198
+ participating in this project you agree to abide by its terms.
199
+
200
+ Check the code of conduct [here](CODE_OF_CONDUCT.md)
201
+
202
+ ## Changelog
203
+
204
+ You can consult the changelog [here](CHANGELOG.md)
205
+
206
+ ## License
207
+
208
+ The gem is available as open source under the terms of the [MIT
209
+ License](http://opensource.org/licenses/MIT).
@@ -0,0 +1,17 @@
1
+ # Releasing Arrival
2
+
3
+ All releases come from the master branch. All other branches won't be maintained
4
+ and will receive bug fix releases only.
5
+
6
+ In order to give support to a new major Rails version, we'll branch off of
7
+ master, name it following the Rails repo convention, such as `v4.2`, and
8
+ we'll keep it open for bug fixes.
9
+
10
+ 1. Update `lib/arrival/version.rb` accordingly
11
+ 2. Review the `CHANGELOG.md` and add a new section following the format
12
+ `[version] - YYYY-MM-DD`. We conform to the guidelines of
13
+ http://keepachangelog.com/
14
+ 3. Commit the changes with the message `Prepare release VERSION`
15
+ 4. Execute the release rake task as `bundle exec rake release`. It creates the
16
+ tag, builds and pushes the gem to Rubygems.
17
+ 5. Announce it! :tada:
@@ -0,0 +1,17 @@
1
+ require 'bundler/gem_tasks'
2
+ require 'rspec/core/rake_task'
3
+
4
+ require './configuration'
5
+ require './test_database'
6
+
7
+ RSpec::Core::RakeTask.new(:spec)
8
+
9
+ task default: :spec
10
+
11
+ namespace :db do
12
+ desc 'Create the test database'
13
+ task :create do
14
+ config = Configuration.new
15
+ TestDatabase.new(config).setup_test_database
16
+ end
17
+ end
@@ -0,0 +1,29 @@
1
+ lib = File.expand_path('../lib', __FILE__)
2
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
3
+
4
+ require 'arrival/version'
5
+
6
+ Gem::Specification.new do |spec|
7
+ spec.name = 'arrival'
8
+ spec.version = Arrival::VERSION
9
+ spec.authors = ['Arno Fleming']
10
+ spec.email = ['arno@wetransfer.com']
11
+
12
+ spec.summary = %(pt-gh-ost runner for ActiveRecord migrations)
13
+ spec.description = %(Execute your ActiveRecord migrations with Github's `gh-ost`, a triggerless online schema changer)
14
+ spec.homepage = 'https://github.com/wetransfer/arrival'
15
+ spec.license = 'MIT'
16
+
17
+ spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
18
+ spec.require_paths = ['lib']
19
+
20
+ spec.add_runtime_dependency 'railties', '~> 5.2.0'
21
+ spec.add_runtime_dependency 'activerecord', '~> 5.2.0'
22
+ spec.add_runtime_dependency 'mysql2', '>= 0.4.0', '<= 0.5.2'
23
+
24
+ spec.add_development_dependency 'rake', '~> 10.0'
25
+ spec.add_development_dependency 'rspec', '~> 3.4', '>= 3.4.0'
26
+ spec.add_development_dependency 'rspec-its', '~> 1.2'
27
+ spec.add_development_dependency 'pry' #,'~> 8.2', '>= 8.2.1'
28
+ spec.add_development_dependency 'climate_control', '~> 0.0.3'
29
+ end
@@ -0,0 +1,14 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'bundler/setup'
4
+ require 'percona_migrator'
5
+
6
+ # You can add fixtures and/or initialization code here to make experimenting
7
+ # with your gem easier. You can also use a different console, if you like.
8
+
9
+ # (If you use this, don't forget to add pry to your Gemfile!)
10
+ require "pry"
11
+ Pry.start
12
+
13
+ # require 'irb'
14
+ # IRB.start
@@ -0,0 +1,16 @@
1
+ #!/usr/bin/env ruby
2
+ # frozen_string_literal: true
3
+
4
+ # This file was generated by Bundler.
5
+ #
6
+ # The application 'rspec' is installed as part of a gem, and
7
+ # this file is here to facilitate running it.
8
+ #
9
+
10
+ require 'pathname'
11
+ ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../../Gemfile', Pathname.new(__FILE__).realpath)
12
+
13
+ require 'rubygems'
14
+ require 'bundler/setup'
15
+
16
+ load Gem.bin_path('rspec-core', 'rspec')
@@ -0,0 +1,7 @@
1
+ #!/bin/bash
2
+ set -euo pipefail
3
+ IFS=$'\n\t'
4
+
5
+ bundle install
6
+
7
+ bundle exec rake db:create
@@ -0,0 +1,4 @@
1
+ username: <%= ENV['PERCONA_DB_USER'] || 'root' %>
2
+ password: <%= ENV['PERCONA_DB_PASSWORD'] || '' %>
3
+ database: <%= ENV['PERCONA_DB_NAME'] || 'arrival_test' %>
4
+ hostname: <%= ENV['PERCONA_DB_HOST'] || 'localhost' %>
@@ -0,0 +1,16 @@
1
+ require 'yaml'
2
+ require 'erb'
3
+
4
+ class Configuration
5
+ CONFIG_PATH = 'config.yml.erb'.freeze
6
+
7
+ attr_reader :config
8
+
9
+ def initialize
10
+ @config = YAML.load(ERB.new(File.read(CONFIG_PATH)).result).freeze
11
+ end
12
+
13
+ def [](key)
14
+ config[key]
15
+ end
16
+ end
@@ -0,0 +1,44 @@
1
+ version: '3'
2
+ services:
3
+ web:
4
+ build: 'web'
5
+ command: bash -c "rm -f tmp/pids/server.pid && bundle exec rails s -p 3000 -b '0.0.0.0'"
6
+ container_name: "web"
7
+ volumes:
8
+ - ./web:/myapp
9
+ ports:
10
+ - "3000:3000"
11
+ depends_on:
12
+ - mysql_master
13
+
14
+ ghost:
15
+ build: ghost
16
+ depends_on:
17
+ - mysql_master
18
+ - mysql_slave
19
+
20
+ mysql_master:
21
+ image: mysql:5.7
22
+ env_file:
23
+ - ./master/mysql_master.env
24
+ container_name: "mysql_master"
25
+ restart: "no"
26
+ ports:
27
+ - 4406:3306
28
+ volumes:
29
+ - ./master/conf/mysql.conf.cnf:/etc/mysql/conf.d/mysql.conf.cnf
30
+ - ./master/data:/var/lib/mysql
31
+
32
+ mysql_slave:
33
+ image: mysql:5.7
34
+ env_file:
35
+ - ./slave/mysql_slave.env
36
+ container_name: "mysql_slave"
37
+ restart: "no"
38
+ ports:
39
+ - 5506:3306
40
+ depends_on:
41
+ - mysql_master
42
+ volumes:
43
+ - ./slave/conf/mysql.conf.cnf:/etc/mysql/conf.d/mysql.conf.cnf
44
+ - ./slave/data:/var/lib/mysql
@@ -0,0 +1,40 @@
1
+ version: '3'
2
+ services:
3
+ mysql_main:
4
+ image: mysql:5.7
5
+ env_file:
6
+ - ./main/mysql_main.env
7
+ container_name: "mysql_main"
8
+ restart: "no"
9
+ # ports:
10
+ # - 4406:3306
11
+ volumes:
12
+ - ./main/conf/mysql.conf.cnf:/etc/mysql/conf.d/mysql.conf.cnf
13
+ - ./main/data:/var/lib/mysql
14
+
15
+ mysql_replica:
16
+ image: mysql:5.7
17
+ env_file:
18
+ - ./replica/mysql_replica.env
19
+ container_name: "mysql_replica"
20
+ restart: "no"
21
+ # ports:
22
+ # - 5506:3306
23
+ depends_on:
24
+ - mysql_main
25
+ volumes:
26
+ - ./replica/conf/mysql.conf.cnf:/etc/mysql/conf.d/mysql.conf.cnf
27
+ - ./replica/data:/var/lib/mysql
28
+
29
+ arrival:
30
+ build: .
31
+ depends_on:
32
+ - mysql_main
33
+ - mysql_replica
34
+ # tty: true
35
+ # stdin_open: true
36
+ # environment:
37
+ # PERCONA_DB_USER: root
38
+ # PERCONA_DB_HOST: db
39
+ # PERCONA_DB_PASSWORD:
40
+ # PERCONA_DB_NAME: arrival_test