arrival 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.codeclimate.yml +8 -0
- data/.github/ISSUE_TEMPLATE/bug_report.md +31 -0
- data/.github/ISSUE_TEMPLATE/feature_request.md +20 -0
- data/.gitignore +13 -0
- data/.rspec +2 -0
- data/.rubocop.yml +59 -0
- data/.travis.yml +20 -0
- data/CHANGELOG.md +184 -0
- data/CODE_OF_CONDUCT.md +50 -0
- data/Dockerfile +39 -0
- data/Gemfile +6 -0
- data/LICENSE.txt +22 -0
- data/README.md +209 -0
- data/RELEASING.md +17 -0
- data/Rakefile +17 -0
- data/arrival.gemspec +29 -0
- data/bin/console +14 -0
- data/bin/rspec +16 -0
- data/bin/setup +7 -0
- data/config.yml.erb +4 -0
- data/configuration.rb +16 -0
- data/docker-compose-inspiration.yml +44 -0
- data/docker-compose.yml +40 -0
- data/lib/active_record/connection_adapters/for_alter.rb +91 -0
- data/lib/active_record/connection_adapters/percona_adapter.rb +158 -0
- data/lib/arrival.rb +68 -0
- data/lib/arrival/alter_argument.rb +43 -0
- data/lib/arrival/cli_generator.rb +85 -0
- data/lib/arrival/command.rb +96 -0
- data/lib/arrival/configuration.rb +19 -0
- data/lib/arrival/connection_details.rb +96 -0
- data/lib/arrival/dsn.rb +24 -0
- data/lib/arrival/errors.rb +39 -0
- data/lib/arrival/log_sanitizers/password_sanitizer.rb +22 -0
- data/lib/arrival/logger.rb +42 -0
- data/lib/arrival/logger_factory.rb +16 -0
- data/lib/arrival/null_logger.rb +15 -0
- data/lib/arrival/option.rb +75 -0
- data/lib/arrival/railtie.rb +28 -0
- data/lib/arrival/runner.rb +62 -0
- data/lib/arrival/user_options.rb +44 -0
- data/lib/arrival/version.rb +3 -0
- data/lib/lhm.rb +23 -0
- data/lib/lhm/adapter.rb +107 -0
- data/lib/lhm/column_with_sql.rb +96 -0
- data/lib/lhm/column_with_type.rb +29 -0
- data/main/conf/mysql.conf.cnf +9 -0
- data/main/mysql_main.env +7 -0
- data/replica/conf/mysql.conf.cnf +10 -0
- data/replica/mysql_replica.env +7 -0
- data/test_database.rb +80 -0
- metadata +220 -0
@@ -0,0 +1,91 @@
|
|
1
|
+
require 'active_record/connection_adapters/mysql/schema_statements'
|
2
|
+
|
3
|
+
module ForAlterStatements
|
4
|
+
class << self
|
5
|
+
def included(_)
|
6
|
+
STDERR.puts 'Including for_alter statements'
|
7
|
+
end
|
8
|
+
end
|
9
|
+
|
10
|
+
def bulk_change_table(table_name, operations) #:nodoc:
|
11
|
+
sqls = operations.flat_map do |command, args|
|
12
|
+
table = args.shift
|
13
|
+
arguments = args
|
14
|
+
|
15
|
+
method = :"#{command}_for_alter"
|
16
|
+
|
17
|
+
raise "Unknown method called : #{method}(#{arguments.inspect})" unless respond_to?(method, true)
|
18
|
+
public_send(method, table, *arguments)
|
19
|
+
end.join(', ')
|
20
|
+
|
21
|
+
execute("ALTER TABLE #{quote_table_name(table_name)} #{sqls}")
|
22
|
+
end
|
23
|
+
|
24
|
+
def change_column_for_alter(table_name, column_name, type, options = {})
|
25
|
+
column = column_for(table_name, column_name)
|
26
|
+
type ||= column.sql_type
|
27
|
+
|
28
|
+
options = {
|
29
|
+
default: column.default,
|
30
|
+
null: column.null,
|
31
|
+
comment: column.comment
|
32
|
+
}.merge(options)
|
33
|
+
|
34
|
+
td = create_table_definition(table_name)
|
35
|
+
cd = td.new_column_definition(column.name, type, options)
|
36
|
+
schema_creation.accept(ActiveRecord::ConnectionAdapters::ChangeColumnDefinition.new(cd, column.name))
|
37
|
+
end
|
38
|
+
|
39
|
+
def rename_column_for_alter(table_name, column_name, new_column_name)
|
40
|
+
column = column_for(table_name, column_name)
|
41
|
+
options = {
|
42
|
+
default: column.default,
|
43
|
+
null: column.null,
|
44
|
+
auto_increment: column.auto_increment?
|
45
|
+
}
|
46
|
+
|
47
|
+
columns_sql = "SHOW COLUMNS FROM #{quote_table_name(table_name)} LIKE #{quote(column_name)}"
|
48
|
+
current_type = exec_query(columns_sql, 'SCHEMA').first['Type']
|
49
|
+
td = create_table_definition(table_name)
|
50
|
+
cd = td.new_column_definition(new_column_name, current_type, options)
|
51
|
+
schema_creation.accept(ActiveRecord::ConnectionAdapters::ChangeColumnDefinition.new(cd, column.name))
|
52
|
+
end
|
53
|
+
|
54
|
+
def add_index_for_alter(table_name, column_name, options = {})
|
55
|
+
index_name, index_type, index_columns, _,
|
56
|
+
index_algorithm, index_using = add_index_options(table_name, column_name, options)
|
57
|
+
|
58
|
+
index_algorithm[0, 0] = ', ' if index_algorithm.present?
|
59
|
+
"ADD #{index_type} INDEX #{quote_column_name(index_name)} #{index_using} (#{index_columns})#{index_algorithm}"
|
60
|
+
end
|
61
|
+
|
62
|
+
def remove_index_for_alter(table_name, options = {})
|
63
|
+
index_name = index_name_for_remove(table_name, options)
|
64
|
+
"DROP INDEX #{quote_column_name(index_name)}"
|
65
|
+
end
|
66
|
+
|
67
|
+
def add_timestamps_for_alter(table_name, options = {})
|
68
|
+
[
|
69
|
+
add_column_for_alter(table_name, :created_at, :datetime, options),
|
70
|
+
add_column_for_alter(table_name, :updated_at, :datetime, options)
|
71
|
+
]
|
72
|
+
end
|
73
|
+
|
74
|
+
def remove_timestamps_for_alter(table_name, _options = {})
|
75
|
+
[remove_column_for_alter(table_name, :updated_at), remove_column_for_alter(table_name, :created_at)]
|
76
|
+
end
|
77
|
+
|
78
|
+
def add_column_for_alter(table_name, column_name, type, options = {})
|
79
|
+
td = create_table_definition(table_name)
|
80
|
+
cd = td.new_column_definition(column_name, type, options)
|
81
|
+
schema_creation.accept(ActiveRecord::ConnectionAdapters::AddColumnDefinition.new(cd))
|
82
|
+
end
|
83
|
+
|
84
|
+
def remove_column_for_alter(_table_name, column_name, _type = nil, _options = {})
|
85
|
+
"DROP COLUMN #{quote_column_name(column_name)}"
|
86
|
+
end
|
87
|
+
|
88
|
+
def remove_columns_for_alter(table_name, *column_names)
|
89
|
+
column_names.map { |column_name| remove_column_for_alter(table_name, column_name) }
|
90
|
+
end
|
91
|
+
end
|
@@ -0,0 +1,158 @@
|
|
1
|
+
require 'active_record/connection_adapters/abstract_mysql_adapter'
|
2
|
+
require 'active_record/connection_adapters/statement_pool'
|
3
|
+
require 'active_record/connection_adapters/mysql2_adapter'
|
4
|
+
require 'arrival'
|
5
|
+
require 'forwardable'
|
6
|
+
|
7
|
+
module ActiveRecord
|
8
|
+
module ConnectionHandling
|
9
|
+
# Establishes a connection to the database that's used by all Active
|
10
|
+
# Record objects.
|
11
|
+
def percona_connection(config)
|
12
|
+
config[:username] = 'root' if config[:username].nil?
|
13
|
+
mysql2_connection = mysql2_connection(config)
|
14
|
+
|
15
|
+
connection_details =Arrival::ConnectionDetails.new(config)
|
16
|
+
verbose = ActiveRecord::Migration.verbose
|
17
|
+
sanitizers = [
|
18
|
+
Arrival::LogSanitizers::PasswordSanitizer.new(connection_details)
|
19
|
+
]
|
20
|
+
percona_logger = Arrival::LoggerFactory.build(sanitizers: sanitizers, verbose: verbose)
|
21
|
+
cli_generator = Arrival::CliGenerator.new(connection_details)
|
22
|
+
|
23
|
+
runner = Arrival::Runner.new(
|
24
|
+
percona_logger,
|
25
|
+
cli_generator,
|
26
|
+
mysql2_connection
|
27
|
+
)
|
28
|
+
|
29
|
+
connection_options = { mysql_adapter: mysql2_connection }
|
30
|
+
|
31
|
+
ConnectionAdapters::ArrivalAdapter.new(
|
32
|
+
runner,
|
33
|
+
logger,
|
34
|
+
connection_options,
|
35
|
+
config
|
36
|
+
)
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
module ConnectionAdapters
|
41
|
+
class ArrivalAdapter < AbstractMysqlAdapter
|
42
|
+
class Column < ActiveRecord::ConnectionAdapters::MySQL::Column
|
43
|
+
def adapter
|
44
|
+
ArrivalAdapter
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
class SchemaCreation < ActiveRecord::ConnectionAdapters::MySQL::SchemaCreation
|
49
|
+
def visit_DropForeignKey(name) # rubocop:disable Naming/MethodName
|
50
|
+
fk_name =
|
51
|
+
if name =~ /^__(.+)/
|
52
|
+
Regexp.last_match(1)
|
53
|
+
else
|
54
|
+
"_#{name}"
|
55
|
+
end
|
56
|
+
|
57
|
+
"DROP FOREIGN KEY #{fk_name}"
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
extend Forwardable
|
62
|
+
|
63
|
+
unless method_defined?(:change_column_for_alter)
|
64
|
+
include ForAlterStatements
|
65
|
+
end
|
66
|
+
|
67
|
+
ADAPTER_NAME = 'Percona'.freeze
|
68
|
+
|
69
|
+
def_delegators :mysql_adapter, :last_inserted_id, :each_hash, :set_field_encoding
|
70
|
+
|
71
|
+
def initialize(connection, _logger, connection_options, _config)
|
72
|
+
@mysql_adapter = connection_options[:mysql_adapter]
|
73
|
+
super
|
74
|
+
@prepared_statements = false
|
75
|
+
end
|
76
|
+
|
77
|
+
def exec_delete(sql, name, binds)
|
78
|
+
execute(to_sql(sql, binds), name)
|
79
|
+
@connection.affected_rows
|
80
|
+
end
|
81
|
+
alias exec_update exec_delete
|
82
|
+
|
83
|
+
def exec_insert(sql, name, binds, pk = nil, sequence_name = nil) # rubocop:disable Lint/UnusedMethodArgument, Metrics/LineLength
|
84
|
+
execute(to_sql(sql, binds), name)
|
85
|
+
end
|
86
|
+
|
87
|
+
def exec_query(sql, name = 'SQL', _binds = [])
|
88
|
+
result = execute(sql, name)
|
89
|
+
ActiveRecord::Result.new(result.fields, result.to_a)
|
90
|
+
end
|
91
|
+
|
92
|
+
# Executes a SELECT query and returns an array of rows. Each row is an
|
93
|
+
# array of field values.
|
94
|
+
|
95
|
+
def select_rows(arel, name = nil, binds = [])
|
96
|
+
select_all(arel, name, binds).rows
|
97
|
+
end
|
98
|
+
|
99
|
+
# Executes a SELECT query and returns an array of record hashes with the
|
100
|
+
# column names as keys and column values as values.
|
101
|
+
def select(sql, name = nil, binds = [])
|
102
|
+
exec_query(sql, name, binds)
|
103
|
+
end
|
104
|
+
|
105
|
+
# Returns true, as this adapter supports migrations
|
106
|
+
def supports_migrations?
|
107
|
+
true
|
108
|
+
end
|
109
|
+
|
110
|
+
# rubocop:disable Metrics/ParameterLists
|
111
|
+
def new_column(field, default, type_metadata, null, table_name, default_function, collation, comment)
|
112
|
+
Column.new(field, default, type_metadata, null, table_name, default_function, collation, comment)
|
113
|
+
end
|
114
|
+
# rubocop:enable Metrics/ParameterLists
|
115
|
+
|
116
|
+
# Adds a new index to the table
|
117
|
+
#
|
118
|
+
# @param table_name [String, Symbol]
|
119
|
+
# @param column_name [String, Symbol]
|
120
|
+
# @param options [Hash] optional
|
121
|
+
def add_index(table_name, column_name, options = {})
|
122
|
+
index_name, index_type, index_columns, index_options = add_index_options(table_name, column_name, options)
|
123
|
+
execute "ALTER TABLE #{quote_table_name(table_name)} ADD #{index_type} INDEX #{quote_column_name(index_name)} (#{index_columns})#{index_options}" # rubocop:disable Metrics/LineLength
|
124
|
+
end
|
125
|
+
|
126
|
+
# Remove the given index from the table.
|
127
|
+
#
|
128
|
+
# @param table_name [String, Symbol]
|
129
|
+
# @param options [Hash] optional
|
130
|
+
def remove_index(table_name, options = {})
|
131
|
+
index_name = index_name_for_remove(table_name, options)
|
132
|
+
execute "ALTER TABLE #{quote_table_name(table_name)} DROP INDEX #{quote_column_name(index_name)}"
|
133
|
+
end
|
134
|
+
|
135
|
+
def schema_creation
|
136
|
+
SchemaCreation.new(self)
|
137
|
+
end
|
138
|
+
|
139
|
+
def change_table(table_name, _options = {})
|
140
|
+
recorder = ActiveRecord::Migration::CommandRecorder.new(self)
|
141
|
+
yield update_table_definition(table_name, recorder)
|
142
|
+
bulk_change_table(table_name, recorder.commands)
|
143
|
+
end
|
144
|
+
|
145
|
+
# Returns the MySQL error number from the exception. The
|
146
|
+
# AbstractMysqlAdapter requires it to be implemented
|
147
|
+
def error_number(_exception); end
|
148
|
+
|
149
|
+
def full_version
|
150
|
+
mysql_adapter.raw_connection.server_info[:version]
|
151
|
+
end
|
152
|
+
|
153
|
+
private
|
154
|
+
|
155
|
+
attr_reader :mysql_adapter
|
156
|
+
end
|
157
|
+
end
|
158
|
+
end
|
data/lib/arrival.rb
ADDED
@@ -0,0 +1,68 @@
|
|
1
|
+
require 'active_record'
|
2
|
+
require 'active_support/all'
|
3
|
+
|
4
|
+
require 'active_record/connection_adapters/for_alter'
|
5
|
+
|
6
|
+
require 'arrival/version'
|
7
|
+
require 'arrival/log_sanitizers/password_sanitizer'
|
8
|
+
require 'arrival/runner'
|
9
|
+
require 'arrival/cli_generator'
|
10
|
+
require 'arrival/logger'
|
11
|
+
require 'arrival/null_logger'
|
12
|
+
require 'arrival/logger_factory'
|
13
|
+
require 'arrival/configuration'
|
14
|
+
require 'arrival/errors'
|
15
|
+
require 'arrival/command'
|
16
|
+
|
17
|
+
require 'arrival/railtie' if defined?(Rails)
|
18
|
+
|
19
|
+
# We need the OS not to buffer the IO to see pt-osc's output while migrating
|
20
|
+
$stdout.sync = true
|
21
|
+
|
22
|
+
module Arrival
|
23
|
+
class << self
|
24
|
+
attr_accessor :configuration
|
25
|
+
end
|
26
|
+
|
27
|
+
def self.configure
|
28
|
+
self.configuration ||= Configuration.new
|
29
|
+
yield(configuration)
|
30
|
+
end
|
31
|
+
|
32
|
+
# Hooks Percona Migrator into Rails migrations by replacing the configured
|
33
|
+
# database adapter
|
34
|
+
def self.load
|
35
|
+
ActiveRecord::Migration.class_eval do
|
36
|
+
alias_method :original_migrate, :migrate
|
37
|
+
|
38
|
+
# Replaces the current connection adapter with the PerconaAdapter and
|
39
|
+
# patches LHM, then it continues with the regular migration process.
|
40
|
+
#
|
41
|
+
# @param direction [Symbol] :up or :down
|
42
|
+
def migrate(direction)
|
43
|
+
reconnect_with_percona
|
44
|
+
include_foreigner if defined?(Foreigner)
|
45
|
+
|
46
|
+
::Lhm.migration = self
|
47
|
+
original_migrate(direction)
|
48
|
+
end
|
49
|
+
|
50
|
+
# Includes the Foreigner's Mysql2Adapter implemention in
|
51
|
+
# ArrivalAdapter to support foreign keys
|
52
|
+
def include_foreigner
|
53
|
+
Foreigner::Adapter.safe_include(
|
54
|
+
:ArrivalAdapter,
|
55
|
+
Foreigner::ConnectionAdapters::Mysql2Adapter
|
56
|
+
)
|
57
|
+
end
|
58
|
+
|
59
|
+
# Make all connections in the connection pool to use PerconaAdapter
|
60
|
+
# instead of the current adapter.
|
61
|
+
def reconnect_with_percona
|
62
|
+
connection_config = ActiveRecord::Base
|
63
|
+
.connection_config.merge(adapter: 'percona')
|
64
|
+
ActiveRecord::Base.establish_connection(connection_config)
|
65
|
+
end
|
66
|
+
end
|
67
|
+
end
|
68
|
+
end
|
@@ -0,0 +1,43 @@
|
|
1
|
+
module Arrival
|
2
|
+
class InvalidAlterStatement < StandardError; end
|
3
|
+
|
4
|
+
# Represents the '--alter' argument of Percona's pt-online-schema-change
|
5
|
+
# See https://www.percona.com/doc/percona-toolkit/2.0/pt-online-schema-change.html
|
6
|
+
class AlterArgument
|
7
|
+
ALTER_TABLE_REGEX = /\AALTER TABLE `(\w+)` /
|
8
|
+
|
9
|
+
attr_reader :table_name
|
10
|
+
|
11
|
+
# Constructor
|
12
|
+
#
|
13
|
+
# @param statement [String]
|
14
|
+
# @raise [InvalidAlterStatement] if the statement is not an ALTER TABLE
|
15
|
+
def initialize(statement)
|
16
|
+
@statement = statement
|
17
|
+
|
18
|
+
match = statement.match(ALTER_TABLE_REGEX)
|
19
|
+
raise InvalidAlterStatement unless match
|
20
|
+
|
21
|
+
@table_name = match.captures[0]
|
22
|
+
end
|
23
|
+
|
24
|
+
# Returns the '--alter' pt-online-schema-change argument as a string. See
|
25
|
+
# https://www.percona.com/doc/percona-toolkit/2.0/pt-online-schema-change.html
|
26
|
+
def to_s
|
27
|
+
"--alter \"#{parsed_statement}\""
|
28
|
+
end
|
29
|
+
|
30
|
+
private
|
31
|
+
|
32
|
+
attr_reader :statement
|
33
|
+
|
34
|
+
# Removes the 'ALTER TABLE' portion of the SQL statement
|
35
|
+
#
|
36
|
+
# @return [String]
|
37
|
+
def parsed_statement
|
38
|
+
@parsed_statement ||= statement
|
39
|
+
.gsub(ALTER_TABLE_REGEX, '')
|
40
|
+
.gsub('`', '\\\`')
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
@@ -0,0 +1,85 @@
|
|
1
|
+
require 'arrival/dsn'
|
2
|
+
require 'arrival/option'
|
3
|
+
require 'arrival/alter_argument'
|
4
|
+
require 'arrival/connection_details'
|
5
|
+
require 'arrival/user_options'
|
6
|
+
|
7
|
+
module Arrival
|
8
|
+
# Generates the equivalent Percona's pt-online-schema-change command to the
|
9
|
+
# given SQL statement
|
10
|
+
#
|
11
|
+
# --no-check-alter is used to allow running CHANGE COLUMN statements. For more details, check:
|
12
|
+
# www.percona.com/doc/percona-toolkit/2.2/pt-online-schema-change.html#cmdoption-pt-online-schema-change--[no]check-alter # rubocop:disable Metrics/LineLength
|
13
|
+
#
|
14
|
+
class CliGenerator
|
15
|
+
# COMMAND_NAME = 'pt-online-schema-change'.freeze
|
16
|
+
COMMAND_NAME = 'gh-ost'.freeze
|
17
|
+
DEFAULT_OPTIONS = Set.new(
|
18
|
+
[
|
19
|
+
Option.new('execute'),
|
20
|
+
Option.new('statistics'),
|
21
|
+
Option.new('alter-foreign-keys-method', 'auto'),
|
22
|
+
Option.new('no-check-alter')
|
23
|
+
]
|
24
|
+
).freeze
|
25
|
+
|
26
|
+
# TODO: Better doc.
|
27
|
+
#
|
28
|
+
# Constructor. Specify any arguments to pass to pt-online-schema-change
|
29
|
+
# passing the PERCONA_ARGS env var when executing the migration
|
30
|
+
#
|
31
|
+
# @param connection_data [Hash]
|
32
|
+
def initialize(connection_details)
|
33
|
+
@connection_details = connection_details
|
34
|
+
end
|
35
|
+
|
36
|
+
# Generates the percona command. Fills all the connection credentials from
|
37
|
+
# the current AR connection, but that can be amended via ENV-vars:
|
38
|
+
# PERCONA_DB_HOST, PERCONA_DB_USER, PERCONA_DB_PASSWORD, PERCONA_DB_NAME
|
39
|
+
# Table name can't not be amended, it populates automatically from the
|
40
|
+
# migration data
|
41
|
+
#
|
42
|
+
# @param table_name [String]
|
43
|
+
# @param statement [String] MySQL statement
|
44
|
+
# @return [String]
|
45
|
+
def generate(table_name, statement)
|
46
|
+
alter_argument = AlterArgument.new(statement)
|
47
|
+
dsn = DSN.new(connection_details.database, table_name)
|
48
|
+
|
49
|
+
"#{command} #{all_options} #{dsn} #{alter_argument}"
|
50
|
+
end
|
51
|
+
|
52
|
+
# Generates the percona command for a raw MySQL statement. Fills all the
|
53
|
+
# connection credentials from the current AR connection, but that can
|
54
|
+
# amended via ENV-vars: PERCONA_DB_HOST, PERCONA_DB_USER,
|
55
|
+
# PERCONA_DB_PASSWORD, PERCONA_DB_NAME Table name can't not be amended, it
|
56
|
+
# populates automatically from the migration data
|
57
|
+
#
|
58
|
+
# @param statement [String] MySQL statement
|
59
|
+
# @return [String]
|
60
|
+
def parse_statement(statement)
|
61
|
+
alter_argument = AlterArgument.new(statement)
|
62
|
+
dsn = DSN.new(connection_details.database, alter_argument.table_name)
|
63
|
+
|
64
|
+
"#{command} #{all_options} #{dsn} #{alter_argument}"
|
65
|
+
end
|
66
|
+
|
67
|
+
private
|
68
|
+
|
69
|
+
attr_reader :connection_details
|
70
|
+
|
71
|
+
def command
|
72
|
+
"#{COMMAND_NAME} #{connection_details}"
|
73
|
+
end
|
74
|
+
|
75
|
+
# Returns all the arguments to execute pt-online-schema-change with
|
76
|
+
#
|
77
|
+
# @return [String]
|
78
|
+
def all_options
|
79
|
+
env_variable_options = UserOptions.new
|
80
|
+
global_configuration_options = UserOptions.new(Arrival.configuration.global_percona_args)
|
81
|
+
options = env_variable_options.merge(global_configuration_options).merge(DEFAULT_OPTIONS)
|
82
|
+
options.to_a.join(' ')
|
83
|
+
end
|
84
|
+
end
|
85
|
+
end
|