departure 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +12 -0
- data/.rspec +2 -0
- data/.travis.yml +16 -0
- data/CHANGELOG.md +95 -0
- data/CODE_OF_CONDUCT.md +50 -0
- data/Gemfile +5 -0
- data/LICENSE.txt +21 -0
- data/README.md +147 -0
- data/Rakefile +17 -0
- data/bin/console +14 -0
- data/bin/setup +7 -0
- data/config.yml +3 -0
- data/configuration.rb +15 -0
- data/departure.gemspec +35 -0
- data/lib/active_record/connection_adapters/percona_adapter.rb +121 -0
- data/lib/departure.rb +65 -0
- data/lib/departure/alter_argument.rb +43 -0
- data/lib/departure/cli_generator.rb +134 -0
- data/lib/departure/configuration.rb +18 -0
- data/lib/departure/errors.rb +35 -0
- data/lib/departure/logger.rb +31 -0
- data/lib/departure/logger_factory.rb +17 -0
- data/lib/departure/null_logger.rb +15 -0
- data/lib/departure/railtie.rb +28 -0
- data/lib/departure/runner.rb +136 -0
- data/lib/departure/version.rb +3 -0
- data/lib/lhm.rb +25 -0
- data/lib/lhm/adapter.rb +109 -0
- data/lib/lhm/column_with_sql.rb +90 -0
- data/lib/lhm/column_with_type.rb +31 -0
- data/test_database.rb +52 -0
- metadata +195 -0
@@ -0,0 +1,121 @@
|
|
1
|
+
require 'active_record/connection_adapters/abstract_mysql_adapter'
|
2
|
+
require 'active_record/connection_adapters/statement_pool'
|
3
|
+
require 'active_record/connection_adapters/mysql2_adapter'
|
4
|
+
require 'departure'
|
5
|
+
require 'forwardable'
|
6
|
+
|
7
|
+
module ActiveRecord
|
8
|
+
class Base
|
9
|
+
# Establishes a connection to the database that's used by all Active
|
10
|
+
# Record objects.
|
11
|
+
def self.percona_connection(config)
|
12
|
+
mysql2_connection = mysql2_connection(config)
|
13
|
+
|
14
|
+
verbose = ActiveRecord::Migration.verbose
|
15
|
+
percona_logger = Departure::LoggerFactory.build(verbose: verbose)
|
16
|
+
cli_generator = Departure::CliGenerator.new(config)
|
17
|
+
|
18
|
+
runner = Departure::Runner.new(
|
19
|
+
percona_logger,
|
20
|
+
cli_generator,
|
21
|
+
mysql2_connection
|
22
|
+
)
|
23
|
+
|
24
|
+
connection_options = { mysql_adapter: mysql2_connection }
|
25
|
+
|
26
|
+
ConnectionAdapters::DepartureAdapter.new(
|
27
|
+
runner,
|
28
|
+
logger,
|
29
|
+
connection_options,
|
30
|
+
config
|
31
|
+
)
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
module ConnectionAdapters
|
36
|
+
class DepartureAdapter < AbstractMysqlAdapter
|
37
|
+
|
38
|
+
class Column < AbstractMysqlAdapter::Column
|
39
|
+
def adapter
|
40
|
+
DepartureAdapter
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
extend Forwardable
|
45
|
+
|
46
|
+
ADAPTER_NAME = 'Percona'.freeze
|
47
|
+
|
48
|
+
def_delegators :mysql_adapter, :last_inserted_id, :each_hash
|
49
|
+
|
50
|
+
def initialize(connection, _logger, connection_options, _config)
|
51
|
+
super
|
52
|
+
@visitor = BindSubstitution.new(self)
|
53
|
+
@mysql_adapter = connection_options[:mysql_adapter]
|
54
|
+
end
|
55
|
+
|
56
|
+
def exec_delete(sql, name, binds)
|
57
|
+
execute(to_sql(sql, binds), name)
|
58
|
+
@connection.affected_rows
|
59
|
+
end
|
60
|
+
alias :exec_update :exec_delete
|
61
|
+
|
62
|
+
def exec_insert(sql, name, binds)
|
63
|
+
execute(to_sql(sql, binds), name)
|
64
|
+
end
|
65
|
+
|
66
|
+
def exec_query(sql, name = 'SQL', _binds = [])
|
67
|
+
result = execute(sql, name)
|
68
|
+
ActiveRecord::Result.new(result.fields, result.to_a)
|
69
|
+
end
|
70
|
+
|
71
|
+
# Executes a SELECT query and returns an array of rows. Each row is an
|
72
|
+
# array of field values.
|
73
|
+
def select_rows(sql, name = nil)
|
74
|
+
execute(sql, name).to_a
|
75
|
+
end
|
76
|
+
|
77
|
+
# Executes a SELECT query and returns an array of record hashes with the
|
78
|
+
# column names as keys and column values as values.
|
79
|
+
def select(sql, name = nil, binds = [])
|
80
|
+
exec_query(sql, name, binds).to_a
|
81
|
+
end
|
82
|
+
|
83
|
+
# Returns true, as this adapter supports migrations
|
84
|
+
def supports_migrations?
|
85
|
+
true
|
86
|
+
end
|
87
|
+
|
88
|
+
def new_column(field, default, type, null, collation)
|
89
|
+
Column.new(field, default, type, null, collation)
|
90
|
+
end
|
91
|
+
|
92
|
+
# Adds a new index to the table
|
93
|
+
#
|
94
|
+
# @param table_name [String, Symbol]
|
95
|
+
# @param column_name [String, Symbol]
|
96
|
+
# @param options [Hash] optional
|
97
|
+
def add_index(table_name, column_name, options = {})
|
98
|
+
index_name, index_type, index_columns, index_options = add_index_options(table_name, column_name, options)
|
99
|
+
execute "ALTER TABLE #{quote_table_name(table_name)} ADD #{index_type} INDEX #{quote_column_name(index_name)} (#{index_columns})#{index_options}"
|
100
|
+
end
|
101
|
+
|
102
|
+
# Remove the given index from the table.
|
103
|
+
#
|
104
|
+
# @param table_name [String, Symbol]
|
105
|
+
# @param options [Hash] optional
|
106
|
+
def remove_index(table_name, options = {})
|
107
|
+
index_name = index_name_for_remove(table_name, options)
|
108
|
+
execute "ALTER TABLE #{quote_table_name(table_name)} DROP INDEX #{quote_column_name(index_name)}"
|
109
|
+
end
|
110
|
+
|
111
|
+
# Returns the MySQL error number from the exception. The
|
112
|
+
# AbstractMysqlAdapter requires it to be implemented
|
113
|
+
def error_number(_exception)
|
114
|
+
end
|
115
|
+
|
116
|
+
private
|
117
|
+
|
118
|
+
attr_reader :mysql_adapter
|
119
|
+
end
|
120
|
+
end
|
121
|
+
end
|
data/lib/departure.rb
ADDED
@@ -0,0 +1,65 @@
|
|
1
|
+
require 'active_record'
|
2
|
+
require 'active_support/all'
|
3
|
+
|
4
|
+
require 'departure/version'
|
5
|
+
require 'departure/runner'
|
6
|
+
require 'departure/cli_generator'
|
7
|
+
require 'departure/logger'
|
8
|
+
require 'departure/null_logger'
|
9
|
+
require 'departure/logger_factory'
|
10
|
+
require 'departure/configuration'
|
11
|
+
require 'departure/errors'
|
12
|
+
|
13
|
+
require 'departure/railtie' if defined?(Rails)
|
14
|
+
|
15
|
+
# We need the OS not to buffer the IO to see pt-osc's output while migrating
|
16
|
+
$stdout.sync = true
|
17
|
+
|
18
|
+
module Departure
|
19
|
+
class << self
|
20
|
+
attr_accessor :configuration
|
21
|
+
end
|
22
|
+
|
23
|
+
def self.configure
|
24
|
+
self.configuration ||= Configuration.new
|
25
|
+
yield(configuration)
|
26
|
+
end
|
27
|
+
|
28
|
+
# Hooks Percona Migrator into Rails migrations by replacing the configured
|
29
|
+
# database adapter
|
30
|
+
def self.load
|
31
|
+
ActiveRecord::Migration.class_eval do
|
32
|
+
alias_method :original_migrate, :migrate
|
33
|
+
|
34
|
+
# Replaces the current connection adapter with the PerconaAdapter and
|
35
|
+
# patches LHM, then it continues with the regular migration process.
|
36
|
+
#
|
37
|
+
# @param direction [Symbol] :up or :down
|
38
|
+
def migrate(direction)
|
39
|
+
reconnect_with_percona
|
40
|
+
include_foreigner if defined?(Foreigner)
|
41
|
+
|
42
|
+
::Lhm.migration = self
|
43
|
+
original_migrate(direction)
|
44
|
+
end
|
45
|
+
|
46
|
+
# Includes the Foreigner's Mysql2Adapter implemention in
|
47
|
+
# DepartureAdapter to support foreign keys
|
48
|
+
def include_foreigner
|
49
|
+
Foreigner::Adapter.safe_include(
|
50
|
+
:DepartureAdapter,
|
51
|
+
Foreigner::ConnectionAdapters::Mysql2Adapter
|
52
|
+
)
|
53
|
+
end
|
54
|
+
|
55
|
+
# Make all connections in the connection pool to use PerconaAdapter
|
56
|
+
# instead of the current adapter.
|
57
|
+
def reconnect_with_percona
|
58
|
+
connection_config = ActiveRecord::Base.connection_config
|
59
|
+
ActiveRecord::Base.establish_connection(
|
60
|
+
connection_config.merge(adapter: 'percona')
|
61
|
+
)
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end
|
65
|
+
end
|
@@ -0,0 +1,43 @@
|
|
1
|
+
module Departure
|
2
|
+
class InvalidAlterStatement < StandardError; end
|
3
|
+
|
4
|
+
# Represents the '--alter' argument of Percona's pt-online-schema-change
|
5
|
+
# See https://www.percona.com/doc/percona-toolkit/2.0/pt-online-schema-change.html
|
6
|
+
class AlterArgument
|
7
|
+
ALTER_TABLE_REGEX = /\AALTER TABLE `(\w+)` /
|
8
|
+
|
9
|
+
attr_reader :table_name
|
10
|
+
|
11
|
+
# Constructor
|
12
|
+
#
|
13
|
+
# @param statement [String]
|
14
|
+
# @raise [InvalidAlterStatement] if the statement is not an ALTER TABLE
|
15
|
+
def initialize(statement)
|
16
|
+
@statement = statement
|
17
|
+
|
18
|
+
match = statement.match(ALTER_TABLE_REGEX)
|
19
|
+
raise InvalidAlterStatement unless match
|
20
|
+
|
21
|
+
@table_name = match.captures[0]
|
22
|
+
end
|
23
|
+
|
24
|
+
# Returns the '--alter' pt-online-schema-change argument as a string. See
|
25
|
+
# https://www.percona.com/doc/percona-toolkit/2.0/pt-online-schema-change.html
|
26
|
+
def to_s
|
27
|
+
"--alter \"#{parsed_statement}\""
|
28
|
+
end
|
29
|
+
|
30
|
+
private
|
31
|
+
|
32
|
+
attr_reader :statement
|
33
|
+
|
34
|
+
# Removes the 'ALTER TABLE' portion of the SQL statement
|
35
|
+
#
|
36
|
+
# @return [String]
|
37
|
+
def parsed_statement
|
38
|
+
@parsed_statement ||= statement
|
39
|
+
.gsub(ALTER_TABLE_REGEX, '')
|
40
|
+
.gsub('`','\\\`')
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
@@ -0,0 +1,134 @@
|
|
1
|
+
require 'departure/alter_argument'
|
2
|
+
|
3
|
+
module Departure
|
4
|
+
|
5
|
+
# Represents the 'DSN' argument of Percona's pt-online-schema-change
|
6
|
+
# See https://www.percona.com/doc/percona-toolkit/2.0/pt-online-schema-change.html#dsn-options
|
7
|
+
class DSN
|
8
|
+
|
9
|
+
# Constructor
|
10
|
+
#
|
11
|
+
# @param database [String, Symbol]
|
12
|
+
# @param table_name [String, Symbol]
|
13
|
+
def initialize(database, table_name)
|
14
|
+
@database = database
|
15
|
+
@table_name = table_name
|
16
|
+
end
|
17
|
+
|
18
|
+
# Returns the pt-online-schema-change DSN string. See
|
19
|
+
# https://www.percona.com/doc/percona-toolkit/2.0/pt-online-schema-change.html#dsn-options
|
20
|
+
def to_s
|
21
|
+
"D=#{database},t=#{table_name}"
|
22
|
+
end
|
23
|
+
|
24
|
+
private
|
25
|
+
|
26
|
+
attr_reader :table_name, :database
|
27
|
+
end
|
28
|
+
|
29
|
+
# Generates the equivalent Percona's pt-online-schema-change command to the
|
30
|
+
# given SQL statement
|
31
|
+
class CliGenerator # Command
|
32
|
+
BASE_COMMAND = 'pt-online-schema-change'
|
33
|
+
BASE_OPTIONS = %w(
|
34
|
+
--execute
|
35
|
+
--statistics
|
36
|
+
--recursion-method=none
|
37
|
+
--alter-foreign-keys-method=auto
|
38
|
+
)
|
39
|
+
|
40
|
+
# Constructor
|
41
|
+
#
|
42
|
+
# @param connection_data [Hash]
|
43
|
+
def initialize(connection_data)
|
44
|
+
@connection_data = connection_data
|
45
|
+
init_base_command
|
46
|
+
add_connection_details
|
47
|
+
end
|
48
|
+
|
49
|
+
# Generates the percona command. Fills all the connection credentials from
|
50
|
+
# the current AR connection, but that can be amended via ENV-vars:
|
51
|
+
# PERCONA_DB_HOST, PERCONA_DB_USER, PERCONA_DB_PASSWORD, PERCONA_DB_NAME
|
52
|
+
# Table name can't not be amended, it populates automatically from the
|
53
|
+
# migration data
|
54
|
+
#
|
55
|
+
# @param table_name [String]
|
56
|
+
# @param statement [String] MySQL statement
|
57
|
+
# @return [String]
|
58
|
+
def generate(table_name, statement)
|
59
|
+
alter_argument = AlterArgument.new(statement)
|
60
|
+
dsn = DSN.new(database, table_name)
|
61
|
+
|
62
|
+
"#{self} #{dsn} #{alter_argument}"
|
63
|
+
end
|
64
|
+
|
65
|
+
# Generates the percona command for a raw MySQL statement. Fills all the
|
66
|
+
# connection credentials from the current AR connection, but that can
|
67
|
+
# amended via ENV-vars: PERCONA_DB_HOST, PERCONA_DB_USER,
|
68
|
+
# PERCONA_DB_PASSWORD, PERCONA_DB_NAME Table name can't not be amended, it
|
69
|
+
# populates automatically from the migration data
|
70
|
+
#
|
71
|
+
# @param statement [String] MySQL statement
|
72
|
+
# @return [String]
|
73
|
+
def parse_statement(statement)
|
74
|
+
alter_argument = AlterArgument.new(statement)
|
75
|
+
dsn = DSN.new(database, alter_argument.table_name)
|
76
|
+
|
77
|
+
"#{self} #{dsn} #{alter_argument}"
|
78
|
+
end
|
79
|
+
|
80
|
+
private
|
81
|
+
|
82
|
+
attr_reader :connection_data
|
83
|
+
|
84
|
+
# Sets up the command with its options
|
85
|
+
def init_base_command
|
86
|
+
@command = [BASE_COMMAND, BASE_OPTIONS.join(' ')]
|
87
|
+
end
|
88
|
+
|
89
|
+
# Adds the host, user and password, if present, to the command
|
90
|
+
def add_connection_details
|
91
|
+
@command.push("-h #{host}")
|
92
|
+
@command.push("-u #{user}")
|
93
|
+
@command.push("-p #{password}") if password.present?
|
94
|
+
end
|
95
|
+
|
96
|
+
# Returns the command as a string that can be executed in a shell
|
97
|
+
#
|
98
|
+
# @return [String]
|
99
|
+
def to_s
|
100
|
+
@command.join(' ')
|
101
|
+
end
|
102
|
+
|
103
|
+
# Returns the database host name, defaulting to localhost. If PERCONA_DB_HOST
|
104
|
+
# is passed its value will be used instead
|
105
|
+
#
|
106
|
+
# @return [String]
|
107
|
+
def host
|
108
|
+
ENV['PERCONA_DB_HOST'] || connection_data[:host] || 'localhost'
|
109
|
+
end
|
110
|
+
|
111
|
+
# Returns the database user. If PERCONA_DB_USER is passed its value will be
|
112
|
+
# used instead
|
113
|
+
#
|
114
|
+
# @return [String]
|
115
|
+
def user
|
116
|
+
ENV['PERCONA_DB_USER'] || connection_data[:username]
|
117
|
+
end
|
118
|
+
|
119
|
+
# Returns the database user's password. If PERCONA_DB_PASSWORD is passed its
|
120
|
+
# value will be used instead
|
121
|
+
#
|
122
|
+
# @return [String]
|
123
|
+
def password
|
124
|
+
ENV['PERCONA_DB_PASSWORD'] || connection_data[:password]
|
125
|
+
end
|
126
|
+
|
127
|
+
# TODO: Doesn't the abstract adapter already handle this somehow?
|
128
|
+
# Returns the database name. If PERCONA_DB_NAME is passed its value will be
|
129
|
+
# used instead
|
130
|
+
def database
|
131
|
+
ENV['PERCONA_DB_NAME'] || connection_data[:database]
|
132
|
+
end
|
133
|
+
end
|
134
|
+
end
|
@@ -0,0 +1,18 @@
|
|
1
|
+
module Departure
|
2
|
+
class Configuration
|
3
|
+
attr_accessor :tmp_path
|
4
|
+
|
5
|
+
def initialize
|
6
|
+
@tmp_path = '.'.freeze
|
7
|
+
@error_log_filename = 'departure_error.log'.freeze
|
8
|
+
end
|
9
|
+
|
10
|
+
def error_log_path
|
11
|
+
File.join(tmp_path, error_log_filename)
|
12
|
+
end
|
13
|
+
|
14
|
+
private
|
15
|
+
|
16
|
+
attr_reader :error_log_filename
|
17
|
+
end
|
18
|
+
end
|
@@ -0,0 +1,35 @@
|
|
1
|
+
module Departure
|
2
|
+
class Error < StandardError; end
|
3
|
+
|
4
|
+
# Used when for whatever reason we couldn't get the spawned process'
|
5
|
+
# status.
|
6
|
+
class NoStatusError < Error
|
7
|
+
def message
|
8
|
+
'Status could not be retrieved'.freeze
|
9
|
+
end
|
10
|
+
end
|
11
|
+
|
12
|
+
# Used when the spawned process failed by receiving a signal.
|
13
|
+
# pt-online-schema-change returns "SIGSEGV (signal 11)" on failures.
|
14
|
+
class SignalError < Error
|
15
|
+
attr_reader :status
|
16
|
+
|
17
|
+
# Constructor
|
18
|
+
#
|
19
|
+
# @param status [Process::Status]
|
20
|
+
def initialize(status)
|
21
|
+
super
|
22
|
+
@status = status
|
23
|
+
end
|
24
|
+
|
25
|
+
def message
|
26
|
+
status.to_s
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
class CommandNotFoundError < Error
|
31
|
+
def message
|
32
|
+
'Please install pt-online-schema-change. Check: https://www.percona.com/doc/percona-toolkit for further details'
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
@@ -0,0 +1,31 @@
|
|
1
|
+
module Departure
|
2
|
+
# Copies the ActiveRecord::Migration #say and #write plus a new
|
3
|
+
# #write_no_newline to log the migration's status. It's not possible to reuse
|
4
|
+
# the from ActiveRecord::Migration because the migration's instance can't be
|
5
|
+
# seen from the connection adapter.
|
6
|
+
class Logger
|
7
|
+
|
8
|
+
# Outputs the message through the stdout, following the
|
9
|
+
# ActiveRecord::Migration log format
|
10
|
+
#
|
11
|
+
# @param message [String]
|
12
|
+
# @param subitem [Boolean] whether to show message as a nested log item
|
13
|
+
def say(message, subitem = false)
|
14
|
+
write "#{subitem ? " ->" : "--"} #{message}"
|
15
|
+
end
|
16
|
+
|
17
|
+
# Outputs the text through the stdout adding a new line at the end
|
18
|
+
#
|
19
|
+
# @param text [String]
|
20
|
+
def write(text = '')
|
21
|
+
puts(text)
|
22
|
+
end
|
23
|
+
|
24
|
+
# Outputs the text through the stdout without adding a new line at the end
|
25
|
+
#
|
26
|
+
# @param text [String]
|
27
|
+
def write_no_newline(text)
|
28
|
+
print(text)
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|