ar_mysql_flexmaster 0.0.4
Sign up to get free protection for your applications and to get access to all the features.
- data/.gitignore +19 -0
- data/Gemfile +6 -0
- data/LICENSE +22 -0
- data/README.md +29 -0
- data/Rakefile +11 -0
- data/ar_mysql_flexmaster.gemspec +22 -0
- data/bin/master_cut +101 -0
- data/lib/active_record/connection_adapters/mysql_flexmaster_adapter.rb +146 -0
- data/lib/ar_mysql_flexmaster.rb +4 -0
- data/test/ar_flexmaster_test.rb +186 -0
- data/test/boot_mysql_env.rb +34 -0
- data/test/boot_slave +16 -0
- data/test/integration/no_traffic_test.rb +23 -0
- data/test/integration/with_queries_to_be_killed_test.rb +37 -0
- data/test/integration/wrong_setup_test.rb +32 -0
- data/test/mysql_isolated_server.rb +114 -0
- metadata +156 -0
data/.gitignore
ADDED
data/Gemfile
ADDED
data/LICENSE
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
Copyright (c) 2012 Ben Osheroff
|
2
|
+
|
3
|
+
MIT License
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
6
|
+
a copy of this software and associated documentation files (the
|
7
|
+
"Software"), to deal in the Software without restriction, including
|
8
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
9
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
10
|
+
permit persons to whom the Software is furnished to do so, subject to
|
11
|
+
the following conditions:
|
12
|
+
|
13
|
+
The above copyright notice and this permission notice shall be
|
14
|
+
included in all copies or substantial portions of the Software.
|
15
|
+
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
17
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
18
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
19
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
20
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
21
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
22
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,29 @@
|
|
1
|
+
# ArMysqlFlexmaster
|
2
|
+
|
3
|
+
TODO: Write a gem description
|
4
|
+
|
5
|
+
## Installation
|
6
|
+
|
7
|
+
Add this line to your application's Gemfile:
|
8
|
+
|
9
|
+
gem 'ar_mysql_flexmaster'
|
10
|
+
|
11
|
+
And then execute:
|
12
|
+
|
13
|
+
$ bundle
|
14
|
+
|
15
|
+
Or install it yourself as:
|
16
|
+
|
17
|
+
$ gem install ar_mysql_flexmaster
|
18
|
+
|
19
|
+
## Usage
|
20
|
+
|
21
|
+
TODO: Write usage instructions here
|
22
|
+
|
23
|
+
## Contributing
|
24
|
+
|
25
|
+
1. Fork it
|
26
|
+
2. Create your feature branch (`git checkout -b my-new-feature`)
|
27
|
+
3. Commit your changes (`git commit -am 'Added some feature'`)
|
28
|
+
4. Push to the branch (`git push origin my-new-feature`)
|
29
|
+
5. Create new Pull Request
|
data/Rakefile
ADDED
@@ -0,0 +1,11 @@
|
|
1
|
+
#!/usr/bin/env rake
|
2
|
+
require 'yaggy'
|
3
|
+
require 'rake/testtask'
|
4
|
+
|
5
|
+
Yaggy.gem(File.expand_path("ar_mysql_flexmaster.gemspec", File.dirname(__FILE__)), :push_gem => true)
|
6
|
+
|
7
|
+
Rake::TestTask.new(:test) do |test|
|
8
|
+
test.libs << 'lib' << 'test'
|
9
|
+
test.pattern = 'test/**/*_test.rb'
|
10
|
+
test.verbose = true
|
11
|
+
end
|
@@ -0,0 +1,22 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
|
3
|
+
Gem::Specification.new do |gem|
|
4
|
+
gem.authors = ["Ben Osheroff"]
|
5
|
+
gem.email = ["ben@zendesk.com"]
|
6
|
+
gem.description = %q{ar_mysql_flexmaster allows configuring N mysql servers in database.yml and auto-selects which is a master at runtime}
|
7
|
+
gem.summary = %q{select a master at runtime from a list}
|
8
|
+
gem.homepage = ""
|
9
|
+
|
10
|
+
gem.files = `git ls-files`.split($\)
|
11
|
+
gem.executables = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
|
12
|
+
gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
|
13
|
+
gem.name = "ar_mysql_flexmaster"
|
14
|
+
gem.require_paths = ["lib"]
|
15
|
+
gem.version = "0.0.4"
|
16
|
+
|
17
|
+
gem.add_runtime_dependency("mysql2")
|
18
|
+
gem.add_runtime_dependency("activerecord")
|
19
|
+
gem.add_runtime_dependency("activesupport")
|
20
|
+
gem.add_development_dependency("appraisal")
|
21
|
+
gem.add_development_dependency("yaggy")
|
22
|
+
end
|
data/bin/master_cut
ADDED
@@ -0,0 +1,101 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'bundler/setup'
|
4
|
+
require 'mysql2'
|
5
|
+
require 'socket'
|
6
|
+
require 'pp'
|
7
|
+
|
8
|
+
Thread.abort_on_exception = false
|
9
|
+
$old_master, $new_master, $username, $password = *ARGV
|
10
|
+
unless $old_master && $new_master && $username && $password
|
11
|
+
puts "Usage: master_cut OLD_MASTER NEW_MASTER USERNAME PASSWORD"
|
12
|
+
exit
|
13
|
+
end
|
14
|
+
|
15
|
+
def open_cx(host)
|
16
|
+
host, port = host.split(":")
|
17
|
+
port = port.to_i if port
|
18
|
+
Mysql2::Client.new(:host => host, :username => $username, :password => $password, :port => port)
|
19
|
+
end
|
20
|
+
|
21
|
+
def set_rw(cx)
|
22
|
+
cx.query("SET GLOBAL READ_ONLY=0")
|
23
|
+
end
|
24
|
+
|
25
|
+
def set_ro(cx)
|
26
|
+
cx.query("SET GLOBAL READ_ONLY=1")
|
27
|
+
end
|
28
|
+
|
29
|
+
$swapped_ok = false
|
30
|
+
|
31
|
+
def fail(reason)
|
32
|
+
puts "Failed preflight check: #{reason}"
|
33
|
+
exit false
|
34
|
+
end
|
35
|
+
|
36
|
+
def preflight_check
|
37
|
+
cx = open_cx($old_master)
|
38
|
+
rw = cx.query("select @@read_only as read_only").first['read_only']
|
39
|
+
fail("old-master #{$old_master} is read-only!") if rw != 0
|
40
|
+
|
41
|
+
slave_cx = open_cx($new_master)
|
42
|
+
rw = slave_cx.query("select @@read_only as read_only").first['read_only']
|
43
|
+
fail("new-master #{$old_master} is read-write!") if rw != 1
|
44
|
+
|
45
|
+
slave_info = slave_cx.query("show slave status").first
|
46
|
+
fail("slave is stopped!") unless slave_info['Slave_IO_Running'] == 'Yes' && slave_info['Slave_SQL_Running'] == 'Yes'
|
47
|
+
fail("slave is delayed") if slave_info['Seconds_Behind_Master'].nil? || slave_info['Seconds_Behind_Master'] > 0
|
48
|
+
|
49
|
+
master_ip, slave_master_ip = [$old_master, slave_info['Master_Host']].map do |h|
|
50
|
+
h = h.split(':').first
|
51
|
+
Socket.gethostbyname(h)[3].unpack("CCCC")
|
52
|
+
end
|
53
|
+
|
54
|
+
if master_ip != slave_master_ip
|
55
|
+
fail("slave does not appear to be replicating off master! (master: #{master_ip.join('.')}, slave's master: #{slave_master_ip.join('.')})")
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
def process_kill_thread
|
60
|
+
Thread.new do
|
61
|
+
cx = open_cx($old_master)
|
62
|
+
sleep 5
|
63
|
+
while !$swapped_ok
|
64
|
+
my_id = cx.query("SELECT CONNECTION_ID() as id").first['id']
|
65
|
+
processlist = cx.query("show processlist")
|
66
|
+
processlist.each do |process|
|
67
|
+
next if process['Info'] =~ /SET GLOBAL READ_ONLY/
|
68
|
+
next if process['Id'].to_i == my_id.to_i
|
69
|
+
puts "killing #{process}"
|
70
|
+
cx.query("kill #{process['Id']}")
|
71
|
+
end
|
72
|
+
sleep 0.1
|
73
|
+
end
|
74
|
+
end
|
75
|
+
end
|
76
|
+
|
77
|
+
def swap_thread
|
78
|
+
Thread.new do
|
79
|
+
master = open_cx($old_master)
|
80
|
+
slave = open_cx($new_master)
|
81
|
+
set_ro(master)
|
82
|
+
slave.query("slave stop")
|
83
|
+
new_master_info = slave.query("show master status").first
|
84
|
+
set_rw(slave)
|
85
|
+
$swapped_ok = true
|
86
|
+
puts "Swapped #{$old_master} and #{$new_master}"
|
87
|
+
puts "New master information at time of swap: "
|
88
|
+
pp new_master_info
|
89
|
+
exit
|
90
|
+
end
|
91
|
+
end
|
92
|
+
|
93
|
+
preflight_check
|
94
|
+
|
95
|
+
threads = []
|
96
|
+
threads << process_kill_thread
|
97
|
+
threads << swap_thread
|
98
|
+
threads.each(&:join)
|
99
|
+
|
100
|
+
|
101
|
+
|
@@ -0,0 +1,146 @@
|
|
1
|
+
require 'active_record/connection_adapters/mysql2_adapter'
|
2
|
+
require 'timeout'
|
3
|
+
|
4
|
+
module ActiveRecord
|
5
|
+
class Base
|
6
|
+
def self.mysql_flexmaster_connection(config)
|
7
|
+
config = config.symbolize_keys
|
8
|
+
|
9
|
+
# fallback to :host or :localhost
|
10
|
+
config[:hosts] ||= config.key?(:host) ? [config[:host]] : ['localhost']
|
11
|
+
|
12
|
+
hosts = config[:hosts] || [config[:host]]
|
13
|
+
|
14
|
+
config[:username] = 'root' if config[:username].nil?
|
15
|
+
|
16
|
+
if Mysql2::Client.const_defined? :FOUND_ROWS
|
17
|
+
config[:flags] = Mysql2::Client::FOUND_ROWS
|
18
|
+
end
|
19
|
+
|
20
|
+
ConnectionAdapters::MysqlFlexmasterAdapter.new(logger, config)
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
module ConnectionAdapters
|
25
|
+
class MysqlFlexmasterAdapter < Mysql2Adapter
|
26
|
+
class NoActiveMasterException < Exception; end
|
27
|
+
|
28
|
+
CHECK_EVERY_N_SELECTS = 10
|
29
|
+
DEFAULT_CONNECT_TIMEOUT = 5
|
30
|
+
DEFAULT_TX_HOLD_TIMEOUT = 5
|
31
|
+
|
32
|
+
def initialize(logger, config)
|
33
|
+
@select_counter = 0
|
34
|
+
@config = config
|
35
|
+
@is_master = !config[:slave]
|
36
|
+
@tx_hold_timeout = @config[:tx_hold_timeout] || DEFAULT_TX_HOLD_TIMEOUT
|
37
|
+
@connection_timeout = @config[:connection_timeout] || DEFAULT_CONNECT_TIMEOUT
|
38
|
+
connection = find_correct_host
|
39
|
+
raise NoActiveMasterException unless connection
|
40
|
+
super(connection, logger, [], config)
|
41
|
+
end
|
42
|
+
|
43
|
+
def begin_db_transaction
|
44
|
+
if !cx_correct? && open_transactions == 0
|
45
|
+
refind_correct_host
|
46
|
+
end
|
47
|
+
super
|
48
|
+
end
|
49
|
+
|
50
|
+
def execute(sql, name = nil)
|
51
|
+
if open_transactions == 0 && sql =~ /^(INSERT|UPDATE|DELETE|ALTER|CHANGE)/ && !cx_correct?
|
52
|
+
refind_correct_host
|
53
|
+
else
|
54
|
+
@select_counter += 1
|
55
|
+
if (@select_counter % CHECK_EVERY_N_SELECTS == 0) && !cx_correct?
|
56
|
+
# on select statements, check every 10 times to see if we need to switch masters,
|
57
|
+
# but don't hold off anything if we fail
|
58
|
+
refind_correct_host(1, 0)
|
59
|
+
end
|
60
|
+
end
|
61
|
+
super
|
62
|
+
end
|
63
|
+
|
64
|
+
private
|
65
|
+
|
66
|
+
def connect
|
67
|
+
@connection = find_correct_host
|
68
|
+
raise NoActiveMasterException unless @connection
|
69
|
+
end
|
70
|
+
|
71
|
+
def refind_correct_host(tries = nil, sleep_interval = nil)
|
72
|
+
tries ||= @tx_hold_timeout.to_f / 0.1
|
73
|
+
sleep_interval ||= 0.1
|
74
|
+
tries.to_i.times do
|
75
|
+
cx = find_correct_host
|
76
|
+
if cx
|
77
|
+
flush_column_information
|
78
|
+
@connection = cx
|
79
|
+
return
|
80
|
+
end
|
81
|
+
sleep(sleep_interval)
|
82
|
+
end
|
83
|
+
raise NoActiveMasterException
|
84
|
+
end
|
85
|
+
|
86
|
+
def hosts_and_ports
|
87
|
+
@hosts_and_ports ||= @config[:hosts].map do |hoststr|
|
88
|
+
host, port = hoststr.split(':')
|
89
|
+
port = port.to_i unless port.nil?
|
90
|
+
[host, port]
|
91
|
+
end
|
92
|
+
end
|
93
|
+
|
94
|
+
def find_correct_host
|
95
|
+
cxs = hosts_and_ports.map do |host, port|
|
96
|
+
initialize_connection(host, port)
|
97
|
+
end
|
98
|
+
|
99
|
+
correct_cxs = cxs.select { |cx| cx && cx_correct?(cx) }
|
100
|
+
|
101
|
+
if @is_master
|
102
|
+
# for master connections, we make damn sure that we have just one master
|
103
|
+
if correct_cxs.size == 1
|
104
|
+
return correct_cxs.first
|
105
|
+
else
|
106
|
+
# nothing read-write, or too many read-write
|
107
|
+
# (should we manually close the connections?)
|
108
|
+
return nil
|
109
|
+
end
|
110
|
+
else
|
111
|
+
# for slave connections, we just return a random RO candidate
|
112
|
+
return correct_cxs.shuffle.first
|
113
|
+
end
|
114
|
+
end
|
115
|
+
|
116
|
+
def initialize_connection(host, port)
|
117
|
+
begin
|
118
|
+
Timeout::timeout(@connection_timeout) do
|
119
|
+
cfg = @config.merge(:host => host, :port => port)
|
120
|
+
Mysql2::Client.new(cfg).tap do |cx|
|
121
|
+
cx.query_options.merge!(:as => :array)
|
122
|
+
end
|
123
|
+
end
|
124
|
+
rescue Mysql2::Error
|
125
|
+
rescue Timeout::Error
|
126
|
+
end
|
127
|
+
end
|
128
|
+
|
129
|
+
def flush_column_information
|
130
|
+
ActiveRecord::Base.descendants.each do |k|
|
131
|
+
k.reset_column_information
|
132
|
+
end
|
133
|
+
end
|
134
|
+
|
135
|
+
def cx_correct?(cx = @connection)
|
136
|
+
res = cx.query("SELECT @@read_only as ro").first
|
137
|
+
|
138
|
+
if @is_master
|
139
|
+
res.first == 0
|
140
|
+
else
|
141
|
+
res.first == 1
|
142
|
+
end
|
143
|
+
end
|
144
|
+
end
|
145
|
+
end
|
146
|
+
end
|
@@ -0,0 +1,186 @@
|
|
1
|
+
require 'bundler/setup'
|
2
|
+
require 'ar_mysql_flexmaster'
|
3
|
+
require 'active_record'
|
4
|
+
require_relative 'boot_mysql_env'
|
5
|
+
require 'test/unit'
|
6
|
+
|
7
|
+
File.open(File.dirname(File.expand_path(__FILE__)) + "/database.yml", "w+") do |f|
|
8
|
+
f.write <<-EOL
|
9
|
+
test:
|
10
|
+
adapter: mysql_flexmaster
|
11
|
+
username: flex
|
12
|
+
hosts: ["127.0.0.1:#{$mysql_master.port}", "127.0.0.1:#{$mysql_slave.port}"]
|
13
|
+
password:
|
14
|
+
database: flexmaster_test
|
15
|
+
|
16
|
+
test_slave:
|
17
|
+
adapter: mysql_flexmaster
|
18
|
+
username: flex
|
19
|
+
slave: true
|
20
|
+
hosts: ["127.0.0.1:#{$mysql_slave.port}", "127.0.0.1:#{$mysql_slave_2.port}"]
|
21
|
+
password:
|
22
|
+
database: flexmaster_test
|
23
|
+
EOL
|
24
|
+
end
|
25
|
+
|
26
|
+
ActiveRecord::Base.configurations = YAML::load(IO.read(File.dirname(__FILE__) + '/database.yml'))
|
27
|
+
ActiveRecord::Base.establish_connection("test")
|
28
|
+
|
29
|
+
class User < ActiveRecord::Base
|
30
|
+
end
|
31
|
+
|
32
|
+
class UserSlave < ActiveRecord::Base
|
33
|
+
establish_connection(:test_slave)
|
34
|
+
set_table_name "users"
|
35
|
+
end
|
36
|
+
|
37
|
+
# $mysql_master and $mysql_slave are separate references to the master and slave that we
|
38
|
+
# use to send control-channel commands on
|
39
|
+
|
40
|
+
class TestArFlexmaster < Test::Unit::TestCase
|
41
|
+
def setup
|
42
|
+
ActiveRecord::Base.establish_connection("test")
|
43
|
+
|
44
|
+
$mysql_master.set_rw(true)
|
45
|
+
$mysql_slave.set_rw(false)
|
46
|
+
$mysql_slave_2.set_rw(false)
|
47
|
+
end
|
48
|
+
|
49
|
+
def test_should_raise_without_a_rw_master
|
50
|
+
[$mysql_master, $mysql_slave].each do |m|
|
51
|
+
m.set_rw(false)
|
52
|
+
end
|
53
|
+
|
54
|
+
assert_raises(ActiveRecord::ConnectionAdapters::MysqlFlexmasterAdapter::NoActiveMasterException) do
|
55
|
+
ActiveRecord::Base.connection
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
def test_should_select_the_master_on_boot
|
60
|
+
assert main_connection_is_master?
|
61
|
+
end
|
62
|
+
|
63
|
+
def test_should_hold_txs_until_timeout_then_abort
|
64
|
+
ActiveRecord::Base.connection
|
65
|
+
|
66
|
+
$mysql_master.set_rw(false)
|
67
|
+
start_time = Time.now.to_i
|
68
|
+
assert_raises(ActiveRecord::ConnectionAdapters::MysqlFlexmasterAdapter::NoActiveMasterException) do
|
69
|
+
User.create(:name => "foo")
|
70
|
+
end
|
71
|
+
end_time = Time.now.to_i
|
72
|
+
assert end_time - start_time >= 5
|
73
|
+
end
|
74
|
+
|
75
|
+
def test_should_hold_txs_and_then_continue
|
76
|
+
ActiveRecord::Base.connection
|
77
|
+
$mysql_master.set_rw(false)
|
78
|
+
Thread.new do
|
79
|
+
sleep 1
|
80
|
+
$mysql_slave.set_rw(true)
|
81
|
+
end
|
82
|
+
User.create(:name => "foo")
|
83
|
+
assert !main_connection_is_master?
|
84
|
+
assert User.first(:conditions => {:name => "foo"})
|
85
|
+
end
|
86
|
+
|
87
|
+
def test_should_hold_implicit_txs_and_then_continue
|
88
|
+
User.create!(:name => "foo")
|
89
|
+
$mysql_master.set_rw(false)
|
90
|
+
Thread.new do
|
91
|
+
sleep 1
|
92
|
+
$mysql_slave.set_rw(true)
|
93
|
+
end
|
94
|
+
User.update_all(:name => "bar")
|
95
|
+
assert !main_connection_is_master?
|
96
|
+
assert_equal "bar", User.first.name
|
97
|
+
end
|
98
|
+
|
99
|
+
def test_should_let_in_flight_txs_crash
|
100
|
+
User.transaction do
|
101
|
+
$mysql_master.set_rw(false)
|
102
|
+
assert_raises(ActiveRecord::StatementInvalid) do
|
103
|
+
User.update_all(:name => "bar")
|
104
|
+
end
|
105
|
+
end
|
106
|
+
end
|
107
|
+
|
108
|
+
def test_should_eventually_pick_up_new_master_on_selects
|
109
|
+
ActiveRecord::Base.connection
|
110
|
+
$mysql_master.set_rw(false)
|
111
|
+
$mysql_slave.set_rw(true)
|
112
|
+
assert main_connection_is_master?
|
113
|
+
100.times do
|
114
|
+
u = User.first
|
115
|
+
end
|
116
|
+
assert !main_connection_is_master?
|
117
|
+
end
|
118
|
+
|
119
|
+
def test_altering_on_the_slave
|
120
|
+
User.create!(:name => "foo")
|
121
|
+
$mysql_slave.connection.query("ALTER TABLE flexmaster_test.users add column xtra int(10)")
|
122
|
+
assert !User.column_names.include?("xtra")
|
123
|
+
$mysql_master.set_rw(false)
|
124
|
+
$mysql_slave.set_rw(true)
|
125
|
+
|
126
|
+
# pick up the new connection and column info
|
127
|
+
User.create!(:name => "foo")
|
128
|
+
|
129
|
+
# now we should be able to use it.
|
130
|
+
assert User.create!(:name => "foobar", :xtra => 5)
|
131
|
+
|
132
|
+
# teardown
|
133
|
+
$mysql_slave.connection.query("ALTER TABLE flexmaster_test.users drop column xtra")
|
134
|
+
User.reset_column_information
|
135
|
+
end
|
136
|
+
|
137
|
+
def test_should_choose_a_random_slave_connection
|
138
|
+
h = {}
|
139
|
+
10.times do
|
140
|
+
port = UserSlave.connection.execute("show global variables like 'port'").first.last.to_i
|
141
|
+
h[port] = 1
|
142
|
+
UserSlave.connection.reconnect!
|
143
|
+
end
|
144
|
+
assert_equal 2, h.size
|
145
|
+
end
|
146
|
+
|
147
|
+
def test_should_flip_the_slave_after_it_becomes_master
|
148
|
+
UserSlave.first
|
149
|
+
User.create!
|
150
|
+
$mysql_master.set_rw(false)
|
151
|
+
$mysql_slave.set_rw(true)
|
152
|
+
11.times do
|
153
|
+
UserSlave.first
|
154
|
+
end
|
155
|
+
assert_equal $mysql_slave_2.port, port_for_class(UserSlave)
|
156
|
+
end
|
157
|
+
|
158
|
+
def test_xxx_non_responsive_master
|
159
|
+
ActiveRecord::Base.configurations["test"]["hosts"] << "127.0.0.2:1235"
|
160
|
+
start_time = Time.now.to_i
|
161
|
+
User.connection.reconnect!
|
162
|
+
assert Time.now.to_i - start_time >= 5
|
163
|
+
ActiveRecord::Base.configurations["test"]["hosts"].pop
|
164
|
+
end
|
165
|
+
|
166
|
+
def test_yyy_shooting_the_master_in_the_head
|
167
|
+
User.create!
|
168
|
+
Process.kill("TERM", $mysql_master.pid)
|
169
|
+
$mysql_slave.set_rw(true)
|
170
|
+
User.connection.reconnect!
|
171
|
+
User.create!
|
172
|
+
UserSlave.first
|
173
|
+
assert !main_connection_is_master?
|
174
|
+
end
|
175
|
+
|
176
|
+
private
|
177
|
+
|
178
|
+
def port_for_class(klass)
|
179
|
+
klass.connection.execute("show global variables like 'port'").first.last.to_i
|
180
|
+
end
|
181
|
+
|
182
|
+
def main_connection_is_master?
|
183
|
+
port = port_for_class(ActiveRecord::Base)
|
184
|
+
port == $mysql_master.port
|
185
|
+
end
|
186
|
+
end
|
@@ -0,0 +1,34 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require_relative "mysql_isolated_server"
|
4
|
+
require 'debugger'
|
5
|
+
|
6
|
+
mysql_master = MysqlIsolatedServer.new
|
7
|
+
mysql_master.boot!
|
8
|
+
mysql_master.connection.query("set global server_id=1")
|
9
|
+
|
10
|
+
puts "mysql master booted on port #{mysql_master.port} -- access with mysql -uroot -h127.0.0.1 --port=#{mysql_master.port} mysql"
|
11
|
+
|
12
|
+
mysql_slave = MysqlIsolatedServer.new
|
13
|
+
mysql_slave.boot!
|
14
|
+
mysql_slave.connection.query("set global server_id=2")
|
15
|
+
|
16
|
+
puts "mysql slave booted on port #{mysql_slave.port} -- access with mysql -uroot -h127.0.0.1 --port=#{mysql_slave.port} mysql"
|
17
|
+
|
18
|
+
mysql_slave_2 = MysqlIsolatedServer.new
|
19
|
+
mysql_slave_2.boot!
|
20
|
+
mysql_slave_2.connection.query("set global server_id=3")
|
21
|
+
|
22
|
+
puts "mysql chained slave booted on port #{mysql_slave_2.port} -- access with mysql -uroot -h127.0.0.1 --port=#{mysql_slave_2.port} mysql"
|
23
|
+
|
24
|
+
mysql_slave.make_slave_of(mysql_master)
|
25
|
+
mysql_slave_2.make_slave_of(mysql_slave)
|
26
|
+
|
27
|
+
mysql_master.connection.query("GRANT ALL ON flexmaster_test.* to flex@localhost")
|
28
|
+
mysql_master.connection.query("CREATE DATABASE flexmaster_test")
|
29
|
+
mysql_master.connection.query("CREATE TABLE flexmaster_test.users (id INT(10) NOT NULL AUTO_INCREMENT PRIMARY KEY, name varchar(20))")
|
30
|
+
mysql_master.connection.query("INSERT INTO flexmaster_test.users set name='foo'")
|
31
|
+
|
32
|
+
$mysql_master = mysql_master
|
33
|
+
$mysql_slave = mysql_slave
|
34
|
+
$mysql_slave_2 = mysql_slave_2
|
data/test/boot_slave
ADDED
@@ -0,0 +1,16 @@
|
|
1
|
+
require_relative 'mysql_isolated_server'
|
2
|
+
|
3
|
+
# yeah, not technically isolated
|
4
|
+
master = MysqlIsolatedServer.new(port: 3306)
|
5
|
+
|
6
|
+
slave = MysqlIsolatedServer.new(data_path: "/Users/ben/.zendesk/var/mysql", allow_output: true, params: "--relay-log=footwa --skip-slave-start", port: 41756)
|
7
|
+
slave.boot!
|
8
|
+
puts "mysql slave booted on port #{slave.port} -- access with mysql -uroot -h127.0.0.1 --port=#{slave.port} mysql"
|
9
|
+
slave.connection.query("set global server_id=123")
|
10
|
+
slave.make_slave_of(master)
|
11
|
+
slave.set_rw(false)
|
12
|
+
|
13
|
+
uid_server = MysqlIsolatedServer.new(data_path: "/Users/ben/.zendesk/var/mysql", allow_output: true, params: "--skip-slave-start", port: 41757)
|
14
|
+
uid_server.boot!
|
15
|
+
puts "mysql uid server booted on port #{uid_server.port} -- access with mysql -uroot -h127.0.0.1 --port=#{uid_server.port} mysql"
|
16
|
+
sleep
|
@@ -0,0 +1,23 @@
|
|
1
|
+
require 'bundler/setup'
|
2
|
+
require 'mysql2'
|
3
|
+
require_relative '../boot_mysql_env'
|
4
|
+
master_cut_script = File.expand_path(File.dirname(__FILE__)) + "/../../bin/master_cut"
|
5
|
+
|
6
|
+
$mysql_master.connection.query("set GLOBAL READ_ONLY=0")
|
7
|
+
$mysql_slave.connection.query("set GLOBAL READ_ONLY=1")
|
8
|
+
|
9
|
+
puts "testing basic cutover..."
|
10
|
+
|
11
|
+
system "#{master_cut_script} 127.0.0.1:#{$mysql_master.port} 127.0.0.1:#{$mysql_slave.port} root ''"
|
12
|
+
if $mysql_master.connection.query("select @@read_only as ro").first['ro'] != 1
|
13
|
+
puts "Master is not readonly!"
|
14
|
+
exit 1
|
15
|
+
end
|
16
|
+
|
17
|
+
if $mysql_slave.connection.query("select @@read_only as ro").first['ro'] != 0
|
18
|
+
puts "Slave is not readwrite!"
|
19
|
+
exit 1
|
20
|
+
end
|
21
|
+
|
22
|
+
puts "everything seemed to go ok..."
|
23
|
+
|
@@ -0,0 +1,37 @@
|
|
1
|
+
require 'bundler/setup'
|
2
|
+
require 'mysql2'
|
3
|
+
require_relative '../boot_mysql_env'
|
4
|
+
master_cut_script = File.expand_path(File.dirname(__FILE__)) + "/../../bin/master_cut"
|
5
|
+
|
6
|
+
puts "testing with long running queries..."
|
7
|
+
|
8
|
+
$mysql_master.connection.query("set GLOBAL READ_ONLY=0")
|
9
|
+
$mysql_slave.connection.query("set GLOBAL READ_ONLY=1")
|
10
|
+
$mysql_master.connection.send(:reconnect=, true)
|
11
|
+
$mysql_slave.connection.send(:reconnect=, true)
|
12
|
+
|
13
|
+
thread = Thread.new {
|
14
|
+
begin
|
15
|
+
$mysql_master.connection.query("update flexmaster_test.users set name=sleep(600)")
|
16
|
+
puts "Query did not get killed! Bad."
|
17
|
+
exit 1
|
18
|
+
rescue Exception => e
|
19
|
+
puts e
|
20
|
+
end
|
21
|
+
}
|
22
|
+
|
23
|
+
system "#{master_cut_script} 127.0.0.1:#{$mysql_master.port} 127.0.0.1:#{$mysql_slave.port} root ''"
|
24
|
+
|
25
|
+
thread.join
|
26
|
+
|
27
|
+
if $mysql_master.connection.query("select @@read_only as ro").first['ro'] != 1
|
28
|
+
puts "Master is not readonly!"
|
29
|
+
exit 1
|
30
|
+
end
|
31
|
+
|
32
|
+
if $mysql_slave.connection.query("select @@read_only as ro").first['ro'] != 0
|
33
|
+
puts "Slave is not readwrite!"
|
34
|
+
exit 1
|
35
|
+
end
|
36
|
+
|
37
|
+
puts "everything seemed to go ok..."
|
@@ -0,0 +1,32 @@
|
|
1
|
+
require 'bundler/setup'
|
2
|
+
require 'mysql2'
|
3
|
+
require_relative '../boot_mysql_env'
|
4
|
+
|
5
|
+
|
6
|
+
def assert_script_failed
|
7
|
+
master_cut_script = File.expand_path(File.dirname(__FILE__)) + "/../../bin/master_cut"
|
8
|
+
if system "#{master_cut_script} 127.0.0.1:#{$mysql_master.port} 127.0.0.1:#{$mysql_slave.port} root ''"
|
9
|
+
puts "Script returned ok instead of false!"
|
10
|
+
exit 1
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
puts "testing cutover with incorrect master config..."
|
15
|
+
$mysql_master.connection.query("set GLOBAL READ_ONLY=0")
|
16
|
+
$mysql_slave.connection.query("set GLOBAL READ_ONLY=0")
|
17
|
+
assert_script_failed
|
18
|
+
|
19
|
+
puts "testing cutover with incorrect slave config..."
|
20
|
+
$mysql_master.connection.query("set GLOBAL READ_ONLY=0")
|
21
|
+
$mysql_slave.connection.query("set GLOBAL READ_ONLY=0")
|
22
|
+
assert_script_failed
|
23
|
+
|
24
|
+
puts "testing cutover with stopped slave"
|
25
|
+
$mysql_master.connection.query("set GLOBAL READ_ONLY=0")
|
26
|
+
$mysql_slave.connection.query("set GLOBAL READ_ONLY=1")
|
27
|
+
$mysql_slave.connection.query("slave stop")
|
28
|
+
assert_script_failed
|
29
|
+
|
30
|
+
|
31
|
+
puts "Tests passed."
|
32
|
+
|
@@ -0,0 +1,114 @@
|
|
1
|
+
require 'tmpdir'
|
2
|
+
require 'socket'
|
3
|
+
require 'mysql2'
|
4
|
+
|
5
|
+
class MysqlIsolatedServer
|
6
|
+
attr_reader :pid, :base, :port
|
7
|
+
MYSQL_BASE_DIR="/usr"
|
8
|
+
|
9
|
+
def initialize(options = {})
|
10
|
+
@base = Dir.mktmpdir("/tmp/mysql_isolated")
|
11
|
+
@mysql_data_dir="#{@base}/mysqld"
|
12
|
+
@mysql_socket="#{@mysql_data_dir}/mysqld.sock"
|
13
|
+
@params = options[:params]
|
14
|
+
@load_data_path = options[:data_path]
|
15
|
+
@port = options[:port]
|
16
|
+
@allow_output = options[:allow_output]
|
17
|
+
end
|
18
|
+
|
19
|
+
def make_slave_of(master)
|
20
|
+
master_binlog_info = master.connection.query("show master status").first
|
21
|
+
connection.query(<<-EOL
|
22
|
+
change master to master_host='127.0.0.1',
|
23
|
+
master_port=#{master.port},
|
24
|
+
master_user='root', master_password='',
|
25
|
+
master_log_file='#{master_binlog_info['File']}',
|
26
|
+
master_log_pos=#{master_binlog_info['Position']}
|
27
|
+
EOL
|
28
|
+
)
|
29
|
+
connection.query("SLAVE START")
|
30
|
+
end
|
31
|
+
|
32
|
+
def connection
|
33
|
+
@cx ||= Mysql2::Client.new(:host => "127.0.0.1", :port => @port, :username => "root", :password => "", :database => "mysql")
|
34
|
+
end
|
35
|
+
|
36
|
+
def set_rw(rw)
|
37
|
+
ro = rw ? 0 : 1
|
38
|
+
connection.query("SET GLOBAL READ_ONLY=#{ro}")
|
39
|
+
end
|
40
|
+
|
41
|
+
|
42
|
+
def boot!
|
43
|
+
@port ||= grab_free_port
|
44
|
+
system("rm -Rf #{@mysql_data_dir}")
|
45
|
+
system("mkdir #{@mysql_data_dir}")
|
46
|
+
if @load_data_path
|
47
|
+
system("cp -a #{@load_data_path}/* #{@mysql_data_dir}")
|
48
|
+
system("rm -f #{@mysql_data_dir}/relay-log.info")
|
49
|
+
else
|
50
|
+
mysql_install_db = `which mysql_install_db`
|
51
|
+
idb_path = File.dirname(mysql_install_db)
|
52
|
+
system("(cd #{idb_path}/..; mysql_install_db --datadir=#{@mysql_data_dir}) >/dev/null 2>&1")
|
53
|
+
end
|
54
|
+
|
55
|
+
exec_server <<-EOL
|
56
|
+
mysqld --no-defaults --default-storage-engine=innodb \
|
57
|
+
--datadir=#{@mysql_data_dir} --pid-file=#{@base}/mysqld.pid --port=#{@port} \
|
58
|
+
#{@params} --socket=#{@mysql_data_dir}/mysql.sock --log-bin --log-slave-updates
|
59
|
+
EOL
|
60
|
+
|
61
|
+
while !system("mysql -h127.0.0.1 --port=#{@port} --database=mysql -u root -e 'select 1' >/dev/null 2>&1")
|
62
|
+
sleep(0.1)
|
63
|
+
end
|
64
|
+
|
65
|
+
system("mysql_tzinfo_to_sql /usr/share/zoneinfo 2>/dev/null | mysql --database=mysql --port=#{@port} -u root mysql >/dev/null")
|
66
|
+
|
67
|
+
system(%Q(mysql --port=#{@port} --database=mysql -u root -e "SET GLOBAL time_zone='UTC'"))
|
68
|
+
system(%Q(mysql --port=#{@port} --database=mysql -u root -e "GRANT SELECT ON *.* to 'zdslave'@'localhost'"))
|
69
|
+
end
|
70
|
+
|
71
|
+
def grab_free_port
|
72
|
+
while true
|
73
|
+
candidate=9000 + rand(50_000)
|
74
|
+
|
75
|
+
begin
|
76
|
+
socket = Socket.new(:INET, :STREAM, 0)
|
77
|
+
socket.bind(Addrinfo.tcp("127.0.0.1", candidate))
|
78
|
+
socket.close
|
79
|
+
return candidate
|
80
|
+
rescue Exception => e
|
81
|
+
puts e
|
82
|
+
end
|
83
|
+
end
|
84
|
+
end
|
85
|
+
|
86
|
+
attr_reader :pid
|
87
|
+
def exec_server(cmd)
|
88
|
+
cmd.strip!
|
89
|
+
cmd.gsub!(/\\\n/, ' ')
|
90
|
+
devnull = File.open("/dev/null", "w")
|
91
|
+
system("mkdir -p #{base}/tmp")
|
92
|
+
system("chmod 0777 #{base}/tmp")
|
93
|
+
pid = fork do
|
94
|
+
ENV["TMPDIR"] = "#{base}/tmp"
|
95
|
+
if !@allow_output
|
96
|
+
STDOUT.reopen(devnull)
|
97
|
+
STDERR.reopen(devnull)
|
98
|
+
end
|
99
|
+
|
100
|
+
exec(cmd)
|
101
|
+
end
|
102
|
+
at_exit {
|
103
|
+
Process.kill("TERM", pid)
|
104
|
+
system("rm -Rf #{base}")
|
105
|
+
}
|
106
|
+
@pid = pid
|
107
|
+
devnull.close
|
108
|
+
end
|
109
|
+
|
110
|
+
def kill!
|
111
|
+
return unless @pid
|
112
|
+
system("kill -KILL #{@pid}")
|
113
|
+
end
|
114
|
+
end
|
metadata
ADDED
@@ -0,0 +1,156 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: ar_mysql_flexmaster
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.4
|
5
|
+
prerelease:
|
6
|
+
platform: ruby
|
7
|
+
authors:
|
8
|
+
- Ben Osheroff
|
9
|
+
autorequire:
|
10
|
+
bindir: bin
|
11
|
+
cert_chain: []
|
12
|
+
date: 2013-01-08 00:00:00.000000000 Z
|
13
|
+
dependencies:
|
14
|
+
- !ruby/object:Gem::Dependency
|
15
|
+
name: mysql2
|
16
|
+
requirement: !ruby/object:Gem::Requirement
|
17
|
+
none: false
|
18
|
+
requirements:
|
19
|
+
- - ! '>='
|
20
|
+
- !ruby/object:Gem::Version
|
21
|
+
version: '0'
|
22
|
+
type: :runtime
|
23
|
+
prerelease: false
|
24
|
+
version_requirements: !ruby/object:Gem::Requirement
|
25
|
+
none: false
|
26
|
+
requirements:
|
27
|
+
- - ! '>='
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
version: '0'
|
30
|
+
- !ruby/object:Gem::Dependency
|
31
|
+
name: activerecord
|
32
|
+
requirement: !ruby/object:Gem::Requirement
|
33
|
+
none: false
|
34
|
+
requirements:
|
35
|
+
- - ! '>='
|
36
|
+
- !ruby/object:Gem::Version
|
37
|
+
version: '0'
|
38
|
+
type: :runtime
|
39
|
+
prerelease: false
|
40
|
+
version_requirements: !ruby/object:Gem::Requirement
|
41
|
+
none: false
|
42
|
+
requirements:
|
43
|
+
- - ! '>='
|
44
|
+
- !ruby/object:Gem::Version
|
45
|
+
version: '0'
|
46
|
+
- !ruby/object:Gem::Dependency
|
47
|
+
name: activesupport
|
48
|
+
requirement: !ruby/object:Gem::Requirement
|
49
|
+
none: false
|
50
|
+
requirements:
|
51
|
+
- - ! '>='
|
52
|
+
- !ruby/object:Gem::Version
|
53
|
+
version: '0'
|
54
|
+
type: :runtime
|
55
|
+
prerelease: false
|
56
|
+
version_requirements: !ruby/object:Gem::Requirement
|
57
|
+
none: false
|
58
|
+
requirements:
|
59
|
+
- - ! '>='
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '0'
|
62
|
+
- !ruby/object:Gem::Dependency
|
63
|
+
name: appraisal
|
64
|
+
requirement: !ruby/object:Gem::Requirement
|
65
|
+
none: false
|
66
|
+
requirements:
|
67
|
+
- - ! '>='
|
68
|
+
- !ruby/object:Gem::Version
|
69
|
+
version: '0'
|
70
|
+
type: :development
|
71
|
+
prerelease: false
|
72
|
+
version_requirements: !ruby/object:Gem::Requirement
|
73
|
+
none: false
|
74
|
+
requirements:
|
75
|
+
- - ! '>='
|
76
|
+
- !ruby/object:Gem::Version
|
77
|
+
version: '0'
|
78
|
+
- !ruby/object:Gem::Dependency
|
79
|
+
name: yaggy
|
80
|
+
requirement: !ruby/object:Gem::Requirement
|
81
|
+
none: false
|
82
|
+
requirements:
|
83
|
+
- - ! '>='
|
84
|
+
- !ruby/object:Gem::Version
|
85
|
+
version: '0'
|
86
|
+
type: :development
|
87
|
+
prerelease: false
|
88
|
+
version_requirements: !ruby/object:Gem::Requirement
|
89
|
+
none: false
|
90
|
+
requirements:
|
91
|
+
- - ! '>='
|
92
|
+
- !ruby/object:Gem::Version
|
93
|
+
version: '0'
|
94
|
+
description: ar_mysql_flexmaster allows configuring N mysql servers in database.yml
|
95
|
+
and auto-selects which is a master at runtime
|
96
|
+
email:
|
97
|
+
- ben@zendesk.com
|
98
|
+
executables:
|
99
|
+
- master_cut
|
100
|
+
extensions: []
|
101
|
+
extra_rdoc_files: []
|
102
|
+
files:
|
103
|
+
- .gitignore
|
104
|
+
- Gemfile
|
105
|
+
- LICENSE
|
106
|
+
- README.md
|
107
|
+
- Rakefile
|
108
|
+
- ar_mysql_flexmaster.gemspec
|
109
|
+
- bin/master_cut
|
110
|
+
- lib/active_record/connection_adapters/mysql_flexmaster_adapter.rb
|
111
|
+
- lib/ar_mysql_flexmaster.rb
|
112
|
+
- test/ar_flexmaster_test.rb
|
113
|
+
- test/boot_mysql_env.rb
|
114
|
+
- test/boot_slave
|
115
|
+
- test/integration/no_traffic_test.rb
|
116
|
+
- test/integration/with_queries_to_be_killed_test.rb
|
117
|
+
- test/integration/wrong_setup_test.rb
|
118
|
+
- test/mysql_isolated_server.rb
|
119
|
+
homepage: ''
|
120
|
+
licenses: []
|
121
|
+
post_install_message:
|
122
|
+
rdoc_options: []
|
123
|
+
require_paths:
|
124
|
+
- lib
|
125
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
126
|
+
none: false
|
127
|
+
requirements:
|
128
|
+
- - ! '>='
|
129
|
+
- !ruby/object:Gem::Version
|
130
|
+
version: '0'
|
131
|
+
segments:
|
132
|
+
- 0
|
133
|
+
hash: -532349908577144552
|
134
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
135
|
+
none: false
|
136
|
+
requirements:
|
137
|
+
- - ! '>='
|
138
|
+
- !ruby/object:Gem::Version
|
139
|
+
version: '0'
|
140
|
+
segments:
|
141
|
+
- 0
|
142
|
+
hash: -532349908577144552
|
143
|
+
requirements: []
|
144
|
+
rubyforge_project:
|
145
|
+
rubygems_version: 1.8.24
|
146
|
+
signing_key:
|
147
|
+
specification_version: 3
|
148
|
+
summary: select a master at runtime from a list
|
149
|
+
test_files:
|
150
|
+
- test/ar_flexmaster_test.rb
|
151
|
+
- test/boot_mysql_env.rb
|
152
|
+
- test/boot_slave
|
153
|
+
- test/integration/no_traffic_test.rb
|
154
|
+
- test/integration/with_queries_to_be_killed_test.rb
|
155
|
+
- test/integration/wrong_setup_test.rb
|
156
|
+
- test/mysql_isolated_server.rb
|