mysql_warmer 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- data/.document +5 -0
- data/.gitignore +21 -0
- data/LICENSE +20 -0
- data/README.rdoc +37 -0
- data/Rakefile +55 -0
- data/VERSION +1 -0
- data/bin/mysql_sniff +61 -0
- data/bin/mysql_warmer +72 -0
- data/lib/mysql_warmer.rb +6 -0
- data/lib/mysql_warmer/em/mysql.rb +476 -0
- data/mysql_warmer.gemspec +64 -0
- data/test/helper.rb +10 -0
- data/test/test_mysql_warmer.rb +7 -0
- metadata +100 -0
data/.document
ADDED
data/.gitignore
ADDED
data/LICENSE
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
Copyright (c) 2009 Rohith Ravi
|
2
|
+
|
3
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
4
|
+
a copy of this software and associated documentation files (the
|
5
|
+
"Software"), to deal in the Software without restriction, including
|
6
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
7
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
8
|
+
permit persons to whom the Software is furnished to do so, subject to
|
9
|
+
the following conditions:
|
10
|
+
|
11
|
+
The above copyright notice and this permission notice shall be
|
12
|
+
included in all copies or substantial portions of the Software.
|
13
|
+
|
14
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
15
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
16
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
17
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
18
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
19
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
20
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.rdoc
ADDED
@@ -0,0 +1,37 @@
|
|
1
|
+
= mysql_warmer
|
2
|
+
|
3
|
+
The need for mysql_warmer arose out of having to add a newly seeded MySQL slave in to production rotation. Since the internal buffer pool is empty / not tuned for the queries that will be run on it, queries that would run in less than 1s often took > 1much higher with a cold MySQL cache.
|
4
|
+
|
5
|
+
To alleviate the pain of addding a cold / untuned MySQL machine to the cluster, mysql_warmer simply executes the same queries that are being run on a warm slave, on a cold slave. You're expected to run a query reaper on the slave with some reasonable timeout so that the initial flood of slow queries will not kill the machine. And once enough time has gone by, both the warm slave and the cold slave will start exhibiting identical performance characteristics and then we can give production traffic to the new slave.
|
6
|
+
|
7
|
+
== Install
|
8
|
+
|
9
|
+
mysql_warmer has many dependencies. Make sure your system has libpcap installed with the dev headers.
|
10
|
+
|
11
|
+
$ sudo yum install libpcap libpcap-devel
|
12
|
+
$ gem install mysql_warmer
|
13
|
+
|
14
|
+
== Usage
|
15
|
+
|
16
|
+
On the cold slave:
|
17
|
+
|
18
|
+
$ mysql_warmer -u root -p sekret production_db &
|
19
|
+
$ <start a query reaper with a timeout of 10s or similar>
|
20
|
+
|
21
|
+
On the warm slave:
|
22
|
+
|
23
|
+
$ sudo mysql_sniff <cold_slave_ip>
|
24
|
+
|
25
|
+
== Note on Patches/Pull Requests
|
26
|
+
|
27
|
+
* Fork the project.
|
28
|
+
* Make your feature addition or bug fix.
|
29
|
+
* Add tests for it. This is important so I don't break it in a
|
30
|
+
future version unintentionally.
|
31
|
+
* Commit, do not mess with rakefile, version, or history.
|
32
|
+
(if you want to have your own version, that is fine but bump version in a commit by itself I can ignore when I pull)
|
33
|
+
* Send me a pull request. Bonus points for topic branches.
|
34
|
+
|
35
|
+
== Copyright
|
36
|
+
|
37
|
+
Copyright (c) 2010 Rohith Ravi. See LICENSE for details.
|
data/Rakefile
ADDED
@@ -0,0 +1,55 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'rake'
|
3
|
+
|
4
|
+
begin
|
5
|
+
require 'jeweler'
|
6
|
+
Jeweler::Tasks.new do |gem|
|
7
|
+
gem.name = "mysql_warmer"
|
8
|
+
gem.summary = %Q{Couple command line utilities to warm a cold MySQL database}
|
9
|
+
gem.description = %Q{mysql_warmer warms up a cold database by sniffing SQL queries from a hot machine and then repeating those queries on a cold machine}
|
10
|
+
gem.email = "entombedvirus@gmail.com"
|
11
|
+
gem.homepage = "http://github.com/entombedvirus/mysql_warmer"
|
12
|
+
gem.authors = ["Rohith Ravi", "Steven Lumos"]
|
13
|
+
gem.add_dependency "eventmachine", ">= 0.12.9"
|
14
|
+
gem.add_dependency "mysqlplus", ">= 0.1.0"
|
15
|
+
gem.add_dependency "pcap"
|
16
|
+
# gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
|
17
|
+
end
|
18
|
+
Jeweler::GemcutterTasks.new
|
19
|
+
rescue LoadError
|
20
|
+
puts "Jeweler (or a dependency) not available. Install it with: gem install jeweler"
|
21
|
+
end
|
22
|
+
|
23
|
+
require 'rake/testtask'
|
24
|
+
Rake::TestTask.new(:test) do |test|
|
25
|
+
test.libs << 'lib' << 'test'
|
26
|
+
test.pattern = 'test/**/test_*.rb'
|
27
|
+
test.verbose = true
|
28
|
+
end
|
29
|
+
|
30
|
+
begin
|
31
|
+
require 'rcov/rcovtask'
|
32
|
+
Rcov::RcovTask.new do |test|
|
33
|
+
test.libs << 'test'
|
34
|
+
test.pattern = 'test/**/test_*.rb'
|
35
|
+
test.verbose = true
|
36
|
+
end
|
37
|
+
rescue LoadError
|
38
|
+
task :rcov do
|
39
|
+
abort "RCov is not available. In order to run rcov, you must: sudo gem install spicycode-rcov"
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
task :test => :check_dependencies
|
44
|
+
|
45
|
+
task :default => :test
|
46
|
+
|
47
|
+
require 'rake/rdoctask'
|
48
|
+
Rake::RDocTask.new do |rdoc|
|
49
|
+
version = File.exist?('VERSION') ? File.read('VERSION') : ""
|
50
|
+
|
51
|
+
rdoc.rdoc_dir = 'rdoc'
|
52
|
+
rdoc.title = "mysql_warmer #{version}"
|
53
|
+
rdoc.rdoc_files.include('README*')
|
54
|
+
rdoc.rdoc_files.include('lib/**/*.rb')
|
55
|
+
end
|
data/VERSION
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
0.1.0
|
data/bin/mysql_sniff
ADDED
@@ -0,0 +1,61 @@
|
|
1
|
+
#! /usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'pcaplet'
|
4
|
+
require 'socket'
|
5
|
+
require 'optparse'
|
6
|
+
|
7
|
+
options = {}
|
8
|
+
|
9
|
+
OptionParser.new do |opts|
|
10
|
+
opts.banner = 'Usage: sudo mysql_sniff [OPTIONS] <mysql_warmer_host> [mysql_warmer_port]'
|
11
|
+
|
12
|
+
options[:verbose] = false
|
13
|
+
opts.on('-v', '--verbose', 'Outputs the SQL queries sniffed off the wire') do |verbose|
|
14
|
+
options[:verbose] = true
|
15
|
+
end
|
16
|
+
end.parse!
|
17
|
+
|
18
|
+
|
19
|
+
warmer_host = ARGV.shift
|
20
|
+
warmer_port = (ARGV.shift || 10169).to_i
|
21
|
+
|
22
|
+
if warmer_host.nil?
|
23
|
+
puts "Please specify where the mysql_warmer script is running"
|
24
|
+
return
|
25
|
+
end
|
26
|
+
|
27
|
+
MYSQL_COM_QUERY = 0x03
|
28
|
+
|
29
|
+
mysqlsniff = Pcaplet.new('-s 65535 -i eth0')
|
30
|
+
mysqlsniff.add_filter Pcap::Filter.new('tcp port 3306', mysqlsniff.capture)
|
31
|
+
|
32
|
+
query = ''
|
33
|
+
query_length = 0
|
34
|
+
|
35
|
+
mysqlsniff.each_packet do |pkt|
|
36
|
+
next unless pkt.tcp_data
|
37
|
+
if query then
|
38
|
+
if query.length == query_length then
|
39
|
+
if options[:verbose]
|
40
|
+
puts query
|
41
|
+
puts '-' * 80
|
42
|
+
end
|
43
|
+
|
44
|
+
s = TCPSocket.open(warmer_host, warmer_port)
|
45
|
+
s.write(query)
|
46
|
+
s.close
|
47
|
+
|
48
|
+
query = nil
|
49
|
+
query_length = 0
|
50
|
+
else
|
51
|
+
query << pkt.tcp_data
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
if pkt.tcp_data[3] == 0 and pkt.tcp_data[4] == MYSQL_COM_QUERY then
|
56
|
+
query_length = (pkt.tcp_data[0,3] + "\0").unpack('V')[0] - 1
|
57
|
+
next if query_length < 1
|
58
|
+
query = pkt.tcp_data[5..-1]
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
data/bin/mysql_warmer
ADDED
@@ -0,0 +1,72 @@
|
|
1
|
+
#! /usr/bin/env ruby
|
2
|
+
#
|
3
|
+
|
4
|
+
require 'rubygems'
|
5
|
+
require 'mysql_warmer'
|
6
|
+
require 'optparse'
|
7
|
+
|
8
|
+
options = {}
|
9
|
+
|
10
|
+
OptionParser.new do |opts|
|
11
|
+
opts.banner = "Usage: mysql_warmer [options] <db_name>"
|
12
|
+
|
13
|
+
options[:verbose] = false
|
14
|
+
opts.on('-v', '--verbose', 'Prints queries that are being run to warm up the db') do
|
15
|
+
options[:verbose] = true
|
16
|
+
end
|
17
|
+
|
18
|
+
options[:db_opts] = {}
|
19
|
+
opts.on('-u USER', 'Username to use when connecting to the cold database') do |username|
|
20
|
+
options[:db_opts][:username] = username
|
21
|
+
end
|
22
|
+
opts.on('-p PASSWORD', 'Password to use when connecting to the cold database') do |password|
|
23
|
+
options[:db_opts][:password] = password
|
24
|
+
end
|
25
|
+
opts.on('-h HOST', 'Hostname to use when connecting to the cold database') do |hostname|
|
26
|
+
options[:db_opts][:host] = hostname
|
27
|
+
end
|
28
|
+
|
29
|
+
|
30
|
+
options[:listen_port] = 10169
|
31
|
+
opts.on('-l PORT', "Port to listen on for queries. Default is #{options[:listen_port]}") do |port|
|
32
|
+
options[:listen_port] = port.to_i
|
33
|
+
end
|
34
|
+
end.parse!
|
35
|
+
|
36
|
+
options[:db_opts][:database] = ARGV.shift
|
37
|
+
options[:db_opts].update(:on_error => proc {|e| STDERR.puts "%s - %s" % [e.class.name, e.message]}, :connections => 100)
|
38
|
+
|
39
|
+
if options[:db_opts][:database].nil?
|
40
|
+
puts "Please specify a database to run the queries aganist."
|
41
|
+
return
|
42
|
+
end
|
43
|
+
|
44
|
+
SQL = MysqlWarmer::EventedMysql
|
45
|
+
SQL.settings.update options[:db_opts]
|
46
|
+
|
47
|
+
module MysqlSpammer
|
48
|
+
def initialize(options)
|
49
|
+
@options = options
|
50
|
+
@query = ""
|
51
|
+
end
|
52
|
+
|
53
|
+
def receive_data(data)
|
54
|
+
@query << data
|
55
|
+
end
|
56
|
+
|
57
|
+
def unbind
|
58
|
+
return if @query.empty?
|
59
|
+
|
60
|
+
puts @query.inspect if @options[:verbose]
|
61
|
+
SQL.raw(@query) {}
|
62
|
+
end
|
63
|
+
end
|
64
|
+
|
65
|
+
trap("INT") { EM.stop_event_loop; puts "Exiting..." }
|
66
|
+
|
67
|
+
EM.run do
|
68
|
+
EM.start_server '0.0.0.0', options[:listen_port], MysqlSpammer, options
|
69
|
+
puts "Waiting for queries on port %d..." % options[:listen_port]
|
70
|
+
end
|
71
|
+
|
72
|
+
|
data/lib/mysql_warmer.rb
ADDED
@@ -0,0 +1,476 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'eventmachine'
|
3
|
+
require 'mysqlplus'
|
4
|
+
require 'fcntl'
|
5
|
+
|
6
|
+
class Mysql
|
7
|
+
def result
|
8
|
+
@cur_result
|
9
|
+
end
|
10
|
+
end
|
11
|
+
|
12
|
+
module MysqlWarmer
|
13
|
+
class EventedMysql < EM::Connection
|
14
|
+
def initialize mysql, opts
|
15
|
+
@mysql = mysql
|
16
|
+
@fd = mysql.socket
|
17
|
+
@opts = opts
|
18
|
+
@current = nil
|
19
|
+
@@queue ||= []
|
20
|
+
@processing = false
|
21
|
+
@connected = true
|
22
|
+
|
23
|
+
log 'mysql connected'
|
24
|
+
|
25
|
+
self.notify_readable = true
|
26
|
+
EM.add_timer(0){ next_query }
|
27
|
+
end
|
28
|
+
attr_reader :processing, :connected, :opts
|
29
|
+
alias :settings :opts
|
30
|
+
|
31
|
+
DisconnectErrors = [
|
32
|
+
'query: not connected',
|
33
|
+
'MySQL server has gone away',
|
34
|
+
'Lost connection to MySQL server during query'
|
35
|
+
] unless defined? DisconnectErrors
|
36
|
+
|
37
|
+
def notify_readable
|
38
|
+
log 'readable'
|
39
|
+
if item = @current
|
40
|
+
@current = nil
|
41
|
+
start, response, sql, cblk, eblk = item
|
42
|
+
log 'mysql response', Time.now-start, sql
|
43
|
+
arg = case response
|
44
|
+
when :raw
|
45
|
+
result = @mysql.get_result
|
46
|
+
@mysql.instance_variable_set('@cur_result', result)
|
47
|
+
@mysql
|
48
|
+
when :select
|
49
|
+
ret = []
|
50
|
+
result = @mysql.get_result
|
51
|
+
result.each_hash{|h| ret << h }
|
52
|
+
log 'mysql result', ret
|
53
|
+
ret
|
54
|
+
when :update
|
55
|
+
result = @mysql.get_result
|
56
|
+
@mysql.affected_rows
|
57
|
+
when :insert
|
58
|
+
result = @mysql.get_result
|
59
|
+
@mysql.insert_id
|
60
|
+
else
|
61
|
+
result = @mysql.get_result
|
62
|
+
log 'got a result??', result if result
|
63
|
+
nil
|
64
|
+
end
|
65
|
+
|
66
|
+
@processing = false
|
67
|
+
# result.free if result.is_a? Mysql::Result
|
68
|
+
next_query
|
69
|
+
cblk.call(arg) if cblk
|
70
|
+
else
|
71
|
+
log 'readable, but nothing queued?! probably an ERROR state'
|
72
|
+
return close
|
73
|
+
end
|
74
|
+
rescue Mysql::Error => e
|
75
|
+
log 'mysql error', e.message
|
76
|
+
if e.message =~ /Deadlock/
|
77
|
+
@@queue << [response, sql, cblk, eblk]
|
78
|
+
@processing = false
|
79
|
+
next_query
|
80
|
+
elsif DisconnectErrors.include? e.message
|
81
|
+
@@queue << [response, sql, cblk, eblk]
|
82
|
+
return close
|
83
|
+
elsif cb = (eblk || @opts[:on_error])
|
84
|
+
cb.call(e)
|
85
|
+
@processing = false
|
86
|
+
next_query
|
87
|
+
else
|
88
|
+
raise e
|
89
|
+
end
|
90
|
+
# ensure
|
91
|
+
# res.free if res.is_a? Mysql::Result
|
92
|
+
# @processing = false
|
93
|
+
# next_query
|
94
|
+
end
|
95
|
+
|
96
|
+
def unbind
|
97
|
+
log 'mysql disconnect', $!
|
98
|
+
# cp = EventedMysql.instance_variable_get('@connection_pool') and cp.delete(self)
|
99
|
+
@connected = false
|
100
|
+
|
101
|
+
# XXX wait for the next tick until the current fd is removed completely from the reactor
|
102
|
+
#
|
103
|
+
# XXX in certain cases the new FD# (@mysql.socket) is the same as the old, since FDs are re-used
|
104
|
+
# XXX without next_tick in these cases, unbind will get fired on the newly attached signature as well
|
105
|
+
#
|
106
|
+
# XXX do _NOT_ use EM.next_tick here. if a bunch of sockets disconnect at the same time, we want
|
107
|
+
# XXX reconnects to happen after all the unbinds have been processed
|
108
|
+
EM.add_timer(0) do
|
109
|
+
log 'mysql reconnecting'
|
110
|
+
@processing = false
|
111
|
+
@mysql = EventedMysql._connect @opts
|
112
|
+
@fd = @mysql.socket
|
113
|
+
|
114
|
+
@signature = EM.attach_fd @mysql.socket, true
|
115
|
+
EM.set_notify_readable @signature, true
|
116
|
+
log 'mysql connected'
|
117
|
+
EM.instance_variable_get('@conns')[@signature] = self
|
118
|
+
@connected = true
|
119
|
+
# make_socket_blocking
|
120
|
+
next_query
|
121
|
+
end
|
122
|
+
end
|
123
|
+
|
124
|
+
def execute sql, response = nil, cblk = nil, eblk = nil, &blk
|
125
|
+
cblk ||= blk
|
126
|
+
|
127
|
+
begin
|
128
|
+
unless @processing or !@connected
|
129
|
+
# begin
|
130
|
+
# log 'mysql ping', @mysql.ping
|
131
|
+
# # log 'mysql stat', @mysql.stat
|
132
|
+
# # log 'mysql errno', @mysql.errno
|
133
|
+
# rescue
|
134
|
+
# log 'mysql ping failed'
|
135
|
+
# @@queue << [response, sql, blk]
|
136
|
+
# return close
|
137
|
+
# end
|
138
|
+
|
139
|
+
@processing = true
|
140
|
+
|
141
|
+
log 'mysql sending', sql
|
142
|
+
@mysql.send_query(sql)
|
143
|
+
else
|
144
|
+
@@queue << [response, sql, cblk, eblk]
|
145
|
+
return
|
146
|
+
end
|
147
|
+
rescue Mysql::Error => e
|
148
|
+
log 'mysql error', e.message
|
149
|
+
if DisconnectErrors.include? e.message
|
150
|
+
@@queue << [response, sql, cblk, eblk]
|
151
|
+
return close
|
152
|
+
else
|
153
|
+
raise e
|
154
|
+
end
|
155
|
+
end
|
156
|
+
|
157
|
+
log 'queuing', response, sql
|
158
|
+
@current = [Time.now, response, sql, cblk, eblk]
|
159
|
+
end
|
160
|
+
|
161
|
+
def close
|
162
|
+
@connected = false
|
163
|
+
fd = detach
|
164
|
+
log 'detached fd', fd
|
165
|
+
end
|
166
|
+
|
167
|
+
private
|
168
|
+
|
169
|
+
def next_query
|
170
|
+
if @connected and !@processing and pending = @@queue.shift
|
171
|
+
response, sql, cblk, eblk = pending
|
172
|
+
execute(sql, response, cblk, eblk)
|
173
|
+
end
|
174
|
+
end
|
175
|
+
|
176
|
+
def log *args
|
177
|
+
return unless @opts[:logging]
|
178
|
+
p [Time.now, @fd, (@signature[-4..-1] if @signature), *args]
|
179
|
+
end
|
180
|
+
|
181
|
+
public
|
182
|
+
|
183
|
+
def self.connect opts
|
184
|
+
puts EM.respond_to?(:watch)
|
185
|
+
unless EM.respond_to?(:watch) and Mysql.method_defined?(:socket)
|
186
|
+
raise RuntimeError, 'mysqlplus and EM.watch are required for EventedMysql'
|
187
|
+
end
|
188
|
+
|
189
|
+
if conn = _connect(opts)
|
190
|
+
EM.watch conn.socket, self, conn, opts
|
191
|
+
else
|
192
|
+
EM.add_timer(5){ connect opts }
|
193
|
+
end
|
194
|
+
end
|
195
|
+
|
196
|
+
self::Mysql = ::Mysql unless defined? self::Mysql
|
197
|
+
|
198
|
+
# stolen from sequel
|
199
|
+
def self._connect opts
|
200
|
+
opts = settings.merge(opts)
|
201
|
+
|
202
|
+
conn = Mysql.init
|
203
|
+
|
204
|
+
# set encoding _before_ connecting
|
205
|
+
if charset = opts[:charset] || opts[:encoding]
|
206
|
+
conn.options(Mysql::SET_CHARSET_NAME, charset)
|
207
|
+
end
|
208
|
+
|
209
|
+
conn.options(Mysql::OPT_LOCAL_INFILE, 'client')
|
210
|
+
|
211
|
+
conn.real_connect(
|
212
|
+
opts[:host] || 'localhost',
|
213
|
+
opts[:user] || 'root',
|
214
|
+
opts[:password],
|
215
|
+
opts[:database],
|
216
|
+
opts[:port],
|
217
|
+
opts[:socket],
|
218
|
+
0 +
|
219
|
+
# XXX multi results require multiple callbacks to parse
|
220
|
+
# Mysql::CLIENT_MULTI_RESULTS +
|
221
|
+
# Mysql::CLIENT_MULTI_STATEMENTS +
|
222
|
+
(opts[:compress] == false ? 0 : Mysql::CLIENT_COMPRESS)
|
223
|
+
)
|
224
|
+
|
225
|
+
# increase timeout so mysql server doesn't disconnect us
|
226
|
+
# this is especially bad if we're disconnected while EM.attach is
|
227
|
+
# still in progress, because by the time it gets to EM, the FD is
|
228
|
+
# no longer valid, and it throws a c++ 'bad file descriptor' error
|
229
|
+
# (do not use a timeout of -1 for unlimited, it does not work on mysqld > 5.0.60)
|
230
|
+
conn.query("set @@wait_timeout = #{opts[:timeout] || 2592000}")
|
231
|
+
|
232
|
+
# we handle reconnecting (and reattaching the new fd to EM)
|
233
|
+
conn.reconnect = false
|
234
|
+
|
235
|
+
# By default, MySQL 'where id is null' selects the last inserted id
|
236
|
+
# Turn this off. http://dev.rubyonrails.org/ticket/6778
|
237
|
+
conn.query("set SQL_AUTO_IS_NULL=0")
|
238
|
+
|
239
|
+
# get results for queries
|
240
|
+
conn.query_with_result = true
|
241
|
+
|
242
|
+
conn
|
243
|
+
rescue Mysql::Error => e
|
244
|
+
if cb = opts[:on_error]
|
245
|
+
cb.call(e)
|
246
|
+
nil
|
247
|
+
else
|
248
|
+
raise e
|
249
|
+
end
|
250
|
+
end
|
251
|
+
end
|
252
|
+
|
253
|
+
class EventedMysql
|
254
|
+
def self.settings
|
255
|
+
@settings ||= { :connections => 4, :logging => false }
|
256
|
+
end
|
257
|
+
|
258
|
+
def self.execute query, type = nil, cblk = nil, eblk = nil, &blk
|
259
|
+
unless nil#connection = connection_pool.find{|c| not c.processing and c.connected }
|
260
|
+
@n ||= 0
|
261
|
+
connection = connection_pool[@n]
|
262
|
+
@n = 0 if (@n+=1) >= connection_pool.size
|
263
|
+
end
|
264
|
+
|
265
|
+
connection.execute(query, type, cblk, eblk, &blk)
|
266
|
+
end
|
267
|
+
|
268
|
+
%w[ select insert update raw ].each do |type| class_eval %[
|
269
|
+
|
270
|
+
def self.#{type} query, cblk = nil, eblk = nil, &blk
|
271
|
+
execute query, :#{type}, cblk, eblk, &blk
|
272
|
+
end
|
273
|
+
|
274
|
+
] end
|
275
|
+
|
276
|
+
def self.all query, type = nil, &blk
|
277
|
+
responses = 0
|
278
|
+
connection_pool.each do |c|
|
279
|
+
c.execute(query, type) do
|
280
|
+
responses += 1
|
281
|
+
blk.call if blk and responses == @connection_pool.size
|
282
|
+
end
|
283
|
+
end
|
284
|
+
end
|
285
|
+
|
286
|
+
def self.connection_pool
|
287
|
+
@connection_pool ||= (1..settings[:connections]).map{ EventedMysql.connect(settings) }
|
288
|
+
# p ['connpool', settings[:connections], @connection_pool.size]
|
289
|
+
# (1..(settings[:connections]-@connection_pool.size)).each do
|
290
|
+
# @connection_pool << EventedMysql.connect(settings)
|
291
|
+
# end unless settings[:connections] == @connection_pool.size
|
292
|
+
# @connection_pool
|
293
|
+
end
|
294
|
+
end
|
295
|
+
|
296
|
+
if __FILE__ == $0 and require 'em/spec'
|
297
|
+
|
298
|
+
EM.describe EventedMysql, 'individual connections' do
|
299
|
+
|
300
|
+
should 'create a new connection' do
|
301
|
+
@mysql = EventedMysql.connect :host => '127.0.0.1',
|
302
|
+
:port => 3306,
|
303
|
+
:database => 'test',
|
304
|
+
:logging => false
|
305
|
+
|
306
|
+
@mysql.class.should == EventedMysql
|
307
|
+
done
|
308
|
+
end
|
309
|
+
|
310
|
+
should 'execute sql' do
|
311
|
+
start = Time.now
|
312
|
+
|
313
|
+
@mysql.execute('select sleep(0.2)'){
|
314
|
+
(Time.now-start).should.be.close 0.2, 0.1
|
315
|
+
done
|
316
|
+
}
|
317
|
+
end
|
318
|
+
|
319
|
+
should 'reconnect when disconnected' do
|
320
|
+
@mysql.close
|
321
|
+
@mysql.execute('select 1+2'){
|
322
|
+
:connected.should == :connected
|
323
|
+
done
|
324
|
+
}
|
325
|
+
end
|
326
|
+
|
327
|
+
# to test, run:
|
328
|
+
# mysqladmin5 -u root kill `mysqladmin5 -u root processlist | grep "select sleep(5)+1" | cut -d'|' -f2`
|
329
|
+
#
|
330
|
+
# should 're-run query if disconnected during query' do
|
331
|
+
# @mysql.execute('select sleep(5)+1', :select){ |res|
|
332
|
+
# res.first['sleep(5)+1'].should == '1'
|
333
|
+
# done
|
334
|
+
# }
|
335
|
+
# end
|
336
|
+
|
337
|
+
should 'run select queries and return results' do
|
338
|
+
@mysql.execute('select 1+2', :select){ |res|
|
339
|
+
res.size.should == 1
|
340
|
+
res.first['1+2'].should == '3'
|
341
|
+
done
|
342
|
+
}
|
343
|
+
end
|
344
|
+
|
345
|
+
should 'queue up queries and execute them in order' do
|
346
|
+
@mysql.execute('select 1+2', :select)
|
347
|
+
@mysql.execute('select 2+3', :select)
|
348
|
+
@mysql.execute('select 3+4', :select){ |res|
|
349
|
+
res.first['3+4'].should == '7'
|
350
|
+
done
|
351
|
+
}
|
352
|
+
end
|
353
|
+
|
354
|
+
should 'continue processing queries after hitting an error' do
|
355
|
+
@mysql.settings.update :on_error => proc{|e|}
|
356
|
+
|
357
|
+
@mysql.execute('select 1+ from table'){}
|
358
|
+
@mysql.execute('select 1+1 as num', :select){ |res|
|
359
|
+
res[0]['num'].should == '2'
|
360
|
+
done
|
361
|
+
}
|
362
|
+
end
|
363
|
+
|
364
|
+
should 'have raw mode which yields the mysql object' do
|
365
|
+
@mysql.execute('select 1+2 as num', :raw){ |mysql|
|
366
|
+
mysql.should.is_a? Mysql
|
367
|
+
mysql.result.all_hashes.should == [{'num' => '3'}]
|
368
|
+
done
|
369
|
+
}
|
370
|
+
end
|
371
|
+
|
372
|
+
should 'allow custom error callbacks for each query' do
|
373
|
+
@mysql.settings.update :on_error => proc{ should.flunk('default errback invoked') }
|
374
|
+
|
375
|
+
@mysql.execute('select 1+ from table', :select, proc{
|
376
|
+
should.flunk('callback invoked')
|
377
|
+
}, proc{ |e|
|
378
|
+
done
|
379
|
+
})
|
380
|
+
end
|
381
|
+
|
382
|
+
end
|
383
|
+
|
384
|
+
EM.describe EventedMysql, 'connection pools' do
|
385
|
+
|
386
|
+
EventedMysql.settings.update :connections => 3
|
387
|
+
|
388
|
+
should 'run queries in parallel' do
|
389
|
+
n = 0
|
390
|
+
EventedMysql.select('select sleep(0.25)'){ n+=1 }
|
391
|
+
EventedMysql.select('select sleep(0.25)'){ n+=1 }
|
392
|
+
EventedMysql.select('select sleep(0.25)'){ n+=1 }
|
393
|
+
|
394
|
+
EM.add_timer(0.30){
|
395
|
+
n.should == 3
|
396
|
+
done
|
397
|
+
}
|
398
|
+
end
|
399
|
+
|
400
|
+
end
|
401
|
+
|
402
|
+
SQL = EventedMysql
|
403
|
+
def SQL(query, &blk) SQL.select(query, &blk) end
|
404
|
+
|
405
|
+
# XXX this should get cleaned up automatically after reactor stops
|
406
|
+
SQL.instance_variable_set('@connection_pool', nil)
|
407
|
+
|
408
|
+
|
409
|
+
EM.describe SQL, 'sql api' do
|
410
|
+
|
411
|
+
should 'run a query on all connections' do
|
412
|
+
SQL.all('use test'){
|
413
|
+
:done.should == :done
|
414
|
+
done
|
415
|
+
}
|
416
|
+
end
|
417
|
+
|
418
|
+
should 'execute queries with no results' do
|
419
|
+
SQL.execute('drop table if exists evented_mysql_test'){
|
420
|
+
:table_dropped.should == :table_dropped
|
421
|
+
SQL.execute('create table evented_mysql_test (id int primary key auto_increment, num int not null)'){
|
422
|
+
:table_created.should == :table_created
|
423
|
+
done
|
424
|
+
}
|
425
|
+
}
|
426
|
+
end
|
427
|
+
|
428
|
+
should 'insert rows and return inserted id' do
|
429
|
+
SQL.insert('insert into evented_mysql_test (num) values (10),(11),(12)'){ |id|
|
430
|
+
id.should == 1
|
431
|
+
done
|
432
|
+
}
|
433
|
+
end
|
434
|
+
|
435
|
+
should 'select rows from the database' do
|
436
|
+
SQL.select('select * from evented_mysql_test'){ |res|
|
437
|
+
res.size.should == 3
|
438
|
+
res.first.should == { 'id' => '1', 'num' => '10' }
|
439
|
+
res.last.should == { 'id' => '3', 'num' => '12' }
|
440
|
+
done
|
441
|
+
}
|
442
|
+
end
|
443
|
+
|
444
|
+
should 'update rows and return affected rows' do
|
445
|
+
SQL.update('update evented_mysql_test set num = num + 10'){ |changed|
|
446
|
+
changed.should == 3
|
447
|
+
done
|
448
|
+
}
|
449
|
+
end
|
450
|
+
|
451
|
+
should 'allow access to insert_id in raw mode' do
|
452
|
+
SQL.raw('insert into evented_mysql_test (num) values (20), (21), (22)'){ |mysql|
|
453
|
+
mysql.insert_id.should == 4
|
454
|
+
done
|
455
|
+
}
|
456
|
+
end
|
457
|
+
|
458
|
+
should 'allow access to affected_rows in raw mode' do
|
459
|
+
SQL.raw('update evented_mysql_test set num = num + 10'){ |mysql|
|
460
|
+
mysql.affected_rows.should == 6
|
461
|
+
done
|
462
|
+
}
|
463
|
+
end
|
464
|
+
|
465
|
+
should 'fire error callback with exceptions' do
|
466
|
+
SQL.settings.update :on_error => proc{ |e|
|
467
|
+
e.class.should == Mysql::Error
|
468
|
+
done
|
469
|
+
}
|
470
|
+
SQL.select('select 1+ from table'){}
|
471
|
+
end
|
472
|
+
|
473
|
+
end
|
474
|
+
|
475
|
+
end
|
476
|
+
end
|
@@ -0,0 +1,64 @@
|
|
1
|
+
# Generated by jeweler
|
2
|
+
# DO NOT EDIT THIS FILE DIRECTLY
|
3
|
+
# Instead, edit Jeweler::Tasks in Rakefile, and run the gemspec command
|
4
|
+
# -*- encoding: utf-8 -*-
|
5
|
+
|
6
|
+
Gem::Specification.new do |s|
|
7
|
+
s.name = %q{mysql_warmer}
|
8
|
+
s.version = "0.1.0"
|
9
|
+
|
10
|
+
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
11
|
+
s.authors = ["Rohith Ravi", "Steven Lumos"]
|
12
|
+
s.date = %q{2010-01-21}
|
13
|
+
s.description = %q{mysql_warmer warms up a cold database by sniffing SQL queries from a hot machine and then repeating those queries on a cold machine}
|
14
|
+
s.email = %q{entombedvirus@gmail.com}
|
15
|
+
s.executables = ["mysql_sniff", "mysql_warmer"]
|
16
|
+
s.extra_rdoc_files = [
|
17
|
+
"LICENSE",
|
18
|
+
"README.rdoc"
|
19
|
+
]
|
20
|
+
s.files = [
|
21
|
+
".document",
|
22
|
+
".gitignore",
|
23
|
+
"LICENSE",
|
24
|
+
"README.rdoc",
|
25
|
+
"Rakefile",
|
26
|
+
"VERSION",
|
27
|
+
"bin/mysql_sniff",
|
28
|
+
"bin/mysql_warmer",
|
29
|
+
"lib/mysql_warmer.rb",
|
30
|
+
"lib/mysql_warmer/em/mysql.rb",
|
31
|
+
"mysql_warmer.gemspec",
|
32
|
+
"test/helper.rb",
|
33
|
+
"test/test_mysql_warmer.rb"
|
34
|
+
]
|
35
|
+
s.homepage = %q{http://github.com/entombedvirus/mysql_warmer}
|
36
|
+
s.rdoc_options = ["--charset=UTF-8"]
|
37
|
+
s.require_paths = ["lib"]
|
38
|
+
s.rubygems_version = %q{1.3.5}
|
39
|
+
s.summary = %q{Couple command line utilities to warm a cold MySQL database}
|
40
|
+
s.test_files = [
|
41
|
+
"test/helper.rb",
|
42
|
+
"test/test_mysql_warmer.rb"
|
43
|
+
]
|
44
|
+
|
45
|
+
if s.respond_to? :specification_version then
|
46
|
+
current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
|
47
|
+
s.specification_version = 3
|
48
|
+
|
49
|
+
if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
|
50
|
+
s.add_runtime_dependency(%q<eventmachine>, [">= 0.12.9"])
|
51
|
+
s.add_runtime_dependency(%q<mysqlplus>, [">= 0.1.0"])
|
52
|
+
s.add_runtime_dependency(%q<pcap>, [">= 0"])
|
53
|
+
else
|
54
|
+
s.add_dependency(%q<eventmachine>, [">= 0.12.9"])
|
55
|
+
s.add_dependency(%q<mysqlplus>, [">= 0.1.0"])
|
56
|
+
s.add_dependency(%q<pcap>, [">= 0"])
|
57
|
+
end
|
58
|
+
else
|
59
|
+
s.add_dependency(%q<eventmachine>, [">= 0.12.9"])
|
60
|
+
s.add_dependency(%q<mysqlplus>, [">= 0.1.0"])
|
61
|
+
s.add_dependency(%q<pcap>, [">= 0"])
|
62
|
+
end
|
63
|
+
end
|
64
|
+
|
data/test/helper.rb
ADDED
metadata
ADDED
@@ -0,0 +1,100 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: mysql_warmer
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Rohith Ravi
|
8
|
+
- Steven Lumos
|
9
|
+
autorequire:
|
10
|
+
bindir: bin
|
11
|
+
cert_chain: []
|
12
|
+
|
13
|
+
date: 2010-01-21 00:00:00 -08:00
|
14
|
+
default_executable:
|
15
|
+
dependencies:
|
16
|
+
- !ruby/object:Gem::Dependency
|
17
|
+
name: eventmachine
|
18
|
+
type: :runtime
|
19
|
+
version_requirement:
|
20
|
+
version_requirements: !ruby/object:Gem::Requirement
|
21
|
+
requirements:
|
22
|
+
- - ">="
|
23
|
+
- !ruby/object:Gem::Version
|
24
|
+
version: 0.12.9
|
25
|
+
version:
|
26
|
+
- !ruby/object:Gem::Dependency
|
27
|
+
name: mysqlplus
|
28
|
+
type: :runtime
|
29
|
+
version_requirement:
|
30
|
+
version_requirements: !ruby/object:Gem::Requirement
|
31
|
+
requirements:
|
32
|
+
- - ">="
|
33
|
+
- !ruby/object:Gem::Version
|
34
|
+
version: 0.1.0
|
35
|
+
version:
|
36
|
+
- !ruby/object:Gem::Dependency
|
37
|
+
name: pcap
|
38
|
+
type: :runtime
|
39
|
+
version_requirement:
|
40
|
+
version_requirements: !ruby/object:Gem::Requirement
|
41
|
+
requirements:
|
42
|
+
- - ">="
|
43
|
+
- !ruby/object:Gem::Version
|
44
|
+
version: "0"
|
45
|
+
version:
|
46
|
+
description: mysql_warmer warms up a cold database by sniffing SQL queries from a hot machine and then repeating those queries on a cold machine
|
47
|
+
email: entombedvirus@gmail.com
|
48
|
+
executables:
|
49
|
+
- mysql_sniff
|
50
|
+
- mysql_warmer
|
51
|
+
extensions: []
|
52
|
+
|
53
|
+
extra_rdoc_files:
|
54
|
+
- LICENSE
|
55
|
+
- README.rdoc
|
56
|
+
files:
|
57
|
+
- .document
|
58
|
+
- .gitignore
|
59
|
+
- LICENSE
|
60
|
+
- README.rdoc
|
61
|
+
- Rakefile
|
62
|
+
- VERSION
|
63
|
+
- bin/mysql_sniff
|
64
|
+
- bin/mysql_warmer
|
65
|
+
- lib/mysql_warmer.rb
|
66
|
+
- lib/mysql_warmer/em/mysql.rb
|
67
|
+
- mysql_warmer.gemspec
|
68
|
+
- test/helper.rb
|
69
|
+
- test/test_mysql_warmer.rb
|
70
|
+
has_rdoc: true
|
71
|
+
homepage: http://github.com/entombedvirus/mysql_warmer
|
72
|
+
licenses: []
|
73
|
+
|
74
|
+
post_install_message:
|
75
|
+
rdoc_options:
|
76
|
+
- --charset=UTF-8
|
77
|
+
require_paths:
|
78
|
+
- lib
|
79
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
80
|
+
requirements:
|
81
|
+
- - ">="
|
82
|
+
- !ruby/object:Gem::Version
|
83
|
+
version: "0"
|
84
|
+
version:
|
85
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
86
|
+
requirements:
|
87
|
+
- - ">="
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: "0"
|
90
|
+
version:
|
91
|
+
requirements: []
|
92
|
+
|
93
|
+
rubyforge_project:
|
94
|
+
rubygems_version: 1.3.5
|
95
|
+
signing_key:
|
96
|
+
specification_version: 3
|
97
|
+
summary: Couple command line utilities to warm a cold MySQL database
|
98
|
+
test_files:
|
99
|
+
- test/helper.rb
|
100
|
+
- test/test_mysql_warmer.rb
|