mysqlplus 0.1.1
Sign up to get free protection for your applications and to get access to all the features.
- data/README +40 -0
- data/Rakefile +52 -0
- data/TODO_LIST +13 -0
- data/ext/error_const.h +536 -0
- data/ext/extconf.rb +88 -0
- data/ext/mysql.c +2489 -0
- data/lib/mysqlplus.rb +21 -0
- data/mysqlplus.gemspec +35 -0
- data/test/c_threaded_test.rb +36 -0
- data/test/evented_test.rb +31 -0
- data/test/native_threaded_test.rb +36 -0
- data/test/test_all_hashes.rb +43 -0
- data/test/test_failure.rb +17 -0
- data/test/test_helper.rb +199 -0
- data/test/test_many_requests.rb +7 -0
- data/test/test_parsing_while_response_is_being_read.rb +48 -0
- data/test/test_threaded_sequel.rb +24 -0
- metadata +70 -0
data/lib/mysqlplus.rb
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
require 'mysql' # this should load the mysqlplus version of mysql.so, as we assume the user has installed mysql as a gem and have not done any previous "require 'mysql'" to have loaded the other
|
2
|
+
|
3
|
+
#
|
4
|
+
# Mysqlplus library gives you a [slightly modified] version of the Mysql class
|
5
|
+
# See http://www.kitebird.com/articles/ruby-mysql.html for details, as well as the test directory within the library
|
6
|
+
#
|
7
|
+
class Mysql
|
8
|
+
|
9
|
+
def ruby_async_query(sql, timeout = nil) # known to deadlock TODO
|
10
|
+
send_query(sql)
|
11
|
+
select [ (@sockets ||= {})[socket] ||= IO.new(socket) ], nil, nil, nil
|
12
|
+
get_result
|
13
|
+
end
|
14
|
+
|
15
|
+
begin
|
16
|
+
alias_method :async_query, :c_async_query
|
17
|
+
rescue NameError => e
|
18
|
+
raise LoadError.new "error loading mysqlplus--this may mean you ran a require 'mysql' before a require 'mysqplus', which much come first"
|
19
|
+
end
|
20
|
+
|
21
|
+
end
|
data/mysqlplus.gemspec
ADDED
@@ -0,0 +1,35 @@
|
|
1
|
+
Gem::Specification.new do |s|
|
2
|
+
s.name = "mysqlplus"
|
3
|
+
s.version = "0.1.1"
|
4
|
+
s.date = "2009-03-22"
|
5
|
+
s.summary = "Enhanced Ruby MySQL driver"
|
6
|
+
s.email = "oldmoe@gmail.com"
|
7
|
+
s.homepage = "http://github.com/oldmoe/mysqlplus"
|
8
|
+
s.description = "Enhanced Ruby MySQL driver"
|
9
|
+
s.has_rdoc = true
|
10
|
+
s.authors = ["Muhammad A. Ali"]
|
11
|
+
s.platform = Gem::Platform::RUBY
|
12
|
+
s.files = %w[
|
13
|
+
README
|
14
|
+
Rakefile
|
15
|
+
TODO_LIST
|
16
|
+
ext/error_const.h
|
17
|
+
ext/extconf.rb
|
18
|
+
ext/mysql.c
|
19
|
+
lib/mysqlplus.rb
|
20
|
+
mysqlplus.gemspec
|
21
|
+
test/c_threaded_test.rb
|
22
|
+
test/evented_test.rb
|
23
|
+
test/native_threaded_test.rb
|
24
|
+
test/test_all_hashes.rb
|
25
|
+
test/test_failure.rb
|
26
|
+
test/test_helper.rb
|
27
|
+
test/test_many_requests.rb
|
28
|
+
test/test_parsing_while_response_is_being_read.rb
|
29
|
+
test/test_threaded_sequel.rb
|
30
|
+
]
|
31
|
+
s.rdoc_options = ["--main", "README"]
|
32
|
+
s.extra_rdoc_files = ["README"]
|
33
|
+
s.extensions << "ext/extconf.rb"
|
34
|
+
end
|
35
|
+
|
@@ -0,0 +1,36 @@
|
|
1
|
+
require File.dirname(__FILE__) + '/test_helper'
|
2
|
+
|
3
|
+
ThreadedMysqlTest.new( 10, "Threaded, C, very small overhead" ) do |test|
|
4
|
+
test.setup{ Mysql.real_connect('localhost','root') }
|
5
|
+
test.per_query_overhead = 0.005
|
6
|
+
test.query_with = :c_async_query
|
7
|
+
test.run!
|
8
|
+
end
|
9
|
+
|
10
|
+
ThreadedMysqlTest.new( 10, "Threaded, C, small overhead" ) do |test|
|
11
|
+
test.setup{ Mysql.real_connect('localhost','root') }
|
12
|
+
test.per_query_overhead = 0.1
|
13
|
+
test.query_with = :c_async_query
|
14
|
+
test.run!
|
15
|
+
end
|
16
|
+
|
17
|
+
ThreadedMysqlTest.new( 10, "Threaded, C, medium overhead" ) do |test|
|
18
|
+
test.setup{ Mysql.real_connect('localhost','root') }
|
19
|
+
test.per_query_overhead = 1
|
20
|
+
test.query_with = :c_async_query
|
21
|
+
test.run!
|
22
|
+
end
|
23
|
+
|
24
|
+
ThreadedMysqlTest.new( 10, "Threaded, C, large overhead" ) do |test|
|
25
|
+
test.setup{ Mysql.real_connect('localhost','root') }
|
26
|
+
test.per_query_overhead = 3
|
27
|
+
test.query_with = :c_async_query
|
28
|
+
test.run!
|
29
|
+
end
|
30
|
+
|
31
|
+
ThreadedMysqlTest.new( 10, "Threaded, C, random overhead" ) do |test|
|
32
|
+
test.setup{ Mysql.real_connect('localhost','root') }
|
33
|
+
test.per_query_overhead = :random
|
34
|
+
test.query_with = :c_async_query
|
35
|
+
test.run!
|
36
|
+
end
|
@@ -0,0 +1,31 @@
|
|
1
|
+
require File.dirname(__FILE__) + '/test_helper'
|
2
|
+
|
3
|
+
EventedMysqlTest.new( 10, "Evented, very small overhead" ) do |test|
|
4
|
+
test.setup{ Mysql.real_connect('localhost','root') }
|
5
|
+
test.per_query_overhead = 0.005
|
6
|
+
test.run!
|
7
|
+
end
|
8
|
+
|
9
|
+
EventedMysqlTest.new( 10, "Evented, small overhead" ) do |test|
|
10
|
+
test.setup{ Mysql.real_connect('localhost','root') }
|
11
|
+
test.per_query_overhead = 0.1
|
12
|
+
test.run!
|
13
|
+
end
|
14
|
+
|
15
|
+
EventedMysqlTest.new( 10, "Evented, medium overhead" ) do |test|
|
16
|
+
test.setup{ Mysql.real_connect('localhost','root') }
|
17
|
+
test.per_query_overhead = 1
|
18
|
+
test.run!
|
19
|
+
end
|
20
|
+
|
21
|
+
EventedMysqlTest.new( 10, "Evented, large overhead" ) do |test|
|
22
|
+
test.setup{ Mysql.real_connect('localhost','root') }
|
23
|
+
test.per_query_overhead = 3
|
24
|
+
test.run!
|
25
|
+
end
|
26
|
+
|
27
|
+
EventedMysqlTest.new( 10, "Evented, random overhead" ) do |test|
|
28
|
+
test.setup{ Mysql.real_connect('localhost','root') }
|
29
|
+
test.per_query_overhead = :random
|
30
|
+
test.run!
|
31
|
+
end
|
@@ -0,0 +1,36 @@
|
|
1
|
+
require File.dirname(__FILE__) + '/test_helper'
|
2
|
+
|
3
|
+
ThreadedMysqlTest.new( 10, "Threaded, native Ruby, very small overhead" ) do |test|
|
4
|
+
test.setup{ Mysql.real_connect('localhost','root') }
|
5
|
+
test.per_query_overhead = 0.005
|
6
|
+
test.query_with = :async_query
|
7
|
+
test.run!
|
8
|
+
end
|
9
|
+
|
10
|
+
ThreadedMysqlTest.new( 10, "Threaded, native Ruby, small overhead" ) do |test|
|
11
|
+
test.setup{ Mysql.real_connect('localhost','root') }
|
12
|
+
test.per_query_overhead = 0.1
|
13
|
+
test.query_with = :async_query
|
14
|
+
test.run!
|
15
|
+
end
|
16
|
+
|
17
|
+
ThreadedMysqlTest.new( 10, "Threaded, native Ruby, medium overhead" ) do |test|
|
18
|
+
test.setup{ Mysql.real_connect('localhost','root') }
|
19
|
+
test.per_query_overhead = 1
|
20
|
+
test.query_with = :async_query
|
21
|
+
test.run!
|
22
|
+
end
|
23
|
+
|
24
|
+
ThreadedMysqlTest.new( 10, "Threaded, native Ruby, large overhead" ) do |test|
|
25
|
+
test.setup{ Mysql.real_connect('localhost','root') }
|
26
|
+
test.per_query_overhead = 3
|
27
|
+
test.query_with = :async_query
|
28
|
+
test.run!
|
29
|
+
end
|
30
|
+
|
31
|
+
ThreadedMysqlTest.new( 10, "Threaded, native Ruby, random overhead" ) do |test|
|
32
|
+
test.setup{ Mysql.real_connect('localhost','root') }
|
33
|
+
test.per_query_overhead = :random
|
34
|
+
test.query_with = :async_query
|
35
|
+
test.run!
|
36
|
+
end
|
@@ -0,0 +1,43 @@
|
|
1
|
+
# shows the effect of using .all_hashes instead of looping on each hash
|
2
|
+
# run it by substiting in a 'long' [many row] query for the query variable and toggling use_all_hashes here at the top
|
3
|
+
# note that we load all the rows first, then run .all_hashes on the result [to see more easily the effect of all hashes]
|
4
|
+
# on my machine and a 200_000 row table, it took 3.38s versus 3.65s
|
5
|
+
require 'rubygems'
|
6
|
+
require 'mysqlplus'
|
7
|
+
|
8
|
+
use_the_all_hashes_method = true
|
9
|
+
|
10
|
+
$count = 5
|
11
|
+
|
12
|
+
$start = Time.now
|
13
|
+
|
14
|
+
$connections = []
|
15
|
+
$count.times do
|
16
|
+
$connections << Mysql.real_connect('localhost','root', '', 'local_leadgen_dev')
|
17
|
+
end
|
18
|
+
|
19
|
+
puts 'connection pool ready'
|
20
|
+
|
21
|
+
$threads = []
|
22
|
+
$count.times do |i|
|
23
|
+
$threads << Thread.new do
|
24
|
+
|
25
|
+
query = "select * from campus_zips"
|
26
|
+
puts "sending query on connection #{i}"
|
27
|
+
conn = $connections[i]
|
28
|
+
result = conn.async_query(query)
|
29
|
+
if use_the_all_hashes_method
|
30
|
+
saved = result.all_hashes
|
31
|
+
else
|
32
|
+
saved = []
|
33
|
+
result.each_hash {|h| saved << h }
|
34
|
+
end
|
35
|
+
result.free
|
36
|
+
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
puts 'waiting on threads'
|
41
|
+
$threads.each{|t| t.join }
|
42
|
+
|
43
|
+
puts Time.now - $start
|
@@ -0,0 +1,17 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'mysqlplus'
|
3
|
+
begin
|
4
|
+
Mysql.real_connect('fakehost','root', '', 'local_leadgen_dev')
|
5
|
+
rescue Mysql::Error
|
6
|
+
end
|
7
|
+
begin
|
8
|
+
Mysql.real_connect('localhost','root', '', 'faketable')
|
9
|
+
rescue Mysql::Error
|
10
|
+
end
|
11
|
+
begin
|
12
|
+
Mysql.real_connect('localhost', 'root', 'pass', 'db', 3307)# bad port
|
13
|
+
rescue Mysql::Error
|
14
|
+
end
|
15
|
+
print "pass"
|
16
|
+
|
17
|
+
|
data/test/test_helper.rb
ADDED
@@ -0,0 +1,199 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'mysqlplus'
|
3
|
+
|
4
|
+
class MysqlTest
|
5
|
+
|
6
|
+
class NotImplemented < StandardError
|
7
|
+
end
|
8
|
+
|
9
|
+
attr_accessor :queries,
|
10
|
+
:context,
|
11
|
+
:connections,
|
12
|
+
:connection_signature,
|
13
|
+
:start,
|
14
|
+
:done,
|
15
|
+
:query_with,
|
16
|
+
:per_query_overhead,
|
17
|
+
:timeout
|
18
|
+
|
19
|
+
def initialize( queries, context = '' )
|
20
|
+
@queries = queries
|
21
|
+
@context = context
|
22
|
+
@done = []
|
23
|
+
@query_with = :async_query
|
24
|
+
@per_query_overhead = 3
|
25
|
+
@timeout = 20
|
26
|
+
yield self if block_given?
|
27
|
+
end
|
28
|
+
|
29
|
+
def setup( &block )
|
30
|
+
@start = Time.now
|
31
|
+
@connection_signature = block
|
32
|
+
end
|
33
|
+
|
34
|
+
def run!
|
35
|
+
c_or_native_ruby_async_query do
|
36
|
+
present_context if context?
|
37
|
+
prepare
|
38
|
+
yield
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
def per_query_overhead=( overhead )
|
43
|
+
@per_query_overhead = ( overhead == :random ) ? rand() : overhead
|
44
|
+
end
|
45
|
+
|
46
|
+
protected
|
47
|
+
|
48
|
+
def prepare
|
49
|
+
raise NotImplemented
|
50
|
+
end
|
51
|
+
|
52
|
+
def teardown
|
53
|
+
raise NotImplemented
|
54
|
+
end
|
55
|
+
|
56
|
+
def log( message, prefix = '' )
|
57
|
+
puts "[#{timestamp}] #{prefix} #{message}"
|
58
|
+
end
|
59
|
+
|
60
|
+
def with_logging( message )
|
61
|
+
log( message, 'Start' )
|
62
|
+
yield
|
63
|
+
log( message, 'End' )
|
64
|
+
end
|
65
|
+
|
66
|
+
def timestamp
|
67
|
+
Time.now - @start
|
68
|
+
end
|
69
|
+
|
70
|
+
def context?
|
71
|
+
@context != ''
|
72
|
+
end
|
73
|
+
|
74
|
+
def present_context
|
75
|
+
log "#############################################"
|
76
|
+
log "# #{@context}"
|
77
|
+
log "#############################################"
|
78
|
+
end
|
79
|
+
|
80
|
+
def c_or_native_ruby_async_query
|
81
|
+
if @query_with == :c_async_query
|
82
|
+
log "** using C based async_query"
|
83
|
+
else
|
84
|
+
log "** using native Ruby async_query"
|
85
|
+
end
|
86
|
+
yield
|
87
|
+
end
|
88
|
+
|
89
|
+
def dispatch_query( connection, sql, timeout = nil )
|
90
|
+
connection.send( @query_with, sql, timeout )
|
91
|
+
end
|
92
|
+
|
93
|
+
end
|
94
|
+
|
95
|
+
class EventedMysqlTest < MysqlTest
|
96
|
+
|
97
|
+
attr_accessor :sockets
|
98
|
+
|
99
|
+
def initialize( queries, context = '' )
|
100
|
+
@sockets = []
|
101
|
+
@connections = {}
|
102
|
+
super( queries, context )
|
103
|
+
end
|
104
|
+
|
105
|
+
def setup( &block )
|
106
|
+
super( &block )
|
107
|
+
with_logging 'Setup connection pool' do
|
108
|
+
@queries.times do
|
109
|
+
connection = @connection_signature.call
|
110
|
+
@connections[ IO.new(connection.socket) ] = connection
|
111
|
+
@sockets = @connections.keys
|
112
|
+
end
|
113
|
+
end
|
114
|
+
end
|
115
|
+
|
116
|
+
def run!
|
117
|
+
super do
|
118
|
+
catch :END_EVENT_LOOP do
|
119
|
+
loop do
|
120
|
+
result = select( @sockets,nil,nil,nil )
|
121
|
+
if result
|
122
|
+
result.first.each do |conn|
|
123
|
+
@connections[conn].get_result.each{|res| log( "Result for socket #{conn.fileno} : #{res}" ) }
|
124
|
+
@done << nil
|
125
|
+
if done?
|
126
|
+
teardown
|
127
|
+
end
|
128
|
+
end
|
129
|
+
end
|
130
|
+
end
|
131
|
+
end
|
132
|
+
end
|
133
|
+
end
|
134
|
+
|
135
|
+
protected
|
136
|
+
|
137
|
+
def prepare
|
138
|
+
@connections.each_value do |conn|
|
139
|
+
conn.send_query( "select sleep(#{@per_query_overhead})" )
|
140
|
+
end
|
141
|
+
end
|
142
|
+
|
143
|
+
def teardown
|
144
|
+
log "done"
|
145
|
+
throw :END_EVENT_LOOP
|
146
|
+
end
|
147
|
+
|
148
|
+
def done?
|
149
|
+
@done.size == @queries
|
150
|
+
end
|
151
|
+
|
152
|
+
end
|
153
|
+
|
154
|
+
class ThreadedMysqlTest < MysqlTest
|
155
|
+
|
156
|
+
attr_accessor :threads
|
157
|
+
|
158
|
+
def initialize( queries, context = '' )
|
159
|
+
@connections = []
|
160
|
+
@threads = []
|
161
|
+
super( queries, context )
|
162
|
+
end
|
163
|
+
|
164
|
+
def setup( &block )
|
165
|
+
super( &block )
|
166
|
+
with_logging "Setup connection pool" do
|
167
|
+
@queries.times do
|
168
|
+
@connections << @connection_signature.call
|
169
|
+
end
|
170
|
+
end
|
171
|
+
end
|
172
|
+
|
173
|
+
def run!
|
174
|
+
super do
|
175
|
+
with_logging "waiting on threads" do
|
176
|
+
@threads.each{|t| t.join }
|
177
|
+
end
|
178
|
+
end
|
179
|
+
end
|
180
|
+
|
181
|
+
protected
|
182
|
+
|
183
|
+
def prepare
|
184
|
+
with_logging "prepare" do
|
185
|
+
@queries.times do |conn|
|
186
|
+
@threads << Thread.new do
|
187
|
+
|
188
|
+
log "sending query on connection #{conn}"
|
189
|
+
|
190
|
+
dispatch_query( @connections[conn], "select sleep(#{@per_query_overhead})", @timeout ).each do |result|
|
191
|
+
log "connection #{conn} done"
|
192
|
+
end
|
193
|
+
|
194
|
+
end
|
195
|
+
end
|
196
|
+
end
|
197
|
+
end
|
198
|
+
|
199
|
+
end
|
@@ -0,0 +1,48 @@
|
|
1
|
+
# This is an example of using Mysql::ResultSet#use_result [see docs for what that does]
|
2
|
+
# this function is useful for those who have large query results and want to be able to parse them
|
3
|
+
# as they come in, instead of having to wait for the query to finish before doing parsing
|
4
|
+
# for me, running this on a query with 200_000 lines decreases total time to create an array of results
|
5
|
+
# from .82s to .62s
|
6
|
+
# you can experiment with it by changing the query here to be a long one, and toggling the do_the_use_query_optimization variable
|
7
|
+
# this also has the interesting property of 'freeing' Ruby to do thread changes mid-query.
|
8
|
+
require 'rubygems'
|
9
|
+
require 'mysqlplus'
|
10
|
+
|
11
|
+
do_the_use_query_optimization = true
|
12
|
+
|
13
|
+
$count = 5
|
14
|
+
|
15
|
+
$start = Time.now
|
16
|
+
|
17
|
+
$connections = []
|
18
|
+
$count.times do
|
19
|
+
$connections << Mysql.real_connect('localhost','root', '', 'local_leadgen_dev')
|
20
|
+
end
|
21
|
+
|
22
|
+
puts 'connection pool ready'
|
23
|
+
|
24
|
+
$threads = []
|
25
|
+
$count.times do |i|
|
26
|
+
$threads << Thread.new do
|
27
|
+
|
28
|
+
puts "sending query on connection #{i}"
|
29
|
+
conn = $connections[i]
|
30
|
+
saved = []
|
31
|
+
query = "select * from campus_zips"
|
32
|
+
if do_the_use_query_optimization
|
33
|
+
conn.query_with_result=false
|
34
|
+
result = conn.async_query(query)
|
35
|
+
res = result.use_result
|
36
|
+
res.each_hash { |h| saved << h }
|
37
|
+
res.free
|
38
|
+
else
|
39
|
+
conn.async_query(query).each_hash {|h| saved << h }
|
40
|
+
end
|
41
|
+
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
puts 'waiting on threads'
|
46
|
+
$threads.each{|t| t.join }
|
47
|
+
|
48
|
+
puts Time.now - $start
|