mysqlplus 0.1.1

Sign up to get free protection for your applications and to get access to all the features.
data/lib/mysqlplus.rb ADDED
@@ -0,0 +1,21 @@
1
+ require 'mysql' # this should load the mysqlplus version of mysql.so, as we assume the user has installed mysql as a gem and have not done any previous "require 'mysql'" to have loaded the other
2
+
3
+ #
4
+ # Mysqlplus library gives you a [slightly modified] version of the Mysql class
5
+ # See http://www.kitebird.com/articles/ruby-mysql.html for details, as well as the test directory within the library
6
+ #
7
+ class Mysql
8
+
9
+ def ruby_async_query(sql, timeout = nil) # known to deadlock TODO
10
+ send_query(sql)
11
+ select [ (@sockets ||= {})[socket] ||= IO.new(socket) ], nil, nil, nil
12
+ get_result
13
+ end
14
+
15
+ begin
16
+ alias_method :async_query, :c_async_query
17
+ rescue NameError => e
18
+ raise LoadError.new "error loading mysqlplus--this may mean you ran a require 'mysql' before a require 'mysqplus', which much come first"
19
+ end
20
+
21
+ end
data/mysqlplus.gemspec ADDED
@@ -0,0 +1,35 @@
1
+ Gem::Specification.new do |s|
2
+ s.name = "mysqlplus"
3
+ s.version = "0.1.1"
4
+ s.date = "2009-03-22"
5
+ s.summary = "Enhanced Ruby MySQL driver"
6
+ s.email = "oldmoe@gmail.com"
7
+ s.homepage = "http://github.com/oldmoe/mysqlplus"
8
+ s.description = "Enhanced Ruby MySQL driver"
9
+ s.has_rdoc = true
10
+ s.authors = ["Muhammad A. Ali"]
11
+ s.platform = Gem::Platform::RUBY
12
+ s.files = %w[
13
+ README
14
+ Rakefile
15
+ TODO_LIST
16
+ ext/error_const.h
17
+ ext/extconf.rb
18
+ ext/mysql.c
19
+ lib/mysqlplus.rb
20
+ mysqlplus.gemspec
21
+ test/c_threaded_test.rb
22
+ test/evented_test.rb
23
+ test/native_threaded_test.rb
24
+ test/test_all_hashes.rb
25
+ test/test_failure.rb
26
+ test/test_helper.rb
27
+ test/test_many_requests.rb
28
+ test/test_parsing_while_response_is_being_read.rb
29
+ test/test_threaded_sequel.rb
30
+ ]
31
+ s.rdoc_options = ["--main", "README"]
32
+ s.extra_rdoc_files = ["README"]
33
+ s.extensions << "ext/extconf.rb"
34
+ end
35
+
@@ -0,0 +1,36 @@
1
+ require File.dirname(__FILE__) + '/test_helper'
2
+
3
+ ThreadedMysqlTest.new( 10, "Threaded, C, very small overhead" ) do |test|
4
+ test.setup{ Mysql.real_connect('localhost','root') }
5
+ test.per_query_overhead = 0.005
6
+ test.query_with = :c_async_query
7
+ test.run!
8
+ end
9
+
10
+ ThreadedMysqlTest.new( 10, "Threaded, C, small overhead" ) do |test|
11
+ test.setup{ Mysql.real_connect('localhost','root') }
12
+ test.per_query_overhead = 0.1
13
+ test.query_with = :c_async_query
14
+ test.run!
15
+ end
16
+
17
+ ThreadedMysqlTest.new( 10, "Threaded, C, medium overhead" ) do |test|
18
+ test.setup{ Mysql.real_connect('localhost','root') }
19
+ test.per_query_overhead = 1
20
+ test.query_with = :c_async_query
21
+ test.run!
22
+ end
23
+
24
+ ThreadedMysqlTest.new( 10, "Threaded, C, large overhead" ) do |test|
25
+ test.setup{ Mysql.real_connect('localhost','root') }
26
+ test.per_query_overhead = 3
27
+ test.query_with = :c_async_query
28
+ test.run!
29
+ end
30
+
31
+ ThreadedMysqlTest.new( 10, "Threaded, C, random overhead" ) do |test|
32
+ test.setup{ Mysql.real_connect('localhost','root') }
33
+ test.per_query_overhead = :random
34
+ test.query_with = :c_async_query
35
+ test.run!
36
+ end
@@ -0,0 +1,31 @@
1
+ require File.dirname(__FILE__) + '/test_helper'
2
+
3
+ EventedMysqlTest.new( 10, "Evented, very small overhead" ) do |test|
4
+ test.setup{ Mysql.real_connect('localhost','root') }
5
+ test.per_query_overhead = 0.005
6
+ test.run!
7
+ end
8
+
9
+ EventedMysqlTest.new( 10, "Evented, small overhead" ) do |test|
10
+ test.setup{ Mysql.real_connect('localhost','root') }
11
+ test.per_query_overhead = 0.1
12
+ test.run!
13
+ end
14
+
15
+ EventedMysqlTest.new( 10, "Evented, medium overhead" ) do |test|
16
+ test.setup{ Mysql.real_connect('localhost','root') }
17
+ test.per_query_overhead = 1
18
+ test.run!
19
+ end
20
+
21
+ EventedMysqlTest.new( 10, "Evented, large overhead" ) do |test|
22
+ test.setup{ Mysql.real_connect('localhost','root') }
23
+ test.per_query_overhead = 3
24
+ test.run!
25
+ end
26
+
27
+ EventedMysqlTest.new( 10, "Evented, random overhead" ) do |test|
28
+ test.setup{ Mysql.real_connect('localhost','root') }
29
+ test.per_query_overhead = :random
30
+ test.run!
31
+ end
@@ -0,0 +1,36 @@
1
+ require File.dirname(__FILE__) + '/test_helper'
2
+
3
+ ThreadedMysqlTest.new( 10, "Threaded, native Ruby, very small overhead" ) do |test|
4
+ test.setup{ Mysql.real_connect('localhost','root') }
5
+ test.per_query_overhead = 0.005
6
+ test.query_with = :async_query
7
+ test.run!
8
+ end
9
+
10
+ ThreadedMysqlTest.new( 10, "Threaded, native Ruby, small overhead" ) do |test|
11
+ test.setup{ Mysql.real_connect('localhost','root') }
12
+ test.per_query_overhead = 0.1
13
+ test.query_with = :async_query
14
+ test.run!
15
+ end
16
+
17
+ ThreadedMysqlTest.new( 10, "Threaded, native Ruby, medium overhead" ) do |test|
18
+ test.setup{ Mysql.real_connect('localhost','root') }
19
+ test.per_query_overhead = 1
20
+ test.query_with = :async_query
21
+ test.run!
22
+ end
23
+
24
+ ThreadedMysqlTest.new( 10, "Threaded, native Ruby, large overhead" ) do |test|
25
+ test.setup{ Mysql.real_connect('localhost','root') }
26
+ test.per_query_overhead = 3
27
+ test.query_with = :async_query
28
+ test.run!
29
+ end
30
+
31
+ ThreadedMysqlTest.new( 10, "Threaded, native Ruby, random overhead" ) do |test|
32
+ test.setup{ Mysql.real_connect('localhost','root') }
33
+ test.per_query_overhead = :random
34
+ test.query_with = :async_query
35
+ test.run!
36
+ end
@@ -0,0 +1,43 @@
1
+ # shows the effect of using .all_hashes instead of looping on each hash
2
+ # run it by substiting in a 'long' [many row] query for the query variable and toggling use_all_hashes here at the top
3
+ # note that we load all the rows first, then run .all_hashes on the result [to see more easily the effect of all hashes]
4
+ # on my machine and a 200_000 row table, it took 3.38s versus 3.65s
5
+ require 'rubygems'
6
+ require 'mysqlplus'
7
+
8
+ use_the_all_hashes_method = true
9
+
10
+ $count = 5
11
+
12
+ $start = Time.now
13
+
14
+ $connections = []
15
+ $count.times do
16
+ $connections << Mysql.real_connect('localhost','root', '', 'local_leadgen_dev')
17
+ end
18
+
19
+ puts 'connection pool ready'
20
+
21
+ $threads = []
22
+ $count.times do |i|
23
+ $threads << Thread.new do
24
+
25
+ query = "select * from campus_zips"
26
+ puts "sending query on connection #{i}"
27
+ conn = $connections[i]
28
+ result = conn.async_query(query)
29
+ if use_the_all_hashes_method
30
+ saved = result.all_hashes
31
+ else
32
+ saved = []
33
+ result.each_hash {|h| saved << h }
34
+ end
35
+ result.free
36
+
37
+ end
38
+ end
39
+
40
+ puts 'waiting on threads'
41
+ $threads.each{|t| t.join }
42
+
43
+ puts Time.now - $start
@@ -0,0 +1,17 @@
1
+ require 'rubygems'
2
+ require 'mysqlplus'
3
+ begin
4
+ Mysql.real_connect('fakehost','root', '', 'local_leadgen_dev')
5
+ rescue Mysql::Error
6
+ end
7
+ begin
8
+ Mysql.real_connect('localhost','root', '', 'faketable')
9
+ rescue Mysql::Error
10
+ end
11
+ begin
12
+ Mysql.real_connect('localhost', 'root', 'pass', 'db', 3307)# bad port
13
+ rescue Mysql::Error
14
+ end
15
+ print "pass"
16
+
17
+
@@ -0,0 +1,199 @@
1
+ require 'rubygems'
2
+ require 'mysqlplus'
3
+
4
+ class MysqlTest
5
+
6
+ class NotImplemented < StandardError
7
+ end
8
+
9
+ attr_accessor :queries,
10
+ :context,
11
+ :connections,
12
+ :connection_signature,
13
+ :start,
14
+ :done,
15
+ :query_with,
16
+ :per_query_overhead,
17
+ :timeout
18
+
19
+ def initialize( queries, context = '' )
20
+ @queries = queries
21
+ @context = context
22
+ @done = []
23
+ @query_with = :async_query
24
+ @per_query_overhead = 3
25
+ @timeout = 20
26
+ yield self if block_given?
27
+ end
28
+
29
+ def setup( &block )
30
+ @start = Time.now
31
+ @connection_signature = block
32
+ end
33
+
34
+ def run!
35
+ c_or_native_ruby_async_query do
36
+ present_context if context?
37
+ prepare
38
+ yield
39
+ end
40
+ end
41
+
42
+ def per_query_overhead=( overhead )
43
+ @per_query_overhead = ( overhead == :random ) ? rand() : overhead
44
+ end
45
+
46
+ protected
47
+
48
+ def prepare
49
+ raise NotImplemented
50
+ end
51
+
52
+ def teardown
53
+ raise NotImplemented
54
+ end
55
+
56
+ def log( message, prefix = '' )
57
+ puts "[#{timestamp}] #{prefix} #{message}"
58
+ end
59
+
60
+ def with_logging( message )
61
+ log( message, 'Start' )
62
+ yield
63
+ log( message, 'End' )
64
+ end
65
+
66
+ def timestamp
67
+ Time.now - @start
68
+ end
69
+
70
+ def context?
71
+ @context != ''
72
+ end
73
+
74
+ def present_context
75
+ log "#############################################"
76
+ log "# #{@context}"
77
+ log "#############################################"
78
+ end
79
+
80
+ def c_or_native_ruby_async_query
81
+ if @query_with == :c_async_query
82
+ log "** using C based async_query"
83
+ else
84
+ log "** using native Ruby async_query"
85
+ end
86
+ yield
87
+ end
88
+
89
+ def dispatch_query( connection, sql, timeout = nil )
90
+ connection.send( @query_with, sql, timeout )
91
+ end
92
+
93
+ end
94
+
95
+ class EventedMysqlTest < MysqlTest
96
+
97
+ attr_accessor :sockets
98
+
99
+ def initialize( queries, context = '' )
100
+ @sockets = []
101
+ @connections = {}
102
+ super( queries, context )
103
+ end
104
+
105
+ def setup( &block )
106
+ super( &block )
107
+ with_logging 'Setup connection pool' do
108
+ @queries.times do
109
+ connection = @connection_signature.call
110
+ @connections[ IO.new(connection.socket) ] = connection
111
+ @sockets = @connections.keys
112
+ end
113
+ end
114
+ end
115
+
116
+ def run!
117
+ super do
118
+ catch :END_EVENT_LOOP do
119
+ loop do
120
+ result = select( @sockets,nil,nil,nil )
121
+ if result
122
+ result.first.each do |conn|
123
+ @connections[conn].get_result.each{|res| log( "Result for socket #{conn.fileno} : #{res}" ) }
124
+ @done << nil
125
+ if done?
126
+ teardown
127
+ end
128
+ end
129
+ end
130
+ end
131
+ end
132
+ end
133
+ end
134
+
135
+ protected
136
+
137
+ def prepare
138
+ @connections.each_value do |conn|
139
+ conn.send_query( "select sleep(#{@per_query_overhead})" )
140
+ end
141
+ end
142
+
143
+ def teardown
144
+ log "done"
145
+ throw :END_EVENT_LOOP
146
+ end
147
+
148
+ def done?
149
+ @done.size == @queries
150
+ end
151
+
152
+ end
153
+
154
+ class ThreadedMysqlTest < MysqlTest
155
+
156
+ attr_accessor :threads
157
+
158
+ def initialize( queries, context = '' )
159
+ @connections = []
160
+ @threads = []
161
+ super( queries, context )
162
+ end
163
+
164
+ def setup( &block )
165
+ super( &block )
166
+ with_logging "Setup connection pool" do
167
+ @queries.times do
168
+ @connections << @connection_signature.call
169
+ end
170
+ end
171
+ end
172
+
173
+ def run!
174
+ super do
175
+ with_logging "waiting on threads" do
176
+ @threads.each{|t| t.join }
177
+ end
178
+ end
179
+ end
180
+
181
+ protected
182
+
183
+ def prepare
184
+ with_logging "prepare" do
185
+ @queries.times do |conn|
186
+ @threads << Thread.new do
187
+
188
+ log "sending query on connection #{conn}"
189
+
190
+ dispatch_query( @connections[conn], "select sleep(#{@per_query_overhead})", @timeout ).each do |result|
191
+ log "connection #{conn} done"
192
+ end
193
+
194
+ end
195
+ end
196
+ end
197
+ end
198
+
199
+ end
@@ -0,0 +1,7 @@
1
+ require 'rubygems'
2
+ require 'mysqlplus'
3
+ a = Mysql.real_connect('localhost','root')
4
+ 100.times { a.query("select sleep(0)") }
5
+ print "pass"
6
+
7
+
@@ -0,0 +1,48 @@
1
+ # This is an example of using Mysql::ResultSet#use_result [see docs for what that does]
2
+ # this function is useful for those who have large query results and want to be able to parse them
3
+ # as they come in, instead of having to wait for the query to finish before doing parsing
4
+ # for me, running this on a query with 200_000 lines decreases total time to create an array of results
5
+ # from .82s to .62s
6
+ # you can experiment with it by changing the query here to be a long one, and toggling the do_the_use_query_optimization variable
7
+ # this also has the interesting property of 'freeing' Ruby to do thread changes mid-query.
8
+ require 'rubygems'
9
+ require 'mysqlplus'
10
+
11
+ do_the_use_query_optimization = true
12
+
13
+ $count = 5
14
+
15
+ $start = Time.now
16
+
17
+ $connections = []
18
+ $count.times do
19
+ $connections << Mysql.real_connect('localhost','root', '', 'local_leadgen_dev')
20
+ end
21
+
22
+ puts 'connection pool ready'
23
+
24
+ $threads = []
25
+ $count.times do |i|
26
+ $threads << Thread.new do
27
+
28
+ puts "sending query on connection #{i}"
29
+ conn = $connections[i]
30
+ saved = []
31
+ query = "select * from campus_zips"
32
+ if do_the_use_query_optimization
33
+ conn.query_with_result=false
34
+ result = conn.async_query(query)
35
+ res = result.use_result
36
+ res.each_hash { |h| saved << h }
37
+ res.free
38
+ else
39
+ conn.async_query(query).each_hash {|h| saved << h }
40
+ end
41
+
42
+ end
43
+ end
44
+
45
+ puts 'waiting on threads'
46
+ $threads.each{|t| t.join }
47
+
48
+ puts Time.now - $start