jrmey-mysqlplus 0.1.3

data/lib/mysqlplus.rb

@@ -0,0 +1,23 @@
+require File.dirname(__FILE__) + '/mysql' # load our version of mysql--note
+# if someone does a require 'mysql' after a require 'mysqlplus' then their screen will be littered with warnings
+# and the "old" mysql will override the "new" mysqlplus, so be careful.
+
+#
+# The mysqlplus library is a  [slightly updated] fork of the Mysql class, with asynchronous capability added
+# See http://www.kitebird.com/articles/ruby-mysql.html for details, as well as the test directory within the gem
+#
+class Mysql
+  
+  def ruby_async_query(sql, timeout = nil) # known to deadlock TODO
+    send_query(sql)
+    select [ (@sockets ||= {})[socket] ||= IO.new(socket) ], nil, nil, nil
+    get_result
+  end
+
+  begin
+    alias_method :async_query, :c_async_query  
+  rescue NameError => e
+    raise LoadError.new("error loading mysqlplus--this may mean you ran a require 'mysql' before a require 'mysqplus', which must come first -- possibly also run gem uninstall mysql")
+  end
+  
+end

data/mysqlplus.gemspec

@@ -0,0 +1,30 @@
+Gem::Specification.new do |s|
+  s.name     = "jrmey-mysqlplus"
+  s.version  = "0.1.3"
+  s.date     = "2010-07-04"
+  s.summary  = "Enhanced Ruby MySQL driver with Ruby 1.9 support"
+  s.email    = "jeremysuriel@gmail.com"
+  s.homepage = "http://github.com/jrmey/mysqlplus"
+  s.description = "Enhanced Ruby MySQL driver"
+  s.has_rdoc = true
+  s.authors  = ["Muhammad A. Ali", "Jeremy Suriel"]
+  s.platform = Gem::Platform::RUBY
+  s.files    = %w[
+    README
+    Rakefile
+    TODO_LIST
+    ext/error_const.h
+    ext/extconf.rb
+    ext/mysql.c
+    lib/mysqlplus.rb
+    mysqlplus.gemspec
+  ] + Dir.glob('test/*')
+  s.rdoc_options = ["--main", "README"]
+  s.extra_rdoc_files = ["README"]
+  s.extensions << "ext/extconf.rb"
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 3
+  end
+end

data/test/RUN_ALL_TESTS.RB

@@ -0,0 +1,9 @@
+# I suppose if all the tests don't blow up, that probably means pass
+require 'mysqlplus'
+for file in Dir.glob('*_test.rb') do
+ puts 'testing ' + file
+ # fork so we don't run out of connections to the mysql db, as few tests ever clean up their old processes
+ pid = Process.fork { load file }
+ Process.wait(pid)
+end
+puts 'successful'

data/test/all_hashes_test.rb

@@ -0,0 +1,37 @@
+require 'create_test_db'
+
+use_the_all_hashes_method = true 
+
+$count = 5
+
+$start = Time.now
+
+$connections = []
+$count.times do  
+  $connections << Mysql.real_connect('localhost','root', '', 'local_test_db')
+end
+
+
+$threads = []
+$count.times do |i|
+  $threads << Thread.new do
+
+    query = "select * from test_table"
+    puts "sending query on connection #{i}"
+    conn = $connections[i]
+    result = conn.async_query(query)
+    if use_the_all_hashes_method
+      saved =  result.all_hashes
+    else
+      saved = []
+      result.each_hash {|h| saved << h } 
+    end
+    result.free
+
+  end
+end
+
+puts 'waiting on threads'
+$threads.each{|t| t.join }
+
+puts Time.now - $start

data/test/async_query_with_block_test.rb

@@ -0,0 +1,7 @@
+require File.dirname(__FILE__) + '/test_helper'
+
+m = Mysql.real_connect('localhost','root','','mysql')
+
+m.c_async_query( 'SELECT * FROM user' ) do |result|
+  puts result.inspect
+end  

data/test/c_threaded_test.rb

@@ -0,0 +1,36 @@
+require File.dirname(__FILE__) + '/test_helper'
+
+ThreadedMysqlTest.new( 10, "Threaded, C, very small overhead"  ) do |test|
+  test.setup{ Mysql.real_connect('localhost','root') }
+  test.per_query_overhead = 0.005
+  test.query_with = :c_async_query 
+  test.run!
+end
+
+ThreadedMysqlTest.new( 10, "Threaded, C, small overhead"  ) do |test|
+  test.setup{ Mysql.real_connect('localhost','root') }
+  test.per_query_overhead = 0.1
+  test.query_with = :c_async_query 
+  test.run!
+end
+
+ThreadedMysqlTest.new( 10, "Threaded, C, medium overhead"  ) do |test|
+  test.setup{ Mysql.real_connect('localhost','root') }
+  test.per_query_overhead = 1
+  test.query_with = :c_async_query 
+  test.run!
+end
+
+ThreadedMysqlTest.new( 10, "Threaded, C, large overhead"  ) do |test|
+  test.setup{ Mysql.real_connect('localhost','root') }
+  test.per_query_overhead = 3
+  test.query_with = :c_async_query 
+  test.run!
+end
+
+ThreadedMysqlTest.new( 10, "Threaded, C, random overhead"  ) do |test|
+  test.setup{ Mysql.real_connect('localhost','root') }
+  test.per_query_overhead = :random
+  test.query_with = :c_async_query 
+  test.run!
+end

data/test/connect_failure2_test.rb

@@ -0,0 +1,22 @@
+# If this script returns without the word pass
+# you may have compiled mysqlplus using ruby and
+# run it using a different version of ruby
+
+if RUBY_VERSION >= "1.9.1"
+ require 'mysqlplus'
+ require 'socket'
+ require 'timeout'
+ TCPServer.new '0.0.0.0', 8002
+ Thread.new { 
+  sleep 2
+  print "pass"
+  system("kill -9 #{Process.pid}")
+ }
+ Timeout::timeout(1) {
+  # uncomment this line to do the 'real' test
+  # which hangs otherwise (blows up if code is bad, otherwise hangs)
+  Mysql.real_connect '127.0.0.1', 'root', 'pass', 'db', 8002
+ }
+ raise 'should never get here'
+end
+

data/test/connect_failure_test.rb

@@ -0,0 +1,17 @@
+require 'mysqlplus'
+begin
+  Mysql.real_connect('fakehost','root', '', 'local_leadgen_dev')
+rescue Mysql::Error
+end
+begin
+  Mysql.real_connect('localhost','root', '', 'faketable')
+rescue Mysql::Error
+end
+begin
+  Mysql.real_connect('localhost', 'root', 'pass', 'db', 3307)# bad port
+rescue Mysql::Error
+end
+
+print "pass"
+
+

data/test/create_test_db.rb

@@ -0,0 +1,22 @@
+# To run first execute:
+=begin
+create database local_test_db;
+use local_test_db;
+CREATE TABLE test_table (
+         c1 INT, 
+         c2 VARCHAR(20)
+     );
+=end
+# This script shows the effect of using .all_hashes instead of looping on each hash 
+# run it by substiting in a 'long' [many row] query for the query variable and toggling use_all_hashes here at the top
+# note that we load all the rows first, then run .all_hashes on the result [to see more easily the effect of all hashes]
+# on my machine and a 200_000 row table, it took 3.38s versus 3.65s for the old .each_hash way [note also that .each_hash is 
+# almost as fast, now, as .all_hashes--they've both been optimized]
+require 'mysqlplus'
+
+puts 'initing db'
+# init the DB
+conn = Mysql.real_connect('localhost', 'root', '', 'local_test_db')
+conn.query("delete from test_table")
+200_000.times {conn.query(" insert into test_table (c1, c2) values (3, 'ABCDEFG')")}
+puts 'connection pool ready'

data/test/evented_test.rb

@@ -0,0 +1,31 @@
+require File.dirname(__FILE__) + '/test_helper'
+
+EventedMysqlTest.new( 10, "Evented, very small overhead" ) do |test|
+  test.setup{ Mysql.real_connect('localhost','root') }
+  test.per_query_overhead = 0.005
+  test.run!
+end
+
+EventedMysqlTest.new( 10, "Evented, small overhead" ) do |test|
+  test.setup{ Mysql.real_connect('localhost','root') }
+  test.per_query_overhead = 0.1
+  test.run!
+end
+
+EventedMysqlTest.new( 10, "Evented, medium overhead" ) do |test|
+  test.setup{ Mysql.real_connect('localhost','root') }
+  test.per_query_overhead = 1
+  test.run!
+end
+
+EventedMysqlTest.new( 10, "Evented, large overhead" ) do |test|
+  test.setup{ Mysql.real_connect('localhost','root') }
+  test.per_query_overhead = 3
+  test.run!
+end
+
+EventedMysqlTest.new( 10, "Evented, random overhead" ) do |test|
+  test.setup{ Mysql.real_connect('localhost','root') }
+  test.per_query_overhead = :random
+  test.run!
+end

data/test/gc_benchmark_test.rb

@@ -0,0 +1,40 @@
+require 'mysqlplus'
+require 'benchmark'
+
+with_gc = Mysql.real_connect('localhost','root','','mysql')
+without_gc = Mysql.real_connect('localhost','root','','mysql')
+without_gc.disable_gc = true
+
+$gc_stats = []
+
+def countable_gc?
+  GC.respond_to? :count
+end
+
+def gc_counts( label, scope )
+  $gc_stats << "Objects #{scope} ( #{label} ) #{GC.count}"
+end
+
+def with_gc_counts( label )
+  gc_counts( label, 'before' ) if countable_gc?
+  yield
+  gc_counts( label, 'after' ) if countable_gc?
+end
+
+n = 1000
+
+Benchmark.bmbm do |x|
+  x.report( 'With GC' ) do
+    with_gc_counts( 'With GC' ) do
+      n.times{ with_gc.c_async_query( 'SELECT * FROM user' ) }
+    end
+  end  
+  GC.start  
+  x.report( 'Without GC' ) do
+    with_gc_counts( 'Without GC' ) do
+      n.times{ without_gc.c_async_query( 'SELECT * FROM user' ) }
+    end
+  end
+end
+
+puts $gc_stats.join( ' | ' )

data/test/many_requests_test.rb

@@ -0,0 +1,6 @@
+require 'mysqlplus'
+a = Mysql.real_connect('localhost','root')
+100.times { a.query("select sleep(0)") }
+print "pass"
+
+

data/test/native_threaded_test.rb

@@ -0,0 +1,36 @@
+require File.dirname(__FILE__) + '/test_helper'
+
+ThreadedMysqlTest.new( 10, "Threaded, native Ruby, very small overhead"  ) do |test|
+  test.setup{ Mysql.real_connect('localhost','root') }
+  test.per_query_overhead = 0.005
+  test.query_with = :async_query   
+  test.run!
+end
+
+ThreadedMysqlTest.new( 10, "Threaded, native Ruby, small overhead"  ) do |test|
+  test.setup{ Mysql.real_connect('localhost','root') }
+  test.per_query_overhead = 0.1
+  test.query_with = :async_query     
+  test.run!
+end
+
+ThreadedMysqlTest.new( 10, "Threaded, native Ruby, medium overhead"  ) do |test|
+  test.setup{ Mysql.real_connect('localhost','root') }
+  test.per_query_overhead = 1
+  test.query_with = :async_query     
+  test.run!
+end
+
+ThreadedMysqlTest.new( 10, "Threaded, native Ruby, large overhead"  ) do |test|
+  test.setup{ Mysql.real_connect('localhost','root') }
+  test.per_query_overhead = 3
+  test.query_with = :async_query     
+  test.run!
+end
+
+ThreadedMysqlTest.new( 10, "Threaded, native Ruby, random overhead"  ) do |test|
+  test.setup{ Mysql.real_connect('localhost','root') }
+  test.per_query_overhead = :random
+  test.query_with = :async_query     
+  test.run!
+end

data/test/out_of_sync_test.rb

@@ -0,0 +1,34 @@
+require File.dirname(__FILE__) + '/test_helper'
+
+m = Mysql.real_connect('localhost','root')
+m.reconnect = true
+$count = 0
+class << m
+  def safe_query( query )
+    begin
+      send_query( query )
+    rescue => e
+      $count += 1
+      puts e.message
+    end
+  end
+  
+end
+
+m.safe_query( 'select sleep(1)' )
+m.safe_query( 'select sleep(1)' )#raises
+m.simulate_disconnect #fires mysql_library_end
+m.safe_query( 'select sleep(1)' )
+m.safe_query( 'select sleep(1)' )#raises
+m.close
+m.connect('localhost','root')
+m.safe_query( 'select sleep(1)' )
+m.safe_query( 'select sleep(1)' )#raises
+m.simulate_disconnect
+raise unless $count == 3
+m.safe_query( 'BEGIN' )
+m.safe_query( 'select sleep(1)' ) # raises
+m.get_result()
+m.safe_query( 'COMMIT' )
+m.get_result
+raise unless $count == 4

data/test/query_with_result_false_test.rb

@@ -0,0 +1,47 @@
+# This is an example of using Mysql::ResultSet#use_result [see docs for what that does]
+# this function is useful for those who have large query results and want to be able to parse them
+# as they come in, instead of having to wait for the query to finish before doing parsing
+# for me, running this on a query with 200_000 lines decreases total time to create an array of results 
+# from .82s to .62s
+# you can experiment with it by changing the query here to be a long one, and toggling the do_the_use_query_optimization variable
+# this also has the interesting property of 'freeing' Ruby to do thread changes mid-query.
+require 'create_test_db'
+
+do_the_use_query_optimization = true
+
+$count = 5
+
+$start = Time.now
+
+$connections = []
+$count.times do  
+  $connections << Mysql.real_connect('localhost','root', '', 'local_test_db')
+end
+
+puts 'connection pool ready'
+
+$threads = []
+$count.times do |i|
+  $threads << Thread.new do
+
+    puts "sending query on connection #{i}"
+    conn = $connections[i]
+    saved = []
+    query = "select * from test_table"
+    if do_the_use_query_optimization
+      conn.query_with_result=false
+      result = conn.async_query(query)
+      res = result.use_result
+      res.each_hash { |h| saved << h }
+      res.free
+    else
+      conn.async_query(query).each_hash {|h| saved << h }
+    end
+
+  end
+end
+
+puts 'waiting on threads'
+$threads.each{|t| t.join }
+
+puts Time.now - $start

data/test/reconnected_test.rb

@@ -0,0 +1,18 @@
+require File.dirname(__FILE__) + '/test_helper'
+
+$m = Mysql.real_connect('localhost','root')
+#$m.reconnect = true
+
+def assert_reconnected
+  puts $m.reconnected?().inspect
+  sleep 1
+  yield
+  puts $m.reconnected?().inspect
+end
+
+assert_reconnected do
+  $m.simulate_disconnect
+end
+assert_reconnected do
+  $m.close
+end

data/test/test_helper.rb

@@ -0,0 +1,198 @@
+require 'mysqlplus'
+
+class MysqlTest
+  
+  class NotImplemented < StandardError
+  end
+  
+  attr_accessor :queries,
+                :context,
+                :connections,
+                :connection_signature,
+                :start,
+                :done,
+                :query_with,
+                :per_query_overhead,
+                :timeout
+  
+  def initialize( queries, context = '' )
+    @queries = queries
+    @context = context
+    @done = []
+    @query_with = :async_query
+    @per_query_overhead = 3
+    @timeout = 20
+    yield self if block_given?
+  end
+  
+  def setup( &block )
+    @start = Time.now
+    @connection_signature = block
+  end
+  
+  def run!
+    c_or_native_ruby_async_query do
+      present_context if context?
+      prepare
+      yield
+    end  
+  end
+  
+  def per_query_overhead=( overhead )
+    @per_query_overhead = ( overhead == :random ) ? rand() : overhead
+  end
+  
+  protected
+
+  def prepare
+    raise NotImplemented
+  end
+  
+  def teardown
+    raise NotImplemented
+  end
+  
+  def log( message, prefix = '' )
+    puts "[#{timestamp}] #{prefix} #{message}"
+  end
+  
+  def with_logging( message )
+    log( message, 'Start' )
+    yield
+    log( message, 'End' )
+  end
+  
+  def timestamp
+    Time.now - @start
+  end
+
+  def context?
+    @context != ''
+  end
+  
+  def present_context
+    log "#############################################"
+    log "#  #{@context}" 
+    log "#############################################"   
+  end
+  
+  def c_or_native_ruby_async_query
+    if @query_with == :c_async_query
+      log "** using C based async_query"
+    else
+      log "** using native Ruby async_query"
+    end
+    yield
+  end
+  
+  def dispatch_query( connection, sql, timeout = nil )
+    connection.send( @query_with, sql, timeout )
+  end
+  
+end
+
+class EventedMysqlTest < MysqlTest
+  
+  attr_accessor :sockets
+  
+  def initialize( queries, context = '' )
+    @sockets = []
+    @connections = {}
+    super( queries, context )
+  end  
+  
+  def setup( &block )
+    super( &block )
+    with_logging 'Setup connection pool' do
+      @queries.times do 
+        connection = @connection_signature.call
+        @connections[ IO.new(connection.socket) ] = connection
+        @sockets = @connections.keys
+      end
+    end  
+  end
+  
+  def run!
+    super do
+      catch :END_EVENT_LOOP do
+        loop do
+          result = select( @sockets,nil,nil,nil )
+          if result
+            result.first.each do |conn|
+              @connections[conn].get_result.each{|res| log( "Result for socket #{conn.fileno} : #{res}" ) }
+              @done << nil
+              if done?
+                teardown
+              end  
+            end 
+          end
+        end    
+      end
+    end
+  end
+  
+  protected
+  
+  def prepare
+    @connections.each_value do |conn|
+      conn.send_query( "select sleep(#{@per_query_overhead})" ) 
+    end
+  end
+  
+  def teardown
+    log "done"
+    throw :END_EVENT_LOOP
+  end
+  
+  def done?
+    @done.size == @queries
+  end
+  
+end
+
+class ThreadedMysqlTest < MysqlTest
+  
+  attr_accessor :threads
+  
+  def initialize( queries, context = '' )
+    @connections = []
+    @threads = []
+    super( queries, context )
+  end
+  
+  def setup( &block )
+    super( &block )
+    with_logging "Setup connection pool" do
+      @queries.times do 
+        @connections << @connection_signature.call
+      end
+    end
+  end
+  
+  def run!
+    super do
+      with_logging "waiting on threads" do
+        @threads.each{|t| t.join }
+      end
+    end  
+  end 
+  
+  protected
+  
+  def prepare
+    with_logging "prepare" do
+      @queries.times do |conn|
+        @threads << Thread.new do
+
+          log "sending query on connection #{conn}"
+
+          dispatch_query( @connections[conn], "select sleep(#{@per_query_overhead})", @timeout ).each do |result|
+            log "connection #{conn} done"
+          end 
+        
+        end
+      end  
+    end
+  end
+  
+end

data/test/threaded_sequel_test.rb

@@ -0,0 +1,24 @@
+require 'mysqlplus'
+require 'rubygems'
+require 'sequel'
+
+class Mysql
+  unless method_defined? :sync_query
+    alias :sync_query :query
+    alias :query :async_query
+  end
+end
+
+DB = Sequel.connect('mysql://root@localhost', :max_connections => 20)
+
+start = Time.now
+
+(0..10).map do
+  Thread.new do
+
+    p DB['select sleep(2)'].all
+
+  end
+end.map{|t| t.join }
+
+p (Time.now - start)