mysqlplus 0.1.1

data/lib/mysqlplus.rb

@@ -0,0 +1,21 @@
+require 'mysql' # this should load the mysqlplus version of mysql.so, as we assume the user has installed mysql as a gem and have not done any previous "require 'mysql'" to have loaded the other
+
+#
+# Mysqlplus library gives you a [slightly modified] version of the Mysql class
+# See http://www.kitebird.com/articles/ruby-mysql.html for details, as well as the test directory within the library
+#
+class Mysql
+  
+  def ruby_async_query(sql, timeout = nil) # known to deadlock TODO
+    send_query(sql)
+    select [ (@sockets ||= {})[socket] ||= IO.new(socket) ], nil, nil, nil
+    get_result
+  end
+
+  begin
+    alias_method :async_query, :c_async_query  
+  rescue NameError => e
+    raise LoadError.new "error loading mysqlplus--this may mean you ran a require 'mysql' before a require 'mysqplus', which much come first"
+  end
+  
+end

data/mysqlplus.gemspec

@@ -0,0 +1,35 @@
+Gem::Specification.new do |s|
+  s.name     = "mysqlplus"
+  s.version  = "0.1.1"
+  s.date     = "2009-03-22"
+  s.summary  = "Enhanced Ruby MySQL driver"
+  s.email    = "oldmoe@gmail.com"
+  s.homepage = "http://github.com/oldmoe/mysqlplus"
+  s.description = "Enhanced Ruby MySQL driver"
+  s.has_rdoc = true
+  s.authors  = ["Muhammad A. Ali"]
+  s.platform = Gem::Platform::RUBY
+  s.files    = %w[
+    README
+    Rakefile
+    TODO_LIST
+    ext/error_const.h
+    ext/extconf.rb
+    ext/mysql.c
+    lib/mysqlplus.rb
+    mysqlplus.gemspec
+    test/c_threaded_test.rb
+    test/evented_test.rb
+    test/native_threaded_test.rb
+    test/test_all_hashes.rb
+    test/test_failure.rb
+    test/test_helper.rb
+    test/test_many_requests.rb
+    test/test_parsing_while_response_is_being_read.rb
+    test/test_threaded_sequel.rb
+  ]
+  s.rdoc_options = ["--main", "README"]
+  s.extra_rdoc_files = ["README"]
+  s.extensions << "ext/extconf.rb"
+end
+

data/test/c_threaded_test.rb

@@ -0,0 +1,36 @@
+require File.dirname(__FILE__) + '/test_helper'
+
+ThreadedMysqlTest.new( 10, "Threaded, C, very small overhead"  ) do |test|
+  test.setup{ Mysql.real_connect('localhost','root') }
+  test.per_query_overhead = 0.005
+  test.query_with = :c_async_query 
+  test.run!
+end
+
+ThreadedMysqlTest.new( 10, "Threaded, C, small overhead"  ) do |test|
+  test.setup{ Mysql.real_connect('localhost','root') }
+  test.per_query_overhead = 0.1
+  test.query_with = :c_async_query 
+  test.run!
+end
+
+ThreadedMysqlTest.new( 10, "Threaded, C, medium overhead"  ) do |test|
+  test.setup{ Mysql.real_connect('localhost','root') }
+  test.per_query_overhead = 1
+  test.query_with = :c_async_query 
+  test.run!
+end
+
+ThreadedMysqlTest.new( 10, "Threaded, C, large overhead"  ) do |test|
+  test.setup{ Mysql.real_connect('localhost','root') }
+  test.per_query_overhead = 3
+  test.query_with = :c_async_query 
+  test.run!
+end
+
+ThreadedMysqlTest.new( 10, "Threaded, C, random overhead"  ) do |test|
+  test.setup{ Mysql.real_connect('localhost','root') }
+  test.per_query_overhead = :random
+  test.query_with = :c_async_query 
+  test.run!
+end

data/test/evented_test.rb

@@ -0,0 +1,31 @@
+require File.dirname(__FILE__) + '/test_helper'
+
+EventedMysqlTest.new( 10, "Evented, very small overhead" ) do |test|
+  test.setup{ Mysql.real_connect('localhost','root') }
+  test.per_query_overhead = 0.005
+  test.run!
+end
+
+EventedMysqlTest.new( 10, "Evented, small overhead" ) do |test|
+  test.setup{ Mysql.real_connect('localhost','root') }
+  test.per_query_overhead = 0.1
+  test.run!
+end
+
+EventedMysqlTest.new( 10, "Evented, medium overhead" ) do |test|
+  test.setup{ Mysql.real_connect('localhost','root') }
+  test.per_query_overhead = 1
+  test.run!
+end
+
+EventedMysqlTest.new( 10, "Evented, large overhead" ) do |test|
+  test.setup{ Mysql.real_connect('localhost','root') }
+  test.per_query_overhead = 3
+  test.run!
+end
+
+EventedMysqlTest.new( 10, "Evented, random overhead" ) do |test|
+  test.setup{ Mysql.real_connect('localhost','root') }
+  test.per_query_overhead = :random
+  test.run!
+end

data/test/native_threaded_test.rb

@@ -0,0 +1,36 @@
+require File.dirname(__FILE__) + '/test_helper'
+
+ThreadedMysqlTest.new( 10, "Threaded, native Ruby, very small overhead"  ) do |test|
+  test.setup{ Mysql.real_connect('localhost','root') }
+  test.per_query_overhead = 0.005
+  test.query_with = :async_query   
+  test.run!
+end
+
+ThreadedMysqlTest.new( 10, "Threaded, native Ruby, small overhead"  ) do |test|
+  test.setup{ Mysql.real_connect('localhost','root') }
+  test.per_query_overhead = 0.1
+  test.query_with = :async_query     
+  test.run!
+end
+
+ThreadedMysqlTest.new( 10, "Threaded, native Ruby, medium overhead"  ) do |test|
+  test.setup{ Mysql.real_connect('localhost','root') }
+  test.per_query_overhead = 1
+  test.query_with = :async_query     
+  test.run!
+end
+
+ThreadedMysqlTest.new( 10, "Threaded, native Ruby, large overhead"  ) do |test|
+  test.setup{ Mysql.real_connect('localhost','root') }
+  test.per_query_overhead = 3
+  test.query_with = :async_query     
+  test.run!
+end
+
+ThreadedMysqlTest.new( 10, "Threaded, native Ruby, random overhead"  ) do |test|
+  test.setup{ Mysql.real_connect('localhost','root') }
+  test.per_query_overhead = :random
+  test.query_with = :async_query     
+  test.run!
+end

data/test/test_all_hashes.rb

@@ -0,0 +1,43 @@
+# shows the effect of using .all_hashes instead of looping on each hash 
+# run it by substiting in a 'long' [many row] query for the query variable and toggling use_all_hashes here at the top
+# note that we load all the rows first, then run .all_hashes on the result [to see more easily the effect of all hashes]
+# on my machine and a 200_000 row table, it took 3.38s versus 3.65s
+require 'rubygems'
+require 'mysqlplus'
+
+use_the_all_hashes_method = true 
+
+$count = 5
+
+$start = Time.now
+
+$connections = []
+$count.times do  
+  $connections << Mysql.real_connect('localhost','root', '', 'local_leadgen_dev')
+end
+
+puts 'connection pool ready'
+
+$threads = []
+$count.times do |i|
+  $threads << Thread.new do
+
+    query = "select * from campus_zips"
+    puts "sending query on connection #{i}"
+    conn = $connections[i]
+    result = conn.async_query(query)
+    if use_the_all_hashes_method
+      saved =  result.all_hashes
+    else
+      saved = []
+      result.each_hash {|h| saved << h } 
+    end
+    result.free
+
+  end
+end
+
+puts 'waiting on threads'
+$threads.each{|t| t.join }
+
+puts Time.now - $start

data/test/test_failure.rb

@@ -0,0 +1,17 @@
+require 'rubygems'
+require 'mysqlplus'
+begin
+  Mysql.real_connect('fakehost','root', '', 'local_leadgen_dev')
+rescue Mysql::Error
+end
+begin
+  Mysql.real_connect('localhost','root', '', 'faketable')
+rescue Mysql::Error
+end
+begin
+  Mysql.real_connect('localhost', 'root', 'pass', 'db', 3307)# bad port
+rescue Mysql::Error
+end
+print "pass"
+
+

data/test/test_helper.rb

@@ -0,0 +1,199 @@
+require 'rubygems'
+require 'mysqlplus'
+
+class MysqlTest
+  
+  class NotImplemented < StandardError
+  end
+  
+  attr_accessor :queries,
+                :context,
+                :connections,
+                :connection_signature,
+                :start,
+                :done,
+                :query_with,
+                :per_query_overhead,
+                :timeout
+  
+  def initialize( queries, context = '' )
+    @queries = queries
+    @context = context
+    @done = []
+    @query_with = :async_query
+    @per_query_overhead = 3
+    @timeout = 20
+    yield self if block_given?
+  end
+  
+  def setup( &block )
+    @start = Time.now
+    @connection_signature = block
+  end
+  
+  def run!
+    c_or_native_ruby_async_query do
+      present_context if context?
+      prepare
+      yield
+    end  
+  end
+  
+  def per_query_overhead=( overhead )
+    @per_query_overhead = ( overhead == :random ) ? rand() : overhead
+  end
+  
+  protected
+
+  def prepare
+    raise NotImplemented
+  end
+  
+  def teardown
+    raise NotImplemented
+  end
+  
+  def log( message, prefix = '' )
+    puts "[#{timestamp}] #{prefix} #{message}"
+  end
+  
+  def with_logging( message )
+    log( message, 'Start' )
+    yield
+    log( message, 'End' )
+  end
+  
+  def timestamp
+    Time.now - @start
+  end
+
+  def context?
+    @context != ''
+  end
+  
+  def present_context
+    log "#############################################"
+    log "#  #{@context}" 
+    log "#############################################"   
+  end
+  
+  def c_or_native_ruby_async_query
+    if @query_with == :c_async_query
+      log "** using C based async_query"
+    else
+      log "** using native Ruby async_query"
+    end
+    yield
+  end
+  
+  def dispatch_query( connection, sql, timeout = nil )
+    connection.send( @query_with, sql, timeout )
+  end
+  
+end
+
+class EventedMysqlTest < MysqlTest
+  
+  attr_accessor :sockets
+  
+  def initialize( queries, context = '' )
+    @sockets = []
+    @connections = {}
+    super( queries, context )
+  end  
+  
+  def setup( &block )
+    super( &block )
+    with_logging 'Setup connection pool' do
+      @queries.times do 
+        connection = @connection_signature.call
+        @connections[ IO.new(connection.socket) ] = connection
+        @sockets = @connections.keys
+      end
+    end  
+  end
+  
+  def run!
+    super do
+      catch :END_EVENT_LOOP do
+        loop do
+          result = select( @sockets,nil,nil,nil )
+          if result
+            result.first.each do |conn|
+              @connections[conn].get_result.each{|res| log( "Result for socket #{conn.fileno} : #{res}" ) }
+              @done << nil
+              if done?
+                teardown
+              end  
+            end 
+          end
+        end    
+      end
+    end
+  end
+  
+  protected
+  
+  def prepare
+    @connections.each_value do |conn|
+      conn.send_query( "select sleep(#{@per_query_overhead})" ) 
+    end
+  end
+  
+  def teardown
+    log "done"
+    throw :END_EVENT_LOOP
+  end
+  
+  def done?
+    @done.size == @queries
+  end
+  
+end
+
+class ThreadedMysqlTest < MysqlTest
+  
+  attr_accessor :threads
+  
+  def initialize( queries, context = '' )
+    @connections = []
+    @threads = []
+    super( queries, context )
+  end
+  
+  def setup( &block )
+    super( &block )
+    with_logging "Setup connection pool" do
+      @queries.times do 
+        @connections << @connection_signature.call
+      end
+    end
+  end
+  
+  def run!
+    super do
+      with_logging "waiting on threads" do
+        @threads.each{|t| t.join }
+      end
+    end  
+  end 
+  
+  protected
+  
+  def prepare
+    with_logging "prepare" do
+      @queries.times do |conn|
+        @threads << Thread.new do
+
+          log "sending query on connection #{conn}"
+
+          dispatch_query( @connections[conn], "select sleep(#{@per_query_overhead})", @timeout ).each do |result|
+            log "connection #{conn} done"
+          end 
+        
+        end
+      end  
+    end
+  end
+  
+end

data/test/test_many_requests.rb

@@ -0,0 +1,7 @@
+require 'rubygems'
+require 'mysqlplus'
+a = Mysql.real_connect('localhost','root')
+100.times { a.query("select sleep(0)") }
+print "pass"
+
+

data/test/test_parsing_while_response_is_being_read.rb

@@ -0,0 +1,48 @@
+# This is an example of using Mysql::ResultSet#use_result [see docs for what that does]
+# this function is useful for those who have large query results and want to be able to parse them
+# as they come in, instead of having to wait for the query to finish before doing parsing
+# for me, running this on a query with 200_000 lines decreases total time to create an array of results 
+# from .82s to .62s
+# you can experiment with it by changing the query here to be a long one, and toggling the do_the_use_query_optimization variable
+# this also has the interesting property of 'freeing' Ruby to do thread changes mid-query.
+require 'rubygems'
+require 'mysqlplus'
+
+do_the_use_query_optimization = true
+
+$count = 5
+
+$start = Time.now
+
+$connections = []
+$count.times do  
+  $connections << Mysql.real_connect('localhost','root', '', 'local_leadgen_dev')
+end
+
+puts 'connection pool ready'
+
+$threads = []
+$count.times do |i|
+  $threads << Thread.new do
+
+    puts "sending query on connection #{i}"
+    conn = $connections[i]
+    saved = []
+    query = "select * from campus_zips"
+    if do_the_use_query_optimization
+      conn.query_with_result=false
+      result = conn.async_query(query)
+      res = result.use_result
+      res.each_hash { |h| saved << h }
+      res.free
+    else
+      conn.async_query(query).each_hash {|h| saved << h }
+    end
+
+  end
+end
+
+puts 'waiting on threads'
+$threads.each{|t| t.join }
+
+puts Time.now - $start