oldmoe-mysqlplus 0.1.0 → 0.1.1

data/README

@@ -20,6 +20,14 @@ An enhanced MySQL database driver. With support for async operations and threade
       --with-mysql-dir=/usr/local/mysql \
       --with-mysql-lib=/usr/local/mysql/lib \
       --with-mysql-include=/usr/local/mysql/include
+== Using
+  to use within rails
+  add require 'mysqlplus' to the top of environment.rb
+  this instantiates the Mysql class that you can use.
+  and it will automagically use async_query instead of query, so it's a drop in replacement.
+
+  Same with other scripts that want to use it--just require 'mysqlplus' BEFORE you require 'mysql' and it will 
+  load the asynchronous version, then ignore the sequent require 'mysql' call.
 
 === Credits
 

data/TODO_LIST

@@ -0,0 +1,13 @@
+TODO list:
+
+Is there a quick, cheap, easy way to test for writability so we don't have to use select on the way in? Does it take long anyway?
+
+Docs for how to use with rails, etc. [there is mysqlplus_adapter, so maybe we're good there].
+
+Merge in lourens' branch -- I like the double check for not using send_query twice in a row! [rdp]
+
+look into http://coderrr.wordpress.com/2009/01/11/ruby-and-mysqlplus-select-deadlock/
+
+Some of the test/* anticipate a certain database existing--todo create it first, then drop it [?]
+
+critical todo list:

data/ext/extconf.rb

@@ -1,10 +1,24 @@
 require 'mkmf'
 
+dirs = ENV['PATH'].split(':') + %w[
+  /opt
+  /opt/local
+  /opt/local/mysql
+  /opt/local/lib/mysql5
+  /usr
+  /usr/local
+  /usr/local/mysql
+  /usr/local/mysql-*
+  /usr/local/lib/mysql5
+].map{|dir| "#{dir}/bin" }
+
+GLOB = "{#{dirs.join(',')}}/{mysql_config,mysql_config5}"
+
 if /mswin32/ =~ RUBY_PLATFORM
   inc, lib = dir_config('mysql')
   exit 1 unless have_library("libmysql")
-elsif mc = with_config('mysql-config') then
-  mc = 'mysql_config' if mc == true
+elsif mc = (with_config('mysql-config') || Dir[GLOB].first) then
+  mc = Dir[GLOB].first if mc == true
   cflags = `#{mc} --cflags`.chomp
   exit 1 if $? != 0
   libs = `#{mc} --libs`.chomp
@@ -31,8 +45,8 @@ else
   exit 1
 end
 
-if have_func('rb_thread_blocking_region') and have_macro('RB_UBF_DFL', 'ruby.h')
-  flags << "-DHAVE_TBR"
+if have_func('rb_thread_blocking_region') and have_macro('RUBY_UBF_IO', 'ruby.h')
+  $CPPFLAGS << " -DHAVE_TBR"
 end
 
 # make mysql constant

data/ext/mysql.c

@@ -190,8 +190,10 @@ static VALUE mysqlres2obj(MYSQL_RES* res)
     resp->res = res;
     resp->freed = Qfalse;
     rb_obj_call_init(obj, 0, NULL);
+    /* disabled until it can be reviewed further--rely on the normal GC for now.
     if (++store_result_count > GC_STORE_RESULT_LIMIT)
 	rb_gc();
+    */
     return obj;
 }
 
@@ -719,18 +721,43 @@ static VALUE use_result(VALUE obj)
 }
 
 static VALUE res_free(VALUE);
+
+typedef struct {
+	MYSQL *m;
+	const char *data;
+	unsigned long len;
+} QueryArgs;
+
+static VALUE blocking_query(void *data)
+{
+    QueryArgs *args = (QueryArgs *) data;
+    return (VALUE) mysql_real_query(args->m, args->data, args->len);
+}
+
 /*	query(sql)	*/
 static VALUE query(VALUE obj, VALUE sql)
 {
     int loop = 0;
     MYSQL* m = GetHandler(obj);
+    QueryArgs args;
+    int result;
+    
     Check_Type(sql, T_STRING);
     if (GetMysqlStruct(obj)->connection == Qfalse) {
         rb_raise(eMysql, "query: not connected");
     }
     if (rb_block_given_p()) {
-	if (mysql_real_query(m, RSTRING_PTR(sql), RSTRING_LEN(sql)) != 0)
+        #ifdef RUBY_VM
+	    args.m = m;
+	    args.data = RSTRING_PTR(sql);
+	    args.len = RSTRING_LEN(sql);
+	    result = (int) rb_thread_blocking_region(blocking_query, &args, RUBY_UBF_PROCESS, 0);
+	#else
+	    result = mysql_real_query(m, RSTRING_PTR(sql), RSTRING_LEN(sql));
+	#endif
+	if (result != 0)
 	    mysql_raise(m);
+	
 	do {
 	    MYSQL_RES* res = mysql_store_result(m);
 	    if (res == NULL) {
@@ -749,7 +776,16 @@ static VALUE query(VALUE obj, VALUE sql)
 #endif
 	return obj;
     }
-    if (mysql_real_query(m, RSTRING_PTR(sql), RSTRING_LEN(sql)) != 0)
+    
+    #ifdef RUBY_VM
+	args.m = m;
+	args.data = RSTRING_PTR(sql);
+	args.len = RSTRING_LEN(sql);
+	result = (int) rb_thread_blocking_region(blocking_query, &args, RUBY_UBF_PROCESS, 0);
+    #else
+	result = mysql_real_query(m, RSTRING_PTR(sql), RSTRING_LEN(sql));
+    #endif
+    if (result != 0)
 	mysql_raise(m);
     if (GetMysqlStruct(obj)->query_with_result == Qfalse)
 	return obj;
@@ -764,13 +800,12 @@ static VALUE socket(VALUE obj)
     MYSQL* m = GetHandler(obj);
     return INT2NUM(m->net.fd);
 }
-
 /* socket_type */
 static VALUE socket_type(VALUE obj)
 {
     MYSQL* m = GetHandler(obj);
     VALUE description = vio_description( m->net.vio );
-    return NILorSTRING( description );
+    return (VALUE) NILorSTRING( description );
 }
 
 /* blocking */
@@ -824,42 +859,33 @@ static VALUE get_result(VALUE obj)
     return store_result(obj);
 }
 
-/* async_query(sql,timeout=nil) */
-static VALUE async_query(int argc, VALUE* argv, VALUE obj)
+static void schedule(VALUE obj, VALUE timeout)
 {
-  MYSQL* m = GetHandler(obj); 
-  VALUE sql, timeout;
-  fd_set read;
-  int ret;
+    MYSQL* m = GetHandler(obj);
+    fd_set read;
 
-  rb_scan_args(argc, argv, "11", &sql, &timeout);
+    timeout = ( NIL_P(timeout) ? m->net.read_timeout : INT2NUM(timeout) );
 
-  send_query(obj,sql);
+    struct timeval tv = { tv_sec: timeout, tv_usec: 0 };
 
-  timeout = ( NIL_P(timeout) ? m->net.read_timeout : INT2NUM(timeout) );
+    FD_ZERO(&read);
+    FD_SET(m->net.fd, &read);
 
-  VALUE args[1];
-  args[0] = timeout;
+    if (rb_thread_select(m->net.fd + 1, &read, NULL, NULL, &tv) < 0) {
+      rb_raise(eMysql, "query: timeout");
+    }
+}
 
-  struct timeval tv = { tv_sec: timeout, tv_usec: 0 };
+/* async_query(sql,timeout=nil) */
+static VALUE async_query(int argc, VALUE* argv, VALUE obj)
+{
+  VALUE sql, timeout;
 
-  FD_ZERO(&read);
-  FD_SET(m->net.fd, &read);
+  rb_scan_args(argc, argv, "11", &sql, &timeout);
 
-  for(;;) {
-      ret = rb_thread_select(m->net.fd + 1, &read, NULL, NULL, &tv);
-      if (ret < 0) {
-        rb_raise(eMysql, "query: timeout");
-      }
-              
-      if (ret == 0) {
-        continue;
-      }
+  send_query(obj,sql);
 
-      if ( vio_poll_read( m->net.vio, INT2NUM(timeout) ) == 0 ) {
-        break;
-      }
-  }
+  schedule(obj, timeout);
 
   return get_result(obj);
 }
@@ -1136,7 +1162,7 @@ static VALUE process_all_hashes(VALUE obj, VALUE with_table, int build_array, in
 {
     MYSQL_RES* res = GetMysqlRes(obj);
     unsigned int n = mysql_num_fields(res);
-    VALUE ary;
+    VALUE ary = Qnil;
     if(build_array)
   	ary = rb_ary_new();
     MYSQL_ROW row = mysql_fetch_row(res); // grab one off the top, to determine the rows
@@ -1203,6 +1229,8 @@ static VALUE process_all_hashes(VALUE obj, VALUE with_table, int build_array, in
    
     if(yield)
 	return obj;
+
+    return Qnil; /* we should never get here -- this takes out a compiler warning */
 }
 
 /*	fetch_hash2 (internal)	*/
@@ -1586,12 +1614,12 @@ static VALUE stmt_execute(int argc, VALUE *argv, VALUE obj)
                     s->param.bind[i].buffer = &(s->param.buffer[i]);
                     t.second_part = 0;
                     t.neg = 0;
-                    t.second = FIX2INT(RARRAY(a)->ptr[0]);
-                    t.minute = FIX2INT(RARRAY(a)->ptr[1]);
-                    t.hour = FIX2INT(RARRAY(a)->ptr[2]);
-                    t.day = FIX2INT(RARRAY(a)->ptr[3]);
-                    t.month = FIX2INT(RARRAY(a)->ptr[4]);
-                    t.year = FIX2INT(RARRAY(a)->ptr[5]);
+                    t.second = FIX2INT(rb_ary_entry(a, 0));
+                    t.minute = FIX2INT(rb_ary_entry(a, 1));
+                    t.hour = FIX2INT(rb_ary_entry(a, 2));
+                    t.day = FIX2INT(rb_ary_entry(a, 3));
+                    t.month = FIX2INT(rb_ary_entry(a, 4));
+                    t.year = FIX2INT(rb_ary_entry(a, 5));
                     *(MYSQL_TIME*)&(s->param.buffer[i]) = t;
                 } else if (CLASS_OF(argv[i]) == cMysqlTime) {
                     MYSQL_TIME t;

data/lib/mysqlplus.rb

@@ -1,19 +1,21 @@
-require 'mysql'
+require 'mysql' # this should load the mysqlplus version of mysql.so, as we assume the user has installed mysql as a gem and have not done any previous "require 'mysql'" to have loaded the other
 
+#
+# Mysqlplus library gives you a [slightly modified] version of the Mysql class
+# See http://www.kitebird.com/articles/ruby-mysql.html for details, as well as the test directory within the library
+#
 class Mysql
   
-  def async_query(sql, timeout = nil)
+  def ruby_async_query(sql, timeout = nil) # known to deadlock TODO
     send_query(sql)
     select [ (@sockets ||= {})[socket] ||= IO.new(socket) ], nil, nil, nil
     get_result
   end
-  
-end
 
-class Mysql::Result
-  def all_hashes
-    rows = []
-    each_hash { |row| rows << row }
-    rows
+  begin
+    alias_method :async_query, :c_async_query  
+  rescue NameError => e
+    raise LoadError.new "error loading mysqlplus--this may mean you ran a require 'mysql' before a require 'mysqplus', which much come first"
   end
+  
 end

data/mysqlplus.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
   s.name     = "mysqlplus"
-  s.version  = "0.1.0"
-  s.date     = "2008-08-13"
+  s.version  = "0.1.1"
+  s.date     = "2009-03-22"
   s.summary  = "Enhanced Ruby MySQL driver"
   s.email    = "oldmoe@gmail.com"
   s.homepage = "http://github.com/oldmoe/mysqlplus"
@@ -9,19 +9,25 @@ Gem::Specification.new do |s|
   s.has_rdoc = true
   s.authors  = ["Muhammad A. Ali"]
   s.platform = Gem::Platform::RUBY
-  s.files    = [ 
-		"mysqlplus.gemspec", 
-		"README",
-		"Rakefile",
-		"lib/mysqlplus.rb",
-		"test/test_helper.rb",
-		"test/native_threaded_test.rb",
-		"test/c_threaded_test.rb",
-		"test/evented_test.rb",
-		"ext/error_const.h",
-		"ext/extconf.rb",
-		"ext/mysql.c"
-	]
+  s.files    = %w[
+    README
+    Rakefile
+    TODO_LIST
+    ext/error_const.h
+    ext/extconf.rb
+    ext/mysql.c
+    lib/mysqlplus.rb
+    mysqlplus.gemspec
+    test/c_threaded_test.rb
+    test/evented_test.rb
+    test/native_threaded_test.rb
+    test/test_all_hashes.rb
+    test/test_failure.rb
+    test/test_helper.rb
+    test/test_many_requests.rb
+    test/test_parsing_while_response_is_being_read.rb
+    test/test_threaded_sequel.rb
+  ]
   s.rdoc_options = ["--main", "README"]
   s.extra_rdoc_files = ["README"]
   s.extensions << "ext/extconf.rb"

data/test/c_threaded_test.rb

@@ -3,34 +3,34 @@ require File.dirname(__FILE__) + '/test_helper'
 ThreadedMysqlTest.new( 10, "Threaded, C, very small overhead"  ) do |test|
   test.setup{ Mysql.real_connect('localhost','root') }
   test.per_query_overhead = 0.005
-  test.c_async_query = true
+  test.query_with = :c_async_query 
   test.run!
 end
 
 ThreadedMysqlTest.new( 10, "Threaded, C, small overhead"  ) do |test|
   test.setup{ Mysql.real_connect('localhost','root') }
   test.per_query_overhead = 0.1
-  test.c_async_query = true
+  test.query_with = :c_async_query 
   test.run!
 end
 
 ThreadedMysqlTest.new( 10, "Threaded, C, medium overhead"  ) do |test|
   test.setup{ Mysql.real_connect('localhost','root') }
   test.per_query_overhead = 1
-  test.c_async_query = true
+  test.query_with = :c_async_query 
   test.run!
 end
 
 ThreadedMysqlTest.new( 10, "Threaded, C, large overhead"  ) do |test|
   test.setup{ Mysql.real_connect('localhost','root') }
   test.per_query_overhead = 3
-  test.c_async_query = true
+  test.query_with = :c_async_query 
   test.run!
 end
 
 ThreadedMysqlTest.new( 10, "Threaded, C, random overhead"  ) do |test|
   test.setup{ Mysql.real_connect('localhost','root') }
   test.per_query_overhead = :random
-  test.c_async_query = true
+  test.query_with = :c_async_query 
   test.run!
 end

data/test/native_threaded_test.rb

@@ -3,29 +3,34 @@ require File.dirname(__FILE__) + '/test_helper'
 ThreadedMysqlTest.new( 10, "Threaded, native Ruby, very small overhead"  ) do |test|
   test.setup{ Mysql.real_connect('localhost','root') }
   test.per_query_overhead = 0.005
+  test.query_with = :async_query   
   test.run!
 end
 
 ThreadedMysqlTest.new( 10, "Threaded, native Ruby, small overhead"  ) do |test|
   test.setup{ Mysql.real_connect('localhost','root') }
   test.per_query_overhead = 0.1
+  test.query_with = :async_query     
   test.run!
 end
 
 ThreadedMysqlTest.new( 10, "Threaded, native Ruby, medium overhead"  ) do |test|
   test.setup{ Mysql.real_connect('localhost','root') }
   test.per_query_overhead = 1
+  test.query_with = :async_query     
   test.run!
 end
 
 ThreadedMysqlTest.new( 10, "Threaded, native Ruby, large overhead"  ) do |test|
   test.setup{ Mysql.real_connect('localhost','root') }
   test.per_query_overhead = 3
+  test.query_with = :async_query     
   test.run!
 end
 
 ThreadedMysqlTest.new( 10, "Threaded, native Ruby, random overhead"  ) do |test|
   test.setup{ Mysql.real_connect('localhost','root') }
   test.per_query_overhead = :random
+  test.query_with = :async_query     
   test.run!
 end

data/test/test_all_hashes.rb

@@ -0,0 +1,43 @@
+# shows the effect of using .all_hashes instead of looping on each hash 
+# run it by substiting in a 'long' [many row] query for the query variable and toggling use_all_hashes here at the top
+# note that we load all the rows first, then run .all_hashes on the result [to see more easily the effect of all hashes]
+# on my machine and a 200_000 row table, it took 3.38s versus 3.65s
+require 'rubygems'
+require 'mysqlplus'
+
+use_the_all_hashes_method = true 
+
+$count = 5
+
+$start = Time.now
+
+$connections = []
+$count.times do  
+  $connections << Mysql.real_connect('localhost','root', '', 'local_leadgen_dev')
+end
+
+puts 'connection pool ready'
+
+$threads = []
+$count.times do |i|
+  $threads << Thread.new do
+
+    query = "select * from campus_zips"
+    puts "sending query on connection #{i}"
+    conn = $connections[i]
+    result = conn.async_query(query)
+    if use_the_all_hashes_method
+      saved =  result.all_hashes
+    else
+      saved = []
+      result.each_hash {|h| saved << h } 
+    end
+    result.free
+
+  end
+end
+
+puts 'waiting on threads'
+$threads.each{|t| t.join }
+
+puts Time.now - $start

data/test/test_failure.rb

@@ -0,0 +1,17 @@
+require 'rubygems'
+require 'mysqlplus'
+begin
+  Mysql.real_connect('fakehost','root', '', 'local_leadgen_dev')
+rescue Mysql::Error
+end
+begin
+  Mysql.real_connect('localhost','root', '', 'faketable')
+rescue Mysql::Error
+end
+begin
+  Mysql.real_connect('localhost', 'root', 'pass', 'db', 3307)# bad port
+rescue Mysql::Error
+end
+print "pass"
+
+

data/test/test_helper.rb

@@ -12,7 +12,7 @@ class MysqlTest
                 :connection_signature,
                 :start,
                 :done,
-                :c_async_query,
+                :query_with,
                 :per_query_overhead,
                 :timeout
   
@@ -20,7 +20,7 @@ class MysqlTest
     @queries = queries
     @context = context
     @done = []
-    @c_async_query = false
+    @query_with = :async_query
     @per_query_overhead = 3
     @timeout = 20
     yield self if block_given?
@@ -78,7 +78,7 @@ class MysqlTest
   end
   
   def c_or_native_ruby_async_query
-    if @c_async_query
+    if @query_with == :c_async_query
       log "** using C based async_query"
     else
       log "** using native Ruby async_query"
@@ -86,9 +86,8 @@ class MysqlTest
     yield
   end
   
-  def c_or_native_async_query( connection, sql, timeout = nil )
-    method = @c_async_query ? :c_async_query : :async_query
-    connection.send( method, sql, timeout )
+  def dispatch_query( connection, sql, timeout = nil )
+    connection.send( @query_with, sql, timeout )
   end
   
 end
@@ -188,7 +187,7 @@ class ThreadedMysqlTest < MysqlTest
 
           log "sending query on connection #{conn}"
 
-          c_or_native_async_query( @connections[conn], "select sleep(#{@per_query_overhead})", @timeout ).each do |result|
+          dispatch_query( @connections[conn], "select sleep(#{@per_query_overhead})", @timeout ).each do |result|
             log "connection #{conn} done"
           end 
         

data/test/test_many_requests.rb

@@ -0,0 +1,7 @@
+require 'rubygems'
+require 'mysqlplus'
+a = Mysql.real_connect('localhost','root')
+100.times { a.query("select sleep(0)") }
+print "pass"
+
+

data/test/test_parsing_while_response_is_being_read.rb

@@ -0,0 +1,48 @@
+# This is an example of using Mysql::ResultSet#use_result [see docs for what that does]
+# this function is useful for those who have large query results and want to be able to parse them
+# as they come in, instead of having to wait for the query to finish before doing parsing
+# for me, running this on a query with 200_000 lines decreases total time to create an array of results 
+# from .82s to .62s
+# you can experiment with it by changing the query here to be a long one, and toggling the do_the_use_query_optimization variable
+# this also has the interesting property of 'freeing' Ruby to do thread changes mid-query.
+require 'rubygems'
+require 'mysqlplus'
+
+do_the_use_query_optimization = true
+
+$count = 5
+
+$start = Time.now
+
+$connections = []
+$count.times do  
+  $connections << Mysql.real_connect('localhost','root', '', 'local_leadgen_dev')
+end
+
+puts 'connection pool ready'
+
+$threads = []
+$count.times do |i|
+  $threads << Thread.new do
+
+    puts "sending query on connection #{i}"
+    conn = $connections[i]
+    saved = []
+    query = "select * from campus_zips"
+    if do_the_use_query_optimization
+      conn.query_with_result=false
+      result = conn.async_query(query)
+      res = result.use_result
+      res.each_hash { |h| saved << h }
+      res.free
+    else
+      conn.async_query(query).each_hash {|h| saved << h }
+    end
+
+  end
+end
+
+puts 'waiting on threads'
+$threads.each{|t| t.join }
+
+puts Time.now - $start

data/test/test_threaded_sequel.rb

@@ -0,0 +1,24 @@
+require 'rubygems'
+require 'sequel'
+
+require 'mysqlplus'
+class Mysql
+  unless method_defined? :sync_query
+    alias :sync_query :query
+    alias :query :async_query
+  end
+end
+
+DB = Sequel.connect('mysql://root@localhost', :max_connections => 20)
+
+start = Time.now
+
+(0..10).map do
+  Thread.new do
+
+    p DB['select sleep(2)'].all
+
+  end
+end.map{|t| t.join }
+
+p (Time.now - start)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: oldmoe-mysqlplus
 version: !ruby/object:Gem::Version 
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors: 
 - Muhammad A. Ali
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2008-08-13 00:00:00 -07:00
+date: 2009-03-22 00:00:00 -07:00
 default_executable: 
 dependencies: []
 
@@ -22,17 +22,23 @@ extensions:
 extra_rdoc_files: 
 - README
 files: 
-- mysqlplus.gemspec
 - README
 - Rakefile
-- lib/mysqlplus.rb
-- test/test_helper.rb
-- test/native_threaded_test.rb
-- test/c_threaded_test.rb
-- test/evented_test.rb
+- TODO_LIST
 - ext/error_const.h
 - ext/extconf.rb
 - ext/mysql.c
+- lib/mysqlplus.rb
+- mysqlplus.gemspec
+- test/c_threaded_test.rb
+- test/evented_test.rb
+- test/native_threaded_test.rb
+- test/test_all_hashes.rb
+- test/test_failure.rb
+- test/test_helper.rb
+- test/test_many_requests.rb
+- test/test_parsing_while_response_is_being_read.rb
+- test/test_threaded_sequel.rb
 has_rdoc: true
 homepage: http://github.com/oldmoe/mysqlplus
 post_install_message: