mysql2 0.2.7 → 0.2.8

data/CHANGELOG.md

@@ -1,5 +1,12 @@
 # Changelog
 
+## 0.2.8 (June 14th, 2011)
+* disable async support, and access to the underlying file descriptor under Windows. It's never worked reliably and ruby-core has a lot of work to do in order to make it possible.
+* added support for turning eager-casting off. This is especially useful in ORMs that will lazily cast values upon access.
+* added a warning if a 0.2.x release is being used with ActiveRecord 3.1 since both the 0.2.x releases and AR 3.1 have mysql2 adapters, we want you to use the one in AR 3.1
+* added Mysql2::Client.escape (class-level method)
+* disabled eager-casting in the bundled ActiveRecord adapter (for Rails 3.0 or less)
+
 ## 0.2.7 (March 28th, 2011)
 * various fixes for em_mysql2 and fiber usage
 * use our own Mysql2IndexDefinition class for better compatibility across ActiveRecord versions

data/README.md

@@ -0,0 +1,324 @@
+# Mysql2 - A modern, simple and very fast Mysql library for Ruby - binding to libmysql
+
+The Mysql2 gem is meant to serve the extremely common use-case of connecting, querying and iterating on results.
+Some database libraries out there serve as direct 1:1 mappings of the already complex C API's available.
+This one is not.
+
+It also forces the use of UTF-8 [or binary] for the connection [and all strings in 1.9, unless Encoding.default_internal is set then it'll convert from UTF-8 to that encoding] and uses encoding-aware MySQL API calls where it can.
+
+The API consists of two clases:
+
+Mysql2::Client - your connection to the database
+
+Mysql2::Result - returned from issuing a #query on the connection. It includes Enumerable.
+
+## Installing
+
+``` sh
+gem install mysql2
+```
+
+You may have to specify --with-mysql-config=/some/random/path/bin/mysql_config
+
+## Usage
+
+Connect to a database:
+
+``` ruby
+# this takes a hash of options, almost all of which map directly
+# to the familiar database.yml in rails
+# See http://api.rubyonrails.org/classes/ActiveRecord/ConnectionAdapters/MysqlAdapter.html
+client = Mysql2::Client.new(:host => "localhost", :username => "root")
+```
+
+Then query it:
+
+``` ruby
+results = client.query("SELECT * FROM users WHERE group='githubbers'")
+```
+
+Need to escape something first?
+
+``` ruby
+escaped = client.escape("gi'thu\"bbe\0r's")
+results = client.query("SELECT * FROM users WHERE group='#{escaped}'")
+```
+
+Finally, iterate over the results:
+
+``` ruby
+results.each do |row|
+  # conveniently, row is a hash
+  # the keys are the fields, as you'd expect
+  # the values are pre-built ruby primitives mapped from their corresponding field types in MySQL
+  # Here's an otter: http://farm1.static.flickr.com/130/398077070_b8795d0ef3_b.jpg
+end
+```
+
+Or, you might just keep it simple:
+
+``` ruby
+client.query("SELECT * FROM users WHERE group='githubbers'").each do |row|
+  # do something with row, it's ready to rock
+end
+```
+
+How about with symbolized keys?
+
+``` ruby
+# NOTE: the :symbolize_keys and future options will likely move to the #query method soon
+client.query("SELECT * FROM users WHERE group='githubbers'").each(:symbolize_keys => true) do |row|
+  # do something with row, it's ready to rock
+end
+```
+
+You can get the headers and the columns in the order that they were returned
+by the query like this:
+
+``` ruby
+headers = results.fields # <= that's an array of field names, in order
+results.each(:as => :array) do |row|
+# Each row is an array, ordered the same as the query results
+# An otter's den is called a "holt" or "couch"
+end
+```
+
+## Cascading config
+
+The default config hash is at:
+
+``` ruby
+Mysql2::Client.default_query_options
+```
+
+which defaults to:
+
+``` ruby
+{:async => false, :as => :hash, :symbolize_keys => false}
+```
+
+that can be used as so:
+
+``` ruby
+# these are the defaults all Mysql2::Client instances inherit
+Mysql2::Client.default_query_options.merge!(:as => :array)
+```
+
+or
+
+``` ruby
+# this will change the defaults for all future results returned by the #query method _for this connection only_
+c = Mysql2::Client.new
+c.query_options.merge!(:symbolize_keys => true)
+```
+
+or
+
+``` ruby
+# this will set the options for the Mysql2::Result instance returned from the #query method
+c = Mysql2::Client.new
+c.query(sql, :symbolize_keys => true)
+```
+
+## Result types
+
+### Array of Arrays
+
+Pass the `:as => :array` option to any of the above methods of configuration
+
+### Array of Hashes
+
+The default result type is set to :hash, but you can override a previous setting to something else with :as => :hash
+
+### Others...
+
+I may add support for `:as => :csv` or even `:as => :json` to allow for *much* more efficient generation of those data types from result sets.
+If you'd like to see either of these (or others), open an issue and start bugging me about it ;)
+
+### Timezones
+
+Mysql2 now supports two timezone options:
+
+``` ruby
+:database_timezone # this is the timezone Mysql2 will assume fields are already stored as, and will use this when creating the initial Time objects in ruby
+:application_timezone # this is the timezone Mysql2 will convert to before finally handing back to the caller
+```
+
+In other words, if `:database_timezone` is set to `:utc` - Mysql2 will create the Time objects using `Time.utc(...)` from the raw value libmysql hands over initially.
+Then, if `:application_timezone` is set to say - `:local` - Mysql2 will then convert the just-created UTC Time object to local time.
+
+Both options only allow two values - `:local` or `:utc` - with the exception that `:application_timezone` can be [and defaults to] nil
+
+### Casting "boolean" columns
+
+You can now tell Mysql2 to cast `tinyint(1)` fields to boolean values in Ruby with the `:cast_booleans` option.
+
+``` ruby
+client = Mysql2::Client.new
+result = client.query("SELECT * FROM table_with_boolean_field", :cast_booleans => true)
+```
+
+### Skipping casting
+
+Mysql2 casting is fast, but not as fast as not casting data.  In rare cases where typecasting is not needed, it will be faster to disable it by providing :cast => false.
+
+``` ruby
+client = Mysql2::Client.new
+result = client.query("SELECT * FROM table", :cast => false)
+```
+
+Here are the results from the `query_without_mysql_casting.rb` script in the benchmarks folder:
+
+``` sh
+                           user     system      total        real
+Mysql2 (cast: true)    0.340000   0.000000   0.340000 (  0.405018)
+Mysql2 (cast: false)   0.160000   0.010000   0.170000 (  0.209937)
+Mysql                  0.080000   0.000000   0.080000 (  0.129355)
+do_mysql               0.520000   0.010000   0.530000 (  0.574619)
+```
+
+Although Mysql2 performs reasonably well at retrieving uncasted data, it (currently) is not as fast as the Mysql gem.  In spite of this small disadvantage, Mysql2 still sports a friendlier interface and doesn't block the entire ruby process when querying.
+
+### Async
+
+NOTE: Not supported on Windows.
+
+`Mysql2::Client` takes advantage of the MySQL C API's (undocumented) non-blocking function mysql_send_query for *all* queries.
+But, in order to take full advantage of it in your Ruby code, you can do:
+
+``` ruby
+client.query("SELECT sleep(5)", :async => true)
+```
+
+Which will return nil immediately. At this point you'll probably want to use some socket monitoring mechanism
+like EventMachine or even IO.select. Once the socket becomes readable, you can do:
+
+``` ruby
+# result will be a Mysql2::Result instance
+result = client.async_result
+```
+
+NOTE: Because of the way MySQL's query API works, this method will block until the result is ready.
+So if you really need things to stay async, it's best to just monitor the socket with something like EventMachine.
+If you need multiple query concurrency take a look at using a connection pool.
+
+### Row Caching
+
+By default, Mysql2 will cache rows that have been created in Ruby (since this happens lazily).
+This is especially helpful since it saves the cost of creating the row in Ruby if you were to iterate over the collection again.
+
+If you only plan on using each row once, then it's much more efficient to disable this behavior by setting the `:cache_rows` option to false.
+This would be helpful if you wanted to iterate over the results in a streaming manner. Meaning the GC would cleanup rows you don't need anymore as you're iterating over the result set.
+
+## ActiveRecord
+
+To use the ActiveRecord driver (with our without rails), all you should need to do is have this gem installed and set the adapter in your database.yml to "mysql2".
+That was easy right? :)
+
+NOTE: as of 0.3.0, and ActiveRecord 3.1 - the ActiveRecord adapter has been pulled out of this gem and into ActiveRecord itself. If you need to use mysql2 with
+Rails versions < 3.1 make sure and specify `gem "mysql2", "~> 0.2.7"` in your Gemfile
+
+## Asynchronous ActiveRecord
+
+You can also use Mysql2 with asynchronous Rails (first introduced at http://www.mikeperham.com/2010/04/03/introducing-phat-an-asynchronous-rails-app/) by
+setting the adapter in your database.yml to "em_mysql2".  You must be running Ruby 1.9, thin and the rack-fiber_pool middleware for it to work.
+
+## Sequel
+
+The Sequel adapter was pulled out into Sequel core (will be part of the next release) and can be used by specifying the "mysql2://" prefix to your connection specification.
+
+## EventMachine
+
+The mysql2 EventMachine deferrable api allows you to make async queries using EventMachine,
+while specifying callbacks for success for failure. Here's a simple example:
+
+``` ruby
+require 'mysql2/em'
+
+EM.run do
+  client1 = Mysql2::EM::Client.new
+  defer1 = client1.query "SELECT sleep(3) as first_query"
+  defer1.callback do |result|
+    puts "Result: #{result.to_a.inspect}"
+  end
+
+  client2 = Mysql2::EM::Client.new
+  defer2 = client2.query "SELECT sleep(1) second_query"
+  defer2.callback do |result|
+    puts "Result: #{result.to_a.inspect}"
+  end
+end
+```
+
+## Lazy Everything
+
+Well... almost ;)
+
+Field name strings/symbols are shared across all the rows so only one object is ever created to represent the field name for an entire dataset.
+
+Rows themselves are lazily created in ruby-land when an attempt to yield it is made via #each.
+For example, if you were to yield 4 rows from a 100 row dataset, only 4 hashes will be created. The rest will sit and wait in C-land until you want them (or when the GC goes to cleanup your `Mysql2::Result` instance).
+Now say you were to iterate over that same collection again, this time yielding 15 rows - the 4 previous rows that had already been turned into ruby hashes would be pulled from an internal cache, then 11 more would be created and stored in that cache.
+Once the entire dataset has been converted into ruby objects, Mysql2::Result will free the Mysql C result object as it's no longer needed.
+
+This caching behavior can be disabled by setting the :cache_rows option to false.
+
+As for field values themselves, I'm workin on it - but expect that soon.
+
+## Compatibility
+
+The specs pass on my system (SL 10.6.3, x86_64) in these rubies:
+
+* 1.8.7-p249
+* ree-1.8.7-2010.01
+* 1.9.1-p378
+* ruby-trunk
+* rbx-head - broken at the moment, working with the rbx team for a solution
+
+The ActiveRecord driver should work on 2.3.5 and 3.0
+
+## Yeah... but why?
+
+Someone: Dude, the Mysql gem works fiiiiiine.
+
+Me: It sure does, but it only hands you nil and strings for field values. Leaving you to convert
+them into proper Ruby types in Ruby-land - which is slow as balls.
+
+
+Someone: OK fine, but do_mysql can already give me back values with Ruby objects mapped to MySQL types.
+
+Me: Yep, but it's API is considerably more complex *and* can be ~2x slower.
+
+## Benchmarks
+
+Performing a basic "SELECT * FROM" query on a table with 30k rows and fields of nearly every Ruby-representable data type,
+then iterating over every row using an #each like method yielding a block:
+
+These results are from the `query_with_mysql_casting.rb` script in the benchmarks folder
+
+``` sh
+ user       system     total       real
+Mysql2
+ 0.750000   0.180000   0.930000 (  1.821655)
+do_mysql
+ 1.650000   0.200000   1.850000 (  2.811357)
+Mysql
+ 7.500000   0.210000   7.710000 (  8.065871)
+```
+
+## Development
+
+To run the tests, you can use RVM and Bundler to create a pristine environment for mysql2 development/hacking.
+Use 'bundle install' to install the necessary development and testing gems:
+
+``` sh
+bundle install
+rake
+```
+
+## Special Thanks
+
+* Eric Wong - for the contribution (and the informative explanations) of some thread-safety, non-blocking I/O and cleanup patches. You rock dude
+* Yury Korolev (http://github.com/yury) - for TONS of help testing the ActiveRecord adapter
+* Aaron Patterson (http://github.com/tenderlove) - tons of contributions, suggestions and general badassness
+* Mike Perham (http://github.com/mperham) - Async ActiveRecord adapter (uses Fibers and EventMachine)

data/benchmark/active_record_threaded.rb

@@ -0,0 +1,42 @@
+# encoding: UTF-8
+$LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__) + '/../lib')
+
+require 'rubygems'
+require 'benchmark'
+require 'active_record'
+
+times = 25
+
+
+# mysql2
+mysql2_opts = {
+  :adapter => 'mysql2',
+  :database => 'test',
+  :pool => times
+}
+ActiveRecord::Base.establish_connection(mysql2_opts)
+x = Benchmark.realtime do
+  threads = []
+  times.times do
+    threads << Thread.new { ActiveRecord::Base.connection.execute("select sleep(1)") }
+  end
+  threads.each {|t| t.join }
+end
+puts "mysql2: #{x} seconds"
+
+
+# mysql
+mysql2_opts = {
+  :adapter => 'mysql',
+  :database => 'test',
+  :pool => times
+}
+ActiveRecord::Base.establish_connection(mysql2_opts)
+x = Benchmark.realtime do
+  threads = []
+  times.times do
+    threads << Thread.new { ActiveRecord::Base.connection.execute("select sleep(1)") }
+  end
+  threads.each {|t| t.join }
+end
+puts "mysql: #{x} seconds"

data/benchmark/query_without_mysql_casting.rb

@@ -14,9 +14,18 @@ sql = "SELECT * FROM mysql2_test LIMIT 100"
 Benchmark.bmbm do |x|
   mysql2 = Mysql2::Client.new(:host => "localhost", :username => "root")
   mysql2.query "USE #{database}"
-  x.report "Mysql2" do
+  x.report "Mysql2 (cast: true)" do
     number_of.times do
-      mysql2_result = mysql2.query sql, :symbolize_keys => true
+      mysql2_result = mysql2.query sql, :symbolize_keys => true, :cast => true
+      mysql2_result.each do |res|
+        # puts res.inspect
+      end
+    end
+  end
+
+  x.report "Mysql2 (cast: false)" do
+    number_of.times do
+      mysql2_result = mysql2.query sql, :symbolize_keys => true, :cast => false
       mysql2_result.each do |res|
         # puts res.inspect
       end

data/benchmark/threaded.rb

@@ -0,0 +1,44 @@
+# encoding: UTF-8
+$LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__) + '/../lib')
+
+require 'rubygems'
+require 'benchmark'
+require 'active_record'
+
+mysql2_opts = {
+  :adapter => 'mysql2',
+  :database => 'test',
+  :pool => 25
+}
+ActiveRecord::Base.establish_connection(mysql2_opts)
+x = Benchmark.realtime do
+  threads = []
+  25.times do
+    threads << Thread.new { ActiveRecord::Base.connection.execute("select sleep(1)") }
+  end
+  threads.each {|t| t.join }
+end
+puts x
+
+mysql2_opts = {
+  :adapter => 'mysql',
+  :database => 'test',
+  :pool => 25
+}
+ActiveRecord::Base.establish_connection(mysql2_opts)
+x = Benchmark.realtime do
+  threads = []
+  25.times do
+    threads << Thread.new { ActiveRecord::Base.connection.execute("select sleep(1)") }
+  end
+  threads.each {|t| t.join }
+end
+puts x
+
+# these results are similar on 1.8.7, 1.9.2 and rbx-head
+#
+# $ bundle exec ruby benchmarks/threaded.rb
+# 1.0774750709533691
+#
+# and using the mysql gem
+# 25.099437952041626

data/ext/mysql2/client.c

@@ -139,10 +139,6 @@ static VALUE nogvl_close(void *ptr) {
     flags = fcntl(wrapper->client->net.fd, F_GETFL);
     if (flags > 0 && !(flags & O_NONBLOCK))
       fcntl(wrapper->client->net.fd, F_SETFL, flags | O_NONBLOCK);
-#else
-    u_long iMode;
-    iMode = 1;
-    ioctlsocket(wrapper->client->net.fd, FIONBIO, &iMode);
 #endif
 
     mysql_close(wrapper->client);
@@ -171,6 +167,31 @@ static VALUE allocate(VALUE klass) {
   return obj;
 }
 
+static VALUE rb_mysql_client_escape(VALUE klass, VALUE str) {
+  unsigned char *newStr;
+  VALUE rb_str;
+  unsigned long newLen, oldLen;
+
+  Check_Type(str, T_STRING);
+
+  oldLen = RSTRING_LEN(str);
+  newStr = xmalloc(oldLen*2+1);
+
+  newLen = mysql_escape_string((char *)newStr, StringValuePtr(str), oldLen);
+  if (newLen == oldLen) {
+    // no need to return a new ruby string if nothing changed
+    xfree(newStr);
+    return str;
+  } else {
+    rb_str = rb_str_new((const char*)newStr, newLen);
+#ifdef HAVE_RUBY_ENCODING_H
+    rb_enc_copy(rb_str, str);
+#endif
+    xfree(newStr);
+    return rb_str;
+  }
+}
+
 static VALUE rb_connect(VALUE self, VALUE user, VALUE pass, VALUE host, VALUE port, VALUE database, VALUE socket, VALUE flags) {
   struct nogvl_connect_args args;
   GET_CLIENT(self);
@@ -331,6 +352,7 @@ static VALUE rb_mysql_client_query(int argc, VALUE * argv, VALUE self) {
     return rb_raise_mysql2_error(wrapper);
   }
 
+#ifndef _WIN32
   read_timeout = rb_iv_get(self, "@read_timeout");
 
   tvp = NULL;
@@ -355,27 +377,11 @@ static VALUE rb_mysql_client_query(int argc, VALUE * argv, VALUE self) {
     for(;;) {
       int fd_set_fd = fd;
 
-#ifdef _WIN32
-      WSAPROTOCOL_INFO wsa_pi;
-      // dupicate the SOCKET from libmysql
-      int r = WSADuplicateSocket(fd, GetCurrentProcessId(), &wsa_pi);
-      SOCKET s = WSASocket(wsa_pi.iAddressFamily, wsa_pi.iSocketType, wsa_pi.iProtocol, &wsa_pi, 0, 0);
-      // create the CRT fd so ruby can get back to the SOCKET
-      fd_set_fd = _open_osfhandle(s, O_RDWR|O_BINARY);
-#endif
-
       FD_ZERO(&fdset);
       FD_SET(fd_set_fd, &fdset);
 
       retval = rb_thread_select(fd_set_fd + 1, &fdset, NULL, NULL, tvp);
 
-#ifdef _WIN32
-      // cleanup the CRT fd
-      _close(fd_set_fd);
-      // cleanup the duplicated SOCKET
-      closesocket(s);
-#endif
-
       if (retval == 0) {
         rb_raise(cMysql2Error, "Timeout waiting for a response from the last query. (waited %d seconds)", FIX2INT(read_timeout));
       }
@@ -395,9 +401,13 @@ static VALUE rb_mysql_client_query(int argc, VALUE * argv, VALUE self) {
   } else {
     return Qnil;
   }
+#else
+  // this will just block until the result is ready
+  return rb_mysql_client_async_result(self);
+#endif
 }
 
-static VALUE rb_mysql_client_escape(VALUE self, VALUE str) {
+static VALUE rb_mysql_client_real_escape(VALUE self, VALUE str) {
   unsigned char *newStr;
   VALUE rb_str;
   unsigned long newLen, oldLen;
@@ -492,8 +502,13 @@ static VALUE rb_mysql_client_server_info(VALUE self) {
 
 static VALUE rb_mysql_client_socket(VALUE self) {
   GET_CLIENT(self);
+#ifndef _WIN32
   REQUIRE_OPEN_DB(wrapper);
-  return INT2NUM(wrapper->client->net.fd);
+  int fd_set_fd = wrapper->client->net.fd;
+  return INT2NUM(fd_set_fd);
+#else
+  rb_raise(cMysql2Error, "Raw access to the mysql file descriptor isn't supported on Windows");
+#endif
 }
 
 static VALUE rb_mysql_client_last_id(VALUE self) {
@@ -523,8 +538,7 @@ static VALUE rb_mysql_client_thread_id(VALUE self) {
   return ULL2NUM(retVal);
 }
 
-static VALUE nogvl_ping(void *ptr)
-{
+static VALUE nogvl_ping(void *ptr) {
   MYSQL *client = ptr;
 
   return mysql_ping(client) == 0 ? Qtrue : Qfalse;
@@ -533,7 +547,11 @@ static VALUE nogvl_ping(void *ptr)
 static VALUE rb_mysql_client_ping(VALUE self) {
   GET_CLIENT(self);
 
-  return rb_thread_blocking_region(nogvl_ping, wrapper->client, RUBY_UBF_IO, 0);
+  if (wrapper->closed) {
+    return Qfalse;
+  } else {
+    return rb_thread_blocking_region(nogvl_ping, wrapper->client, RUBY_UBF_IO, 0);
+  }
 }
 
 #ifdef HAVE_RUBY_ENCODING_H
@@ -654,9 +672,11 @@ void init_mysql2_client() {
 
   rb_define_alloc_func(cMysql2Client, allocate);
 
+  rb_define_singleton_method(cMysql2Client, "escape", rb_mysql_client_escape, 1);
+
   rb_define_method(cMysql2Client, "close", rb_mysql_client_close, 0);
   rb_define_method(cMysql2Client, "query", rb_mysql_client_query, -1);
-  rb_define_method(cMysql2Client, "escape", rb_mysql_client_escape, 1);
+  rb_define_method(cMysql2Client, "escape", rb_mysql_client_real_escape, 1);
   rb_define_method(cMysql2Client, "info", rb_mysql_client_info, 0);
   rb_define_method(cMysql2Client, "server_info", rb_mysql_client_server_info, 0);
   rb_define_method(cMysql2Client, "socket", rb_mysql_client_socket, 0);