julien51-em-mysql 0.3.0

data/README

@@ -0,0 +1,40 @@
+Async MySQL driver for Ruby/EventMachine
+  (c) 2008 Aman Gupta (tmm1)
+
+
+Requires mysqlplus.
+
+  require 'em/mysql'
+
+  # alias SQL for simpler syntax
+
+  SQL = EventedMysql
+  def SQL(query, &blk) SQL.select(query, &blk) end
+
+  
+  # setup connection details and allow 4 connections to the server
+
+  SQL.settings.update :host => 'localhost',
+                      :port => 3306,
+                      :database => 'test',
+                      :connections => 4
+  
+
+  # use 4 connections to execute queries in parallel
+
+  SQL('select sleep(0.25)'){ p 'done' }
+  SQL('select sleep(0.25)'){ p 'done' }
+  SQL('select sleep(0.25)'){ p 'done' }
+  SQL('select sleep(0.25)'){ p 'done' }
+
+Also includes a sequel async wrapper
+
+  require 'sequel'
+  require 'sequel/async'
+
+  DB = Sequel.connect(:adapter => 'mysql', :user => 'root', :database => 'test', ...)
+  EventedMysql.settings.update(..., :on_error => proc{|e| log 'error', e })
+
+  DB[:table].where(:field => 'value').async_update(:field => 'new value')
+
+For more info, see the comments in lib/sequel/async.rb

data/em-mysql.gemspec

@@ -0,0 +1,21 @@
+spec = Gem::Specification.new do |s|
+  s.name = 'em-mysql'
+  s.version = '0.3.0'
+  s.date = '2009-06-23'
+  s.summary = 'Async MySQL client API for Ruby/EventMachine'
+  s.email = "em-mysql@tmm1.net"
+  s.homepage = "http://github.com/tmm1/em-mysql"
+  s.description = 'Async MySQL client API for Ruby/EventMachine'
+  s.has_rdoc = false
+  s.authors = ["Aman Gupta"]
+  s.add_dependency('eventmachine', '>= 0.12.8')
+
+  # git ls-files
+  s.files = %w[
+    README
+    em-mysql.gemspec
+    lib/em/mysql.rb
+    lib/sequel/async.rb
+    test.rb
+  ]
+end

data/lib/em/mysql.rb

@@ -0,0 +1,492 @@
+require 'rubygems'
+require 'eventmachine'
+require 'mysqlplus'
+require 'fcntl'
+
+class Mysql
+  def result
+    @cur_result
+  end
+end
+
+class EventedMysql < EM::Connection
+  def initialize mysql, opts
+    @mysql = mysql
+    @fd = mysql.socket
+    @opts = opts
+    @current = nil
+    @@queue ||= []
+    @processing = false
+    @connected = true
+
+    log 'mysql connected'
+    make_socket_blocking
+    EM.add_timer(0){ next_query }
+  end
+  attr_reader :processing, :connected, :opts
+  alias :settings :opts
+
+  DisconnectErrors = [
+    'query: not connected',
+    'MySQL server has gone away',
+    'Lost connection to MySQL server during query'
+  ] unless defined? DisconnectErrors
+
+  def notify_readable
+    log 'readable'
+    if item = @current
+      @current = nil
+      start, response, sql, cblk, eblk = item
+      log 'mysql response', Time.now-start, sql
+      arg = case response
+            when :raw
+              result = @mysql.get_result
+              @mysql.instance_variable_set('@cur_result', result)
+              @mysql
+            when :select
+              ret = []
+              result = @mysql.get_result
+              result.each_hash{|h| ret << h }
+              log 'mysql result', ret
+              ret
+            when :update
+              result = @mysql.get_result
+              @mysql.affected_rows
+            when :insert
+              result = @mysql.get_result
+              @mysql.insert_id
+            else
+              result = @mysql.get_result
+              log 'got a result??', result if result
+              nil
+            end
+
+      @processing = false
+      # result.free if result.is_a? Mysql::Result
+      next_query
+      cblk.call(arg) if cblk
+    else
+      log 'readable, but nothing queued?! probably an ERROR state'
+      return close
+    end
+  rescue Mysql::Error => e
+    log 'mysql error', e.message
+    if e.message =~ /Deadlock/
+      @@queue << [response, sql, cblk, eblk]
+      @processing = false
+      next_query
+    elsif DisconnectErrors.include? e.message
+      @@queue << [response, sql, cblk, eblk]
+      return close
+    elsif cb = (eblk || @opts[:on_error])
+      cb.call(e)
+      @processing = false
+      next_query
+    else
+      raise e
+    end
+  # ensure
+  #   res.free if res.is_a? Mysql::Result
+  #   @processing = false
+  #   next_query
+  end
+
+  def unbind
+    log 'mysql disconnect', $!, *($! ? $!.backtrace[0..5] : [])
+    # cp = EventedMysql.instance_variable_get('@connection_pool') and cp.delete(self)
+    @connected = false
+
+    # XXX wait for the next tick until the current fd is removed completely from the reactor
+    #
+    # XXX in certain cases the new FD# (@mysql.socket) is the same as the old, since FDs are re-used
+    # XXX without next_tick in these cases, unbind will get fired on the newly attached signature as well
+    #
+    # XXX do _NOT_ use EM.next_tick here. if a bunch of sockets disconnect at the same time, we want
+    # XXX reconnects to happen after all the unbinds have been processed
+    EM.add_timer(0) do
+      log 'mysql reconnecting'
+      @processing = false
+      @mysql = EventedMysql._connect @opts
+      @fd = @mysql.socket
+
+      @signature = EM.attach_fd @mysql.socket, true, false
+      log 'mysql connected'
+      EM.instance_variable_get('@conns')[@signature] = self
+      @connected = true
+      make_socket_blocking
+      next_query
+    end
+  end
+
+  def execute sql, response = nil, cblk = nil, eblk = nil, &blk
+    cblk ||= blk
+
+    begin
+      unless @processing or !@connected
+        # begin
+        #   log 'mysql ping', @mysql.ping
+        #   # log 'mysql stat', @mysql.stat
+        #   # log 'mysql errno', @mysql.errno
+        # rescue
+        #   log 'mysql ping failed'
+        #   @@queue << [response, sql, blk]
+        #   return close
+        # end
+
+        @processing = true
+
+        log 'mysql sending', sql
+        @mysql.send_query(sql)
+      else
+        @@queue << [response, sql, cblk, eblk]
+        return
+      end
+    rescue Mysql::Error => e
+      log 'mysql error', e.message
+      if DisconnectErrors.include? e.message
+        @@queue << [response, sql, cblk, eblk]
+        return close
+      else
+        raise e
+      end
+    end
+
+    log 'queuing', response, sql
+    @current = [Time.now, response, sql, cblk, eblk]
+  end
+  
+  def close
+    @connected = false
+    @mysql.close
+    # IO.pipe
+    # EM.add_timer(0){ close_connection }
+    # close_connection
+    fd = detach
+    @io.close if @io
+    @io = nil
+    log 'detached fd', fd
+  end
+
+  private
+  
+  def make_socket_blocking
+    if defined?(Fcntl::F_GETFL)
+      @io = IO.for_fd(@mysql.socket)
+      m = @io.fcntl(Fcntl::F_GETFL, 0)
+      @io.fcntl(Fcntl::F_SETFL, ~Fcntl::O_NONBLOCK & m)
+    end
+  end
+
+  def next_query
+    if @connected and !@processing and pending = @@queue.shift
+      response, sql, cblk, eblk = pending
+      execute(sql, response, cblk, eblk)
+    end
+  end
+  
+  def log *args
+    return unless @opts[:logging]
+    p [Time.now, @fd, (@signature[-4..-1] if @signature), *args]
+  end
+
+  public
+
+  def self.connect opts
+    unless EM.respond_to?(:attach) and Mysql.method_defined?(:socket)
+      raise RuntimeError, 'mysqlplus and EM.attach are required for EventedMysql'
+    end
+
+    if conn = _connect(opts)
+      EM.attach conn.socket, self, conn, opts
+    else
+      EM.add_timer(5){ connect opts }
+    end
+  end
+
+  self::Mysql = ::Mysql unless defined? self::Mysql
+
+  # stolen from sequel
+  def self._connect opts
+    opts = settings.merge(opts)
+
+    conn = Mysql.init
+
+    # set encoding _before_ connecting
+    if charset = opts[:charset] || opts[:encoding]
+      conn.options(Mysql::SET_CHARSET_NAME, charset)
+    end
+
+    conn.options(Mysql::OPT_LOCAL_INFILE, 'client')
+
+    conn.real_connect(
+      opts[:host] || 'localhost',
+      opts[:user] || 'root',
+      opts[:password],
+      opts[:database],
+      opts[:port],
+      opts[:socket],
+      0 +
+      # XXX multi results require multiple callbacks to parse
+      # Mysql::CLIENT_MULTI_RESULTS +
+      # Mysql::CLIENT_MULTI_STATEMENTS +
+      (opts[:compress] == false ? 0 : Mysql::CLIENT_COMPRESS)
+    )
+    
+    # increase timeout so mysql server doesn't disconnect us
+    # this is especially bad if we're disconnected while EM.attach is
+    # still in progress, because by the time it gets to EM, the FD is
+    # no longer valid, and it throws a c++ 'bad file descriptor' error
+    # (do not use a timeout of -1 for unlimited, it does not work on mysqld > 5.0.60)
+    conn.query("set @@wait_timeout = #{opts[:timeout] || 2592000}")
+
+    # we handle reconnecting (and reattaching the new fd to EM)
+    conn.reconnect = false
+
+    # By default, MySQL 'where id is null' selects the last inserted id
+    # Turn this off. http://dev.rubyonrails.org/ticket/6778
+    conn.query("set SQL_AUTO_IS_NULL=0")
+
+    # get results for queries
+    conn.query_with_result = true
+
+    conn
+  rescue Mysql::Error => e
+    if cb = opts[:on_error]
+      cb.call(e)
+      nil
+    else
+      raise e
+    end
+  end
+end
+
+class EventedMysql
+  def self.settings
+    @settings ||= { :connections => 4, :logging => false }
+  end
+
+  def self.execute query, type = nil, cblk = nil, eblk = nil, &blk
+    unless nil#connection = connection_pool.find{|c| not c.processing and c.connected }
+      @n ||= 0
+      connection = connection_pool[@n]
+      @n = 0 if (@n+=1) >= connection_pool.size
+    end
+
+    connection.execute(query, type, cblk, eblk, &blk)
+  end
+
+  %w[ select insert update raw ].each do |type| class_eval %[
+
+    def self.#{type} query, cblk = nil, eblk = nil, &blk
+      execute query, :#{type}, cblk, eblk, &blk
+    end
+
+  ] end
+
+  def self.all query, type = nil, &blk
+    responses = 0
+    connection_pool.each do |c|
+      c.execute(query, type) do
+        responses += 1
+        blk.call if blk and responses == @connection_pool.size
+      end
+    end
+  end
+
+  def self.connection_pool
+    @connection_pool ||= (1..settings[:connections]).map{ EventedMysql.connect(settings) }
+    # p ['connpool', settings[:connections], @connection_pool.size]
+    # (1..(settings[:connections]-@connection_pool.size)).each do
+    #   @connection_pool << EventedMysql.connect(settings)
+    # end unless settings[:connections] == @connection_pool.size
+    # @connection_pool
+  end
+
+  def self.reset!
+    @connection_pool.each do |c|
+      c.close
+    end
+    @connection_pool = nil
+  end
+end
+
+if __FILE__ == $0 and require 'em/spec'
+
+  EM.describe EventedMysql, 'individual connections' do
+
+    should 'create a new connection' do
+      @mysql = EventedMysql.connect :host => '127.0.0.1',
+                                    :port => 3306,
+                                    :database => 'test',
+                                    :logging => false
+
+      @mysql.class.should == EventedMysql
+      done
+    end
+      
+    should 'execute sql' do
+      start = Time.now
+
+      @mysql.execute('select sleep(0.2)'){
+        (Time.now-start).should.be.close 0.2, 0.1
+        done
+      }
+    end
+
+    should 'reconnect when disconnected' do
+      @mysql.close
+      @mysql.execute('select 1+2'){
+        :connected.should == :connected
+        done
+      }
+    end
+
+    # to test, run:
+    #   mysqladmin5 -u root kill `mysqladmin5 -u root processlist | grep "select sleep(5)+1" | cut -d'|' -f2`
+    #
+    # should 're-run query if disconnected during query' do
+    #   @mysql.execute('select sleep(5)+1', :select){ |res|
+    #     res.first['sleep(5)+1'].should == '1'
+    #     done
+    #   }
+    # end
+  
+    should 'run select queries and return results' do
+      @mysql.execute('select 1+2', :select){ |res|
+        res.size.should == 1
+        res.first['1+2'].should == '3'
+        done
+      }
+    end
+
+    should 'queue up queries and execute them in order' do
+      @mysql.execute('select 1+2', :select)
+      @mysql.execute('select 2+3', :select)
+      @mysql.execute('select 3+4', :select){ |res|
+        res.first['3+4'].should == '7'
+        done
+      }
+    end
+  
+    should 'continue processing queries after hitting an error' do
+      @mysql.settings.update :on_error => proc{|e|}
+
+      @mysql.execute('select 1+ from table'){}
+      @mysql.execute('select 1+1 as num', :select){ |res|
+        res[0]['num'].should == '2'
+        done
+      }
+    end
+
+    should 'have raw mode which yields the mysql object' do
+      @mysql.execute('select 1+2 as num', :raw){ |mysql|
+        mysql.should.is_a? Mysql
+        mysql.result.all_hashes.should == [{'num' => '3'}]
+        done
+      }
+    end
+
+    should 'allow custom error callbacks for each query' do
+      @mysql.settings.update :on_error => proc{ should.flunk('default errback invoked') }
+
+      @mysql.execute('select 1+ from table', :select, proc{
+        should.flunk('callback invoked')
+      }, proc{ |e|
+        done
+      })
+    end
+
+  end
+
+  EM.describe EventedMysql, 'connection pools' do
+
+    EventedMysql.settings.update :connections => 3
+
+    should 'run queries in parallel' do
+      n = 0
+      EventedMysql.select('select sleep(0.25)'){ n+=1 }
+      EventedMysql.select('select sleep(0.25)'){ n+=1 }
+      EventedMysql.select('select sleep(0.25)'){ n+=1 }
+
+      EM.add_timer(0.30){
+        n.should == 3
+        done
+      }
+    end
+
+  end
+
+  SQL = EventedMysql
+  def SQL(query, &blk) SQL.select(query, &blk) end
+
+  # XXX this should get cleaned up automatically after reactor stops
+  SQL.reset!
+
+
+  EM.describe SQL, 'sql api' do
+    
+    should 'run a query on all connections' do
+      SQL.all('use test'){
+        :done.should == :done
+        done
+      }
+    end
+    
+    should 'execute queries with no results' do
+      SQL.execute('drop table if exists evented_mysql_test'){
+        :table_dropped.should == :table_dropped
+        SQL.execute('create table evented_mysql_test (id int primary key auto_increment, num int not null)'){
+          :table_created.should == :table_created
+          done
+        }
+      }
+    end
+    
+    should 'insert rows and return inserted id' do
+      SQL.insert('insert into evented_mysql_test (num) values (10),(11),(12)'){ |id|
+        id.should == 1
+        done
+      }
+    end
+
+    should 'select rows from the database' do
+      SQL.select('select * from evented_mysql_test'){ |res|
+        res.size.should == 3
+        res.first.should == { 'id' => '1', 'num' => '10' }
+        res.last.should  == { 'id' => '3', 'num' => '12' }
+        done
+      }
+    end
+
+    should 'update rows and return affected rows' do
+      SQL.update('update evented_mysql_test set num = num + 10'){ |changed|
+        changed.should == 3
+        done
+      }
+    end
+
+    should 'allow access to insert_id in raw mode' do
+      SQL.raw('insert into evented_mysql_test (num) values (20), (21), (22)'){ |mysql|
+        mysql.insert_id.should == 4
+        done
+      }
+    end
+
+    should 'allow access to affected_rows in raw mode' do
+      SQL.raw('update evented_mysql_test set num = num + 10'){ |mysql|
+        mysql.affected_rows.should == 6
+        done
+      }
+    end
+
+    should 'fire error callback with exceptions' do
+      SQL.settings.update :on_error => proc{ |e|
+        e.class.should == Mysql::Error
+        done
+      }
+      SQL.select('select 1+ from table'){}
+    end
+
+  end
+
+end

data/lib/sequel/async.rb

@@ -0,0 +1,184 @@
+# async sequel extensions, for use with em-mysql
+#
+#   require 'em/mysql'
+#   DB = Sequel.connect(:adapter => 'mysql', :user => 'root', :database => 'test', ...)
+#   EventedMysql.settings.update(..., :on_error => proc{|e| log 'error', e })
+#
+#   def log *args
+#     p [Time.now, *args]
+#   end
+#
+#   DB[:table].where(:id < 100).async_update(:field => 'new value') do |num_updated|
+#     log "done updating #{num_updated} rows"
+#   end
+#
+#   DB[:table].async_insert(:field => 'value') do |insert_id|
+#     log "inserted row #{insert_id}"
+#   end
+#
+#   DB[:table].async_multi_insert([:field], [ ['one'], ['two'], ['three'] ]) do
+#     log "done inserting 3 rows"
+#   end
+#
+#   DB[:table].limit(10).async_each do |row|
+#     log "got a row", row
+#   end; log "this will be printed before the query returns"
+#
+#   DB[:table].async_all do |rows|
+#     DB[:table].async_multi_insert([:field], rows.map{|r| "new_#{r[:field]}" })
+#   end
+#
+#   DB[:table].async_all do |rows|
+#     num = rows.size
+#
+#     rows.each{ |r|
+#       DB[:table].where(:id => r[:id]).async_update(:field => rand(10000).to_s) do
+#         num = num-1
+#         if num == 0
+#           log "last update completed"
+#         end
+#       end
+#     }
+#   end
+#
+#   DB[:table].async_count do |num_rows|
+#     log "table has #{num_rows} rows"
+#   end
+
+require 'sequel'
+raise 'need Sequel >= 3.2.0' unless Sequel::MAJOR == 3 and Sequel::MINOR >= 2
+
+module Sequel
+  class Dataset
+    def async_insert *args, &cb
+      EventedMysql.insert insert_sql(*args), &cb
+      nil
+    end
+
+    def async_insert_ignore *args, &cb
+      EventedMysql.insert insert_ignore.insert_sql(*args), &cb
+      nil
+    end
+
+    def async_update *args, &cb
+      EventedMysql.update update_sql(*args), &cb
+      nil
+    end
+
+    def async_delete &cb
+      EventedMysql.execute delete_sql, &cb
+      nil
+    end
+
+    def async_multi_insert *args, &cb
+      EventedMysql.execute multi_insert_sql(*args).first, &cb
+      nil
+    end
+
+    def async_multi_insert_ignore *args, &cb
+      EventedMysql.execute insert_ignore.multi_insert_sql(*args).first, &cb
+      nil
+    end
+
+    def async_fetch_rows sql, iter = :each
+      EventedMysql.raw(sql) do |m|
+        r = m.result
+
+        i = -1
+        cols = r.fetch_fields.map{|f| [output_identifier(f.name), Sequel::MySQL::MYSQL_TYPES[f.type], i+=1]}
+        @columns = cols.map{|c| c.first}
+        rows = []
+        while row = r.fetch_row
+          h = {}
+          cols.each{|n, p, i| v = row[i]; h[n] = (v && p) ? p.call(v) : v}
+          if iter == :each
+            yield h
+          else
+            rows << h
+          end
+        end
+        yield rows if iter == :all
+      end
+      nil
+    end
+
+    def async_each
+      async_fetch_rows(select_sql, :each) do |r|
+        if row_proc = @row_proc
+          yield row_proc.call(r)
+        else
+          yield r
+        end
+      end
+      nil
+    end
+
+    def async_all
+      async_fetch_rows(sql, :all) do |rows|
+        if row_proc = @row_proc
+          yield(rows.map{|r| row_proc.call(r) })
+        else
+          yield(rows)
+        end
+      end
+      nil
+    end
+
+    def async_count &cb
+      if options_overlap(COUNT_FROM_SELF_OPTS)
+        from_self.async_count(&cb)
+      else
+        clone(STOCK_COUNT_OPTS).async_each{|r|
+          yield r.is_a?(Hash) ? r.values.first.to_i : r.values.values.first.to_i
+        }
+      end
+      nil
+    end
+  end
+
+  class Model
+    def async_update *args, &cb
+      this.async_update(*args, &cb)
+      set(*args)
+      self
+    end
+
+    def async_delete &cb
+      this.async_delete(&cb)
+      nil
+    end
+
+    class << self
+      [ :async_insert,
+        :async_insert_ignore,
+        :async_multi_insert,
+        :async_multi_insert_ignore,
+        :async_each,
+        :async_all,
+        :async_update,
+        :async_count ].each do |method|
+        class_eval %[
+          def #{method} *args, &cb
+            dataset.#{method}(*args, &cb)
+          end
+        ]
+      end
+
+      # async version of Model#[]
+      def async_lookup args
+        unless Hash === args
+          args = primary_key_hash(args)
+        end
+
+        dataset.where(args).limit(1).async_all{ |rows|
+          if rows.any?
+            yield rows.first
+          else
+            yield nil
+          end
+        }
+        nil
+      end
+    end
+  end
+end

data/test.rb

@@ -0,0 +1,59 @@
+require 'lib/em/mysql'
+
+# EM.kqueue
+# EM.epoll
+EM.run{
+  EM.start_server '127.0.0.1', 12345 do |c|
+    def c.receive_data data
+      p 'sending http response'
+      send_data "hello"
+      close_connection_after_writing
+    end
+  end
+
+  SQL = EventedMysql
+  def SQL(query, &blk) SQL.select(query, &blk) end
+
+  if true
+
+    SQL.settings.update :logging => true,
+                        :database => 'test',
+                        :connections => 1
+
+    SQL.execute('select 1+2')
+
+    EM.add_timer(1){
+      3.times do SQL.select('select sleep(0.5)+1'){|r| p(r) } end
+    }
+
+  elsif false
+
+    SQL.settings.update :logging => true,
+                        :database => 'test',
+                        :connections => 10
+
+    EM.add_timer(2.5){ SQL.all('use test') }
+
+  else
+
+    SQL.settings.update :logging => true,
+                        :database => 'test',
+                        :connections => 10,
+                        :timeout => 1
+
+    n = 0
+
+    SQL.execute('drop table if exists testingabc'){
+      SQL.execute('create table testingabc (a int, b int, c int)'){
+        EM.add_periodic_timer(0.2) do
+          cur_num = n+=1
+          SQL.execute("insert into testingabc values (1,2,#{cur_num})"){
+            SQL("select * from testingabc where c = #{cur_num} limit 1"){ |res| puts;puts }
+          }
+        end
+      }
+    }
+
+  end
+
+}

metadata

@@ -0,0 +1,67 @@
+--- !ruby/object:Gem::Specification 
+name: julien51-em-mysql
+version: !ruby/object:Gem::Version 
+  version: 0.3.0
+platform: ruby
+authors: 
+- Aman Gupta
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2009-06-23 00:00:00 -07:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: eventmachine
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 0.12.8
+    version: 
+description: Async MySQL client API for Ruby/EventMachine
+email: em-mysql@tmm1.net
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- README
+- em-mysql.gemspec
+- lib/em/mysql.rb
+- lib/sequel/async.rb
+- test.rb
+has_rdoc: false
+homepage: http://github.com/tmm1/em-mysql
+licenses: 
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.5
+signing_key: 
+specification_version: 2
+summary: Async MySQL client API for Ruby/EventMachine
+test_files: []
+