tmm1-em-mysql 0.2.0

data/README

@@ -0,0 +1,28 @@
+Async MySQL driver for Ruby/EventMachine
+  (c) 2008 Aman Gupta (tmm1)
+
+
+Requires mysqlplus and the EM.attach patch.
+
+  require 'em/mysql'
+
+  # alias SQL for simpler syntax
+
+  SQL = EventedMysql
+  def SQL(query, &blk) SQL.select(query, &blk) end
+
+  
+  # setup connection details and allow 4 connections to the server
+
+  SQL.settings.update :host => 'localhost',
+                      :port => 3306,
+                      :database => 'test',
+                      :connections => 4
+  
+
+  # use 4 connections to execute queries in parallel
+
+  SQL('select sleep(0.25)'){ p 'done' }
+  SQL('select sleep(0.25)'){ p 'done' }
+  SQL('select sleep(0.25)'){ p 'done' }
+  SQL('select sleep(0.25)'){ p 'done' }

data/lib/em/mysql.rb

@@ -0,0 +1,459 @@
+require 'rubygems'
+require 'eventmachine'
+require 'mysqlplus'
+
+class EventedMysql < EM::Connection
+  def initialize mysql, opts
+    @mysql = mysql
+    @fd = mysql.socket
+    @opts = opts
+    @queue = []
+    @pending = []
+    @processing = false
+    @connected = true
+
+    log 'mysql connected'
+  end
+  attr_reader :processing, :connected, :opts
+  alias :settings :opts
+
+  def connection_completed
+    @connected = true
+    next_query
+  end
+
+  DisconnectErrors = [
+    'query: not connected',
+    'MySQL server has gone away',
+    'Lost connection to MySQL server during query'
+  ] unless defined? DisconnectErrors
+
+  def notify_readable
+    log 'readable'
+    if item = @queue.shift
+      start, response, sql, cblk, eblk = item
+      log 'mysql response', Time.now-start, sql
+      arg = case response
+            when :raw
+              @mysql
+            when :select
+              ret = []
+              result = @mysql.get_result
+              result.each_hash{|h| ret << h }
+              log 'mysql result', ret
+              ret
+            when :update
+              result = @mysql.get_result
+              @mysql.affected_rows
+            when :insert
+              result = @mysql.get_result
+              @mysql.insert_id
+            else
+              result = @mysql.get_result
+              log 'got a result??', result if result
+              nil
+            end
+
+      @processing = false
+      # result.free if result.is_a? Mysql::Result
+      next_query
+      cblk.call(arg) if cblk
+    else
+      log 'readable, but nothing queued?! probably an ERROR state'
+      return close
+    end
+  rescue Mysql::Error => e
+    log 'mysql error', e.message
+    if DisconnectErrors.include? e.message
+      @pending << [response, sql, cblk, eblk]
+      return close
+    elsif cb = (eblk || @opts[:on_error])
+      cb.call(e)
+      @processing = false
+      next_query
+    else
+      raise e
+    end
+  # ensure
+  #   res.free if res.is_a? Mysql::Result
+  #   @processing = false
+  #   next_query
+  end
+
+  def unbind
+    log 'mysql disconnect', $!
+    # cp = EventedMysql.instance_variable_get('@connection_pool') and cp.delete(self)
+    @connected = false
+
+    # XXX wait for the next tick until the current fd is removed completely from the reactor
+    #
+    # XXX in certain cases the new FD# (@mysql.socket) is the same as the old, since FDs are re-used
+    # XXX without next_tick in these cases, unbind will get fired on the newly attached signature as well
+    #
+    # XXX do _NOT_ use EM.next_tick here. if a bunch of sockets disconnect at the same time, we want
+    # XXX reconnects to happen after all the unbinds have been processed
+    EM.add_timer(0) do
+      log 'mysql reconnecting'
+      @processing = false
+      @mysql = EventedMysql._connect @opts
+      @fd = @mysql.socket
+
+      @signature = EM.attach_fd @mysql.socket, true, false
+      log 'mysql connected'
+      EM.instance_variable_get('@conns')[@signature] = self
+    end
+  end
+
+  def execute sql, response = nil, cblk = nil, eblk = nil, &blk
+    cblk ||= blk
+
+    begin
+      unless @processing or !@connected
+        # begin
+        #   log 'mysql ping', @mysql.ping
+        #   # log 'mysql stat', @mysql.stat
+        #   # log 'mysql errno', @mysql.errno
+        # rescue
+        #   log 'mysql ping failed'
+        #   @pending << [response, sql, blk]
+        #   return close
+        # end
+
+        @processing = true
+
+        log 'mysql sending', sql
+        @mysql.send_query(sql)
+      else
+        @pending << [response, sql, cblk, eblk]
+        return
+      end
+    rescue Mysql::Error => e
+      log 'mysql error', e.message
+      if DisconnectErrors.include? e.message
+        @pending << [response, sql, cblk, eblk]
+        return close
+      else
+        raise e
+      end
+    end
+
+    log 'queuing', response, sql
+    @queue << [Time.now, response, sql, cblk, eblk]
+  end
+  
+  def close
+    @connected = false
+    # @mysql.close
+    # IO.pipe
+    # EM.add_timer(0){ close_connection }
+    # close_connection
+    fd = detach
+    log 'detached fd', fd
+  end
+
+  private
+  
+  def next_query
+    if @connected and !@processing and pending = @pending.shift
+      response, sql, cblk, eblk = pending
+      execute(sql, response, cblk, eblk)
+    end
+  end
+  
+  def log *args
+    return unless @opts[:logging]
+    p [Time.now, @fd, (@signature[-4..-1] if @signature), *args]
+  end
+
+  public
+
+  def self.connect opts
+    unless EM.respond_to?(:attach) and Mysql.method_defined?(:socket)
+      raise RuntimeError, 'mysqlplus and EM.attach are required for EventedMysql'
+    end
+
+    if conn = _connect(opts)
+      EM.attach conn.socket, self, conn, opts
+    else
+      EM.add_timer(5){ connect opts }
+    end
+  end
+
+  self::Mysql = ::Mysql unless defined? self::Mysql
+
+  # stolen from sequel
+  def self._connect opts
+    opts = settings.merge(opts)
+
+    conn = Mysql.init
+
+    # set encoding _before_ connecting
+    if charset = opts[:charset] || opts[:encoding]
+      conn.options(Mysql::SET_CHARSET_NAME, charset)
+    end
+
+    conn.options(Mysql::OPT_LOCAL_INFILE, 'client')
+
+    conn.real_connect(
+      opts[:host] || 'localhost',
+      opts[:user] || 'root',
+      opts[:password],
+      opts[:database],
+      opts[:port],
+      opts[:socket],
+      0 +
+      # XXX multi results require multiple callbacks to parse
+      # Mysql::CLIENT_MULTI_RESULTS +
+      # Mysql::CLIENT_MULTI_STATEMENTS +
+
+      # XXX this should check for opts[:compression]
+      Mysql::CLIENT_COMPRESS
+    )
+    
+    # increase timeout so mysql server doesn't disconnect us
+    # this is especially bad if we're disconnected while EM.attach is
+    # still in progress, because by the time it gets to EM, the FD is
+    # no longer valid, and it throws a c++ 'bad file descriptor' error
+    # (do not use a timeout of -1 for unlimited, it does not work on mysqld > 5.0.60)
+    conn.query("set @@wait_timeout = #{opts[:timeout] || 2592000}")
+
+    # we handle reconnecting (and reattaching the new fd to EM)
+    conn.reconnect = false
+
+    # By default, MySQL 'where id is null' selects the last inserted id
+    # Turn this off. http://dev.rubyonrails.org/ticket/6778
+    conn.query("set SQL_AUTO_IS_NULL=0")
+
+    # get results for queries
+    conn.query_with_result = true
+
+    # if encoding = opts[:encoding] || opts[:charset]
+    #   conn.options(Mysql::SET_CHARSET_NAME, encoding) rescue nil
+    #   conn.query("set names '#{encoding}'")
+    #   conn.query("set character_set_connection = '#{encoding}'")
+    #   conn.query("set character_set_client = '#{encoding}'")
+    #   conn.query("set character_set_database = '#{encoding}'")
+    #   conn.query("set character_set_server = '#{encoding}'")
+    #   conn.query("set character_set_results = '#{encoding}'")
+    # end
+
+    conn
+  rescue Mysql::Error => e
+    if cb = opts[:on_error]
+      cb.call(e)
+      nil
+    else
+      raise e
+    end
+  end
+end
+
+class EventedMysql
+  def self.settings
+    @settings ||= { :connections => 4, :logging => false }
+  end
+
+  def self.execute query, type = nil, cblk = nil, eblk = nil, &blk
+    unless nil#connection = connection_pool.find{|c| not c.processing and c.connected }
+      @n ||= 0
+      connection = connection_pool[@n]
+      @n = 0 if (@n+=1) >= connection_pool.size
+    end
+
+    connection.execute(query, type, cblk, eblk, &blk)
+  end
+
+  %w[ select insert update raw ].each do |type| class_eval %[
+
+    def self.#{type} query, cblk = nil, eblk = nil, &blk
+      execute query, :#{type}, cblk, eblk, &blk
+    end
+
+  ] end
+
+  def self.all query, type = nil, &blk
+    responses = 0
+    connection_pool.each do |c|
+      c.execute(query, type) do
+        responses += 1
+        blk.call if blk and responses == @connection_pool.size
+      end
+    end
+  end
+
+  def self.connection_pool
+    @connection_pool ||= (1..settings[:connections]).map{ EventedMysql.connect(settings) }
+    # p ['connpool', settings[:connections], @connection_pool.size]
+    # (1..(settings[:connections]-@connection_pool.size)).each do
+    #   @connection_pool << EventedMysql.connect(settings)
+    # end unless settings[:connections] == @connection_pool.size
+    # @connection_pool
+  end
+end
+
+if __FILE__ == $0 and require 'em/spec'
+
+  EM.describe EventedMysql, 'individual connections' do
+
+    should 'create a new connection' do
+      @mysql = EventedMysql.connect :host => '127.0.0.1',
+                                    :port => 3306,
+                                    :database => 'test',
+                                    :logging => false
+
+      @mysql.class.should == EventedMysql
+      done
+    end
+      
+    should 'execute sql' do
+      start = Time.now
+
+      @mysql.execute('select sleep(0.2)'){
+        (Time.now-start).should.be.close 0.2, 0.1
+        done
+      }
+    end
+
+    should 'reconnect when disconnected' do
+      @mysql.close
+      @mysql.execute('select 1+2'){
+        :connected.should == :connected
+        done
+      }
+    end
+
+    # to test, run:
+    #   mysqladmin5 -u root kill `mysqladmin5 -u root processlist | grep "select sleep(5)+1" | cut -d'|' -f2`
+    #
+    # should 're-run query if disconnected during query' do
+    #   @mysql.execute('select sleep(5)+1', :select){ |res|
+    #     res.first['sleep(5)+1'].should == '1'
+    #     done
+    #   }
+    # end
+  
+    should 'run select queries and return results' do
+      @mysql.execute('select 1+2', :select){ |res|
+        res.size.should == 1
+        res.first['1+2'].should == '3'
+        done
+      }
+    end
+
+    should 'queue up queries and execute them in order' do
+      @mysql.execute('select 1+2', :select)
+      @mysql.execute('select 2+3', :select)
+      @mysql.execute('select 3+4', :select){ |res|
+        res.first['3+4'].should == '7'
+        done
+      }
+    end
+  
+    should 'continue processing queries after hitting an error' do
+      @mysql.settings.update :on_error => proc{|e|}
+
+      @mysql.execute('select 1+ from table'){}
+      @mysql.execute('select 1+1 as num', :select){ |res|
+        res[0]['num'].should == '2'
+        done
+      }
+    end
+
+    should 'have raw mode which yields the mysql object' do
+      @mysql.execute('select 1+2 as num', :raw){ |mysql|
+        mysql.should.is_a? Mysql
+        mysql.get_result.all_hashes.should == [{'num' => '3'}]
+        done
+      }
+    end
+
+    should 'allow custom error callbacks for each query' do
+      @mysql.settings.update :on_error => proc{ should.flunk('default errback invoked') }
+
+      @mysql.execute('select 1+ from table', :select, proc{
+        should.flunk('callback invoked')
+      }, proc{ |e|
+        done
+      })
+    end
+
+  end
+
+  EM.describe EventedMysql, 'connection pools' do
+
+    EventedMysql.settings.update :connections => 3
+
+    should 'run queries in parallel' do
+      n = 0
+      EventedMysql.select('select sleep(0.25)'){ n+=1 }
+      EventedMysql.select('select sleep(0.25)'){ n+=1 }
+      EventedMysql.select('select sleep(0.25)'){ n+=1 }
+
+      EM.add_timer(0.30){
+        n.should == 3
+        done
+      }
+    end
+
+  end
+
+  SQL = EventedMysql
+  def SQL(query, &blk) SQL.select(query, &blk) end
+
+  # XXX this should get cleaned up automatically after reactor stops
+  SQL.instance_variable_set('@connection_pool', nil)
+
+
+  EM.describe SQL, 'sql api' do
+    
+    should 'run a query on all connections' do
+      SQL.all('use test'){
+        :done.should == :done
+        done
+      }
+    end
+    
+    should 'execute queries with no results' do
+      SQL.execute('drop table if exists evented_mysql_test'){
+        :table_dropped.should == :table_dropped
+        SQL.execute('create table evented_mysql_test (id int primary key auto_increment, num int not null)'){
+          :table_created.should == :table_created
+          done
+        }
+      }
+    end
+    
+    should 'insert rows and return inserted id' do
+      SQL.insert('insert into evented_mysql_test (num) values (10),(11),(12)'){ |id|
+        id.should == 1
+        done
+      }
+    end
+
+    should 'select rows from the database' do
+      SQL.select('select * from evented_mysql_test'){ |res|
+        res.size.should == 3
+        res.first.should == { 'id' => '1', 'num' => '10' }
+        res.last.should  == { 'id' => '3', 'num' => '12' }
+        done
+      }
+    end
+
+    should 'update rows and return affected rows' do
+      SQL.update('update evented_mysql_test set num = num + 10'){ |changed|
+        changed.should == 3
+        done
+      }
+    end
+
+    should 'fire error callback with exceptions' do
+      SQL.settings.update :on_error => proc{ |e|
+        e.class.should == Mysql::Error
+        done
+      }
+      SQL.select('select 1+ from table'){}
+    end
+
+  end
+
+end

data/lib/sequel/async.rb

@@ -0,0 +1,157 @@
+# async sequel extensions, for use with em-mysql
+#
+#   require 'em/mysql'
+#   DB = Sequel.connect(...)
+#   ADB = EventedMysql.connect(..., :on_error => proc{|e| log 'error', e })
+#
+#   def log *args
+#     p [Time.now, *args]
+#   end
+#
+#   DB[:table].where(:id < 100).async_update do |num_updated|
+#     log "done updating #{num_updated} rows"
+#   end
+#
+#   DB[:table].async_insert(:field => 'value') do |insert_id|
+#     log "inserted row #{insert_id}"
+#   end
+#
+#   DB[:table].async_multi_insert([:field], [ ['one'], ['two'], ['three'] ]) do
+#     log "done inserting 3 rows"
+#   end
+#
+#   DB[:table].limit(10).async_each do |row|
+#     log "got a row", row
+#   end; log "this will be printed before the query returns"
+#
+#   DB[:table].async_all do |rows|
+#     DB[:table].async_multi_insert([:field], rows.map{|r| "new_#{r[:field]}" })
+#   end
+#
+#   DB[:table].async_all do |rows|
+#     num = rows.size
+#
+#     rows.each{ |r|
+#       DB[:table].where(:id => r[:id]).async_update(:field => rand(10000).to_s) do
+#         num = num-1
+#         if num == 0
+#           log "last update completed"
+#         end
+#       end
+#     }
+#   end
+#
+#   DB[:table].async_count do |num_rows|
+#     log "table has #{num_rows} rows"
+#   end
+
+module Sequel
+  class Dataset
+    def async_insert *args, &cb
+      ADB.insert insert_sql(*args), &cb
+      nil
+    end
+
+    def async_update *args, &cb
+      ADB.update update_sql(*args), &cb
+      nil
+    end
+
+    def async_delete &cb
+      ADB.execute delete_sql, &cb
+      nil
+    end
+
+    def async_multi_insert *args, &cb
+      ADB.execute multi_insert_sql(*args).first, &cb
+      nil
+    end
+
+    def async_multi_insert_ignore *args, &cb
+      ADB.execute multi_insert_sql(*args).first.sub(/insert/i, "INSERT IGNORE"), &cb
+      nil
+    end
+
+    def async_each *args
+      ADB.select(select_sql(*args)) do |rows|
+        rows.each{|r|
+          r = transform_load(r) if @transform
+          r = row_proc[r] if row_proc
+          yield r
+        }
+      end
+      nil
+    end
+
+    def async_all
+      ADB.select(sql) do |rows|
+        if row_proc or transform
+          yield(rows.map{|r|
+            r = transform_load(r) if @transform
+            r = row_proc[r] if row_proc
+            r
+          })
+        else
+          yield(rows)
+        end
+      end
+      nil
+    end
+
+    def async_count &cb
+      if options_overlap(COUNT_FROM_SELF_OPTS)
+        from_self.async_count(&cb)
+      else
+        naked.async_each(STOCK_COUNT_OPTS){|r|
+          yield r.values.first.to_i
+        }
+      end
+      nil
+    end
+  end
+
+  class Model
+    def async_update *args, &cb
+      this.async_update(*args, &cb)
+      set(*args)
+      self
+    end
+
+    def async_delete &cb
+      this.async_delete(&cb)
+      nil
+    end
+
+    class << self
+      [ :async_insert,
+        :async_multi_insert,
+        :async_multi_insert_ignore,
+        :async_each,
+        :async_all,
+        :async_update,
+        :async_count ].each do |method|
+        class_eval %[
+          def #{method} *args, &cb
+            dataset.#{method}(*args, &cb)
+          end
+        ]
+      end
+
+      # async version of Model#[]
+      def async_lookup args
+        unless Hash === args
+          args = primary_key_hash(args)
+        end
+
+        dataset.where(args).limit(1).async_all{ |rows|
+          if rows.any?
+            yield rows.first
+          else
+            yield nil
+          end
+        }
+        nil
+      end
+    end
+  end
+end

data/test.rb

@@ -0,0 +1,59 @@
+require 'lib/em/mysql'
+
+# EM.kqueue
+# EM.epoll
+EM.run{
+  EM.start_server '127.0.0.1', 12345 do |c|
+    def c.receive_data data
+      p 'sending http response'
+      send_data "hello"
+      close_connection_after_writing
+    end
+  end
+
+  SQL = EventedMysql
+  def SQL(query, &blk) SQL.select(query, &blk) end
+
+  if true
+
+    SQL.settings.update :logging => true,
+                        :database => 'test',
+                        :connections => 1
+
+    SQL.execute('select 1+2')
+
+    EM.add_timer(1){
+      3.times do SQL.select('select sleep(0.5)+1'){|r| p(r) } end
+    }
+
+  elsif false
+
+    SQL.settings.update :logging => true,
+                        :database => 'test',
+                        :connections => 10
+
+    EM.add_timer(2.5){ SQL.all('use test') }
+
+  else
+
+    SQL.settings.update :logging => true,
+                        :database => 'test',
+                        :connections => 10,
+                        :timeout => 1
+
+    n = 0
+
+    SQL.execute('drop table if exists testingabc'){
+      SQL.execute('create table testingabc (a int, b int, c int)'){
+        EM.add_periodic_timer(0.2) do
+          cur_num = n+=1
+          SQL.execute("insert into testingabc values (1,2,#{cur_num})"){
+            SQL("select * from testingabc where c = #{cur_num} limit 1"){ |res| puts;puts }
+          }
+        end
+      }
+    }
+
+  end
+
+}

metadata

@@ -0,0 +1,65 @@
+--- !ruby/object:Gem::Specification 
+name: tmm1-em-mysql
+version: !ruby/object:Gem::Version 
+  version: 0.2.0
+platform: ruby
+authors: 
+- Aman Gupta
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2009-02-18 00:00:00 -08:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: eventmachine
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 0.12.4
+    version: 
+description: Async MySQL client API for Ruby/EventMachine
+email: em-mysql@tmm1.net
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- README
+- lib/em/mysql.rb
+- lib/sequel/async.rb
+- test.rb
+has_rdoc: false
+homepage: http://github.com/tmm1/em-mysql
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.2.0
+signing_key: 
+specification_version: 2
+summary: Async MySQL client API for Ruby/EventMachine
+test_files: []
+