em-synchrony 0.3.0.beta.1 → 1.0.0

data/.gitignore

@@ -1,3 +1,4 @@
 Gemfile.lock
 .bundle
 misc
+pkg

data/.rspec

@@ -0,0 +1 @@
+-I.

data/Gemfile

@@ -3,13 +3,15 @@ source :gemcutter
 gem 'eventmachine', :git => 'git://github.com/eventmachine/eventmachine.git'
 
 group :development do
-  gem 'rspec', '~> 2.0.0'
+  gem 'rspec'
   gem 'em-http-request', :git => 'git://github.com/igrigorik/em-http-request'
   gem 'remcached'
-  gem 'em-mongo'
+  # gem 'em-mongo', :git => 'https://github.com/bcg/em-mongo.git'
+  gem 'activerecord', '>= 3.1.0.rc6'
+  gem 'em-mongo', '~> 0.3.6'
   gem 'bson_ext'
-  gem 'mysqlplus'
-  gem 'em-mysqlplus'
+  gem 'mysql2'
   gem 'em-redis', '~> 0.3.0'
+  gem 'em-hiredis'
   gem 'mongo'
-end
+end

data/README.md

@@ -1,141 +1,171 @@
 # EM-Synchrony
 
-Collection of convenience classes and primitives to help untangle evented code, plus a number of patched EM clients to make them Fiber aware. To learn more, please see: [Untangling Evented Code with Ruby Fibers](http://www.igvita.com/2010/03/22/untangling-evented-code-with-ruby-fibers). Word of warning: even though Fibers have been backported to Ruby 1.8.x, these classes assume Ruby 1.9.x (if you're using fibers in production, you should be on 1.9.x anyway).
+Collection of convenience classes and primitives to help untangle evented code, plus a number of patched EM clients to make them Fiber aware. To learn more, please see: [Untangling Evented Code with Ruby Fibers](http://www.igvita.com/2010/03/22/untangling-evented-code-with-ruby-fibers).
 
- * Fiber aware connection pool with sync/async query support
- * Fiber aware iterator to allow concurrency control & mixing of sync / async
+ * Fiber aware ConnectionPool with sync/async query support
+ * Fiber aware Iterator to allow concurrency control & mixing of sync / async
  * Fiber aware async inline support: turns any async function into sync
- * Fiber aware multi-request interface for any callback enabled clients
+ * Fiber aware Multi-request interface for any callback enabled clients
  * Fiber aware TCPSocket replacement, powered by EventMachine
  * Fiber aware Thread, Mutex, ConditionVariable clases
  * Fiber aware sleep
 
 Supported clients:
 
- * em-http-request: .get, etc are synchronous, while .aget, etc are async
- * em-memcached & remcached: .get, etc, and .multi_* methods are synchronous
- * em-redis: synchronous connect, .a{cmd} are async
- * em-mysqlplus: .query is synchronous, while .aquery is async
- * em-mongo: .find, .first are synchronous
- * mongoid: all functions synchronous, plus Rails compatability
- * bitly v2 and v3: synchronous api calls with EM::HttpRequest.
+ * [mysql2](http://github.com/igrigorik/em-synchrony/blob/master/spec/mysql2_spec.rb): .query is synchronous, while .aquery is async (see specs)
+ * [activerecord](http://github.com/igrigorik/em-synchrony/blob/master/spec/activerecord_spec.rb): require synchrony/activerecord, set your AR adapter to em_mysql2 and you should be good to go
+ * [em-http-request](http://github.com/igrigorik/em-synchrony/blob/master/spec/http_spec.rb): .get, etc are synchronous, while .aget, etc are async
+ * [em-memcached](http://github.com/igrigorik/em-synchrony/blob/master/spec/memcache_spec.rb) & [remcached](http://github.com/igrigorik/em-synchrony/blob/master/spec/remcached_spec.rb): .get, etc, and .multi_* methods are synchronous
+ * [em-mongo](http://github.com/igrigorik/em-synchrony/blob/master/spec/em-mongo_spec.rb): .find, .first are synchronous
+ * [mongoid](http://github.com/igrigorik/em-synchrony/blob/master/spec/mongo_spec.rb): all functions synchronous, plus Rails compatibility
+ * em-jack: a[method]'s are async, and all regular jack method's are synchronous
 
+Other clients with native Fiber support:
+
+ * redis: contains [synchrony code](https://github.com/ezmobius/redis-rb/blob/master/test/synchrony_driver.rb) right within the driver
+ * synchrony also supports [em-redis](http://github.com/igrigorik/em-synchrony/blob/master/spec/redis_spec.rb) and em-hiredis (see specs), but unless you specifically need either of those, use the official redis gem
 
 ## Fiber-aware Iterator: mixing sync / async code
+
 Allows you to perform each, map, inject on a collection of any asynchronous tasks. To advance the iterator, simply call iter.next, or iter.return(result). The iterator will not exit until you advance through the entire collection. Additionally, you can specify the desired concurrency level! Ex: crawling a web-site, but you want to have at most 5 connections open at any one time.
 
-    require "em-synchrony/em-http"
-    EM.synchrony do
-        concurrency = 2
-        urls = ['http://url.1.com', 'http://url2.com']
+```ruby
+require "em-synchrony"
+require "em-synchrony/em-http"
+
+EM.synchrony do
+    concurrency = 2
+    urls = ['http://url.1.com', 'http://url2.com']
+
+    # iterator will execute async blocks until completion, .each, .inject also work!
+    results = EM::Synchrony::Iterator.new(urls, concurrency).map do |url, iter|
+
+        # fire async requests, on completion advance the iterator
+        http = EventMachine::HttpRequest.new(url).aget
+        http.callback { iter.return(http) }
+        http.errback { iter.return(http) }
+    end
+
+    p results # all completed requests
+    EventMachine.stop
+end
+```
+
+Or, you can use FiberIterator to hide the async nature of em-http:
 
-        # iterator will execute async blocks until completion, .each, .inject also work!
-        results = EM::Synchrony::Iterator.new(urls, concurrency).map do |url, iter|
+```ruby
+require "em-synchrony"
+require "em-synchrony/em-http"
 
-            # fire async requests, on completion advance the iterator
-            http = EventMachine::HttpRequest.new(url).aget
-            http.callback { iter.return(http) }
-        end
+EM.synchrony do
+    concurrency = 2
+    urls = ['http://url.1.com', 'http://url2.com']
+    results = []
 
-        p results # all completed requests
-        EventMachine.stop
+    EM::Synchrony::FiberIterator.new(urls, concurrency).each do |url|
+        resp = EventMachine::HttpRequest.new(url).get
+    results.push resp.response
     end
 
+    p results # all completed requests
+    EventMachine.stop
+end
+```
+
 ## Fiber-aware ConnectionPool shared by a fiber:
 Allows you to create a pool of resources which are then shared by one or more fibers. A good example is a collection of long-lived database connections. The pool will automatically block and wake up the fibers as the connections become available.
 
-    require "em-synchrony/em-mysqlplus"
-    EventMachine.synchrony do
-        db = EventMachine::Synchrony::ConnectionPool.new(size: 2) do
-            EventMachine::MySQL.new(host: "localhost")
-        end
+```ruby
+require "em-synchrony"
+require "em-synchrony/mysql2"
 
-        multi = EventMachine::Synchrony::Multi.new
-        multi.add :a, db.aquery("select sleep(1)")
-        multi.add :b, db.aquery("select sleep(1)")
-        res = multi.perform
+EventMachine.synchrony do
+    db = EventMachine::Synchrony::ConnectionPool.new(size: 2) do
+        Mysql2::EM::Client.new
+    end
 
-        p "Look ma, no callbacks, and parallel MySQL requests!"
-        p res
+    multi = EventMachine::Synchrony::Multi.new
+    multi.add :a, db.aquery("select sleep(1)")
+    multi.add :b, db.aquery("select sleep(1)")
+    res = multi.perform
 
-        EventMachine.stop
-    end
+    p "Look ma, no callbacks, and parallel MySQL requests!"
+    p res
+
+    EventMachine.stop
+end
+```
 
-## Fiber-aware Multi inteface: parallel HTTP requests
+## Fiber-aware Multi interface: parallel HTTP requests
 Allows you to fire simultaneous requests and wait for all of them to complete (success or error) before advancing. Concurrently fetching many HTTP pages at once is a good example; parallel SQL queries is another. Technically, this functionality can be also achieved by using the Synchrony Iterator shown above.
 
-    require "em-synchrony/em-http"
-    EventMachine.synchrony do
-        multi = EventMachine::Synchrony::Multi.new
-        multi.add :a, EventMachine::HttpRequest.new("http://www.postrank.com").aget
-        multi.add :b, EventMachine::HttpRequest.new("http://www.postrank.com").apost
-        res = multi.perform
+```ruby
+require "em-synchrony"
+require "em-synchrony/em-http"
+EventMachine.synchrony do
+    multi = EventMachine::Synchrony::Multi.new
+    multi.add :a, EventMachine::HttpRequest.new("http://www.postrank.com").aget
+    multi.add :b, EventMachine::HttpRequest.new("http://www.postrank.com").apost
+    res = multi.perform
 
-        p "Look ma, no callbacks, and parallel HTTP requests!"
-        p res
+    p "Look ma, no callbacks, and parallel HTTP requests!"
+    p res
 
-        EventMachine.stop
-    end
+    EventMachine.stop
+end
+```
 
 ## Fiber-aware & EventMachine backed TCPSocket:
 This is dangerous territory - you've been warned. You can patch your base TCPSocket class to make any/all libraries depending on TCPSocket be actually powered by EventMachine and Fibers under the hood.
 
-    require "lib/em-synchrony"
-    require "net/http"
+```ruby
+require "em-synchrony"
+require "lib/em-synchrony"
+require "net/http"
 
-    EM.synchrony do
-      # replace default Socket code to use EventMachine Sockets instead
-      TCPSocket = EventMachine::Synchrony::TCPSocket
+EM.synchrony do
+  # replace default Socket code to use EventMachine Sockets instead
+  TCPSocket = EventMachine::Synchrony::TCPSocket
 
-      Net::HTTP.get_print 'www.google.com', '/index.html'
-      EM.stop
-    end
+  Net::HTTP.get_print 'www.google.com', '/index.html'
+  EM.stop
+end
+```
 
 ## Inline synchronization & Fiber sleep:
 Allows you to inline/synchronize any callback interface to behave as if it was a blocking call. Simply pass any callback object to Synchrony.sync and it will do the right thing: the fiber will be resumed once the callback/errback fires. Likewise, use Synchrony.sleep to avoid blocking the main thread if you need to put one of your workers to sleep.
 
-    EM.synchrony do
-      # pass a callback enabled client to sync to automatically resume it when callback fires
-      result = EM::Synchrony.sync EventMachine::HttpRequest.new('http://www.gooogle.com/').aget
-      p result
+```ruby
+require "em-synchrony"
+require "em-synchrony/em-http"
+EM.synchrony do
+  # pass a callback enabled client to sync to automatically resume it when callback fires
+  result = EM::Synchrony.sync EventMachine::HttpRequest.new('http://www.gooogle.com/').aget
+  p result
 
-      # pause exection for 2 seconds
-      EM::Synchrony.sleep(2)
+  # pause exection for 2 seconds
+  EM::Synchrony.sleep(2)
 
-      EM.stop
-    end
+  EM.stop
+end
+```
 
-## Patched clients
+## Async ActiveRecord:
 
-EM-Synchrony ships with a number of patched Eventmachine clients, which allow you to patch your asynchronous drivers on the fly to behave as if they were actually blocking clients:
+Allows you to use async ActiveRecord within Rails and outside of Rails (see [async-rails](https://github.com/igrigorik/async-rails)). If you need to control the connection pool size, use [rack/fiber_pool](https://github.com/mperham/rack-fiber_pool/).
 
- * [em-http-request](http://github.com/igrigorik/em-synchrony/blob/master/spec/http_spec.rb)
- * [em-mysqlplus](http://github.com/igrigorik/em-synchrony/blob/master/spec/mysqlplus_spec.rb)
- * [em-redis](http://github.com/igrigorik/em-synchrony/blob/master/spec/redis_spec.rb)
- * [em-memcached](http://github.com/igrigorik/em-synchrony/blob/master/spec/memcache_spec.rb) & [remcached](http://github.com/igrigorik/em-synchrony/blob/master/spec/remcached_spec.rb)
- * [em-mongo](http://github.com/igrigorik/em-synchrony/blob/master/spec/em-mongo_spec.rb) & [mongoid](http://github.com/igrigorik/em-synchrony/blob/master/spec/mongo_spec.rb)
+```ruby
+require "em-synchrony"
+require "em-synchrony/activerecord"
 
-# License
-
-(The MIT License)
+ActiveRecord::Base.establish_connection(
+  :adapter => 'em_mysql2',
+  :database => 'widgets'
+)
 
-Copyright (c) 2010 Ilya Grigorik
+result = Widget.all.to_a
+```
 
-Permission is hereby granted, free of charge, to any person obtaining
-a copy of this software and associated documentation files (the
-'Software'), to deal in the Software without restriction, including
-without limitation the rights to use, copy, modify, merge, publish,
-distribute, sublicense, and/or sell copies of the Software, and to
-permit persons to whom the Software is furnished to do so, subject to
-the following conditions:
-
-The above copyright notice and this permission notice shall be
-included in all copies or substantial portions of the Software.
+# License
 
-THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
-IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
-CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
-TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
-SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+The MIT License - Copyright (c) 2011 Ilya Grigorik

data/Rakefile

@@ -1,4 +1,10 @@
 require 'bundler'
+require 'rspec/core/rake_task'
 
 Bundler.setup
-Bundler::GemHelper.install_tasks
+Bundler::GemHelper.install_tasks
+
+desc "Run all RSpec tests"
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/em-synchrony.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path("../lib", __FILE__)
 
 Gem::Specification.new do |s|
   s.name        = "em-synchrony"
-  s.version     = "0.3.0.beta.1"
+  s.version     = "1.0.0"
   s.platform    = Gem::Platform::RUBY
   s.authors     = ["Ilya Grigorik"]
   s.email       = ["ilya@igvita.com"]
@@ -19,4 +19,4 @@ Gem::Specification.new do |s|
   s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
   s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
   s.require_paths = ["lib"]
-end
+end

data/lib/em-synchrony.rb

@@ -12,6 +12,7 @@ require "em-synchrony/thread"
 require "em-synchrony/em-multi"
 require "em-synchrony/tcpsocket"
 require "em-synchrony/connection_pool"
+require "em-synchrony/keyboard"
 require "em-synchrony/iterator"  if EventMachine::VERSION > '0.12.10'
 
 module EventMachine
@@ -44,18 +45,41 @@ module EventMachine
     #
     def self.sync(df)
       f = Fiber.current
-      df.callback { |r| f.resume(r) }
-      df.errback { |r| f.resume(r) }
+      xback = proc {|r|
+        if f == Fiber.current
+          return r
+        else
+          f.resume r
+        end
+      }
+      df.callback &xback
+      df.errback &xback
 
       Fiber.yield
     end
 
     # a Fiber-aware sleep function using an EM timer
-    def self.sleep( secs )
+    def self.sleep(secs)
       fiber = Fiber.current
-      EM::Timer.new(secs) {  fiber.resume  }
+      EM::Timer.new(secs) { fiber.resume }
       Fiber.yield
     end
-  end
 
+    def self.add_timer(interval, &blk)
+      EM.add_timer(interval) do
+        Fiber.new { blk.call }.resume
+      end
+    end
+
+    def self.add_periodic_timer(interval, &blk)
+      EM.add_periodic_timer(interval) do
+        Fiber.new { blk.call }.resume
+      end
+    end
+    
+    # routes to EM::Synchrony::Keyboard
+    def self.gets
+      EventMachine::Synchrony::Keyboard.new.gets
+    end
+  end
 end

data/lib/em-synchrony/active_record/connection_adapters/em_mysql2_adapter.rb

@@ -0,0 +1,18 @@
+# encoding: utf-8
+
+# AR adapter for using a fibered mysql2 connection with EM
+# This adapter should be used within Thin or Unicorn with the rack-fiber_pool middleware.
+# Just update your database.yml's adapter to be 'em_mysql2'
+
+require 'active_record/connection_adapters/abstract_adapter'
+require 'active_record/connection_adapters/mysql2_adapter'
+
+module ActiveRecord
+  class Base
+    def self.em_mysql2_connection(config)
+      client = Mysql2::EM::Client.new(config.symbolize_keys)
+      options = [config[:host], config[:username], config[:password], config[:database], config[:port], config[:socket], 0]
+      ConnectionAdapters::Mysql2Adapter.new(client, logger, options, config)
+    end
+  end
+end

data/lib/em-synchrony/active_record/patches.rb

@@ -0,0 +1,132 @@
+# Necessary monkeypatching to make AR fiber-friendly.
+
+module ActiveRecord
+  module ConnectionAdapters
+
+    def self.fiber_pools
+      @fiber_pools ||= []
+    end
+    def self.register_fiber_pool(fp)
+      fiber_pools << fp
+    end
+
+    class FiberedMonitor
+      class Queue
+        def initialize
+          @queue = []
+        end
+
+        def wait(timeout)
+          t = timeout || 5
+          fiber = Fiber.current
+          x = EM::Timer.new(t) do
+            @queue.delete(fiber)
+            fiber.resume(false)
+          end
+          @queue << fiber
+          Fiber.yield.tap do
+            x.cancel
+          end
+        end
+
+        def signal
+          fiber = @queue.pop
+          fiber.resume(true) if fiber
+        end
+      end
+
+      def synchronize
+        yield
+      end
+
+      def new_cond
+        Queue.new
+      end
+    end
+
+    # ActiveRecord's connection pool is based on threads.  Since we are working
+    # with EM and a single thread, multiple fiber design, we need to provide
+    # our own connection pool that keys off of Fiber.current so that different
+    # fibers running in the same thread don't try to use the same connection.
+    class ConnectionPool
+      def initialize(spec)
+        @spec = spec
+
+        # The cache of reserved connections mapped to threads
+        @reserved_connections = {}
+
+        # The mutex used to synchronize pool access
+        @connection_mutex = FiberedMonitor.new
+        @queue = @connection_mutex.new_cond
+
+        # default 5 second timeout unless on ruby 1.9
+        @timeout = spec.config[:wait_timeout] || 5
+
+        # default max pool size to 5
+        @size = (spec.config[:pool] && spec.config[:pool].to_i) || 5
+
+        @connections = []
+        @checked_out = []
+        @automatic_reconnect = true
+        @tables = {}
+
+        @columns     = Hash.new do |h, table_name|
+          h[table_name] = with_connection do |conn|
+
+            # Fetch a list of columns
+            conn.columns(table_name, "#{table_name} Columns").tap do |columns|
+
+              # set primary key information
+              columns.each do |column|
+                column.primary = column.name == primary_keys[table_name]
+              end
+            end
+          end
+        end
+
+        @columns_hash = Hash.new do |h, table_name|
+          h[table_name] = Hash[columns[table_name].map { |col|
+            [col.name, col]
+          }]
+        end
+
+        @primary_keys = Hash.new do |h, table_name|
+          h[table_name] = with_connection do |conn|
+            table_exists?(table_name) ? conn.primary_key(table_name) : 'id'
+          end
+        end
+      end
+
+      def clear_stale_cached_connections!
+        cache = @reserved_connections
+        keys = Set.new(cache.keys)
+
+        ActiveRecord::ConnectionAdapters.fiber_pools.each do |pool|
+          pool.busy_fibers.each_pair do |object_id, fiber|
+            keys.delete(object_id)
+          end
+        end
+
+        keys.each do |key|
+          next unless cache.has_key?(key)
+          checkin cache[key]
+          cache.delete(key)
+        end
+      end
+
+      private
+
+      def current_connection_id #:nodoc:
+        Fiber.current.object_id
+      end
+
+      def checkout_and_verify(c)
+        @checked_out << c
+        c.run_callbacks :checkout
+        c.verify!
+        c
+      end
+    end
+
+  end
+end