em-synchrony 0.1.5 → 0.2.0

data/Gemfile

@@ -0,0 +1,15 @@
+source :gemcutter
+
+gem 'eventmachine', :git => 'git://github.com/eventmachine/eventmachine.git'
+
+group :development do
+  gem 'rspec', '~> 2.0.0'
+  gem 'em-http-request'
+  gem 'remcached'
+  gem 'em-mongo'
+  gem 'bson_ext'
+  gem 'mysqlplus'
+  gem 'em-mysqlplus'
+  gem 'em-redis'
+  gem 'mongo'
+end

data/README.md

@@ -1,33 +1,28 @@
 # EM-Synchrony
 
-Blog post: [Untangling Evented Code with Ruby Fibers](http://www.igvita.com/2010/03/22/untangling-evented-code-with-ruby-fibers)
+Collection of convenience classes and primitives to help untangle evented code, plus a number of patched EM clients to make them Fiber aware. To learn more, please see: [Untangling Evented Code with Ruby Fibers](http://www.igvita.com/2010/03/22/untangling-evented-code-with-ruby-fibers). Word of warning: even though Fibers have been backported to Ruby 1.8.x, these classes assume Ruby 1.9.x (if you're using fibers in production, you should be on 1.9.x anyway).
 
-Collection of convenience classes and patches to common EventMachine clients to
-make them Fiber aware and friendly. Word of warning: even though fibers have been
-backported to Ruby 1.8.x, these classes assume Ruby 1.9 (if you're using fibers
-in production, you should be on 1.9 anyway)
+ * Fiber aware connection pool with sync/async query support
+ * Fiber aware iterator to allow concurrency control & mixing of sync / async
+ * Fiber aware async inline support: turns any async function into sync
+ * Fiber aware multi-request interface for any callback enabled clients
+ * Fiber aware TCPSocket replacement, powered by EventMachine
+ * Fiber aware Thread, Mutex, ConditionVariable clases
+ * Fiber aware sleep
 
-Features:
+Supported clients:
 
- * Fiber aware connection pool with sync/async query support
- * Multi request interface which accepts any callback enabled client
- * Fibered iterator to allow concurrency control & mixing of sync / async
  * em-http-request: .get, etc are synchronous, while .aget, etc are async
+ * em-memcached & remcached: .get, etc, and .multi_* methods are synchronous
+ * em-redis: synchronous connect, .a{cmd} are async
  * em-mysqlplus: .query is synchronous, while .aquery is async
- * remcached: .get, etc, and .multi_* methods are synchronous
- * bitly: synchronous api calls with EM::HttpRequest.
+ * em-mongo: .find, .first are synchronous
+ * mongoid: all functions synchronous, plus Rails compatability
+ * bitly v2 and v3: synchronous api calls with EM::HttpRequest.
 
-## Example with async em-http client:
-    require "em-synchrony/em-http"
-    EventMachine.synchrony do
-        res = EventMachine::HttpRequest.new("http://www.postrank.com").get
-        p "Look ma, no callbacks!"
-        p res
 
-        EventMachine.stop
-    end
-
-## EM Iterator & mixing sync / async code
+## Fiber-aware Iterator: mixing sync / async code
+Allows you to perform each, map, inject on a collection of any asynchronous tasks. To advance the iterator, simply call iter.next, or iter.return(result). The iterator will not exit until you advance through the entire collection. Additionally, you can specify the desired concurrency level! Ex: crawling a web-site, but you want to have at most 5 connections open at any one time.
 
     require "em-synchrony/em-http"
     EM.synchrony do
@@ -46,7 +41,8 @@ Features:
         EventMachine.stop
     end
 
-## Example connection pool shared by a fiber:
+## Fiber-aware ConnectionPool shared by a fiber:
+Allows you to create a pool of resources which are then shared by one or more fibers. A good example is a collection of long-lived database connections. The pool will automatically block and wake up the fibers as the connections become available.
 
     require "em-synchrony/em-mysqlplus"
     EventMachine.synchrony do
@@ -65,7 +61,8 @@ Features:
         EventMachine.stop
     end
 
-## Example with multi-request async em-http client:
+## Fiber-aware Multi inteface: parallel HTTP requests
+Allows you to fire simultaneous requests and wait for all of them to complete (success or error) before advancing. Concurrently fetching many HTTP pages at once is a good example; parallel SQL queries is another. Technically, this functionality can be also achieved by using the Synchrony Iterator shown above.
 
     require "em-synchrony/em-http"
     EventMachine.synchrony do
@@ -80,17 +77,44 @@ Features:
         EventMachine.stop
     end
 
-## Example with async Bitly client:
+## Fiber-aware & EventMachine backed TCPSocket:
+This is dangerous territory - you've been warned. You can patch your base TCPSocket class to make any/all libraries depending on TCPSocket be actually powered by EventMachine and Fibers under the hood.
+
+    require "lib/em-synchrony"
+    require "net/http"
+
+    EM.synchrony do
+      # replace default Socket code to use EventMachine Sockets instead
+      TCPSocket = EventMachine::Synchrony::TCPSocket
+
+      Net::HTTP.get_print 'www.google.com', '/index.html'
+      EM.stop
+    end
+
+## Inline synchronization & Fiber sleep:
+Allows you to inline/synchronize any callback interface to behave as if it was a blocking call. Simply pass any callback object to Synchrony.sync and it will do the right thing: the fiber will be resumed once the callback/errback fires. Likewise, use Synchrony.sleep to avoid blocking the main thread if you need to put one of your workers to sleep.
 
-    require "em-synchrony/em-bitly"
     EM.synchrony do
-        bitly = Bitly.new('[INSERT_LOGIN]', '[INSERT_API_KEY]')
-        url = 'http://github.com/igrigorik/em-synchrony'
-        short = bitly.shorten(url)
+      # pass a callback enabled client to sync to automatically resume it when callback fires
+      result = EM::Synchrony.sync EventMachine::HttpRequest.new('http://www.gooogle.com/').aget
+      p result
 
-        p "Short #{url} => #{short.jmp_url}"
+      # pause exection for 2 seconds
+      EM::Synchrony.sleep(2)
+
+      EM.stop
     end
 
+## Patched clients
+
+EM-Synchrony ships with a number of patched Eventmachine clients, which allow you to patch your asynchronous drivers on the fly to behave as if they were actually blocking clients:
+
+ * [em-http-request](http://github.com/igrigorik/em-synchrony/blob/master/spec/http_spec.rb)
+ * [em-mysqlplus](http://github.com/igrigorik/em-synchrony/blob/master/spec/mysqlplus_spec.rb)
+ * [em-redis](http://github.com/igrigorik/em-synchrony/blob/master/spec/redis_spec.rb)
+ * [em-memcached](http://github.com/igrigorik/em-synchrony/blob/master/spec/memcache_spec.rb) & [remcached](http://github.com/igrigorik/em-synchrony/blob/master/spec/remcached_spec.rb)
+ * [em-mongo](http://github.com/igrigorik/em-synchrony/blob/master/spec/em-mongo_spec.rb) & [mongoid](http://github.com/igrigorik/em-synchrony/blob/master/spec/mongo_spec.rb)
+
 # License
 
 (The MIT License)
@@ -114,4 +138,4 @@ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
 CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
-SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -1,20 +1,4 @@
-require "rake"
+require 'bundler'
 
-begin
-  require "jeweler"
-  Jeweler::Tasks.new do |gemspec|
-    gemspec.name = "em-synchrony"
-    gemspec.summary = "Fiber aware EventMachine libraries"
-    gemspec.description = gemspec.summary
-    gemspec.email = "ilya@igvita.com"
-    gemspec.homepage = "http://github.com/igrigorik/em-synchrony"
-    gemspec.authors = ["Ilya Grigorik"]
-    gemspec.required_ruby_version = ">= 1.9"
-    gemspec.add_dependency("eventmachine", ">= 0.12.9")
-    gemspec.rubyforge_project = "em-synchrony"
-  end
-
-  Jeweler::GemcutterTasks.new
-rescue LoadError
-  puts "Jeweler not available: gem install jeweler"
-end
+Bundler.setup
+Bundler::GemHelper.install_tasks

data/em-synchrony.gemspec

@@ -0,0 +1,23 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+
+Gem::Specification.new do |s|
+  s.name        = "em-synchrony"
+  s.version     = "0.2.0"
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ["Ilya Grigorik"]
+  s.email       = ["ilya@igvita.com"]
+  s.homepage    = "http://github.com/igrigorik/em-synchrony"
+  s.summary     = %q{Fiber aware EventMachine libraries}
+  s.description = s.summary
+
+  s.rubyforge_project = "em-synchrony"
+
+  s.required_ruby_version = Gem::Requirement.new(">= 1.9")
+  s.add_runtime_dependency("eventmachine", [">= 0.12.9"])
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+end

data/examples/bitly.rb

@@ -0,0 +1,25 @@
+require 'lib/em-synchrony'
+
+require "em-synchrony/em-bitly"
+EM.synchrony do
+  bitly = Bitly.new('[INSERT_LOGIN]', '[INSERT_API_KEY]')
+  url = 'http://github.com/igrigorik/em-synchrony'
+  short = bitly.shorten(url)
+
+  p "Short #{url} => #{short.jmp_url}"
+  EM.stop
+end
+
+
+Bitly.use_api_version_3
+EM.synchrony do
+  bitly = Bitly.new('[INSERT_LOGIN]', '[INSERT_API_KEY]')
+
+  url = 'http://github.com/igrigorik/em-synchrony'
+  domain='nyti.ms'
+
+  pro = bitly.bitly_pro_domain(domain)
+  p "Domain #{domain} pro=#{pro}"
+
+  EM.stop
+end

data/examples/go/README.md

@@ -0,0 +1,50 @@
+# CSP Experiments with Ruby
+
+Partly an exercise to help myself wrap my head around Go's concurrency, partly an experiment to see how much of the syntax & behavior of Go's CSP model can be modelled in Ruby... As it turns out, it's not hard to almost replicate the look and feel.
+
+Note: none of the Ruby examples actually give you the parallelism of Go.
+
+## Notes
+ * Instead of explicitly using locks to mediate access to shared data, Go encourages the use of channels to pass references to data between goroutines.
+
+ * Channels combine communication — the exchange of a value—with synchronization — guaranteeing that two calculations (goroutines) are in a known state.
+
+go.rb implements an (un)bounded Channel interface, and with some help from Fibers & Ruby 1.9, we can also implement the goroutine look and feel pretty easily. In fact, with CSP semantics, its not hard to imagine a MVM (multi-VM) Ruby where each VM still has a GIL, but where data sharing is done via communication of references between VM's.
+
+## Simple channel example in Go
+
+    package main
+
+    import (
+      "fmt"
+      "time"
+    )
+
+    func main() {
+      c := make(chan string)
+
+      go func() {
+        time.Sleep(1)
+        c <- "go go go sleep 1!"
+       }()
+
+       fmt.Printf("%v\n", <-c)  // Wait for goroutine to finish
+    }
+
+## Equivalent in Ruby
+
+    require 'go'
+
+    EM.synchrony do
+      c = Channel.new
+
+      go {
+        sleep(1)
+        c << 'go go go sleep 1!'
+      }
+
+      puts c.pop
+
+      EM.stop
+    end
+

data/examples/go/channel.go

@@ -0,0 +1,17 @@
+package main
+
+import (
+  "fmt"
+  "time"
+)
+
+func main() {
+  c := make(chan string)
+
+  go func() {
+    time.Sleep(1)
+    c <- "go go go sleep 1!"
+   }()
+
+   fmt.Printf("%v\n", <-c)  // Wait for goroutine to finish
+}

data/examples/go/channel.rb

@@ -0,0 +1,14 @@
+require 'go'
+
+EM.synchrony do
+  c = Channel.new
+
+  go {
+    sleep(1)
+    c << 'go go go sleep 1!'
+  }
+
+  puts c.pop
+
+  EM.stop
+end

data/examples/go/consumer-publisher.go

@@ -0,0 +1,34 @@
+package main
+
+import (
+  "fmt"
+  "runtime"
+)
+
+func producer(c chan int, N int, s chan bool) {
+  for i := 0; i < N; i++ {
+    fmt.Printf("producer: %d\n", i)
+    c <- i
+  }
+  s <- true
+}
+
+func consumer(c chan int, N int, s chan bool) {
+  for i := 0; i < N; i++ {
+    fmt.Printf("consumer got: %d\n", <- c)
+  }
+  s <- true
+}
+
+func main() {
+  runtime.GOMAXPROCS(2)
+
+  c := make(chan int)
+  s := make(chan bool)
+
+  go producer(c, 10, s)
+  go consumer(c, 10, s)
+
+  <- s
+  <- s
+}

data/examples/go/consumer-publisher.rb

@@ -0,0 +1,62 @@
+require 'go'
+
+EM.synchrony do
+  def producer(c, n, s)
+    n.times do |i|
+      puts "producer: #{i}"
+      c << i
+    end
+
+    s << "producer finished"
+  end
+
+  consumer = ->(c, n, s) do
+    n.times do |i|
+      puts "consumer 1 got: #{c.pop}"
+    end
+
+    s << "consumer finished"
+  end
+
+  c = Channel.new(size: 2)
+  s = Channel.new
+  n = 10
+
+  # mix the syntax, just for fun...
+  go c,n,s, &method(:producer)
+  go c,n-1,s, &consumer
+
+  go c,s do |c, s|
+    puts "consumer 2 got: #{c.pop}"
+    s << "consumer 2 finished"
+  end
+
+  3.times { puts s.pop }
+
+  EM.stop
+end
+
+# (M=6c0863) igrigorik /git/em-synchrony/examples/go> ruby consumer-publisher.rb
+# producer: 0
+# producer: 1
+# consumer 1 got: [0]
+# producer: 2
+# consumer 2 got: [1]
+# producer: 3
+# consumer 1 got: [2]
+# producer: 4
+# consumer 2 finished
+# consumer 1 got: [3]
+# producer: 5
+# consumer 1 got: [4]
+# producer: 6
+# consumer 1 got: [5]
+# producer: 7
+# consumer 1 got: [6]
+# producer: 8
+# consumer 1 got: [7]
+# producer: 9
+# consumer 1 got: [8]
+# consumer 1 got: [9]
+# producer finished
+# consumer finished

data/examples/go/go.rb

@@ -0,0 +1,49 @@
+require 'em-synchrony'
+
+module Kernel
+  def go(*args, &blk)
+    EM.next_tick do
+      Fiber.new { blk.call(*args) }.resume
+    end
+  end
+end
+
+class Channel < EM::Queue
+  def initialize(opts = {})
+    @limit = opts[:size]
+    @prodq = []
+    @size  = 0
+
+    super()
+  end
+
+  def size; @size; end
+  def empty?; size == 0; end
+
+  def pop
+    f = Fiber.current
+    clb = Proc.new do |*args|
+      @size -= 1
+      f.resume(args)
+      @prodq.shift.call if !@prodq.empty?
+    end
+
+    super(&clb)
+    Fiber.yield
+  end
+
+  def push(*items)
+    f = Fiber.current
+    @size += 1
+
+    EM.next_tick { super(*items) }
+
+    # if the queue is bounded, then suspend the producer
+    # until someone consumes a pending message
+    if @limit && size >= @limit
+      @prodq.push -> { f.resume }
+      Fiber.yield
+    end
+  end
+  alias :<< :push
+end