em-synchrony 0.1.2 → 0.1.4

data/README.md

@@ -1,71 +1,86 @@
 # EM-Synchrony
 
+http://www.igvita.com/2010/03/22/untangling-evented-code-with-ruby-fibers
+
 Collection of convenience classes and patches to common EventMachine clients to 
 make them Fiber aware and friendly. Word of warning: even though fibers have been
 backported to Ruby 1.8.x, these classes assume Ruby 1.9 (if you're using fibers
-in production, you should be on 1.9 anyway) 
+in production, you should be on 1.9 anyway)
 
 Features:
 
  * Fiber aware connection pool with sync/async query support
  * Multi request interface which accepts any callback enabled client
+ * Fibered iterator to allow concurrency control & mixing of sync / async
  * em-http-request: .get, etc are synchronous, while .aget, etc are async
  * em-mysqlplus: .query is synchronous, while .aquery is async
  * remcached: .get, etc, and .multi_* methods are synchronous
 
 ## Example with async em-http client:
 
-	EventMachine.run do
-      Fiber.new {
-        res = EventMachine::HttpRequest.new("http://www.postrank.com").get
+	EventMachine.synchrony do
+      res = EventMachine::HttpRequest.new("http://www.postrank.com").get
 		
-		p "Look ma, no callbacks!"
-		p res
+	  p "Look ma, no callbacks!"
+	  p res
 
-        EventMachine.stop
-      }.resume
+      EventMachine.stop
     end
 
-## Example with multi-request async em-http client:
+## EM Iterator & mixing sync / async code
 
-	EventMachine.run do
-	  Fiber.new {
-		
- 	    multi = EventMachine::Synchrony::Multi.new
-        multi.add :a, EventMachine::HttpRequest.new("http://www.postrank.com").aget
-        multi.add :b, EventMachine::HttpRequest.new("http://www.postrank.com").apost
-        res = multi.perform
-	
-		p "Look ma, no callbacks, and parallel requests!"
-		p res
+	EM.synchrony do
+	  concurrency = 2
+	  urls = ['http://url.1.com', 'http://url2.com']
+
+	  # iterator will execute async blocks until completion, .each, .inject also work!
+	  results = EM::Synchrony::Iterator.new(urls, concurrency).map do |url, iter|
+
+		# fire async requests, on completion advance the iterator
+	    http = EventMachine::HttpRequest.new(url).aget
+	    http.callback { iter.return(http) }
+	  end
 
-	    EventMachine.stop
-	  }.resume
+	  p results # all completed requests
+
+	  EventMachine.stop
 	end
 
 ## Example connection pool shared by a fiber:
 
-	EventMachine.run do
-
+	EventMachine.synchrony do
 	  db = EventMachine::Synchrony::ConnectionPool.new(size: 2) do
 	    EventMachine::MySQL.new(host: "localhost")
 	  end
 
-	  Fiber.new {
-	    start = now
+	  start = now
 
-	    multi = EventMachine::Synchrony::Multi.new
-	    multi.add :a, db.aquery("select sleep(1)")
-	    multi.add :b, db.aquery("select sleep(1)")
-	    res = multi.perform
+	  multi = EventMachine::Synchrony::Multi.new
+	  multi.add :a, db.aquery("select sleep(1)")
+	  multi.add :b, db.aquery("select sleep(1)")
+	  res = multi.perform
 
-		p "Look ma, no callbacks, and parallel requests!"
-		p res
+	  p "Look ma, no callbacks, and parallel MySQL requests!"
+	  p res
 
-	    EventMachine.stop
-	  }.resume
+	  EventMachine.stop
 	end
 
+
+## Example with multi-request async em-http client:
+
+	EventMachine.synchrony do
+ 	  multi = EventMachine::Synchrony::Multi.new
+      multi.add :a, EventMachine::HttpRequest.new("http://www.postrank.com").aget
+      multi.add :b, EventMachine::HttpRequest.new("http://www.postrank.com").apost
+      res = multi.perform
+	 
+	  p "Look ma, no callbacks, and parallel HTTP requests!"
+	  p res
+     
+	  EventMachine.stop
+	end
+	
 # License
 
 (The MIT License)

data/VERSION

@@ -1 +1 @@
-0.1.2
+0.1.4

data/examples/all.rb

@@ -0,0 +1,27 @@
+require "lib/em-synchrony"
+
+EM.synchrony do
+
+  # open 4 concurrent MySQL connections
+  db = EventMachine::Synchrony::ConnectionPool.new(size: 4) do
+    EventMachine::MySQL.new(host: "localhost")
+  end
+
+  # perform 4 http requests in parallel, and collect responses
+  multi = EventMachine::Synchrony::Multi.new
+  multi.add :page1, EventMachine::HttpRequest.new("http://service.com/page1").aget
+  multi.add :page2, EventMachine::HttpRequest.new("http://service.com/page2").aget
+  multi.add :page3, EventMachine::HttpRequest.new("http://service.com/page3").aget
+  multi.add :page4, EventMachine::HttpRequest.new("http://service.com/page4").aget
+  data = multi.perform.responses[:callback].values
+
+  # insert fetched HTTP data into a mysql database, using at most 2 connections at a time
+  # - note that we're writing async code within the callback!
+  EM::Synchrony::Iterator.new(data, 2).each do |page, iter|
+    db.aquery("INSERT INTO table (data) VALUES(#{page});")
+    db.callback { iter.return(http) }
+  end
+
+  puts "All done! Stopping event loop."
+  EventMachine.stop
+end

data/lib/em-synchrony.rb

@@ -5,9 +5,23 @@ require "eventmachine"
 require "fiber"
 
 require "em-synchrony/em-multi"
+# require "em-synchrony/iterator" # iterators are not release in EM yet
+require "em-synchrony/connection_pool"
+
 require "em-synchrony/em-http"
 require "em-synchrony/em-mysql"
-# require "em-synchrony/em-jack"
 require "em-synchrony/em-remcached"
 
-require "em-synchrony/connection_pool"
+module EventMachine
+
+  # A convenience method for wrapping EM.run body within
+  # a Ruby Fiber such that async operations can be transparently
+  # paused and resumed based on IO scheduling.
+  def self.synchrony(blk=nil, tail=nil, &block)
+    blk ||= block
+    context = Proc.new { Fiber.new { blk.call }.resume }
+
+    self.run(context, tail)
+  end
+
+end

data/lib/em-synchrony/iterator.rb

@@ -0,0 +1,52 @@
+require "em/iterator"
+
+module EventMachine
+  module Synchrony
+
+    class Iterator < EM::Iterator
+
+      # synchronous iterator which will wait until all the
+      # jobs are done before returning. Unfortunately this
+      # means that you loose ability to choose concurrency
+      # on the fly (see iterator documentation in EM)
+      def each(foreach=nil, after=nil, &blk)
+        fiber = Fiber.current
+
+        fe = (foreach || blk)
+        cb = Proc.new do
+          after.call if after
+          fiber.resume
+        end
+
+        Fiber.yield super(fe, cb)
+      end
+
+      def map(&block)
+        fiber = Fiber.current
+        result = nil
+
+        after = Proc.new {|res| result = res; fiber.resume }
+        super(block, after)
+
+        Fiber.yield
+        result
+      end
+
+      def inject(obj, foreach = nil, after = nil, &block)
+        if foreach and after
+          super(obj, foreach, after)
+        else
+          fiber = Fiber.current
+          result = nil
+
+          after = Proc.new {|res| result = res; fiber.resume}
+          super(obj, block, after)
+
+          Fiber.yield
+          result
+        end
+      end
+
+    end
+  end
+end

data/spec/beanstalk_spec.rb

@@ -1,12 +1,14 @@
-require 'spec/helper'
+require 'spec/helper/all'
 
 DELAY = 0.25
 
+__END__
 describe EMJack do
 
   it "should fire sequential Beanstalk requests" do
+    pending
+    
     EventMachine.run do
-
       Fiber.new {
         jack = EMJack::Connection.new
 
@@ -19,8 +21,9 @@ describe EMJack do
   end
 
   it "should fire multiple requests in parallel" do
+    pending
+    
     EventMachine.run do
-
       Fiber.new {
         jack = EMJack::Connection.new
 

data/spec/http_spec.rb

@@ -5,47 +5,45 @@ DELAY = 0.25
 
 describe EventMachine::HttpRequest do
   it "should fire sequential requests" do
-    EventMachine.run do
-      @s = StubServer.new("HTTP/1.0 200 OK\r\nConnection: close\r\n\r\nFoo", DELAY)
-
-      Fiber.new {
-        start = now
-        order = []
-        order.push :get  if EventMachine::HttpRequest.new(URL).get
-        order.push :post if EventMachine::HttpRequest.new(URL).post
-        order.push :head if EventMachine::HttpRequest.new(URL).head
-        order.push :post if EventMachine::HttpRequest.new(URL).delete
-        order.push :put  if EventMachine::HttpRequest.new(URL).put
-
-        (now - start.to_f).should be_within(DELAY * order.size * 0.15).of(DELAY * order.size)
-        order.should == [:get, :post, :head, :post, :put]
-
-        EventMachine.stop
-      }.resume
+    EventMachine.synchrony do
+      s = StubServer.new("HTTP/1.0 200 OK\r\nConnection: close\r\n\r\nFoo", DELAY)
+
+      start = now
+      order = []
+      order.push :get  if EventMachine::HttpRequest.new(URL).get
+      order.push :post if EventMachine::HttpRequest.new(URL).post
+      order.push :head if EventMachine::HttpRequest.new(URL).head
+      order.push :post if EventMachine::HttpRequest.new(URL).delete
+      order.push :put  if EventMachine::HttpRequest.new(URL).put
+
+      (now - start.to_f).should be_within(DELAY * order.size * 0.15).of(DELAY * order.size)
+      order.should == [:get, :post, :head, :post, :put]
+
+      s.stop
+      EventMachine.stop
     end
   end
 
   it "should fire simultaneous requests via Multi interface" do
-    EventMachine.run do
-      @s = StubServer.new("HTTP/1.0 200 OK\r\nConnection: close\r\n\r\nFoo", DELAY)
-
-      Fiber.new {
-        start = now
-
-        multi = EventMachine::Synchrony::Multi.new
-        multi.add :a, EventMachine::HttpRequest.new(URL).aget
-        multi.add :b, EventMachine::HttpRequest.new(URL).apost
-        multi.add :c, EventMachine::HttpRequest.new(URL).ahead
-        multi.add :d, EventMachine::HttpRequest.new(URL).adelete
-        multi.add :e, EventMachine::HttpRequest.new(URL).aput
-        res = multi.perform
-        
-        (now - start.to_f).should be_within(DELAY * 0.15).of(DELAY)
-        res.responses[:callback].size.should == 5
-        res.responses[:errback].size.should == 0
-
-        EventMachine.stop
-      }.resume
+    EventMachine.synchrony do
+      s = StubServer.new("HTTP/1.0 200 OK\r\nConnection: close\r\n\r\nFoo", DELAY)
+
+      start = now
+
+      multi = EventMachine::Synchrony::Multi.new
+      multi.add :a, EventMachine::HttpRequest.new(URL).aget
+      multi.add :b, EventMachine::HttpRequest.new(URL).apost
+      multi.add :c, EventMachine::HttpRequest.new(URL).ahead
+      multi.add :d, EventMachine::HttpRequest.new(URL).adelete
+      multi.add :e, EventMachine::HttpRequest.new(URL).aput
+      res = multi.perform
+
+      (now - start.to_f).should be_within(DELAY * 0.15).of(DELAY)
+      res.responses[:callback].size.should == 5
+      res.responses[:errback].size.should == 0
+
+      s.stop
+      EventMachine.stop
     end
   end
 end

data/spec/iterator_spec.rb

@@ -0,0 +1,68 @@
+require "spec/helper/all"
+require "em-synchrony/iterator"
+
+describe EventMachine::Synchrony::Iterator do
+
+  it "should wait until the iterator is done" do
+    EM.synchrony do
+
+      results = []
+      i = EM::Synchrony::Iterator.new(1..50, 10).each do |num, iter|
+        results.push num
+        iter.next
+      end
+
+      results.should == (1..50).to_a
+      results.size.should == 50
+      EventMachine.stop
+    end
+  end
+
+  it "should map values within the iterator" do
+    EM.synchrony do
+      results = EM::Synchrony::Iterator.new(1..50, 10).map do |num, iter|
+        iter.return(num + 1)
+      end
+
+      results.should == (2..51).to_a
+      results.size.should == 50
+      EventMachine.stop
+    end
+  end
+
+  it "should sum values within the iterator" do
+    EM.synchrony do
+      data = (1..50).to_a
+      res = EM::Synchrony::Iterator.new(data, 10).inject(0) do |total, num, iter|
+        total += num
+        iter.return(total)
+      end
+
+      res.should == data.inject(:+)
+      EventMachine.stop
+    end
+  end
+
+  it "should fire async http requests in blocks of 2" do
+    EM.synchrony do
+      num_urls = 4
+      concurrency = 2
+      delay = 0.25
+      
+      s = StubServer.new("HTTP/1.0 200 OK\r\nConnection: close\r\n\r\nFoo", delay)
+      urls = ['http://localhost:8081/'] * num_urls
+      
+      start = now
+      results = EM::Synchrony::Iterator.new(urls, concurrency).map do |url, iter|
+        http = EventMachine::HttpRequest.new(url).aget
+        http.callback { iter.return(http) }
+      end
+
+      results.size.should == 4
+      (now - start.to_f).should be_within(delay * 0.15).of(delay * (num_urls / concurrency))
+      
+      EventMachine.stop
+    end
+  end
+
+end

data/spec/mysql_spec.rb

@@ -7,23 +7,21 @@ QUERY = "select sleep(#{DELAY})"
 describe EventMachine::MySQL do
 
   it "should fire sequential, synchronous requests" do
-    EventMachine.run do
-      Fiber.new {
-        db = EventMachine::MySQL.new(host: "localhost")
-        start = now
-        res = []
+    EventMachine.synchrony do
+      db = EventMachine::MySQL.new(host: "localhost")
+      start = now
+      res = []
 
-        res.push db.query(QUERY)
-        res.push db.query(QUERY)
-        (now - start.to_f).should be_within(DELAY * res.size * 0.15).of(DELAY * res.size)
+      res.push db.query(QUERY)
+      res.push db.query(QUERY)
+      (now - start.to_f).should be_within(DELAY * res.size * 0.15).of(DELAY * res.size)
 
-        EventMachine.stop
-      }.resume
+      EventMachine.stop
     end
   end
 
   it "should have accept a callback, errback on async queries" do
-    EventMachine.run do
+    EventMachine.synchrony do
       db = EventMachine::MySQL.new(host: "localhost")
 
       res = db.aquery(QUERY)
@@ -36,58 +34,54 @@ describe EventMachine::MySQL do
   end
 
   it "should fire simultaneous requests via Multi interface" do
-    EventMachine.run do
+    EventMachine.synchrony do
 
       db = EventMachine::Synchrony::ConnectionPool.new(size: 2) do
         EventMachine::MySQL.new(host: "localhost")
       end
 
-      Fiber.new {
-        start = now
+      start = now
 
-        multi = EventMachine::Synchrony::Multi.new
-        multi.add :a, db.aquery(QUERY)
-        multi.add :b, db.aquery(QUERY)
-        res = multi.perform
+      multi = EventMachine::Synchrony::Multi.new
+      multi.add :a, db.aquery(QUERY)
+      multi.add :b, db.aquery(QUERY)
+      res = multi.perform
 
-        (now - start.to_f).should be_within(DELAY * 0.15).of(DELAY)
-        res.responses[:callback].size.should == 2
-        res.responses[:errback].size.should == 0
+      (now - start.to_f).should be_within(DELAY * 0.15).of(DELAY)
+      res.responses[:callback].size.should == 2
+      res.responses[:errback].size.should == 0
 
-        EventMachine.stop
-      }.resume
+      EventMachine.stop
     end
   end
 
   it "should fire sequential and simultaneous MySQL requests" do
-    EventMachine.run do
+    EventMachine.synchrony do
       db = EventMachine::Synchrony::ConnectionPool.new(size: 3) do
         EventMachine::MySQL.new(host: "localhost")
       end
 
-      Fiber.new {
-        start = now
-        res = []
+      start = now
+      res = []
 
-        res.push db.query(QUERY)
-        res.push db.query(QUERY)
-        (now - start.to_f).should be_within(DELAY * res.size * 0.15).of(DELAY * res.size)
+      res.push db.query(QUERY)
+      res.push db.query(QUERY)
+      (now - start.to_f).should be_within(DELAY * res.size * 0.15).of(DELAY * res.size)
 
-        start = now
+      start = now
 
-        multi = EventMachine::Synchrony::Multi.new
-        multi.add :a, db.aquery(QUERY)
-        multi.add :b, db.aquery(QUERY)
-        multi.add :c, db.aquery(QUERY)
-        res = multi.perform
+      multi = EventMachine::Synchrony::Multi.new
+      multi.add :a, db.aquery(QUERY)
+      multi.add :b, db.aquery(QUERY)
+      multi.add :c, db.aquery(QUERY)
+      res = multi.perform
 
-        (now - start.to_f).should be_within(DELAY * 0.15).of(DELAY)
-        res.responses[:callback].size.should == 3
-        res.responses[:errback].size.should == 0
+      (now - start.to_f).should be_within(DELAY * 0.15).of(DELAY)
+      res.responses[:callback].size.should == 3
+      res.responses[:errback].size.should == 0
 
-        EventMachine.stop
-      }.resume
+      EventMachine.stop
     end
   end
 
-end
+end

data/spec/remcached_spec.rb

@@ -4,47 +4,38 @@ require "remcached"
 describe Memcached do
 
   it "should yield until connection is ready" do
-    EventMachine.run do
-      Fiber.new {
-        Memcached.connect %w(localhost)
-        Memcached.usable?.should be_true
-        EventMachine.stop
-      }.resume
+    EventMachine.synchrony do
+      Memcached.connect %w(localhost)
+      Memcached.usable?.should be_true
+      EventMachine.stop
     end
   end
 
   it "should fire sequential memcached requests" do
-    EventMachine.run do
-      Fiber.new {
-
-        Memcached.connect %w(localhost)
-        Memcached.get(key: 'hai') do |res|
-          res[:value].should match('Not found')
-        end
-
-        Memcached.set(key: 'hai', value: 'win')
-        Memcached.add(key: 'count')
-        Memcached.delete(key: 'hai')           
-        
-        EventMachine.stop
-      }.resume
+    EventMachine.synchrony do
+
+      Memcached.connect %w(localhost)
+      Memcached.get(key: 'hai') do |res|
+        res[:value].should match('Not found')
+      end
+
+      Memcached.set(key: 'hai', value: 'win')
+      Memcached.add(key: 'count')
+      Memcached.delete(key: 'hai')
+
+      EventMachine.stop
     end
   end
-  
+
   it "should fire multi memcached requests" do
-    pending "remcached borked? opened a ticket"
-    
-    EventMachine.run do
-      Fiber.new {
-
-        Memcached.connect %w(localhost)
-        
-        Memcached.multi_get([{:key => 'foo'},{:key => 'bar'}, {:key => 'test'}]) do |res|
-          p res
-        end
-                
+    EventMachine.synchrony do
+      pending "patch mult-get"
+
+      Memcached.connect %w(localhost)
+      Memcached.multi_get([{:key => 'foo'},{:key => 'bar'}, {:key => 'test'}]) do |res|
+        # TODO
         EventMachine.stop
-      }.resume
+      end
     end
   end
 

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 0
   - 1
-  - 2
-  version: 0.1.2
+  - 4
+  version: 0.1.4
 platform: ruby
 authors: 
 - Ilya Grigorik
@@ -14,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-03-19 00:00:00 -04:00
+date: 2010-03-22 00:00:00 -04:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -50,12 +50,14 @@ files:
 - lib/em-synchrony/em-multi.rb
 - lib/em-synchrony/em-mysql.rb
 - lib/em-synchrony/em-remcached.rb
+- lib/em-synchrony/iterator.rb
 - spec/beanstalk_spec.rb
 - spec/connection_pool_spec.rb
 - spec/helper/all.rb
 - spec/helper/stub-http-server.rb
 - spec/helper/tolerance_matcher.rb
 - spec/http_spec.rb
+- spec/iterator_spec.rb
 - spec/mysql_spec.rb
 - spec/remcached_spec.rb
 has_rdoc: true
@@ -96,5 +98,7 @@ test_files:
 - spec/helper/stub-http-server.rb
 - spec/helper/tolerance_matcher.rb
 - spec/http_spec.rb
+- spec/iterator_spec.rb
 - spec/mysql_spec.rb
 - spec/remcached_spec.rb
+- examples/all.rb