eventmachine 0.12.10 → 1.0.0.beta.1

data/java/src/com/rubyeventmachine/EmReactor.java

@@ -48,6 +48,7 @@ public class EmReactor {
 	public final int EM_SSL_HANDSHAKE_COMPLETED = 108;
 	public final int EM_SSL_VERIFY = 109;
 	public final int EM_PROXY_TARGET_UNBOUND = 110;
+    public final int EM_PROXY_COMPLETED = 111;
 
 	private Selector mySelector;
 	private TreeMap<Long, ArrayList<Long>> Timers;

data/lib/em/channel.rb

@@ -35,7 +35,7 @@ module EventMachine
     # Add items to the channel, which are pushed out to all subscribers.
     def push(*items)
       items = items.dup
-      EM.schedule { @subs.values.each { |s| items.each { |i| s.call i } } }
+      EM.schedule { items.each { |i| @subs.values.each { |s| s.call i } } }
     end
     alias << push
 

data/lib/em/connection.rb

@@ -36,7 +36,7 @@ module EventMachine
       allocate.instance_eval do
         # Store signature
         @signature = sig
-        associate_callback_target sig
+        # associate_callback_target sig
 
         # Call a superclass's #initialize if it has one
         initialize(*args)
@@ -134,6 +134,11 @@ module EventMachine
     def proxy_target_unbound
     end
 
+    # EventMachine::Connection#proxy_completed is called when the reactor finished proxying all
+    # of the requested bytes.
+    def proxy_completed
+    end
+
     # EventMachine::Connection#proxy_incoming_to is called only by user code. It sets up
     # a low-level proxy relay for all data inbound for this connection, to the connection given
     # as the argument. This is essentially just a helper method for enable_proxy.

data/lib/em/deferrable.rb

@@ -51,6 +51,13 @@ module EventMachine
       end
     end
 
+    # Cancels an outstanding callback to &block if any. Undoes the action of #callback.
+    #
+    def cancel_callback block
+      @callbacks ||= []
+      @callbacks.delete block
+    end
+
     # Specify a block to be executed if and when the Deferrable object receives
     # a status of :failed. See #set_deferred_status for more information.
     #--
@@ -69,6 +76,13 @@ module EventMachine
       end
     end
 
+    # Cancels an outstanding errback to &block if any. Undoes the action of #errback.
+    #
+    def cancel_errback block
+      @errbacks ||= []
+      @errbacks.delete block
+    end
+
     # Sets the "disposition" (status) of the Deferrable object. See also the large set of
     # sugarings for this method.
     # Note that if you call this method without arguments,
@@ -150,10 +164,10 @@ module EventMachine
     # the Timeout expires (passing no arguments to the object's errbacks).
     # Setting the status at any time prior to a call to the expiration of the timeout
     # will cause the timer to be cancelled.
-    def timeout seconds
+    def timeout seconds, *args
       cancel_timeout
       me = self
-      @deferred_timeout = EventMachine::Timer.new(seconds) {me.fail}
+      @deferred_timeout = EventMachine::Timer.new(seconds) {me.fail(*args)}
     end
 
     # Cancels an outstanding timeout if any. Undoes the action of #timeout.

data/lib/em/iterator.rb

@@ -0,0 +1,270 @@
+module EventMachine
+  # A simple iterator for concurrent asynchronous work.
+  #
+  # Unlike ruby's built-in iterators, the end of the current iteration cycle is signaled manually,
+  # instead of happening automatically after the yielded block finishes executing. For example:
+  #
+  #  (0..10).each{ |num| }
+  #
+  # becomes:
+  #
+  #  EM::Iterator.new(0..10).each{ |num,iter| iter.next }
+  #
+  # This is especially useful when doing asynchronous work via reactor libraries and
+  # functions. For example, given a sync and async http api:
+  #
+  #  response = sync_http_get(url); ...
+  #  async_http_get(url){ |response| ... }
+  #
+  # a synchronous iterator such as:
+  #
+  #  responses = urls.map{ |url| sync_http_get(url) }
+  #  ...
+  #  puts 'all done!'
+  #
+  # could be written as:
+  #
+  #  EM::Iterator.new(urls).map(proc{ |url,iter|
+  #    async_http_get(url){ |res|
+  #      iter.return(res)
+  #    }
+  #  }, proc{ |responses|
+  #    ...
+  #    puts 'all done!'
+  #  })
+  #
+  # Now, you can take advantage of the asynchronous api to issue requests in parallel. For example,
+  # to fetch 10 urls at a time, simply pass in a concurrency of 10:
+  #
+  #  EM::Iterator.new(urls, 10).each do |url,iter|
+  #    async_http_get(url){ iter.next }
+  #  end
+  #
+  class Iterator
+    # Create a new parallel async iterator with specified concurrency.
+    #
+    #  i = EM::Iterator.new(1..100, 10)
+    #
+    # will create an iterator over the range that processes 10 items at a time. Iteration
+    # is started via #each, #map or #inject
+    #
+    def initialize(list, concurrency = 1)
+      raise ArgumentError, 'argument must be an array' unless list.respond_to?(:to_a)
+      @list = list.to_a.dup
+      @concurrency = concurrency
+
+      @started = false
+      @ended = false
+    end
+
+    # Change the concurrency of this iterator. Workers will automatically be spawned or destroyed
+    # to accomodate the new concurrency level.
+    #
+    def concurrency=(val)
+      old = @concurrency
+      @concurrency = val
+
+      spawn_workers if val > old and @started and !@ended
+    end
+    attr_reader :concurrency
+
+    # Iterate over a set of items using the specified block or proc.
+    #
+    #  EM::Iterator.new(1..100).each do |num, iter|
+    #    puts num
+    #    iter.next
+    #  end
+    #
+    # An optional second proc is invoked after the iteration is complete.
+    #
+    #  EM::Iterator.new(1..100).each(
+    #    proc{ |num,iter| iter.next },
+    #    proc{ puts 'all done' }
+    #  )
+    #
+    def each(foreach=nil, after=nil, &blk)
+      raise ArgumentError, 'proc or block required for iteration' unless foreach ||= blk
+      raise RuntimeError, 'cannot iterate over an iterator more than once' if @started or @ended
+
+      @started = true
+      @pending = 0
+      @workers = 0
+
+      all_done = proc{
+        after.call if after and @ended and @pending == 0
+      }
+
+      @process_next = proc{
+        # p [:process_next, :pending=, @pending, :workers=, @workers, :ended=, @ended, :concurrency=, @concurrency, :list=, @list]
+        unless @ended or @workers > @concurrency
+          if @list.empty?
+            @ended = true
+            @workers -= 1
+            all_done.call
+          else
+            item = @list.shift
+            @pending += 1
+
+            is_done = false
+            on_done = proc{
+              raise RuntimeError, 'already completed this iteration' if is_done
+              is_done = true
+
+              @pending -= 1
+
+              if @ended
+                all_done.call
+              else
+                EM.next_tick(@process_next)
+              end
+            }
+            class << on_done
+              alias :next :call
+            end
+
+            foreach.call(item, on_done)
+          end
+        else
+          @workers -= 1
+        end
+      }
+
+      spawn_workers
+
+      self
+    end
+
+    # Collect the results of an asynchronous iteration into an array.
+    #
+    #  EM::Iterator.new(%w[ pwd uptime uname date ], 2).map(proc{ |cmd,iter|
+    #    EM.system(cmd){ |output,status|
+    #      iter.return(output)
+    #    }
+    #  }, proc{ |results|
+    #    p results
+    #  })
+    #
+    def map(foreach, after)
+      index = 0
+
+      inject([], proc{ |results,item,iter|
+        i = index
+        index += 1
+
+        is_done = false
+        on_done = proc{ |res|
+          raise RuntimeError, 'already returned a value for this iteration' if is_done
+          is_done = true
+
+          results[i] = res
+          iter.return(results)
+        }
+        class << on_done
+          alias :return :call
+          def next
+            raise NoMethodError, 'must call #return on a map iterator'
+          end
+        end
+
+        foreach.call(item, on_done)
+      }, proc{ |results|
+        after.call(results)
+      })
+    end
+
+    # Inject the results of an asynchronous iteration onto a given object.
+    #
+    #  EM::Iterator.new(%w[ pwd uptime uname date ], 2).inject({}, proc{ |hash,cmd,iter|
+    #    EM.system(cmd){ |output,status|
+    #      hash[cmd] = status.exitstatus == 0 ? output.strip : nil
+    #      iter.return(hash)
+    #    }
+    #  }, proc{ |results|
+    #    p results
+    #  })
+    #
+    def inject(obj, foreach, after)
+      each(proc{ |item,iter|
+        is_done = false
+        on_done = proc{ |res|
+          raise RuntimeError, 'already returned a value for this iteration' if is_done
+          is_done = true
+
+          obj = res
+          iter.next
+        }
+        class << on_done
+          alias :return :call
+          def next
+            raise NoMethodError, 'must call #return on an inject iterator'
+          end
+        end
+
+        foreach.call(obj, item, on_done)
+      }, proc{
+        after.call(obj)
+      })
+    end
+
+    private
+
+    # Spawn workers to consume items from the iterator's enumerator based on the current concurrency level.
+    #
+    def spawn_workers
+      EM.next_tick(start_worker = proc{
+        if @workers < @concurrency and !@ended
+          # p [:spawning_worker, :workers=, @workers, :concurrency=, @concurrency, :ended=, @ended]
+          @workers += 1
+          @process_next.call
+          EM.next_tick(start_worker)
+        end
+      })
+      nil
+    end
+  end
+end
+
+if __FILE__ == $0
+  $:.unshift File.join(File.dirname(__FILE__), '..')
+  require 'eventmachine'
+
+  # TODO: real tests
+  # TODO: pass in one object instead of two? .each{ |iter| puts iter.current; iter.next }
+  # TODO: support iter.pause/resume/stop/break/continue?
+  # TODO: create some exceptions instead of using RuntimeError
+  # TODO: support proc instead of enumerable? EM::Iterator.new(proc{ return queue.pop })
+
+  EM.run{
+    EM::Iterator.new(1..50).each{ |num,iter| p num; iter.next }
+    EM::Iterator.new([1,2,3], 10).each{ |num,iter| p num; iter.next }
+
+    i = EM::Iterator.new(1..100, 5)
+    i.each(proc{|num,iter|
+      p num.to_s
+      iter.next
+    }, proc{
+      p :done
+    })
+    EM.add_timer(0.03){
+      i.concurrency = 1
+    }
+    EM.add_timer(0.04){
+      i.concurrency = 3
+    }
+
+    EM::Iterator.new(100..150).map(proc{ |num,iter|
+      EM.add_timer(0.01){ iter.return(num) }
+    }, proc{ |results|
+      p results
+    })
+
+    EM::Iterator.new(%w[ pwd uptime uname date ], 2).inject({}, proc{ |hash,cmd,iter|
+      EM.system(cmd){ |output,status|
+        hash[cmd] = status.exitstatus == 0 ? output.strip : nil
+        iter.return(hash)
+      }
+    }, proc{ |results|
+      p results
+    })
+  }
+end

data/lib/em/protocols.rb

@@ -5,7 +5,7 @@ module EventMachine
   # - Memcache
   # - SmtpClient and SmtpServer
   # - SASLauth and SASLauthclient
-  # - LineAndTextProtocol and LineText2
+  # - LineProtocol, LineAndTextProtocol and LineText2
   # - HeaderAndContentProtocol
   # - Postgres3
   # - ObjectProtocol

data/lib/em/protocols/httpclient.rb

@@ -142,6 +142,11 @@ module EventMachine
             req << "Cookie: #{args[:cookie]}"
           end
 
+          # Allow custom HTTP headers, e.g. SOAPAction
+          args[:custom_headers].each do |k,v|
+            req << "#{k}: #{v}"
+          end if args[:custom_headers]
+
           # Basic-auth stanza contributed by Matt Murphy.
           if args[:basic_auth]
             basic_auth_string = ["#{args[:basic_auth][:username]}:#{args[:basic_auth][:password]}"].pack('m').strip.gsub(/\n/,'')

data/lib/em/protocols/line_protocol.rb

@@ -0,0 +1,28 @@
+module EventMachine
+  module Protocols
+    # LineProtocol will parse out newline terminated strings from a receive_data stream
+    #
+    #  module Server
+    #    include EM::P::LineProtocol
+    #
+    #    def receive_line(line)
+    #      send_data("you said: #{line}")
+    #    end
+    #  end
+    #
+    module LineProtocol
+      def receive_data data # :nodoc:
+        (@buf ||= '') << data
+
+        while line = @buf.slice!(/(.*)\r?\n/)
+          receive_line(line)
+        end
+      end
+
+      # Invoked with lines received over the network
+      def receive_line(line)
+        # stub
+      end
+    end
+  end
+end

data/lib/em/protocols/smtpserver.rb

@@ -32,6 +32,86 @@ module EventMachine
     # and data to user code. User code is responsible for doing the right things with the
     # data in order to get complete and correct SMTP server behavior.
     #
+    # Simple SMTP server example:
+    #
+    #  class EmailServer < EM::P::SmtpServer
+    #    def receive_plain_auth(user, pass)
+    #      true
+    #    end
+    #
+    #    def get_server_domain
+    #      "mock.smtp.server.local"
+    #    end
+    #
+    #    def get_server_greeting
+    #      "mock smtp server greets you with impunity"
+    #    end
+    #
+    #    def receive_sender(sender)
+    #      current.sender = sender
+    #      true
+    #    end
+    #
+    #    def receive_recipient(recipient)
+    #      current.recipient = recipient
+    #      true
+    #    end
+    #
+    #    def receive_message
+    #      current.received = true
+    #      current.completed_at = Time.now
+    #
+    #      p [:received_email, current]
+    #      @current = OpenStruct.new
+    #      true
+    #    end
+    #
+    #    def receive_ehlo_domain(domain)
+    #      @ehlo_domain = domain
+    #      true
+    #    end
+    #
+    #    def receive_data_command
+    #      current.data = ""
+    #      true
+    #    end
+    #
+    #    def receive_data_chunk(data)
+    #      current.data << data.join("\n")
+    #      true
+    #    end
+    #
+    #    def receive_transaction
+    #      if @ehlo_domain
+    #        current.ehlo_domain = @ehlo_domain
+    #        @ehlo_domain = nil
+    #      end
+    #      true
+    #    end
+    #
+    #    def current
+    #      @current ||= OpenStruct.new
+    #    end
+    #
+    #    def self.start(host = 'localhost', port = 1025)
+    #      require 'ostruct'
+    #      @server = EM.start_server host, port, self
+    #    end
+    #
+    #    def self.stop
+    #      if @server
+    #        EM.stop_server @server
+    #        @server = nil
+    #      end
+    #    end
+    #
+    #    def self.running?
+    #      !!@server
+    #    end
+    #  end
+    #
+    #  EM.run{ EmailServer.start }
+    #
     #--
     # Useful paragraphs in RFC-2821:
     # 4.3.2: Concise list of command-reply sequences, in essence a text representation
@@ -114,6 +194,7 @@ module EventMachine
         @@parms[:verbose] and $>.puts ">>> #{ln}"
 
         return process_data_line(ln) if @state.include?(:data)
+        return process_auth_line(ln) if @state.include?(:auth_incomplete)
 
         case ln
         when EhloRegex
@@ -216,7 +297,7 @@ module EventMachine
             send_data "250-STARTTLS\r\n"
           end
           if @@parms[:auth]
-            send_data "250-AUTH PLAIN LOGIN\r\n"
+            send_data "250-AUTH PLAIN\r\n"
           end
           send_data "250-NO-SOLICITING\r\n"
           # TODO, size needs to be configurable.
@@ -259,14 +340,14 @@ module EventMachine
       def process_auth str
         if @state.include?(:auth)
           send_data "503 auth already issued\r\n"
-        elsif str =~ /\APLAIN\s+/i
-          plain = ($'.dup).unpack("m").first # Base64::decode64($'.dup)
-          discard,user,psw = plain.split("\000")
-          if receive_plain_auth user,psw
-            send_data "235 authentication ok\r\n"
-            @state << :auth
+        elsif str =~ /\APLAIN\s?/i
+          if $'.length == 0
+            # we got a partial response, so let the client know to send the rest
+            @state << :auth_incomplete
+            send_data("334 \r\n")
           else
-            send_data "535 invalid authentication\r\n"
+            # we got the initial response, so go ahead & process it
+            process_auth_line($')
           end
           #elsif str =~ /\ALOGIN\s+/i
         else
@@ -274,6 +355,18 @@ module EventMachine
         end
       end
 
+      def process_auth_line(line)
+        plain = line.unpack("m").first
+        discard,user,psw = plain.split("\000")
+        if receive_plain_auth user,psw
+          send_data "235 authentication ok\r\n"
+          @state << :auth
+        else
+          send_data "535 invalid authentication\r\n"
+        end
+        @state.delete :auth_incomplete
+      end
+
       #--
       # Unusually, we can deal with a Deferrable returned from the user application.
       # This was added to deal with a special case in a particular application, but