eventmachine-maglev- 0.12.10 → 1.0.0.beta.4

data/lib/em/deferrable/pool.rb

@@ -0,0 +1,2 @@
+warn "EM::Deferrable::Pool is deprecated, please use EM::Pool"
+EM::Deferrable::Pool = EM::Pool

data/lib/em/deferrable.rb

@@ -25,6 +25,7 @@
 
 module EventMachine
   module Deferrable
+    autoload :Pool, 'em/deferrable/pool'
 
     # Specify a block to be executed if and when the Deferrable object receives
     # a status of :succeeded. See #set_deferred_status for more information.
@@ -49,6 +50,14 @@ module EventMachine
         @callbacks ||= []
         @callbacks.unshift block # << block
       end
+      self
+    end
+
+    # Cancels an outstanding callback to &block if any. Undoes the action of #callback.
+    #
+    def cancel_callback block
+      @callbacks ||= []
+      @callbacks.delete block
     end
 
     # Specify a block to be executed if and when the Deferrable object receives
@@ -67,6 +76,14 @@ module EventMachine
         @errbacks ||= []
         @errbacks.unshift block # << block
       end
+      self
+    end
+
+    # Cancels an outstanding errback to &block if any. Undoes the action of #errback.
+    #
+    def cancel_errback block
+      @errbacks ||= []
+      @errbacks.delete block
     end
 
     # Sets the "disposition" (status) of the Deferrable object. See also the large set of
@@ -150,10 +167,11 @@ module EventMachine
     # the Timeout expires (passing no arguments to the object's errbacks).
     # Setting the status at any time prior to a call to the expiration of the timeout
     # will cause the timer to be cancelled.
-    def timeout seconds
+    def timeout seconds, *args
       cancel_timeout
       me = self
-      @deferred_timeout = EventMachine::Timer.new(seconds) {me.fail}
+      @deferred_timeout = EventMachine::Timer.new(seconds) {me.fail(*args)}
+      self
     end
 
     # Cancels an outstanding timeout if any. Undoes the action of #timeout.

data/lib/em/file_watch.rb

@@ -1,15 +1,24 @@
 module EventMachine
-
-  # This is subclassed from EventMachine::Connection for use with the file monitoring API. Read the
-  # documentation on the instance methods of this class, and for a full explanation see EventMachine.watch_file.
+  # Utility class that is useful for file monitoring. Supported events are
+  #
+  # * File is modified
+  # * File is deleted
+  # * File is moved
+  #
+  # @note On Mac OS X, file watching only works when kqueue is enabled
+  #
+  # @see EventMachine.watch_file
   class FileWatch < Connection
-    # :stopdoc:
+    # @private
     Cmodified = 'modified'.freeze
+    # @private
     Cdeleted = 'deleted'.freeze
+    # @private
     Cmoved = 'moved'.freeze
-    # :startdoc:
 
-    def receive_data(data) #:nodoc:
+
+    # @private
+    def receive_data(data)
       case data
       when Cmodified
         file_modified
@@ -20,35 +29,45 @@ module EventMachine
       end
     end
 
-    # Returns the path that EventMachine::watch_file was originally called with. The current implementation
-    # does not pick up on the new filename after a rename occurs.
+    # Returns the path that is being monitored.
+    #
+    # @note Current implementation does not pick up on the new filename after a rename occurs.
+    #
+    # @return [String]
+    # @see EventMachine.watch_file
     def path
       @path
     end
 
-    # Should be redefined with the user's custom callback that will be fired when the file is modified.
+    # Will be called when the file is modified. Supposed to be redefined by subclasses.
+    #
+    # @abstract
     def file_modified
     end
 
-    # Should be redefined with the user's custom callback that will be fired when the file is deleted.
+    # Will be called when the file is deleted. Supposed to be redefined by subclasses.
     # When the file is deleted, stop_watching will be called after this to make sure everything is
     # cleaned up correctly.
     #
-    # Note that on linux (with inotify), file_deleted will not be called until all open file descriptors to
-    # the file have been closed.
+    # @note On Linux (with {http://en.wikipedia.org/wiki/Inotify inotify}), this method will not be called until *all* open file descriptors to
+    #       the file have been closed.
+    #
+    # @abstract
     def file_deleted
     end
 
-    # Should be redefined with the user's custom callback that will be fired when the file is moved or renamed.
+    # Will be called when the file is moved or renamed. Supposed to be redefined by subclasses.
+    #
+    # @abstract
     def file_moved
     end
 
     # Discontinue monitoring of the file.
-    # This involves cleaning up the underlying monitoring details with kqueue/inotify, and in turn firing unbind.
+    #
+    # This involves cleaning up the underlying monitoring details with kqueue/inotify, and in turn firing {EventMachine::Connection#unbind}.
     # This will be called automatically when a file is deleted. User code may call it as well.
     def stop_watching
       EventMachine::unwatch_filename(@signature)
-    end
-  end
-
-end
+    end # stop_watching
+  end # FileWatch
+end # EventMachine

data/lib/em/iterator.rb

@@ -0,0 +1,270 @@
+module EventMachine
+  # A simple iterator for concurrent asynchronous work.
+  #
+  # Unlike ruby's built-in iterators, the end of the current iteration cycle is signaled manually,
+  # instead of happening automatically after the yielded block finishes executing. For example:
+  #
+  #  (0..10).each{ |num| }
+  #
+  # becomes:
+  #
+  #  EM::Iterator.new(0..10).each{ |num,iter| iter.next }
+  #
+  # This is especially useful when doing asynchronous work via reactor libraries and
+  # functions. For example, given a sync and async http api:
+  #
+  #  response = sync_http_get(url); ...
+  #  async_http_get(url){ |response| ... }
+  #
+  # a synchronous iterator such as:
+  #
+  #  responses = urls.map{ |url| sync_http_get(url) }
+  #  ...
+  #  puts 'all done!'
+  #
+  # could be written as:
+  #
+  #  EM::Iterator.new(urls).map(proc{ |url,iter|
+  #    async_http_get(url){ |res|
+  #      iter.return(res)
+  #    }
+  #  }, proc{ |responses|
+  #    ...
+  #    puts 'all done!'
+  #  })
+  #
+  # Now, you can take advantage of the asynchronous api to issue requests in parallel. For example,
+  # to fetch 10 urls at a time, simply pass in a concurrency of 10:
+  #
+  #  EM::Iterator.new(urls, 10).each do |url,iter|
+  #    async_http_get(url){ iter.next }
+  #  end
+  #
+  class Iterator
+    # Create a new parallel async iterator with specified concurrency.
+    #
+    #  i = EM::Iterator.new(1..100, 10)
+    #
+    # will create an iterator over the range that processes 10 items at a time. Iteration
+    # is started via #each, #map or #inject
+    #
+    def initialize(list, concurrency = 1)
+      raise ArgumentError, 'argument must be an array' unless list.respond_to?(:to_a)
+      @list = list.to_a.dup
+      @concurrency = concurrency
+
+      @started = false
+      @ended = false
+    end
+
+    # Change the concurrency of this iterator. Workers will automatically be spawned or destroyed
+    # to accomodate the new concurrency level.
+    #
+    def concurrency=(val)
+      old = @concurrency
+      @concurrency = val
+
+      spawn_workers if val > old and @started and !@ended
+    end
+    attr_reader :concurrency
+
+    # Iterate over a set of items using the specified block or proc.
+    #
+    #  EM::Iterator.new(1..100).each do |num, iter|
+    #    puts num
+    #    iter.next
+    #  end
+    #
+    # An optional second proc is invoked after the iteration is complete.
+    #
+    #  EM::Iterator.new(1..100).each(
+    #    proc{ |num,iter| iter.next },
+    #    proc{ puts 'all done' }
+    #  )
+    #
+    def each(foreach=nil, after=nil, &blk)
+      raise ArgumentError, 'proc or block required for iteration' unless foreach ||= blk
+      raise RuntimeError, 'cannot iterate over an iterator more than once' if @started or @ended
+
+      @started = true
+      @pending = 0
+      @workers = 0
+
+      all_done = proc{
+        after.call if after and @ended and @pending == 0
+      }
+
+      @process_next = proc{
+        # p [:process_next, :pending=, @pending, :workers=, @workers, :ended=, @ended, :concurrency=, @concurrency, :list=, @list]
+        unless @ended or @workers > @concurrency
+          if @list.empty?
+            @ended = true
+            @workers -= 1
+            all_done.call
+          else
+            item = @list.shift
+            @pending += 1
+
+            is_done = false
+            on_done = proc{
+              raise RuntimeError, 'already completed this iteration' if is_done
+              is_done = true
+
+              @pending -= 1
+
+              if @ended
+                all_done.call
+              else
+                EM.next_tick(@process_next)
+              end
+            }
+            class << on_done
+              alias :next :call
+            end
+
+            foreach.call(item, on_done)
+          end
+        else
+          @workers -= 1
+        end
+      }
+
+      spawn_workers
+
+      self
+    end
+
+    # Collect the results of an asynchronous iteration into an array.
+    #
+    #  EM::Iterator.new(%w[ pwd uptime uname date ], 2).map(proc{ |cmd,iter|
+    #    EM.system(cmd){ |output,status|
+    #      iter.return(output)
+    #    }
+    #  }, proc{ |results|
+    #    p results
+    #  })
+    #
+    def map(foreach, after)
+      index = 0
+
+      inject([], proc{ |results,item,iter|
+        i = index
+        index += 1
+
+        is_done = false
+        on_done = proc{ |res|
+          raise RuntimeError, 'already returned a value for this iteration' if is_done
+          is_done = true
+
+          results[i] = res
+          iter.return(results)
+        }
+        class << on_done
+          alias :return :call
+          def next
+            raise NoMethodError, 'must call #return on a map iterator'
+          end
+        end
+
+        foreach.call(item, on_done)
+      }, proc{ |results|
+        after.call(results)
+      })
+    end
+
+    # Inject the results of an asynchronous iteration onto a given object.
+    #
+    #  EM::Iterator.new(%w[ pwd uptime uname date ], 2).inject({}, proc{ |hash,cmd,iter|
+    #    EM.system(cmd){ |output,status|
+    #      hash[cmd] = status.exitstatus == 0 ? output.strip : nil
+    #      iter.return(hash)
+    #    }
+    #  }, proc{ |results|
+    #    p results
+    #  })
+    #
+    def inject(obj, foreach, after)
+      each(proc{ |item,iter|
+        is_done = false
+        on_done = proc{ |res|
+          raise RuntimeError, 'already returned a value for this iteration' if is_done
+          is_done = true
+
+          obj = res
+          iter.next
+        }
+        class << on_done
+          alias :return :call
+          def next
+            raise NoMethodError, 'must call #return on an inject iterator'
+          end
+        end
+
+        foreach.call(obj, item, on_done)
+      }, proc{
+        after.call(obj)
+      })
+    end
+
+    private
+
+    # Spawn workers to consume items from the iterator's enumerator based on the current concurrency level.
+    #
+    def spawn_workers
+      EM.next_tick(start_worker = proc{
+        if @workers < @concurrency and !@ended
+          # p [:spawning_worker, :workers=, @workers, :concurrency=, @concurrency, :ended=, @ended]
+          @workers += 1
+          @process_next.call
+          EM.next_tick(start_worker)
+        end
+      })
+      nil
+    end
+  end
+end
+
+if __FILE__ == $0
+  $:.unshift File.join(File.dirname(__FILE__), '..')
+  require 'eventmachine'
+
+  # TODO: real tests
+  # TODO: pass in one object instead of two? .each{ |iter| puts iter.current; iter.next }
+  # TODO: support iter.pause/resume/stop/break/continue?
+  # TODO: create some exceptions instead of using RuntimeError
+  # TODO: support proc instead of enumerable? EM::Iterator.new(proc{ return queue.pop })
+
+  EM.run{
+    EM::Iterator.new(1..50).each{ |num,iter| p num; iter.next }
+    EM::Iterator.new([1,2,3], 10).each{ |num,iter| p num; iter.next }
+
+    i = EM::Iterator.new(1..100, 5)
+    i.each(proc{|num,iter|
+      p num.to_s
+      iter.next
+    }, proc{
+      p :done
+    })
+    EM.add_timer(0.03){
+      i.concurrency = 1
+    }
+    EM.add_timer(0.04){
+      i.concurrency = 3
+    }
+
+    EM::Iterator.new(100..150).map(proc{ |num,iter|
+      EM.add_timer(0.01){ iter.return(num) }
+    }, proc{ |results|
+      p results
+    })
+
+    EM::Iterator.new(%w[ pwd uptime uname date ], 2).inject({}, proc{ |hash,cmd,iter|
+      EM.system(cmd){ |output,status|
+        hash[cmd] = status.exitstatus == 0 ? output.strip : nil
+        iter.return(hash)
+      }
+    }, proc{ |results|
+      p results
+    })
+  }
+end

data/lib/em/pool.rb

@@ -0,0 +1,146 @@
+module EventMachine
+  # = EventMachine::Pool
+  #
+  # A simple async resource pool based on a resource and work queue. Resources
+  # are enqueued and work waits for resources to become available.
+  #
+  # Example:
+  #
+  #    EM.run do
+  #      pool  = EM::Pool.new
+  #      spawn = lambda { pool.add EM::HttpRequest.new('http://example.org') }
+  #      10.times { spawn[] }
+  #      done, scheduled = 0, 0
+  #    
+  #      check = lambda do
+  #        done += 1
+  #        if done >= scheduled
+  #          EM.stop
+  #        end
+  #      end
+  #    
+  #      pool.on_error { |conn| spawn[] }
+  #    
+  #      100.times do
+  #        pool.perform do |conn|
+  #          req = conn.get :path => '/', :keepalive => true
+  #    
+  #          req.callback do
+  #            p [:success, conn.object_id, i, req.response.size]
+  #            check[]
+  #          end
+  #    
+  #          req.errback { check[] }
+  #    
+  #          req
+  #        end
+  #      end
+  #    end
+  #
+  # Resources are expected to be controlled by an object responding to a
+  # deferrable/completion style API with callback and errback blocks.
+  #
+  class Pool
+
+    def initialize
+      @resources = EM::Queue.new
+      @removed = []
+      @contents = []
+      @on_error = nil
+    end
+
+    def add resource
+      @contents << resource
+      requeue resource
+    end
+
+    def remove resource
+      @contents.delete resource
+      @removed << resource
+    end
+
+    # Returns a list for introspection purposes only. You should *NEVER* call
+    # modification or work oriented methods on objects in this list. A good
+    # example use case is periodic statistics collection against a set of
+    # connection resources.
+    #
+    # For example: 
+    #     pool.contents.inject(0) { |sum, connection| connection.num_bytes }
+    def contents
+      @contents.dup
+    end
+
+    # Define a default catch-all for when the deferrables returned by work
+    # blocks enter a failed state. By default all that happens is that the
+    # resource is returned to the pool. If on_error is defined, this block is
+    # responsible for re-adding the resource to the pool if it is still usable.
+    # In other words, it is generally assumed that on_error blocks explicitly
+    # handle the rest of the lifetime of the resource.
+    def on_error *a, &b
+      @on_error = EM::Callback(*a, &b)
+    end
+
+    # Perform a given #call-able object or block. The callable object will be
+    # called with a resource from the pool as soon as one is available, and is
+    # expected to return a deferrable.
+    # 
+    # The deferrable will have callback and errback added such that when the
+    # deferrable enters a finished state, the object is returned to the pool.
+    #
+    # If on_error is defined, then objects are not automatically returned to the
+    # pool.
+    def perform(*a, &b)
+      work = EM::Callback(*a, &b)
+
+      @resources.pop do |resource|
+        if removed? resource
+          @removed.delete resource
+          reschedule work
+        else
+          process work, resource
+        end
+      end
+    end
+    alias reschedule perform
+
+    # A peek at the number of enqueued jobs waiting for resources
+    def num_waiting
+      @resources.num_waiting
+    end
+
+    # Removed will show resources in a partial pruned state. Resources in the
+    # removed list may not appear in the contents list if they are currently in
+    # use.
+    def removed? resource
+      @removed.include? resource
+    end
+
+    protected
+    def requeue resource
+      @resources.push resource
+    end
+
+    def failure resource
+      if @on_error
+        @on_error.call resource
+      else
+        requeue resource
+      end
+    end
+
+    def completion deferrable, resource
+      deferrable.callback { requeue resource }
+      deferrable.errback  { failure resource }
+    end
+
+    def process work, resource
+      deferrable = work.call resource
+      if deferrable.kind_of?(EM::Deferrable)
+        completion deferrable, resource
+      else
+        raise ArgumentError, "deferrable expected from work"
+      end
+    end
+
+  end
+end

data/lib/em/process_watch.rb

@@ -3,12 +3,13 @@ module EventMachine
   # This is subclassed from EventMachine::Connection for use with the process monitoring API. Read the
   # documentation on the instance methods of this class, and for a full explanation see EventMachine.watch_process.
   class ProcessWatch < Connection
-    # :stopdoc:
+    # @private
     Cfork = 'fork'.freeze
+    # @private
     Cexit = 'exit'.freeze
-    # :startdoc:
 
-    def receive_data(data) # :nodoc:
+    # @private
+    def receive_data(data)
       case data
       when Cfork
         process_forked
@@ -41,4 +42,4 @@ module EventMachine
     end
   end
 
-end
+end

data/lib/em/processes.rb

@@ -39,7 +39,8 @@ module EventMachine
   class DeferrableChildProcess < EventMachine::Connection
     include EventMachine::Deferrable
 
-    def initialize # :nodoc:
+    # @private
+    def initialize
       super
       @data = []
     end
@@ -60,16 +61,19 @@ module EventMachine
       EventMachine.popen( cmd, DeferrableChildProcess )
     end
 
-    def receive_data data # :nodoc:
+    # @private
+    def receive_data data
       @data << data
     end
 
-    def unbind # :nodoc:
+    # @private
+    def unbind
       succeed( @data.join )
     end
   end
 
-  class SystemCmd < EventMachine::Connection # :nodoc:
+  # @private
+  class SystemCmd < EventMachine::Connection
     def initialize cb
       @cb = cb
       @output = []

data/lib/em/protocols/httpclient.rb

@@ -3,7 +3,7 @@
 # Author:: Francis Cianfrocca (gmail: blackhedd)
 # Homepage::  http://rubyeventmachine.com
 # Date:: 16 July 2006
-# 
+#
 # See EventMachine and EventMachine::Connection for documentation and
 # usage examples.
 #
@@ -11,25 +11,26 @@
 #
 # Copyright (C) 2006-07 by Francis Cianfrocca. All Rights Reserved.
 # Gmail: blackhedd
-# 
+#
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of either: 1) the GNU General Public License
 # as published by the Free Software Foundation; either version 2 of the
 # License, or (at your option) any later version; or 2) Ruby's License.
-# 
+#
 # See the file COPYING for complete licensing information.
 #
 #---------------------------------------------------------------------------
 #
-# 
+#
 
 
 
 module EventMachine
   module Protocols
 
-    # === Usage
+    # <b>Note:</b> This class is deprecated and will be removed. Please use EM-HTTP-Request instead.
     #
+    # @example
     #  EventMachine.run {
     #    http = EventMachine::Protocols::HttpClient.request(
     #      :host => server,
@@ -62,11 +63,21 @@ module EventMachine
 
       MaxPostContentLength = 20 * 1024 * 1024
 
-      # === Arg list
-      # :host => 'ip/dns', :port => fixnum, :verb => 'GET', :request => 'path',
-      # :basic_auth => {:username => '', :password => ''}, :content => 'content',
-      # :contenttype => 'text/plain', :query_string => '', :host_header => '',
-      # :cookie => ''
+      def initialize
+        warn "HttpClient is deprecated and will be removed. EM-Http-Request should be used instead."
+      end
+
+      # @param args [Hash] The request arguments
+      # @option args [String] :host The host IP/DNS name
+      # @option args [Integer] :port The port to connect too
+      # @option args [String] :verb The request type [GET | POST | DELETE | PUT]
+      # @option args [String] :request The request path
+      # @option args [Hash] :basic_auth The basic auth credentials (:username and :password)
+      # @option args [String] :content The request content
+      # @option args [String] :contenttype The content type (e.g. text/plain)
+      # @option args [String] :query_string The query string
+      # @option args [String] :host_header The host header to set
+      # @option args [String] :cookie Cookies to set
       def self.request( args = {} )
         args[:port] ||= 80
         EventMachine.connect( args[:host], args[:port], self ) {|c|
@@ -142,6 +153,11 @@ module EventMachine
             req << "Cookie: #{args[:cookie]}"
           end
 
+          # Allow custom HTTP headers, e.g. SOAPAction
+          args[:custom_headers].each do |k,v|
+            req << "#{k}: #{v}"
+          end if args[:custom_headers]
+
           # Basic-auth stanza contributed by Matt Murphy.
           if args[:basic_auth]
             basic_auth_string = ["#{args[:basic_auth][:username]}:#{args[:basic_auth][:password]}"].pack('m').strip.gsub(/\n/,'')
@@ -260,4 +276,4 @@ module EventMachine
     end
 
   end
-end
+end