eventmachine 1.0.0.beta.3 → 1.0.0.beta.4

data/lib/em/queue.rb

@@ -1,29 +1,31 @@
 module EventMachine
   # A cross thread, reactor scheduled, linear queue.
   #
-  # This class provides a simple "Queue" like abstraction on top of the reactor
+  # This class provides a simple queue abstraction on top of the reactor
   # scheduler. It services two primary purposes:
+  #
   # * API sugar for stateful protocols
-  # * Pushing processing onto the same thread as the reactor
+  # * Pushing processing onto the reactor thread
   #
-  # See examples/ex_queue.rb for a detailed example.
+  # @example
   #
   #  q = EM::Queue.new
   #  q.push('one', 'two', 'three')
   #  3.times do
-  #    q.pop{ |msg| puts(msg) }
+  #    q.pop { |msg| puts(msg) }
   #  end
   #
   class Queue
-    # Create a new queue
     def initialize
       @items = []
       @popq  = []
     end
 
     # Pop items off the queue, running the block on the reactor thread. The pop
-    # will not happen immediately, but at some point in the future, either in 
+    # will not happen immediately, but at some point in the future, either in
     # the next tick, if the queue has data, or when the queue is populated.
+    #
+    # @return [NilClass] nil
     def pop(*a, &b)
       cb = EM::Callback(*a, &b)
       EM.schedule do
@@ -37,7 +39,7 @@ module EventMachine
     end
 
     # Push items onto the queue in the reactor thread. The items will not appear
-    # in the queue immediately, but will be scheduled for addition during the 
+    # in the queue immediately, but will be scheduled for addition during the
     # next reactor tick.
     def push(*items)
       EM.schedule do
@@ -47,16 +49,23 @@ module EventMachine
     end
     alias :<< :push
 
-    # N.B. This is a peek, it's not thread safe, and may only tend toward 
-    # accuracy.
+    # @return [Boolean]
+    # @note This is a peek, it's not thread safe, and may only tend toward accuracy.
     def empty?
       @items.empty?
     end
 
-    # N.B. This is a peek, it's not thread safe, and may only tend toward 
-    # accuracy.
+    # @return [Integer] Queue size
+    # @note This is a peek, it's not thread safe, and may only tend toward accuracy.
     def size
       @items.size
     end
-  end
-end
+
+    # @return [Integer] Waiting size
+    # @note This is a peek at the number of jobs that are currently waiting on the Queue
+    def num_waiting
+      @popq.size
+    end
+
+  end # Queue
+end # EventMachine

data/lib/em/resolver.rb

@@ -6,6 +6,8 @@ module EventMachine
         Request.new(socket, hostname)
       end
 
+      @socket = @nameservers = nil
+
       def self.socket
         if !@socket || (@socket && @socket.error?)
           @socket = Socket.open
@@ -58,6 +60,10 @@ module EventMachine
         EventMachine::open_datagram_socket('0.0.0.0', 0, self)
       end
 
+      def initialize
+        @nameserver = nil
+      end
+
       def post_init
         @requests = {}
         EM.add_periodic_timer(0.1, &method(:tick))
@@ -99,88 +105,88 @@ module EventMachine
         begin
           msg = Resolv::DNS::Message.decode data
         rescue
-          else
-            req = @requests[msg.id]
-            if req
-              @requests.delete(msg.id)
-              req.receive_answer(msg)
-            end
+        else
+          req = @requests[msg.id]
+          if req
+            @requests.delete(msg.id)
+            req.receive_answer(msg)
           end
         end
       end
+    end
 
-      class Request
-        include Deferrable
-        attr_accessor :retry_interval, :max_tries
+    class Request
+      include Deferrable
+      attr_accessor :retry_interval, :max_tries
 
-        def initialize(socket, hostname)
-          @socket = socket
-          @hostname = hostname
-          @tries = 0
-          @last_send = Time.at(0)
-          @retry_interval = 3
-          @max_tries = 5
+      def initialize(socket, hostname)
+        @socket = socket
+        @hostname = hostname
+        @tries = 0
+        @last_send = Time.at(0)
+        @retry_interval = 3
+        @max_tries = 5
 
-          if addrs = Resolver.hosts[hostname]
-            succeed addrs
-          else
-            EM.next_tick { tick }
-          end
+        if addrs = Resolver.hosts[hostname]
+          succeed addrs
+        else
+          EM.next_tick { tick }
         end
+      end
 
-        def tick
-          # Break early if nothing to do
-          return if @last_send + @retry_interval > Time.now
-          if @tries < @max_tries
-            send
-          else
-            fail 'retries exceeded'
-          end
+      def tick
+        # Break early if nothing to do
+        return if @last_send + @retry_interval > Time.now
+        if @tries < @max_tries
+          send
+        else
+          fail 'retries exceeded'
         end
+      end
 
-        def receive_answer(msg)
-          addrs = []
-          msg.each_answer do |name,ttl,data|
-            if data.kind_of?(Resolv::DNS::Resource::IN::A) ||
-                data.kind_of?(Resolv::DNS::Resource::IN::AAAA)
-              addrs << data.address.to_s
-            end
+      def receive_answer(msg)
+        addrs = []
+        msg.each_answer do |name,ttl,data|
+          if data.kind_of?(Resolv::DNS::Resource::IN::A) ||
+              data.kind_of?(Resolv::DNS::Resource::IN::AAAA)
+            addrs << data.address.to_s
           end
+        end
 
-          if addrs.empty?
-            fail "rcode=#{msg.rcode}"
-          else
-            succeed addrs
-          end
+        if addrs.empty?
+          fail "rcode=#{msg.rcode}"
+        else
+          succeed addrs
         end
+      end
 
-        private
+      private
 
-          def send
-            @tries += 1
-            @last_send = Time.now
-            @socket.send_packet(packet.encode)
-          end
+        def send
+          @tries += 1
+          @last_send = Time.now
+          @socket.send_packet(packet.encode)
+        end
 
-          def id
-            begin
-              @id = rand(65535)
-              @socket.register_request(@id, self)
-            rescue RequestIdAlreadyUsed
-              retry
-            end unless defined?(@id)
+        def id
+          begin
+            @id = rand(65535)
+            @socket.register_request(@id, self)
+          rescue RequestIdAlreadyUsed
+            retry
+          end unless defined?(@id)
 
-            @id
-          end
+          @id
+        end
 
-          def packet
-            msg = Resolv::DNS::Message.new
-            msg.id = id
-            msg.rd = 1
-            msg.add_question @hostname, Resolv::DNS::Resource::IN::A
-            msg
-          end
+        def packet
+          msg = Resolv::DNS::Message.new
+          msg.id = id
+          msg.rd = 1
+          msg.add_question @hostname, Resolv::DNS::Resource::IN::A
+          msg
+        end
 
-      end
     end
   end
+end

data/lib/em/spawnable.rb

@@ -55,7 +55,8 @@ module EventMachine
 
   end
 
-  class YieldBlockFromSpawnedProcess # :nodoc:
+  # @private
+  class YieldBlockFromSpawnedProcess
     def initialize block, notify
       @block = [block,notify]
     end
@@ -71,11 +72,13 @@ module EventMachine
     s
   end
 
-  def self.yield &block # :nodoc:
+  # @private
+  def self.yield &block
     return YieldBlockFromSpawnedProcess.new( block, false )
   end
 
-  def self.yield_and_notify &block # :nodoc:
+  # @private
+  def self.yield_and_notify &block
     return YieldBlockFromSpawnedProcess.new( block, true )
   end
 end

data/lib/em/streamer.rb

@@ -1,30 +1,24 @@
-#--
-#
-# Author:: Francis Cianfrocca (gmail: blackhedd)
-# Homepage::  http://rubyeventmachine.com
-# Date:: 16 Jul 2006
-# 
-# See EventMachine and EventMachine::Connection for documentation and
-# usage examples.
-#
-#----------------------------------------------------------------------------
-#
-# Copyright (C) 2006-07 by Francis Cianfrocca. All Rights Reserved.
-# Gmail: blackhedd
-# 
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of either: 1) the GNU General Public License
-# as published by the Free Software Foundation; either version 2 of the
-# License, or (at your option) any later version; or 2) Ruby's License.
-# 
-# See the file COPYING for complete licensing information.
-#
-#---------------------------------------------------------------------------
-#
-# 
-
-
 module EventMachine
+  # Streams a file over a given connection. Streaming begins once the object is
+  # instantiated. Typically FileStreamer instances are not reused.
+  #
+  # Streaming uses buffering for files larger than 16K and uses so-called fast file reader (a C++ extension)
+  # if available (it is part of eventmachine gem itself).
+  #
+  # @example
+  #
+  #  module FileSender
+  #    def post_init
+  #      streamer = EventMachine::FileStreamer.new(self, '/tmp/bigfile.tar')
+  #      streamer.callback{
+  #        # file was sent successfully
+  #        close_connection_after_writing
+  #      }
+  #    end
+  #  end
+  #
+  #
+  # @author Francis Cianfrocca
   class FileStreamer
     include Deferrable
 
@@ -35,19 +29,10 @@ module EventMachine
     # Send 16k chunks at a time
     ChunkSize = 16384
 
-    # Stream a file over a given connection. An optional :http_chunks => true argument will
-    # use HTTP 1.1 style chunked-encoding semantics.
-    #
-    #  module FileSender
-    #    def post_init
-    #      streamer = EventMachine::FileStreamer.new(self, '/tmp/bigfile.tar')
-    #      streamer.callback{
-    #        # file was sent successfully
-    #        close_connection_after_writing
-    #      }
-    #    end
-    #  end
+    # @param [EventMachine::Connection] connection
+    # @param [String] filename File path
     #
+    # @option args [Boolean] :http_chunks (false) Use HTTP 1.1 style chunked-encoding semantics.
     def initialize connection, filename, args = {}
       @connection = connection
       @http_chunks = args[:http_chunks]
@@ -64,7 +49,8 @@ module EventMachine
       end
     end
 
-    def stream_without_mapping filename # :nodoc:
+    # @private
+    def stream_without_mapping filename
       if @http_chunks
         @connection.send_data "#{@size.to_s(16)}\r\n"
         @connection.send_file_data filename
@@ -76,7 +62,8 @@ module EventMachine
     end
     private :stream_without_mapping
 
-    def stream_with_mapping filename # :nodoc:
+    # @private
+    def stream_with_mapping filename
       ensure_mapping_extension_is_present
 
       @position = 0
@@ -86,6 +73,7 @@ module EventMachine
     private :stream_with_mapping
 
     # Used internally to stream one chunk at a time over multiple reactor ticks
+    # @private
     def stream_one_chunk
       loop {
         if @position < @size
@@ -111,7 +99,7 @@ module EventMachine
       }
     end
 
-    #--
+    #
     # We use an outboard extension class to get memory-mapped files.
     # It's outboard to avoid polluting the core distro, but that means
     # there's a "hidden" dependency on it. The first time we get here in
@@ -120,11 +108,11 @@ module EventMachine
     # mapped files will work fine without it. This is a somewhat difficult
     # compromise between usability and proper modularization.
     #
-    def ensure_mapping_extension_is_present # :nodoc:
+    # @private
+    def ensure_mapping_extension_is_present
       @@fastfilereader ||= (require 'fastfilereaderext')
     end
     private :ensure_mapping_extension_is_present
 
-  end
-end
-
+  end # FileStreamer
+end # EventMachine

data/lib/em/threaded_resource.rb

@@ -0,0 +1,90 @@
+module EventMachine
+  # = EventMachine::ThreadedResource
+  #
+  # A threaded resource is a "quick and dirty" wrapper around the concept of
+  # wiring up synchronous code into a standard EM::Pool. This is useful to keep
+  # interfaces coherent and provide a simple approach at "making an interface
+  # async-ish".
+  #
+  # General usage is to wrap libraries that do not support EventMachine, or to
+  # have a specific number of dedicated high-cpu worker resources.
+  #
+  # == Basic Usage example
+  #
+  # This example requires the cassandra gem. The cassandra gem contains an
+  # EventMachine interface, but it's sadly Fiber based and thus only works on
+  # 1.9. It also requires (potentially) complex stack switching logic to reach
+  # completion of nested operations. By contrast this approach provides a block
+  # in which normal synchronous code can occur, but makes no attempt to wire the
+  # IO into EventMachines C++ IO implementations, instead relying on the reactor
+  # pattern in rb_thread_select.
+  #
+  #    cassandra_dispatcher = ThreadedResource.new do
+  #      Cassandra.new('allthethings', '127.0.0.1:9160')
+  #    end
+  #
+  #    pool = EM::Pool.new
+  #
+  #    pool.add cassandra_dispatcher
+  #
+  #    # If we don't care about the result:
+  #    pool.perform do |dispatcher|
+  #      # The following blcok executes inside a dedicated thread, and should not
+  #      # access EventMachine things:
+  #      dispatcher.dispatch do |cassandra|
+  #        cassandra.insert(:Things, '10', 'stuff' => 'things')
+  #      end
+  #    end
+  #
+  #    # Example where we care about the result:
+  #    pool.perform do |dispatcher|
+  #      # The dispatch block is executed in the resources thread.
+  #      completion = dispatcher.dispatch do |cassandra|
+  #        cassandra.get(:Things, '10', 'stuff')
+  #      end
+  #
+  #      # This block will be yielded on the EM thread:
+  #      completion.callback do |result|
+  #        EM.do_something_with(result)
+  #      end
+  #
+  #      completion
+  #    end
+  class ThreadedResource
+
+    # The block should return the resource that will be yielded in a dispatch.
+    def initialize
+      @resource = yield
+
+      @running = true
+      @queue   = ::Queue.new
+      @thread  = Thread.new do
+        @queue.pop.call while @running
+      end
+    end
+
+    # Called on the EM thread, generally in a perform block to return a
+    # completion for the work.
+    def dispatch
+      completion = EM::Completion.new
+      @queue << lambda do
+        begin
+          result = yield @resource
+          completion.succeed result
+        rescue Exception => e
+          completion.fail e
+        end
+      end
+      completion
+    end
+
+    # Kill the internal thread. should only be used to cleanup - generally
+    # only required for tests.
+    def shutdown
+      @running = false
+      @queue << lambda {}
+      @thread.join
+    end
+
+  end
+end