eventmachine-mkroman 1.3.0.dev.1

data/lib/em/queue.rb

@@ -0,0 +1,80 @@
+module EventMachine
+  # A cross thread, reactor scheduled, linear queue.
+  #
+  # This class provides a simple queue abstraction on top of the reactor
+  # scheduler. It services two primary purposes:
+  #
+  # * API sugar for stateful protocols
+  # * Pushing processing onto the reactor thread
+  #
+  # @example
+  #
+  #  q = EM::Queue.new
+  #  q.push('one', 'two', 'three')
+  #  3.times do
+  #    q.pop { |msg| puts(msg) }
+  #  end
+  #
+  class Queue
+    def initialize
+      @sink  = []
+      @drain = []
+      @popq  = []
+    end
+
+    # Pop items off the queue, running the block on the reactor thread. The pop
+    # will not happen immediately, but at some point in the future, either in
+    # the next tick, if the queue has data, or when the queue is populated.
+    #
+    # @return [NilClass] nil
+    def pop(*a, &b)
+      cb = EM::Callback(*a, &b)
+      EM.schedule do
+        if @drain.empty?
+          @drain = @sink
+          @sink = []
+        end
+        if @drain.empty?
+          @popq << cb
+        else
+          cb.call @drain.shift
+        end
+      end
+      nil # Always returns nil
+    end
+
+    # Push items onto the queue in the reactor thread. The items will not appear
+    # in the queue immediately, but will be scheduled for addition during the
+    # next reactor tick.
+    def push(*items)
+      EM.schedule do
+        @sink.push(*items)
+        unless @popq.empty?
+          @drain = @sink
+          @sink = []
+          @popq.shift.call @drain.shift until @drain.empty? || @popq.empty?
+        end
+      end
+    end
+    alias :<< :push
+
+    # @return [Boolean]
+    # @note This is a peek, it's not thread safe, and may only tend toward accuracy.
+    def empty?
+      @drain.empty? && @sink.empty?
+    end
+
+    # @return [Integer] Queue size
+    # @note This is a peek, it's not thread safe, and may only tend toward accuracy.
+    def size
+      @drain.size + @sink.size
+    end
+
+    # @return [Integer] Waiting size
+    # @note This is a peek at the number of jobs that are currently waiting on the Queue
+    def num_waiting
+      @popq.size
+    end
+
+  end # Queue
+end # EventMachine

data/lib/em/resolver.rb

@@ -0,0 +1,232 @@
+module EventMachine
+  module DNS
+    class Resolver
+
+      def self.windows?
+        if RUBY_PLATFORM =~ /mswin32|cygwin|mingw|bccwin/
+          require 'win32/resolv'
+          true
+        else
+          false
+        end
+      end
+
+      HOSTS_FILE = windows? ? Win32::Resolv.get_hosts_path : '/etc/hosts'
+
+      @hosts = nil
+      @nameservers = nil
+      @socket = nil
+
+      def self.resolve(hostname)
+        Request.new(socket, hostname)
+      end
+
+      def self.socket
+        if @socket && @socket.error?
+          @socket = Socket.open
+        else
+          @socket ||= Socket.open
+        end
+      end
+
+      def self.nameservers=(ns)
+        @nameservers = ns
+      end
+
+      def self.nameservers
+        return @nameservers if @nameservers
+
+        if windows?
+          _, ns = Win32::Resolv.get_resolv_info
+          return @nameservers = ns || []
+        end
+
+        @nameservers = []
+        IO.readlines('/etc/resolv.conf').each do |line|
+          if line =~ /^nameserver (.+)$/
+            @nameservers << $1.split(/\s+/).first
+          end
+        end
+
+        @nameservers
+      rescue
+        @nameservers = []
+      end
+
+      def self.nameserver
+        nameservers.shuffle.first
+      end
+
+      def self.hosts
+        return @hosts if @hosts
+
+        @hosts = {}
+        IO.readlines(HOSTS_FILE).each do |line|
+          next if line =~ /^#/
+          addr, host = line.split(/\s+/)
+
+          next unless addr && host
+          @hosts[host] ||= []
+          @hosts[host] << addr
+        end
+
+        @hosts
+      rescue
+        @hosts = {}
+      end
+    end
+
+    class RequestIdAlreadyUsed < RuntimeError; end
+
+    class Socket < EventMachine::Connection
+      def self.open
+        EventMachine::open_datagram_socket('0.0.0.0', 0, self)
+      end
+
+      def initialize
+        @nameserver = nil
+      end
+
+      def post_init
+        @requests = {}
+      end
+
+      def start_timer
+        @timer ||= EM.add_periodic_timer(0.1, &method(:tick))
+      end
+
+      def stop_timer
+        EM.cancel_timer(@timer)
+        @timer = nil
+      end
+
+      def unbind
+      end
+
+      def tick
+        @requests.each do |id,req|
+          req.tick
+        end
+      end
+
+      def register_request(id, req)
+        if @requests.has_key?(id)
+          raise RequestIdAlreadyUsed
+        else
+          @requests[id] = req
+        end
+
+        start_timer
+      end
+
+      def deregister_request(id, req)
+        @requests.delete(id)
+        stop_timer if @requests.length == 0
+      end
+
+      def send_packet(pkt)
+        send_datagram(pkt, nameserver, 53)
+      end
+
+      def nameserver=(ns)
+        @nameserver = ns
+      end
+
+      def nameserver
+        @nameserver || Resolver.nameserver
+      end
+
+      # Decodes the packet, looks for the request and passes the
+      # response over to the requester
+      def receive_data(data)
+        msg = nil
+        begin
+          msg = Resolv::DNS::Message.decode data
+        rescue
+        else
+          req = @requests[msg.id]
+          if req
+            @requests.delete(msg.id)
+            stop_timer if @requests.length == 0
+            req.receive_answer(msg)
+          end
+        end
+      end
+    end
+
+    class Request
+      include Deferrable
+      attr_accessor :retry_interval, :max_tries
+
+      def initialize(socket, hostname)
+        @socket = socket
+        @hostname = hostname
+        @tries = 0
+        @last_send = Time.at(0)
+        @retry_interval = 3
+        @max_tries = 5
+
+        if addrs = Resolver.hosts[hostname]
+          succeed addrs
+        else
+          EM.next_tick { tick }
+        end
+      end
+
+      def tick
+        # Break early if nothing to do
+        return if @last_send + @retry_interval > Time.now
+        if @tries < @max_tries
+          send
+        else
+          @socket.deregister_request(@id, self)
+          fail 'retries exceeded'
+        end
+      end
+
+      def receive_answer(msg)
+        addrs = []
+        msg.each_answer do |name,ttl,data|
+          if data.kind_of?(Resolv::DNS::Resource::IN::A) ||
+              data.kind_of?(Resolv::DNS::Resource::IN::AAAA)
+            addrs << data.address.to_s
+          end
+        end
+
+        if addrs.empty?
+          fail "rcode=#{msg.rcode}"
+        else
+          succeed addrs
+        end
+      end
+
+      private
+
+        def send
+          @tries += 1
+          @last_send = Time.now
+          @socket.send_packet(packet.encode)
+        end
+
+        def id
+          begin
+            @id = rand(65535)
+            @socket.register_request(@id, self)
+          rescue RequestIdAlreadyUsed
+            retry
+          end unless defined?(@id)
+
+          @id
+        end
+
+        def packet
+          msg = Resolv::DNS::Message.new
+          msg.id = id
+          msg.rd = 1
+          msg.add_question @hostname, Resolv::DNS::Resource::IN::A
+          msg
+        end
+
+    end
+  end
+end

data/lib/em/spawnable.rb

@@ -0,0 +1,84 @@
+#--
+#
+# Author:: Francis Cianfrocca (gmail: blackhedd)
+# Homepage::  http://rubyeventmachine.com
+# Date:: 25 Aug 2007
+# 
+# See EventMachine and EventMachine::Connection for documentation and
+# usage examples.
+#
+#----------------------------------------------------------------------------
+#
+# Copyright (C) 2006-07 by Francis Cianfrocca. All Rights Reserved.
+# Gmail: blackhedd
+# 
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of either: 1) the GNU General Public License
+# as published by the Free Software Foundation; either version 2 of the
+# License, or (at your option) any later version; or 2) Ruby's License.
+# 
+# See the file COPYING for complete licensing information.
+#
+#---------------------------------------------------------------------------
+#
+#
+
+module EventMachine
+  # Support for Erlang-style processes.
+  #
+  class SpawnedProcess
+    # Send a message to the spawned process
+    def notify *x
+      me = self
+      EM.next_tick {
+        # A notification executes in the context of this
+        # SpawnedProcess object. That makes self and notify
+        # work as one would expect.
+        #
+        y = me.call(*x)
+        if y and y.respond_to?(:pull_out_yield_block)
+          a,b = y.pull_out_yield_block
+          set_receiver a
+          self.notify if b
+        end
+      }
+    end
+    alias_method :resume, :notify
+    alias_method :run, :notify # for formulations like (EM.spawn {xxx}).run
+
+    def set_receiver blk
+      (class << self ; self ; end).class_eval do
+        remove_method :call if method_defined? :call
+        define_method :call, blk
+      end
+    end
+
+  end
+
+  # @private
+  class YieldBlockFromSpawnedProcess
+    def initialize block, notify
+      @block = [block,notify]
+    end
+    def pull_out_yield_block
+      @block
+    end
+  end
+
+  # Spawn an erlang-style process
+  def self.spawn &block
+    s = SpawnedProcess.new
+    s.set_receiver block
+    s
+  end
+
+  # @private
+  def self.yield &block
+    return YieldBlockFromSpawnedProcess.new( block, false )
+  end
+
+  # @private
+  def self.yield_and_notify &block
+    return YieldBlockFromSpawnedProcess.new( block, true )
+  end
+end

data/lib/em/streamer.rb

@@ -0,0 +1,118 @@
+module EventMachine
+  # Streams a file over a given connection. Streaming begins once the object is
+  # instantiated. Typically FileStreamer instances are not reused.
+  #
+  # Streaming uses buffering for files larger than 16K and uses so-called fast file reader (a C++ extension)
+  # if available (it is part of eventmachine gem itself).
+  #
+  # @example
+  #
+  #  module FileSender
+  #    def post_init
+  #      streamer = EventMachine::FileStreamer.new(self, '/tmp/bigfile.tar')
+  #      streamer.callback{
+  #        # file was sent successfully
+  #        close_connection_after_writing
+  #      }
+  #    end
+  #  end
+  #
+  #
+  # @author Francis Cianfrocca
+  class FileStreamer
+    include Deferrable
+
+    # Use mapped streamer for files bigger than 16k
+    MappingThreshold = 16384
+    # Wait until next tick to send more data when 50k is still in the outgoing buffer
+    BackpressureLevel = 50000
+    # Send 16k chunks at a time
+    ChunkSize = 16384
+
+    # @param [EventMachine::Connection] connection
+    # @param [String] filename File path
+    #
+    # @option args [Boolean] :http_chunks (false) Use HTTP 1.1 style chunked-encoding semantics.
+    def initialize connection, filename, args = {}
+      @connection = connection
+      @http_chunks = args[:http_chunks]
+
+      if File.exist?(filename)
+        @size = File.size(filename)
+        if @size <= MappingThreshold
+          stream_without_mapping filename
+        else
+          stream_with_mapping filename
+        end
+      else
+        fail "file not found"
+      end
+    end
+
+    # @private
+    def stream_without_mapping filename
+      if @http_chunks
+        @connection.send_data "#{@size.to_s(16)}\r\n"
+        @connection.send_file_data filename
+        @connection.send_data "\r\n0\r\n\r\n"
+      else
+        @connection.send_file_data filename
+      end
+      succeed
+    end
+    private :stream_without_mapping
+
+    # @private
+    def stream_with_mapping filename
+      ensure_mapping_extension_is_present
+
+      @position = 0
+      @mapping = EventMachine::FastFileReader::Mapper.new filename
+      stream_one_chunk
+    end
+    private :stream_with_mapping
+
+    # Used internally to stream one chunk at a time over multiple reactor ticks
+    # @private
+    def stream_one_chunk
+      loop {
+        if @position < @size
+          if @connection.get_outbound_data_size > BackpressureLevel
+            EventMachine::next_tick {stream_one_chunk}
+            break
+          else
+            len = @size - @position
+            len = ChunkSize if (len > ChunkSize)
+
+            @connection.send_data( "#{len.to_s(16)}\r\n" ) if @http_chunks
+            @connection.send_data( @mapping.get_chunk( @position, len ))
+            @connection.send_data("\r\n") if @http_chunks
+
+            @position += len
+          end
+        else
+          @connection.send_data "0\r\n\r\n" if @http_chunks
+          @mapping.close
+          succeed
+          break
+        end
+      }
+    end
+
+    #
+    # We use an outboard extension class to get memory-mapped files.
+    # It's outboard to avoid polluting the core distro, but that means
+    # there's a "hidden" dependency on it. The first time we get here in
+    # any run, try to load up the dependency extension. User code will see
+    # a LoadError if it's not available, but code that doesn't require
+    # mapped files will work fine without it. This is a somewhat difficult
+    # compromise between usability and proper modularization.
+    #
+    # @private
+    def ensure_mapping_extension_is_present
+      @@fastfilereader ||= (require 'fastfilereaderext')
+    end
+    private :ensure_mapping_extension_is_present
+
+  end # FileStreamer
+end # EventMachine

data/lib/em/threaded_resource.rb

@@ -0,0 +1,90 @@
+module EventMachine
+  # = EventMachine::ThreadedResource
+  #
+  # A threaded resource is a "quick and dirty" wrapper around the concept of
+  # wiring up synchronous code into a standard EM::Pool. This is useful to keep
+  # interfaces coherent and provide a simple approach at "making an interface
+  # async-ish".
+  #
+  # General usage is to wrap libraries that do not support EventMachine, or to
+  # have a specific number of dedicated high-cpu worker resources.
+  #
+  # == Basic Usage example
+  #
+  # This example requires the cassandra gem. The cassandra gem contains an
+  # EventMachine interface, but it's sadly Fiber based and thus only works on
+  # 1.9. It also requires (potentially) complex stack switching logic to reach
+  # completion of nested operations. By contrast this approach provides a block
+  # in which normal synchronous code can occur, but makes no attempt to wire the
+  # IO into EventMachines C++ IO implementations, instead relying on the reactor
+  # pattern in rb_thread_select.
+  #
+  #    cassandra_dispatcher = ThreadedResource.new do
+  #      Cassandra.new('allthethings', '127.0.0.1:9160')
+  #    end
+  #
+  #    pool = EM::Pool.new
+  #
+  #    pool.add cassandra_dispatcher
+  #
+  #    # If we don't care about the result:
+  #    pool.perform do |dispatcher|
+  #      # The following block executes inside a dedicated thread, and should not
+  #      # access EventMachine things:
+  #      dispatcher.dispatch do |cassandra|
+  #        cassandra.insert(:Things, '10', 'stuff' => 'things')
+  #      end
+  #    end
+  #
+  #    # Example where we care about the result:
+  #    pool.perform do |dispatcher|
+  #      # The dispatch block is executed in the resources thread.
+  #      completion = dispatcher.dispatch do |cassandra|
+  #        cassandra.get(:Things, '10', 'stuff')
+  #      end
+  #
+  #      # This block will be yielded on the EM thread:
+  #      completion.callback do |result|
+  #        EM.do_something_with(result)
+  #      end
+  #
+  #      completion
+  #    end
+  class ThreadedResource
+
+    # The block should return the resource that will be yielded in a dispatch.
+    def initialize
+      @resource = yield
+
+      @running = true
+      @queue   = ::Queue.new
+      @thread  = Thread.new do
+        @queue.pop.call while @running
+      end
+    end
+
+    # Called on the EM thread, generally in a perform block to return a
+    # completion for the work.
+    def dispatch
+      completion = EM::Completion.new
+      @queue << lambda do
+        begin
+          result = yield @resource
+          completion.succeed result
+        rescue => e
+          completion.fail e
+        end
+      end
+      completion
+    end
+
+    # Kill the internal thread. should only be used to cleanup - generally
+    # only required for tests.
+    def shutdown
+      @running = false
+      @queue << lambda {}
+      @thread.join
+    end
+
+  end
+end

data/lib/em/tick_loop.rb

@@ -0,0 +1,85 @@
+module EventMachine
+  # Creates and immediately starts an EventMachine::TickLoop
+  def self.tick_loop(*a, &b)
+    TickLoop.new(*a, &b).start
+  end
+
+  # A TickLoop is useful when one needs to distribute amounts of work
+  # throughout ticks in order to maintain response times. It is also useful for
+  # simple repeated checks and metrics.
+  # @example
+  #    # Here we run through an array one item per tick until it is empty, 
+  #    # printing each element.
+  #    # When the array is empty, we return :stop from the callback, and the
+  #    # loop will terminate.
+  #    # When the loop terminates, the on_stop callbacks will be called.  
+  #    EM.run do
+  #      array = (1..100).to_a
+  #      
+  #      tickloop = EM.tick_loop do
+  #        if array.empty?
+  #          :stop
+  #        else
+  #          puts array.shift
+  #        end
+  #      end
+  #      
+  #      tickloop.on_stop { EM.stop }
+  #    end
+  #
+  class TickLoop
+
+    # Arguments: A callback (EM::Callback) to call each tick. If the call
+    # returns +:stop+ then the loop will be stopped. Any other value is 
+    # ignored.
+    def initialize(*a, &b)
+      @work = EM::Callback(*a, &b)
+      @stops = []
+      @stopped = true
+    end
+
+    # Arguments: A callback (EM::Callback) to call once on the next stop (or
+    # immediately if already stopped).
+    def on_stop(*a, &b)
+      if @stopped
+        EM::Callback(*a, &b).call
+      else
+        @stops << EM::Callback(*a, &b)
+      end
+    end
+
+    # Stop the tick loop immediately, and call it's on_stop callbacks.
+    def stop
+      @stopped = true
+      until @stops.empty?
+        @stops.shift.call
+      end
+    end
+
+    # Query if the loop is stopped.
+    def stopped?
+      @stopped
+    end
+
+    # Start the tick loop, will raise argument error if the loop is already
+    # running.
+    def start
+      raise ArgumentError, "double start" unless @stopped
+      @stopped = false
+      schedule
+    end
+
+    private
+    def schedule
+      EM.next_tick do
+        next if @stopped
+        if @work.call == :stop
+          stop
+        else
+          schedule
+        end
+      end
+      self
+    end
+  end
+end