arya-pandemic 0.2.3 → 0.3.0

data/examples/client/pandemic_client.yml

@@ -0,0 +1,3 @@
+servers:
+  - localhost:4000
+  - localhost:4001

data/examples/server/pandemic_server.yml

@@ -0,0 +1,3 @@
+servers:
+  - localhost:4000
+  - localhost:4001

data/examples/server/word_count_server.rb

@@ -0,0 +1,47 @@
+require 'rubygems'
+require 'pandemic'
+require 'json'
+
+class WordCounter < Pandemic::ServerSide::Handler
+  def map(request, servers)
+    # select only the alive servers (non-disconnected)
+    only_alive = servers.keys.select{|k| servers[k] != :disconnected}
+    
+    mapping = {}
+    intervals = (request.body.size / only_alive.size.to_f).floor
+    
+    pos = 0
+    only_alive.size.times do |i|
+      if i == only_alive.size - 1 # last peer gets the rest
+        mapping[only_alive[i]] = request.body[pos..-1]
+      else
+        next_pos = request.body[(pos + intervals)..-1].index(/ /) + pos + intervals
+        mapping[only_alive[i]] = request.body[pos...next_pos]
+        pos = next_pos
+      end
+    end
+    mapping
+  end
+  
+  def process(text)
+    counts = Hash.new(0)
+    text.scan(/\w+/) do |word|
+      counts[word.strip.downcase] += 1
+    end
+    counts.to_json
+  end
+  
+  def reduce(request)
+    total_counts = Hash.new(0)
+    request.responses.each do |counts|
+      JSON.parse(counts).each do |word, count|
+        total_counts[word] += count
+      end
+    end
+    total_counts.to_json
+  end
+end
+
+server = epidemic!
+server.handler = WordCounter
+server.start.join

data/lib/pandemic.rb

@@ -17,6 +17,7 @@ require 'pandemic/server_side/server'
 require 'pandemic/server_side/peer'
 require 'pandemic/server_side/request'
 require 'pandemic/server_side/handler'
+require 'pandemic/server_side/processor'
 
 require 'pandemic/client_side/config'
 require 'pandemic/client_side/cluster_connection'
@@ -33,13 +34,13 @@ require 'pandemic/client_side/pandemize'
 TCP_NO_DELAY_AVAILABLE =
     RUBY_VERSION < '1.9' ? Socket.constants.include?('TCP_NODELAY') : Socket.constants.include?(:TCP_NODELAY)
 
-def epidemic!
+def epidemic!(bind_to = nil)
   if $pandemic_logger.nil?
     $pandemic_logger = Logger.new("pandemic.log")
     $pandemic_logger.level = Logger::INFO
     $pandemic_logger.datetime_format = "%Y-%m-%d %H:%M:%S "
   end
-  Pandemic::ServerSide::Server.boot
+  Pandemic::ServerSide::Server.boot(bind_to)
 end
 
 ::Pandemize = Pandemic::ClientSide::Pandemize

data/lib/pandemic/mutex_counter.rb

@@ -11,6 +11,7 @@ module Pandemic
     def real_total
       @mutex.synchronize { (@resets * @max) + @counter }
     end
+    alias_method :to_i, :real_total
     
     def value
       @mutex.synchronize { @counter }
@@ -25,5 +26,25 @@ module Pandemic
         @counter += 1
       end
     end
+    alias_method :next, :inc
+    alias_method :succ, :inc
+    
+    # decr only to zero
+    def decr
+      @mutex.synchronize do
+        if @counter > 0
+          @counter -= 1
+        else
+          if @resets > 1
+            @resets -= 1 
+            @counter = @max
+          end
+        end
+        @counter
+      end
+    end
+    alias_method :pred, :decr
+    alias_method :prev, :decr
+    
   end
 end

data/lib/pandemic/server_side/config.rb

@@ -2,7 +2,7 @@ module Pandemic
   module ServerSide
     class Config
       class << self
-        attr_accessor :bind_to, :servers, :response_timeout
+        attr_accessor :bind_to, :servers, :response_timeout, :fork_for_processor
         def load
           path = extract_config_path
           yaml = YAML.load_file(path)
@@ -10,9 +10,11 @@ module Pandemic
           @server_map = yaml['servers'] || []
           @servers = @server_map.is_a?(Hash) ? @server_map.values : @server_map 
           @servers = @servers.collect { |s| s.is_a?(Hash) ? s.keys.first : s }
-        
+          
           @response_timeout = (yaml['response_timeout'] || 1).to_f
           @bind_to = extract_bind_to
+          @fork_for_processor = yaml['fork_for_processor']
+          
           raise "Interface to bind to is nil." unless @bind_to
         end
         

data/lib/pandemic/server_side/handler.rb

@@ -22,6 +22,10 @@ module Pandemic
       def process(body)
         body
       end
+      
+      def filter_alive(servers)
+        servers.keys.select{|k| servers[k] != :disconnected}
+      end        
     end
   end
 end

data/lib/pandemic/server_side/processor.rb

@@ -0,0 +1,90 @@
+module Pandemic
+  module ServerSide
+    class Processor
+      def initialize(handler)
+        read_from_parent, write_to_child = IO.pipe
+        read_from_child, write_to_parent = IO.pipe
+        
+        @child_process_id = fork
+        if @child_process_id
+          # I'm the parent
+          write_to_parent.close
+          read_from_parent.close
+          @out = write_to_child
+          @in = read_from_child
+        else
+          # I'm the child
+          write_to_child.close
+          read_from_child.close
+          @out = write_to_parent
+          @in = read_from_parent
+          @handler = handler.new
+          wait_for_jobs
+        end
+      end
+      
+      def process(body)
+        if parent?
+          @out.puts(body.size.to_s)
+          @out.write(body)
+          ready, = IO.select([@in], nil, nil)
+          if ready
+            size = @in.gets.to_i
+            result = @in.read(size)
+            return result
+          end
+        else
+          return @handler.process(body)
+        end
+      end
+      
+      def close(status = 0)
+        if parent? && child_alive?
+          Process.detach(@child_process_id)
+          @out.puts(status.to_s)
+          @out.close
+          @in.close
+        else
+          Process.exit!(status)
+        end
+      end
+      
+      def closed?
+        !child_alive?
+      end
+      
+      private
+      def wait_for_jobs
+        if child?
+          while true
+            ready, = IO.select([@in], nil, nil)
+            if ready
+              size = @in.gets.to_i
+              if size > 0
+                body = @in.read(size)
+                result = process(body)
+                @out.puts(result.size.to_s)
+                @out.write(result)
+              else
+                self.close(size)
+                break
+              end
+            end
+          end
+        end
+      end
+      
+      def parent?
+        !!@child_process_id
+      end
+      
+      def child?
+        !parent?
+      end
+      
+      def child_alive?
+        parent? && !@in.closed?
+      end
+    end
+  end
+end

data/lib/pandemic/server_side/server.rb

@@ -4,10 +4,10 @@ module Pandemic
       include Util
       class StopServer < Exception; end
       class << self
-        def boot
+        def boot(bind_to = nil)
           Config.load
           # Process.setrlimit(Process::RLIMIT_NOFILE, 4096) # arbitrary high number of max file descriptors.
-          server = self.new
+          server = self.new(bind_to || Config.bind_to)
           set_signal_traps(server)
           server
         end
@@ -26,23 +26,25 @@ module Pandemic
         end
       end
       attr_reader :host, :port, :running
-      def initialize
-        @host, @port = host_port(Config.bind_to)
-        
+      def initialize(bind_to)
+        @host, @port = host_port(bind_to)
         @clients = []
         @total_clients = 0
         @clients_mutex = Mutex.new
+        @num_jobs_processed = MutexCounter.new
+        @num_jobs_entered = MutexCounter.new
         
         @peers = {}
         @servers = Config.servers
         @servers.each do |peer|
-          next if peer == Config.bind_to # not a peer, it's itself
+          next if peer == bind_to # not a peer, it's itself
           @peers[peer] = Peer.new(peer, self)
         end
       end
       
       def handler=(handler)
         @handler = handler
+        @handler_instance = handler.new
       end
     
       def start
@@ -113,7 +115,7 @@ module Pandemic
     
       def handle_client_request(request)
         info("Handling client request")
-        map = @handler.map(request, connection_statuses)
+        map = @handler_instance.map(request, connection_statuses)
         request.max_responses = map.size
         debug("Sending client request to #{map.size} handlers (#{request.hash})")
         
@@ -129,7 +131,7 @@ module Pandemic
             begin
               request.add_response(self.process(map[signature]))
             rescue Exception => e
-              warn("Unhandled exception in local processing: #{e.inspect}")
+              warn("Unhandled exception in local processing: #{e.inspect}#{e.backtrace.join("\n")}}")
             end
           end
         end
@@ -138,11 +140,18 @@ module Pandemic
         request.wait_for_responses
         
         debug("Done waiting for responses, calling reduce")
-        @handler.reduce(request)
+        @handler_instance.reduce(request)
       end
     
       def process(body)
-        @handler.process(body)
+        @num_jobs_entered.inc
+        response = if Config.fork_for_processor
+          self.processor.with_connection {|con| con.process(body) } 
+        else
+          @handler_instance.process(body)
+        end
+        @num_jobs_processed.inc
+        response
       end
     
       def signature
@@ -166,6 +175,14 @@ module Pandemic
         end
       end
       
+      def processor
+        @processor ||= begin
+          processor = ConnectionPool.new
+          processor.create_connection { Processor.new(@handler) }
+          processor
+        end
+      end
+      
       private
       def print_stats(connection)
         begin
@@ -180,6 +197,8 @@ module Pandemic
           str << "Total Requests: #{stats[:num_requests]}"
           str << "Pending Requests: #{stats[:pending_requests]}"
           str << "Late Responses: #{stats[:late_responses]}"
+          str << "Total Jobs Processed: #{stats[:total_jobs_processed]}"
+          str << "Pending Jobs: #{stats[:jobs_pending]}"
           connection.puts(str.join("\n"))
         end while (s = connection.gets) && (s.strip == "stats" || s.strip == "")
         connection.close if connection && !connection.closed?
@@ -212,10 +231,10 @@ module Pandemic
               end
               counts
             end
-        
+        results[:total_jobs_processed] = @num_jobs_processed.to_i
+        results[:jobs_pending] = @num_jobs_entered.to_i - results[:total_jobs_processed]
         results[:num_requests] = Request.total_request_count
         results[:late_responses] = Request.total_late_responses
-
         results[:pending_requests] = @clients_mutex.synchronize do
           @clients.inject(0) do |pending, client|
             pending + (client.received_requests - client.responded_requests)

data/pandemic.gemspec

@@ -2,15 +2,15 @@
 
 Gem::Specification.new do |s|
   s.name = %q{pandemic}
-  s.version = "0.2.3"
+  s.version = "0.3.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 1.2") if s.respond_to? :required_rubygems_version=
   s.authors = ["Arya Asemanfar"]
-  s.date = %q{2009-04-10}
+  s.date = %q{2009-05-27}
   s.description = %q{Distribute MapReduce to any of the workers and it will spread, like a pandemic.}
   s.email = %q{aryaasemanfar@gmail.com}
-  s.extra_rdoc_files = ["lib/pandemic/client_side/cluster_connection.rb", "lib/pandemic/client_side/config.rb", "lib/pandemic/client_side/connection.rb", "lib/pandemic/client_side/connection_proxy.rb", "lib/pandemic/client_side/pandemize.rb", "lib/pandemic/connection_pool.rb", "lib/pandemic/mutex_counter.rb", "lib/pandemic/server_side/client.rb", "lib/pandemic/server_side/config.rb", "lib/pandemic/server_side/handler.rb", "lib/pandemic/server_side/peer.rb", "lib/pandemic/server_side/request.rb", "lib/pandemic/server_side/server.rb", "lib/pandemic/util.rb", "lib/pandemic.rb", "README.markdown"]
-  s.files = ["lib/pandemic/client_side/cluster_connection.rb", "lib/pandemic/client_side/config.rb", "lib/pandemic/client_side/connection.rb", "lib/pandemic/client_side/connection_proxy.rb", "lib/pandemic/client_side/pandemize.rb", "lib/pandemic/connection_pool.rb", "lib/pandemic/mutex_counter.rb", "lib/pandemic/server_side/client.rb", "lib/pandemic/server_side/config.rb", "lib/pandemic/server_side/handler.rb", "lib/pandemic/server_side/peer.rb", "lib/pandemic/server_side/request.rb", "lib/pandemic/server_side/server.rb", "lib/pandemic/util.rb", "lib/pandemic.rb", "Manifest", "MIT-LICENSE", "pandemic.gemspec", "Rakefile", "README.markdown", "test/client_test.rb", "test/connection_pool_test.rb", "test/functional_test.rb", "test/handler_test.rb", "test/mutex_counter_test.rb", "test/peer_test.rb", "test/server_test.rb", "test/test_helper.rb", "test/util_test.rb"]
+  s.extra_rdoc_files = ["lib/pandemic/client_side/cluster_connection.rb", "lib/pandemic/client_side/config.rb", "lib/pandemic/client_side/connection.rb", "lib/pandemic/client_side/connection_proxy.rb", "lib/pandemic/client_side/pandemize.rb", "lib/pandemic/connection_pool.rb", "lib/pandemic/mutex_counter.rb", "lib/pandemic/server_side/client.rb", "lib/pandemic/server_side/config.rb", "lib/pandemic/server_side/handler.rb", "lib/pandemic/server_side/peer.rb", "lib/pandemic/server_side/processor.rb", "lib/pandemic/server_side/request.rb", "lib/pandemic/server_side/server.rb", "lib/pandemic/util.rb", "lib/pandemic.rb", "README.markdown"]
+  s.files = ["examples/client/client.rb", "examples/client/constitution.txt", "examples/client/pandemic_client.yml", "examples/server/pandemic_server.yml", "examples/server/word_count_server.rb", "lib/pandemic/client_side/cluster_connection.rb", "lib/pandemic/client_side/config.rb", "lib/pandemic/client_side/connection.rb", "lib/pandemic/client_side/connection_proxy.rb", "lib/pandemic/client_side/pandemize.rb", "lib/pandemic/connection_pool.rb", "lib/pandemic/mutex_counter.rb", "lib/pandemic/server_side/client.rb", "lib/pandemic/server_side/config.rb", "lib/pandemic/server_side/handler.rb", "lib/pandemic/server_side/peer.rb", "lib/pandemic/server_side/processor.rb", "lib/pandemic/server_side/request.rb", "lib/pandemic/server_side/server.rb", "lib/pandemic/util.rb", "lib/pandemic.rb", "Manifest", "MIT-LICENSE", "Rakefile", "README.markdown", "pandemic.gemspec", "test/client_test.rb", "test/connection_pool_test.rb", "test/functional_test.rb", "test/handler_test.rb", "test/mutex_counter_test.rb", "test/peer_test.rb", "test/processor_test.rb", "test/server_test.rb", "test/test_helper.rb", "test/util_test.rb"]
   s.has_rdoc = true
   s.homepage = %q{https://github.com/arya/pandemic/}
   s.rdoc_options = ["--line-numbers", "--inline-source", "--title", "Pandemic", "--main", "README.markdown"]
@@ -18,7 +18,7 @@ Gem::Specification.new do |s|
   s.rubyforge_project = %q{pandemic}
   s.rubygems_version = %q{1.3.1}
   s.summary = %q{Distribute MapReduce to any of the workers and it will spread, like a pandemic.}
-  s.test_files = ["test/client_test.rb", "test/connection_pool_test.rb", "test/functional_test.rb", "test/handler_test.rb", "test/mutex_counter_test.rb", "test/peer_test.rb", "test/server_test.rb", "test/test_helper.rb", "test/util_test.rb"]
+  s.test_files = ["test/client_test.rb", "test/connection_pool_test.rb", "test/functional_test.rb", "test/handler_test.rb", "test/mutex_counter_test.rb", "test/peer_test.rb", "test/processor_test.rb", "test/server_test.rb", "test/test_helper.rb", "test/util_test.rb"]
 
   if s.respond_to? :specification_version then
     current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION

data/test/functional_test.rb

@@ -12,7 +12,7 @@ class FunctionalTest < Test::Unit::TestCase
       def process(body)
         body.reverse
       end
-    end.new
+    end
     server.start
     
     client = Class.new do
@@ -30,7 +30,7 @@ class FunctionalTest < Test::Unit::TestCase
       def process(body)
         body.reverse
       end
-    end.new
+    end
     
     ARGV.replace(["-i", "0", "-c", "test/pandemic_server.yml"]) # :(    
     server = epidemic!

data/test/mutex_counter_test.rb

@@ -16,6 +16,24 @@ class MutexCounterTest < Test::Unit::TestCase
       assert_equal 1, @counter.value
     end
     
+    should "decrement to 0 after one call to inc" do
+      assert_equal 0, @counter.value
+      assert_equal 1, @counter.inc
+      assert_equal 1, @counter.value
+      assert_equal 0, @counter.decr
+      assert_equal 0, @counter.value
+    end
+    
+    should "only decrement to 0" do
+      assert_equal 0, @counter.value
+      assert_equal 1, @counter.inc
+      assert_equal 1, @counter.value
+      assert_equal 0, @counter.decr
+      assert_equal 0, @counter.value
+      assert_equal 0, @counter.decr
+      assert_equal 0, @counter.value
+    end
+    
     should "be thread safe" do
       # Not exactly a perfect test, but I'm not sure how to actually test 
       # this without putting some code in the counter for this reason.
@@ -42,5 +60,14 @@ class MutexCounterTest < Test::Unit::TestCase
       assert_equal 1, @counter.value
       assert_equal 11, @counter.real_total
     end
+    
+    should "maintain correct 'real_total' with decrement" do
+      11.times { @counter.inc }
+      assert_equal 1, @counter.value
+      assert_equal 11, @counter.real_total
+      assert_equal 0, @counter.decr
+      assert_equal 10, @counter.real_total
+    end
+    
   end
 end