arya-pandemic 0.2 → 0.2.1

data/MIT-LICENSE

@@ -0,0 +1,22 @@
+Copyright (c) 2009 Arya Asemanfar
+
+Permission is hereby granted, free of charge, to any person
+obtaining a copy of this software and associated documentation
+files (the "Software"), to deal in the Software without
+restriction, including without limitation the rights to use,
+copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the
+Software is furnished to do so, subject to the following
+conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -2,13 +2,13 @@ require 'rubygems'
 require 'rake'
 require 'echoe'
 
-Echoe.new('pandemic', '0.2') do |p|
+Echoe.new('pandemic', '0.2.1') do |p|
   p.description    = "Distribute MapReduce to any of the workers and it will spread, like a pandemic."
-  p.url            = ""
+  p.url            = "https://github.com/arya/pandemic/"
   p.author         = "Arya Asemanfar"
   p.email          = "aryaasemanfar@gmail.com"
-  p.ignore_pattern = ["tmp/*", "script/*", 'config.yml']
-  p.development_dependencies = []
+  p.ignore_pattern = ["tmp/*", "script/*", 'test/*']
+  p.development_dependencies = ["shoulda", "mocha"]
 end
 
-Dir["#{File.dirname(__FILE__)}/tasks/*.rake"].sort.each { |ext| load ext }
+Dir["#{File.dirname(__FILE__)}/tasks/*.rake"].sort.each { |ext| load ext }

data/lib/pandemic.rb

@@ -25,15 +25,17 @@ require 'pandemic/client_side/connection_proxy'
 require 'pandemic/client_side/pandemize'
 
 # TODO:
+# - tests
 # - IO timeouts/robustness
 # - documentation
 # - PING/PONG?
 
-$logger = Logger.new(STDOUT)
-$logger.level = Logger::DEBUG
-$logger.datetime_format = "%Y-%m-%d %H:%M:%S "
-
 def epidemic!
+  if $pandemic_logger.nil?
+    $pandemic_logger = Logger.new("pandemic.log")
+    $pandemic_logger.level = Logger::INFO
+    $pandemic_logger.datetime_format = "%Y-%m-%d %H:%M:%S "
+  end
   Pandemic::ServerSide::Server.boot
 end
 

data/lib/pandemic/client_side/cluster_connection.rb

@@ -4,6 +4,7 @@ module Pandemic
       class NotEnoughConnectionsTimeout < Exception; end
       class NoNodesAvailable < Exception; end
       class LostConnectionToNode < Exception; end
+      class NodeTimedOut < Exception; end
       
       include Util
       def initialize
@@ -16,6 +17,9 @@ module Pandemic
         @connection_proxies = {}
         @queue = @mutex.new_cond # TODO: there should be a queue for each group
         
+        @response_timeout = Config.response_timeout
+        @response_timeout = nil if @response_timeout <= 0
+        
         Config.servers.each_with_index do |server_addr, key|
           @connection_proxies[key] = ConnectionProxy.new(key, self)
           host, port = host_port(server_addr)
@@ -41,7 +45,8 @@ module Pandemic
           begin
             socket.write("#{body.size}\n#{body}")
             socket.flush
-            # IO.select([socket])
+            is_ready = IO.select([socket], nil, nil, @response_timeout)
+            raise NodeTimedOut if is_ready.nil?
             response_size = socket.gets
             if response_size
               socket.read(response_size.strip.to_i)

data/lib/pandemic/client_side/config.rb

@@ -5,7 +5,7 @@ module Pandemic
         @@load_mutex = Mutex.new
         attr_accessor :config_path, :loaded
         attr_accessor :servers, :max_connections_per_server, :min_connections_per_server,
-                      :connection_wait_timeout
+                      :connection_wait_timeout, :response_timeout
         def load
           @@load_mutex.synchronize do
             return if self.loaded
@@ -20,6 +20,7 @@ module Pandemic
             @max_connections_per_server = (yaml['max_connections_per_server'] || 1).to_i
             @min_connections_per_server = (yaml['min_connections_per_server'] || 1).to_i
             @connection_wait_timeout = (yaml['connection_wait_timeout'] || 1).to_f
+            @response_timeout = (yaml['response_timeout'] || 1).to_f
             self.loaded = true
           end
         end

data/lib/pandemic/connection_pool.rb

@@ -4,34 +4,39 @@ module Pandemic
     class CreateConnectionUndefinedException < Exception; end
     include Util
     def initialize(options = {})
+      @connected = false
       @mutex = Monitor.new
       @queue = @mutex.new_cond
       @available = []
       @connections = []
       @max_connections = options[:max_connections] || 10
+      @min_connections = options[:min_connections] || 1
       @timeout = options[:timeout] || 3
     end
     
     def add_connection!
-      # bang because we're ignorings the max connections
-      conn = create_connection
-      if conn
-        @mutex.synchronize do
-          @connections << conn
-          @available << conn
-        end
+      # bang because we're ignoring the max connections
+      @mutex.synchronize do
+        conn = create_connection
+        @available << conn if conn && !conn.closed?
       end
     end
     
     def create_connection(&block)
       if block.nil?
         if @create_connection
-          @create_connection.call
+          conn = @create_connection.call
+          if conn && !conn.closed?
+            @connections << conn
+            @connected = true 
+            conn
+          end
         else
           raise CreateConnectionUndefinedException.new("You must specify a block to create connections")
         end
       else
         @create_connection = block
+        connect
       end
     end
     
@@ -45,30 +50,61 @@ module Pandemic
             connection.close
           end
         end
+        @connections.delete(connection)
+        @available.delete(connection)
+        # this is within the mutex of the caller
+        @connected = false if @connections.empty?
       else
         @destroy_connection = block
       end
     end
     
+    def status_check(connection = nil, &block)
+      if block.nil?
+        if @status_check
+          @status_check.call(connection)
+        else
+          connection && !connection.closed?
+        end
+      else
+        @status_check = block
+      end
+    end    
+    
     def connected?
-      @mutex.synchronize { @connections.size > 0 }
+      @connected
+    end
+    
+    def connect
+      if !connected?
+        @min_connections.times { add_connection! }
+        grim_reaper
+      end
+    end
+    
+    def available_count
+      @available.size
+    end
+    
+    def connections_count
+      @connections.size
     end
     
     def disconnect
       @mutex.synchronize do
         return if @disconnecting
         @disconnecting = true
-        @available.each do |conn|
+        @connected = false # we don't want anyone thinking they can use this connection
+        @grim_reaper.kill if @grim_reaper && @grim_reaper.alive?
+        
+        @available.dup.each do |conn|
           destroy_connection(conn)
-          @connections.delete(conn)
         end
-        @available = []
+        
         while @connections.size > 0 && @queue.wait
-          @available.each do |conn|
+          @available.dup.each do |conn|
             destroy_connection(conn)
-            @connections.delete(conn)
           end
-          @available = []
         end
         @disconnecting = false
       end
@@ -94,7 +130,6 @@ module Pandemic
             connection = @available.pop
             break
           elsif @connections.size < @max_connections && (connection = create_connection)
-            @connections << connection
             break
           elsif @queue.wait(@timeout)
             next
@@ -113,5 +148,46 @@ module Pandemic
       end
     end
     
+    def grim_reaper
+      @grim_reaper.kill if @grim_reaper && @grim_reaper.alive?
+      @grim_reaper = Thread.new do
+        usage_history = []
+        loop do
+          if connected?
+            @mutex.synchronize do
+              dead = []
+
+              @connections.each do |conn|
+                dead << conn if !status_check(conn)
+              end
+
+              dead.each { |c| destroy_connection(c) }
+              
+              # restore to minimum number of connections if it's too low
+              [@min_connections - @connections.size, 0].max.times do
+                add_connection!
+              end
+
+              usage_history.push(@available.size)
+              if usage_history.size >= 10
+                # kill the minimum number of available connections over the last 10 checks
+                # or the total connections minux the min connections, whichever is lower.
+                # this ensures that you never go below min connections
+                to_kill = [usage_history.min, @connections.size - @min_connections].min
+                [to_kill, 0].max.times do
+                  destroy_connection(@connections.last)
+                end
+                usage_history = []
+              end
+
+            end # end mutex
+            sleep 30
+          else
+            break
+          end # end if connected
+        end
+      end
+    end
+    
   end
 end

data/lib/pandemic/mutex_counter.rb

@@ -1,19 +1,24 @@
 module Pandemic
   class MutexCounter
     MAX = (2 ** 30) - 1
-    def initialize
+    def initialize(max = MAX)
       @mutex = Mutex.new
       @counter = 0
       @resets = 0
+      @max = max
     end
 
     def real_total
-      @mutex.synchronize { (@resets * MAX) + @counter }
+      @mutex.synchronize { (@resets * @max) + @counter }
+    end
+    
+    def value
+      @mutex.synchronize { @counter }
     end
 
     def inc
       @mutex.synchronize do
-        if @counter >= MAX
+        if @counter >= @max
           @counter = 0  # to avoid Bignum, it's about 4x slower
           @resets += 1
         end

data/lib/pandemic/server_side/client.rb

@@ -11,6 +11,7 @@ module Pandemic
         @server = server
         @received_requests = 0
         @responded_requests = 0
+        @current_request = nil
       end
     
       def listen
@@ -38,6 +39,8 @@ module Pandemic
                   response = handle_request(body)
                   
                   debug("Writing response to client")
+                  
+                  # the connection could be closed, we'll let it be rescued if it is.
                   @connection.write("#{response.size}\n#{response}")
                   @connection.flush
                   @responded_requests += 1
@@ -46,10 +49,14 @@ module Pandemic
               end
             rescue DisconnectClient
               info("Closing client connection")
-              @connection.close unless @connection.nil? || @connection.closed?
+              close_connection
+            rescue Errno::EPIPE
+              info("Connection to client lost")
+              close_connection
             rescue Exception => e
               warn("Unhandled exception in client listen thread: #{e.inspect}")
             ensure
+              @current_request.cancel! if @current_request
               @server.client_closed(self)
             end
           end
@@ -60,12 +67,21 @@ module Pandemic
       def close
         @listener_thread.raise(DisconnectClient)
       end
+      
+
     
       def handle_request(request)
-        @server.handle_client_request(Request.new(request))
+        @current_request = Request.new(request)
+        response = @server.handle_client_request(@current_request)
+        @current_request = nil
+        return response
       end
       
       private
+      def close_connection
+        @connection.close unless @connection.nil? || @connection.closed?
+      end
+      
       def signature
         @signature ||= @connection.peeraddr.values_at(3,1).join(":")
       end

data/lib/pandemic/server_side/peer.rb

@@ -15,9 +15,8 @@ module Pandemic
       end
       
       def connect
-        return if connected?
         debug("Forced connection to peer")
-        1.times { @connection_pool.add_connection! }
+        @connection_pool.connect
       end
       
       def disconnect
@@ -31,6 +30,7 @@ module Pandemic
     
       def client_request(request, body)
         debug("Sending client's request to peer")
+        debug("Connection pool has #{@connection_pool.available_count} of #{@connection_pool.connections_count} connections available")
         # TODO: Consider adding back threads here if it will be faster that way in Ruby 1.9
         @connection_pool.with_connection do |connection|
           if connection && !connection.closed?
@@ -49,6 +49,7 @@ module Pandemic
         debug("Adding incoming connection")
 
         connect # if we're not connected, we should be
+
         
         thread = Thread.new(conn) do |connection|
           begin
@@ -89,10 +90,18 @@ module Pandemic
         
         @connection_pool.create_connection do
           connection = nil
+          retries = 0
           begin
             connection = TCPSocket.new(@host, @port)
-          rescue Errno::ETIMEDOUT, Errno::ECONNREFUSED
+          rescue Errno::ETIMEDOUT, Errno::ECONNREFUSED => e
             connection = nil
+            debug("Connection timeout or refused: #{e.inspect}")
+            if retries == 0
+              debug("Retrying connection")
+              retries += 1
+              sleep 0.01
+              retry
+            end
           rescue Exception => e
             warn("Unhandled exception in create connection block: #{e.inspect}")
           end
@@ -102,7 +111,6 @@ module Pandemic
           end
           connection
         end
-        
       end
     
       def handle_incoming_request(request, connection)

data/lib/pandemic/server_side/request.rb

@@ -34,20 +34,35 @@ module Pandemic
           @responses << response
           if @max_responses && @responses.size >= @max_responses
             debug("Hit max responses, waking up waiting thread")
-            @waiting_thread.wakeup if @waiting_thread && @waiting_thread.status == "sleep"
+            wakeup_waiting_thread
             @complete = true
           end
         end
       end
+      
+      def wakeup_waiting_thread
+        @waiting_thread.wakeup if @waiting_thread && @waiting_thread.status == "sleep"
+      end
     
       def responses
         @responses # its frozen so we don't have to worry about mutex
       end
+      
+      def cancel!
+        # consider telling peers that they should stop, but for now this is fine.
+        @responses_mutex.synchronize do
+          wakeup_waiting_thread
+        end
+      end
     
       def wait_for_responses
         return if @complete
         @waiting_thread = Thread.current
-        sleep Config.response_timeout
+        if Config.response_timeout <= 0
+          Thread.stop
+        else
+          sleep Config.response_timeout
+        end
         # there is a race case where if the sleep finishes, 
         # and response comes in and has the mutex, and then array is frozen
         # it would be ideal to use monitor wait/signal here but the monitor implementation is currently flawed

data/lib/pandemic/server_side/server.rb

@@ -148,7 +148,7 @@ module Pandemic
       def signature
         "#{@host}:#{@port}"
       end
-      
+            
       def connection_statuses
         @servers.inject({}) do |statuses, server|
           if server == signature

data/lib/pandemic/util.rb

@@ -5,18 +5,7 @@ module Pandemic
     end
     
     def logger
-      $logger
-    end
-    
-    def bm(title, &block)
-      @times ||= Hash.new(0)
-      begin
-        start = Time.now.to_f
-        yield block
-      ensure
-        @times[title] += (Time.now.to_f - start)
-        $stdout.puts("#{title} #{@times[title]}")
-      end
+      $pandemic_logger
     end
     
     def with_mutex(obj)

data/pandemic.gemspec

@@ -2,30 +2,37 @@
 
 Gem::Specification.new do |s|
   s.name = %q{pandemic}
-  s.version = "0.2"
+  s.version = "0.2.1"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 1.2") if s.respond_to? :required_rubygems_version=
   s.authors = ["Arya Asemanfar"]
-  s.date = %q{2009-03-10}
+  s.date = %q{2009-03-26}
   s.description = %q{Distribute MapReduce to any of the workers and it will spread, like a pandemic.}
   s.email = %q{aryaasemanfar@gmail.com}
   s.extra_rdoc_files = ["lib/pandemic/client_side/cluster_connection.rb", "lib/pandemic/client_side/config.rb", "lib/pandemic/client_side/connection.rb", "lib/pandemic/client_side/connection_proxy.rb", "lib/pandemic/client_side/pandemize.rb", "lib/pandemic/connection_pool.rb", "lib/pandemic/mutex_counter.rb", "lib/pandemic/server_side/client.rb", "lib/pandemic/server_side/config.rb", "lib/pandemic/server_side/handler.rb", "lib/pandemic/server_side/peer.rb", "lib/pandemic/server_side/request.rb", "lib/pandemic/server_side/server.rb", "lib/pandemic/util.rb", "lib/pandemic.rb", "README.markdown"]
-  s.files = ["lib/pandemic/client_side/cluster_connection.rb", "lib/pandemic/client_side/config.rb", "lib/pandemic/client_side/connection.rb", "lib/pandemic/client_side/connection_proxy.rb", "lib/pandemic/client_side/pandemize.rb", "lib/pandemic/connection_pool.rb", "lib/pandemic/mutex_counter.rb", "lib/pandemic/server_side/client.rb", "lib/pandemic/server_side/config.rb", "lib/pandemic/server_side/handler.rb", "lib/pandemic/server_side/peer.rb", "lib/pandemic/server_side/request.rb", "lib/pandemic/server_side/server.rb", "lib/pandemic/util.rb", "lib/pandemic.rb", "Rakefile", "README.markdown", "Manifest", "pandemic.gemspec"]
+  s.files = ["lib/pandemic/client_side/cluster_connection.rb", "lib/pandemic/client_side/config.rb", "lib/pandemic/client_side/connection.rb", "lib/pandemic/client_side/connection_proxy.rb", "lib/pandemic/client_side/pandemize.rb", "lib/pandemic/connection_pool.rb", "lib/pandemic/mutex_counter.rb", "lib/pandemic/server_side/client.rb", "lib/pandemic/server_side/config.rb", "lib/pandemic/server_side/handler.rb", "lib/pandemic/server_side/peer.rb", "lib/pandemic/server_side/request.rb", "lib/pandemic/server_side/server.rb", "lib/pandemic/util.rb", "lib/pandemic.rb", "Manifest", "MIT-LICENSE", "pandemic.gemspec", "Rakefile", "README.markdown", "test/client_test.rb", "test/connection_pool_test.rb", "test/functional_test.rb", "test/handler_test.rb", "test/mutex_counter_test.rb", "test/peer_test.rb", "test/server_test.rb", "test/test_helper.rb", "test/util_test.rb"]
   s.has_rdoc = true
-  s.homepage = %q{}
+  s.homepage = %q{https://github.com/arya/pandemic/}
   s.rdoc_options = ["--line-numbers", "--inline-source", "--title", "Pandemic", "--main", "README.markdown"]
   s.require_paths = ["lib"]
   s.rubyforge_project = %q{pandemic}
   s.rubygems_version = %q{1.3.1}
   s.summary = %q{Distribute MapReduce to any of the workers and it will spread, like a pandemic.}
+  s.test_files = ["test/client_test.rb", "test/connection_pool_test.rb", "test/functional_test.rb", "test/handler_test.rb", "test/mutex_counter_test.rb", "test/peer_test.rb", "test/server_test.rb", "test/test_helper.rb", "test/util_test.rb"]
 
   if s.respond_to? :specification_version then
     current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
     s.specification_version = 2
 
     if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
+      s.add_development_dependency(%q<shoulda>, [">= 0"])
+      s.add_development_dependency(%q<mocha>, [">= 0"])
     else
+      s.add_dependency(%q<shoulda>, [">= 0"])
+      s.add_dependency(%q<mocha>, [">= 0"])
     end
   else
+    s.add_dependency(%q<shoulda>, [">= 0"])
+    s.add_dependency(%q<mocha>, [">= 0"])
   end
 end

data/test/client_test.rb

@@ -0,0 +1,80 @@
+require 'test_helper'
+
+class ClientTest < Test::Unit::TestCase
+  include TestHelper
+  
+  context "with a client object" do
+    setup do
+      @server = mock()
+      @server.expects(:running).returns(true).then.returns(false)
+      @connection = mock()
+      @connection.expects(:peeraddr).returns(['','','',''])
+      @connection.expects(:nil?).returns(false).at_least_once
+      @client = Pandemic::ServerSide::Client.new(@connection, @server)
+    end
+    
+    should "read size from the connection" do
+      @connection.expects(:gets).returns("5\n")
+      @server.expects(:client_closed).with(@client)
+      @client.listen
+      wait_for_threads
+    end
+    
+    should "read body from the connection" do
+      @connection.expects(:gets).returns("5\n")
+      @connection.expects(:read).with(5).returns("hello")
+      @server.expects(:client_closed).with(@client)
+      @client.listen    
+      wait_for_threads
+    end
+    
+    should "call handle request body on server" do
+      @connection.expects(:gets).returns("5\n")
+      @connection.expects(:read).with(5).returns("hello")
+      
+      request = mock()
+      Pandemic::ServerSide::Request.expects(:new).returns(request)
+      @server.expects(:handle_client_request).with(request)
+      
+      @server.expects(:client_closed).with(@client)
+      @client.listen    
+      wait_for_threads
+    end
+    
+    should "write response back to client" do
+      @connection.expects(:gets).returns("5\n")
+      @connection.expects(:read).with(5).returns("hello")
+      
+      request = mock()
+      response = "olleh"
+      Pandemic::ServerSide::Request.expects(:new).returns(request)
+      @server.expects(:handle_client_request).with(request).returns(response)
+      
+      @connection.expects(:write).with("5\n#{response}")
+      @connection.expects(:flush)
+      
+      @server.expects(:client_closed).with(@client)
+      @client.listen    
+      wait_for_threads
+    end
+    
+    should "close the connection on nil response" do
+      @connection.expects(:gets).returns(nil)
+      @connection.expects(:close)
+      
+      @server.expects(:client_closed).with(@client)
+      @client.listen
+      wait_for_threads
+    end
+    
+    should "close the connection on disconnect" do
+      @connection.expects(:gets).raises(Pandemic::ServerSide::Client::DisconnectClient)
+      @connection.expects(:closed?).returns(false)
+      @connection.expects(:close)
+      
+      @server.expects(:client_closed).with(@client)
+      @client.listen
+      wait_for_threads
+    end
+  end
+end

data/test/connection_pool_test.rb

@@ -0,0 +1,133 @@
+require 'test_helper'
+
+class ConnectionPoolTest < Test::Unit::TestCase
+  include TestHelper
+  context "without a create connection block" do
+    setup do
+      @connection_pool = Pandemic::ConnectionPool.new
+    end
+    
+    should "raise an exception when trying to connect" do
+      assert_raises Pandemic::ConnectionPool::CreateConnectionUndefinedException do
+        @connection_pool.connect
+      end
+    end
+  end
+  
+  context "with a connection pool" do
+    setup do
+      @connection_pool = Pandemic::ConnectionPool.new
+      @connection_creator = mock()
+      @connection_destroyer = mock()
+    end
+    
+    should "call create connection after its defined" do
+      @connection_creator.expects(:create).once
+      @connection_pool.create_connection do
+        @connection_creator.create
+      end
+    end
+    
+    should "call destroyer on disconnect" do
+      @connection_creator.expects(:create).once
+      @connection_destroyer.expects(:destroy).once
+      @connection_pool.create_connection do
+        @connection_creator.create
+        conn = mock()
+        conn.expects(:closed?).returns(false).at_least(0)
+        conn
+      end
+      
+      @connection_pool.destroy_connection do
+        @connection_destroyer.destroy
+      end
+      
+      @connection_pool.disconnect
+    end
+    
+  end
+  
+  context "with a max connections of 2" do
+    setup do
+      @connection_pool = Pandemic::ConnectionPool.new(:max_connections => 2, :timeout => 0.01)
+      @connection_creator = mock()
+    end
+    
+    should "raise timeout exception when no connections available" do
+      @connection_creator.expects(:create).twice
+      @connection_pool.create_connection do
+        @connection_creator.create
+        conn = mock()
+        conn.expects(:closed?).returns(false).at_least(0)
+        conn
+      end
+      
+      assert_raises Pandemic::ConnectionPool::TimedOutWaitingForConnectionException do
+        @connection_pool.with_connection do |conn1|
+          @connection_pool.with_connection do |conn2|
+            @connection_pool.with_connection do |conn3|
+              fail("there should only be two connections")
+            end
+          end
+        end
+      end
+    end
+    
+    should "should checkin a connection and allow someone check the same one out" do
+      @connection_creator.expects(:create).twice
+      @connection_pool.create_connection do
+        @connection_creator.create
+        conn = mock()
+        conn.expects(:closed?).returns(false).at_least(0)
+        conn
+      end
+      
+      @connection_pool.with_connection do |conn1|
+        conn2, conn3 = nil, nil
+        
+        @connection_pool.with_connection do |conn|
+          conn2 = conn
+        end
+        @connection_pool.with_connection do |conn|
+          conn3 = conn
+        end
+        
+        assert_equal conn2, conn3
+      end
+    end
+    
+    should "should checkin connection even if there is an exception" do
+      @connection_creator.expects(:create).once
+      @connection_pool.create_connection do
+        @connection_creator.create
+        conn = mock()
+        conn.expects(:closed?).returns(false).at_least(0)
+        conn
+      end
+      before = @connection_pool.available_count
+      begin
+        @connection_pool.with_connection do |conn1|
+          raise TestException
+        end
+      rescue TestException
+      end
+      after = @connection_pool.available_count
+      
+      assert_equal before, after
+    end
+  end
+  
+  context "with a min connections of 2" do
+    setup do
+      @connection_pool = Pandemic::ConnectionPool.new(:min_connections => 2)
+      @connection_creator = mock()
+    end
+    
+    should "call create connection twice" do
+      @connection_creator.expects(:create).twice
+      @connection_pool.create_connection do
+        @connection_creator.create
+      end
+    end
+  end
+end

data/test/functional_test.rb

@@ -0,0 +1,56 @@
+require 'test_helper'
+
+class FunctionalTest < Test::Unit::TestCase
+  include TestHelper
+  should "work" do
+    ignore_threads = Thread.list
+    ARGV.replace(["-i", "0", "-c", "test/pandemic_server.yml"]) # :(
+    Pandemic::ClientSide::Config.config_path = "test/pandemic_client.yml"
+    
+    server = epidemic!
+    server.handler = Class.new(Pandemic::ServerSide::Handler) do
+      def process(body)
+        body.reverse
+      end
+    end.new
+    server.start
+    
+    client = Class.new do
+      include Pandemize
+    end.new
+    client.extend(Pandemize)
+    assert_equal "dlrow olleh", client.pandemic.request("hello world")
+    server.stop
+    wait_for_threads(ignore_threads)
+  end
+  
+  should "work with multiple peers" do
+    ignore_threads = Thread.list
+    handler = Class.new(Pandemic::ServerSide::Handler) do
+      def process(body)
+        body.reverse
+      end
+    end.new
+    
+    ARGV.replace(["-i", "0", "-c", "test/pandemic_server.yml"]) # :(    
+    server = epidemic!
+    server.handler = handler
+    server.start
+    
+    ARGV.replace(["-i", "1", "-c", "test/pandemic_server.yml"]) # :(    
+    server2 = epidemic!
+    server2.handler = handler
+    server2.start
+    
+    Pandemic::ClientSide::Config.config_path = "test/pandemic_client.yml"
+    
+    client = Class.new do
+      include Pandemize
+    end.new
+    client.extend(Pandemize)
+    assert_equal "raboofraboof", client.pandemic.request("foobar")
+    server.stop
+    server2.stop
+    wait_for_threads(ignore_threads)
+  end
+end

data/test/handler_test.rb

@@ -0,0 +1,31 @@
+require 'test_helper'
+
+class HandlerTest < Test::Unit::TestCase
+  include TestHelper
+  
+  context "with a request object" do
+    setup do
+      @request = mock()
+      @servers = {
+        1 => :self,
+        2 => :disconnected,
+        3 => :connected
+      }
+      @handler = Pandemic::ServerSide::Handler.new
+    end
+    
+    should "concatenate all the results" do
+      @request.expects(:responses).once.returns(%w{1 2 3})
+      assert_equal "123", @handler.reduce(@request)
+    end
+    
+    should "map to all non-disconnected nodes" do
+      @request.expects(:body).twice.returns("123")
+      map = @handler.map(@request, @servers)
+      # see setup for @servers
+      assert_equal 2, map.size
+      assert_equal "123", map[1]
+      assert_equal "123", map[3]
+    end
+  end
+end

data/test/mutex_counter_test.rb

@@ -0,0 +1,46 @@
+require 'test_helper'
+
+class MutexCounterTest < Test::Unit::TestCase
+  context "with a new counter" do 
+    setup do
+      @counter = Pandemic::MutexCounter.new
+    end
+
+    should "start at 0" do
+      assert_equal 0, @counter.value
+    end
+
+    should "increment to 1 after one call to inc" do
+      assert_equal 0, @counter.value
+      assert_equal 1, @counter.inc
+      assert_equal 1, @counter.value
+    end
+    
+    should "be thread safe" do
+      # Not exactly a perfect test, but I'm not sure how to actually test 
+      # this without putting some code in the counter for this reason.
+      threads = []
+      5.times { threads << Thread.new { 100.times { @counter.inc }}}
+      threads.each {|t| t.join } # make sure they're all done
+      assert_equal 500, @counter.value
+    end
+  end
+    
+  context "with a max of 10" do
+    setup do
+      @counter = Pandemic::MutexCounter.new(10)
+    end
+    
+    should "cycle from 1 to 10" do
+      expected = (1..10).to_a + [1]
+      actual = (1..11).collect { @counter.inc }
+      assert_equal expected, actual
+    end
+    
+    should "have the correct 'real_total'" do
+      11.times { @counter.inc }
+      assert_equal 1, @counter.value
+      assert_equal 11, @counter.real_total
+    end
+  end
+end

data/test/peer_test.rb

@@ -0,0 +1,48 @@
+require 'test_helper'
+
+class PeerTest < Test::Unit::TestCase
+  include TestHelper
+  
+  should "initialize a new connection pool" do
+    connection_pool = mock()
+    Pandemic::ConnectionPool.expects(:new).returns(connection_pool)
+    connection_pool.expects(:create_connection)
+    
+    server = mock()
+    peer = Pandemic::ServerSide::Peer.new("localhost:4000", server)
+  end
+  
+  should "create a tcp socket" do
+    connection_pool = mock()
+    Pandemic::ConnectionPool.expects(:new).returns(connection_pool)
+    connection_pool.expects(:create_connection).yields
+    TCPSocket.expects(:new).with("localhost", 4000)
+    
+    server = mock()
+    peer = Pandemic::ServerSide::Peer.new("localhost:4000", server)
+  end
+  
+  context "with conn. pool" do
+    setup do
+      @connection_pool = mock()
+      Pandemic::ConnectionPool.expects(:new).returns(@connection_pool)
+      @connection_pool.expects(:create_connection)
+
+      @server = mock()
+      @peer = Pandemic::ServerSide::Peer.new("localhost:4000", @server)
+    end
+    
+    should "send client request to peer connection" do
+      request, body = stub(:hash => "asdasdfadsf"), "hello world"
+      @connection_pool.expects(:available_count => 1, :connections_count => 1)
+      conn = mock()
+      @connection_pool.expects(:with_connection).yields(conn)
+      
+      conn.stubs(:closed? => false)
+      conn.expects(:write).with("PROCESS #{request.hash} #{body.size}\n#{body}")
+      conn.expects(:flush)
+      
+      @peer.client_request(request, body)
+    end
+  end
+end

data/test/server_test.rb

@@ -0,0 +1,111 @@
+require 'test_helper'
+
+class ServerTest < Test::Unit::TestCase
+  include TestHelper
+  
+  should "initialize peers" do
+    Pandemic::ServerSide::Config.expects(:bind_to).at_least_once.returns("localhost:4000")
+    Pandemic::ServerSide::Config.expects(:servers).returns(["localhost:4000", "localhost:4001"])
+    Pandemic::ServerSide::Peer.expects(:new).with("localhost:4001", is_a(Pandemic::ServerSide::Server))
+    @server = Pandemic::ServerSide::Server.new    
+  end
+  
+  context "with a server" do
+    setup do
+      Pandemic::ServerSide::Config.expects(:bind_to).at_least_once.returns("localhost:4000")
+      Pandemic::ServerSide::Config.expects(:servers).returns(["localhost:4000", "localhost:4001"])
+      @peer = mock()
+      Pandemic::ServerSide::Peer.expects(:new).with("localhost:4001", is_a(Pandemic::ServerSide::Server)).returns(@peer)
+      @server = Pandemic::ServerSide::Server.new
+    end
+    
+    should "start a TCPServer, and connect to peers" do
+      ignore_threads = Thread.list
+      @tcpserver = mock()
+      TCPServer.expects(:new).with("localhost", 4000).returns(@tcpserver)
+      @peer.expects(:connect).once
+      @tcpserver.expects(:accept).twice.returns(nil).then.raises(Pandemic::ServerSide::Server::StopServer)
+      @tcpserver.expects(:close)
+      @peer.expects(:disconnect)
+      @server.handler = mock()
+      @server.start
+      wait_for_threads(ignore_threads)
+    end
+    
+    should "create a new client object for a client connection" do
+      ignore_threads = Thread.list
+      
+      @tcpserver = mock()
+      TCPServer.expects(:new).with("localhost", 4000).returns(@tcpserver)
+      @peer.expects(:connect).once
+      
+      @conn = mock(:peeraddr => ['','','',''])
+      @tcpserver.expects(:accept).twice.returns(@conn).then.raises(Pandemic::ServerSide::Server::StopServer)
+      client = mock()
+      @conn.expects(:setsockopt).with(Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1)
+      
+      Pandemic::ServerSide::Client.expects(:new).with(@conn, @server).returns(client)
+
+      
+      @conn.expects(:gets).returns("CLIENT\n")
+      @tcpserver.expects(:close)
+      @peer.expects(:disconnect)
+      client.expects(:listen).returns(client)
+      client.expects(:close).at_most_once # optional due to threaded nature, this may not actually happen
+      @server.handler = mock()
+      @server.start
+      wait_for_threads(ignore_threads)
+    end
+    
+    should "connect with the corresponding peer object" do
+      ignore_threads = Thread.list
+      @tcpserver = mock()
+      TCPServer.expects(:new).with("localhost", 4000).returns(@tcpserver)
+      @peer.expects(:connect).once
+      
+      @conn = mock(:peeraddr => ['','','',''])
+      @tcpserver.expects(:accept).twice.returns(@conn).then.raises(Pandemic::ServerSide::Server::StopServer)
+      @tcpserver.expects(:close)
+      @peer.expects(:disconnect)
+      
+      @conn.expects(:setsockopt).with(Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1)
+      @conn.expects(:gets).returns("SERVER localhost:4001\n")
+      
+      @peer.expects(:host).returns("localhost")
+      @peer.expects(:port).returns(4001)
+      @peer.expects(:add_incoming_connection).with(@conn)
+      @server.handler = mock()
+      
+      @server.start
+      wait_for_threads(ignore_threads)
+    end
+   
+    should "call process on handler" do
+      handler = mock()
+      handler.expects(:process).with("body")
+      @server.handler = handler
+      @server.process("body")
+    end
+    
+    should "map request, distribute to peers, and reduce" do
+      handler = mock()
+      request = mock()
+      request.expects(:hash).at_least_once.returns("abcddef134123")
+      @peer.expects(:connected?).returns(true)
+      handler.expects(:map).with(request, is_a(Hash)).returns({"localhost:4000" => "1", "localhost:4001" => "2"})
+      request.expects(:max_responses=).with(2)
+      @peer.expects(:client_request).with(request, "2")
+      
+      handler.expects(:process).with("1").returns("2")
+      request.expects(:add_response).with("2")
+      
+      request.expects(:wait_for_responses).once
+      handler.expects(:reduce).with(request)
+      
+      @server.handler = handler
+      
+      @server.handle_client_request(request)
+      # wait_for_threads
+    end
+  end
+end

data/test/test_helper.rb

@@ -0,0 +1,24 @@
+TEST_DIR = File.dirname(__FILE__)
+%w(lib test).each do |dir|
+  $LOAD_PATH.unshift "#{TEST_DIR}/../#{dir}"
+end
+
+require 'test/unit'
+require 'pandemic'
+require 'rubygems'
+require 'shoulda'
+require 'mocha'
+
+
+blackhole = StringIO.new
+$pandemic_logger = Logger.new(blackhole)
+
+module TestHelper
+  class TestException < Exception; end
+  def wait_for_threads(ignore = [Thread.current])
+    Thread.list.each do |thread|
+      next if ignore.include?(thread)
+      thread.join
+    end
+  end
+end

data/test/util_test.rb

@@ -0,0 +1,21 @@
+require 'test_helper'
+
+class UtilTest < Test::Unit::TestCase
+  context "with the module methods" do
+    setup do
+      @util = Object.new
+      @util.extend(Pandemic::Util)
+    end
+    
+    should "parse out host and port" do
+      assert_equal ["localhost", 4000], @util.host_port("localhost:4000")
+    end
+    
+    should "include the monitor mixin" do
+      object = Object.new
+      assert !object.respond_to?(:synchronize)
+      @util.with_mutex(object)
+      assert object.respond_to?(:synchronize)
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: arya-pandemic
 version: !ruby/object:Gem::Version 
-  version: "0.2"
+  version: 0.2.1
 platform: ruby
 authors: 
 - Arya Asemanfar
@@ -9,10 +9,29 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-03-10 00:00:00 -07:00
+date: 2009-03-26 00:00:00 -07:00
 default_executable: 
-dependencies: []
-
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: shoulda
+  type: :development
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: mocha
+  type: :development
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
 description: Distribute MapReduce to any of the workers and it will spread, like a pandemic.
 email: aryaasemanfar@gmail.com
 executables: []
@@ -52,12 +71,22 @@ files:
 - lib/pandemic/server_side/server.rb
 - lib/pandemic/util.rb
 - lib/pandemic.rb
-- Rakefile
-- README.markdown
 - Manifest
+- MIT-LICENSE
 - pandemic.gemspec
+- Rakefile
+- README.markdown
+- test/client_test.rb
+- test/connection_pool_test.rb
+- test/functional_test.rb
+- test/handler_test.rb
+- test/mutex_counter_test.rb
+- test/peer_test.rb
+- test/server_test.rb
+- test/test_helper.rb
+- test/util_test.rb
 has_rdoc: true
-homepage: ""
+homepage: https://github.com/arya/pandemic/
 post_install_message: 
 rdoc_options: 
 - --line-numbers
@@ -87,5 +116,13 @@ rubygems_version: 1.2.0
 signing_key: 
 specification_version: 2
 summary: Distribute MapReduce to any of the workers and it will spread, like a pandemic.
-test_files: []
-
+test_files: 
+- test/client_test.rb
+- test/connection_pool_test.rb
+- test/functional_test.rb
+- test/handler_test.rb
+- test/mutex_counter_test.rb
+- test/peer_test.rb
+- test/server_test.rb
+- test/test_helper.rb
+- test/util_test.rb