arya-pandemic 0.4.1 → 0.4.3

data/Manifest

@@ -10,6 +10,7 @@ lib/pandemic/client_side/connection_proxy.rb
 lib/pandemic/client_side/pandemize.rb
 lib/pandemic/connection_pool.rb
 lib/pandemic/mutex_counter.rb
+lib/pandemic/requests_per_second.rb
 lib/pandemic/server_side/client.rb
 lib/pandemic/server_side/config.rb
 lib/pandemic/server_side/handler.rb

data/README.markdown

@@ -119,6 +119,10 @@ The servers are going to try to bind to localhost:4000 and localhost:4001 so mak
 By default, the handler runs in the same Ruby process as Pandemic. By setting the fork\_for\_processor to true in pandemic\_server.yml, you can have Pandemic fork to new processes to run the process method. This is particularly useful when your process method goes to MySQL which locks the entire process until MySQL returns.
 
 ## Change History
+**Version 0.4.2**
+
+ * Log file and log level now specify-able as hash attributes to the _epidemic!_ method.
+
 **Version 0.3.1**
 
  * Changed map to partition to more accurately reflect what it does. This breaks backwards compatibility, but all you have to do is rename your method.

data/Rakefile

@@ -2,7 +2,7 @@ require 'rubygems'
 require 'rake'
 require 'echoe'
 
-Echoe.new('pandemic', '0.4.1') do |p|
+Echoe.new('pandemic', '0.4.3') do |p|
   p.description    = "A framework for distributing work for real-time services and offline tasks."
   p.url            = "https://github.com/arya/pandemic/"
   p.author         = "Arya Asemanfar"

data/lib/pandemic.rb

@@ -10,6 +10,7 @@ require 'logger'
 require 'pandemic/util'
 require 'pandemic/connection_pool'
 require 'pandemic/mutex_counter'
+require 'pandemic/requests_per_second'
 
 require 'pandemic/server_side/config'
 require 'pandemic/server_side/client'
@@ -25,22 +26,16 @@ require 'pandemic/client_side/connection'
 require 'pandemic/client_side/connection_proxy'
 require 'pandemic/client_side/pandemize'
 
-# TODO:
-# - tests
-# - IO timeouts/robustness
-# - documentation
-# - PING/PONG?
-
 TCP_NO_DELAY_AVAILABLE =
     RUBY_VERSION < '1.9' ? Socket.constants.include?('TCP_NODELAY') : Socket.constants.include?(:TCP_NODELAY)
 
-def epidemic!(bind_to = nil)
+def epidemic!(options = {})
   if $pandemic_logger.nil?
-    $pandemic_logger = Logger.new("pandemic.log")
-    $pandemic_logger.level = Logger::INFO
+    $pandemic_logger = Logger.new(options[:log_file] || "pandemic.log")
+    $pandemic_logger.level = options[:log_level] || Logger::INFO
     $pandemic_logger.datetime_format = "%Y-%m-%d %H:%M:%S "
   end
-  Pandemic::ServerSide::Server.boot(bind_to)
+  Pandemic::ServerSide::Server.boot(options[:bind_to])
 end
 
 ::Pandemize = Pandemic::ClientSide::Pandemize

data/lib/pandemic/requests_per_second.rb

@@ -0,0 +1,31 @@
+module Pandemic
+  class RequestsPerSecond
+    def initialize(sample_size = 10)
+      @hits = Array.new(sample_size + 2)
+      @last_hit_at = nil
+    end
+
+    def hit(now = Time.now.to_i)
+      key = now % @hits.size
+      if @hits[key].nil? || @hits[key][0] != now
+        @hits[key] = [now, 0]
+      end
+      @hits[key][1] += 1
+      @last_hit_at = now
+    end
+
+    def rps(now = Time.now.to_i)
+      return 0 if @last_hit_at.nil?
+      entries_to_go_back = @hits.size - (now - @last_hit_at) - 2
+      return 0 if entries_to_go_back <= 0
+      sum = 0
+      entries_to_go_back.times do |i|
+        now -= 1 
+        if @hits[now % @hits.size] && @hits[now % @hits.size][0] == now
+          sum += @hits[now % @hits.size][1] 
+        end
+      end
+      return sum.to_f / (@hits.size - 2)
+    end
+  end
+end

data/lib/pandemic/server_side/server.rb

@@ -33,8 +33,9 @@ module Pandemic
         @clients_mutex = Mutex.new
         @num_jobs_processed = MutexCounter.new
         @num_jobs_entered = MutexCounter.new
+        @requests_per_second = RequestsPerSecond.new(10)
         
-        @peers = {}
+        @peers = with_mutex({})
         @servers = Config.servers
         @servers.each do |peer|
           next if peer == bind_to # not a peer, it's itself
@@ -64,7 +65,7 @@ module Pandemic
                 debug("Listening")
                 conn = @listener.accept
                 Thread.new(conn) { |c| handle_connection(c) }
-              rescue Errno::ECONNABORTED, Errno::EINTR 
+              rescue Errno::ECONNABORTED, Errno::EINTR # TODO: what else can wrong here? this should be more robust.
                 debug("Connection accepted aborted")
                 conn.close if conn && !conn.closed?
               end
@@ -96,8 +97,17 @@ module Pandemic
             debug("Recognized as peer")
             host, port = host_port($1)
             matching_peer = @peers.values.detect { |peer| [peer.host, peer.port] == [host, port] }
-            debug("Found matching peer")
-            matching_peer.add_incoming_connection(connection) unless matching_peer.nil?
+            if matching_peer
+              debug("Found matching peer")
+            else
+              debug("Didn't find matching peer, adding it")
+              matching_peer = @peers.synchronize do
+                hostport = "#{host}:#{port}"
+                @servers.push(hostport) unless @servers.include?(hostport)
+                @peers[hostport] ||= Peer.new(hostport, self)
+              end
+            end
+            matching_peer.add_incoming_connection(connection)
           elsif identification =~ /^CLIENT$/
             debug("Recognized as client")
             @clients_mutex.synchronize do
@@ -139,6 +149,8 @@ module Pandemic
           end
         end
         
+        @requests_per_second.hit
+        
         debug("Waiting for responses")
         request.wait_for_responses
         
@@ -202,6 +214,8 @@ module Pandemic
           str << "Late Responses: #{stats[:late_responses]}"
           str << "Total Jobs Processed: #{stats[:total_jobs_processed]}"
           str << "Pending Jobs: #{stats[:jobs_pending]}"
+          str << "Requests per Second (10 sec): #{"%.1f" % stats[:rps_10]}"
+          str << "Requests per Second (lifetime): #{"%.1f" % stats[:rps_lifetime]}"
           connection.puts(str.join("\n"))
         end while (s = connection.gets) && (s.strip == "stats" || s.strip == "")
         connection.close if connection && !connection.closed?
@@ -243,8 +257,11 @@ module Pandemic
             pending + (client.received_requests - client.responded_requests.to_i)
           end
         end
+        results[:rps_10] = @requests_per_second.rps
         
         results[:uptime] = Time.now - @running_since
+        results[:rps_lifetime] = results[:num_requests] / results[:uptime]
+        
         results
       end
       

data/pandemic.gemspec

@@ -2,15 +2,15 @@
 
 Gem::Specification.new do |s|
   s.name = %q{pandemic}
-  s.version = "0.4.1"
+  s.version = "0.4.3"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 1.2") if s.respond_to? :required_rubygems_version=
   s.authors = ["Arya Asemanfar"]
-  s.date = %q{2009-07-12}
+  s.date = %q{2009-07-20}
   s.description = %q{A framework for distributing work for real-time services and offline tasks.}
   s.email = %q{aryaasemanfar@gmail.com}
-  s.extra_rdoc_files = ["lib/pandemic/client_side/cluster_connection.rb", "lib/pandemic/client_side/config.rb", "lib/pandemic/client_side/connection.rb", "lib/pandemic/client_side/connection_proxy.rb", "lib/pandemic/client_side/pandemize.rb", "lib/pandemic/connection_pool.rb", "lib/pandemic/mutex_counter.rb", "lib/pandemic/server_side/client.rb", "lib/pandemic/server_side/config.rb", "lib/pandemic/server_side/handler.rb", "lib/pandemic/server_side/peer.rb", "lib/pandemic/server_side/processor.rb", "lib/pandemic/server_side/request.rb", "lib/pandemic/server_side/server.rb", "lib/pandemic/util.rb", "lib/pandemic.rb", "README.markdown"]
-  s.files = ["examples/client/client.rb", "examples/client/constitution.txt", "examples/client/pandemic_client.yml", "examples/server/pandemic_server.yml", "examples/server/word_count_server.rb", "lib/pandemic/client_side/cluster_connection.rb", "lib/pandemic/client_side/config.rb", "lib/pandemic/client_side/connection.rb", "lib/pandemic/client_side/connection_proxy.rb", "lib/pandemic/client_side/pandemize.rb", "lib/pandemic/connection_pool.rb", "lib/pandemic/mutex_counter.rb", "lib/pandemic/server_side/client.rb", "lib/pandemic/server_side/config.rb", "lib/pandemic/server_side/handler.rb", "lib/pandemic/server_side/peer.rb", "lib/pandemic/server_side/processor.rb", "lib/pandemic/server_side/request.rb", "lib/pandemic/server_side/server.rb", "lib/pandemic/util.rb", "lib/pandemic.rb", "Manifest", "MIT-LICENSE", "pandemic.gemspec", "Rakefile", "README.markdown", "test/client_test.rb", "test/connection_pool_test.rb", "test/functional_test.rb", "test/handler_test.rb", "test/mutex_counter_test.rb", "test/peer_test.rb", "test/processor_test.rb", "test/server_test.rb", "test/test_helper.rb", "test/util_test.rb"]
+  s.extra_rdoc_files = ["lib/pandemic/client_side/cluster_connection.rb", "lib/pandemic/client_side/config.rb", "lib/pandemic/client_side/connection.rb", "lib/pandemic/client_side/connection_proxy.rb", "lib/pandemic/client_side/pandemize.rb", "lib/pandemic/connection_pool.rb", "lib/pandemic/mutex_counter.rb", "lib/pandemic/requests_per_second.rb", "lib/pandemic/server_side/client.rb", "lib/pandemic/server_side/config.rb", "lib/pandemic/server_side/handler.rb", "lib/pandemic/server_side/peer.rb", "lib/pandemic/server_side/processor.rb", "lib/pandemic/server_side/request.rb", "lib/pandemic/server_side/server.rb", "lib/pandemic/util.rb", "lib/pandemic.rb", "README.markdown"]
+  s.files = ["examples/client/client.rb", "examples/client/constitution.txt", "examples/client/pandemic_client.yml", "examples/server/pandemic_server.yml", "examples/server/word_count_server.rb", "lib/pandemic/client_side/cluster_connection.rb", "lib/pandemic/client_side/config.rb", "lib/pandemic/client_side/connection.rb", "lib/pandemic/client_side/connection_proxy.rb", "lib/pandemic/client_side/pandemize.rb", "lib/pandemic/connection_pool.rb", "lib/pandemic/mutex_counter.rb", "lib/pandemic/requests_per_second.rb", "lib/pandemic/server_side/client.rb", "lib/pandemic/server_side/config.rb", "lib/pandemic/server_side/handler.rb", "lib/pandemic/server_side/peer.rb", "lib/pandemic/server_side/processor.rb", "lib/pandemic/server_side/request.rb", "lib/pandemic/server_side/server.rb", "lib/pandemic/util.rb", "lib/pandemic.rb", "Manifest", "MIT-LICENSE", "pandemic.gemspec", "Rakefile", "README.markdown", "test/client_test.rb", "test/connection_pool_test.rb", "test/functional_test.rb", "test/handler_test.rb", "test/mutex_counter_test.rb", "test/peer_test.rb", "test/processor_test.rb", "test/server_test.rb", "test/test_helper.rb", "test/util_test.rb"]
   s.homepage = %q{https://github.com/arya/pandemic/}
   s.rdoc_options = ["--line-numbers", "--inline-source", "--title", "Pandemic", "--main", "README.markdown"]
   s.require_paths = ["lib"]

data/test/server_test.rb

@@ -3,6 +3,33 @@ require 'test_helper'
 class ServerTest < Test::Unit::TestCase
   include TestHelper
   
+  context "without a peer list" do
+    setup do
+      Pandemic::ServerSide::Config.expects(:servers).returns(["localhost:4000"])
+      @server = Pandemic::ServerSide::Server.new("localhost:4000")
+    end
+    
+    should "create a peer connection when a unknown peer tries to connect" do
+      ignore_threads = Thread.list
+      @tcpserver = mock()
+      TCPServer.expects(:new).with("localhost", 4000).returns(@tcpserver)
+      
+      @conn = mock(:peeraddr => ['','','',''])
+      @tcpserver.expects(:accept).twice.returns(@conn).then.raises(Pandemic::ServerSide::Server::StopServer)
+      peer = mock()
+      @conn.expects(:setsockopt).with(Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1)
+      
+      @conn.expects(:gets).returns("SERVER localhost:4001\n")
+      Pandemic::ServerSide::Peer.expects(:new).with("localhost:4001", is_a(Pandemic::ServerSide::Server)).returns(peer)
+      @tcpserver.expects(:close)
+      peer.stubs(:disconnect) # the StopServer signal could be called before the peer is added. race case not important for this test
+      
+      @server.handler = mock(:new)
+      @server.start
+      wait_for_threads(ignore_threads)
+    end
+  end
+  
   should "initialize peers" do
     Pandemic::ServerSide::Config.expects(:servers).returns(["localhost:4000", "localhost:4001"])
     Pandemic::ServerSide::Peer.expects(:new).with("localhost:4001", is_a(Pandemic::ServerSide::Server))

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: arya-pandemic
 version: !ruby/object:Gem::Version 
-  version: 0.4.1
+  version: 0.4.3
 platform: ruby
 authors: 
 - Arya Asemanfar
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-07-12 00:00:00 -07:00
+date: 2009-07-20 00:00:00 -07:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -46,6 +46,7 @@ extra_rdoc_files:
 - lib/pandemic/client_side/pandemize.rb
 - lib/pandemic/connection_pool.rb
 - lib/pandemic/mutex_counter.rb
+- lib/pandemic/requests_per_second.rb
 - lib/pandemic/server_side/client.rb
 - lib/pandemic/server_side/config.rb
 - lib/pandemic/server_side/handler.rb
@@ -69,6 +70,7 @@ files:
 - lib/pandemic/client_side/pandemize.rb
 - lib/pandemic/connection_pool.rb
 - lib/pandemic/mutex_counter.rb
+- lib/pandemic/requests_per_second.rb
 - lib/pandemic/server_side/client.rb
 - lib/pandemic/server_side/config.rb
 - lib/pandemic/server_side/handler.rb