pandemic 0.5.4

data/lib/pandemic/server_side/processor.rb

@@ -0,0 +1,90 @@
+module Pandemic
+  module ServerSide
+    class Processor
+      def initialize(handler)
+        read_from_parent, write_to_child = IO.pipe
+        read_from_child, write_to_parent = IO.pipe
+        
+        @child_process_id = fork
+        if @child_process_id
+          # I'm the parent
+          write_to_parent.close
+          read_from_parent.close
+          @out = write_to_child
+          @in = read_from_child
+        else
+          # I'm the child
+          write_to_child.close
+          read_from_child.close
+          @out = write_to_parent
+          @in = read_from_parent
+          @handler = handler.new
+          wait_for_jobs
+        end
+      end
+      
+      def process(body)
+        if parent?
+          @out.puts(body.size.to_s)
+          @out.write(body)
+          ready, = IO.select([@in], nil, nil)
+          if ready
+            size = @in.gets.to_i
+            result = @in.read(size)
+            return result
+          end
+        else
+          return @handler.process(body)
+        end
+      end
+      
+      def close(status = 0)
+        if parent? && child_alive?
+          Process.detach(@child_process_id)
+          @out.puts(status.to_s)
+          @out.close
+          @in.close
+        else
+          Process.exit!(status)
+        end
+      end
+      
+      def closed?
+        !child_alive?
+      end
+      
+      private
+      def wait_for_jobs
+        if child?
+          while true
+            ready, = IO.select([@in], nil, nil)
+            if ready
+              size = @in.gets.to_i
+              if size > 0
+                body = @in.read(size)
+                result = process(body)
+                @out.puts(result.size.to_s)
+                @out.write(result)
+              else
+                self.close(size)
+                break
+              end
+            end
+          end
+        end
+      end
+      
+      def parent?
+        !!@child_process_id
+      end
+      
+      def child?
+        !parent?
+      end
+      
+      def child_alive?
+        parent? && !@in.closed?
+      end
+    end
+  end
+end

data/lib/pandemic/server_side/request.rb

@@ -0,0 +1,92 @@
+module Pandemic
+  module ServerSide
+    class Request    
+      include Util
+      
+      @@request_count = MutexCounter.new
+      @@late_responses = MutexCounter.new
+      attr_reader :body
+      attr_accessor :max_responses
+      attr_accessor :data
+      
+      def self.total_request_count
+        @@request_count.real_total
+      end
+      
+      def self.total_late_responses
+        @@late_responses.real_total
+      end
+      
+      def initialize(body)
+        @request_number = @@request_count.inc
+        @body = body
+        @responses = []
+        @responses_mutex = Monitor.new
+        @waiter = @responses_mutex.new_cond
+        @complete = false
+      end
+    
+      def add_response(response)
+        @responses_mutex.synchronize do
+          if @responses.frozen? # too late
+            @@late_responses.inc
+            return  
+          end
+          # debug("Adding response")
+          @responses << response
+          if @max_responses && @responses.size >= @max_responses
+            # debug("Hit max responses, waking up waiting thread")
+            wakeup_waiting_thread
+            @complete = true
+          end
+        end
+      end
+      
+      def wakeup_waiting_thread
+        @waiter.signal if @waiter
+      end
+    
+      def responses
+        @responses # its frozen so we don't have to worry about mutex
+      end
+      
+      def cancel!
+        # consider telling peers that they should stop, but for now this is fine.
+        @responses_mutex.synchronize do
+          wakeup_waiting_thread
+        end
+      end
+    
+      def wait_for_responses
+        @responses_mutex.synchronize do
+          return if @complete
+          if Config.response_timeout <= 0
+            @waiter.wait
+          elsif !MONITOR_TIMEOUT_AVAILABLE
+            Thread.new do
+              sleep Config.response_timeout
+              wakeup_waiting_thread
+            end
+            @waiter.wait
+          else
+            @waiter.wait(Config.response_timeout)
+          end
+          @responses.freeze
+        end
+      end
+      
+      def hash
+        @hash ||= Digest::MD5.hexdigest("#{@request_number} #{@body}")[0,10]
+      end
+      
+      private
+      def debug(msg)
+        logger.debug("Request #{hash}") {msg}
+      end
+      
+      def info(msg)
+        logger.info("Request #{hash}") {msg}
+      end
+    end
+  end
+end

data/lib/pandemic/server_side/server.rb

@@ -0,0 +1,285 @@
+module Pandemic
+  module ServerSide
+    class Server
+      include Util
+      class StopServer < Exception; end
+      class << self
+        def boot(bind_to = nil)
+          Config.load
+          # Process.setrlimit(Process::RLIMIT_NOFILE, 4096) # arbitrary high number of max file descriptors.
+          server = self.new(bind_to || Config.bind_to)
+          set_signal_traps(server)
+          server
+        end
+      
+        private
+        def set_signal_traps(server)
+          interrupt_tries = 0
+          Signal.trap(Signal.list["INT"]) do
+            interrupt_tries += 1
+            if interrupt_tries == 1
+              server.stop
+            else
+              exit
+            end
+          end
+        end
+      end
+      attr_reader :host, :port, :running
+      def initialize(bind_to)
+        write_pid_file
+        
+        @host, @port = host_port(bind_to)
+        @clients = []
+        @total_clients = 0
+        @clients_mutex = Mutex.new
+        @num_jobs_processed = MutexCounter.new
+        @num_jobs_entered = MutexCounter.new
+        @requests_per_second = RequestsPerSecond.new(10)
+        
+        @peers = with_mutex({})
+        @servers = Config.servers
+        @servers.each do |peer|
+          next if peer == bind_to # not a peer, it's itself
+          @peers[peer] = Peer.new(peer, self)
+        end
+      end
+      
+      def handler=(handler)
+        @handler = handler
+        @handler_instance = handler.new
+      end
+    
+      def start
+        raise "You must specify a handler" unless @handler
+        
+        @listener = TCPServer.new(@host, @port)
+        @running = true
+        @running_since = Time.now
+        
+        # debug("Connecting to peers")
+        @peers.values.each { |peer| peer.connect }
+
+        @listener_thread = Thread.new do
+          begin
+            while @running
+              begin
+                # debug("Listening")
+                conn = @listener.accept
+                Thread.new(conn) { |c| handle_connection(c) }
+              rescue Errno::ECONNABORTED, Errno::EINTR # TODO: what else can wrong here? this should be more robust.
+                debug("Connection accepted aborted")
+                conn.close if conn && !conn.closed?
+              end
+            end
+          rescue StopServer
+            info("Stopping server")
+            remove_pid_file
+            @listener.close if @listener
+            @peers.values.each { |p| p.disconnect }
+            @clients.each {|c| c.close }
+            self.processor.disconnect if Config.fork_for_processor
+          rescue Exception => e
+            warn("Unhandled exception in server listening thread:\n#{e.inspect}\n#{e.backtrace.join("\n")}")
+          end
+        end
+      end
+    
+      def stop
+        @running = false
+        @listener_thread.raise(StopServer)
+      end
+    
+      def handle_connection(connection)
+        begin
+          connection.setsockopt(Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1) if TCP_NO_DELAY_AVAILABLE
+        
+          identification = connection.gets.strip
+          # info("Incoming connection from #{connection.peeraddr.values_at(3,1).join(":")} (#{identification})")
+          if identification =~ /^SERVER ([a-zA-Z0-9.]+:[0-9]+)$/
+            # debug("Recognized as peer")
+            host, port = host_port($1)
+            matching_peer = @peers.values.detect { |peer| [peer.host, peer.port] == [host, port] }
+            if matching_peer
+              # debug("Found matching peer")
+            else
+              # debug("Didn't find matching peer, adding it")
+              matching_peer = @peers.synchronize do
+                hostport = "#{host}:#{port}"
+                @servers.push(hostport) unless @servers.include?(hostport)
+                @peers[hostport] ||= Peer.new(hostport, self)
+              end
+            end
+            matching_peer.add_incoming_connection(connection)
+          elsif identification =~ /^CLIENT$/
+            # debug("Recognized as client")
+            @clients_mutex.synchronize do
+              @clients << Client.new(connection, self).listen
+              @total_clients += 1
+            end
+          elsif identification =~ /^stats$/
+            # debug("Stats request received")
+            print_stats(connection)
+          else
+            debug("Unrecognized connection. Closing.")
+            connection.close # i dunno you
+          end
+        rescue Exception => e
+          warn("Unhandled exception in handle connection method:\n#{e.inspect}\n#{e.backtrace.join("\n")}")
+        end
+      end
+    
+      def handle_client_request(request)
+        # info("Handling client request")
+        map = @handler_instance.partition(request, connection_statuses)
+        request.max_responses = map.size
+        # debug("Sending client request to #{map.size} handlers (#{request.hash})")
+        
+        map.each do |peer, body|
+          if @peers[peer]
+            @peers[peer].client_request(request, body)
+          end
+        end
+        
+        if map[signature]
+          # debug("Processing #{request.hash}")
+          Thread.new do
+            begin
+              request.add_response(self.process(map[signature]))
+            rescue Exception => e
+              warn("Unhandled exception in local processing:\n#{e.inspect}#{e.backtrace.join("\n")}}")
+            end
+          end
+        end
+        
+        @requests_per_second.hit
+        
+        # debug("Waiting for responses")
+        request.wait_for_responses
+        
+        # debug("Done waiting for responses, calling reduce")
+        @handler_instance.reduce(request)
+      end
+    
+      def process(body)
+        @num_jobs_entered.inc
+        response = if Config.fork_for_processor
+          self.processor.with_connection {|con| con.process(body) } 
+        else
+          @handler_instance.process(body)
+        end
+        @num_jobs_processed.inc
+        response
+      end
+    
+      def signature
+        @signature ||= "#{@host}:#{@port}"
+      end
+            
+      def connection_statuses
+        @servers.inject({}) do |statuses, server|
+          if server == signature
+            statuses[server] = :self
+          else
+            statuses[server] = @peers[server].connected? ? :connected : :disconnected
+          end
+          statuses
+        end
+      end
+      
+      def client_closed(client)
+        @clients_mutex.synchronize do
+          @clients.delete(client)
+        end
+      end
+      
+      def processor
+        @processor ||= begin
+          processor = ConnectionPool.new
+          processor.create_connection { Processor.new(@handler) }
+          processor
+        end
+      end
+      
+      private
+      def print_stats(connection)
+        begin
+          stats = collect_stats
+          str = []
+          str << "Uptime: #{stats[:uptime]}"
+          str << "Number of Threads: #{stats[:num_threads]}"
+          str << "Connected Clients: #{stats[:connected_clients]}"
+          str << "Clients Ever: #{stats[:total_clients]}"
+          str << "Connected Peers: #{stats[:connected_peers]}"
+          str << "Disconnected Peers: #{stats[:disconnected_peers]}"
+          str << "Total Requests: #{stats[:num_requests]}"
+          str << "Pending Requests: #{stats[:pending_requests]}"
+          str << "Late Responses: #{stats[:late_responses]}"
+          str << "Total Jobs Processed: #{stats[:total_jobs_processed]}"
+          str << "Pending Jobs: #{stats[:jobs_pending]}"
+          str << "Requests per Second (10 sec): #{"%.1f" % stats[:rps_10]}"
+          str << "Requests per Second (lifetime): #{"%.1f" % stats[:rps_lifetime]}"
+          connection.puts(str.join("\n"))
+        end while (s = connection.gets) && (s.strip == "stats" || s.strip == "")
+        connection.close if connection && !connection.closed?
+      end
+      
+      def debug(msg)
+        logger.debug("Server #{signature}") {msg}
+      end
+      
+      def info(msg)
+        logger.info("Server #{signature}") {msg}
+      end
+      
+      def warn(msg)
+        logger.warn("Server #{signature}") {msg}
+      end
+      
+      def collect_stats
+        results = {}
+        results[:num_threads] = Thread.list.size
+        results[:connected_clients], results[:total_clients] = \
+            @clients_mutex.synchronize { [@clients.size, @total_clients] }
+            
+        results[:connected_peers], results[:disconnected_peers] = \
+            connection_statuses.inject([0,0]) do |counts, (server,status)|              
+              if status == :connected
+                counts[0] += 1
+              elsif status == :disconnected
+                counts[1] += 1
+              end
+              counts
+            end
+        results[:total_jobs_processed] = @num_jobs_processed.to_i
+        results[:jobs_pending] = @num_jobs_entered.to_i - results[:total_jobs_processed]
+        results[:num_requests] = Request.total_request_count
+        results[:late_responses] = Request.total_late_responses
+        results[:pending_requests] = @clients_mutex.synchronize do
+          @clients.inject(0) do |pending, client|
+            pending + (client.received_requests - client.responded_requests.to_i)
+          end
+        end
+        results[:rps_10] = @requests_per_second.rps
+        
+        results[:uptime] = Time.now - @running_since
+        results[:rps_lifetime] = results[:num_requests] / results[:uptime]
+        
+        results
+      end
+      
+      def remove_pid_file
+        File.unlink(Config.pid_file) if Config.pid_file && File.exists?(Config.pid_file)
+      end
+
+      def write_pid_file
+        return if Config.pid_file.nil?
+        File.open(Config.pid_file,"w") do |f|
+          f.write(Process.pid)
+          File.chmod(0644, Config.pid_file)
+        end      
+      end
+      
+    end
+  end
+end