packet 0.1.0 → 0.1.1

data/lib/nbio.rb

@@ -1,13 +1,26 @@
 module Packet
   module NbioHelper
-    # nonblocking method of reading data
-    # when method returns nil it probably means that client diconnected
+    def packet_classify(original_string)
+      word_parts = original_string.split('_')
+      return word_parts.map { |x| x.capitalize}.join
+    end
+
+    def gen_worker_key(worker_name,job_key = nil)
+      return worker_name if job_key.nil?
+      return "#{worker_name}_#{job_key}".to_sym
+    end
+
     def read_data(t_sock)
       sock_data = ""
       begin
-        while(sock_data << t_sock.read_nonblock(1023)); end
+        while(t_data = t_sock.recv_nonblock(1023))
+          raise DisconnectError.new(t_sock) if t_data.empty?
+          sock_data << t_data
+        end
       rescue Errno::EAGAIN
         return sock_data
+      rescue Errno::EWOULDBLOCK
+        return sock_data
       rescue
         raise DisconnectError.new(t_sock)
       end
@@ -25,19 +38,9 @@ module Packet
         p_sock.write_nonblock(t_data)
       rescue Errno::EAGAIN
         return
+      rescue Errno::EPIPE
+        raise DisconnectError.new(p_sock)
       end
-
-#       loop do
-#         begin
-#           written_length = p_sock.write_nonblock(t_data)
-#         rescue Errno::EAGAIN
-#           break
-#         end
-#         break if written_length >= t_length
-#         t_data = t_data[written_length..-1]
-#         break if t_data.empty?
-#         t_length = t_data.length
-#       end
     end
 
     # method writes data to socket in a non blocking manner, but doesn't care if there is a error writing data
@@ -47,6 +50,8 @@ module Packet
         p_sock.write_nonblock(t_data)
       rescue Errno::EAGAIN
         return
+      rescue Errno::EPIPE
+        raise DisconnectError.new(p_sock)
       end
     end
 
@@ -56,26 +61,18 @@ module Packet
       dump_length = object_dump.length.to_s
       length_str = dump_length.rjust(9,'0')
       final_data = length_str + object_dump
-
-#       total_length = final_data.length
-#       loop do
-#         begin
-#           written_length = p_sock.write_nonblock(final_data)
-#         rescue Errno::EAGAIN
-#           break
-#         end
-#         break if written_length >= total_length
-#         final_data = final_data[written_length..-1]
-#         break if final_data.empty?
-#         total_length = final_data.length
-#       end
-
       begin
         p_sock.write_nonblock(final_data)
       rescue Errno::EAGAIN
+        puts "EAGAIN Error while writing socket"
         return
+      rescue Errno::EINTR
+        puts "Interrupt error"
+        return
+      rescue Errno::EPIPE
+        puts "Pipe error"
+        raise DisconnectError.new(p_sock)
       end
     end
-
   end
 end

data/lib/packet.rb

@@ -1,15 +1,15 @@
 require "socket"
 require "yaml"
 require "forwardable"
-require "attribute_accessors"
-require "buftok"
+require "ostruct"
+require "thread"
+
 require "bin_parser"
 
-require "ostruct"
-require "socket"
 
 require "packet_guid"
-require "ruby_hacks"
+require "class_helpers"
+require "thread_pool"
 require "double_keyed_hash"
 require "event"
 
@@ -33,5 +33,5 @@ require "worker"
 PACKET_APP = File.expand_path'../' unless defined?(PACKET_APP)
 
 module Packet
-  VERSION='0.1.0'
+  VERSION='0.1.1'
 end

data/lib/packet_master.rb

@@ -1,64 +1,58 @@
-# FIXME: Some code is duplicated between worker class and this Reactor class, that can be fixed
-# with help of creation of Connection class and enabling automatic inheritance of that class and
-# mixing in of methods from that class.
 module Packet
   class Reactor
     include Core
+    #set_thread_pool_size(20)
     attr_accessor :fd_writers, :msg_writers,:msg_reader
+    attr_accessor :result_hash
+
     attr_accessor :live_workers
     after_connection :provide_workers
 
+    def self.server_logger= (log_file_name)
+      @@server_logger = log_file_name
+    end
+
     def self.run
       master_reactor_instance = new
+      # master_reactor_instance.result_hash = {}
       master_reactor_instance.live_workers = DoubleKeyedHash.new
       yield(master_reactor_instance)
       master_reactor_instance.load_workers
       master_reactor_instance.start_reactor
     end # end of run method
 
+    def set_result_hash(hash)
+      @result_hash = hash
+    end
+
+    def update_result(worker_key,result)
+      @result_hash ||= {}
+      @result_hash[worker_key.to_sym] = result
+    end
+
     def provide_workers(handler_instance,t_sock)
       class << handler_instance
         extend Forwardable
         attr_accessor :workers,:connection,:reactor, :initialized,:signature
+        attr_accessor :thread_pool
         include NbioHelper
-        def send_data p_data
-          begin
-            write_data(p_data,connection)
-          rescue Errno::EPIPE
-            # probably a callback, when there is a error in writing to the socket
-          end
-        end
-        def invoke_init
-          @initialized = true
-          post_init
-        end
-
-        def close_connection
-          unbind
-          reactor.remove_connection(connection)
-        end
-
-        def close_connection_after_writing
-          connection.flush
-          unbind
-          reactor.remove_connection(connection)
-        end
-
+        include Connection
         def ask_worker(*args)
           worker_name = args.shift
           data_options = *args
+          worker_name_key = gen_worker_key(worker_name,data_options[:job_key])
           data_options[:client_signature] = connection.fileno
-          workers[worker_name].send_request(data_options)
+          reactor.live_workers[worker_name_key].send_request(data_options)
         end
 
-        def send_object p_object
-          dump_object(p_object,connection)
-        end
-        def_delegators :@reactor, :start_server, :connect, :add_periodic_timer, :add_timer, :cancel_timer,:reconnect, :start_worker
+        def_delegators(:@reactor, :start_server, :connect, :add_periodic_timer, \
+                         :add_timer, :cancel_timer,:reconnect, :start_worker,:delete_worker)
+
       end
       handler_instance.workers = @live_workers
       handler_instance.connection = t_sock
       handler_instance.reactor = self
+      handler_instance.thread_pool = @thread_pool
     end
 
     # FIXME: right now, each worker is tied to its connection and this can be problematic
@@ -75,41 +69,42 @@ module Packet
       end
     end
 
-    # method loads workers in new processes
-    # FIXME: this method can be fixed, so as worker code can be actually, required
-    # only in forked process and hence saving upon the memory involved
-    # where worker is actually required in master as well as in worker.
-    def load_workers
+    def delete_worker(worker_options = {})
+      worker_name = worker_options[:worker]
+      worker_name_key = gen_worker_key(worker_name,worker_options[:job_key])
+      worker_options[:method] = :exit
+      @live_workers[worker_name_key].send_request(worker_options)
+    end
 
+    def load_workers
       if defined?(WORKER_ROOT)
         worker_root = WORKER_ROOT
       else
         worker_root = "#{PACKET_APP}/worker"
       end
       t_workers = Dir["#{worker_root}/**/*.rb"]
-      return if t_workers.blank?
+      return if t_workers.empty?
       t_workers.each do |b_worker|
         worker_name = File.basename(b_worker,".rb")
         require worker_name
-        worker_klass = Object.const_get(worker_name.classify)
+        worker_klass = Object.const_get(packet_classify(worker_name))
         next if worker_klass.no_auto_load
         fork_and_load(worker_klass)
       end
-
-      # FIXME: easiest and yet perhaps a bit ugly, its just to make sure that from each
-      # worker proxy one can access other workers
-      @live_workers.each do |key,worker_instance|
-        worker_instance.workers = @live_workers
-      end
     end
 
-    def start_worker(worker_name,options = {})
-      require worker_name.to_s
-      worker_klass = Object.const_get(worker_name.classify)
-      fork_and_load(worker_klass,options)
+    def start_worker(worker_options = { })
+      worker_name = worker_options[:worker].to_s
+      worker_name_key = gen_worker_key(worker_name,worker_options[:job_key])
+      return if @live_workers[worker_name_key]
+      worker_options.delete(:worker)
+      require worker_name
+      worker_klass = Object.const_get(packet_classify(worker_name))
+      fork_and_load(worker_klass,worker_options)
     end
 
     # method forks given worker file in a new process
+    # method should use job_key if provided in options hash.
     def fork_and_load(worker_klass,worker_options = { })
       t_worker_name = worker_klass.worker_name
       worker_pimp = worker_klass.worker_proxy.to_s
@@ -122,20 +117,23 @@ module Packet
 
       if((pid = fork()).nil?)
         $0 = "ruby #{worker_klass.worker_name}"
-        master_write_end.close
-        master_read_end.close
-        master_write_fd.close
-        # master_write_end.close if master_write_end
-        worker_klass.start_worker(:write_end => worker_write_end,:read_end => worker_read_end,:read_fd => worker_read_fd,:options => worker_options)
+        [master_write_end,master_read_end,master_write_fd].each { |x| x.close }
+
+        worker_klass.start_worker(:write_end => worker_write_end,:read_end => worker_read_end,\
+                                  :read_fd => worker_read_fd,:options => worker_options)
       end
       Process.detach(pid)
 
-      unless worker_pimp.blank?
+      worker_name_key = gen_worker_key(t_worker_name,worker_options[:job_key])
+
+      if worker_pimp && !worker_pimp.empty?
         require worker_pimp
-        pimp_klass = Object.const_get(worker_pimp.classify)
-        @live_workers[t_worker_name,master_read_end.fileno] = pimp_klass.new(master_write_end,pid,self)
+        pimp_klass = Object.const_get(packet_classify(worker_pimp))
+        @live_workers[worker_name_key,master_read_end.fileno] = pimp_klass.new(master_write_end,pid,self)
       else
-        @live_workers[t_worker_name,master_read_end.fileno] = Packet::MetaPimp.new(master_write_end,pid,self)
+        t_pimp = Packet::MetaPimp.new(master_write_end,pid,self)
+        t_pimp.worker_key = worker_name_key
+        @live_workers[worker_name_key,master_read_end.fileno] = t_pimp
       end
 
       worker_read_end.close
@@ -143,6 +141,5 @@ module Packet
       worker_read_fd.close
       read_ios << master_read_end
     end # end of fork_and_load method
-
   end # end of Reactor class
 end # end of Packet module

data/lib/pimp.rb

@@ -1,6 +1,7 @@
 module Packet
   class Pimp
     include NbioHelper
+    extend ClassHelpers
     extend Forwardable
     iattr_accessor :pimp_name
     attr_accessor :lifeline, :pid, :signature

data/lib/thread_pool.rb

@@ -0,0 +1,54 @@
+module Packet
+  class WorkData
+    attr_accessor :data,:block
+    def initialize(args,&block)
+      @data = args
+      @block = block
+    end
+  end
+
+  class ThreadPool
+    attr_accessor :size
+    attr_accessor :threads
+    attr_accessor :work_queue
+    def initialize(size)
+      @size = size
+      @threads = []
+      @work_queue = Queue.new
+      @running_tasks = Queue.new
+      @size.times { add_thread }
+    end
+    def defer(*args,&block)
+      @work_queue << WorkData.new(args,&block)
+    end
+
+    def add_thread
+      @threads << Thread.new do
+        while true
+          task = @work_queue.pop
+          @running_tasks << task
+          block_arity = task.block.arity
+          begin
+            block_arity == 0 ? task.block.call : task.block.call(*(task.data))
+          rescue
+            puts $!
+            puts $!.backtrace
+          end
+          @running_tasks.pop
+        end
+      end
+    end
+
+    # method ensures exclusive run of deferred tasks for 0.5 seconds, so as they do get a chance to run.
+    def exclusive_run
+      if @running_tasks.empty? && @work_queue.empty?
+        return
+      else
+        sleep(0.005)
+        return
+      end
+    end
+  end # end of ThreadPool class
+
+end # end of Packet module
+

data/lib/timer_store.rb

@@ -0,0 +1,63 @@
+=begin
+ There are many ordered hash implementation of ordered hashes, but this one is for packet.
+ Nothing more, nothing less.
+=end
+
+module Packet
+  class TimerStore
+    attr_accessor :order
+    def initialize
+      @order = []
+      @container = { }
+    end
+
+    def store(timer)
+      int_time = timer.scheduled_time.to_i
+      @container[int_time] ||= []
+      @container[int_time] << timer
+
+      if @container.empty?
+        @order << int_time
+        return
+      end
+      if @order.last <= key
+        @order << int_time
+      else
+        index = bin_search_for_key(o,@order.length - 1,int_time)
+        @order.insert(index,int_time)
+      end
+    end
+
+    def bin_search_for_key(lower_index,upper_index,key)
+      return upper_index if(upper_index - lower_index <= 1)
+      pivot = (lower_index + upper_index)/2
+      if @order[pivot] == key
+        return pivot
+      elsif @order[pivot] < key
+        bin_search_for_key(pivot,upper_index,key)
+      else
+        bin_search_for_key(lower_index,pivot,key)
+      end
+    end
+  end
+
+  def each
+    @order.each_with_index do |x,i|
+      if x <= Time.now.to_i
+        @container[x].each { |timer| yield x }
+        @container.delete(x)
+        @order.delete_at(i)
+      end
+    end
+  end
+
+  def delete(timer)
+    int_time = timer.scheduled_time
+    @container[int_time] && @container[int_time].delete(timer)
+
+    if(!@container[int_time] || @container[int_time].empty?)
+      @order.delete(timer)
+    end
+  end
+end
+

data/lib/worker.rb

@@ -5,6 +5,7 @@ module Packet
     iattr_accessor :fd_reader,:msg_writer,:msg_reader,:worker_name
     iattr_accessor :worker_proxy
     iattr_accessor :no_auto_load
+
     attr_accessor :worker_started, :worker_options
     after_connection :provide_workers
 
@@ -60,40 +61,15 @@ module Packet
       class << handler_instance
         extend Forwardable
         attr_accessor :worker, :connection, :reactor, :initialized, :signature
+        attr_accessor :thread_pool
         include NbioHelper
-        def send_data p_data
-          begin
-            write_data(p_data,connection)
-          rescue Errno::EPIPE
-            # probably a callback
-          end
-        end
-
-        def invoke_init
-          @initialized = true
-          post_init
-        end
-
-        def close_connection
-          unbind
-          reactor.remove_connection(connection)
-        end
-
-        def close_connection_after_writing
-          connection.flush
-          unbind
-          reactor.remove_connection(connection)
-        end
-
-        def send_object p_object
-          dump_object(p_object,connection)
-        end
-
+        include Connection
         def_delegators :@reactor, :start_server, :connect, :add_periodic_timer, :add_timer, :cancel_timer,:reconnect
       end
       handler_instance.connection = connection
       handler_instance.worker = self
       handler_instance.reactor = self
+      handler_instance.thread_pool = @thread_pool
     end
 
     def log log_data