packet 0.1.2 → 0.1.3

data/Rakefile

@@ -16,7 +16,7 @@ NAME = "packet"
 $LOAD_PATH.unshift __DIR__+'/lib'
 require 'packet'
 
-CLEAN.include ['**/.*.sw?', '*.gem', '.config']
+CLEAN.include ['**/.*.sw?', '*.gem', '.config','*.rbc']
 
 
 @windows = (PLATFORM =~ /win32/)

data/TODO

@@ -3,6 +3,8 @@
 ** TODO Remove class and class instance attributes from core classes.
 ** TODO Cleanup callback mechanism
 ** TODO Implement a sample worker, which provides capabilities to execute mysql statements async.
+** TODO Implement more cleaner write thingies.
+** TODO Implement more cleaner timers.
 
 
 

data/examples/write_bulk.rb

@@ -0,0 +1,41 @@
+require "socket"
+require "thread"
+
+# sock = TCPSocket.open("localhost",11007)
+#data = File.open("netbeans.jpg").read
+data = File.open("nginx.dat").read
+# p data.length
+
+threads = []
+500.times do
+  #   sock.write(data)
+  #   select([sock],nil,nil,nil)
+  #   read_data = ""
+
+  #   loop do
+  #     begin
+  #       while(read_data << sock.read_nonblock(1023)); end
+  #     rescue Errno::EAGAIN
+  #       break
+  #     rescue
+  #       break
+  #     end
+  #   end
+
+  threads << Thread.new do
+    sock = TCPSocket.open("localhost",11007)
+    #   p read_data.length
+    written_length = sock.write(data)
+    p "Write Length: #{written_length}"
+    read_length = sock.read(written_length)
+    p "Read length: #{read_length.length}"
+  end
+
+#   #   p read_data.length
+#   written_length = sock.write(data)
+#   #p "Write Length: #{written_length}"
+#   read_length = sock.read(written_length)
+#   #p "Read length: #{read_length.length}"
+end
+
+threads.each { |x| x.join }

data/lib/packet.rb

@@ -1,37 +1,30 @@
+$:.unshift(File.dirname(__FILE__)) unless
+  $:.include?(File.dirname(__FILE__)) || $:.include?(File.expand_path(File.dirname(__FILE__)))
+
 require "socket"
 require "yaml"
 require "forwardable"
 require "ostruct"
 require "thread"
 
-require "bin_parser"
-
-
-require "packet_guid"
-require "class_helpers"
-require "thread_pool"
-require "double_keyed_hash"
-require "event"
-
-require "periodic_event"
-require "disconnect_error"
-require "callback"
-
-require "nbio"
-require "pimp"
-require "meta_pimp"
-require "core"
-
-require "packet_master"
-require "connection"
-require "worker"
-
-# This file is just a runner of things and hence does basic initialization of thingies required for running
-# the application.
-
+require "packet/packet_parser"
+require "packet/packet_guid"
+require "packet/packet_helper"
+require "packet/double_keyed_hash"
+require "packet/packet_event"
+require "packet/packet_periodic_event"
+require "packet/disconnect_error"
+require "packet/packet_callback"
+require "packet/packet_nbio"
+require "packet/packet_pimp"
+require "packet/packet_meta_pimp"
+require "packet/packet_core"
+require "packet/packet_master"
+require "packet/packet_connection"
+require "packet/packet_worker"
 
 PACKET_APP = File.expand_path'../' unless defined?(PACKET_APP)
 
 module Packet
-  VERSION='0.1.2'
+  VERSION='0.1.3'
 end

data/lib/{disconnect_error.rb → packet/disconnect_error.rb}

@@ -1,8 +1,9 @@
 module Packet
   class DisconnectError < RuntimeError
-    attr_accessor :disconnected_socket
-    def initialize(t_sock)
+    attr_accessor :disconnected_socket,:data
+    def initialize(t_sock,data = nil)
       @disconnected_socket = t_sock
+      @data = data
     end
   end
 end

data/lib/{connection.rb → packet/packet_connection.rb}

@@ -3,21 +3,27 @@
 
 module Packet
   module Connection
+    attr_accessor :outbound_data,:connection_live
+
     def send_data p_data
+      @outbound_data << p_data
       begin
-        write_data(p_data,connection)
-      rescue DisconnectError => sock_error
+        write_and_schedule(connection) 
+      rescue DisconnectError => sock
         close_connection
       end
     end
 
     def invoke_init
       @initialized = true
+      @connection_live = true
+      @outbound_data = []
       post_init if respond_to?(:post_init)
     end
 
-    def close_connection
+    def close_connection(sock = nil)
       unbind if respond_to?(:unbind)
+      reactor.cancel_write(connection)
       reactor.remove_connection(connection)
     end
 
@@ -29,5 +35,6 @@ module Packet
     def send_object p_object
       dump_object(p_object,connection)
     end
+
   end # end of class Connection
 end # end of module Packet

data/lib/{core.rb → packet/packet_core.rb}

@@ -6,11 +6,11 @@ module Packet
       base_klass.instance_eval do
         @@connection_callbacks ||= {}
 
-        iattr_accessor :thread_pool_size
         cattr_accessor :connection_callbacks
         attr_accessor :read_ios, :write_ios, :listen_sockets
-        attr_accessor :connection_completion_awaited
-        attr_accessor :connections, :thread_pool, :windows_flag
+        attr_accessor :connection_completion_awaited,:write_scheduled
+        attr_accessor :connections, :windows_flag
+        attr_accessor :internal_scheduled_write,:outbound_data,:reactor
         include CommonMethods
       end
     end
@@ -70,7 +70,6 @@ module Packet
           client_socket,client_sockaddr = sock_io.accept_nonblock
           client_socket.setsockopt(Socket::IPPROTO_TCP,Socket::TCP_NODELAY,1)
         rescue Errno::EAGAIN, Errno::ECONNABORTED, Errno::EPROTO, Errno::EINTR
-          puts "not ready yet"
           return
         end
         read_ios << client_socket
@@ -96,18 +95,12 @@ module Packet
 
       # method removes the connection and closes the socket
       def remove_connection(t_sock)
-        @read_ios.delete(t_sock)
-        @write_ios.delete(t_sock)
-        connections.delete(t_sock.fileno)
-        t_sock.close
-      end
-
-      def socket_really_connected?(t_sock)
+        read_ios.delete(t_sock)
+        write_ios.delete(t_sock)
         begin
-          t_data = read_data(t_sock)
-          return true
-        rescue DisconnectError
-          return false
+          connections.delete(t_sock.fileno)
+          t_sock.close
+        rescue
         end
       end
 
@@ -126,34 +119,70 @@ module Packet
         Signal.trap("INT") { shutdown }
         loop do
           check_for_timer_events
-          user_thread_window #=> let user level threads run for a while
-          ready_fds = select(@read_ios,@write_ios,nil,0.005)
-          #next if ready_fds.blank?
-
-          next if !ready_fds or ready_fds.empty?
-
-          ready_fds = ready_fds.flatten.compact
-          ready_fds.each do |t_sock|
-            if(unix? && t_sock.is_a?(UNIXSocket))
-              handle_internal_messages(t_sock)
-            else
-              handle_external_messages(t_sock)
-            end
+          ready_read_fds,ready_write_fds,read_error_fds = select(read_ios,write_ios,nil,0.005)
+
+          if ready_read_fds && !ready_read_fds.empty?
+            handle_read_event(ready_read_fds)
+          elsif ready_write_fds && !ready_write_fds.empty?
+            handle_write_event(ready_write_fds)
+          end
+        end
+
+      end
+
+      def schedule_write(t_sock)
+        fileno = t_sock.fileno
+        if UNIXSocket === t_sock && internal_scheduled_write[fileno].nil?
+          write_ios << t_sock
+          internal_scheduled_write[t_sock.fileno] ||= self
+        elsif write_scheduled[fileno].nil?
+          write_ios << t_sock
+          write_scheduled[fileno] ||= connections[fileno].instance
+        end
+      end
+
+      def cancel_write(t_sock)
+        fileno = t_sock.fileno
+        if UNIXSocket === t_sock
+          internal_scheduled_write.delete(fileno)
+        else
+          write_scheduled.delete(fileno)
+        end
+        write_ios.delete(t_sock)
+      end
+
+      def handle_write_event(p_ready_fds)
+        p_ready_fds.each do |sock_fd|
+          fileno = sock_fd.fileno
+          if UNIXSocket == sock_fd && internal_scheduled_write[fileno]
+            write_and_schedule(sock_fd)
+          elsif extern_opts = connection_completion_awaited[fileno]
+            complete_connection(sock_fd,extern_opts)
+          elsif handler_instance = write_scheduled[fileno]
+            handler_instance.write_scheduled(sock_fd)
           end
         end
       end
 
-      def user_thread_window
-        # run_user_threads if respond_to?(:run_user_threads)
-        @thread_pool.exclusive_run
+      def handle_read_event(p_ready_fds)
+        ready_fds = p_ready_fds.flatten.compact
+        ready_fds.each do |t_sock|
+          if(unix? && t_sock.is_a?(UNIXSocket))
+            handle_internal_messages(t_sock)
+          else
+            handle_external_messages(t_sock)
+          end
+        end
       end
 
       def terminate_me
         # FIXME: close the open sockets
+        # @thread_pool.kill_all
         exit
       end
 
       def shutdown
+        # @thread_pool.kill_all
         # FIXME: close the open sockets
         exit
       end
@@ -166,8 +195,6 @@ module Packet
         sock_fd = t_sock.fileno
         if sock_opts = listen_sockets[sock_fd]
           accept_connection(sock_opts)
-        elsif extern_opts = connection_completion_awaited[sock_fd]
-          complete_connection(t_sock,extern_opts)
         else
           read_external_socket(t_sock)
         end
@@ -179,9 +206,7 @@ module Packet
           t_data = read_data(t_sock)
           handler_instance.receive_data(t_data) if handler_instance.respond_to?(:receive_data)
         rescue DisconnectError => sock_error
-          handler_instance.unbind if handler_instance.respond_to?(:unbind)
-          connections.delete(t_sock.fileno)
-          read_ios.delete(t_sock)
+          handler_instance.close_connection
         end
       end
 
@@ -202,17 +227,28 @@ module Packet
         @timer_hash.delete(t_timer.timer_signature)
       end
 
+      def binding_str
+        @binding += 1
+        "BIND_#{@binding}"
+      end
+
       def initialize
         @read_ios ||= []
         @write_ios ||= []
         @connection_completion_awaited ||= {}
+        @write_scheduled ||= {}
+        @internal_scheduled_write ||= {}
+        # internal outbound data
+        @outbound_data = []
         @connections ||= {}
         @listen_sockets ||= {}
+        @binding = 0
 
         # @timer_hash = Packet::TimerStore
         @timer_hash ||= {}
-        @thread_pool = ThreadPool.new(thread_pool_size || 20)
+        # @thread_pool = ThreadPool.new(thread_pool_size || 20)
         @windows_flag = windows?
+        @reactor = self
       end
 
       def windows?
@@ -233,6 +269,15 @@ module Packet
           end
         end
       end
+      
+      # close the connection with internal specified socket
+      def close_connection(sock = nil)
+        begin
+          read_ios.delete(sock.fileno)
+          write_ios.delete(sock.fileno)
+          sock.close
+        rescue; end
+      end
 
       def initialize_handler(p_module)
         return p_module if(!p_module.is_a?(Class) and !p_module.is_a?(Module))
@@ -253,10 +298,9 @@ module Packet
           handler_instance.unbind if handler_instance.respond_to?(:unbind)
           return
         end
-        t_signature = Guid.hexdigest
-        handler_instance.signature = t_signature
+        handler_instance.signature = binding_str
         connections[t_socket.fileno] =
-          OpenStruct.new(:socket => t_socket, :instance => handler_instance, :signature => t_signature,:sock_addr => sock_addr)
+          OpenStruct.new(:socket => t_socket, :instance => handler_instance, :signature => handler_instance.signature,:sock_addr => sock_addr)
         block.call(handler_instance) if block
         handler_instance.connection_completed if handler_instance.respond_to?(:connection_completed)
       end

data/lib/{event.rb → packet/packet_event.rb}

@@ -1,6 +1,6 @@
 module Packet
   class Event
-    attr_accessor :timer_signature, :block, :cancel_flag
+    attr_accessor :timer_signature, :block, :cancel_flag, :scheduled_time
     def initialize(elapsed_time,&block)
       @cancel_flag = false
       @timer_signature = Guid.hexdigest

data/lib/{packet_master.rb → packet/packet_master.rb}

@@ -14,7 +14,7 @@ module Packet
 
     def self.run
       master_reactor_instance = new
-      # master_reactor_instance.result_hash = {}
+      master_reactor_instance.result_hash = {}
       master_reactor_instance.live_workers = DoubleKeyedHash.new
       yield(master_reactor_instance)
       master_reactor_instance.load_workers
@@ -34,7 +34,6 @@ module Packet
       class << handler_instance
         extend Forwardable
         attr_accessor :workers,:connection,:reactor, :initialized,:signature
-        attr_accessor :thread_pool
         include NbioHelper
         include Connection
         def ask_worker(*args)
@@ -52,7 +51,6 @@ module Packet
       handler_instance.workers = @live_workers
       handler_instance.connection = t_sock
       handler_instance.reactor = self
-      handler_instance.thread_pool = @thread_pool
     end
 
     # FIXME: right now, each worker is tied to its connection and this can be problematic
@@ -65,10 +63,16 @@ module Packet
         # t_data = Marshal.load(raw_data)
         worker_instance.receive_data(raw_data) if worker_instance.respond_to?(:receive_data)
       rescue DisconnectError => sock_error
-        read_ios.delete(t_sock)
+        remove_worker(t_sock)
       end
     end
+    
 
+    def remove_worker(t_sock)
+      @live_workers.delete(t_sock.fileno)
+      read_ios.delete(t_sock)
+    end
+    
     def delete_worker(worker_options = {})
       worker_name = worker_options[:worker]
       worker_name_key = gen_worker_key(worker_name,worker_options[:job_key])
@@ -100,10 +104,14 @@ module Packet
       worker_options.delete(:worker)
       begin
         require worker_name
+        worker_klass = Object.const_get(packet_classify(worker_name))
+        fork_and_load(worker_klass,worker_options)
       rescue LoadError
+        puts "no such worker #{worker_name}" 
+      rescue MissingSourceFile
+        puts "no such worker #{worker_name}" 
+        return
       end
-      worker_klass = Object.const_get(packet_classify(worker_name))
-      fork_and_load(worker_klass,worker_options)
     end
 
     # method forks given worker file in a new process
@@ -116,14 +124,14 @@ module Packet
       master_read_end,worker_write_end = UNIXSocket.pair(Socket::SOCK_STREAM)
       # socket to which master process is going to write
       worker_read_end,master_write_end = UNIXSocket.pair(Socket::SOCK_STREAM)
-      worker_read_fd,master_write_fd = UNIXSocket.pair
+      # worker_read_fd,master_write_fd = UNIXSocket.pair
 
       if((pid = fork()).nil?)
         $0 = "ruby #{worker_klass.worker_name}"
-        [master_write_end,master_read_end,master_write_fd].each { |x| x.close }
+        [master_write_end,master_read_end].each { |x| x.close }
 
         worker_klass.start_worker(:write_end => worker_write_end,:read_end => worker_read_end,\
-                                  :read_fd => worker_read_fd,:options => worker_options)
+                                  :options => worker_options)
       end
       Process.detach(pid)
 
@@ -136,12 +144,12 @@ module Packet
       else
         t_pimp = Packet::MetaPimp.new(master_write_end,pid,self)
         t_pimp.worker_key = worker_name_key
+        t_pimp.worker_name = t_worker_name
         @live_workers[worker_name_key,master_read_end.fileno] = t_pimp
       end
 
       worker_read_end.close
       worker_write_end.close
-      worker_read_fd.close
       read_ios << master_read_end
     end # end of fork_and_load method
   end # end of Reactor class