drbqs 0.0.17 → 0.0.18

data/lib/drbqs/worker/serialize.rb

@@ -0,0 +1,66 @@
+module DRbQS
+  class Worker
+    class Serialize
+      HEADER_BYTE_SIZE = 4
+
+      class Unpacker
+        def initialize
+          @chunk = ''
+          @next_size = nil
+        end
+
+        def feed(data)
+          @chunk << data
+        end
+
+        def next_object
+          unless @next_size
+            if @chunk.bytesize >= HEADER_BYTE_SIZE
+              @chunk.force_encoding('BINARY')
+              @next_size = @chunk[0, HEADER_BYTE_SIZE].unpack('N')[0]
+              @chunk = @chunk[HEADER_BYTE_SIZE..-1]
+            else
+              return nil
+            end
+          end
+          if @next_size && @chunk.bytesize >= @next_size
+            data = @chunk[0, @next_size]
+            @chunk = @chunk[@next_size..-1]
+            @next_size = nil
+            [:loaded, Marshal.load(data)]
+          else
+            nil
+          end
+        end
+        private :next_object
+
+        def each(&block)
+          if block_given?
+            while ary = next_object
+              sym, obj = ary
+              yield(obj)
+            end
+          else
+            to_enum(:each)
+          end
+        end
+
+        def feed_each(data, &block)
+          feed(data)
+          each(&block)
+        end
+      end
+
+      def self.dump(obj)
+        str = Marshal.dump(obj)
+        [str.size].pack('N') << str
+      end
+
+      def self.load(s)
+        size = s[0, HEADER_BYTE_SIZE].unpack('N')[0]
+        data = s[HEADER_BYTE_SIZE, size]
+        Marshal.load(data)
+      end
+    end
+  end
+end

data/lib/drbqs/worker/worker.rb

@@ -0,0 +1,133 @@
+require 'drbqs/worker/worker_process_set'
+
+module DRbQS
+  # We can use DRbQS::Worker to send some child processes.
+  # Note that DRbQS::Worker is not used in DRbQS::Node class and then
+  # is not included in main part of DRbQS.
+  class Worker
+    attr_reader :process
+
+    def initialize(opts = {})
+      @process = DRbQS::Worker::ProcessSet.new(opts[:class])
+      if opts[:key]
+        opts[:key].each do |key|
+          @process.create_process(key)
+        end
+      end
+      @state = Hash.new { |h, k| h[k] = Hash.new }
+      @task_pool = {}
+      @task_group = Hash.new { |h, k| h[k] = Array.new }
+      @task_num = 0
+    end
+
+    def calculating?
+      !@task_pool.empty?
+    end
+
+    def sleep(*keys)
+      keys.each do |key|
+        @state[key][:sleep] = true
+      end
+    end
+
+    def wakeup(*keys)
+      keys.each do |key|
+        @state[key][:sleep] = false
+      end
+    end
+
+    def group(grp, *keys)
+      keys.each do |key|
+        (@state[key][:group] ||= []) << grp
+      end
+    end
+
+    def on_result(&block)
+      @process.on_result do |proc_key, ary|
+        task_id, result = ary
+        if task_data = @task_pool.delete(task_id)
+          task = task_data[:task]
+          @task_group[task.group].delete(task_id)
+          task.exec_hook(self, result)
+        end
+        block.call(proc_key, ary)
+      end
+    end
+
+    def on_error(&block)
+      @process.on_error(&block)
+    end
+
+    def send_task(proc_key, task_id, task)
+      dumped = [task_id] + task.simple_drb_args
+      @process.send_task(proc_key, dumped)
+    end
+    private :send_task
+
+    def add_task(task, broadcast = nil)
+      if broadcast
+        @process.all_processes.each do |proc_key|
+          send_task(proc_key, nil, task)
+        end
+      else
+        task_id = (@task_num += 1)
+        @task_pool[task_id] = { :task => task }
+        @task_group[task.group] << task_id
+        task_id
+      end
+    end
+
+    # This method sends a stored task for each process that is not calculating a task
+    # and responds signals from child processes.
+    def step
+      @process.waiting_processes.each do |proc_key|
+        if @state[proc_key][:sleep]
+          next
+        end
+        catch(:add) do
+          grps = (@state[proc_key][:group] || []) + [DRbQS::Task::DEFAULT_GROUP]
+          grps.each do |gr|
+            @task_group[gr].each do |task_id|
+              task_data = @task_pool[task_id]
+              if !task_data[:calculate]
+                send_task(proc_key, task_id, task_data[:task])
+                @task_pool[task_id][:calculate] = true
+                throw :add
+              end
+            end
+          end
+        end
+      end
+      @process.respond_signal
+    end
+
+    # Wait finish of task +task_id+ with sleep +interval_time+.
+    # @param [Fixnum] task_id
+    # @param [Numeric] interval_time An argument of Kernel#sleep.
+    def wait(task_id, interval_time)
+      while @task_pool[task_id]
+        unless step
+          Kernel.sleep(interval_time)
+        end
+      end
+    end
+
+    # Wait finishes of all tasks with sleep +interval_time+.
+    # @param [Numeric] interval_time An argument of Kernel#sleep.
+    def waitall(interval_time)
+      while calculating?
+        unless step
+          Kernel.sleep(interval_time)
+        end
+      end
+    end
+
+    # Send signal to exit to all child processes and wait the completion
+    # with sleep +interval_time+.
+    # @param [Numeric] interval_time An argument of Kernel#sleep.
+    def finish(interval_time = 1)
+      @process.prepare_to_exit
+      @process.waitall(interval_time)
+    end
+  end
+end

data/lib/drbqs/worker/worker_process_set.rb

@@ -0,0 +1,219 @@
+require 'drbqs/worker/serialize'
+require 'drbqs/worker/forked_process'
+require 'drbqs/utility/temporary.rb'
+
+module DRbQS
+  class Worker
+    READ_BYTE_SIZE = 1024
+
+    class ProcessSet
+      attr_reader :process
+
+      def initialize(process_class = nil)
+        @process_class = process_class || DRbQS::Worker::SimpleForkedProcess
+        @process = {}
+        @result = Queue.new
+        @on_error = nil
+        @on_result = nil
+      end
+
+      def on_error(&block)
+        @on_error = block
+      end
+
+      def on_result(&block)
+        @on_result = block
+      end
+
+      def process_create(key)
+        io_r, io_w = IO.pipe('BINARY')
+        io_r2, io_w2 = IO.pipe('BINARY')
+        parent_pid = Process.pid
+        pid = fork do
+          $PROGRAM_NAME = "[worker:#{key.to_s}] for drbqs-node (PID #{parent_pid})"
+          io_w.close
+          io_r2.close
+          worker = @process_class.new(io_r, io_w2)
+          worker.start
+        end
+        @process[key] = {
+          :pid => pid, :out => io_w, :in => io_r2,
+          :unpacker => DRbQS::Worker::Serialize::Unpacker.new,
+          :task => []
+        }
+      end
+      private :process_create
+
+      def get_process(key)
+        if @process[key]
+          @process[key]
+        else
+          process_create(key)
+        end
+      end
+      private :get_process
+
+      def create_process(*keys)
+        keys.each do |key|
+          get_process(key)
+        end
+      end
+
+      # Return true if the process of +key+ exists.
+      # @param [Symbol,nil] key Key of child process or nil.
+      def exist?(key)
+        @process[key]
+      end
+
+      def has_process?
+        !@process.empty?
+      end
+
+      # Return true if the process of +key+ is calculating.
+      def calculating?(key)
+        @process[key] && !@process[key][:task].empty?
+      end
+
+      # Return true if the process +key+ does not calculate any tasks.
+      def waiting?(key)
+        !calculating?(key)
+      end
+
+      # Return keys of processes not calculating a task.
+      def waiting_processes
+        @process.keys.select do |key|
+          @process[key][:task].empty?
+        end
+      end
+
+      def all_processes
+        @process.keys
+      end
+
+      def output_to_io(io, obj)
+        io.print Serialize.dump(obj)
+        io.flush
+      end
+      private :output_to_io
+
+      def send_object(key, obj)
+        if h = get_process(key)
+          output_to_io(h[:out], obj)
+          h
+        else
+          nil
+        end
+      end
+      private :send_object
+
+      # @param [Array] dumped_task_ary is [task_id, obj, method_name, args].
+      def send_task(key, dumped_task_ary)
+        if h = send_object(key, dumped_task_ary)
+          if dumped_task_ary[0]
+            h[:task] << dumped_task_ary[0]
+          end
+        else
+          raise "Process #{key.inspect} does not exist."
+        end
+      end
+
+      def prepare_to_exit(key = nil)
+        if key
+          if h = send_object(key, :prepare_to_exit)
+            h[:exit] = true
+          end
+        else
+          @process.each do |key, h|
+            prepare_to_exit(key)
+          end
+        end
+      end
+
+      def delete_process(key)
+        if h = get_process(key)
+          Process.detach(h[:pid])
+          output_to_io(h[:out], :exit)
+        else
+          nil
+        end
+      end
+      private :delete_process
+
+      # Read IOs and respond signals from chiled processes.
+      # If there no signal from childe processes then the method returns false.
+      # Otherwise, true.
+      # Types of signals are :result, :node_error, :finish_preparing_to_exit.
+      #  - :result
+      #    Execute callback set by DRbQS::Worker::ProcessSet#on_result.
+      #  - :node_error
+      #    Execute callback set by DRbQS::Worker::ProcessSet#on_error.
+      #  - :finish_preparing_to_exit
+      #    Send :exit signale to the process and delete from list of child processes.
+      def respond_signal
+        num = 0
+        to_be_deleted = []
+        @process.each do |key, h|
+          if !h[:task].empty? || h[:exit]
+            begin
+              data = h[:in].read_nonblock(READ_BYTE_SIZE)
+              h[:unpacker].feed_each(data) do |ary|
+                num += 1
+                response_type, response = ary
+                case response_type
+                when :result
+                  task_id, result = response
+                  h[:task].delete(task_id)
+                  if @on_result
+                    @on_result.call(key, [task_id, result])
+                  else
+                    $stderr.puts "The instance of DRbQS::Worker::ProcessSet can not deal with results from child processes."
+                  end
+                when :node_error
+                  if @on_error
+                    @on_error.call(key, response)
+                  else
+                    $stderr.puts "The instance of DRbQS::Worker::ProcessSet can not deal with error from child processes."
+                  end
+                when :finish_preparing_to_exit
+                  delete_process(key)
+                  to_be_deleted << key
+                end
+              end
+            rescue IO::WaitReadable
+            end
+          end
+        end
+        to_be_deleted.each do |key|
+          @process.delete(key)
+        end
+        to_be_deleted.clear
+        num > 0
+      end
+
+      def kill_all_processes
+        @process.each do |key, h|
+          Process.detach(h[:pid])
+          Process.kill("KILL", h[:pid])
+        end
+        @process.clear
+      end
+
+      WAITALL_INTERVAL_TIME = 0.1
+
+      def waitall(interval_time = nil)
+        unless @process.all? { |key, h| h[:exit] }
+          return nil
+        end
+        t = interval_time || WAITALL_INTERVAL_TIME
+        until @process.empty?
+          respond_signal
+          Kernel.sleep(t)
+        end
+        until Process.waitall == []
+          Kernel.sleep(t)
+        end
+        true
+      end
+    end
+  end
+end

data/spec/integration_test/01_basic_usage_spec.rb

@@ -6,7 +6,7 @@ require_relative 'definition/task_obj_definition.rb'
 describe DRbQS do
   before(:all) do
     @tasks = 5.times.map do |i|
-      DRbQS::Task.new(Test1.new, :echo, args: [i])
+      DRbQS::Task.new(TestCount.new, :echo, args: [i])
     end
     @process_id, @uri = drbqs_fork_server(14010, :task => @tasks)
     @node = DRbQS::Node.new(@uri, :log_file => $stdout, :continue => true)
@@ -42,10 +42,11 @@ describe DRbQS do
     lambda do
       @node.calculate
     end.should_not raise_error
-    Test1.get_execute_echo_number.should == @tasks.size
+    TestCount.get_execute_echo_number.should == @tasks.size
   end
 
   after(:all) do
+    TestCount.clear
     lambda do
       drbqs_wait_kill_server(@process_id)
     end.should_not raise_error

data/spec/integration_test/06_node_exit_after_task_spec.rb

@@ -15,8 +15,9 @@ describe DRbQS::Server do
 
   it "should send node exit" do
     execute_node = DRbQS::Execution::ExecuteNode.new(@uri, nil, nil)
-    execute_node.execute(1)
-    client_process_id = execute_node.pid[0]
+    client_process_id = fork do
+      execute_node.execute(1)
+    end
     sleep(@wait)
     @manage.send_node_exit_after_task(1)
     th = Process.detach(client_process_id)