drbqs 0.0.17 → 0.0.18

data/lib/drbqs/server/server_hook.rb

@@ -26,15 +26,15 @@ module DRbQS
       end
       private :create_proc_name
 
-      def add(key, name = nil, &block)
+      def add(key, opts = {}, &block)
         unless block_given?
           raise ArgumentError, "The main part of hook must be specified as a block."
         end
         if (n = @argument_number[key]) && (block.arity != n)
           raise ArgumentError, "Invalid argument number of hook of #{key.inspect}."
         end
-        name ||= create_proc_name(key)
-        @hook[key] << [name, block]
+        name = opts[:name] || create_proc_name(key)
+        @hook[key] << [name, block, opts[:repeat]]
         name
       end
 
@@ -68,10 +68,24 @@ module DRbQS
         @hook[key].map { |a| a[0] }
       end
 
+      def delete_unused_hook
+        @hook.keys.each do |key|
+          @hook[key].delete_if do |ary|
+            ary[2] && ary[2] == 0
+          end
+        end
+      end
+      private :delete_unused_hook
+
       def exec(key, *args, &cond)
+        delete_unused_hook
         @hook[key].each do |ary|
-          if !cond || cond.call(ary[0])
-            ary[1].call(*args)
+          name, proc, repeat = ary
+          if !cond || cond.call(name)
+            if !repeat || repeat > 0
+              proc.call(*args)
+              ary[2] -= 1 if repeat
+            end
           else
             return nil
           end

data/lib/drbqs/server/test/node.rb

@@ -6,26 +6,51 @@ module DRbQS
       def initialize(log_level, transfer, queue)
         super(nil, :log_file => $stdout, :log_level => log_level)
         DRbQS::Transfer::Client.set(transfer.get_client(true)) if transfer
-        @task_client = DRbQS::Node::TaskClient.new(nil, queue, nil)
+        @task_client = DRbQS::Node::TaskClient.new(nil, queue, nil, [DRbQS::Node::SAME_HOST_GROUP], 1)
+        @special_task_number = 0
       end
 
       def server_on_same_host?
         true
       end
 
+      def subdirectory_name(task_id)
+        if task_id
+          sprintf("T%08d", task_id)
+        else
+          sprintf("S%08d", (@special_task_number += 1))
+        end
+      end
+      private :subdirectory_name
+
+      def execute_task(task_id, marshal_obj, method_sym, args)
+        DRbQS::Temporary.set_sub_directory(subdirectory_name(task_id))
+        result = DRbQS::Task.execute_task(marshal_obj, method_sym, args)
+        if files = DRbQS::Transfer.dequeue_all
+          transfer_file(files)
+        end
+        if subdir = DRbQS::Temporary.subdirectory
+          FileUtils.rm_r(subdir)
+        end
+        result
+      end
+      private :execute_task
+
       def calc
         if ary = @task_client.get_task
           task_id, marshal_obj, method_sym, args = ary
-          result = execute_task(marshal_obj, method_sym, args)
+          result = execute_task(task_id, marshal_obj, method_sym, args)
           return [task_id, result]
         end
         nil
       end
 
-      def finalize(finalization_task)
-        if finalization_task
-          args = finalization_task.drb_args(nil)[1..-1]
-          execute_task(*args)
+      def finalize(finalization_task_ary)
+        if finalization_task_ary
+          finalization_task_ary.each do |task|
+            args = task.simple_drb_args
+            execute_task(nil, *args)
+          end
         end
         clear_node_files
       end

data/lib/drbqs/setting/node.rb

@@ -7,6 +7,7 @@ module DRbQS
         super(:all_keys_defined => true, :log_level => true, :daemon => true) do
           register_key(:load, :check => [:>, 0], :add => true)
           register_key(:process, :check => 1, :default => [1])
+          register_key(:group, :add => true)
           register_key(:loadavg, :check => 1)
           register_key(:log_prefix, :check => 1, :default => [LOG_PREFIX_DEFAULT])
           register_key(:log_stdout, :bool => true)
@@ -28,6 +29,7 @@ module DRbQS
       def parse!
         super
         parse_load
+        parse_group
         parse_loadavg
         if !get(:log_stdout)
           @options[:log_prefix] = get_first(:log_prefix) do |val|
@@ -49,7 +51,7 @@ module DRbQS
       private :parse_load
 
       def parse_loadavg
-        @options[:node_opts] = {}
+        @options[:node_opts] ||= {}
         if args = get(:loadavg)
           max_loadavg, sleep_time = args[0].split(':', -1)
           @options[:node_opts][:max_loadavg] = max_loadavg && max_loadavg.size > 0 ? max_loadavg.to_f : nil
@@ -58,6 +60,14 @@ module DRbQS
       end
       private :parse_loadavg
 
+      def parse_group
+        if args = get(:group)
+          @options[:node_opts] ||= {}
+          @options[:node_opts][:group] = args.map { |a| a.intern }
+        end
+      end
+      private :parse_group
+
       def exec(io = nil)
         return true if exec_as_daemon
 
@@ -76,7 +86,6 @@ module DRbQS
 
         exec_node = DRbQS::Execution::ExecuteNode.new(@uri, @options[:log_prefix], @options[:log_level], @options[:node_opts])
         exec_node.execute(@process_num)
-        exec_node.wait
         true
       end
     end

data/lib/drbqs/setting/server.rb

@@ -133,7 +133,6 @@ module DRbQS
         end
         exec_node = DRbQS::Execution::ExecuteNode.new(uri, node_log_file, @options[:log_level])
         exec_node.execute(@execute_node_number, NODE_INTERVAL_TIME)
-        exec_node.wait
       end
       private :execute_node_and_wait
 
@@ -149,6 +148,7 @@ module DRbQS
         end
         uri = current_server_uri
         wait_server_process(uri, server_pid)
+        $PROGRAM_NAME = "drbqs-node with drbqs-server (PID #{server_pid})"
         execute_node_and_wait(uri)
       end
       private :command_server_with_nodes

data/lib/drbqs/task/task.rb

@@ -8,11 +8,13 @@ module DRbQS
   # The tasks defined by this class are sent to nodes and
   # calculated by the nodes.
   class Task
+    DEFAULT_GROUP = :default
+
     attr_reader :obj
     attr_reader :args
     attr_reader :method_name
     attr_reader :hook
-    attr_accessor :note
+    attr_accessor :note, :group
 
     # Nodes calculate by obj.method_name(*opts[:args]) and send the result to their server.
     # Then the server executes &hook with a server instance and an object of result.
@@ -26,6 +28,7 @@ module DRbQS
     # @option opts [String] :note Note for a task
     # @option opts [Symbol] :hook Method name for hook
     #  that takes two arguments server and the result object.
+    # @option opts [Symbol] :group Group of nodes to execute the task.
     # @param [Proc] hook A server execute hook as a callback when the server receive the result
     #  hook take two arguments: a DRbQS::Server object and a result of task.
     # @note Changes of obj on a node are not sent to a server.
@@ -49,10 +52,15 @@ module DRbQS
       end
       @note = opts[:note]
       @hook = hook || opts[:hook]
+      @group = opts[:group] || DRbQS::Task::DEFAULT_GROUP
+    end
+
+    def simple_drb_args
+      [@marshal_obj, @method_name, @marshal_args]
     end
 
     def drb_args(task_id)
-      [task_id, @marshal_obj, @method_name, @marshal_args]
+      [@group, task_id] + simple_drb_args
     end
 
     def exec_hook(server, result)
@@ -130,13 +138,13 @@ module DRbQS
       class ContainerWithoutHook < DRbQS::Task::TaskSet::Container
         def initialize(task_ary)
           @data = task_ary.map.with_index do |task, i|
-            task.drb_args(i)
+            task.drb_args(i)[2..-1]
           end
         end
 
         def exec
           @data.map do |ary|
-            DRbQS::Task.execute_task(*ary[1..-1])
+            DRbQS::Task.execute_task(*ary)
           end
         end
       end
@@ -147,13 +155,14 @@ module DRbQS
         @original_note = task_ary.map do |task|
           task.note
         end.compact!
+        group_sym = get_group_sym(task_ary)
         if task_ary.all? { |task| !(Proc === task.hook) }
           container = DRbQS::Task::TaskSet::ContainerTask.new(task_ary)
-          super(container, :exec, hook: :exec_all_hooks, note: note_string)
+          super(container, :exec, hook: :exec_all_hooks, note: note_string, group: group_sym)
         else
           container = DRbQS::Task::TaskSet::ContainerWithoutHook.new(task_ary)
           @original_task = task_ary
-          super(container, :exec, note: note_string) do |srv, result|
+          super(container, :exec, note: note_string, group: group_sym) do |srv, result|
             result.each_with_index do |res, i|
               @original_task[i].exec_hook(srv, res)
             end
@@ -161,6 +170,17 @@ module DRbQS
         end
       end
 
+      def get_group_sym(task_ary)
+        group_names = task_ary.map do |task|
+          task.group
+        end
+        group_names.compact!
+        group_names.uniq!
+        raise ArgumentError, "Can not collect tasks with different groups." if group_names.size > 1
+        group_names[0]
+      end
+      private :get_group_sym
+
       def note_string
         str = "TaskSet"
         unless @original_note.empty?

data/lib/drbqs/task/task_generator.rb

@@ -1,7 +1,8 @@
 module DRbQS
   class Task
     class Generator
-      # @param [Hash] data Names of instance variables and their values,
+      # @param [Hash] data This argument is unnecessary and so deprecated
+      #  Names of instance variables and their values,
       #  which can be accessed in {DRbQS::Task::Generator#set}.
       def initialize(data = {})
         @registrar = DRbQS::Task::Registrar.new(data)

data/lib/drbqs/utility/temporary.rb

@@ -3,15 +3,36 @@ require 'tmpdir'
 module DRbQS
   module Temporary
     @@root = nil
+    @@subdir = nil
     @@filename = nil
 
-    # Return FileName object to generate names of temporary files on DRbQS nodes.
-    def self.filename
-      unless @@filename
+    def self.set_root_directory
+      if !@@root
         pid = Process.pid
         @@root = File.join(Dir.tmpdir, sprintf("drbqs_%s_%d_%d", ENV['USER'], pid, rand(10000)))
         FileUtils.mkdir_p(@@root, :mode => 0700)
-        @@filename = FileName.new(File.join(@@root, sprintf("temp_%d_%d", pid, rand(10000))))
+      end
+    end
+
+    def self.set_sub_directory(dir)
+      self.set_root_directory
+      @@filename = nil
+      @@subdir = File.join(@@root, dir)
+    end
+
+    def self.subdirectory
+      @@subdir && File.exist?(@@subdir) ? @@subdir : nil
+    end
+
+    # Return FileName object to generate names of temporary files on DRbQS nodes.
+    def self.filename
+      unless @@filename
+        self.set_root_directory
+        if @@subdir
+          @@filename = FileName.new(File.join(@@subdir, sprintf("temp_%d", rand(10000))))
+        else
+          @@filename = FileName.new(File.join(@@root, sprintf("temp_%d", rand(10000))))
+        end
       end
       @@filename
     end
@@ -27,7 +48,7 @@ module DRbQS
       if basename
         File.join(self.directory, basename)
       else
-        filename.create(:add => :always)
+        filename.create(:add => :always, :directory => :parent)
       end
     end
 
@@ -35,7 +56,7 @@ module DRbQS
     def self.delete
       if @@root
         FileUtils.rm_r(@@root)
-        FileUtils.mkdir_p(@@root)
+        FileUtils.mkdir_p(@@root, :mode => 0700)
       end
     end
 

data/lib/drbqs/utility/transfer/transfer_client.rb

@@ -68,19 +68,17 @@ module DRbQS
           @transfer = transfer
         end
 
-        def transfer_to_server
-          if files = DRbQS::Transfer.dequeue_all
-            if @transfer
-              begin
-                @transfer.transfer(files)
-              rescue Exception => err
-                err_new = err.class.new("#{err.to_s} (#{err.class}); Can not send file: #{files.join(", ")}")
-                err_new.set_backtrace(err.backtrace)
-                raise err_new
-              end
-            else
-              raise "Server does not set transfer settings. Can not send file: #{files.join(", ")}"
+        def transfer_to_server(files)
+          if files && @transfer
+            begin
+              @transfer.transfer(files)
+            rescue Exception => err
+              err_new = err.class.new("#{err.to_s} (#{err.class}); Can not send file: #{files.join(", ")}")
+              err_new.set_backtrace(err.backtrace)
+              raise err_new
             end
+          else
+            raise "Server does not set transfer settings. Can not send file: #{files.join(", ")}"
           end
         end
       end

data/lib/drbqs/version.rb

@@ -1,3 +1,3 @@
 module DRbQS
-  VERSION = '0.0.17'
+  VERSION = '0.0.18'
 end

data/lib/drbqs/worker.rb

@@ -0,0 +1,2 @@
+require 'drbqs/task/task'
+require 'drbqs/worker/worker'

data/lib/drbqs/worker/forked_process.rb

@@ -0,0 +1,100 @@
+module DRbQS
+  class Worker
+    class SimpleForkedProcess
+      def initialize(io_r, io_w)
+        @io_r = io_r
+        @io_w = io_w
+        @queue = []
+        @special_task_number = 0
+      end
+
+      def calculate(marshal_obj, method_sym, args)
+        DRbQS::Task.execute_task(marshal_obj, method_sym, args)
+      end
+
+      def send_response(obj)
+        @io_w.print DRbQS::Worker::Serialize.dump(obj)
+        @io_w.flush
+      end
+      private :send_response
+
+      def handle_task(obj)
+        task_id, marshal_obj, method_sym, args = obj
+        begin
+          res = calculate(marshal_obj, method_sym, args)
+          if task_id
+            send_response([:result, [task_id, res]])
+          end
+        rescue => err
+          send_response([:node_error, err])
+        end
+      end
+      private :handle_task
+
+      def start
+        unpacker = DRbQS::Worker::Serialize::Unpacker.new
+        loop do
+          if @queue.empty?
+            begin
+              chunk = @io_r.readpartial(READ_BYTE_SIZE)
+              unpacker.feed_each(chunk) do |ary|
+                @queue << ary
+              end
+            rescue EOFError
+              break
+            end
+          else
+            obj = @queue.shift
+            case obj
+            when Array
+              handle_task(obj)
+            when :prepare_to_exit
+              send_response([:finish_preparing_to_exit])
+            when :exit
+              break
+            else
+              send_response([:node_error, "Invalid object from server."])
+            end  
+          end
+        end
+      end
+    end
+
+    class ForkedProcess < DRbQS::Worker::SimpleForkedProcess
+      def calculate(marshal_obj, method_sym, args)
+        result = super(marshal_obj, method_sym, args)
+        transfer_files = DRbQS::Transfer.dequeue_all
+        { :result => result, :transfer => transfer_files }
+      end
+
+      def subdirectory_name(task_id)
+        if task_id
+          sprintf("T%08d", task_id)
+        else
+          sprintf("S%08d", (@special_task_number += 1))
+        end
+      end
+      private :subdirectory_name
+
+      def handle_task(obj)
+        task_id, marshal_obj, method_sym, args = obj
+        DRbQS::Temporary.set_sub_directory(subdirectory_name(task_id))
+        begin
+          result_hash = calculate(marshal_obj, method_sym, args)
+          # If task_id is nil then the task is initialization or finalization.
+          # So we do not return results.
+          if task_id
+            if subdir = DRbQS::Temporary.subdirectory
+              result_hash[:tmp] = subdir
+            end
+            result_hash[:id] = task_id
+            send_response([:result, [task_id, result_hash]])
+          end
+        rescue => err
+          send_response([:node_error, err])
+        end
+      end
+      private :handle_task
+    end
+  end
+end