drbqs 0.0.19 → 0.1.0

data/lib/drbqs/execute/register.rb

@@ -18,6 +18,7 @@ module DRbQS
       def __register__(type, setting, name, template, args, &block)
         ary = [name.intern, { :type => type.intern, :template => template, :args => args }]
         type = type.to_s
+        # Note that if type is :server, then args[0] is hostname
         case block.arity
         when 2
           if DRbQS::Setting::SSH === setting
@@ -33,6 +34,10 @@ module DRbQS
           end
           ary[1][:ssh] = true
           ary[1][:setting] = ssh_setting
+
+          if type == "server" && !template && !args[0]
+            raise ArgumentError, "No hostname of SSH server"
+          end
         when 1
           if DRbQS::Setting::SSH === setting
             raise ArgumentError, "Inherited definition is over ssh."
@@ -40,6 +45,7 @@ module DRbQS
           yield(setting.value)
           ary[1][:ssh] = false
           ary[1][:setting] = setting
+          ary[1][:unix_domain_socket] = (!template && !args[0])
         else
           raise ArgumentError, "Block must take one or two arguments."
         end
@@ -89,12 +95,21 @@ module DRbQS
       # If we omit the 'connect' method then the program tries to connect
       # the name specified as first argument.
       # 
-      # @param [Symbol,String] name Server name
-      # @param [Hash] opts The options of server
-      # @option opts [Boolean] :template Template for other servers to load, not actual server
-      # @option opts [Symbol] :load Inherit definition of other server
+      # @overload server(name, hostname, opts = {})
+      #  @param [Symbol,String] name Server name
+      #  @param [Symbol,String] hostname Hostname of server to access over SSH
+      #  @param [Hash] opts The options of server
+      #  @option opts [Boolean] :template Template for other servers to load, not actual server
+      #  @option opts [Symbol] :load Inherit definition of other server
+      # 
+      # @overload server(name, opts = {})
+      #  If hostname is omitted, the server uses UNIX domain socket and executes some nodes together.
+      #  @param [Symbol,String] name Server name
+      #  @param [Hash] opts The options of server
+      #  @option opts [Boolean] :template Template for other servers to load, not actual server
+      #  @option opts [Symbol] :load Inherit definition of other server
       # 
-      # @example A server on localhost
+      # @example A server on localhost (Connections from nodes is over SSH)
       #  server :server_local, "example.com" do |srv|
       #    srv.load "server_definition.rb"
       #    srv.acl "/path/to/acl"
@@ -104,7 +119,7 @@ module DRbQS
       #    srv.sftp_host "example.com"
       #  end
       # 
-      # @example A server over ssh
+      # @example A server executed over SSH
       #  server :server_ssh, "example.co.jp" do |srv, ssh|
       #    srv.load "server_definition.rb"
       #    srv.acl "/path/to/acl"
@@ -120,36 +135,28 @@ module DRbQS
       #    ssh.output "/path/to/output"
       #    ssh.nice 10
       #  end
+      #
+      # @example A server on localhost that uses UNIX domain socket (No node on other computer)
+      #  server :server_unix_domain_socket do |srv|
+      #    srv.load "server_definition.rb"
+      #    srv.execute_node 4
+      #  end
       def server(name, *args, &block)
+        unless block_given?
+          raise ArgumentError, "Block to define settings is not given."
+        end
         name = name.intern
         if ind = @__server__.index { |n, data| name == n }
           old_data = @__server__.delete_at(ind)
         else
           old_data = nil
         end
-        unless block_given?
-          raise ArgumentError, "Block to define settings is not given."
-        end
-        case args.size
-        when 2
-          hostname = args[0]
-          opts = args[1]
-          unless Hash === opts
-            raise ArgumentError, "Options must be hash."
-          end
-        when 1
-          if Hash === args[0]
-            hostname = nil
-            opts = args[0]
-          else
-            hostname = args[0]
-            opts = {}
-          end
-        else
-          unless old_data
-            raise ArgumentError, "Invalid argument size."
-          end
+        opts = args.extract_options!
+        opts.assert_valid_keys(:template, :load)
+        if args.size > 1
+          raise ArgumentError, "Invalid number of arguments."
         end
+        hostname = args[0]
         if old_data
           if opts[:load]
             raise ArgumentError, "Can not set both reconfiguring and loading."
@@ -159,9 +166,6 @@ module DRbQS
         else
           load_def = opts[:load]
         end
-        if !opts[:template] && !hostname
-          raise ArgumentError, "Definition of server '#{name}' needs hostname."
-        end
         __register_server__(opts[:template], name, load_def, hostname, &block)
       end
 
@@ -203,6 +207,7 @@ module DRbQS
       # @example Node group
       #  node :node_group, :group => [:node_local, :node_ssh]
       def node(name, opts = {}, &block)
+        opts.assert_valid_keys(:template, :load, :group)
         name = name.intern
         load_def = opts[:load]
         if ind = @__node__.index { |n, data| name == n }
@@ -233,7 +238,7 @@ module DRbQS
         end
       end
 
-      # @param [Array] *args Symbols of servers
+      # @param [Array] args Symbols of servers
       # @example Clear server definitions
       #  clear_server :server1, :server2
       def clear_server(*args)
@@ -244,7 +249,7 @@ module DRbQS
         end
       end
 
-      # @param [Array] *args Symbols of nodes
+      # @param [Array] args Symbols of nodes
       # @example Clear node definitions
       #  clear_node :node1, :node2
       def clear_node(*args)
@@ -259,10 +264,12 @@ module DRbQS
       # @param [Hash] val the pair of key and value
       # @option val [Fixnum] :port Port number of a server
       # @option val [Symbol] :server Server executed by default
+      # @option val [Array] :node Nodes executed by default if server has no node
       # @option val [String] :log Path of log of a server and nods on localhost
       # @example Set default value
       #  default :port => 13456, :server => :server_local, :log => "/tmp/drbqs_execute_log"
       def default(val = {})
+        val.assert_valid_keys(:port, :server, :node, :log)
         val.delete_if { |key, v| !v }
         if val[:server]
           val[:server] = val[:server].intern
@@ -290,6 +297,7 @@ module DRbQS
       # @example Set usage
       #  usage :message => 'Calculate some value', :server => 'server.rb'
       def usage(opts = {})
+        opts.assert_valid_keys(:message, :server)
         @__usage__.merge!(opts)
       end
 

data/lib/drbqs/ext/task/command_task.rb

@@ -4,26 +4,23 @@ module DRbQS
 
     # Execute a command and transfer files if needed.
     class CommandExecute
-
-      # :transfer    String or Array
-      # :compress    true or false
-      def initialize(cmd, opts = {})
-        @cmd = cmd
-        unless (Array === @cmd || String === @cmd)
-          raise ArgumentError, "Invalid command: #{@cmd.inspect}"
+      def initialize(cmds, opts = {})
+        opts.assert_valid_keys(:transfer, :compress)
+        @cmds = cmds
+        if String === @cmds
+          @cmds = [@cmds]
+        elsif !(Array === @cmds)
+          raise ArgumentError, "Invalid command: #{@cmds.inspect}"
         end
         @transfer = opts[:transfer]
         @compress = opts[:compress]
       end
 
       def exec
-        case @cmd
-        when Array
-          @cmd.each { |c| system(c) }
-        when String
-          system(@cmd)
+        exit_status_ary = @cmds.map do |cmd|
+          system(cmd)
+          $?.exitstatus
         end
-        exit_status = $?.exitstatus
         if @transfer
           if @transfer.respond_to?(:each)
             @transfer.each { |path| DRbQS::Transfer.enqueue(path, :compress => @compress) }
@@ -31,13 +28,19 @@ module DRbQS
             DRbQS::Transfer.enqueue(@transfer, :compress => @compress)
           end
         end
-        exit_status
+        exit_status_ary
       end
     end
 
-    # &hook takes a server instance and exit number of command.
-    def initialize(cmd, opts = {}, &hook)
-      super(DRbQS::CommandTask::CommandExecute.new(cmd, opts), :exec, &hook)
+    # @override initialize(*cmds, opts = {}, &hook)
+    #  @param [Array] cmds An array of commands which are string or array, that is, arguments of the method 'system'.
+    #  @param [Hash] opts Hash of options
+    #  @param [Proc] hook Block same as that of DRbQS::Task object, which takes a server instance and an array of exit statuses.
+    #  @option opts [String,Array] :transfer Paths to be transfered after finish of commands
+    #  @option opts [boolean] :compress Compress files before transfering or not
+    def initialize(*args, &hook)
+      opts = args.extract_options!
+      super(DRbQS::CommandTask::CommandExecute.new(args, opts), :exec, &hook)
     end
   end
 end

data/lib/drbqs/node/connection.rb

@@ -12,8 +12,7 @@ module DRbQS
       end
 
       def create_id_string
-        t = Time.now
-        sprintf("#{Socket.gethostname}:%d", Process.pid)
+        "#{Socket.gethostname}:#{Process.pid}:#{Time.now.strftime('%s')}"
       end
       private :create_id_string
 

data/lib/drbqs/node/node.rb

@@ -1,5 +1,4 @@
 require 'drbqs/task/task'
-require 'drbqs/utility/temporary'
 require 'drbqs/utility/transfer/transfer_client'
 require 'drbqs/node/connection'
 require 'drbqs/node/task_client'
@@ -18,7 +17,7 @@ module DRbQS
     WAIT_NEW_TASK_TIME = 1
     SAME_HOST_GROUP = :local
 
-    # @param [String] acces_uri Set the uri of server
+    # @param [String] access_uri Set the uri of server
     # @param [Hash] opts Options of a node
     # @option opts [Fixnum] :process Number of worker processes
     # @option opts [Array] :group An array of group symbols
@@ -62,7 +61,7 @@ module DRbQS
         transfer_file(files)
       end
       if subdir = result_hash[:tmp]
-        FileUtils.rm_r(result_hash[:tmp])
+        FileUtils.rm_r(subdir)
       end
       @task_client.queue_result(task_id, result_hash[:result])
     end
@@ -223,7 +222,7 @@ module DRbQS
     private :send_task_to_worker
 
     def clear_node_files
-      DRbQS::Temporary.delete_all
+      DRbQS::Temporary.delete
       @config.list.node.delete(Process.pid)
     end
     private :clear_node_files

data/lib/drbqs/server/server.rb

@@ -161,12 +161,12 @@ module DRbQS
     end
     private :all_tasks_assigned?
 
-    # @param [DRbQS::task] task
+    # @param [Array] tasks An array of DRbQS::task objects, which are executed at initialization
     def set_initialization_task(*tasks)
       @message.set_initialization_tasks(tasks)
     end
 
-    # @param [DRbQS::task] task
+    # @param [Array] tasks An array of DRbQS::task objects, which are executed at initialization
     def set_finalization_task(*tasks)
       @finalization_task.concat(tasks)
     end
@@ -175,10 +175,10 @@ module DRbQS
     # @note When we set both :empty_queue and task generators,
     #  hook of :empty_queue is prior to task generators.
     # @param [:empty_queue,:process_data,:finish] key Set the type of hook.
+    # @param [Proc] block The block is obligatory and takes server itself as an argument.
     # @option opts [Fixnum] :repeat If we execute the hook specified times then the hook is deleted.
     #   If the value is nil, the hook is repeated without limit.
     # @option opts [String] :name Name of the hook. If the value is nil then the name is automatically created.
-    # @param [Proc] &block block is obligatory and takes server itself as an argument.
     def add_hook(key, opts = {}, &block)
       if key == :process_data
         if @hook.number_of_hook(:process_data) != 0
@@ -395,6 +395,11 @@ module DRbQS
     end
     private :first_task_generator_init
 
+    def clear_server_files
+      DRbQS::Temporary.delete
+    end
+    private :clear_server_files
+
     def wait
       first_task_generator_init
       loop do
@@ -407,6 +412,7 @@ module DRbQS
           sleep(WAIT_TIME_NEW_RESULT)
         end
       end
+      clear_server_files
     end
   end
 end

data/lib/drbqs/setting/base.rb

@@ -10,6 +10,7 @@ module DRbQS
     # A base class having options of commands.
     # We must define a method 'exec' this method in a child class.
     class Base
+      include DRbQS::Misc
       extend Forwardable
 
       attr_reader :source

data/lib/drbqs/setting/data_container.rb

@@ -3,7 +3,7 @@ module DRbQS
     class Source
       class DataContainer < BasicObject
         attr_accessor :argument
-        attr_reader :__data__
+        attr_reader :__data__, :__array__
 
         def initialize(array_class)
           @argument = []
@@ -27,7 +27,7 @@ module DRbQS
       end
 
       def self.clone_container(obj)
-        cl = DRbQS::Setting::Source::DataContainer.new(Array)
+        cl = DRbQS::Setting::Source::DataContainer.new(obj.__array__)
         cl.argument = obj.argument.clone
         obj.__data__.each do |key, val|
           cl.__data__[key] = val.clone

data/lib/drbqs/setting/execute.rb

@@ -10,7 +10,7 @@ module DRbQS
           [:port, :server, :node].each do |key|
             register_key(key, :check => 1)
           end
-          [:no_server, :no_node, :information, :help].each do |key|
+          [:no_server, :no_node, :wait_server_finish, :information, :help].each do |key|
             register_key(key, :bool => true)
           end
           set_argument_condition(:<=, 1)
@@ -43,6 +43,7 @@ module DRbQS
         else
           @mode = nil
         end
+        @wait_server_finish = set?(:wait_server_finish)
         if @mode != :help && !@definition
           raise DRbQS::Setting::InvalidArgument, "Definition file must be specified."
         end
@@ -50,7 +51,10 @@ module DRbQS
 
       def exec(io = nil)
         process_def = DRbQS::ProcessDefinition.new(@server, @node, @port, io)
-        process_def.load(@definition) if @definition
+        if @definition
+          process_def.load(@definition)
+          process_def.test_consistency
+        end
         case @mode
         when :help
           io.puts process_def.usage if io
@@ -63,6 +67,9 @@ module DRbQS
           unless @no_node
             process_def.execute_node
           end
+          if !@no_server && @wait_server_finish
+            process_def.wait_server_finish
+          end
         end
         true
       end

data/lib/drbqs/setting/node.rb

@@ -69,7 +69,12 @@ module DRbQS
       private :parse_group
 
       def exec(io = nil)
-        return true if exec_as_daemon
+        if @__daemon__
+          fork do
+            exec_as_daemon
+          end
+          return true
+        end
 
         @options[:load].each do |path|
           io.puts "load #{path}" if io
@@ -80,6 +85,7 @@ module DRbQS
         end
 
         if io
+          io.puts "PID: #{Process.pid}"
           io.puts "Connect to #{@uri}"
           io.puts "Execute #{@process_num} processes"
         end

data/lib/drbqs/setting/server.rb

@@ -25,9 +25,12 @@ module DRbQS
 
       def parse_test
         @test_opts[:profile] = get(:profile)
-        @test_opts[:printer] = get_first(:printer) do |val|
+        @test_opts[:printer] = get_first(:profile_printer) do |val|
           val.intern
         end
+        if @test_opts[:printer] && !@test_opts[:profile]
+          @test_opts[:profile] = true
+        end
         if test = get_first(:test)
           @command_type = "test_#{test.to_s}"
         end

data/lib/drbqs/task/registrar.rb

@@ -11,8 +11,7 @@ module DRbQS
       end
 
       # Add tasks to server.
-      # @param [DRbQS::Task] arg Add an ojbect of DRbQS::Task.
-      # @param [Array] arg Add all elements of an array of objects of DRbQS::Task.
+      # @param [DRbQS::Task,Array] arg DRbQS::Task object or array of DRbQS::Task objects, which is added to pool of tasks
       def add(arg)
         case arg
         when DRbQS::Task

data/lib/drbqs/utility/misc.rb

@@ -16,6 +16,11 @@ module DRbQS
       end
     end
 
+    def uri_drbunix(path)
+      "drbunix:#{path}"
+    end
+    module_function :uri_drbunix
+
     # @param [Hash] opts The arguments of URI of server
     # @option opts [Fixnum] :port Port number of a server
     # @option opts [String] :host Hostname of a server
@@ -33,7 +38,7 @@ module DRbQS
         elsif File.exist?(path)
           raise ArgumentError, "File #{path} already exists."
         end
-        "drbunix:#{path}"
+        uri_drbunix(path)
       end
     end
     module_function :create_uri