drbqs 0.0.13 → 0.0.14

data/lib/drbqs/server/transfer_setting.rb

@@ -0,0 +1,30 @@
+require 'drbqs/utility/transfer/transfer_client'
+
+module DRbQS
+  class Server
+    class TransferSetting
+      attr_accessor :host, :user, :directory
+
+      def initialize(host, user, directory)
+        @host = host
+        @user = user
+        @directory = directory
+        @created = false
+      end
+
+      def create(directory, opts = {})
+        return nil if @created
+        @directory = directory || @directory
+        return nil if !@directory
+        @created = true
+        transfer_client = DRbQS::TransferClient.new(@directory)
+        transfer_client.make_directory
+        if host = opts[:host] || @host
+          user = opts[:user] || @user || ENV['USER']
+          transfer_client.set_sftp(user, host)
+        end
+        transfer_client
+      end
+    end
+  end
+end

data/lib/drbqs/task/command_task.rb

@@ -0,0 +1,43 @@
+module DRbQS
+  # Execute a command and transfer files if needed.
+  class CommandExecute
+
+    # :transfer    String or Array
+    # :compress    true or false
+    def initialize(cmd, opts = {})
+      @cmd = cmd
+      unless (Array === @cmd || String === @cmd)
+        raise ArgumentError, "Invalid command: #{@cmd.inspect}"
+      end
+      @transfer = opts[:transfer]
+      @compress = opts[:compress]
+    end
+
+    def exec
+      case @cmd
+      when Array
+        @cmd.each { |c| system(c) }
+      when String
+        system(@cmd)
+      end
+      exit_status = $?.exitstatus
+      if @transfer
+        if @transfer.respond_to?(:each)
+          @transfer.each { |path| DRbQS::FileTransfer.enqueue(path, :compress => @compress) }
+        else
+          DRbQS::FileTransfer.enqueue(@transfer, :compress => @compress)
+        end
+      end
+      exit_status
+    end
+  end
+
+  # Class to define tasks such that we execute a command.
+  class CommandTask < Task
+
+    # &hook takes a server instance and exit number of command.
+    def initialize(cmd, opts = {}, &hook)
+      super(CommandExecute.new(cmd, opts), :exec, &hook)
+    end
+  end
+end

data/lib/drbqs/{task.rb → task/task.rb}

@@ -1,3 +1,6 @@
+require 'drbqs/utility/transfer/file_transfer'
+require 'drbqs/task/task_generator'
+
 module DRbQS
 
   # The tasks defined by this class are sent to nodes and
@@ -34,51 +37,22 @@ module DRbQS
         @args == other.instance_variable_get(:@args)
     end
 
-    def self.execute_task(marshal_obj, method_sym, args)
-      obj = Marshal.load(marshal_obj)
-      obj.__send__(method_sym, *args)
-    end
-  end
-
-  class CommandExecute
-
-    # :transfer    String or Array
-    # :compress    true or false
-    def initialize(cmd, opts = {})
-      @cmd = cmd
-      unless (Array === @cmd || String === @cmd)
-        raise ArgumentError, "Invalid command: #{@cmd.inspect}"
-      end
-      @transfer = opts[:transfer]
-      @compress = opts[:compress]
-    end
-
-    def exec
-      case @cmd
-      when Array
-        @cmd.each { |c| system(c) }
-      when String
-        system(@cmd)
+    def exec_hook(server, result)
+      if @hook
+        @hook.call(server, result)
+        true
+      else
+        nil
       end
-      exit_status = $?.exitstatus
-      if @transfer
-        if Array === @transfer
-          @transfer.each { |path| DRbQS::FileTransfer.enqueue(path, @compress) }
-        else
-          DRbQS::FileTransfer.enqueue(@transfer, @compress)
-        end
-      end
-      exit_status
     end
-  end
 
-  class CommandTask < Task
-    # &hook takes a server instance and exit number of command.
-    def initialize(cmd, opts = {}, &hook)
-      super(CommandExecute.new(cmd, {}), :exec, &hook)
+    def self.execute_task(marshal_obj, method_sym, args)
+      obj = Marshal.load(marshal_obj)
+      obj.__send__(method_sym, *args)
     end
   end
 
+  # Class to group a number of objects to process tasks.
   class TaskContainer
     def initialize(task_ary)
       @data = task_ary.map.with_index do |task, i|
@@ -104,6 +78,8 @@ module DRbQS
     end
   end
 
+  # Task to group a number of tasks,
+  # which uses TaskContainer and manages hooks of the tasks.
   class TaskSet < Task
     def initialize(task_ary)
       @original_hook = task_ary.map do |task|
@@ -118,4 +94,7 @@ module DRbQS
       end
     end
   end
+
 end
+
+require 'drbqs/task/command_task'

data/lib/drbqs/{task_generator.rb → task/task_generator.rb}

@@ -68,6 +68,8 @@ module DRbQS
       end
     end
 
+    # Initialize fider to create tasks.
+    # This method must be called in thread to create tasks.
     def init
       @fiber_init.call if @fiber_init
     end

data/lib/drbqs/utility/argument.rb

@@ -0,0 +1,27 @@
+module DRbQS
+  module CommandLineArgument
+    def split_arguments(argv, split = '--')
+      if n = argv.index(split)
+        [argv[0..(n - 1)], argv[(n + 1)..-1]]
+      else
+        [argv, []]
+      end
+    end
+    module_function :split_arguments
+
+    def check_argument_size(argv, *args)
+      n = argv.size
+      args.each_slice(2).each do |ary|
+        if ary.size == 2
+          unless n.__send__(*ary)
+            raise "Invalid arguments number. Please refer to documents."
+          end
+        else
+          raise ArgumentError, "Invalid argument to check array size."
+        end
+      end
+      true
+    end
+    module_function :check_argument_size
+  end
+end

data/lib/drbqs/utility/command_line/command_base.rb

@@ -0,0 +1,27 @@
+module DRbQS
+  class CommandBase
+    include DRbQS::CommandLineArgument
+    @@command_name = File.basename($PROGRAM_NAME)
+
+    def self.exec(argv)
+      obj = self.new
+      obj.parse_option(argv)
+      obj.exec
+    end
+
+    def exit_normally
+      Kernel.exit(0)
+    end
+    private :exit_normally
+
+    def exit_unusually
+      Kernel.exit(1)
+    end
+    private :exit_unusually
+
+    def exit_invalid_option
+      Kernel.exit(2)
+    end
+    private :exit_invalid_option
+  end
+end

data/lib/drbqs/utility/command_line/command_manage.rb

@@ -0,0 +1,121 @@
+module DRbQS
+  class CommandManage < CommandBase
+    HELP_MESSAGE =<<HELP
+Usage: #{@@command_name} <command> [arguments ...]
+  Manage DRbQS server by sending messages.
+  <command> is 'signal', 'status', 'process', or 'initialize'.
+
+  #{@@command_name} signal <uri> server-exit
+  #{@@command_name} signal <uri> node-exit-after-task <node_number>
+  #{@@command_name} status <uri>
+  #{@@command_name} process list
+  #{@@command_name} process clear
+  #{@@command_name} initialize
+
+HELP
+
+    def parse_option(argv)
+      begin
+        OptionParser.new(HELP_MESSAGE) do |opt|
+          opt.on('--debug', 'Set $DEBUG true.') do |v|
+            $DEBUG = true
+          end
+          opt.parse!(argv)
+        end
+      rescue OptionParser::InvalidOption
+        $stderr.print "error: Invalid Option\n\n" << HELP_MESSAGE
+        exit_invalid_option
+      rescue OptionParser::InvalidArgument
+        $stderr.print "error: Invalid Argument\n\n" << HELP_MESSAGE
+        exit_invalid_option
+      end
+      @mode = argv.shift
+      @argv = argv
+      @manage = DRbQS::Manage.new
+    end
+
+    def command_initialize
+      check_argument_size(@argv, :>=, 0, :<=, 1)
+      @manage.set_home_directory(@argv[0])
+      @manage.create_config
+      exit_normally
+    end
+    private :command_initialize
+
+    def command_process
+      check_argument_size(@argv, :>=, 0, :<=, 1)
+      if @argv[0] == 'clear'
+        @manage.clear_process
+        exit_normally
+      end
+      list = @manage.list_process
+      $stdout.puts "Server"
+      list[:server].each do |uri, data|
+        $stdout.puts "#{uri}"
+        data.each do |k, v|
+          $stdout.puts sprintf("  %-10s  %s", k, v)
+        end
+      end
+      $stdout.puts "\nNode"
+      list[:node].each do |pid, data|
+        $stdout.puts "#{pid}"
+        data.each do |k, v|
+          $stdout.puts sprintf("  %-10s  %s", k, v)
+        end
+      end
+      exit_normally
+    end
+    private :command_process
+
+    def command_status
+      check_argument_size(@argv, :==, 1)
+      @manage.set_uri(@argv[0])
+      if status = @manage.get_status
+        $stdout.puts status
+      end
+      exit_normally
+    end
+    private :command_status
+
+    def command_signal
+      @manage.set_uri(@argv[0])
+      signal = @argv[1]
+      case signal
+      when 'server-exit'
+        check_argument_size(@argv, :==, 2)
+        @manage.send_exit_signal
+        exit_normally
+      when 'node-exit-after-task'
+        check_argument_size(@argv, :==, 3)
+        node_id = @argv[2].to_i
+        @manage.send_node_exit_after_task(node_id)
+        exit_normally
+      else
+        $stderr.print "error: Invalid signal '#{signal}'\n\n" << HELP_MESSAGE
+        exit_unusually
+      end
+    end
+    private :command_signal
+
+    def exec
+      case @mode
+      when 'initialize'
+        command_initialize
+      when 'process'
+        command_process
+      when 'status'
+        command_status
+      when 'signal'
+        command_signal
+      end
+      $stderr.print "error: Invalid command '#{@mode}'\n\n" << HELP_MESSAGE
+      exit_invalid_option
+    rescue DRb::DRbConnError => err
+      $stderr.puts "error: Can not connect server: #{err.to_s}"
+      exit_unusually
+    rescue => err
+      $stderr.print "error: #{err.to_s}\n" << err.backtrace.join("\n")
+      exit_unusually
+    end
+  end
+end

data/lib/drbqs/utility/command_line/command_node.rb

@@ -0,0 +1,103 @@
+module DRbQS
+  class CommandNode < CommandBase
+    HELP_MESSAGE =<<HELP
+Usage: #{@@command_name} [<uri>] [<process_number>] [options ...]
+  Start DRbQS nodes connecting to <uri>.
+
+HELP
+
+    LOG_PREFIX_DEFAULT = 'drbqs_node'
+    LOG_LEVEL_DEFAULT = Logger::ERROR
+
+    def parse_option(argv)
+      options = {
+        :log_prefix => LOG_PREFIX_DEFAULT,
+        :log_level => LOG_LEVEL_DEFAULT,
+        :load => []
+      }
+
+      begin
+        OptionParser.new(HELP_MESSAGE) do |opt|
+          opt.on('-l', '--load FILE', String, 'Add a file to load.') do |v|
+            options[:load] << File.expand_path(v)
+          end
+          opt.on('--log-prefix STR', String, "Set the prefix of log files. The default is '#{LOG_PREFIX_DEFAULT}'.") do |v|
+            options[:log_prefix] = v
+          end
+          opt.on('--log-level LEVEL', String,
+                 "Set the log level. The value accepts 'fatal', 'error', 'warn', 'info', and 'debug'. The default is 'error'.") do |v|
+            if /^(fatal)|(error)|(warn)|(info)|(debug)$/i =~ v
+              options[:log_level] = eval("Logger::#{v.upcase}")
+            else
+              $stderr.print "error: Invalid log level.\n\n" << HELP_MESSAGE
+              exit_invalid_option
+            end
+          end
+          opt.on('--log-stdout', 'Use stdout for outputting logs. This option cancels --log-prefix.') do |v|
+            options[:log_prefix] = nil
+          end
+          opt.on('--debug', 'Set $DEBUG true.') do |v|
+            $DEBUG = true
+          end
+          opt.parse!(argv)
+        end
+      rescue OptionParser::InvalidOption
+        $stderr.print "error: Invalid Option\n\n" << HELP_MESSAGE
+        exit_invalid_option
+      rescue OptionParser::InvalidArgument
+        $stderr.print "error: Invalid Argument\n\n" << HELP_MESSAGE
+        exit_invalid_option
+      end
+      @options = options
+      @argv = argv
+    end
+
+    def parse_argv_array
+      process_num = 1
+      uri = nil
+      @argv.each do |arg|
+        if /^\d+$/ =~ arg
+          process_num = arg.to_i
+        elsif uri
+          $stderr.print "error: More than one uris is set.\n\n" << HELP_MESSAGE
+          exit_invalid_option
+        else
+          uri = arg
+        end
+      end
+      uri ||= DRbQS::Misc.create_uri
+      [process_num, uri]
+    end
+    private :parse_argv_array
+
+    def exec
+      if @argv.size > 2
+        $stderr.print "error: Too many arguments.\n\n" << HELP_MESSAGE
+        exit_invalid_option
+      end
+      process_num, uri = parse_argv_array
+
+      @options[:load].each do |v|
+        $stdout.puts "load #{v}"
+        load v
+      end
+
+      $stdout.puts "Connect to #{uri}"
+      $stdout.puts "Execute #{process_num} processes"
+
+      if @options[:log_prefix]
+        if /\/$/ =~ @options[:log_prefix]
+          @options[:log_prefix] += 'out'
+        end
+      end
+
+      exec_node = DRbQS::ExecuteNode.new(uri, @options[:log_prefix], @options[:log_level])
+      exec_node.execute(process_num)
+      exec_node.wait
+      exit_normally
+    rescue => err
+      $stderr.print "error: #{err.to_s}\n" << err.backtrace.join("\n")
+      exit_unusually
+    end
+  end
+end

data/lib/drbqs/utility/command_line/command_server.rb

@@ -0,0 +1,165 @@
+module DRbQS
+  class CommandServer < CommandBase
+    @@command_name = File.basename($PROGRAM_NAME)
+
+    HELP_MESSAGE =<<HELP
+Usage: #{@@command_name} <definition.rb> [other files ...] [options ...]
+       #{@@command_name} <definition.rb> [other files ...] [options ...] -- [server options ...]
+  Start DRbQS server of definition files.
+
+HELP
+
+    NODE_INTERVAL_TIME = 1
+
+    def parse_option(argv)
+      @options = {
+        :log_file => STDOUT
+      }
+      @command_type = :server_start
+
+      test_opts = {}
+      @execute_node_number = nil
+      @command_argv, @server_argv = split_arguments(argv)
+
+      begin
+        OptionParser.new(HELP_MESSAGE) do |opt|
+          opt.on('-p PORT', '--port', Integer, 'Set the port number of server.') do |v|
+            @options[:port] = v
+          end
+          opt.on('-u PATH', '--unix', String, 'Set the path of unix domain socket.') do |v|
+            @options[:unix] = v
+          end
+          opt.on('--acl FILE', String, 'Set a file to define ACL.') do |v|
+            @options[:acl] = v
+          end
+          opt.on('--log-file STR', String, "Set the path of log file. If this options is not set, use STDOUT.") do |v|
+            @options[:log_file] = v
+          end
+          opt.on('--log-level LEVEL', String,
+                 "Set the log level. The value accepts 'fatal', 'error', 'warn', 'info', and 'debug'. The default is 'error'.") do |v|
+            if /^(fatal)|(error)|(warn)|(info)|(debug)$/i =~ v
+              @options[:log_level] = eval("Logger::#{v.upcase}")
+            else
+              $stderr.print "error: Invalid log level.\n\n" << HELP_MESSAGE
+              exit_invalid_option
+            end
+          end
+          opt.on('--file-directory DIR', String, 'Set the file archive directory.') do |v|
+            @options[:file_directory] = v
+          end
+          opt.on('--sftp-user USER', String, 'Set the user of sftp destination.') do |v|
+            @options[:sftp_user] = v
+          end
+          opt.on('--sftp-host HOST', String, 'Set the host of sftp destination.') do |v|
+            @options[:sftp_host] = v
+          end
+          opt.on('--profile', 'Use profile for test exec.') do |v|
+            @test_opts[:profile] = true
+          end
+          opt.on('--debug', 'Set $DEBUG true.') do |v|
+            $DEBUG = true
+          end
+          opt.on('--test STR', String, 'Execute test.') do |v|
+            @command_type = "test_#{v}"
+          end
+          opt.on('--execute-node NUM', Integer, 'Execute nodes.') do |v|
+            @execute_node_number = v
+          end
+          opt.on('-h', '--help', 'Show help.') do |v|
+            $stdout.print opt
+            @command_type = :help
+          end
+          opt.parse!(@command_argv)
+        end
+      rescue OptionParser::InvalidOption
+        $stderr.print "error: Invalid Option\n\n" << HELP_MESSAGE
+        exit_invalid_option
+      rescue OptionParser::InvalidArgument
+        $stderr.print "error: Invalid Argument\n\n" << HELP_MESSAGE
+        exit_invalid_option
+      end
+    end
+
+    def command_test
+      s = @command_type.split('_')[1].split(',')
+      type = s[0].intern
+      DRbQS.test_server(@options, type, s[1..-1], @test_opts)
+    end
+    private :command_test
+
+    def command_start_server
+      DRbQS.start_server(@options)
+    end
+    private :command_start_server
+
+    def command_server_with_nodes
+      server_pid = fork do
+        DRbQS.start_server(@options)
+      end
+      uri = DRbQS::Misc.create_uri(@options)
+      manage = DRbQS::Manage.new(:uri => uri)
+      if manage.wait_server_process(server_pid)
+        node_log_file = nil
+        unless IO === @options[:log_file]
+          node_log_file = FileName.create(@options[:log_file], :add => :always, :position => :middle, :delimiter => '', :format => "_node_%02d")
+        end
+        exec_node = DRbQS::ExecuteNode.new(uri, node_log_file, @options[:log_level])
+        exec_node.execute(@execute_node_number, NODE_INTERVAL_TIME)
+        exec_node.wait
+      else
+        $stderr.puts "error: Server has been terminated."
+        exit_unusually
+      end
+    end
+    private :command_server_with_nodes
+
+    def command_server_help
+      begin
+        @command_argv.each do |path|
+          if File.exist?(path)
+            load path
+          end
+        end
+        if mes = DRbQS.option_help_message
+          $stdout.print "\n" << mes
+        end
+      rescue => err
+        $stderr.print "error: Load invalid file.\n#{err.to_s}\n#{err.backtrace.join("\n")}"
+        exit_invalid_option
+      end
+      exit_normally
+    end
+    private :command_server_help
+
+    def exec
+      if @command_type == :help
+        command_server_help
+      end
+      if @command_argv.size == 0 || !(@command_argv.all? { |path| File.exist?(path) })
+        $stderr.print "error: There are nonexistent files.\n\n" << HELP_MESSAGE
+        exit_unusually
+      end
+      @command_argv.each do |path|
+        load path
+      end
+      unless @options[:acl]
+        @options[:acl] = DRbQS::Config.new.get_acl_file
+      end
+      DRbQS.parse_option(@server_argv)
+      case @command_type
+      when /^test/
+        command_test
+      else
+        if @execute_node_number
+          command_server_with_nodes
+        else
+          command_start_server
+        end
+      end
+      exit_normally
+    rescue => err
+      $stderr.print "error: #{err.to_s}\n" << err.backtrace.join("\n")
+      exit_unusually
+    end
+  end
+end