drbqs 0.0.14 → 0.0.15

data/VERSION

@@ -1 +1 @@
-0.0.14
+0.0.15

data/bin/drbqs-execute

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+
+$LOAD_PATH.unshift(File.expand_path(File.dirname(__FILE__) + '/../lib'))
+require 'drbqs/command_line/command_line'
+
+DRbQS::Command::Execute.exec(ARGV)

data/bin/drbqs-manage

@@ -1,6 +1,6 @@
 #!/usr/bin/env ruby
 
 $LOAD_PATH.unshift(File.expand_path(File.dirname(__FILE__) + '/../lib'))
-require 'drbqs/utility/command_line'
+require 'drbqs/command_line/command_line'
 
-DRbQS::CommandManage.exec(ARGV)
+DRbQS::Command::Manage.exec(ARGV)

data/bin/drbqs-node

@@ -1,6 +1,6 @@
 #!/usr/bin/env ruby
 
 $LOAD_PATH.unshift(File.expand_path(File.dirname(__FILE__) + '/../lib'))
-require 'drbqs/utility/command_line'
+require 'drbqs/command_line/command_line'
 
-DRbQS::CommandNode.exec(ARGV)
+DRbQS::Command::Node.exec(ARGV)

data/bin/drbqs-server

@@ -1,6 +1,6 @@
 #!/usr/bin/env ruby
 
 $LOAD_PATH.unshift(File.expand_path(File.dirname(__FILE__) + '/../lib'))
-require 'drbqs/utility/command_line'
+require 'drbqs/command_line/command_line'
 
-DRbQS::CommandServer.exec(ARGV)
+DRbQS::Command::Server.exec(ARGV)

data/bin/drbqs-ssh

@@ -1,6 +1,6 @@
 #!/usr/bin/env ruby
 
 $LOAD_PATH.unshift(File.expand_path(File.dirname(__FILE__) + '/../lib'))
-require 'drbqs/utility/command_line'
+require 'drbqs/command_line/command_line'
 
-DRbQS::CommandSSH.exec(ARGV)
+DRbQS::Command::SSH.exec(ARGV)

data/drbqs.gemspec

@@ -5,14 +5,14 @@
 
 Gem::Specification.new do |s|
   s.name = %q{drbqs}
-  s.version = "0.0.14"
+  s.version = "0.0.15"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Takayuki YAMAGUCHI"]
-  s.date = %q{2011-07-19}
+  s.date = %q{2011-07-29}
   s.description = %q{Task queuing system over network that is implemented by dRuby.}
   s.email = %q{d@ytak.info}
-  s.executables = ["drbqs-manage", "drbqs-node", "drbqs-server", "drbqs-ssh"]
+  s.executables = ["drbqs-execute", "drbqs-manage", "drbqs-node", "drbqs-server", "drbqs-ssh"]
   s.extra_rdoc_files = [
     "LICENSE.txt",
     "README.md"
@@ -25,6 +25,7 @@ Gem::Specification.new do |s|
     "README.md",
     "Rakefile",
     "VERSION",
+    "bin/drbqs-execute",
     "bin/drbqs-manage",
     "bin/drbqs-node",
     "bin/drbqs-server",
@@ -32,19 +33,34 @@ Gem::Specification.new do |s|
     "drbqs.gemspec",
     "example/README.md",
     "example/command/server_def.rb",
-    "example/error/error.rb",
-    "example/error/server_def.rb",
+    "example/error_server/error.rb",
+    "example/error_server/server_def.rb",
+    "example/error_task/error.rb",
+    "example/error_task/server_def.rb",
     "example/server/server.rb",
     "example/sum/server_def.rb",
     "example/sum/sum.rb",
+    "example/sum2/execute_def.rb",
     "example/sum2/server_def.rb",
     "example/sum2/sum.rb",
     "example/transfer/file.rb",
     "example/transfer/server_def.rb",
     "lib/drbqs.rb",
+    "lib/drbqs/command_line/argument.rb",
+    "lib/drbqs/command_line/command_base.rb",
+    "lib/drbqs/command_line/command_execute.rb",
+    "lib/drbqs/command_line/command_line.rb",
+    "lib/drbqs/command_line/command_manage.rb",
+    "lib/drbqs/command_line/command_node.rb",
+    "lib/drbqs/command_line/command_server.rb",
+    "lib/drbqs/command_line/command_ssh.rb",
+    "lib/drbqs/command_line/option_setting.rb",
     "lib/drbqs/config/config.rb",
     "lib/drbqs/config/process_list.rb",
     "lib/drbqs/config/ssh_host.rb",
+    "lib/drbqs/execute/process_define.rb",
+    "lib/drbqs/execute/register.rb",
+    "lib/drbqs/execute/server_define.rb",
     "lib/drbqs/manage/execute_node.rb",
     "lib/drbqs/manage/manage.rb",
     "lib/drbqs/manage/send_signal.rb",
@@ -52,34 +68,47 @@ Gem::Specification.new do |s|
     "lib/drbqs/manage/ssh_shell.rb",
     "lib/drbqs/node/connection.rb",
     "lib/drbqs/node/node.rb",
+    "lib/drbqs/node/state.rb",
     "lib/drbqs/node/task_client.rb",
     "lib/drbqs/server/acl_file.rb",
     "lib/drbqs/server/check_alive.rb",
     "lib/drbqs/server/history.rb",
     "lib/drbqs/server/message.rb",
     "lib/drbqs/server/node_list.rb",
+    "lib/drbqs/server/prof.rb",
     "lib/drbqs/server/queue.rb",
     "lib/drbqs/server/server.rb",
     "lib/drbqs/server/server_hook.rb",
+    "lib/drbqs/server/test/node.rb",
+    "lib/drbqs/server/test/server.rb",
     "lib/drbqs/server/transfer_setting.rb",
+    "lib/drbqs/setting/base.rb",
+    "lib/drbqs/setting/data_container.rb",
+    "lib/drbqs/setting/execute.rb",
+    "lib/drbqs/setting/manage.rb",
+    "lib/drbqs/setting/node.rb",
+    "lib/drbqs/setting/server.rb",
+    "lib/drbqs/setting/setting.rb",
+    "lib/drbqs/setting/source.rb",
+    "lib/drbqs/setting/ssh.rb",
     "lib/drbqs/task/command_task.rb",
     "lib/drbqs/task/task.rb",
     "lib/drbqs/task/task_generator.rb",
-    "lib/drbqs/utility/argument.rb",
-    "lib/drbqs/utility/command_line.rb",
-    "lib/drbqs/utility/command_line/command_base.rb",
-    "lib/drbqs/utility/command_line/command_manage.rb",
-    "lib/drbqs/utility/command_line/command_node.rb",
-    "lib/drbqs/utility/command_line/command_server.rb",
-    "lib/drbqs/utility/command_line/command_ssh.rb",
     "lib/drbqs/utility/misc.rb",
-    "lib/drbqs/utility/server_define.rb",
     "lib/drbqs/utility/temporary.rb",
     "lib/drbqs/utility/transfer/file_transfer.rb",
     "lib/drbqs/utility/transfer/transfer_client.rb",
+    "spec/command_line/command_base_spec.rb",
+    "spec/command_line/commands_spec.rb",
+    "spec/command_line/option_setting_spec.rb",
     "spec/config/config_spec.rb",
     "spec/config/process_list_spec.rb",
     "spec/config/ssh_host_spec.rb",
+    "spec/execute/def/execute1.rb",
+    "spec/execute/def/no_def.rb",
+    "spec/execute/process_define_spec.rb",
+    "spec/execute/register_spec.rb",
+    "spec/execute/server_define_spec.rb",
     "spec/integration_test/01_basic_usage_spec.rb",
     "spec/integration_test/02_use_generator_spec.rb",
     "spec/integration_test/03_use_temporary_file_spec.rb",
@@ -87,6 +116,9 @@ Gem::Specification.new do |s|
     "spec/integration_test/05_server_exit_signal_spec.rb",
     "spec/integration_test/06_node_exit_after_task_spec.rb",
     "spec/integration_test/07_command_server_with_node_spec.rb",
+    "spec/integration_test/08_shutdown_unused_nodes_spec.rb",
+    "spec/integration_test/09_server_process_data_spec.rb",
+    "spec/integration_test/10_test_server_spec.rb",
     "spec/integration_test/definition/server01.rb",
     "spec/integration_test/definition/server02.rb",
     "spec/integration_test/definition/task_obj_definition.rb",
@@ -94,6 +126,7 @@ Gem::Specification.new do |s|
     "spec/manage/send_signal_spec.rb",
     "spec/manage/ssh_shell_spec.rb",
     "spec/node/connection_spec.rb",
+    "spec/node/state_spec.rb",
     "spec/node/task_client_spec.rb",
     "spec/server/acl_file_spec.rb",
     "spec/server/check_alive_spec.rb",
@@ -105,15 +138,19 @@ Gem::Specification.new do |s|
     "spec/server/server_hook_spec.rb",
     "spec/server/server_spec.rb",
     "spec/server/transfer_setting_spec.rb",
+    "spec/setting/base_spec.rb",
+    "spec/setting/data_container_spec.rb",
+    "spec/setting/execute_spec.rb",
+    "spec/setting/manage_spec.rb",
+    "spec/setting/node_spec.rb",
+    "spec/setting/server_spec.rb",
+    "spec/setting/source_spec.rb",
     "spec/spec_helper.rb",
     "spec/task/file_transfer_spec.rb",
     "spec/task/task_generator_spec.rb",
     "spec/task/task_spec.rb",
     "spec/utility/argument_spec.rb",
-    "spec/utility/command_line/command_base_spec.rb",
-    "spec/utility/command_line/commands_spec.rb",
     "spec/utility/misc_spec.rb",
-    "spec/utility/server_define_spec.rb",
     "spec/utility/temporary_spec.rb"
   ]
   s.homepage = %q{http://github.com/ytaka/drbqs}

data/example/error_server/error.rb

@@ -0,0 +1,6 @@
+class TaskTrue
+  def exec
+    sleep(1)
+    true
+  end
+end

data/example/error_server/server_def.rb

@@ -0,0 +1,7 @@
+require_relative 'error.rb'
+
+DRbQS.define_server(:check_alive => 300) do |server, argv, opts|
+  server.queue.add(DRbQS::Task.new(TaskTrue.new, :exec))
+  sleep(1)
+  raise "Error in definition of server."
+end

data/example/sum2/execute_def.rb

@@ -0,0 +1,27 @@
+default :server => :server_local, :port => 13789, :log => '/tmp/drbqs_execute'
+
+usage :message => "Calculate sum of numbers", :server => File.join(File.dirname(__FILE__), 'server_def.rb')
+
+register_server :server_local, "localhost" do |server|
+  server.load File.expand_path(File.join(File.dirname(__FILE__), 'server_def.rb'))
+end
+
+register_server :server_ssh, "localhost" do |server, ssh|
+  ssh.connect 'localhost'
+  ssh.directory File.expand_path(File.dirname(__FILE__))
+  ssh.output "/tmp/drbqs_ssh/server"
+  server.load 'server_def.rb'
+end
+
+register_node :node_local do |node|
+  node.load File.expand_path(File.join(File.dirname(__FILE__), 'sum.rb'))
+  node.process 2
+end
+
+register_node :node_ssh do |node, ssh|
+  ssh.connect 'localhost'
+  ssh.directory File.expand_path(File.dirname(__FILE__))
+  ssh.output "/tmp/drbqs_ssh/node"
+  node.load 'sum.rb'
+  node.process 2
+end

data/lib/drbqs/command_line/argument.rb

@@ -0,0 +1,29 @@
+module DRbQS
+  class Command
+    module Argument
+      def split_arguments(argv, split = '--')
+        if n = argv.index(split)
+          [argv[0..(n - 1)], argv[(n + 1)..-1]]
+        else
+          [argv, []]
+        end
+      end
+      module_function :split_arguments
+
+      def check_argument_size(argv, *args)
+        n = argv.size
+        args.each_slice(2).each do |ary|
+          if ary.size == 2
+            unless n.__send__(*ary)
+              raise ArgumentError, "Invalid arguments number. Please refer to documents."
+            end
+          else
+            raise ArgumentError, "Invalid argument to check array size."
+          end
+        end
+        true
+      end
+      module_function :check_argument_size
+    end
+  end
+end

data/lib/drbqs/command_line/command_base.rb

@@ -0,0 +1,81 @@
+module DRbQS
+  class Command
+    class Base
+      include DRbQS::Misc
+      include DRbQS::Command::Argument
+
+      @@command_name = File.basename($PROGRAM_NAME)
+
+      def self.exec(argv)
+        obj = self.new
+        obj.parse_option(argv)
+        obj.exec
+      end
+
+      def initialize(klass =  DRbQS::Setting::Base, help_message = nil)
+        @help_message = help_message
+        @opt_setting = DRbQS::Command::OptionSetting.new(@help_message, klass.new)
+      end
+
+      def setting
+        @opt_setting.setting
+      end
+
+      def exit_normally
+        Kernel.exit(0)
+      end
+      private :exit_normally
+
+      def exit_unusually
+        Kernel.exit(1)
+      end
+      private :exit_unusually
+
+      def exit_invalid_option
+        Kernel.exit(2)
+      end
+      private :exit_invalid_option
+
+      def option_parser_base(argv, options = {}, &block)
+        @opt_setting.define(options, &block)
+        begin
+          @opt_setting.parse!(argv)
+        rescue DRbQS::Setting::InvalidLogLevel => err
+          $stderr.print err.to_s << "\n\n" << @help_message.to_s
+          exit_invalid_option
+        rescue OptionParser::InvalidOption
+          $stderr.print "error: Invalid Option\n\n" << @help_message.to_s
+          exit_invalid_option
+        rescue OptionParser::InvalidArgument
+          $stderr.print "error: Invalid Argument\n\n" << @help_message.to_s
+          exit_invalid_option
+        end
+        argv
+      end
+      private :option_parser_base
+
+      def parse_arguments!
+        setting.parse!
+      end
+      private :parse_arguments!
+
+      def exec
+        begin
+          parse_arguments!
+          setting.exec($stdout)
+          exit_normally
+        rescue DRb::DRbConnError => err
+          $stderr.puts "error: Can not connect. #{err.to_s}"
+          exit_unusually
+        rescue DRbQS::Setting::InvalidArgument => err
+          mes = "error: Invalid command argument. #{err.to_s}\n\n" << @help_message.to_s
+          $stderr.print mes
+          exit_invalid_option
+        rescue => err
+          output_error(err, $stderr)
+          exit_unusually
+        end
+      end
+    end
+  end
+end

data/lib/drbqs/command_line/command_execute.rb

@@ -0,0 +1,33 @@
+module DRbQS
+  class Command
+    class Execute < DRbQS::Command::Base
+      HELP_MESSAGE =<<HELP
+Usage: #{@@command_name} <definition> [options ...]
+       #{@@command_name} <definition> [options ...] [-- server options ...]
+  Execute DRbQS server and some nodes from definition file.
+
+HELP
+
+      def initialize
+        super(DRbQS::Setting::Execute, HELP_MESSAGE)
+      end
+
+      def parse_option(argv)
+        args, server_args = split_arguments(argv)
+        args = option_parser_base(args) do
+          set(:port, '--port NUM', Integer, 'Set the port number.')
+          set(:server, '--server STR', String, 'Set the key of server.')
+          set(:node, '--node STR', String, 'Set the comma sparated key of nodes.')
+          set(:no_server, '--no-server', 'Not execute server.')
+          set(:no_node, '--no-node', 'Not execute node.')
+          set(:information, '-i', '--information', 'Show information.')
+          set(:help, '-h', '--help', 'Show this command help and usage of definition file.') do |opt|
+            $stdout.print opt
+          end
+        end
+        setting.set_argument(*args)
+        setting.server_argument = server_args
+      end
+    end
+  end
+end

data/lib/drbqs/command_line/command_line.rb

@@ -0,0 +1,19 @@
+require 'drbqs'
+require 'drbqs/manage/manage'
+require 'drbqs/manage/execute_node'
+require 'drbqs/command_line/argument'
+require 'drbqs/command_line/option_setting'
+require 'drbqs/command_line/command_base'
+require 'optparse'
+
+Version = DRbQS::VERSION
+
+module DRbQS
+  class Command
+    autoload :Manage, 'drbqs/command_line/command_manage'
+    autoload :Server, 'drbqs/command_line/command_server'
+    autoload :Node, 'drbqs/command_line/command_node'
+    autoload :SSH, 'drbqs/command_line/command_ssh'
+    autoload :Execute, 'drbqs/command_line/command_execute'
+  end
+end

data/lib/drbqs/command_line/command_manage.rb

@@ -0,0 +1,34 @@
+module DRbQS
+  class Command
+    class Manage < DRbQS::Command::Base
+      HELP_MESSAGE =<<HELP
+Usage: #{@@command_name} <command> [arguments ...]
+  Manage DRbQS server by sending messages.
+  <command> is 'signal', 'status', 'process', or 'initialize'.
+
+  #{@@command_name} signal <uri> server-exit
+  #{@@command_name} signal <uri> node-exit-after-task <node_number>
+  #{@@command_name} signal <uri> node-wake <node_number>
+  #{@@command_name} signal <uri> node-sleep <node_number>
+  #{@@command_name} status <uri>
+  #{@@command_name} history <uri>
+  #{@@command_name} process list
+  #{@@command_name} process clear
+  #{@@command_name} send string <uri> <string>
+  #{@@command_name} send file <uri> <path>
+  #{@@command_name} initialize
+
+HELP
+
+      def initialize
+        super(DRbQS::Setting::Manage, HELP_MESSAGE)
+      end
+
+      def parse_option(argv)
+        argv = option_parser_base(argv, :debug => true) do
+        end
+        setting.set_argument(*argv)
+      end
+    end
+  end
+end

data/lib/drbqs/command_line/command_node.rb

@@ -0,0 +1,26 @@
+module DRbQS
+  class Command
+    class Node < DRbQS::Command::Base
+      HELP_MESSAGE =<<HELP
+Usage: #{@@command_name} <uri> [options ...]
+  Start DRbQS nodes connecting to <uri>.
+
+HELP
+
+      def initialize
+        super(DRbQS::Setting::Node, HELP_MESSAGE)
+      end
+
+      def parse_option(argv)
+        argv = option_parser_base(argv, :log_level => true, :daemon => true, :debug => true) do
+          set(:load, '-l', '--load FILE', String, 'Add a file to load.')
+          set(:process, '-P', '--process NUM', Integer, 'Set the number of node processes to execute.')
+          set(:loadavg, '--loadavg STR', String, 'Set the threshold load average to sleep.')
+          set(:log_prefix, '--log-prefix STR', String, "Set the prefix of log files. The default is '#{setting.default[:log_prefix][0]}'.")
+          set(:log_stdout, '--log-stdout', 'Use stdout for outputting logs. This option cancels --log-prefix.')
+        end
+        setting.set_argument(*argv)
+      end
+    end
+  end
+end

data/lib/drbqs/command_line/command_server.rb

@@ -0,0 +1,41 @@
+module DRbQS
+  class Command
+    class Server < DRbQS::Command::Base
+      @@command_name = File.basename($PROGRAM_NAME)
+
+      HELP_MESSAGE =<<HELP
+Usage: #{@@command_name} <definition.rb> [other files ...] [options ...]
+       #{@@command_name} <definition.rb> [other files ...] [options ...] -- [server options ...]
+  Start DRbQS server of definition files.
+
+HELP
+
+      def initialize
+        super(DRbQS::Setting::Server, HELP_MESSAGE)
+      end
+
+      def parse_option(argv)
+        command_argv, server_argv = split_arguments(argv)
+
+        command_argv = option_parser_base(command_argv, :log_level => true, :daemon => true, :debug => true) do
+          set(:port, '-p PORT', '--port', Integer, 'Set the port number of server.')
+          set(:unix, '-u PATH', '--unix', String, 'Set the path of unix domain socket.')
+          set(:acl, '--acl FILE', String, 'Set a file to define ACL.')
+          set(:log_file, '--log-file STR', String, "Set the path of log file. If this options is not set, use STDOUT.")
+          set(:sftp_user, '--sftp-user USER', String, 'Set the user of sftp destination.')
+          set(:sftp_host, '--sftp-host HOST', String, 'Set the host of sftp destination.')
+          set(:profile, '--profile', 'Use profile for test exec.')
+          set(:profile_printer, '--profile-printer PRINTER', String,
+              'Set the printer type for profile. The value is :flat, :graph, :graphhtml, or :calltree.')
+          set(:test, '--test STR', String, 'Execute test.')
+          set(:execute_node, '--execute-node NUM', Integer, 'Execute nodes.')
+          set(:help, '-h', '--help', 'Show this command help and server specific help.') do |opt|
+            $stdout.print opt
+          end
+        end
+        setting.set_argument(*command_argv)
+        setting.set_server_argument(*server_argv)
+      end
+    end
+  end
+end

data/lib/drbqs/command_line/command_ssh.rb

@@ -0,0 +1,51 @@
+module DRbQS
+  class Command
+    class SSH < DRbQS::Command::Base
+      HELP_MESSAGE =<<HELP
+Usage: #{@@command_name} <command> [arguments ...]
+  Execute command over SSH.
+  <command> is 'list', 'show', 'environment', 'execute', 'server', or 'node'.
+
+  #{@@command_name} list
+  #{@@command_name} show <configuration>
+  #{@@command_name} environment <destination>
+  #{@@command_name} execute <destination> [options ...] -- [arguments ...]
+  #{@@command_name} server <destination> [options ...] -- [arguments ...]
+  #{@@command_name} node <destination> [options ...] -- [arguments ...]
+
+HELP
+
+      def initialize
+        super(DRbQS::Setting::SSH, HELP_MESSAGE)
+      end
+
+      def parse_option(argv)
+        args, mode_args = split_arguments(argv)
+        args = option_parser_base(args, :debug => true) do |opt|
+          set(:directory, '-d DIR', '--directory DIR', String, 'Set the base directory over ssh.')
+          set(:shell, '--shell STR', String, 'Set the shell over ssh')
+          set(:rvm, '--rvm STR', String, 'Ruby version to use on RVM over ssh.')
+          set(:rvm_init, '--rvm-init PATH', String, 'Path of script to initialize RVM over ssh.')
+          set(:output, '-o DIR', '--output DIR', String, 'Directory path that a server and nodes output.')
+          set(:nice, '--nice NUM', Integer, 'Set the value of nice for a server and nodes. The default is 10.')
+        end
+        setting.set_argument(*args)
+        case setting.get_argument[0]
+        when 'server'
+          parser = DRbQS::Command::Server.new
+          parser.parse_option(mode_args)
+          setting.mode_setting = parser.setting
+        when 'node'
+          parser = DRbQS::Command::Node.new
+          parser.parse_option(mode_args)
+          setting.mode_setting = parser.setting
+        when 'execute'
+          setting.configure_mode_setting(:execute) do |mode_setting|
+            mode_setting.argument.concat(mode_args)
+          end
+        end
+      end
+    end
+  end
+end
+

data/lib/drbqs/command_line/option_setting.rb

@@ -0,0 +1,42 @@
+module DRbQS
+  class Command
+    class OptionSetting
+      attr_reader :setting
+
+      def initialize(help_message, setting)
+        @opt_parser = OptionParser.new(help_message)
+        @setting = setting
+      end
+
+      def set(*args, &block)
+        unless @setting
+          raise "Not in method 'define'."
+        end
+        @opt_parser.on(*args[1..-1]) do |v|
+          @setting.set(args[0], v)
+          if block_given?
+            yield(@opt_parser)
+          end
+        end
+      end
+
+      def define(options = {}, &block)
+        instance_eval(&block) if block_given?
+        if options[:log_level]
+          set(:log_level, '--log-level LEVEL', String,
+              "Set the log level. The value accepts 'fatal', 'error', 'warn', 'info', and 'debug'. The default is 'error'.")
+        end
+        if options[:daemon]
+          set(:daemon, '--daemon OUT', String, 'Execute as daemon and set output file for stdout and stderr.')
+        end
+        if options[:debug]
+          set(:debug, '--debug', 'Set $DEBUG true.')
+        end
+      end
+
+      def parse!(argv)
+        @opt_parser.parse!(argv)
+      end
+    end
+  end
+end