drbqs 0.0.13 → 0.0.14

data/Gemfile

@@ -6,12 +6,15 @@ source "http://rubygems.org"
 # Add dependencies to develop your gem here.
 # Include everything needed to run rake, tests, features, etc.
 group :development do
-  gem "rspec", ">= 2.5.0"
-  gem "yard", "~> 0.6.0"
-  gem "bundler", "~> 1.0.0"
-  gem "jeweler", "~> 1.5.2"
+  gem "rspec", ">= 2.6.0"
+  gem "yard", ">= 0.7.2"
+  gem "bundler", ">= 1.0.15"
+  gem "jeweler", ">= 1.6.2"
   gem "rcov", ">= 0"
-  gem "filename", ">= 0.0.5"
-  gem "net-ssh", ">= 2.1.3"
-  gem "net-ssh-shell", ">= 0.1.0"
+  gem "filename", ">= 0.1.0"
+  gem "user_config", ">= 0.0.1"
+  gem "net-ssh", ">= 2.1.4"
+  gem "net-ssh-shell", ">= 0.2.0"
+  gem "net-sftp", ">= 2.0.5"
+  gem "sys-proctable"
 end

data/README.md

@@ -12,7 +12,7 @@ and connect the nodes to the server.
 The behavior of nodes requests tasks, gets tasks from a server, processes the tasks,
 and sends results of the tasks to the server.
 The nodes work repeatedly until they get exit signal from server.
-The server prepares tasks and checks that nodes are alive.
+The server prepares tasks and signals to check that nodes are alive.
 If the server does not communicate with the nodes unexpectedly,
 the server deletes the nodes from node list and
 requeues their calculating tasks.
@@ -23,7 +23,7 @@ after the server accepts them from nodes.
 The tasks are made from objects, an instance method of them, and its arguments.
 Because we use Marshal.dump and Marshal.load for communication of a server and nodes,
 the objects and the arguments must be marshalized.
-And also we tell the server and the nodes the definision of class of the objects and the arguments.
+And also we tell the server and the nodes the definition of class of the objects and the arguments.
 
 ## Requirements
 
@@ -37,7 +37,7 @@ And we use net-ssh and net-ssh-shell to execute servers and nodes over ssh.
 We prepare a class to send tasks over network,
 which has data and a method to deal with tasks.
 
-We make sum.rb as the following.
+For example, we make sum.rb as the following.
 
     class Sum
       def initialize(start_num, end_num)
@@ -92,7 +92,8 @@ when task queue is empty.
 
 ### Task generator
 
-Arguments of DRbQS::TaskGenerator.new define instance variables.
+Arguments of DRbQS::TaskGenerator.new define instance variables,
+which is implemented by Fiber and is executed by server's requests.
 
     task_generator = DRbQS::TaskGenerator.new(:abc => 'ABC', :def => 123, :data => [1, 2, 3])
 
@@ -104,7 +105,7 @@ The above example defines the following instance variables.
 
 Then, DRbQS::TaskGenerator#set method defines generation of tasks.
 The block of the method is evaluated in the context of task_generator.
-For the above example we can use @abc, @def, and @data.
+We can use @abc, @def, and @data on the above example.
 
     task_generator.set do
       @data.each do |i|
@@ -112,8 +113,8 @@ For the above example we can use @abc, @def, and @data.
       end
     end
 
-DRbQS::TaskGenerator#create_add_task creates a task
-and the task is returned by DRbQS::TaskGenerator#new_tasks.
+DRbQS::TaskGenerator#create_add_task set a task
+and DRbQS::TaskGenerator#new_tasks actually creates the task.
 The arguments of DRbQS::TaskGenerator#create_add_task is
 the same as DRbQS::Task.new.
 
@@ -122,18 +123,58 @@ we set the generator by DRbQS::Server#add_task_generator.
 
 ### Start node and connect server
 
-Because nodes needs class Sum,
+Because nodes need class Sum,
 the nodes load sum.rb when they starts.
-Then, we type in terminal.
+Then, we use '-l' option for command 'drbqs-node'.
+That is, we type in terminal.
 
     drbqs-node druby://localhost:13500/ -l sum.rb
 
-To use two cpu cores we execute two processes by the following.
+We execute two processes to use two CPU cores.
 
     drbqs-node 2 druby://localhost:13500/ -l sum.rb
 
 Then, if it succeeds, the calculation starts.
-If it finishes, the server and node ends.
+If it finishes, the server and node end.
+
+## Provided task
+
+### DRbQS::Task
+
+DRbQS::Task is the basic class to define tasks of DRbQS,
+whose objects are consisted of a object having method to process a task
+and hook for returned result.
+Basically, we define objects of DRbQS::Task and
+give the objects to a server.
+
+### DRbQS::TaskSet
+
+DRbQS::TaskSet is a child class of DRbQS::Task and consists of group a number of tasks.
+Objects of the class are generated when we set the option :collect to DRbQS::TaskGenerator#set
+and therefore we are unaware of the objects of DRbQS::TaskSet
+in many cases.
+
+### DRbQS::CommandTask
+
+DRbQS::CommandTask is a class to create tasks to execute some command.
+
+## Temporary file
+
+We can use temporary directories and files on nodes.
+In methods to calculate tasks,
+DRbQS::Temporary.file returns a name of temporary file and
+DRbQS::Temporary.directory returns a name of temporary directory.
+These temporary files and directories are deleted
+after the task is completed.
+
+## File transfer
+
+When a task is finished on a node,
+we can transfer files from a server to a client.
+To be more precise, we enqueue a file by DRbQS::FileTransfer.enqueue
+in methods to calculate tasks and
+files in the queue are automatically transferred.
+Then, original files on nodes are deleted after transferring.
 
 ## Contributing to drbqs
  

data/Rakefile

@@ -21,29 +21,43 @@ Jeweler::Tasks.new do |gem|
   gem.authors = ["Takayuki YAMAGUCHI"]
   # Include your dependencies below. Runtime dependencies are required when using your gem,
   # and development dependencies are only needed for development (ie running rake tasks, tests, etc)
+  gem.add_runtime_dependency 'filename', '>= 0.1.0'
+  gem.add_runtime_dependency "user_config", ">= 0.0.2"
   gem.add_runtime_dependency 'net-ssh', '>= 2.1.3'
   gem.add_runtime_dependency 'net-ssh-shell', '>= 0.1.0'
-  gem.add_runtime_dependency 'filename', '>= 0.0.5'
-  gem.add_development_dependency 'rspec', '>= 2.5.0'
+  gem.add_runtime_dependency "net-sftp", ">= 2.0.5"
+  gem.add_runtime_dependency "sys-proctable"
 end
 Jeweler::RubygemsDotOrgTasks.new
 
-require 'rspec/core'
-require 'rspec/core/rake_task'
-RSpec::Core::RakeTask.new(:spec) do |spec|
-  spec.pattern = FileList['spec/**/*_spec.rb']
+desc "Run all specs"
+task 'spec:all' => ['spec:unit', 'spec:integration']
+
+desc "Run specs of unit test"
+task 'spec:unit' do
+  filelist = FileList['spec/**/*_spec.rb'].delete_if do |path|
+    /integration_test/ =~ path
+  end
+  filelist.each do |path|
+    sh "rspec #{path}"
+  end
+  Rake::Task['clean:temporary'].execute
 end
 
-RSpec::Core::RakeTask.new(:rcov) do |spec|
-  spec.pattern = 'spec/**/*_spec.rb'
-  spec.rcov = true
+desc "Run specs of integration test"
+task 'spec:integration' do
+  FileList['spec/integration_test/**/*_spec.rb'].sort.each do |path|
+    sh "rspec #{path}"
+  end
+  Rake::Task['clean:temporary'].execute
 end
 
-task :default => :spec
+task :default => 'spec:all'
 
 require 'yard'
 YARD::Rake::YardocTask.new
 
+desc "Update version of drbqs.rb"
 task "version:constant" do
   dir = File.dirname(__FILE__)
   path = File.join(dir, 'lib/drbqs.rb')
@@ -53,3 +67,11 @@ task "version:constant" do
     f.print data.sub(/^  VERSION = '.*'$/, "  VERSION = '#{version}'")
   end
 end
+
+desc "Remove temporary home directory for specs."
+task "clean:temporary" do
+  dir = File.join(File.dirname(__FILE__), 'spec', 'home_for_spec')
+  if File.exist?(dir)
+    FileUtils.rm_r(dir)
+  end
+end

data/VERSION

@@ -1 +1 @@
-0.0.13
+0.0.14

data/bin/drbqs-manage

@@ -1,96 +1,6 @@
 #!/usr/bin/env ruby
 
-require 'drbqs'
-require 'optparse'
+$LOAD_PATH.unshift(File.expand_path(File.dirname(__FILE__) + '/../lib'))
+require 'drbqs/utility/command_line'
 
-Version = DRbQS::VERSION
-
-help_message =<<HELP
-Usage: #{File.basename(__FILE__)} <command> [arguments ...]
-Manage DRbQS server by sending messages.
-<command> is 'exit-signal', 'initialize', or 'ssh'.
-
-HELP
-
-def check_argument_size(argv, check_method, n)
-  unless argv.size.__send__(check_method, n)
-    raise "Invalid arguments number. Please refer '#{File.basename(__FILE__)} -h'."
-  end
-end
-
-command = ARGV[0]
-argv, command_args = DRbQS::Manage.split_arguments(ARGV)
-
-options = {}
-if /^ssh/ =~ command
-  check_argument_size(argv, :>=, 2)
-  ssh_host =DRbQS::SSHHost.new
-  path, options = ssh_host.get_options(argv[1])
-  $stdout.puts "Use configuration: #{path}" if path
-  argv[1] = options[:dest] if options[:dest]
-end
-
-begin
-  OptionParser.new(help_message) do |opt|
-    opt.on('--debug', 'Set $DEBUG true.') do |v|
-      $DEBUG = true
-    end
-    opt.on('--dir DIR', String, 'Set the base directory over ssh.') do |v|
-      options[:dir] = v
-    end
-    opt.on('--shell STR', String, 'Set the shell over ssh') do |v|
-      options[:shell] = v
-    end
-    opt.on('--rvm STR', String, 'Ruby version to use on RVM over ssh.') do |v|
-      options[:rvm] = v
-    end
-    opt.on('--rvm-init PATH', String, 'Path of script to initialize RVM over ssh.') do |v|
-      options[:rvm_init] = v
-    end
-    opt.on('--output PATH', String, 'File path that stdout and stderr are output to over ssh.') do |v|
-      options[:output] = v
-    end
-    opt.on('--nice NUM', Integer, 'Set the value for nice command.') do |v|
-      options[:nice] = v
-    end
-    opt.on('--nohup', 'Use nohup command.') do |v|
-      options[:nohup] = true
-    end
-    opt.parse!(argv)
-  end
-rescue OptionParser::InvalidOption
-  $stderr.print <<MES
-error: Invalid Option
-#{help_message}
-MES
-  exit(2)
-rescue OptionParser::InvalidArgument
-  $stderr.print <<MES
-error: Invalid Argument
-#{help_message}
-MES
-  exit(2)
-end
-
-manage = DRbQS::Manage.new
-
-case command
-when 'exit-signal'
-  check_argument_size(argv, :==, 2)
-  uri = argv[1]
-  manage.send_exit_signal(uri)
-when 'status'
-  check_argument_size(argv, :==, 2)
-  uri = argv[1]
-  if status = manage.get_status(uri)
-    $stdout.puts status
-  end
-when 'initialize'
-  check_argument_size(argv, :==, 1)
-  manage.create_config
-when 'ssh'
-  dest = argv[1]
-  manage.execute_over_ssh(dest, options, command_args)
-else
-  raise "Invalid command: #{command}"
-end
+DRbQS::CommandManage.exec(ARGV)

data/bin/drbqs-node

@@ -1,92 +1,6 @@
 #!/usr/bin/env ruby
 
-require 'drbqs'
-require 'optparse'
+$LOAD_PATH.unshift(File.expand_path(File.dirname(__FILE__) + '/../lib'))
+require 'drbqs/utility/command_line'
 
-Version = DRbQS::VERSION
-
-help_message =<<HELP
-Usage: #{File.basename(__FILE__)} [<uri>] [<process_number>] [options ...]
-Start DRbQS nodes connecting to <uri>.
-
-HELP
-
-LOG_PREFIX_DEFAULT = 'drbqs_node'
-LOG_LEVEL_DEFAULT = Logger::ERROR
-
-options = {
-  :log_prefix => LOG_PREFIX_DEFAULT,
-  :log_level => LOG_LEVEL_DEFAULT,
-  :load => []
-}
-
-begin
-  OptionParser.new(help_message) do |opt|
-    opt.on('-l', '--load FILE', String, 'Add a file to load.') do |v|
-      options[:load] << File.expand_path(v)
-    end
-    opt.on('--log-prefix STR', String, "Set the prefix of log files. The default is '#{LOG_PREFIX_DEFAULT}'.") do |v|
-      options[:log_prefix] = v
-    end
-    opt.on('--log-level LEVEL', String,
-           "Set the log level. The value accepts 'fatal', 'error', 'warn', 'info', and 'debug'. The default is 'error'.") do |v|
-      if /^(fatal)|(error)|(warn)|(info)|(debug)$/i =~ v
-        options[:log_level] = eval("Logger::#{v.upcase}")
-      else
-        raise "Invalid log level."
-      end
-    end
-    opt.on('--log-stdout', 'Use stdout for outputting logs. This option cancels --log-prefix.') do |v|
-      options[:log_prefix] = nil
-    end
-    opt.on('--debug', 'Set $DEBUG true.') do |v|
-      $DEBUG = true
-    end
-    opt.parse!(ARGV)
-  end
-rescue OptionParser::InvalidOption
-  $stderr.print <<MES
-error: Invalid Option
-#{help_message}
-MES
-  exit(2)
-rescue OptionParser::InvalidArgument
-  $stderr.print <<MES
-error: Invalid Argument
-#{help_message}
-MES
-  exit(2)
-end
-
-if ARGV.size > 2
-  raise "Too many arguments. Please refer '#{File.basename(__FILE__)} -h'."
-end
-
-process_num = 1
-uri = "druby://localhost:#{DRbQS::ROOT_DEFAULT_PORT}"
-
-ARGV.each do |arg|
-  if /^\d+$/ =~ arg
-    process_num = arg.to_i
-  else
-    uri = arg
-  end
-end
-
-options[:load].each do |v|
-  puts "load #{v}"
-  load v
-end
-
-puts "Connect to #{uri}"
-puts "Execute #{process_num} processes"
-
-if options[:log_prefix]
-  if /\/$/ =~ options[:log_prefix]
-    options[:log_prefix] += 'out'
-  end
-end
-
-exec_node = DRbQS::ExecuteNode.new(uri, options[:log_prefix], options[:log_level])
-exec_node.execute(process_num)
-exec_node.wait
+DRbQS::CommandNode.exec(ARGV)

data/bin/drbqs-server

@@ -1,120 +1,6 @@
 #!/usr/bin/env ruby
 
-require 'drbqs'
-require 'optparse'
+$LOAD_PATH.unshift(File.expand_path(File.dirname(__FILE__) + '/../lib'))
+require 'drbqs/utility/command_line'
 
-Version = DRbQS::VERSION
-
-help_message =<<HELP
-Usage: #{File.basename(__FILE__)} <definition.rb> [other files ...] [options ...]
-       #{File.basename(__FILE__)} <definition.rb> [other files ...] [options ...] -- [server options ...]
-Start DRbQS server of definition files.
-
-HELP
-
-options = {
-  :log_file => STDOUT,
-  :command_type => :server_start,
-  :acl => DRbQS::Config.get_acl_file
-}
-
-test_opts = {}
-execute_node_number = nil
-
-command_argv, server_argv = DRbQS::Manage.split_arguments(ARGV)
-
-begin
-  OptionParser.new(help_message) do |opt|
-    opt.on('-p PORT', '--port', Integer, 'Set the port number of server.') do |v|
-      options[:port] = v
-    end
-    opt.on('--acl FILE', String, 'Set a file to define ACL.') do |v|
-      options[:acl] = v
-    end
-    opt.on('--log-file STR', String, "Set the path of log file. If this options is not set, use STDOUT.") do |v|
-      options[:log_file] = v
-    end
-    opt.on('--log-level LEVEL', String,
-           "Set the log level. The value accepts 'fatal', 'error', 'warn', 'info', and 'debug'. The default is 'error'.") do |v|
-      if /^(fatal)|(error)|(warn)|(info)|(debug)$/i =~ v
-        options[:log_level] = eval("Logger::#{v.upcase}")
-      else
-        raise "Invalid log level."
-      end
-    end
-    opt.on('--file-directory DIR', String, 'Set the file archive directory.') do |v|
-      options[:file_directory] = v
-    end
-    opt.on('--scp-user USER', String, 'Set the user of scp destination.') do |v|
-      options[:scp_user] = v
-    end
-    opt.on('--scp-host HOST', String, 'Set the host of scp destination.') do |v|
-      options[:scp_host] = v
-    end
-    opt.on('--profile', 'Use profile for test exec.') do |v|
-      test_opts[:profile] = true
-    end
-    opt.on('--debug', 'Set $DEBUG true.') do |v|
-      $DEBUG = true
-    end
-    opt.on('--test STR', String, 'Execute test.') do |v|
-      options[:command_type] = "test_#{v}"
-    end
-    opt.on('--execute-node NUM', Integer, 'Execute nodes.') do |v|
-      execute_node_number = v
-    end
-    opt.parse!(command_argv)
-  end
-rescue OptionParser::InvalidOption
-  $stderr.print <<MES
-error: Invalid Option
-#{help_message}
-MES
-  exit(2)
-rescue OptionParser::InvalidArgument
-  $stderr.print <<MES
-error: Invalid Argument
-#{help_message}
-MES
-  exit(2)
-end
-
-if command_argv.size == 0 || !File.exist?(command_argv[0])
-  raise "Invalid arguments. Please refer '#{File.basename(__FILE__)} -h'."
-end
-
-command_argv.each do |path|
-  puts "load #{path}"
-  load path
-end
-
-WAIT_SERVER_TIME = 0.3
-NODE_INTERVAL_TIME = 1
-
-DRbQS.parse_option(server_argv)
-case options[:command_type]
-when /^test/
-  s = options[:command_type].split('_')[1].split(',')
-  type = s[0].intern
-  DRbQS.test_server(options, type, s[1..-1], test_opts)
-else
-  if execute_node_number
-    fork do
-      DRbQS.start_server(options)
-    end
-    uri = "druby://:#{options[:port] || DRbQS::ROOT_DEFAULT_PORT}"
-    manage = DRbQS::Manage.new
-    begin
-      sleep(WAIT_SERVER_TIME)
-    end while !manage.get_status(uri)
-    node_log_file = nil
-    unless IO === options[:log_file]
-      node_log_file = FileName.create(options[:log_file], :add => :always, :position => :middle, :delimiter => '', :format => "_node_%02d")
-    end
-    exec_node = DRbQS::ExecuteNode.new(uri, node_log_file, options[:log_level])
-    exec_node.execute(execute_node_number, NODE_INTERVAL_TIME)
-    exec_node.wait
-  else
-    DRbQS.start_server(options)
-  end
-end
+DRbQS::CommandServer.exec(ARGV)

data/bin/drbqs-ssh

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+
+$LOAD_PATH.unshift(File.expand_path(File.dirname(__FILE__) + '/../lib'))
+require 'drbqs/utility/command_line'
+
+DRbQS::CommandSSH.exec(ARGV)