drbqs 0.0.8 → 0.0.9

data/README.md

@@ -0,0 +1,151 @@
+# drbqs
+
+Task queuing system over network that is implemented by dRuby.
+
+## Summary
+
+To use DRbQS, first, we start a server of DRbQS.
+Second, we execute nodes (on same host or other hosts)
+loading libraries required by the server
+and connect the nodes to the server.
+
+The behavior of nodes requests tasks, gets tasks from a server, processes the tasks,
+and sends results of the tasks to the server.
+The nodes work repeatedly until they get exit signal from server.
+The server prepares tasks and checks that nodes are alive.
+If the server does not communicate with the nodes unexpectedly,
+the server deletes the nodes from node list and
+requeues their calculating tasks.
+We can set hooks for tasks.
+The hooks are executed for results of the tasks
+after the server accepts them from nodes.
+
+The tasks are made from objects, an instance method of them, and its arguments.
+Because we use Marshal.dump and Marshal.load for communication of a server and nodes,
+the objects and the arguments must be marshalized.
+And also we tell the server and the nodes the definision of class of the objects and the arguments.
+
+## Requirements
+
+DRbQS uses Fiber, so ruby requires version 1.9.
+And we use net-ssh and net-ssh-shell to execute servers and nodes over ssh.
+
+## Usage
+
+### Preparation
+
+We prepare a class to send tasks over network,
+which has data and a method to deal with tasks.
+
+We make sum.rb as the following.
+
+    class Sum
+      def initialize(start_num, end_num)
+        @num = [start_num, end_num]
+      end
+    
+      def exec
+        (@num[0]..@num[1]).inject(0) { |sum, i| sum += i }
+      end
+    end
+
+The Sum class calculates sum of numbers from start_num to end_num.
+The task we want to calculate is summation of numbers.
+
+### Start server
+
+We make server.rb as the following.
+
+    require_relative 'sum.rb'
+    
+    DRbQS.define_server(:finish_exit => true) do |server, argv, opts|
+      10.step(100, 10) do |i|
+        task = DRbQS::Task.new(Sum.new(i - 10, i), :exec)
+        server.queue.add(task)
+      end
+    end
+
+In terminal, we load server.rb and execute server of drbqs.
+
+    drbqs-server server.rb -p 13500
+
+### Hook of server
+
+We can use two hooks of server: 'empty_queue' and 'finish'.
+
+    DRbQS.define_server do |server, argv, opts|
+      server.add_hook(:empty_queue) do |srv|
+        srv.queue.add( ... )
+      end
+      
+      server.add_hook(:finish) do |srv|
+        srv.exit
+      end
+    end
+
+'finish' hook usually exit server program, but
+an option :finish_exit for DRbQS.define_server or DRbQS.new
+is nearly same.
+
+We can use 'empty_queue' hook for adding tasks
+when task queue is empty.
+
+### Task generator
+
+Arguments of DRbQS::TaskGenerator.new define instance variables.
+
+    task_generator = DRbQS::TaskGenerator.new(:abc => 'ABC', :def => 123, :data => [1, 2, 3])
+
+The above example defines the following instance variables.
+
+    @abc = 'ABC'
+    @def = 123
+    @data = [1, 2, 3]
+
+Then, DRbQS::TaskGenerator#set method defines generation of tasks.
+The block of the method is evaluated in the context of task_generator.
+For the above example we can use @abc, @def, and @data.
+
+    task_generator.set do
+      @data.each do |i|
+        create_add_task(i, :to_s)
+      end
+    end
+
+DRbQS::TaskGenerator#create_add_task creates a task
+and the task is returned by DRbQS::TaskGenerator#new_tasks.
+The arguments of DRbQS::TaskGenerator#create_add_task is
+the same as DRbQS::Task.new.
+
+To use the generator in DRbQS::Server,
+we set the generator by DRbQS::Server#add_task_generator.
+
+### Start node and connect server
+
+Because nodes needs class Sum,
+the nodes load sum.rb when they starts.
+Then, we type in terminal.
+
+    drbqs-node druby://localhost:13500/ -l sum.rb
+
+To use two cpu cores we execute two processes by the following.
+
+    drbqs-node 2 druby://localhost:13500/ -l sum.rb
+
+Then, if it succeeds, the calculation starts.
+If it finishes, the server and node ends.
+
+## Contributing to drbqs
+ 
+- Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet
+- Check out the issue tracker to make sure someone already hasn't requested it and/or contributed it
+- Fork the project
+- Start a feature/bugfix branch
+- Commit and push until you are happy with your contribution
+- Make sure to add tests for it. This is important so I don't break it in a future version unintentionally.
+- Please try not to mess with the Rakefile, version, or history. If you want to have your own version, or is otherwise necessary, that is fine, but please isolate to its own commit so I can cherry-pick around it.
+
+## Copyright
+
+Copyright (c) 2011 Takayuki YAMAGUCHI. See LICENSE.txt for
+further details.

data/Rakefile

@@ -21,8 +21,9 @@ Jeweler::Tasks.new do |gem|
   gem.authors = ["Takayuki YAMAGUCHI"]
   # Include your dependencies below. Runtime dependencies are required when using your gem,
   # and development dependencies are only needed for development (ie running rake tasks, tests, etc)
-  #  gem.add_runtime_dependency 'jabber4r', '> 0.1'
-  #  gem.add_development_dependency 'rspec', '> 1.2.3'
+  gem.add_runtime_dependency 'net-ssh', '>= 2.1.3'
+  gem.add_runtime_dependency 'net-ssh-shell', '>= 0.1.0'
+  gem.add_development_dependency 'rspec', '>= 2.5.0'
 end
 Jeweler::RubygemsDotOrgTasks.new
 

data/VERSION

@@ -1 +1 @@
-0.0.8
+0.0.9

data/bin/drbqs-manage

@@ -12,10 +12,23 @@ Manage DRbQS server by sending messages.
 
 HELP
 
+def check_argument_size(argv, check_method, n)
+  unless argv.size.__send__(check_method, n)
+    raise "Invalid arguments number. Please refer '#{File.basename(__FILE__)} -h'."
+  end
+end
+
 command = ARGV[0]
 argv, command_args = DRbQS::Manage.split_arguments(ARGV)
 
 options = {}
+if /^ssh/ =~ command
+  check_argument_size(argv, :>=, 2)
+  ssh_host =DRbQS::SSHHost.new
+  path, options = ssh_host.get_options(argv[1])
+  $stdout.puts "Use configuration: #{path}" if path
+  argv[1] = options[:dest] if options[:dest]
+end
 
 begin
   OptionParser.new(help_message) do |opt|
@@ -37,6 +50,12 @@ begin
     opt.on('--output PATH', String, 'File path that stdout and stderr are output to over ssh.') do |v|
       options[:output] = v
     end
+    opt.on('--nice NUM', Integer, 'Set the value for nice command.') do |v|
+      options[:nice] = v
+    end
+    opt.on('--nohup', 'Use nohup command.') do |v|
+      options[:nohup] = true
+    end
     opt.parse!(argv)
   end
 rescue OptionParser::InvalidOption
@@ -53,12 +72,6 @@ MES
   exit(2)
 end
 
-def check_argument_size(argv, check_method, n)
-  unless argv.size.__send__(check_method, n)
-    raise "Invalid arguments number. Please refer '#{File.basename(__FILE__)} -h'."
-  end
-end
-
 manage = DRbQS::Manage.new
 
 case command
@@ -66,13 +79,20 @@ when 'exit-signal'
   check_argument_size(argv, :==, 2)
   uri = argv[1]
   manage.send_exit_signal(uri)
+when 'status'
+  check_argument_size(argv, :==, 2)
+  uri = argv[1]
+  $stdout.puts manage.get_status(uri)
 when 'initialize'
   check_argument_size(argv, :==, 1)
   manage.create_config
 when 'ssh'
-  check_argument_size(argv, :>=, 2)
   dest = argv[1]
   manage.execute_over_ssh(dest, options, command_args)
+when 'new-filename'
+  check_argument_size(argv, :==, 2)
+  fname = DRbQS::FileName.new(argv[1], :add => :auto, :type => :time, :position => :suffix)
+  $stdout.puts fname.create(:directory => true)
 else
   raise "Invalid command: #{command}"
 end

data/bin/drbqs-node

@@ -82,6 +82,7 @@ puts "Connect to #{uri}"
 puts "Execute #{process_num} processes"
 
 if options[:log_prefix]
+  require 'fileutils'
   if /\/$/ =~ options[:log_prefix]
     options[:log_prefix] += 'out'
   end

data/drbqs.gemspec

@@ -5,24 +5,24 @@
 
 Gem::Specification.new do |s|
   s.name = %q{drbqs}
-  s.version = "0.0.8"
+  s.version = "0.0.9"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Takayuki YAMAGUCHI"]
-  s.date = %q{2011-03-20}
+  s.date = %q{2011-03-22}
   s.description = %q{Task queuing system over network that is implemented by dRuby.}
   s.email = %q{d@ytak.info}
   s.executables = ["drbqs-manage", "drbqs-node", "drbqs-server"]
   s.extra_rdoc_files = [
     "LICENSE.txt",
-    "README.rdoc"
+    "README.md"
   ]
   s.files = [
     ".document",
     ".rspec",
     "Gemfile",
     "LICENSE.txt",
-    "README.rdoc",
+    "README.md",
     "Rakefile",
     "VERSION",
     "bin/drbqs-manage",
@@ -39,6 +39,8 @@ Gem::Specification.new do |s|
     "example/sum/sum.rb",
     "example/sum2/server_def.rb",
     "example/sum2/sum.rb",
+    "example/transfer/file.rb",
+    "example/transfer/server_def.rb",
     "lib/drbqs.rb",
     "lib/drbqs/acl_file.rb",
     "lib/drbqs/client.rb",
@@ -51,14 +53,18 @@ Gem::Specification.new do |s|
     "lib/drbqs/server.rb",
     "lib/drbqs/server_define.rb",
     "lib/drbqs/server_hook.rb",
-    "lib/drbqs/ssh_shell.rb",
+    "lib/drbqs/ssh/host.rb",
+    "lib/drbqs/ssh/shell.rb",
+    "lib/drbqs/ssh/transfer.rb",
     "lib/drbqs/task.rb",
     "lib/drbqs/task_client.rb",
     "lib/drbqs/task_generator.rb",
+    "lib/drbqs/utils/filename.rb",
     "spec/acl_file_spec.rb",
     "spec/config_spec.rb",
     "spec/connection_spec.rb",
     "spec/data/acl.txt",
+    "spec/filename_spec.rb",
     "spec/manage_spec.rb",
     "spec/message_spec.rb",
     "spec/node_list_spec.rb",
@@ -73,7 +79,8 @@ Gem::Specification.new do |s|
     "spec/task_spec.rb",
     "spec/test/test1.rb",
     "spec/test1_spec.rb",
-    "spec/test2_spec.rb"
+    "spec/test2_spec.rb",
+    "spec/transfer_spec.rb"
   ]
   s.homepage = %q{http://github.com/ytaka/drbqs}
   s.licenses = ["GPL3"]
@@ -84,6 +91,7 @@ Gem::Specification.new do |s|
     "spec/acl_file_spec.rb",
     "spec/config_spec.rb",
     "spec/connection_spec.rb",
+    "spec/filename_spec.rb",
     "spec/manage_spec.rb",
     "spec/message_spec.rb",
     "spec/node_list_spec.rb",
@@ -98,7 +106,8 @@ Gem::Specification.new do |s|
     "spec/task_spec.rb",
     "spec/test/test1.rb",
     "spec/test1_spec.rb",
-    "spec/test2_spec.rb"
+    "spec/test2_spec.rb",
+    "spec/transfer_spec.rb"
   ]
 
   if s.respond_to? :specification_version then
@@ -110,12 +119,18 @@ Gem::Specification.new do |s|
       s.add_development_dependency(%q<bundler>, ["~> 1.0.0"])
       s.add_development_dependency(%q<jeweler>, ["~> 1.5.2"])
       s.add_development_dependency(%q<rcov>, [">= 0"])
+      s.add_runtime_dependency(%q<net-ssh>, [">= 2.1.3"])
+      s.add_runtime_dependency(%q<net-ssh-shell>, [">= 0.1.0"])
+      s.add_development_dependency(%q<rspec>, [">= 2.5.0"])
     else
       s.add_dependency(%q<rspec>, [">= 2.5.0"])
       s.add_dependency(%q<yard>, ["~> 0.6.0"])
       s.add_dependency(%q<bundler>, ["~> 1.0.0"])
       s.add_dependency(%q<jeweler>, ["~> 1.5.2"])
       s.add_dependency(%q<rcov>, [">= 0"])
+      s.add_dependency(%q<net-ssh>, [">= 2.1.3"])
+      s.add_dependency(%q<net-ssh-shell>, [">= 0.1.0"])
+      s.add_dependency(%q<rspec>, [">= 2.5.0"])
     end
   else
     s.add_dependency(%q<rspec>, [">= 2.5.0"])
@@ -123,6 +138,9 @@ Gem::Specification.new do |s|
     s.add_dependency(%q<bundler>, ["~> 1.0.0"])
     s.add_dependency(%q<jeweler>, ["~> 1.5.2"])
     s.add_dependency(%q<rcov>, [">= 0"])
+    s.add_dependency(%q<net-ssh>, [">= 2.1.3"])
+    s.add_dependency(%q<net-ssh-shell>, [">= 0.1.0"])
+    s.add_dependency(%q<rspec>, [">= 2.5.0"])
   end
 end
 

data/example/transfer/file.rb

@@ -0,0 +1,46 @@
+require 'fileutils'
+
+class CreateFile
+  def initialize(n)
+    @n = n
+  end
+
+  def output_to_file
+    path = "/tmp/transfer_test_#{@n.to_s}.txt"
+    open(path, 'w') { |f| f.puts "file#{@n.to_s}" }
+    path
+  end
+
+  def create
+    DRbQS::FileTransfer.enqueue(output_to_file)
+    nil
+  end
+
+  def create_compress
+    DRbQS::FileTransfer.compress_enqueue(output_to_file)
+    nil
+  end
+end
+
+class CreateDirectory
+  def initialize(n)
+    @n = n
+  end
+
+  def output_to_directory
+    path = "/tmp/transfer_test_#{@n.to_s}/"
+    FileUtils.mkdir_p(path)
+    open(File.join(path, 'tmp.txt'), 'w') { |f| f.puts "file#{@n.to_s}" }
+    path
+  end
+
+  def create
+    DRbQS::FileTransfer.enqueue(output_to_directory)
+    nil
+  end
+
+  def create_compress
+    DRbQS::FileTransfer.compress_enqueue(output_to_directory)
+    nil
+  end
+end

data/example/transfer/server_def.rb

@@ -0,0 +1,19 @@
+# 
+# Usage:
+#  drbqs-server server_def.rb
+# 
+
+require_relative 'file.rb'
+
+DRbQS.define_server(:finish_exit => true) do |server, argv, opts|
+  tgen = DRbQS::TaskGenerator.new(:sleep_time => 2)
+  tgen.set do
+    create_add_task(CreateFile.new(1), :create)
+    create_add_task(CreateFile.new(2), :create_compress)
+    create_add_task(CreateDirectory.new(3), :create)
+    create_add_task(CreateDirectory.new(4), :create_compress)
+  end
+  server.add_task_generator(tgen)
+
+  server.set_file_transfer(ENV['USER'], 'localhost', '/tmp/drbqs_transfer_test/')
+end

data/lib/drbqs.rb

@@ -1,6 +1,5 @@
 require 'thread'
 require 'logger'
-require 'fileutils'
 require 'drb'
 require 'drb/acl'
 require 'rinda/tuplespace'
@@ -15,9 +14,13 @@ module DRbQS
   autoload :TaskGenerator, 'drbqs/task_generator'
   autoload :Manage, 'drbqs/manage'
   autoload :Config, 'drbqs/config'
-  autoload :SSHShell, 'drbqs/ssh_shell'
+  autoload :SSHShell, 'drbqs/ssh/shell'
+  autoload :SSHHost, 'drbqs/ssh/host'
   autoload :CommandTask, 'drbqs/task'
   autoload :CommandExecute, 'drbqs/task'
+  autoload :FileName, 'drbqs/utils/filename'
+  autoload :Transfer, 'drbqs/ssh/transfer'
+  autoload :FileTransfer, 'drbqs/ssh/transfer'
 
   ROOT_DEFAULT_PORT = 13500
 end