drbqs 0.0.15 → 0.0.16

data/docs/GettingStarted.md

@@ -0,0 +1,242 @@
+# Getting started
+
+## Outline
+
+We use example/mandelbrot as an example for use of DRbQS.
+To use DRbQS we needs two files: a file to define class for tasks
+and a file to define a server.
+In addition, it is convenient to create a file
+to define processes of a server and nodes.
+
+## Example
+
+### Class to calculate tasks: mandelbrot.rb
+
+    require 'complex'
+    
+    class Mandelbrot
+      def initialize(iterate, threshold)
+        @iterate = iterate
+        @threshold = threshold
+      end
+    
+      def map(z, c)
+        z * z + c
+      end
+    
+      def iterate_map(z, c, &block)
+        z_old = z
+        @iterate.times do |i|
+          z_new = map(z_old, c)
+          z_old = z_new
+          yield(z_old) if block_given?
+        end
+        z_old
+      end
+    
+      def diverge?(c)
+        iterate_map(Complex(0.0, 0.0), c) do |z|
+          if z.abs > @threshold
+            return true
+          end
+        end
+        false
+      end
+    end
+    
+    class CalcMandelbrot
+      def initialize(mandelbrot)
+        @mandelbrot = mandelbrot
+      end
+    
+      def calc(io, xrange, yrange, step_size)
+        xrange.step(step_size) do |x|
+          yrange.step(step_size) do |y|
+            c = Complex(x, y)
+            unless @mandelbrot.diverge?(c)
+              io.puts "#{c.real}\t#{c.imag}"
+            end
+          end
+        end
+      end
+    
+      def calc_save(basename, *args)
+        file = DRbQS::Temporary.file
+        open(file, 'w') do |f|
+          calc(f, *args)
+        end
+        DRbQS::Transfer.enqueue(file, compress: true, rename: basename) # Return basename.
+      end
+    end
+
+
+### Server: server.rb
+
+    require_relative 'mandelbrot.rb'
+    
+    RESULT_DIR = 'result_mandelbrot'
+    
+    DRbQS.option_parser("Calculate Mandelbrot set.") do |prs, opts|
+      prs.on('-s NUM', '--step NUM', Float, 'Set the step size.') do |v|
+        opts[:step] = v
+      end
+      prs.on('-l NUM', '--limit NUM', Float, 'Set the limit to search.') do |v|
+        opts[:limit] = v
+      end
+      prs.on('-i NUM', '--iterate NUM', Integer, 'Set the iterate number.') do |v|
+        opts[:iterate] = v
+      end
+      prs.on('-t NUM', '--threshold NUM', Float, 'Set the threshold value.') do |v|
+        opts[:threshold] = v
+      end
+    end
+    
+    DRbQS.define_server(file_directory: RESULT_DIR) do |server, argv, opts|
+      step_size = opts[:step] || 0.1
+      limit = opts[:limit] || 2.0
+      iterate = opts[:iterate] || 1000
+      threshold = opts[:threshold] || 5
+    
+      mandelbrot = Mandelbrot.new(iterate, threshold)
+      calc = CalcMandelbrot.new(mandelbrot)
+    
+      ranges = [[-limit..0, -limit..0, step_size], [-limit..0, 0..limit, step_size],
+                [0..limit, -limit..0, step_size], [0..limit, 0..limit, step_size]]
+      ranges.each_with_index do |ranges, i|
+        args_ary = ["%02d.txt" % i] + ranges
+        note_str = "#{ranges[0].inspect} #{ranges[1].inspect}"
+        task = DRbQS::Task.new(calc, :calc_save, args: args_ary, note: note_str) do |srv, result|
+          DRbQS::Transfer.decompress(srv, result)
+        end
+        server.queue.add(task)
+      end
+      server.add_hook(:finish) do |srv|
+        puts "Save results to #{RESULT_DIR}"
+        result_path = File.join(RESULT_DIR, "result.txt")
+        Dir.glob(File.join(RESULT_DIR, "*.txt")).sort.each do |path|
+          open(result_path, 'a+') do |out|
+            out.print File.read(path)
+          end
+          FileUtils.rm(path)
+        end
+      end
+    end
+    
+### Definition of processes (optional): execute.rb
+
+    DIR = File.dirname(__FILE__)
+    
+    server :local_server, "localhost" do |srv|
+      srv.load File.join(DIR, 'server.rb')
+    end
+    
+    node :local_node do |nd|
+      nd.load File.join(DIR, 'mandelbrot.rb')
+      nd.process 2 # For dual core CPU
+    end
+
+## mandelbrot.rb
+
+This file defines class Mandelbrot and CalcMandelbrot.
+The class CalcMandelbrot and its method calc\_save are for DRbQS::Task.
+In the method calc\_save we creates a temporary file by DRbQS::Temporary.file
+and save results to the file.
+Then, we rename the temporary file and add it to queue of transferring.
+The method calc\_save returns basename of the file transferred to the server
+and the returned value will become an argument of hook of the task on the server.
+
+## Definition of server
+
+### A file for drbqs-server
+
+The command "drbqs-server" takes a ruby file that defines a DRbQS server
+and gets settings of the server (port number, settings of sftp, and so on)
+from options of command line.
+In the definition file we load necessary files to process tasks,
+set a parser of command line arguments for the server
+by the method "DRbQS.option\_parser",
+and defines the body of server by the method "DRbQS.define\_server".
+
+### DRbQS.option_parser
+
+DRbQS.option_parser takes a block that has two arguments.
+The first argument is a OptionParser object and
+the second argument is a hash
+that is used to save data from options of command line.
+
+### DRbQS.define_server
+
+The block of DRbQS.define\_server takes three arguments.
+An argument of DRbQS.define\_server is the same as DRbQS::Server.new and
+the first block argument is an object of DRbQS::Server.
+the second block argument is ARGV parsed by DRbQS.option_parser and
+the third block argument is the second block argument of DRbQS.option\_parser.
+Using these variables, we can get arguments from command line.
+
+We set tasks on the block of DRbQS.define\_server.
+We create tasks by DRbQS::Task.new and
+add them to queue of the server as the following
+
+    server.queue.add(task)
+
+In the block of DRbQS::Task.new
+we decompress files transferred from nodes.
+Then, when server finishes, :finish hook concatenates the files of results.
+
+### Execute a server
+
+To execute the server of which port number is 12345,
+we type the following on terminal.
+
+    drbqs-server server.rb -p 12345
+
+We can set options of the server.
+First, we confirm the options of the server.
+
+    drbqs-server server.rb -h
+
+The help messages are displayed.
+
+We set the options of the command drbqs-server, add '--',
+and set the options of the server.
+
+For example,
+
+    drbqs-server server.rb --execute-node 2 -- --limit 3.0 --step 0.1
+
+The option --execute-node is useful,
+it executes also nodes connecting to the server just after the server runs.
+
+## Execute nodes
+
+If the server runs with port 12345, then the command
+
+    drbqs-node -l mandelbrot.rb druby://:12345
+
+connects to the server of druby://:12345 and calculate tasks obtained from the server.
+If we want to execute two node processes then we use either -P or -process option.
+The command
+
+    drbqs-node -P 2 -l mandelbrot.rb druby://:12345
+
+execute two node processes.
+
+If the calculation finishes, we find the directory 'result_mandelbrot'
+including 'result.txt'.
+
+## Execution over SSH
+
+For example, if we want to execute on example.com then we type
+
+    drbqs-ssh server user@example.com -d /path/to/mandelbrot -- server.rb -- --limit 3.0 --step 0.1
+
+and
+
+    drbqs-ssh node user@example.com -d /path/to/mandelbrot -- -l mandelbrot.rb -P 2
+
+## Execute a server and nodes all together
+
+We can execute a server and nods simultaneously as daemon processes.
+We type the following
+
+    drbqs-execute execute.rb

data/drbqs.gemspec

@@ -5,14 +5,14 @@
 
 Gem::Specification.new do |s|
   s.name = %q{drbqs}
-  s.version = "0.0.15"
+  s.version = "0.0.16"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
-  s.authors = ["Takayuki YAMAGUCHI"]
-  s.date = %q{2011-07-29}
+  s.authors = [%q{Takayuki YAMAGUCHI}]
+  s.date = %q{2011-08-03}
   s.description = %q{Task queuing system over network that is implemented by dRuby.}
   s.email = %q{d@ytak.info}
-  s.executables = ["drbqs-execute", "drbqs-manage", "drbqs-node", "drbqs-server", "drbqs-ssh"]
+  s.executables = [%q{drbqs-execute}, %q{drbqs-manage}, %q{drbqs-node}, %q{drbqs-server}, %q{drbqs-ssh}]
   s.extra_rdoc_files = [
     "LICENSE.txt",
     "README.md"
@@ -30,6 +30,8 @@ Gem::Specification.new do |s|
     "bin/drbqs-node",
     "bin/drbqs-server",
     "bin/drbqs-ssh",
+    "docs/FormatExecute.md",
+    "docs/GettingStarted.md",
     "drbqs.gemspec",
     "example/README.md",
     "example/command/server_def.rb",
@@ -37,7 +39,18 @@ Gem::Specification.new do |s|
     "example/error_server/server_def.rb",
     "example/error_task/error.rb",
     "example/error_task/server_def.rb",
+    "example/execute/execute.rb",
+    "example/execute/server.rb",
+    "example/execute/task.rb",
+    "example/mandelbrot/README.md",
+    "example/mandelbrot/execute.rb",
+    "example/mandelbrot/mandelbrot.rb",
+    "example/mandelbrot/server.rb",
     "example/server/server.rb",
+    "example/simple/README.md",
+    "example/simple/execute.rb",
+    "example/simple/server.rb",
+    "example/simple/task.rb",
     "example/sum/server_def.rb",
     "example/sum/sum.rb",
     "example/sum2/execute_def.rb",
@@ -58,10 +71,12 @@ Gem::Specification.new do |s|
     "lib/drbqs/config/config.rb",
     "lib/drbqs/config/process_list.rb",
     "lib/drbqs/config/ssh_host.rb",
+    "lib/drbqs/execute/execute_node.rb",
     "lib/drbqs/execute/process_define.rb",
     "lib/drbqs/execute/register.rb",
     "lib/drbqs/execute/server_define.rb",
-    "lib/drbqs/manage/execute_node.rb",
+    "lib/drbqs/ext/task.rb",
+    "lib/drbqs/ext/task/command_task.rb",
     "lib/drbqs/manage/manage.rb",
     "lib/drbqs/manage/send_signal.rb",
     "lib/drbqs/manage/ssh_execute.rb",
@@ -75,11 +90,11 @@ Gem::Specification.new do |s|
     "lib/drbqs/server/history.rb",
     "lib/drbqs/server/message.rb",
     "lib/drbqs/server/node_list.rb",
-    "lib/drbqs/server/prof.rb",
     "lib/drbqs/server/queue.rb",
     "lib/drbqs/server/server.rb",
     "lib/drbqs/server/server_hook.rb",
     "lib/drbqs/server/test/node.rb",
+    "lib/drbqs/server/test/prof.rb",
     "lib/drbqs/server/test/server.rb",
     "lib/drbqs/server/transfer_setting.rb",
     "lib/drbqs/setting/base.rb",
@@ -91,13 +106,15 @@ Gem::Specification.new do |s|
     "lib/drbqs/setting/setting.rb",
     "lib/drbqs/setting/source.rb",
     "lib/drbqs/setting/ssh.rb",
-    "lib/drbqs/task/command_task.rb",
+    "lib/drbqs/task/registrar.rb",
     "lib/drbqs/task/task.rb",
     "lib/drbqs/task/task_generator.rb",
     "lib/drbqs/utility/misc.rb",
     "lib/drbqs/utility/temporary.rb",
-    "lib/drbqs/utility/transfer/file_transfer.rb",
+    "lib/drbqs/utility/transfer/transfer.rb",
     "lib/drbqs/utility/transfer/transfer_client.rb",
+    "lib/drbqs/utility/transfer/transfer_client_connect.rb",
+    "lib/drbqs/utility/transfer/transfer_file_list.rb",
     "spec/command_line/command_base_spec.rb",
     "spec/command_line/commands_spec.rb",
     "spec/command_line/option_setting_spec.rb",
@@ -105,10 +122,12 @@ Gem::Specification.new do |s|
     "spec/config/process_list_spec.rb",
     "spec/config/ssh_host_spec.rb",
     "spec/execute/def/execute1.rb",
+    "spec/execute/def/execute2.rb",
     "spec/execute/def/no_def.rb",
     "spec/execute/process_define_spec.rb",
     "spec/execute/register_spec.rb",
     "spec/execute/server_define_spec.rb",
+    "spec/ext/task/command_task_spec.rb",
     "spec/integration_test/01_basic_usage_spec.rb",
     "spec/integration_test/02_use_generator_spec.rb",
     "spec/integration_test/03_use_temporary_file_spec.rb",
@@ -126,6 +145,7 @@ Gem::Specification.new do |s|
     "spec/manage/send_signal_spec.rb",
     "spec/manage/ssh_shell_spec.rb",
     "spec/node/connection_spec.rb",
+    "spec/node/node_spec.rb",
     "spec/node/state_spec.rb",
     "spec/node/task_client_spec.rb",
     "spec/server/acl_file_spec.rb",
@@ -146,17 +166,20 @@ Gem::Specification.new do |s|
     "spec/setting/server_spec.rb",
     "spec/setting/source_spec.rb",
     "spec/spec_helper.rb",
-    "spec/task/file_transfer_spec.rb",
+    "spec/task/registrar_spec.rb",
     "spec/task/task_generator_spec.rb",
     "spec/task/task_spec.rb",
     "spec/utility/argument_spec.rb",
     "spec/utility/misc_spec.rb",
-    "spec/utility/temporary_spec.rb"
+    "spec/utility/temporary_spec.rb",
+    "spec/utility/transfer/transfer_client_connect_spec.rb",
+    "spec/utility/transfer/transfer_file_list_spec.rb",
+    "spec/utility/transfer/transfer_spec.rb"
   ]
   s.homepage = %q{http://github.com/ytaka/drbqs}
-  s.licenses = ["GPL3"]
-  s.require_paths = ["lib"]
-  s.rubygems_version = %q{1.6.2}
+  s.licenses = [%q{GPL3}]
+  s.require_paths = [%q{lib}]
+  s.rubygems_version = %q{1.8.5}
   s.summary = %q{dRuby Queueing System}
 
   if s.respond_to? :specification_version then

data/example/command/server_def.rb

@@ -3,12 +3,11 @@
 #  drbqs-server server_def.rb
 # 
 
-DRbQS.define_server(:finish_exit => true) do |server, argv, opts|
-  tgen = DRbQS::TaskGenerator.new(:sleep_time => 2)
-  tgen.set do
+DRbQS.define_server do |server, argv, opts|
+  sleep_time = 2
+  server.task_generator do |reg|
     3.times do |i|
-      add_task(DRbQS::CommandTask.new(["sleep #{@sleep_time}", 'echo hello world']))
+      reg.add(DRbQS::CommandTask.new(["sleep #{sleep_time}", 'echo hello world']))
     end
   end
-  server.add_task_generator(tgen)
 end

data/example/execute/execute.rb

@@ -0,0 +1,41 @@
+#!/usr/bin/env drbqs-execute
+# -*-ruby-*-
+
+usage message: "Message of this file", server: File.join(File.dirname(__FILE__), 'server.rb'), log: "/tmp/drbqs/log"
+
+default server: :server1, port: 12345, node: [:node1, :node3, :node5]
+
+ssh_directory = "/ssh/path/to"
+
+server :server1, 'example.com' do |server, ssh|
+  ssh.directory ssh_directory
+  ssh.output "/path/to/log"
+  ssh.nice 5
+  server.load "server.rb"
+  server.log_level 'error'
+end
+
+server :local, 'localhost' do |server|
+  server.load "server.rb"
+  server.log_level 'error'
+end
+
+node :node_base, template: true do |node, ssh|
+  ssh.directory ssh_directory
+  ssh.output "/path/to/node_ssh"
+  ssh.nice 10
+  node.process 2
+  node.load "server.rb"
+  node.log_level 'error'
+end
+
+ssh_user = 'user_name'
+[1, 2, 3, 4, 5, 6].each do |n|
+  name = "node%02d" % n
+  node name, load: :node_base do |node, ssh|
+    ssh.connect "#{ssh_user}@#{name}.example.com"
+  end
+end
+
+node :even, group: [:node02, :node04, :node06]
+node :odd, group: [:node01, :node03, :node05]

data/example/execute/server.rb

@@ -0,0 +1,14 @@
+require_relative 'task.rb'
+
+DRbQS.option_parser("Usage of server") do |prs, opts|
+  prs.on('-s STR', '--string STR', String, "Set string.") do |v|
+    opts[:string] = v
+  end
+  prs.on('-n NUM', '--number NUM', Integer, "Set number.") do |v|
+    opts[:number] = v
+  end
+end
+
+DRbQS.define_server(finish_exit: true) do |server, argv, opts|
+  
+end