drbqs 0.0.15 → 0.0.16

data/example/mandelbrot/README.md

@@ -0,0 +1,15 @@
+# Mandelbrot set
+
+Calculate Mandelbrot set and save results to directory 'result_mandelbrot'.
+
+## Requirements
+
+This program requires gunzip command.
+
+## Help
+
+    drbqs-server server.rb -h
+
+## Execute
+
+    drbqs-server server.rb --execute-node <process_number>

data/example/mandelbrot/execute.rb

@@ -0,0 +1,10 @@
+DIR = File.dirname(__FILE__)
+
+server :local_server, "localhost" do |srv|
+  srv.load File.join(DIR, 'server.rb')
+end
+
+node :local_node do |nd|
+  nd.load File.join(DIR, 'mandelbrot.rb')
+  nd.process 2 # For dual core CPU
+end

data/example/mandelbrot/mandelbrot.rb

@@ -0,0 +1,56 @@
+require 'complex'
+
+class Mandelbrot
+  def initialize(iterate, threshold)
+    @iterate = iterate
+    @threshold = threshold
+  end
+
+  def map(z, c)
+    z * z + c
+  end
+
+  def iterate_map(z, c, &block)
+    z_old = z
+    @iterate.times do |i|
+      z_new = map(z_old, c)
+      z_old = z_new
+      yield(z_old) if block_given?
+    end
+    z_old
+  end
+
+  def diverge?(c)
+    iterate_map(Complex(0.0, 0.0), c) do |z|
+      if z.abs > @threshold
+        return true
+      end
+    end
+    false
+  end
+end
+
+class CalcMandelbrot
+  def initialize(mandelbrot)
+    @mandelbrot = mandelbrot
+  end
+
+  def calc(io, xrange, yrange, step_size)
+    xrange.step(step_size) do |x|
+      yrange.step(step_size) do |y|
+        c = Complex(x, y)
+        unless @mandelbrot.diverge?(c)
+          io.puts "#{c.real}\t#{c.imag}"
+        end
+      end
+    end
+  end
+
+  def calc_save(basename, *args)
+    file = DRbQS::Temporary.file
+    open(file, 'w') do |f|
+      calc(f, *args)
+    end
+    DRbQS::Transfer.enqueue(file, compress: true, rename: basename) # Return basename.
+  end
+end

data/example/mandelbrot/server.rb

@@ -0,0 +1,49 @@
+require_relative 'mandelbrot.rb'
+
+RESULT_DIR = 'result_mandelbrot'
+
+DRbQS.option_parser("Calculate Mandelbrot set.") do |prs, opts|
+  prs.on('-s NUM', '--step NUM', Float, 'Set the step size.') do |v|
+    opts[:step] = v
+  end
+  prs.on('-l NUM', '--limit NUM', Float, 'Set the limit to search.') do |v|
+    opts[:limit] = v
+  end
+  prs.on('-i NUM', '--iterate NUM', Integer, 'Set the iterate number.') do |v|
+    opts[:iterate] = v
+  end
+  prs.on('-t NUM', '--threshold NUM', Float, 'Set the threshold value.') do |v|
+    opts[:threshold] = v
+  end
+end
+
+DRbQS.define_server(file_directory: RESULT_DIR) do |server, argv, opts|
+  step_size = opts[:step] || 0.1
+  limit = opts[:limit] || 2.0
+  iterate = opts[:iterate] || 1000
+  threshold = opts[:threshold] || 5
+
+  mandelbrot = Mandelbrot.new(iterate, threshold)
+  calc = CalcMandelbrot.new(mandelbrot)
+
+  ranges = [[-limit..0, -limit..0, step_size], [-limit..0, 0..limit, step_size],
+            [0..limit, -limit..0, step_size], [0..limit, 0..limit, step_size]]
+  ranges.each_with_index do |ranges, i|
+    args_ary = ["%02d.txt" % i] + ranges
+    note_str = "#{ranges[0].inspect} #{ranges[1].inspect}"
+    task = DRbQS::Task.new(calc, :calc_save, args: args_ary, note: note_str) do |srv, result|
+      DRbQS::Transfer.decompress(srv, result)
+    end
+    server.queue.add(task)
+  end
+  server.add_hook(:finish) do |srv|
+    puts "Save results to #{RESULT_DIR}"
+    result_path = File.join(RESULT_DIR, "result.txt")
+    Dir.glob(File.join(RESULT_DIR, "*.txt")).sort.each do |path|
+      open(result_path, 'a+') do |out|
+        out.print File.read(path)
+      end
+      FileUtils.rm(path)
+    end
+  end
+end

data/example/server/server.rb

@@ -1,12 +1,9 @@
-task_generator = DRbQS::TaskGenerator.new(:iterate => 3)
-task_generator.set do
+server = DRbQS::Server.new(:port => 13501)
+server.task_generator do |reg|
   5.times do |i|
-    create_add_task(i, :to_s)
+    reg.create_add(i, :to_s)
   end
 end
-
-server = DRbQS::Server.new(:port => 13501, :finish_exit => true)
-server.add_task_generator(task_generator)
 server.set_initialization_task(DRbQS::Task.new(Kernel, :puts, ['hook: initialize']))
 server.set_finalization_task(DRbQS::Task.new(Kernel, :puts, ['hook: finalize']))
 server.set_signal_trap

data/example/simple/README.md

@@ -0,0 +1,18 @@
+# Simple example
+
+Creating one task that calculates sum of three numbers
+and outputing the result.
+
+## drbqs-server and drbqs-node
+
+On first terminal,
+
+    drbqs-server server.rb
+
+and then on second terminal
+
+    drbqs-node druby://:13500 -l task.rb
+
+## drbqs-execute
+
+    drbqs-execute execute.rb

data/example/simple/execute.rb

@@ -0,0 +1,11 @@
+DIR = File.dirname(__FILE__)
+
+default port: 12345
+
+server :local, "localhost" do |srv|
+  srv.load File.join(DIR, 'server.rb')
+end
+
+node :local do |nd|
+  nd.load File.join(DIR, 'task.rb')
+end

data/example/simple/server.rb

@@ -0,0 +1,8 @@
+require_relative 'task.rb'
+
+DRbQS.define_server do |server, argv, opts|
+  task = DRbQS::Task.new(Sum.new(10, 20, 2), :calc) do |srv, result|
+    puts "Result is #{result}"
+  end
+  server.queue.add(task)
+end

data/example/simple/task.rb

@@ -0,0 +1,11 @@
+class Sum
+  def initialize(a, b, c)
+    @a = a
+    @b = b
+    @c = c
+  end
+
+  def calc
+    @a + @b + @c
+  end
+end

data/example/sum/server_def.rb

@@ -17,7 +17,7 @@ DRbQS.define_server(:check_alive => 5) do |server, argv, opts|
   end_num = (argv[1] || 50).to_i
   step_num = opts[:step] || 10
   start_num.step(end_num, step_num) do |i|
-    task = DRbQS::Task.new(Sum.new(i - 10, i), :exec) do |srv, ret|
+    task = DRbQS::Task.new(Sum.new(i - 10, i), :exec, [], "#{i-10} to #{i}") do |srv, ret|
       puts "Receive: #{ret.inspect}"
     end
     server.queue.add(task)

data/example/sum2/execute_def.rb

@@ -2,26 +2,39 @@ default :server => :server_local, :port => 13789, :log => '/tmp/drbqs_execute'
 
 usage :message => "Calculate sum of numbers", :server => File.join(File.dirname(__FILE__), 'server_def.rb')
 
-register_server :server_local, "localhost" do |server|
+server :server_template, :template => true do |server, ssh|
+  server.load 'server_def.rb'
+end
+
+server :server_local, "localhost" do |server|
   server.load File.expand_path(File.join(File.dirname(__FILE__), 'server_def.rb'))
 end
 
-register_server :server_ssh, "localhost" do |server, ssh|
-  ssh.connect 'localhost'
-  ssh.directory File.expand_path(File.dirname(__FILE__))
+ssh_localhost = "#{ENV['USER']}@localhost"
+current_dir = File.expand_path(File.dirname(__FILE__))
+
+server :server_ssh, "localhost" do |server, ssh|
+  ssh.connect ssh_localhost
+  ssh.directory current_dir
   ssh.output "/tmp/drbqs_ssh/server"
   server.load 'server_def.rb'
 end
 
-register_node :node_local do |node|
+node :node_template, template: true do |node|
+  node.load 'sum.rb'
+end
+
+node :node_local do |node|
   node.load File.expand_path(File.join(File.dirname(__FILE__), 'sum.rb'))
   node.process 2
 end
 
-register_node :node_ssh do |node, ssh|
-  ssh.connect 'localhost'
-  ssh.directory File.expand_path(File.dirname(__FILE__))
+node :node_ssh do |node, ssh|
+  ssh.connect ssh_localhost
+  ssh.directory current_dir
   ssh.output "/tmp/drbqs_ssh/node"
   node.load 'sum.rb'
   node.process 2
 end
+
+node :node_group, :group => [:node_local, :node_ssh]

data/example/sum2/server_def.rb

@@ -12,13 +12,14 @@ DRbQS.option_parser do |opt, hash|
   end
 end
 
-DRbQS.define_server(:finish_exit => true) do |server, argv, opts|
-  tgen = DRbQS::TaskGenerator.new(:start_num => (argv[0] || 10).to_i,
-                                  :end_num => (argv[1] || 100).to_i,
-                                  :step_num => opts[:step] || 10)
-  tgen.set do
-    @start_num.step(@end_num, @step_num) do |i|
-      create_add_task(Sum.new(i - 10, i), :exec) do |srv, ret|
+DRbQS.define_server do |server, argv, opts|
+  start_num = (argv[0] || 10).to_i,
+  end_num = (argv[1] || 100).to_i,
+  step_num = opts[:step] || 10
+
+  server.task_generator do |reg|
+    start_num.step(end_num, step_num) do |i|
+      reg.create_add(Sum.new(i - 10, i), :exec) do |srv, ret|
         puts "Receive: #{ret.inspect}"
       end
     end

data/example/transfer/file.rb

@@ -10,13 +10,15 @@ class CreateFile
   end
 
   def create
-    DRbQS::FileTransfer.enqueue(output_to_file)
-    nil
+    path = output_to_file
+    DRbQS::Transfer.enqueue(path)
+    File.basename(path)
   end
 
   def create_compress
-    DRbQS::FileTransfer.compress_enqueue(output_to_file)
-    nil
+    path = output_to_file
+    DRbQS::Transfer.compress_enqueue(path)
+    File.basename(path)
   end
 end
 
@@ -33,12 +35,44 @@ class CreateDirectory
   end
 
   def create
-    DRbQS::FileTransfer.enqueue(output_to_directory)
-    nil
+    dir = output_to_directory
+    DRbQS::Transfer.enqueue(dir)
+    File.basename(dir)
   end
 
   def create_compress
-    DRbQS::FileTransfer.compress_enqueue(output_to_directory)
-    nil
+    dir = output_to_directory
+    DRbQS::Transfer.compress_enqueue(dir)
+    File.basename(dir)
+  end
+end
+
+class ReceiveFile
+  def initialize(file_list)
+    @file_list = file_list
+  end
+
+  def read_file
+    ret = ''
+    @file_list.path.each do |path|
+      if File.directory?(path)
+        raise "Receive directory, not file."
+      end
+      ret << path << "\t" << File.read(path).strip << "\n"
+    end
+    ret
+  end
+
+  def read_directory
+    ret = ''
+    @file_list.path.each do |dir|
+      unless File.directory?(dir)
+        raise "Receive file, not directory."
+      end
+      Dir.glob("#{dir}/**/*.txt").each do |path|
+        ret << path << "\t" << File.read(path).strip << "\n"
+      end
+    end
+    ret
   end
 end

data/example/transfer/server_def.rb

@@ -5,15 +5,49 @@
 
 require_relative 'file.rb'
 
-DRbQS.define_server(:finish_exit => true) do |server, argv, opts|
-  tgen = DRbQS::TaskGenerator.new(:sleep_time => 2)
-  tgen.set do
-    create_add_task(CreateFile.new(1), :create)
-    create_add_task(CreateFile.new(2), :create_compress)
-    create_add_task(CreateDirectory.new(3), :create)
-    create_add_task(CreateDirectory.new(4), :create_compress)
+DRbQS.define_server do |server, argv, opts|
+  server_directory = '/tmp/drbqs_transfer_test/'
+  FileUtils.rm_r(server_directory) if File.exist?(server_directory)
+  server_directory2 = '/tmp/drbqs_transfer_test2/'
+  FileUtils.rm_r(server_directory2) if File.exist?(server_directory2)
+  FileUtils.mkdir_p(server_directory2)
+  test_file = File.join(server_directory2, 'file1.txt')
+  open(test_file, 'w') do |f|
+    f.puts 'ABC'
+  end
+  test_dir = File.join(server_directory2, 'test_dir')
+  test_dir2 = File.join(test_dir, 'abc', 'def')
+  FileUtils.mkdir_p(test_dir2)
+  open(File.join(test_dir2, 'file2.txt'), 'w') do |f|
+    f.puts 'hello'
+  end
+  open(File.join(test_dir2, 'file3.txt'), 'w') do |f|
+    f.puts 'world'
   end
-  server.add_task_generator(tgen)
 
-  server.set_file_transfer('/tmp/drbqs_transfer_test/')
+  server.task_generator do |reg|
+    create_add(CreateFile.new(1), :create) do |srv, result|
+      path = File.join(server_directory, result)
+      puts "#{path} exist? #{File.exist?(path).inspect}"
+    end
+    create_add(CreateFile.new(2), :create_compress) do |srv, result|
+      path = File.join(server_directory, result + '.gz')
+      puts "#{path} exist? #{File.exist?(path).inspect}"
+    end
+    create_add(CreateDirectory.new(3), :create) do |srv, result|
+      path = File.join(server_directory, result)
+      puts "#{path} exist? #{File.exist?(path).inspect}"
+    end
+    create_add(CreateDirectory.new(4), :create_compress) do |srv, result|
+      path = File.join(server_directory, result + '.tar.gz')
+      puts "#{path} exist? #{File.exist?(path).inspect}"
+    end
+    create_add(ReceiveFile.new(DRbQS::Transfer::FileList.new(test_file)), :read_file) do |srv, result|
+      puts result
+    end
+    create_add(ReceiveFile.new(DRbQS::Transfer::FileList.new(test_dir)), :read_directory) do |srv, result|
+      puts result
+    end
+  end
+  server.set_file_transfer(server_directory)
 end

data/lib/drbqs.rb

@@ -20,5 +20,5 @@ module DRbQS
 
   ROOT_DEFAULT_PORT = 13500
 
-  VERSION = '0.0.15'
+  VERSION = '0.0.16'
 end

data/lib/drbqs/command_line/command_execute.rb

@@ -15,9 +15,9 @@ HELP
       def parse_option(argv)
         args, server_args = split_arguments(argv)
         args = option_parser_base(args) do
-          set(:port, '--port NUM', Integer, 'Set the port number.')
-          set(:server, '--server STR', String, 'Set the key of server.')
-          set(:node, '--node STR', String, 'Set the comma sparated key of nodes.')
+          set(:port, '-p NUM', '--port NUM', Integer, 'Set the port number.')
+          set(:server, '-s STR', '--server STR', String, 'Set the key of server.')
+          set(:node, '-n STR', '--node STR', String, 'Set the comma sparated key of nodes.')
           set(:no_server, '--no-server', 'Not execute server.')
           set(:no_node, '--no-node', 'Not execute node.')
           set(:information, '-i', '--information', 'Show information.')

data/lib/drbqs/command_line/command_line.rb

@@ -1,6 +1,6 @@
 require 'drbqs'
 require 'drbqs/manage/manage'
-require 'drbqs/manage/execute_node'
+require 'drbqs/execute/execute_node'
 require 'drbqs/command_line/argument'
 require 'drbqs/command_line/option_setting'
 require 'drbqs/command_line/command_base'