drbqs 0.0.17 → 0.0.18

data/spec/worker/forked_process_spec.rb

@@ -0,0 +1,66 @@
+# -*- coding: utf-8 -*-
+require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
+
+require 'drbqs/worker'
+require 'drbqs/task/task'
+
+describe DRbQS::Worker::ForkedProcess do
+  class CountExec
+    @count = 0
+
+    def self.add_count
+      @count += 1
+    end
+
+    def self.get_count
+      @count
+    end
+
+    def self.clear
+      @count = 0
+    end
+  end
+
+  def print_test_task(io_w, task_id)
+    CountExec.clear
+    ary = DRbQS::Task.new(CountExec, :add_count).simple_drb_args
+    io_w.print DRbQS::Worker::Serialize.dump([task_id] + ary)
+    io_w.print DRbQS::Worker::Serialize.dump(:prepare_to_exit)
+    io_w.print DRbQS::Worker::Serialize.dump(:exit)
+    io_w.flush
+  end
+
+  before(:each) do
+    @io_r, @io_w = IO.pipe('BINARY')
+    @io_r2, @io_w2 = IO.pipe('BINARY')
+  end
+
+  it "should do something" do
+    task_id = 10
+    print_test_task(@io_w, task_id)
+    forked_process = DRbQS::Worker::SimpleForkedProcess.new(@io_r, @io_w2)
+    forked_process.start
+    CountExec.get_count.should == 1
+    result = DRbQS::Worker::Serialize.load(@io_r2.readpartial(1024))
+    result.should == [:result, [task_id, 1]]
+  end
+
+  it "should do something" do
+    DRbQS::Temporary.should_receive(:set_sub_directory)
+    task_id = 20
+    print_test_task(@io_w, task_id)
+    forked_process = DRbQS::Worker::ForkedProcess.new(@io_r, @io_w2)
+    forked_process.start
+    CountExec.get_count.should == 1
+    result = DRbQS::Worker::Serialize.load(@io_r2.readpartial(1024))
+    result[1][0].should == task_id
+    result[1][1].should be_an_instance_of Hash
+  end
+
+  after(:each) do
+    @io_r.close
+    @io_w.close
+    @io_r2.close
+    @io_w2.close
+  end
+end

data/spec/worker/serialize_spec.rb

@@ -0,0 +1,73 @@
+# -*- coding: utf-8 -*-
+require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
+
+require 'drbqs/worker/serialize'
+
+describe DRbQS::Worker::Serialize::Unpacker do
+  before(:all) do
+    @unpacker = DRbQS::Worker::Serialize::Unpacker.new
+  end
+
+  subject do
+    @unpacker
+  end
+
+  it "should dump." do
+    s = "abcあいうえお"
+    DRbQS::Worker::Serialize.dump(s).should be_an_instance_of String
+  end
+
+  it "should unpack." do
+    orig = "abcあいうえお"
+    subject.feed(DRbQS::Worker::Serialize.dump(orig))
+    ary = subject.each.to_a
+    ary.should have(1).item
+    ary[0].should == orig
+  end
+
+  it "should split data string." do
+    orig = "abcあいうえお"
+    s = DRbQS::Worker::Serialize.dump(orig)
+    subject.feed(s[0, 4])
+    subject.each.to_a.should be_empty
+    subject.feed(s[4..-1])
+    ary = subject.each.to_a
+    ary.should have(1).item
+    ary[0].should == orig
+  end
+
+  it "should split data string (2)." do
+    orig = "abcあいうえお"
+    s = DRbQS::Worker::Serialize.dump(orig)
+    subject.feed(s[0, 7])
+    subject.each.to_a.should be_empty
+    subject.feed(s[7..-1])
+    ary = subject.each.to_a
+    ary.should have(1).item
+    ary[0].should == orig
+  end
+
+  it "should unpack multiple objects." do
+    orig = [123, "abcあいうえお", -918, :sym]
+    orig.each do |obj|
+      subject.feed(DRbQS::Worker::Serialize.dump(obj))
+    end
+    ary = subject.each.to_a
+    ary.should have(4).items
+    ary.should == orig
+  end
+
+  it "should unpack multiple objects (2)." do
+    orig = [123, "abcあいうえお", -918, :sym]
+    s = ''
+    orig.each do |obj|
+      s << DRbQS::Worker::Serialize.dump(obj)
+    end
+    subject.feed(s[0, 10])
+    ary = subject.each.to_a
+    subject.feed(s[10..-1])
+    ary.concat(subject.each.to_a)
+    ary.should have(4).items
+    ary.should == orig
+  end
+end

data/spec/worker/worker_process_set_spec.rb

@@ -0,0 +1,104 @@
+# -*- coding: utf-8 -*-
+require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
+
+require 'drbqs/worker'
+require 'drbqs/task/task'
+require 'sys/proctable'
+
+describe DRbQS::Worker::ProcessSet do
+  [DRbQS::Worker::SimpleForkedProcess,
+   DRbQS::Worker::ForkedProcess].each do |klass|
+    context "when creating process #{klass.to_s}" do
+      before(:all) do
+        @worker = DRbQS::Worker::ProcessSet.new(klass)
+        @task_args = DRbQS::Task.new("hello world", :to_s).simple_drb_args
+        @error = []
+        @result = []
+        @worker.on_error do |proc_key, error|
+          @error << [proc_key, error]
+        end
+        @worker.on_result do |proc_key, result|
+          @result << [proc_key, result]
+        end
+        @n = 0
+      end
+
+      subject do
+        @worker
+      end
+
+      def send_task(key)
+        subject.send_task(key, [(@n += 1)] + @task_args)
+      end
+
+      it "should create new process." do
+        subject.create_process(:proc0)
+        subject.exist?(:proc0).should be_true
+        subject.waiting?(:proc0).should be_true
+        subject.calculating?(:proc0).should be_false
+        subject.waiting_processes.should == [:proc0]
+      end
+
+      it "should create new process automatically and execute a task." do
+        key = :proc1
+        send_task(key)
+        pid = subject.process[key][:pid]
+        Sys::ProcTable.ps(pid).should be_true
+        loop do
+          subject.respond_signal
+          unless @result.empty?
+            break
+          end
+          sleep(0.1)
+        end
+        @result.should have(1).item
+        @result[0][0].should == key
+      end
+
+      # Sometimes this spec fails. Later we investigate. (2011-10-21)
+      it "should make process exit." do
+        key = :proc2
+        send_task(key)
+        pid = subject.process[key][:pid]
+        subject.prepare_to_exit(key)
+        loop do
+          subject.respond_signal
+          sleep(0.1)
+          unless subject.exist?(key)
+            break
+          end
+        end
+        Sys::ProcTable.ps(pid).should be_nil
+      end
+
+      it "should make all process exit." do
+        [:proc3, :proc4].each do |key|
+          send_task(key)
+        end
+        subject.prepare_to_exit
+        subject.waitall
+        Process.waitall.should == []
+      end
+
+      it "should kill all processes compellingly." do
+        subject.create_process(:proc5, :proc6)
+        [:proc7, :proc8].each do |key|
+          send_task(key)
+        end
+        subject.kill_all_processes
+        subject.waitall
+        Process.waitall.should == []
+      end
+
+      after(:each) do
+        @result.clear
+        @error.clear
+      end
+
+      after(:all) do
+        subject.kill_all_processes
+        subject.waitall
+      end
+    end
+  end
+end

data/spec/worker/worker_spec.rb

@@ -0,0 +1,127 @@
+# -*- coding: utf-8 -*-
+require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
+
+require 'drbqs/worker'
+
+describe DRbQS::Worker do
+  class SavePID
+    def output(path)
+      pid = Process.pid
+      open(path, 'a+') do |f|
+        f.puts pid
+      end
+      pid
+    end
+  end
+
+  before(:all) do
+    @worker = DRbQS::Worker.new
+    @error = []
+    @result = []
+    @worker.on_error do |proc_key, error|
+      @error << [proc_key, error]
+    end
+    @worker.on_result do |proc_key, result|
+      @result << [proc_key, result]
+    end
+    @worker.process.create_process(:proc1, :proc2, :proc3)
+    @worker.group(:gr1, :proc1)
+    @worker.group(:gr3, :proc3)
+    @n = 0
+    @interval_time = 0.1
+  end
+
+  subject do
+    @worker
+  end
+
+  def file_path(key)
+    File.join(File.dirname(__FILE__), "test_worker", "#{key}.txt")
+  end
+
+  def send_task(key, group = nil, broadcast = nil)
+    path = file_path(key)
+    FileUtils.mkdir(File.dirname(path)) unless File.exist?(File.dirname(path))
+    task = DRbQS::Task.new(SavePID.new, :output, args: [path], group: group) do |wk, res|
+      puts "Hook: receive #{res}"
+    end
+    subject.add_task(task, broadcast)
+  end
+
+  it "should execute a task." do
+    task_key = :task
+    task_id = send_task(task_key)
+    subject.wait(task_id, @interval_time)
+    File.read(file_path(task_key)).lines.to_a[-1].strip.to_i.should == @result[0][1][1]
+  end
+
+  it "should execute a task on process of particular group." do
+    pid = subject.process.process[:proc3][:pid]
+    task_keys = [:task2, :task3]
+    task_keys.each do |key|
+      send_task(key, :gr3)
+    end
+    subject.waitall(@interval_time)
+    task_keys.each do |key|
+      File.read(file_path(key)).lines.to_a[-1].strip.to_i.should == pid
+    end
+  end
+
+  it "should sleep nodes." do
+    subject.sleep(:proc1, :proc2)
+    pid = subject.process.process[:proc3][:pid]
+    task_keys = [:task4, :task5]
+    task_keys.each do |key|
+      send_task(key)
+    end
+    subject.waitall(@interval_time)
+    task_keys.each do |key|
+      File.read(file_path(key)).lines.to_a[-1].strip.to_i.should == pid
+    end
+    subject.wakeup(:proc1, :proc2)
+  end
+
+  it "should wakeup a node." do
+    subject.sleep(:proc1, :proc2, :proc3)
+    subject.wakeup(:proc2)
+    pid = subject.process.process[:proc2][:pid]
+    task_keys = [:task6, :task7]
+    task_keys.each do |key|
+      send_task(key)
+    end
+    subject.waitall(@interval_time)
+    task_keys.each do |key|
+      File.read(file_path(key)).lines.to_a[-1].strip.to_i.should == pid
+    end
+    subject.wakeup(:proc1, :proc3)
+  end
+
+  it "should send to all processes." do
+    ary_pid = subject.process.all_processes.map do |proc_key|
+      subject.process.process[proc_key][:pid]
+    end
+    key = :all
+    send_task(key, nil, true)
+    sleep(1)
+    result_pid = File.read(file_path(key)).lines.map(&:to_i).sort
+    result_pid.should == ary_pid.sort
+  end
+
+  it "should finish." do
+    subject.process.create_process(:proc4, :proc5)
+    subject.finish
+    Process.waitall.should == []
+  end
+
+  after(:each) do
+    @result.clear
+    @error.clear
+  end
+
+  after(:all) do
+    subject.process.kill_all_processes
+    subject.process.waitall
+    path = File.join(File.dirname(__FILE__), "test_worker")
+    FileUtils.rm_r(path) if File.exist?(path)
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: drbqs
 version: !ruby/object:Gem::Version
-  version: 0.0.17
+  version: 0.0.18
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2011-08-27 00:00:00.000000000 Z
+date: 2011-10-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &18201460 !ruby/object:Gem::Requirement
+  requirement: &10367840 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: 2.6.0
   type: :development
   prerelease: false
-  version_requirements: *18201460
+  version_requirements: *10367840
 - !ruby/object:Gem::Dependency
   name: yard
-  requirement: &18200180 !ruby/object:Gem::Requirement
+  requirement: &10366940 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: 0.7.2
   type: :development
   prerelease: false
-  version_requirements: *18200180
+  version_requirements: *10366940
 - !ruby/object:Gem::Dependency
   name: filename
-  requirement: &18199300 !ruby/object:Gem::Requirement
+  requirement: &10366320 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
         version: 0.1.0
   type: :runtime
   prerelease: false
-  version_requirements: *18199300
+  version_requirements: *10366320
 - !ruby/object:Gem::Dependency
   name: user_config
-  requirement: &18216600 !ruby/object:Gem::Requirement
+  requirement: &10365760 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,10 +54,10 @@ dependencies:
         version: 0.0.2
   type: :runtime
   prerelease: false
-  version_requirements: *18216600
+  version_requirements: *10365760
 - !ruby/object:Gem::Dependency
   name: net-ssh
-  requirement: &18214620 !ruby/object:Gem::Requirement
+  requirement: &10365160 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -65,10 +65,10 @@ dependencies:
         version: 2.1.0
   type: :runtime
   prerelease: false
-  version_requirements: *18214620
+  version_requirements: *10365160
 - !ruby/object:Gem::Dependency
   name: net-ssh-shell
-  requirement: &18213260 !ruby/object:Gem::Requirement
+  requirement: &10364460 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -76,10 +76,10 @@ dependencies:
         version: 0.2.0
   type: :runtime
   prerelease: false
-  version_requirements: *18213260
+  version_requirements: *10364460
 - !ruby/object:Gem::Dependency
   name: net-sftp
-  requirement: &18212080 !ruby/object:Gem::Requirement
+  requirement: &10363900 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -87,10 +87,10 @@ dependencies:
         version: 2.0.5
   type: :runtime
   prerelease: false
-  version_requirements: *18212080
+  version_requirements: *10363900
 - !ruby/object:Gem::Dependency
   name: sys-proctable
-  requirement: &18210980 !ruby/object:Gem::Requirement
+  requirement: &10363400 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -98,7 +98,7 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *18210980
+  version_requirements: *10363400
 description: Task queuing system over network that is implemented by dRuby.
 email:
 - d@ytak.info
@@ -135,6 +135,9 @@ files:
 - example/execute/execute.rb
 - example/execute/server.rb
 - example/execute/task.rb
+- example/group/execute.rb
+- example/group/server.rb
+- example/group/sum.rb
 - example/mandelbrot/README.md
 - example/mandelbrot/execute.rb
 - example/mandelbrot/mandelbrot.rb
@@ -209,6 +212,11 @@ files:
 - lib/drbqs/utility/transfer/transfer_client_connect.rb
 - lib/drbqs/utility/transfer/transfer_file_list.rb
 - lib/drbqs/version.rb
+- lib/drbqs/worker.rb
+- lib/drbqs/worker/forked_process.rb
+- lib/drbqs/worker/serialize.rb
+- lib/drbqs/worker/worker.rb
+- lib/drbqs/worker/worker_process_set.rb
 - spec/command_line/command_base_spec.rb
 - spec/command_line/commands_spec.rb
 - spec/command_line/option_setting_spec.rb
@@ -232,6 +240,8 @@ files:
 - spec/integration_test/08_shutdown_unused_nodes_spec.rb
 - spec/integration_test/09_server_process_data_spec.rb
 - spec/integration_test/10_test_server_spec.rb
+- spec/integration_test/11_special_tasks_spec.rb
+- spec/integration_test/12_multiple_workers_spec.rb
 - spec/integration_test/definition/server01.rb
 - spec/integration_test/definition/server02.rb
 - spec/integration_test/definition/task_obj_definition.rb
@@ -269,6 +279,10 @@ files:
 - spec/utility/transfer/transfer_client_connect_spec.rb
 - spec/utility/transfer/transfer_file_list_spec.rb
 - spec/utility/transfer/transfer_spec.rb
+- spec/worker/forked_process_spec.rb
+- spec/worker/serialize_spec.rb
+- spec/worker/worker_process_set_spec.rb
+- spec/worker/worker_spec.rb
 homepage: ''
 licenses: []
 post_install_message: 
@@ -289,8 +303,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: drbqs
-rubygems_version: 1.8.10
+rubygems_version: 1.8.5
 signing_key: 
 specification_version: 3
 summary: dRuby Queueing System
 test_files: []
+has_rdoc: