drbqs 0.0.17 → 0.0.18

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. data/docs/FormatExecute.md +44 -2
  2. data/example/group/execute.rb +19 -0
  3. data/example/group/server.rb +27 -0
  4. data/example/group/sum.rb +9 -0
  5. data/example/mandelbrot/README.md +8 -0
  6. data/example/mandelbrot/execute.rb +4 -0
  7. data/lib/drbqs/command_line/command_node.rb +1 -0
  8. data/lib/drbqs/execute/execute_node.rb +4 -21
  9. data/lib/drbqs/node/connection.rb +1 -2
  10. data/lib/drbqs/node/node.rb +163 -102
  11. data/lib/drbqs/node/state.rb +100 -35
  12. data/lib/drbqs/node/task_client.rb +46 -33
  13. data/lib/drbqs/server/message.rb +13 -7
  14. data/lib/drbqs/server/server.rb +57 -29
  15. data/lib/drbqs/server/server_hook.rb +19 -5
  16. data/lib/drbqs/server/test/node.rb +31 -6
  17. data/lib/drbqs/setting/node.rb +11 -2
  18. data/lib/drbqs/setting/server.rb +1 -1
  19. data/lib/drbqs/task/task.rb +26 -6
  20. data/lib/drbqs/task/task_generator.rb +2 -1
  21. data/lib/drbqs/utility/temporary.rb +27 -6
  22. data/lib/drbqs/utility/transfer/transfer_client.rb +10 -12
  23. data/lib/drbqs/version.rb +1 -1
  24. data/lib/drbqs/worker.rb +2 -0
  25. data/lib/drbqs/worker/forked_process.rb +100 -0
  26. data/lib/drbqs/worker/serialize.rb +66 -0
  27. data/lib/drbqs/worker/worker.rb +133 -0
  28. data/lib/drbqs/worker/worker_process_set.rb +219 -0
  29. data/spec/integration_test/01_basic_usage_spec.rb +3 -2
  30. data/spec/integration_test/06_node_exit_after_task_spec.rb +3 -2
  31. data/spec/integration_test/07_command_server_with_node_spec.rb +1 -0
  32. data/spec/integration_test/08_shutdown_unused_nodes_spec.rb +3 -2
  33. data/spec/integration_test/10_test_server_spec.rb +2 -2
  34. data/spec/integration_test/11_special_tasks_spec.rb +61 -0
  35. data/spec/integration_test/12_multiple_workers_spec.rb +43 -0
  36. data/spec/integration_test/definition/task_obj_definition.rb +33 -6
  37. data/spec/node/connection_spec.rb +6 -6
  38. data/spec/node/node_spec.rb +10 -2
  39. data/spec/node/state_spec.rb +146 -62
  40. data/spec/node/task_client_spec.rb +58 -53
  41. data/spec/server/message_spec.rb +10 -6
  42. data/spec/server/queue_spec.rb +7 -4
  43. data/spec/server/server_hook_spec.rb +28 -1
  44. data/spec/task/task_spec.rb +43 -6
  45. data/spec/utility/temporary_spec.rb +32 -9
  46. data/spec/worker/forked_process_spec.rb +66 -0
  47. data/spec/worker/serialize_spec.rb +73 -0
  48. data/spec/worker/worker_process_set_spec.rb +104 -0
  49. data/spec/worker/worker_spec.rb +127 -0
  50. metadata +34 -19
@@ -0,0 +1,66 @@
1
+ # -*- coding: utf-8 -*-
2
+ require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
3
+
4
+ require 'drbqs/worker'
5
+ require 'drbqs/task/task'
6
+
7
+ describe DRbQS::Worker::ForkedProcess do
8
+ class CountExec
9
+ @count = 0
10
+
11
+ def self.add_count
12
+ @count += 1
13
+ end
14
+
15
+ def self.get_count
16
+ @count
17
+ end
18
+
19
+ def self.clear
20
+ @count = 0
21
+ end
22
+ end
23
+
24
+ def print_test_task(io_w, task_id)
25
+ CountExec.clear
26
+ ary = DRbQS::Task.new(CountExec, :add_count).simple_drb_args
27
+ io_w.print DRbQS::Worker::Serialize.dump([task_id] + ary)
28
+ io_w.print DRbQS::Worker::Serialize.dump(:prepare_to_exit)
29
+ io_w.print DRbQS::Worker::Serialize.dump(:exit)
30
+ io_w.flush
31
+ end
32
+
33
+ before(:each) do
34
+ @io_r, @io_w = IO.pipe('BINARY')
35
+ @io_r2, @io_w2 = IO.pipe('BINARY')
36
+ end
37
+
38
+ it "should do something" do
39
+ task_id = 10
40
+ print_test_task(@io_w, task_id)
41
+ forked_process = DRbQS::Worker::SimpleForkedProcess.new(@io_r, @io_w2)
42
+ forked_process.start
43
+ CountExec.get_count.should == 1
44
+ result = DRbQS::Worker::Serialize.load(@io_r2.readpartial(1024))
45
+ result.should == [:result, [task_id, 1]]
46
+ end
47
+
48
+ it "should do something" do
49
+ DRbQS::Temporary.should_receive(:set_sub_directory)
50
+ task_id = 20
51
+ print_test_task(@io_w, task_id)
52
+ forked_process = DRbQS::Worker::ForkedProcess.new(@io_r, @io_w2)
53
+ forked_process.start
54
+ CountExec.get_count.should == 1
55
+ result = DRbQS::Worker::Serialize.load(@io_r2.readpartial(1024))
56
+ result[1][0].should == task_id
57
+ result[1][1].should be_an_instance_of Hash
58
+ end
59
+
60
+ after(:each) do
61
+ @io_r.close
62
+ @io_w.close
63
+ @io_r2.close
64
+ @io_w2.close
65
+ end
66
+ end
@@ -0,0 +1,73 @@
1
+ # -*- coding: utf-8 -*-
2
+ require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
3
+
4
+ require 'drbqs/worker/serialize'
5
+
6
+ describe DRbQS::Worker::Serialize::Unpacker do
7
+ before(:all) do
8
+ @unpacker = DRbQS::Worker::Serialize::Unpacker.new
9
+ end
10
+
11
+ subject do
12
+ @unpacker
13
+ end
14
+
15
+ it "should dump." do
16
+ s = "abcあいうえお"
17
+ DRbQS::Worker::Serialize.dump(s).should be_an_instance_of String
18
+ end
19
+
20
+ it "should unpack." do
21
+ orig = "abcあいうえお"
22
+ subject.feed(DRbQS::Worker::Serialize.dump(orig))
23
+ ary = subject.each.to_a
24
+ ary.should have(1).item
25
+ ary[0].should == orig
26
+ end
27
+
28
+ it "should split data string." do
29
+ orig = "abcあいうえお"
30
+ s = DRbQS::Worker::Serialize.dump(orig)
31
+ subject.feed(s[0, 4])
32
+ subject.each.to_a.should be_empty
33
+ subject.feed(s[4..-1])
34
+ ary = subject.each.to_a
35
+ ary.should have(1).item
36
+ ary[0].should == orig
37
+ end
38
+
39
+ it "should split data string (2)." do
40
+ orig = "abcあいうえお"
41
+ s = DRbQS::Worker::Serialize.dump(orig)
42
+ subject.feed(s[0, 7])
43
+ subject.each.to_a.should be_empty
44
+ subject.feed(s[7..-1])
45
+ ary = subject.each.to_a
46
+ ary.should have(1).item
47
+ ary[0].should == orig
48
+ end
49
+
50
+ it "should unpack multiple objects." do
51
+ orig = [123, "abcあいうえお", -918, :sym]
52
+ orig.each do |obj|
53
+ subject.feed(DRbQS::Worker::Serialize.dump(obj))
54
+ end
55
+ ary = subject.each.to_a
56
+ ary.should have(4).items
57
+ ary.should == orig
58
+ end
59
+
60
+ it "should unpack multiple objects (2)." do
61
+ orig = [123, "abcあいうえお", -918, :sym]
62
+ s = ''
63
+ orig.each do |obj|
64
+ s << DRbQS::Worker::Serialize.dump(obj)
65
+ end
66
+ subject.feed(s[0, 10])
67
+ ary = subject.each.to_a
68
+ subject.feed(s[10..-1])
69
+ ary.concat(subject.each.to_a)
70
+ ary.should have(4).items
71
+ ary.should == orig
72
+ end
73
+ end
@@ -0,0 +1,104 @@
1
+ # -*- coding: utf-8 -*-
2
+ require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
3
+
4
+ require 'drbqs/worker'
5
+ require 'drbqs/task/task'
6
+ require 'sys/proctable'
7
+
8
+ describe DRbQS::Worker::ProcessSet do
9
+ [DRbQS::Worker::SimpleForkedProcess,
10
+ DRbQS::Worker::ForkedProcess].each do |klass|
11
+ context "when creating process #{klass.to_s}" do
12
+ before(:all) do
13
+ @worker = DRbQS::Worker::ProcessSet.new(klass)
14
+ @task_args = DRbQS::Task.new("hello world", :to_s).simple_drb_args
15
+ @error = []
16
+ @result = []
17
+ @worker.on_error do |proc_key, error|
18
+ @error << [proc_key, error]
19
+ end
20
+ @worker.on_result do |proc_key, result|
21
+ @result << [proc_key, result]
22
+ end
23
+ @n = 0
24
+ end
25
+
26
+ subject do
27
+ @worker
28
+ end
29
+
30
+ def send_task(key)
31
+ subject.send_task(key, [(@n += 1)] + @task_args)
32
+ end
33
+
34
+ it "should create new process." do
35
+ subject.create_process(:proc0)
36
+ subject.exist?(:proc0).should be_true
37
+ subject.waiting?(:proc0).should be_true
38
+ subject.calculating?(:proc0).should be_false
39
+ subject.waiting_processes.should == [:proc0]
40
+ end
41
+
42
+ it "should create new process automatically and execute a task." do
43
+ key = :proc1
44
+ send_task(key)
45
+ pid = subject.process[key][:pid]
46
+ Sys::ProcTable.ps(pid).should be_true
47
+ loop do
48
+ subject.respond_signal
49
+ unless @result.empty?
50
+ break
51
+ end
52
+ sleep(0.1)
53
+ end
54
+ @result.should have(1).item
55
+ @result[0][0].should == key
56
+ end
57
+
58
+ # Sometimes this spec fails. Later we investigate. (2011-10-21)
59
+ it "should make process exit." do
60
+ key = :proc2
61
+ send_task(key)
62
+ pid = subject.process[key][:pid]
63
+ subject.prepare_to_exit(key)
64
+ loop do
65
+ subject.respond_signal
66
+ sleep(0.1)
67
+ unless subject.exist?(key)
68
+ break
69
+ end
70
+ end
71
+ Sys::ProcTable.ps(pid).should be_nil
72
+ end
73
+
74
+ it "should make all process exit." do
75
+ [:proc3, :proc4].each do |key|
76
+ send_task(key)
77
+ end
78
+ subject.prepare_to_exit
79
+ subject.waitall
80
+ Process.waitall.should == []
81
+ end
82
+
83
+ it "should kill all processes compellingly." do
84
+ subject.create_process(:proc5, :proc6)
85
+ [:proc7, :proc8].each do |key|
86
+ send_task(key)
87
+ end
88
+ subject.kill_all_processes
89
+ subject.waitall
90
+ Process.waitall.should == []
91
+ end
92
+
93
+ after(:each) do
94
+ @result.clear
95
+ @error.clear
96
+ end
97
+
98
+ after(:all) do
99
+ subject.kill_all_processes
100
+ subject.waitall
101
+ end
102
+ end
103
+ end
104
+ end
@@ -0,0 +1,127 @@
1
+ # -*- coding: utf-8 -*-
2
+ require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
3
+
4
+ require 'drbqs/worker'
5
+
6
+ describe DRbQS::Worker do
7
+ class SavePID
8
+ def output(path)
9
+ pid = Process.pid
10
+ open(path, 'a+') do |f|
11
+ f.puts pid
12
+ end
13
+ pid
14
+ end
15
+ end
16
+
17
+ before(:all) do
18
+ @worker = DRbQS::Worker.new
19
+ @error = []
20
+ @result = []
21
+ @worker.on_error do |proc_key, error|
22
+ @error << [proc_key, error]
23
+ end
24
+ @worker.on_result do |proc_key, result|
25
+ @result << [proc_key, result]
26
+ end
27
+ @worker.process.create_process(:proc1, :proc2, :proc3)
28
+ @worker.group(:gr1, :proc1)
29
+ @worker.group(:gr3, :proc3)
30
+ @n = 0
31
+ @interval_time = 0.1
32
+ end
33
+
34
+ subject do
35
+ @worker
36
+ end
37
+
38
+ def file_path(key)
39
+ File.join(File.dirname(__FILE__), "test_worker", "#{key}.txt")
40
+ end
41
+
42
+ def send_task(key, group = nil, broadcast = nil)
43
+ path = file_path(key)
44
+ FileUtils.mkdir(File.dirname(path)) unless File.exist?(File.dirname(path))
45
+ task = DRbQS::Task.new(SavePID.new, :output, args: [path], group: group) do |wk, res|
46
+ puts "Hook: receive #{res}"
47
+ end
48
+ subject.add_task(task, broadcast)
49
+ end
50
+
51
+ it "should execute a task." do
52
+ task_key = :task
53
+ task_id = send_task(task_key)
54
+ subject.wait(task_id, @interval_time)
55
+ File.read(file_path(task_key)).lines.to_a[-1].strip.to_i.should == @result[0][1][1]
56
+ end
57
+
58
+ it "should execute a task on process of particular group." do
59
+ pid = subject.process.process[:proc3][:pid]
60
+ task_keys = [:task2, :task3]
61
+ task_keys.each do |key|
62
+ send_task(key, :gr3)
63
+ end
64
+ subject.waitall(@interval_time)
65
+ task_keys.each do |key|
66
+ File.read(file_path(key)).lines.to_a[-1].strip.to_i.should == pid
67
+ end
68
+ end
69
+
70
+ it "should sleep nodes." do
71
+ subject.sleep(:proc1, :proc2)
72
+ pid = subject.process.process[:proc3][:pid]
73
+ task_keys = [:task4, :task5]
74
+ task_keys.each do |key|
75
+ send_task(key)
76
+ end
77
+ subject.waitall(@interval_time)
78
+ task_keys.each do |key|
79
+ File.read(file_path(key)).lines.to_a[-1].strip.to_i.should == pid
80
+ end
81
+ subject.wakeup(:proc1, :proc2)
82
+ end
83
+
84
+ it "should wakeup a node." do
85
+ subject.sleep(:proc1, :proc2, :proc3)
86
+ subject.wakeup(:proc2)
87
+ pid = subject.process.process[:proc2][:pid]
88
+ task_keys = [:task6, :task7]
89
+ task_keys.each do |key|
90
+ send_task(key)
91
+ end
92
+ subject.waitall(@interval_time)
93
+ task_keys.each do |key|
94
+ File.read(file_path(key)).lines.to_a[-1].strip.to_i.should == pid
95
+ end
96
+ subject.wakeup(:proc1, :proc3)
97
+ end
98
+
99
+ it "should send to all processes." do
100
+ ary_pid = subject.process.all_processes.map do |proc_key|
101
+ subject.process.process[proc_key][:pid]
102
+ end
103
+ key = :all
104
+ send_task(key, nil, true)
105
+ sleep(1)
106
+ result_pid = File.read(file_path(key)).lines.map(&:to_i).sort
107
+ result_pid.should == ary_pid.sort
108
+ end
109
+
110
+ it "should finish." do
111
+ subject.process.create_process(:proc4, :proc5)
112
+ subject.finish
113
+ Process.waitall.should == []
114
+ end
115
+
116
+ after(:each) do
117
+ @result.clear
118
+ @error.clear
119
+ end
120
+
121
+ after(:all) do
122
+ subject.process.kill_all_processes
123
+ subject.process.waitall
124
+ path = File.join(File.dirname(__FILE__), "test_worker")
125
+ FileUtils.rm_r(path) if File.exist?(path)
126
+ end
127
+ end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: drbqs
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.17
4
+ version: 0.0.18
5
5
  prerelease:
6
6
  platform: ruby
7
7
  authors:
@@ -9,11 +9,11 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2011-08-27 00:00:00.000000000 Z
12
+ date: 2011-10-21 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: rspec
16
- requirement: &18201460 !ruby/object:Gem::Requirement
16
+ requirement: &10367840 !ruby/object:Gem::Requirement
17
17
  none: false
18
18
  requirements:
19
19
  - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
21
21
  version: 2.6.0
22
22
  type: :development
23
23
  prerelease: false
24
- version_requirements: *18201460
24
+ version_requirements: *10367840
25
25
  - !ruby/object:Gem::Dependency
26
26
  name: yard
27
- requirement: &18200180 !ruby/object:Gem::Requirement
27
+ requirement: &10366940 !ruby/object:Gem::Requirement
28
28
  none: false
29
29
  requirements:
30
30
  - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
32
32
  version: 0.7.2
33
33
  type: :development
34
34
  prerelease: false
35
- version_requirements: *18200180
35
+ version_requirements: *10366940
36
36
  - !ruby/object:Gem::Dependency
37
37
  name: filename
38
- requirement: &18199300 !ruby/object:Gem::Requirement
38
+ requirement: &10366320 !ruby/object:Gem::Requirement
39
39
  none: false
40
40
  requirements:
41
41
  - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
43
43
  version: 0.1.0
44
44
  type: :runtime
45
45
  prerelease: false
46
- version_requirements: *18199300
46
+ version_requirements: *10366320
47
47
  - !ruby/object:Gem::Dependency
48
48
  name: user_config
49
- requirement: &18216600 !ruby/object:Gem::Requirement
49
+ requirement: &10365760 !ruby/object:Gem::Requirement
50
50
  none: false
51
51
  requirements:
52
52
  - - ! '>='
@@ -54,10 +54,10 @@ dependencies:
54
54
  version: 0.0.2
55
55
  type: :runtime
56
56
  prerelease: false
57
- version_requirements: *18216600
57
+ version_requirements: *10365760
58
58
  - !ruby/object:Gem::Dependency
59
59
  name: net-ssh
60
- requirement: &18214620 !ruby/object:Gem::Requirement
60
+ requirement: &10365160 !ruby/object:Gem::Requirement
61
61
  none: false
62
62
  requirements:
63
63
  - - ! '>='
@@ -65,10 +65,10 @@ dependencies:
65
65
  version: 2.1.0
66
66
  type: :runtime
67
67
  prerelease: false
68
- version_requirements: *18214620
68
+ version_requirements: *10365160
69
69
  - !ruby/object:Gem::Dependency
70
70
  name: net-ssh-shell
71
- requirement: &18213260 !ruby/object:Gem::Requirement
71
+ requirement: &10364460 !ruby/object:Gem::Requirement
72
72
  none: false
73
73
  requirements:
74
74
  - - ! '>='
@@ -76,10 +76,10 @@ dependencies:
76
76
  version: 0.2.0
77
77
  type: :runtime
78
78
  prerelease: false
79
- version_requirements: *18213260
79
+ version_requirements: *10364460
80
80
  - !ruby/object:Gem::Dependency
81
81
  name: net-sftp
82
- requirement: &18212080 !ruby/object:Gem::Requirement
82
+ requirement: &10363900 !ruby/object:Gem::Requirement
83
83
  none: false
84
84
  requirements:
85
85
  - - ! '>='
@@ -87,10 +87,10 @@ dependencies:
87
87
  version: 2.0.5
88
88
  type: :runtime
89
89
  prerelease: false
90
- version_requirements: *18212080
90
+ version_requirements: *10363900
91
91
  - !ruby/object:Gem::Dependency
92
92
  name: sys-proctable
93
- requirement: &18210980 !ruby/object:Gem::Requirement
93
+ requirement: &10363400 !ruby/object:Gem::Requirement
94
94
  none: false
95
95
  requirements:
96
96
  - - ! '>='
@@ -98,7 +98,7 @@ dependencies:
98
98
  version: '0'
99
99
  type: :runtime
100
100
  prerelease: false
101
- version_requirements: *18210980
101
+ version_requirements: *10363400
102
102
  description: Task queuing system over network that is implemented by dRuby.
103
103
  email:
104
104
  - d@ytak.info
@@ -135,6 +135,9 @@ files:
135
135
  - example/execute/execute.rb
136
136
  - example/execute/server.rb
137
137
  - example/execute/task.rb
138
+ - example/group/execute.rb
139
+ - example/group/server.rb
140
+ - example/group/sum.rb
138
141
  - example/mandelbrot/README.md
139
142
  - example/mandelbrot/execute.rb
140
143
  - example/mandelbrot/mandelbrot.rb
@@ -209,6 +212,11 @@ files:
209
212
  - lib/drbqs/utility/transfer/transfer_client_connect.rb
210
213
  - lib/drbqs/utility/transfer/transfer_file_list.rb
211
214
  - lib/drbqs/version.rb
215
+ - lib/drbqs/worker.rb
216
+ - lib/drbqs/worker/forked_process.rb
217
+ - lib/drbqs/worker/serialize.rb
218
+ - lib/drbqs/worker/worker.rb
219
+ - lib/drbqs/worker/worker_process_set.rb
212
220
  - spec/command_line/command_base_spec.rb
213
221
  - spec/command_line/commands_spec.rb
214
222
  - spec/command_line/option_setting_spec.rb
@@ -232,6 +240,8 @@ files:
232
240
  - spec/integration_test/08_shutdown_unused_nodes_spec.rb
233
241
  - spec/integration_test/09_server_process_data_spec.rb
234
242
  - spec/integration_test/10_test_server_spec.rb
243
+ - spec/integration_test/11_special_tasks_spec.rb
244
+ - spec/integration_test/12_multiple_workers_spec.rb
235
245
  - spec/integration_test/definition/server01.rb
236
246
  - spec/integration_test/definition/server02.rb
237
247
  - spec/integration_test/definition/task_obj_definition.rb
@@ -269,6 +279,10 @@ files:
269
279
  - spec/utility/transfer/transfer_client_connect_spec.rb
270
280
  - spec/utility/transfer/transfer_file_list_spec.rb
271
281
  - spec/utility/transfer/transfer_spec.rb
282
+ - spec/worker/forked_process_spec.rb
283
+ - spec/worker/serialize_spec.rb
284
+ - spec/worker/worker_process_set_spec.rb
285
+ - spec/worker/worker_spec.rb
272
286
  homepage: ''
273
287
  licenses: []
274
288
  post_install_message:
@@ -289,8 +303,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
289
303
  version: '0'
290
304
  requirements: []
291
305
  rubyforge_project: drbqs
292
- rubygems_version: 1.8.10
306
+ rubygems_version: 1.8.5
293
307
  signing_key:
294
308
  specification_version: 3
295
309
  summary: dRuby Queueing System
296
310
  test_files: []
311
+ has_rdoc: