drbqs 0.0.17 → 0.0.18

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. data/docs/FormatExecute.md +44 -2
  2. data/example/group/execute.rb +19 -0
  3. data/example/group/server.rb +27 -0
  4. data/example/group/sum.rb +9 -0
  5. data/example/mandelbrot/README.md +8 -0
  6. data/example/mandelbrot/execute.rb +4 -0
  7. data/lib/drbqs/command_line/command_node.rb +1 -0
  8. data/lib/drbqs/execute/execute_node.rb +4 -21
  9. data/lib/drbqs/node/connection.rb +1 -2
  10. data/lib/drbqs/node/node.rb +163 -102
  11. data/lib/drbqs/node/state.rb +100 -35
  12. data/lib/drbqs/node/task_client.rb +46 -33
  13. data/lib/drbqs/server/message.rb +13 -7
  14. data/lib/drbqs/server/server.rb +57 -29
  15. data/lib/drbqs/server/server_hook.rb +19 -5
  16. data/lib/drbqs/server/test/node.rb +31 -6
  17. data/lib/drbqs/setting/node.rb +11 -2
  18. data/lib/drbqs/setting/server.rb +1 -1
  19. data/lib/drbqs/task/task.rb +26 -6
  20. data/lib/drbqs/task/task_generator.rb +2 -1
  21. data/lib/drbqs/utility/temporary.rb +27 -6
  22. data/lib/drbqs/utility/transfer/transfer_client.rb +10 -12
  23. data/lib/drbqs/version.rb +1 -1
  24. data/lib/drbqs/worker.rb +2 -0
  25. data/lib/drbqs/worker/forked_process.rb +100 -0
  26. data/lib/drbqs/worker/serialize.rb +66 -0
  27. data/lib/drbqs/worker/worker.rb +133 -0
  28. data/lib/drbqs/worker/worker_process_set.rb +219 -0
  29. data/spec/integration_test/01_basic_usage_spec.rb +3 -2
  30. data/spec/integration_test/06_node_exit_after_task_spec.rb +3 -2
  31. data/spec/integration_test/07_command_server_with_node_spec.rb +1 -0
  32. data/spec/integration_test/08_shutdown_unused_nodes_spec.rb +3 -2
  33. data/spec/integration_test/10_test_server_spec.rb +2 -2
  34. data/spec/integration_test/11_special_tasks_spec.rb +61 -0
  35. data/spec/integration_test/12_multiple_workers_spec.rb +43 -0
  36. data/spec/integration_test/definition/task_obj_definition.rb +33 -6
  37. data/spec/node/connection_spec.rb +6 -6
  38. data/spec/node/node_spec.rb +10 -2
  39. data/spec/node/state_spec.rb +146 -62
  40. data/spec/node/task_client_spec.rb +58 -53
  41. data/spec/server/message_spec.rb +10 -6
  42. data/spec/server/queue_spec.rb +7 -4
  43. data/spec/server/server_hook_spec.rb +28 -1
  44. data/spec/task/task_spec.rb +43 -6
  45. data/spec/utility/temporary_spec.rb +32 -9
  46. data/spec/worker/forked_process_spec.rb +66 -0
  47. data/spec/worker/serialize_spec.rb +73 -0
  48. data/spec/worker/worker_process_set_spec.rb +104 -0
  49. data/spec/worker/worker_spec.rb +127 -0
  50. metadata +34 -19
@@ -3,19 +3,19 @@ require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
3
3
  require 'drbqs/node/task_client'
4
4
 
5
5
  describe DRbQS::Node::TaskClient do
6
- def init_task_client(node_number = 10)
7
- @node_number = node_number
6
+ def init_task_client(opts = {})
7
+ @node_number = opts[:number] || 10
8
8
  @ts_queue = Rinda::TupleSpace.new
9
9
  @ts_result = Rinda::TupleSpace.new
10
- @task_client = DRbQS::Node::TaskClient.new(@node_number, @ts_queue, @ts_result)
10
+ @task_client = DRbQS::Node::TaskClient.new(@node_number, @ts_queue, @ts_result, opts[:group])
11
11
  end
12
12
 
13
- def add_task_to_tuplespace(task_id, task_ary)
14
- @ts_queue.write([task_id] + task_ary)
13
+ def add_task_to_tuplespace(task_id, task_ary, group = DRbQS::Task::DEFAULT_GROUP)
14
+ @ts_queue.write([group, task_id] + task_ary)
15
15
  end
16
16
 
17
17
  def check_empty_queue_tuplespace
18
- @ts_queue.read_all([nil, nil, nil, nil]).size.should == 0
18
+ @ts_queue.read_all([nil, nil, nil, nil, nil]).size.should == 0
19
19
  end
20
20
 
21
21
  subject do
@@ -35,12 +35,12 @@ describe DRbQS::Node::TaskClient do
35
35
  subject.result_empty?.should be_true
36
36
  end
37
37
 
38
- it "should not have calculating task." do
39
- subject.calculating_task.should be_nil
38
+ it "should return no task." do
39
+ subject.get_task.should be_nil
40
40
  end
41
41
 
42
- it "should return no task" do
43
- subject.get_task.should be_nil
42
+ it "should not have group." do
43
+ subject.group.should == []
44
44
  end
45
45
  end
46
46
 
@@ -61,28 +61,54 @@ describe DRbQS::Node::TaskClient do
61
61
  end
62
62
  end
63
63
 
64
+ context "when getting task by group" do
65
+ before(:all) do
66
+ init_task_client(:group => [:grp1])
67
+ @task_id = 3
68
+ @task_ary = ["hello world", :size, []]
69
+ add_task_to_tuplespace(@task_id, @task_ary, :grp1)
70
+ end
71
+
72
+ it "should not return task." do
73
+ client = DRbQS::Node::TaskClient.new(100, @ts_queue, @ts_result, nil)
74
+ client.get_task.should be_nil
75
+ end
76
+
77
+ it "should return task." do
78
+ subject.get_task.should == @task_ary.dup.unshift(@task_id)
79
+ end
80
+
81
+ it "should take out task from tuplespace." do
82
+ check_empty_queue_tuplespace
83
+ end
84
+
85
+ it "should have group." do
86
+ subject.group.should == [:grp1]
87
+ end
88
+ end
89
+
64
90
  context "when getting new task" do
65
91
  before(:all) do
66
- init_task_client(7)
92
+ init_task_client(:number => 7)
67
93
  end
68
94
 
69
95
  it "should return nil" do
70
- subject.add_new_task.should be_nil
96
+ subject.add_new_task(1).should be_nil
71
97
  end
72
98
 
73
99
  it "should return nil" do
74
100
  add_task_to_tuplespace(100, [[1, 3, 5, 7], :size, []])
75
- subject.add_new_task.should be_true
101
+ subject.add_new_task(1).should be_true
76
102
  end
77
103
  end
78
104
 
79
105
  context "when adding new task" do
80
106
  before(:all) do
81
- init_task_client(7)
107
+ init_task_client(:number => 7)
82
108
  @task_id = 3
83
109
  @task_ary = [[1, 3, 5, 7], :size, []]
84
110
  add_task_to_tuplespace(@task_id, @task_ary)
85
- subject.add_new_task
111
+ @get_task_id = subject.add_new_task(1)
86
112
  end
87
113
 
88
114
  it "should have non empty task queue." do
@@ -94,7 +120,7 @@ describe DRbQS::Node::TaskClient do
94
120
  end
95
121
 
96
122
  it "should have calculating task" do
97
- subject.calculating_task.should == @task_id
123
+ @get_task_id.should be_include(@task_id)
98
124
  end
99
125
 
100
126
  it "should take out task from tuplespace." do
@@ -108,16 +134,16 @@ describe DRbQS::Node::TaskClient do
108
134
 
109
135
  context "when dequeueing a task" do
110
136
  before(:all) do
111
- init_task_client(14)
137
+ init_task_client(:number => 14)
112
138
  @task_id = 8
113
139
  @task_ary = [[1, 3, 5, 7], :size, []]
114
140
  add_task_to_tuplespace(@task_id, @task_ary)
115
- subject.add_new_task
141
+ @get_task_id = subject.add_new_task(1)
116
142
  @dequeued_task = subject.dequeue_task
117
143
  end
118
144
 
119
145
  it "should get a task" do
120
- @dequeued_task.should == @task_ary
146
+ @dequeued_task.should == [@task_id] + @task_ary
121
147
  end
122
148
 
123
149
  it "should have empty task queue." do
@@ -129,19 +155,19 @@ describe DRbQS::Node::TaskClient do
129
155
  end
130
156
 
131
157
  it "should have calculating task." do
132
- subject.calculating_task.should == @task_id
158
+ @get_task_id.should be_include(@task_id)
133
159
  end
134
160
  end
135
161
 
136
162
  context "when queueing result" do
137
163
  before(:all) do
138
- init_task_client(2)
164
+ init_task_client(:number => 2)
139
165
  @task_id = 27
140
166
  @task_ary = ["abcdef", :size, []]
141
167
  add_task_to_tuplespace(@task_id, @task_ary)
142
- subject.add_new_task
168
+ @get_task_id = subject.add_new_task(1)
143
169
  @dequeued_task = subject.dequeue_task
144
- subject.queue_result(:result_object)
170
+ subject.queue_result(1, :result_object)
145
171
  end
146
172
 
147
173
  it "should have empty task queue." do
@@ -153,24 +179,24 @@ describe DRbQS::Node::TaskClient do
153
179
  end
154
180
 
155
181
  it "should have calculating task." do
156
- subject.calculating_task.should == @task_id
182
+ @get_task_id.should be_include(@task_id)
157
183
  end
158
184
  end
159
185
 
160
186
  context "when sending result" do
161
187
  before(:all) do
162
- init_task_client(2)
163
- @task_id = 27
188
+ init_task_client(:number => 2)
189
+ @task_id = 28
164
190
  @task_ary = ["abcdef", :size, []]
165
191
  add_task_to_tuplespace(@task_id, @task_ary)
166
- subject.add_new_task
192
+ subject.add_new_task(1)
167
193
  @dequeued_task = subject.dequeue_task
168
- subject.queue_result(:result_object)
194
+ subject.queue_result(@task_id, :result_object)
169
195
  @send_returned_value = subject.send_result
170
196
  end
171
197
 
172
198
  it "should get nil returned value." do
173
- @send_returned_value.should be_nil
199
+ @send_returned_value.should == [@task_id]
174
200
  end
175
201
 
176
202
  it "should have empty task queue." do
@@ -181,35 +207,14 @@ describe DRbQS::Node::TaskClient do
181
207
  subject.result_empty?.should_not be_nil
182
208
  end
183
209
 
184
- it "should not have calculating task." do
185
- subject.calculating_task.should be_nil
186
- end
187
-
188
210
  it "should get result from tuplespace." do
189
211
  @ts_result.take([:result, nil, nil, nil], 0).should be_true
190
212
  end
191
213
  end
192
214
 
193
- context "when setting exit_after_task" do
194
- before(:all) do
195
- init_task_client(2)
196
- @task_id = 27
197
- @task_ary = ["abcdef", :size, []]
198
- add_task_to_tuplespace(@task_id, @task_ary)
199
- subject.add_new_task
200
- subject.set_exit_after_task
201
- @dequeued_task = subject.dequeue_task
202
- subject.queue_result(:result_object)
203
- end
204
-
205
- it "should return true" do
206
- subject.send_result.should be_true
207
- end
208
- end
209
-
210
215
  context "when dumping result queue" do
211
216
  before(:all) do
212
- init_task_client(2)
217
+ init_task_client(:number => 2)
213
218
  end
214
219
 
215
220
  it "should return nil for empty result queue." do
@@ -218,8 +223,8 @@ describe DRbQS::Node::TaskClient do
218
223
 
219
224
  it "should return string for non empty result queue." do
220
225
  results = [:result_object1, :result_object2]
221
- results.each do |sym|
222
- subject.queue_result(sym)
226
+ results.each_with_index do |sym, i|
227
+ subject.queue_result(i, sym)
223
228
  end
224
229
  subject.dump_result_queue.should == Marshal.dump(results)
225
230
  end
@@ -122,19 +122,23 @@ describe DRbQS::Server::Message do
122
122
 
123
123
  context "when setting special tasks" do
124
124
  it "should set initialization task" do
125
+ initialization_pattern = [:initialize, Array]
125
126
  lambda do
126
- @message.take([:initialize, nil, Symbol, nil], 0)
127
+ @message.take(initialization_pattern, 0)
127
128
  end.should raise_error Rinda::RequestExpiredError
128
- @message_server.set_initialization(DRbQS::Task.new(Object.new, :object_id))
129
- @message.take([:initialize, nil, Symbol, nil], 0).should be_true
129
+ @message_server.set_initialization_tasks(DRbQS::Task.new(Object.new, :object_id))
130
+ @message_server.set_initialization_tasks(DRbQS::Task.new(Object.new, :object_id))
131
+ @message.take(initialization_pattern, 0)[1].should have(2).items
130
132
  end
131
133
 
132
134
  it "should set finalization task" do
135
+ finalization_pattern = [:finalize, Array]
133
136
  lambda do
134
- @message.take([:finalization, nil, Symbol, nil], 0)
137
+ @message.take(finalization_pattern, 0)
135
138
  end.should raise_error Rinda::RequestExpiredError
136
- @message_server.set_finalization(DRbQS::Task.new(Object.new, :object_id))
137
- @message.take([:finalization, nil, Symbol, nil], 0).should be_true
139
+ tasks = [DRbQS::Task.new(Object.new, :object_id), DRbQS::Task.new(Object.new, :object_id)]
140
+ @message_server.set_finalization_tasks(tasks)
141
+ @message.take(finalization_pattern, 0)[1].should have(2).items
138
142
  end
139
143
  end
140
144
 
@@ -4,6 +4,8 @@ require 'drbqs/server/queue'
4
4
  require 'drbqs/task/task'
5
5
 
6
6
  describe DRbQS::Server::Queue do
7
+ QUEUE_OBJECT_PATTERN = [nil, nil, nil, nil, nil]
8
+
7
9
  def object_init
8
10
  @ts = {
9
11
  :queue => Rinda::TupleSpace.new,
@@ -86,7 +88,7 @@ describe DRbQS::Server::Queue do
86
88
  end
87
89
 
88
90
  it "should take objects from queue." do
89
- @ts[:queue].take([nil, nil, nil, nil]).should be_true
91
+ @ts[:queue].take(QUEUE_OBJECT_PATTERN).should be_true
90
92
  end
91
93
 
92
94
  it "should return an array of calculating nodes." do
@@ -208,9 +210,10 @@ describe DRbQS::Server::Queue do
208
210
  subject.should_not be_finished
209
211
  end
210
212
 
211
- it "should return task ID." do
212
- (ary = @ts[:queue].take([nil, nil, nil, nil])).should be_true
213
- ary[0].should == @task_id
213
+ it "should return task group and task ID." do
214
+ (ary = @ts[:queue].take(QUEUE_OBJECT_PATTERN)).should be_true
215
+ ary[0].should == DRbQS::Task::DEFAULT_GROUP
216
+ ary[1].should == @task_id
214
217
  end
215
218
 
216
219
  it "should return an array of calculating nodes." do
@@ -21,7 +21,7 @@ describe DRbQS::Server::Hook do
21
21
  it "should add a hook with name." do
22
22
  n = subject.number_of_hook(:finish)
23
23
  name = 'hello'
24
- name_new = subject.add(:finish, name) do |server|
24
+ name_new = subject.add(:finish, name: name) do |server|
25
25
  3 + 4
26
26
  end
27
27
  name_new.should == name
@@ -71,6 +71,33 @@ describe DRbQS::Server::Hook do
71
71
  exec_flag[:second].should be_true
72
72
  end
73
73
 
74
+ it "should execute a hook repeatedly." do
75
+ exec_flag = {}
76
+ subject.add(:finish) do |server|
77
+ exec_flag[:exec] = true
78
+ end
79
+ 10.times do |i|
80
+ exec_flag[:exec] = nil
81
+ subject.exec(:finish)
82
+ exec_flag[:exec].should be_true
83
+ end
84
+ end
85
+
86
+ it "should execute a hook three times." do
87
+ exec_flag = {}
88
+ subject.add(:finish, repeat: 3) do |server|
89
+ exec_flag[:exec] = true
90
+ end
91
+ 3.times do |i|
92
+ exec_flag[:exec] = nil
93
+ subject.exec(:finish)
94
+ exec_flag[:exec].should be_true
95
+ end
96
+ exec_flag[:exec] = nil
97
+ subject.exec(:finish)
98
+ exec_flag[:exec].should be_nil
99
+ end
100
+
74
101
  it "should execute finish_exit that is special proc." do
75
102
  execute = nil
76
103
  subject.set_finish_exit do
@@ -92,6 +92,17 @@ describe DRbQS::Task do
92
92
  Marshal.dump(task)
93
93
  end.should raise_error
94
94
  end
95
+
96
+ it "should get default group." do
97
+ task = DRbQS::Task.new([1, 2, 3], :size)
98
+ task.group.should == DRbQS::Task::DEFAULT_GROUP
99
+ end
100
+
101
+ it "should set group." do
102
+ group = :local
103
+ task = DRbQS::Task.new([1, 2, 3], :size, group: group)
104
+ task.group.should == group
105
+ end
95
106
  end
96
107
 
97
108
  describe DRbQS::Task::TaskSet::ContainerTask do
@@ -172,12 +183,38 @@ describe DRbQS::Task::TaskSet do
172
183
 
173
184
  it "should execute hooks of symbols." do
174
185
  server = mock('server')
175
- obj = [TestTask.new(100), TestTask.new(200), TestTask.new(300)]
176
- tasks = []
177
- tasks << DRbQS::Task.new(obj[0], :some_method, hook: :test_hook)
178
- tasks << DRbQS::Task.new(obj[1], :some_method, hook: :test_hook)
179
- tasks << DRbQS::Task.new(obj[2], :some_method, hook: :test_hook)
186
+ args = [100, 200, 300]
187
+ tasks = args.map do |a|
188
+ DRbQS::Task.new(TestTask.new(a), :some_method, hook: :test_hook)
189
+ end
180
190
  task_set = DRbQS::Task::TaskSet.new(tasks)
181
- task_set.exec_hook(server, [100, 200, 300])
191
+ task_set.exec_hook(server, args)
192
+ end
193
+
194
+ it "should get default group." do
195
+ tasks = 10.times.map do |a|
196
+ DRbQS::Task.new(TestTask.new(a), :some_method)
197
+ end
198
+ task_set = DRbQS::Task::TaskSet.new(tasks)
199
+ task_set.group.should == DRbQS::Task::DEFAULT_GROUP
200
+ end
201
+
202
+ it "should set group." do
203
+ group = :local
204
+ tasks = 10.times.map do |a|
205
+ DRbQS::Task.new(TestTask.new(a), :some_method, group: group)
206
+ end
207
+ task_set = DRbQS::Task::TaskSet.new(tasks)
208
+ task_set.group.should == group
209
+ end
210
+
211
+ it "should raise error for different groups." do
212
+ group = :local
213
+ tasks = 10.times.map do |a|
214
+ DRbQS::Task.new(TestTask.new(a), :some_method, group: "group#{a}".intern)
215
+ end
216
+ lambda do
217
+ DRbQS::Task::TaskSet.new(tasks)
218
+ end.should raise_error ArgumentError
182
219
  end
183
220
  end
@@ -7,18 +7,22 @@ describe DRbQS::Temporary do
7
7
  DRbQS::Temporary.file
8
8
  end
9
9
 
10
+ subject do
11
+ DRbQS::Temporary
12
+ end
13
+
10
14
  it "should create base directory." do
11
- File.exist?(DRbQS::Temporary.root).should be_true
15
+ File.exist?(subject.root).should be_true
12
16
  end
13
17
 
14
18
  it "should create empty directory." do
15
- dir = DRbQS::Temporary.directory
19
+ dir = subject.directory
16
20
  File.directory?(dir).should be_true
17
21
  Dir.entries(dir).should have(2).items
18
22
  end
19
23
 
20
24
  it "should return new path of file." do
21
- path = DRbQS::Temporary.file
25
+ path = subject.file
22
26
  File.exist?(path).should_not be_true
23
27
  open(path, 'w') do |f|
24
28
  f.puts 'hello'
@@ -26,14 +30,33 @@ describe DRbQS::Temporary do
26
30
  end
27
31
 
28
32
  it "should make directory empty." do
29
- Dir.entries(DRbQS::Temporary.root).size.should > 2
30
- DRbQS::Temporary.delete
31
- Dir.entries(DRbQS::Temporary.root).should have(2).items
33
+ Dir.entries(subject.root).size.should > 2
34
+ subject.delete
35
+ Dir.entries(subject.root).should have(2).items
36
+ end
37
+
38
+ it "should set subdirectory." do
39
+ subject.set_sub_directory('abc')
40
+ subject.subdirectory.should be_nil
41
+ end
42
+
43
+ it "should get directory in subdirectory." do
44
+ subject.set_sub_directory('ABCD')
45
+ dir = subject.directory
46
+ File.exist?(subject.subdirectory).should be_true
47
+ dir.should match(/\/ABCD/)
48
+ end
49
+
50
+ it "should get file name in subdirectory." do
51
+ subject.set_sub_directory('EFGH')
52
+ file = subject.file
53
+ File.exist?(subject.subdirectory).should be_true
54
+ file.should match(/\/EFGH\//)
32
55
  end
33
56
 
34
- it "should all directories." do
35
- root = DRbQS::Temporary.root
36
- DRbQS::Temporary.delete_all
57
+ it "should delete all directories." do
58
+ root = subject.root
59
+ subject.delete_all
37
60
  File.exist?(root).should_not be_true
38
61
  end
39
62
  end