drbqs 0.0.17 → 0.0.18

Sign up to get free protection for your applications and to get access to all the features.
Files changed (50) hide show
  1. data/docs/FormatExecute.md +44 -2
  2. data/example/group/execute.rb +19 -0
  3. data/example/group/server.rb +27 -0
  4. data/example/group/sum.rb +9 -0
  5. data/example/mandelbrot/README.md +8 -0
  6. data/example/mandelbrot/execute.rb +4 -0
  7. data/lib/drbqs/command_line/command_node.rb +1 -0
  8. data/lib/drbqs/execute/execute_node.rb +4 -21
  9. data/lib/drbqs/node/connection.rb +1 -2
  10. data/lib/drbqs/node/node.rb +163 -102
  11. data/lib/drbqs/node/state.rb +100 -35
  12. data/lib/drbqs/node/task_client.rb +46 -33
  13. data/lib/drbqs/server/message.rb +13 -7
  14. data/lib/drbqs/server/server.rb +57 -29
  15. data/lib/drbqs/server/server_hook.rb +19 -5
  16. data/lib/drbqs/server/test/node.rb +31 -6
  17. data/lib/drbqs/setting/node.rb +11 -2
  18. data/lib/drbqs/setting/server.rb +1 -1
  19. data/lib/drbqs/task/task.rb +26 -6
  20. data/lib/drbqs/task/task_generator.rb +2 -1
  21. data/lib/drbqs/utility/temporary.rb +27 -6
  22. data/lib/drbqs/utility/transfer/transfer_client.rb +10 -12
  23. data/lib/drbqs/version.rb +1 -1
  24. data/lib/drbqs/worker.rb +2 -0
  25. data/lib/drbqs/worker/forked_process.rb +100 -0
  26. data/lib/drbqs/worker/serialize.rb +66 -0
  27. data/lib/drbqs/worker/worker.rb +133 -0
  28. data/lib/drbqs/worker/worker_process_set.rb +219 -0
  29. data/spec/integration_test/01_basic_usage_spec.rb +3 -2
  30. data/spec/integration_test/06_node_exit_after_task_spec.rb +3 -2
  31. data/spec/integration_test/07_command_server_with_node_spec.rb +1 -0
  32. data/spec/integration_test/08_shutdown_unused_nodes_spec.rb +3 -2
  33. data/spec/integration_test/10_test_server_spec.rb +2 -2
  34. data/spec/integration_test/11_special_tasks_spec.rb +61 -0
  35. data/spec/integration_test/12_multiple_workers_spec.rb +43 -0
  36. data/spec/integration_test/definition/task_obj_definition.rb +33 -6
  37. data/spec/node/connection_spec.rb +6 -6
  38. data/spec/node/node_spec.rb +10 -2
  39. data/spec/node/state_spec.rb +146 -62
  40. data/spec/node/task_client_spec.rb +58 -53
  41. data/spec/server/message_spec.rb +10 -6
  42. data/spec/server/queue_spec.rb +7 -4
  43. data/spec/server/server_hook_spec.rb +28 -1
  44. data/spec/task/task_spec.rb +43 -6
  45. data/spec/utility/temporary_spec.rb +32 -9
  46. data/spec/worker/forked_process_spec.rb +66 -0
  47. data/spec/worker/serialize_spec.rb +73 -0
  48. data/spec/worker/worker_process_set_spec.rb +104 -0
  49. data/spec/worker/worker_spec.rb +127 -0
  50. metadata +34 -19
@@ -3,19 +3,19 @@ require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
3
3
  require 'drbqs/node/task_client'
4
4
 
5
5
  describe DRbQS::Node::TaskClient do
6
- def init_task_client(node_number = 10)
7
- @node_number = node_number
6
+ def init_task_client(opts = {})
7
+ @node_number = opts[:number] || 10
8
8
  @ts_queue = Rinda::TupleSpace.new
9
9
  @ts_result = Rinda::TupleSpace.new
10
- @task_client = DRbQS::Node::TaskClient.new(@node_number, @ts_queue, @ts_result)
10
+ @task_client = DRbQS::Node::TaskClient.new(@node_number, @ts_queue, @ts_result, opts[:group])
11
11
  end
12
12
 
13
- def add_task_to_tuplespace(task_id, task_ary)
14
- @ts_queue.write([task_id] + task_ary)
13
+ def add_task_to_tuplespace(task_id, task_ary, group = DRbQS::Task::DEFAULT_GROUP)
14
+ @ts_queue.write([group, task_id] + task_ary)
15
15
  end
16
16
 
17
17
  def check_empty_queue_tuplespace
18
- @ts_queue.read_all([nil, nil, nil, nil]).size.should == 0
18
+ @ts_queue.read_all([nil, nil, nil, nil, nil]).size.should == 0
19
19
  end
20
20
 
21
21
  subject do
@@ -35,12 +35,12 @@ describe DRbQS::Node::TaskClient do
35
35
  subject.result_empty?.should be_true
36
36
  end
37
37
 
38
- it "should not have calculating task." do
39
- subject.calculating_task.should be_nil
38
+ it "should return no task." do
39
+ subject.get_task.should be_nil
40
40
  end
41
41
 
42
- it "should return no task" do
43
- subject.get_task.should be_nil
42
+ it "should not have group." do
43
+ subject.group.should == []
44
44
  end
45
45
  end
46
46
 
@@ -61,28 +61,54 @@ describe DRbQS::Node::TaskClient do
61
61
  end
62
62
  end
63
63
 
64
+ context "when getting task by group" do
65
+ before(:all) do
66
+ init_task_client(:group => [:grp1])
67
+ @task_id = 3
68
+ @task_ary = ["hello world", :size, []]
69
+ add_task_to_tuplespace(@task_id, @task_ary, :grp1)
70
+ end
71
+
72
+ it "should not return task." do
73
+ client = DRbQS::Node::TaskClient.new(100, @ts_queue, @ts_result, nil)
74
+ client.get_task.should be_nil
75
+ end
76
+
77
+ it "should return task." do
78
+ subject.get_task.should == @task_ary.dup.unshift(@task_id)
79
+ end
80
+
81
+ it "should take out task from tuplespace." do
82
+ check_empty_queue_tuplespace
83
+ end
84
+
85
+ it "should have group." do
86
+ subject.group.should == [:grp1]
87
+ end
88
+ end
89
+
64
90
  context "when getting new task" do
65
91
  before(:all) do
66
- init_task_client(7)
92
+ init_task_client(:number => 7)
67
93
  end
68
94
 
69
95
  it "should return nil" do
70
- subject.add_new_task.should be_nil
96
+ subject.add_new_task(1).should be_nil
71
97
  end
72
98
 
73
99
  it "should return nil" do
74
100
  add_task_to_tuplespace(100, [[1, 3, 5, 7], :size, []])
75
- subject.add_new_task.should be_true
101
+ subject.add_new_task(1).should be_true
76
102
  end
77
103
  end
78
104
 
79
105
  context "when adding new task" do
80
106
  before(:all) do
81
- init_task_client(7)
107
+ init_task_client(:number => 7)
82
108
  @task_id = 3
83
109
  @task_ary = [[1, 3, 5, 7], :size, []]
84
110
  add_task_to_tuplespace(@task_id, @task_ary)
85
- subject.add_new_task
111
+ @get_task_id = subject.add_new_task(1)
86
112
  end
87
113
 
88
114
  it "should have non empty task queue." do
@@ -94,7 +120,7 @@ describe DRbQS::Node::TaskClient do
94
120
  end
95
121
 
96
122
  it "should have calculating task" do
97
- subject.calculating_task.should == @task_id
123
+ @get_task_id.should be_include(@task_id)
98
124
  end
99
125
 
100
126
  it "should take out task from tuplespace." do
@@ -108,16 +134,16 @@ describe DRbQS::Node::TaskClient do
108
134
 
109
135
  context "when dequeueing a task" do
110
136
  before(:all) do
111
- init_task_client(14)
137
+ init_task_client(:number => 14)
112
138
  @task_id = 8
113
139
  @task_ary = [[1, 3, 5, 7], :size, []]
114
140
  add_task_to_tuplespace(@task_id, @task_ary)
115
- subject.add_new_task
141
+ @get_task_id = subject.add_new_task(1)
116
142
  @dequeued_task = subject.dequeue_task
117
143
  end
118
144
 
119
145
  it "should get a task" do
120
- @dequeued_task.should == @task_ary
146
+ @dequeued_task.should == [@task_id] + @task_ary
121
147
  end
122
148
 
123
149
  it "should have empty task queue." do
@@ -129,19 +155,19 @@ describe DRbQS::Node::TaskClient do
129
155
  end
130
156
 
131
157
  it "should have calculating task." do
132
- subject.calculating_task.should == @task_id
158
+ @get_task_id.should be_include(@task_id)
133
159
  end
134
160
  end
135
161
 
136
162
  context "when queueing result" do
137
163
  before(:all) do
138
- init_task_client(2)
164
+ init_task_client(:number => 2)
139
165
  @task_id = 27
140
166
  @task_ary = ["abcdef", :size, []]
141
167
  add_task_to_tuplespace(@task_id, @task_ary)
142
- subject.add_new_task
168
+ @get_task_id = subject.add_new_task(1)
143
169
  @dequeued_task = subject.dequeue_task
144
- subject.queue_result(:result_object)
170
+ subject.queue_result(1, :result_object)
145
171
  end
146
172
 
147
173
  it "should have empty task queue." do
@@ -153,24 +179,24 @@ describe DRbQS::Node::TaskClient do
153
179
  end
154
180
 
155
181
  it "should have calculating task." do
156
- subject.calculating_task.should == @task_id
182
+ @get_task_id.should be_include(@task_id)
157
183
  end
158
184
  end
159
185
 
160
186
  context "when sending result" do
161
187
  before(:all) do
162
- init_task_client(2)
163
- @task_id = 27
188
+ init_task_client(:number => 2)
189
+ @task_id = 28
164
190
  @task_ary = ["abcdef", :size, []]
165
191
  add_task_to_tuplespace(@task_id, @task_ary)
166
- subject.add_new_task
192
+ subject.add_new_task(1)
167
193
  @dequeued_task = subject.dequeue_task
168
- subject.queue_result(:result_object)
194
+ subject.queue_result(@task_id, :result_object)
169
195
  @send_returned_value = subject.send_result
170
196
  end
171
197
 
172
198
  it "should get nil returned value." do
173
- @send_returned_value.should be_nil
199
+ @send_returned_value.should == [@task_id]
174
200
  end
175
201
 
176
202
  it "should have empty task queue." do
@@ -181,35 +207,14 @@ describe DRbQS::Node::TaskClient do
181
207
  subject.result_empty?.should_not be_nil
182
208
  end
183
209
 
184
- it "should not have calculating task." do
185
- subject.calculating_task.should be_nil
186
- end
187
-
188
210
  it "should get result from tuplespace." do
189
211
  @ts_result.take([:result, nil, nil, nil], 0).should be_true
190
212
  end
191
213
  end
192
214
 
193
- context "when setting exit_after_task" do
194
- before(:all) do
195
- init_task_client(2)
196
- @task_id = 27
197
- @task_ary = ["abcdef", :size, []]
198
- add_task_to_tuplespace(@task_id, @task_ary)
199
- subject.add_new_task
200
- subject.set_exit_after_task
201
- @dequeued_task = subject.dequeue_task
202
- subject.queue_result(:result_object)
203
- end
204
-
205
- it "should return true" do
206
- subject.send_result.should be_true
207
- end
208
- end
209
-
210
215
  context "when dumping result queue" do
211
216
  before(:all) do
212
- init_task_client(2)
217
+ init_task_client(:number => 2)
213
218
  end
214
219
 
215
220
  it "should return nil for empty result queue." do
@@ -218,8 +223,8 @@ describe DRbQS::Node::TaskClient do
218
223
 
219
224
  it "should return string for non empty result queue." do
220
225
  results = [:result_object1, :result_object2]
221
- results.each do |sym|
222
- subject.queue_result(sym)
226
+ results.each_with_index do |sym, i|
227
+ subject.queue_result(i, sym)
223
228
  end
224
229
  subject.dump_result_queue.should == Marshal.dump(results)
225
230
  end
@@ -122,19 +122,23 @@ describe DRbQS::Server::Message do
122
122
 
123
123
  context "when setting special tasks" do
124
124
  it "should set initialization task" do
125
+ initialization_pattern = [:initialize, Array]
125
126
  lambda do
126
- @message.take([:initialize, nil, Symbol, nil], 0)
127
+ @message.take(initialization_pattern, 0)
127
128
  end.should raise_error Rinda::RequestExpiredError
128
- @message_server.set_initialization(DRbQS::Task.new(Object.new, :object_id))
129
- @message.take([:initialize, nil, Symbol, nil], 0).should be_true
129
+ @message_server.set_initialization_tasks(DRbQS::Task.new(Object.new, :object_id))
130
+ @message_server.set_initialization_tasks(DRbQS::Task.new(Object.new, :object_id))
131
+ @message.take(initialization_pattern, 0)[1].should have(2).items
130
132
  end
131
133
 
132
134
  it "should set finalization task" do
135
+ finalization_pattern = [:finalize, Array]
133
136
  lambda do
134
- @message.take([:finalization, nil, Symbol, nil], 0)
137
+ @message.take(finalization_pattern, 0)
135
138
  end.should raise_error Rinda::RequestExpiredError
136
- @message_server.set_finalization(DRbQS::Task.new(Object.new, :object_id))
137
- @message.take([:finalization, nil, Symbol, nil], 0).should be_true
139
+ tasks = [DRbQS::Task.new(Object.new, :object_id), DRbQS::Task.new(Object.new, :object_id)]
140
+ @message_server.set_finalization_tasks(tasks)
141
+ @message.take(finalization_pattern, 0)[1].should have(2).items
138
142
  end
139
143
  end
140
144
 
@@ -4,6 +4,8 @@ require 'drbqs/server/queue'
4
4
  require 'drbqs/task/task'
5
5
 
6
6
  describe DRbQS::Server::Queue do
7
+ QUEUE_OBJECT_PATTERN = [nil, nil, nil, nil, nil]
8
+
7
9
  def object_init
8
10
  @ts = {
9
11
  :queue => Rinda::TupleSpace.new,
@@ -86,7 +88,7 @@ describe DRbQS::Server::Queue do
86
88
  end
87
89
 
88
90
  it "should take objects from queue." do
89
- @ts[:queue].take([nil, nil, nil, nil]).should be_true
91
+ @ts[:queue].take(QUEUE_OBJECT_PATTERN).should be_true
90
92
  end
91
93
 
92
94
  it "should return an array of calculating nodes." do
@@ -208,9 +210,10 @@ describe DRbQS::Server::Queue do
208
210
  subject.should_not be_finished
209
211
  end
210
212
 
211
- it "should return task ID." do
212
- (ary = @ts[:queue].take([nil, nil, nil, nil])).should be_true
213
- ary[0].should == @task_id
213
+ it "should return task group and task ID." do
214
+ (ary = @ts[:queue].take(QUEUE_OBJECT_PATTERN)).should be_true
215
+ ary[0].should == DRbQS::Task::DEFAULT_GROUP
216
+ ary[1].should == @task_id
214
217
  end
215
218
 
216
219
  it "should return an array of calculating nodes." do
@@ -21,7 +21,7 @@ describe DRbQS::Server::Hook do
21
21
  it "should add a hook with name." do
22
22
  n = subject.number_of_hook(:finish)
23
23
  name = 'hello'
24
- name_new = subject.add(:finish, name) do |server|
24
+ name_new = subject.add(:finish, name: name) do |server|
25
25
  3 + 4
26
26
  end
27
27
  name_new.should == name
@@ -71,6 +71,33 @@ describe DRbQS::Server::Hook do
71
71
  exec_flag[:second].should be_true
72
72
  end
73
73
 
74
+ it "should execute a hook repeatedly." do
75
+ exec_flag = {}
76
+ subject.add(:finish) do |server|
77
+ exec_flag[:exec] = true
78
+ end
79
+ 10.times do |i|
80
+ exec_flag[:exec] = nil
81
+ subject.exec(:finish)
82
+ exec_flag[:exec].should be_true
83
+ end
84
+ end
85
+
86
+ it "should execute a hook three times." do
87
+ exec_flag = {}
88
+ subject.add(:finish, repeat: 3) do |server|
89
+ exec_flag[:exec] = true
90
+ end
91
+ 3.times do |i|
92
+ exec_flag[:exec] = nil
93
+ subject.exec(:finish)
94
+ exec_flag[:exec].should be_true
95
+ end
96
+ exec_flag[:exec] = nil
97
+ subject.exec(:finish)
98
+ exec_flag[:exec].should be_nil
99
+ end
100
+
74
101
  it "should execute finish_exit that is special proc." do
75
102
  execute = nil
76
103
  subject.set_finish_exit do
@@ -92,6 +92,17 @@ describe DRbQS::Task do
92
92
  Marshal.dump(task)
93
93
  end.should raise_error
94
94
  end
95
+
96
+ it "should get default group." do
97
+ task = DRbQS::Task.new([1, 2, 3], :size)
98
+ task.group.should == DRbQS::Task::DEFAULT_GROUP
99
+ end
100
+
101
+ it "should set group." do
102
+ group = :local
103
+ task = DRbQS::Task.new([1, 2, 3], :size, group: group)
104
+ task.group.should == group
105
+ end
95
106
  end
96
107
 
97
108
  describe DRbQS::Task::TaskSet::ContainerTask do
@@ -172,12 +183,38 @@ describe DRbQS::Task::TaskSet do
172
183
 
173
184
  it "should execute hooks of symbols." do
174
185
  server = mock('server')
175
- obj = [TestTask.new(100), TestTask.new(200), TestTask.new(300)]
176
- tasks = []
177
- tasks << DRbQS::Task.new(obj[0], :some_method, hook: :test_hook)
178
- tasks << DRbQS::Task.new(obj[1], :some_method, hook: :test_hook)
179
- tasks << DRbQS::Task.new(obj[2], :some_method, hook: :test_hook)
186
+ args = [100, 200, 300]
187
+ tasks = args.map do |a|
188
+ DRbQS::Task.new(TestTask.new(a), :some_method, hook: :test_hook)
189
+ end
180
190
  task_set = DRbQS::Task::TaskSet.new(tasks)
181
- task_set.exec_hook(server, [100, 200, 300])
191
+ task_set.exec_hook(server, args)
192
+ end
193
+
194
+ it "should get default group." do
195
+ tasks = 10.times.map do |a|
196
+ DRbQS::Task.new(TestTask.new(a), :some_method)
197
+ end
198
+ task_set = DRbQS::Task::TaskSet.new(tasks)
199
+ task_set.group.should == DRbQS::Task::DEFAULT_GROUP
200
+ end
201
+
202
+ it "should set group." do
203
+ group = :local
204
+ tasks = 10.times.map do |a|
205
+ DRbQS::Task.new(TestTask.new(a), :some_method, group: group)
206
+ end
207
+ task_set = DRbQS::Task::TaskSet.new(tasks)
208
+ task_set.group.should == group
209
+ end
210
+
211
+ it "should raise error for different groups." do
212
+ group = :local
213
+ tasks = 10.times.map do |a|
214
+ DRbQS::Task.new(TestTask.new(a), :some_method, group: "group#{a}".intern)
215
+ end
216
+ lambda do
217
+ DRbQS::Task::TaskSet.new(tasks)
218
+ end.should raise_error ArgumentError
182
219
  end
183
220
  end
@@ -7,18 +7,22 @@ describe DRbQS::Temporary do
7
7
  DRbQS::Temporary.file
8
8
  end
9
9
 
10
+ subject do
11
+ DRbQS::Temporary
12
+ end
13
+
10
14
  it "should create base directory." do
11
- File.exist?(DRbQS::Temporary.root).should be_true
15
+ File.exist?(subject.root).should be_true
12
16
  end
13
17
 
14
18
  it "should create empty directory." do
15
- dir = DRbQS::Temporary.directory
19
+ dir = subject.directory
16
20
  File.directory?(dir).should be_true
17
21
  Dir.entries(dir).should have(2).items
18
22
  end
19
23
 
20
24
  it "should return new path of file." do
21
- path = DRbQS::Temporary.file
25
+ path = subject.file
22
26
  File.exist?(path).should_not be_true
23
27
  open(path, 'w') do |f|
24
28
  f.puts 'hello'
@@ -26,14 +30,33 @@ describe DRbQS::Temporary do
26
30
  end
27
31
 
28
32
  it "should make directory empty." do
29
- Dir.entries(DRbQS::Temporary.root).size.should > 2
30
- DRbQS::Temporary.delete
31
- Dir.entries(DRbQS::Temporary.root).should have(2).items
33
+ Dir.entries(subject.root).size.should > 2
34
+ subject.delete
35
+ Dir.entries(subject.root).should have(2).items
36
+ end
37
+
38
+ it "should set subdirectory." do
39
+ subject.set_sub_directory('abc')
40
+ subject.subdirectory.should be_nil
41
+ end
42
+
43
+ it "should get directory in subdirectory." do
44
+ subject.set_sub_directory('ABCD')
45
+ dir = subject.directory
46
+ File.exist?(subject.subdirectory).should be_true
47
+ dir.should match(/\/ABCD/)
48
+ end
49
+
50
+ it "should get file name in subdirectory." do
51
+ subject.set_sub_directory('EFGH')
52
+ file = subject.file
53
+ File.exist?(subject.subdirectory).should be_true
54
+ file.should match(/\/EFGH\//)
32
55
  end
33
56
 
34
- it "should all directories." do
35
- root = DRbQS::Temporary.root
36
- DRbQS::Temporary.delete_all
57
+ it "should delete all directories." do
58
+ root = subject.root
59
+ subject.delete_all
37
60
  File.exist?(root).should_not be_true
38
61
  end
39
62
  end