drbqs 0.0.17 → 0.0.18
Sign up to get free protection for your applications and to get access to all the features.
- data/docs/FormatExecute.md +44 -2
- data/example/group/execute.rb +19 -0
- data/example/group/server.rb +27 -0
- data/example/group/sum.rb +9 -0
- data/example/mandelbrot/README.md +8 -0
- data/example/mandelbrot/execute.rb +4 -0
- data/lib/drbqs/command_line/command_node.rb +1 -0
- data/lib/drbqs/execute/execute_node.rb +4 -21
- data/lib/drbqs/node/connection.rb +1 -2
- data/lib/drbqs/node/node.rb +163 -102
- data/lib/drbqs/node/state.rb +100 -35
- data/lib/drbqs/node/task_client.rb +46 -33
- data/lib/drbqs/server/message.rb +13 -7
- data/lib/drbqs/server/server.rb +57 -29
- data/lib/drbqs/server/server_hook.rb +19 -5
- data/lib/drbqs/server/test/node.rb +31 -6
- data/lib/drbqs/setting/node.rb +11 -2
- data/lib/drbqs/setting/server.rb +1 -1
- data/lib/drbqs/task/task.rb +26 -6
- data/lib/drbqs/task/task_generator.rb +2 -1
- data/lib/drbqs/utility/temporary.rb +27 -6
- data/lib/drbqs/utility/transfer/transfer_client.rb +10 -12
- data/lib/drbqs/version.rb +1 -1
- data/lib/drbqs/worker.rb +2 -0
- data/lib/drbqs/worker/forked_process.rb +100 -0
- data/lib/drbqs/worker/serialize.rb +66 -0
- data/lib/drbqs/worker/worker.rb +133 -0
- data/lib/drbqs/worker/worker_process_set.rb +219 -0
- data/spec/integration_test/01_basic_usage_spec.rb +3 -2
- data/spec/integration_test/06_node_exit_after_task_spec.rb +3 -2
- data/spec/integration_test/07_command_server_with_node_spec.rb +1 -0
- data/spec/integration_test/08_shutdown_unused_nodes_spec.rb +3 -2
- data/spec/integration_test/10_test_server_spec.rb +2 -2
- data/spec/integration_test/11_special_tasks_spec.rb +61 -0
- data/spec/integration_test/12_multiple_workers_spec.rb +43 -0
- data/spec/integration_test/definition/task_obj_definition.rb +33 -6
- data/spec/node/connection_spec.rb +6 -6
- data/spec/node/node_spec.rb +10 -2
- data/spec/node/state_spec.rb +146 -62
- data/spec/node/task_client_spec.rb +58 -53
- data/spec/server/message_spec.rb +10 -6
- data/spec/server/queue_spec.rb +7 -4
- data/spec/server/server_hook_spec.rb +28 -1
- data/spec/task/task_spec.rb +43 -6
- data/spec/utility/temporary_spec.rb +32 -9
- data/spec/worker/forked_process_spec.rb +66 -0
- data/spec/worker/serialize_spec.rb +73 -0
- data/spec/worker/worker_process_set_spec.rb +104 -0
- data/spec/worker/worker_spec.rb +127 -0
- metadata +34 -19
@@ -3,19 +3,19 @@ require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
|
|
3
3
|
require 'drbqs/node/task_client'
|
4
4
|
|
5
5
|
describe DRbQS::Node::TaskClient do
|
6
|
-
def init_task_client(
|
7
|
-
@node_number =
|
6
|
+
def init_task_client(opts = {})
|
7
|
+
@node_number = opts[:number] || 10
|
8
8
|
@ts_queue = Rinda::TupleSpace.new
|
9
9
|
@ts_result = Rinda::TupleSpace.new
|
10
|
-
@task_client = DRbQS::Node::TaskClient.new(@node_number, @ts_queue, @ts_result)
|
10
|
+
@task_client = DRbQS::Node::TaskClient.new(@node_number, @ts_queue, @ts_result, opts[:group])
|
11
11
|
end
|
12
12
|
|
13
|
-
def add_task_to_tuplespace(task_id, task_ary)
|
14
|
-
@ts_queue.write([task_id] + task_ary)
|
13
|
+
def add_task_to_tuplespace(task_id, task_ary, group = DRbQS::Task::DEFAULT_GROUP)
|
14
|
+
@ts_queue.write([group, task_id] + task_ary)
|
15
15
|
end
|
16
16
|
|
17
17
|
def check_empty_queue_tuplespace
|
18
|
-
@ts_queue.read_all([nil, nil, nil, nil]).size.should == 0
|
18
|
+
@ts_queue.read_all([nil, nil, nil, nil, nil]).size.should == 0
|
19
19
|
end
|
20
20
|
|
21
21
|
subject do
|
@@ -35,12 +35,12 @@ describe DRbQS::Node::TaskClient do
|
|
35
35
|
subject.result_empty?.should be_true
|
36
36
|
end
|
37
37
|
|
38
|
-
it "should
|
39
|
-
subject.
|
38
|
+
it "should return no task." do
|
39
|
+
subject.get_task.should be_nil
|
40
40
|
end
|
41
41
|
|
42
|
-
it "should
|
43
|
-
subject.
|
42
|
+
it "should not have group." do
|
43
|
+
subject.group.should == []
|
44
44
|
end
|
45
45
|
end
|
46
46
|
|
@@ -61,28 +61,54 @@ describe DRbQS::Node::TaskClient do
|
|
61
61
|
end
|
62
62
|
end
|
63
63
|
|
64
|
+
context "when getting task by group" do
|
65
|
+
before(:all) do
|
66
|
+
init_task_client(:group => [:grp1])
|
67
|
+
@task_id = 3
|
68
|
+
@task_ary = ["hello world", :size, []]
|
69
|
+
add_task_to_tuplespace(@task_id, @task_ary, :grp1)
|
70
|
+
end
|
71
|
+
|
72
|
+
it "should not return task." do
|
73
|
+
client = DRbQS::Node::TaskClient.new(100, @ts_queue, @ts_result, nil)
|
74
|
+
client.get_task.should be_nil
|
75
|
+
end
|
76
|
+
|
77
|
+
it "should return task." do
|
78
|
+
subject.get_task.should == @task_ary.dup.unshift(@task_id)
|
79
|
+
end
|
80
|
+
|
81
|
+
it "should take out task from tuplespace." do
|
82
|
+
check_empty_queue_tuplespace
|
83
|
+
end
|
84
|
+
|
85
|
+
it "should have group." do
|
86
|
+
subject.group.should == [:grp1]
|
87
|
+
end
|
88
|
+
end
|
89
|
+
|
64
90
|
context "when getting new task" do
|
65
91
|
before(:all) do
|
66
|
-
init_task_client(7)
|
92
|
+
init_task_client(:number => 7)
|
67
93
|
end
|
68
94
|
|
69
95
|
it "should return nil" do
|
70
|
-
subject.add_new_task.should be_nil
|
96
|
+
subject.add_new_task(1).should be_nil
|
71
97
|
end
|
72
98
|
|
73
99
|
it "should return nil" do
|
74
100
|
add_task_to_tuplespace(100, [[1, 3, 5, 7], :size, []])
|
75
|
-
subject.add_new_task.should be_true
|
101
|
+
subject.add_new_task(1).should be_true
|
76
102
|
end
|
77
103
|
end
|
78
104
|
|
79
105
|
context "when adding new task" do
|
80
106
|
before(:all) do
|
81
|
-
init_task_client(7)
|
107
|
+
init_task_client(:number => 7)
|
82
108
|
@task_id = 3
|
83
109
|
@task_ary = [[1, 3, 5, 7], :size, []]
|
84
110
|
add_task_to_tuplespace(@task_id, @task_ary)
|
85
|
-
subject.add_new_task
|
111
|
+
@get_task_id = subject.add_new_task(1)
|
86
112
|
end
|
87
113
|
|
88
114
|
it "should have non empty task queue." do
|
@@ -94,7 +120,7 @@ describe DRbQS::Node::TaskClient do
|
|
94
120
|
end
|
95
121
|
|
96
122
|
it "should have calculating task" do
|
97
|
-
|
123
|
+
@get_task_id.should be_include(@task_id)
|
98
124
|
end
|
99
125
|
|
100
126
|
it "should take out task from tuplespace." do
|
@@ -108,16 +134,16 @@ describe DRbQS::Node::TaskClient do
|
|
108
134
|
|
109
135
|
context "when dequeueing a task" do
|
110
136
|
before(:all) do
|
111
|
-
init_task_client(14)
|
137
|
+
init_task_client(:number => 14)
|
112
138
|
@task_id = 8
|
113
139
|
@task_ary = [[1, 3, 5, 7], :size, []]
|
114
140
|
add_task_to_tuplespace(@task_id, @task_ary)
|
115
|
-
subject.add_new_task
|
141
|
+
@get_task_id = subject.add_new_task(1)
|
116
142
|
@dequeued_task = subject.dequeue_task
|
117
143
|
end
|
118
144
|
|
119
145
|
it "should get a task" do
|
120
|
-
@dequeued_task.should == @task_ary
|
146
|
+
@dequeued_task.should == [@task_id] + @task_ary
|
121
147
|
end
|
122
148
|
|
123
149
|
it "should have empty task queue." do
|
@@ -129,19 +155,19 @@ describe DRbQS::Node::TaskClient do
|
|
129
155
|
end
|
130
156
|
|
131
157
|
it "should have calculating task." do
|
132
|
-
|
158
|
+
@get_task_id.should be_include(@task_id)
|
133
159
|
end
|
134
160
|
end
|
135
161
|
|
136
162
|
context "when queueing result" do
|
137
163
|
before(:all) do
|
138
|
-
init_task_client(2)
|
164
|
+
init_task_client(:number => 2)
|
139
165
|
@task_id = 27
|
140
166
|
@task_ary = ["abcdef", :size, []]
|
141
167
|
add_task_to_tuplespace(@task_id, @task_ary)
|
142
|
-
subject.add_new_task
|
168
|
+
@get_task_id = subject.add_new_task(1)
|
143
169
|
@dequeued_task = subject.dequeue_task
|
144
|
-
subject.queue_result(:result_object)
|
170
|
+
subject.queue_result(1, :result_object)
|
145
171
|
end
|
146
172
|
|
147
173
|
it "should have empty task queue." do
|
@@ -153,24 +179,24 @@ describe DRbQS::Node::TaskClient do
|
|
153
179
|
end
|
154
180
|
|
155
181
|
it "should have calculating task." do
|
156
|
-
|
182
|
+
@get_task_id.should be_include(@task_id)
|
157
183
|
end
|
158
184
|
end
|
159
185
|
|
160
186
|
context "when sending result" do
|
161
187
|
before(:all) do
|
162
|
-
init_task_client(2)
|
163
|
-
@task_id =
|
188
|
+
init_task_client(:number => 2)
|
189
|
+
@task_id = 28
|
164
190
|
@task_ary = ["abcdef", :size, []]
|
165
191
|
add_task_to_tuplespace(@task_id, @task_ary)
|
166
|
-
subject.add_new_task
|
192
|
+
subject.add_new_task(1)
|
167
193
|
@dequeued_task = subject.dequeue_task
|
168
|
-
subject.queue_result(:result_object)
|
194
|
+
subject.queue_result(@task_id, :result_object)
|
169
195
|
@send_returned_value = subject.send_result
|
170
196
|
end
|
171
197
|
|
172
198
|
it "should get nil returned value." do
|
173
|
-
@send_returned_value.should
|
199
|
+
@send_returned_value.should == [@task_id]
|
174
200
|
end
|
175
201
|
|
176
202
|
it "should have empty task queue." do
|
@@ -181,35 +207,14 @@ describe DRbQS::Node::TaskClient do
|
|
181
207
|
subject.result_empty?.should_not be_nil
|
182
208
|
end
|
183
209
|
|
184
|
-
it "should not have calculating task." do
|
185
|
-
subject.calculating_task.should be_nil
|
186
|
-
end
|
187
|
-
|
188
210
|
it "should get result from tuplespace." do
|
189
211
|
@ts_result.take([:result, nil, nil, nil], 0).should be_true
|
190
212
|
end
|
191
213
|
end
|
192
214
|
|
193
|
-
context "when setting exit_after_task" do
|
194
|
-
before(:all) do
|
195
|
-
init_task_client(2)
|
196
|
-
@task_id = 27
|
197
|
-
@task_ary = ["abcdef", :size, []]
|
198
|
-
add_task_to_tuplespace(@task_id, @task_ary)
|
199
|
-
subject.add_new_task
|
200
|
-
subject.set_exit_after_task
|
201
|
-
@dequeued_task = subject.dequeue_task
|
202
|
-
subject.queue_result(:result_object)
|
203
|
-
end
|
204
|
-
|
205
|
-
it "should return true" do
|
206
|
-
subject.send_result.should be_true
|
207
|
-
end
|
208
|
-
end
|
209
|
-
|
210
215
|
context "when dumping result queue" do
|
211
216
|
before(:all) do
|
212
|
-
init_task_client(2)
|
217
|
+
init_task_client(:number => 2)
|
213
218
|
end
|
214
219
|
|
215
220
|
it "should return nil for empty result queue." do
|
@@ -218,8 +223,8 @@ describe DRbQS::Node::TaskClient do
|
|
218
223
|
|
219
224
|
it "should return string for non empty result queue." do
|
220
225
|
results = [:result_object1, :result_object2]
|
221
|
-
results.
|
222
|
-
subject.queue_result(sym)
|
226
|
+
results.each_with_index do |sym, i|
|
227
|
+
subject.queue_result(i, sym)
|
223
228
|
end
|
224
229
|
subject.dump_result_queue.should == Marshal.dump(results)
|
225
230
|
end
|
data/spec/server/message_spec.rb
CHANGED
@@ -122,19 +122,23 @@ describe DRbQS::Server::Message do
|
|
122
122
|
|
123
123
|
context "when setting special tasks" do
|
124
124
|
it "should set initialization task" do
|
125
|
+
initialization_pattern = [:initialize, Array]
|
125
126
|
lambda do
|
126
|
-
@message.take(
|
127
|
+
@message.take(initialization_pattern, 0)
|
127
128
|
end.should raise_error Rinda::RequestExpiredError
|
128
|
-
@message_server.
|
129
|
-
@
|
129
|
+
@message_server.set_initialization_tasks(DRbQS::Task.new(Object.new, :object_id))
|
130
|
+
@message_server.set_initialization_tasks(DRbQS::Task.new(Object.new, :object_id))
|
131
|
+
@message.take(initialization_pattern, 0)[1].should have(2).items
|
130
132
|
end
|
131
133
|
|
132
134
|
it "should set finalization task" do
|
135
|
+
finalization_pattern = [:finalize, Array]
|
133
136
|
lambda do
|
134
|
-
@message.take(
|
137
|
+
@message.take(finalization_pattern, 0)
|
135
138
|
end.should raise_error Rinda::RequestExpiredError
|
136
|
-
|
137
|
-
@
|
139
|
+
tasks = [DRbQS::Task.new(Object.new, :object_id), DRbQS::Task.new(Object.new, :object_id)]
|
140
|
+
@message_server.set_finalization_tasks(tasks)
|
141
|
+
@message.take(finalization_pattern, 0)[1].should have(2).items
|
138
142
|
end
|
139
143
|
end
|
140
144
|
|
data/spec/server/queue_spec.rb
CHANGED
@@ -4,6 +4,8 @@ require 'drbqs/server/queue'
|
|
4
4
|
require 'drbqs/task/task'
|
5
5
|
|
6
6
|
describe DRbQS::Server::Queue do
|
7
|
+
QUEUE_OBJECT_PATTERN = [nil, nil, nil, nil, nil]
|
8
|
+
|
7
9
|
def object_init
|
8
10
|
@ts = {
|
9
11
|
:queue => Rinda::TupleSpace.new,
|
@@ -86,7 +88,7 @@ describe DRbQS::Server::Queue do
|
|
86
88
|
end
|
87
89
|
|
88
90
|
it "should take objects from queue." do
|
89
|
-
@ts[:queue].take(
|
91
|
+
@ts[:queue].take(QUEUE_OBJECT_PATTERN).should be_true
|
90
92
|
end
|
91
93
|
|
92
94
|
it "should return an array of calculating nodes." do
|
@@ -208,9 +210,10 @@ describe DRbQS::Server::Queue do
|
|
208
210
|
subject.should_not be_finished
|
209
211
|
end
|
210
212
|
|
211
|
-
it "should return task ID." do
|
212
|
-
(ary = @ts[:queue].take(
|
213
|
-
ary[0].should ==
|
213
|
+
it "should return task group and task ID." do
|
214
|
+
(ary = @ts[:queue].take(QUEUE_OBJECT_PATTERN)).should be_true
|
215
|
+
ary[0].should == DRbQS::Task::DEFAULT_GROUP
|
216
|
+
ary[1].should == @task_id
|
214
217
|
end
|
215
218
|
|
216
219
|
it "should return an array of calculating nodes." do
|
@@ -21,7 +21,7 @@ describe DRbQS::Server::Hook do
|
|
21
21
|
it "should add a hook with name." do
|
22
22
|
n = subject.number_of_hook(:finish)
|
23
23
|
name = 'hello'
|
24
|
-
name_new = subject.add(:finish, name) do |server|
|
24
|
+
name_new = subject.add(:finish, name: name) do |server|
|
25
25
|
3 + 4
|
26
26
|
end
|
27
27
|
name_new.should == name
|
@@ -71,6 +71,33 @@ describe DRbQS::Server::Hook do
|
|
71
71
|
exec_flag[:second].should be_true
|
72
72
|
end
|
73
73
|
|
74
|
+
it "should execute a hook repeatedly." do
|
75
|
+
exec_flag = {}
|
76
|
+
subject.add(:finish) do |server|
|
77
|
+
exec_flag[:exec] = true
|
78
|
+
end
|
79
|
+
10.times do |i|
|
80
|
+
exec_flag[:exec] = nil
|
81
|
+
subject.exec(:finish)
|
82
|
+
exec_flag[:exec].should be_true
|
83
|
+
end
|
84
|
+
end
|
85
|
+
|
86
|
+
it "should execute a hook three times." do
|
87
|
+
exec_flag = {}
|
88
|
+
subject.add(:finish, repeat: 3) do |server|
|
89
|
+
exec_flag[:exec] = true
|
90
|
+
end
|
91
|
+
3.times do |i|
|
92
|
+
exec_flag[:exec] = nil
|
93
|
+
subject.exec(:finish)
|
94
|
+
exec_flag[:exec].should be_true
|
95
|
+
end
|
96
|
+
exec_flag[:exec] = nil
|
97
|
+
subject.exec(:finish)
|
98
|
+
exec_flag[:exec].should be_nil
|
99
|
+
end
|
100
|
+
|
74
101
|
it "should execute finish_exit that is special proc." do
|
75
102
|
execute = nil
|
76
103
|
subject.set_finish_exit do
|
data/spec/task/task_spec.rb
CHANGED
@@ -92,6 +92,17 @@ describe DRbQS::Task do
|
|
92
92
|
Marshal.dump(task)
|
93
93
|
end.should raise_error
|
94
94
|
end
|
95
|
+
|
96
|
+
it "should get default group." do
|
97
|
+
task = DRbQS::Task.new([1, 2, 3], :size)
|
98
|
+
task.group.should == DRbQS::Task::DEFAULT_GROUP
|
99
|
+
end
|
100
|
+
|
101
|
+
it "should set group." do
|
102
|
+
group = :local
|
103
|
+
task = DRbQS::Task.new([1, 2, 3], :size, group: group)
|
104
|
+
task.group.should == group
|
105
|
+
end
|
95
106
|
end
|
96
107
|
|
97
108
|
describe DRbQS::Task::TaskSet::ContainerTask do
|
@@ -172,12 +183,38 @@ describe DRbQS::Task::TaskSet do
|
|
172
183
|
|
173
184
|
it "should execute hooks of symbols." do
|
174
185
|
server = mock('server')
|
175
|
-
|
176
|
-
tasks =
|
177
|
-
|
178
|
-
|
179
|
-
tasks << DRbQS::Task.new(obj[2], :some_method, hook: :test_hook)
|
186
|
+
args = [100, 200, 300]
|
187
|
+
tasks = args.map do |a|
|
188
|
+
DRbQS::Task.new(TestTask.new(a), :some_method, hook: :test_hook)
|
189
|
+
end
|
180
190
|
task_set = DRbQS::Task::TaskSet.new(tasks)
|
181
|
-
task_set.exec_hook(server,
|
191
|
+
task_set.exec_hook(server, args)
|
192
|
+
end
|
193
|
+
|
194
|
+
it "should get default group." do
|
195
|
+
tasks = 10.times.map do |a|
|
196
|
+
DRbQS::Task.new(TestTask.new(a), :some_method)
|
197
|
+
end
|
198
|
+
task_set = DRbQS::Task::TaskSet.new(tasks)
|
199
|
+
task_set.group.should == DRbQS::Task::DEFAULT_GROUP
|
200
|
+
end
|
201
|
+
|
202
|
+
it "should set group." do
|
203
|
+
group = :local
|
204
|
+
tasks = 10.times.map do |a|
|
205
|
+
DRbQS::Task.new(TestTask.new(a), :some_method, group: group)
|
206
|
+
end
|
207
|
+
task_set = DRbQS::Task::TaskSet.new(tasks)
|
208
|
+
task_set.group.should == group
|
209
|
+
end
|
210
|
+
|
211
|
+
it "should raise error for different groups." do
|
212
|
+
group = :local
|
213
|
+
tasks = 10.times.map do |a|
|
214
|
+
DRbQS::Task.new(TestTask.new(a), :some_method, group: "group#{a}".intern)
|
215
|
+
end
|
216
|
+
lambda do
|
217
|
+
DRbQS::Task::TaskSet.new(tasks)
|
218
|
+
end.should raise_error ArgumentError
|
182
219
|
end
|
183
220
|
end
|
@@ -7,18 +7,22 @@ describe DRbQS::Temporary do
|
|
7
7
|
DRbQS::Temporary.file
|
8
8
|
end
|
9
9
|
|
10
|
+
subject do
|
11
|
+
DRbQS::Temporary
|
12
|
+
end
|
13
|
+
|
10
14
|
it "should create base directory." do
|
11
|
-
File.exist?(
|
15
|
+
File.exist?(subject.root).should be_true
|
12
16
|
end
|
13
17
|
|
14
18
|
it "should create empty directory." do
|
15
|
-
dir =
|
19
|
+
dir = subject.directory
|
16
20
|
File.directory?(dir).should be_true
|
17
21
|
Dir.entries(dir).should have(2).items
|
18
22
|
end
|
19
23
|
|
20
24
|
it "should return new path of file." do
|
21
|
-
path =
|
25
|
+
path = subject.file
|
22
26
|
File.exist?(path).should_not be_true
|
23
27
|
open(path, 'w') do |f|
|
24
28
|
f.puts 'hello'
|
@@ -26,14 +30,33 @@ describe DRbQS::Temporary do
|
|
26
30
|
end
|
27
31
|
|
28
32
|
it "should make directory empty." do
|
29
|
-
Dir.entries(
|
30
|
-
|
31
|
-
Dir.entries(
|
33
|
+
Dir.entries(subject.root).size.should > 2
|
34
|
+
subject.delete
|
35
|
+
Dir.entries(subject.root).should have(2).items
|
36
|
+
end
|
37
|
+
|
38
|
+
it "should set subdirectory." do
|
39
|
+
subject.set_sub_directory('abc')
|
40
|
+
subject.subdirectory.should be_nil
|
41
|
+
end
|
42
|
+
|
43
|
+
it "should get directory in subdirectory." do
|
44
|
+
subject.set_sub_directory('ABCD')
|
45
|
+
dir = subject.directory
|
46
|
+
File.exist?(subject.subdirectory).should be_true
|
47
|
+
dir.should match(/\/ABCD/)
|
48
|
+
end
|
49
|
+
|
50
|
+
it "should get file name in subdirectory." do
|
51
|
+
subject.set_sub_directory('EFGH')
|
52
|
+
file = subject.file
|
53
|
+
File.exist?(subject.subdirectory).should be_true
|
54
|
+
file.should match(/\/EFGH\//)
|
32
55
|
end
|
33
56
|
|
34
|
-
it "should all directories." do
|
35
|
-
root =
|
36
|
-
|
57
|
+
it "should delete all directories." do
|
58
|
+
root = subject.root
|
59
|
+
subject.delete_all
|
37
60
|
File.exist?(root).should_not be_true
|
38
61
|
end
|
39
62
|
end
|