drbqs 0.0.17 → 0.0.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/docs/FormatExecute.md +44 -2
- data/example/group/execute.rb +19 -0
- data/example/group/server.rb +27 -0
- data/example/group/sum.rb +9 -0
- data/example/mandelbrot/README.md +8 -0
- data/example/mandelbrot/execute.rb +4 -0
- data/lib/drbqs/command_line/command_node.rb +1 -0
- data/lib/drbqs/execute/execute_node.rb +4 -21
- data/lib/drbqs/node/connection.rb +1 -2
- data/lib/drbqs/node/node.rb +163 -102
- data/lib/drbqs/node/state.rb +100 -35
- data/lib/drbqs/node/task_client.rb +46 -33
- data/lib/drbqs/server/message.rb +13 -7
- data/lib/drbqs/server/server.rb +57 -29
- data/lib/drbqs/server/server_hook.rb +19 -5
- data/lib/drbqs/server/test/node.rb +31 -6
- data/lib/drbqs/setting/node.rb +11 -2
- data/lib/drbqs/setting/server.rb +1 -1
- data/lib/drbqs/task/task.rb +26 -6
- data/lib/drbqs/task/task_generator.rb +2 -1
- data/lib/drbqs/utility/temporary.rb +27 -6
- data/lib/drbqs/utility/transfer/transfer_client.rb +10 -12
- data/lib/drbqs/version.rb +1 -1
- data/lib/drbqs/worker.rb +2 -0
- data/lib/drbqs/worker/forked_process.rb +100 -0
- data/lib/drbqs/worker/serialize.rb +66 -0
- data/lib/drbqs/worker/worker.rb +133 -0
- data/lib/drbqs/worker/worker_process_set.rb +219 -0
- data/spec/integration_test/01_basic_usage_spec.rb +3 -2
- data/spec/integration_test/06_node_exit_after_task_spec.rb +3 -2
- data/spec/integration_test/07_command_server_with_node_spec.rb +1 -0
- data/spec/integration_test/08_shutdown_unused_nodes_spec.rb +3 -2
- data/spec/integration_test/10_test_server_spec.rb +2 -2
- data/spec/integration_test/11_special_tasks_spec.rb +61 -0
- data/spec/integration_test/12_multiple_workers_spec.rb +43 -0
- data/spec/integration_test/definition/task_obj_definition.rb +33 -6
- data/spec/node/connection_spec.rb +6 -6
- data/spec/node/node_spec.rb +10 -2
- data/spec/node/state_spec.rb +146 -62
- data/spec/node/task_client_spec.rb +58 -53
- data/spec/server/message_spec.rb +10 -6
- data/spec/server/queue_spec.rb +7 -4
- data/spec/server/server_hook_spec.rb +28 -1
- data/spec/task/task_spec.rb +43 -6
- data/spec/utility/temporary_spec.rb +32 -9
- data/spec/worker/forked_process_spec.rb +66 -0
- data/spec/worker/serialize_spec.rb +73 -0
- data/spec/worker/worker_process_set_spec.rb +104 -0
- data/spec/worker/worker_spec.rb +127 -0
- metadata +34 -19
@@ -3,19 +3,19 @@ require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
|
|
3
3
|
require 'drbqs/node/task_client'
|
4
4
|
|
5
5
|
describe DRbQS::Node::TaskClient do
|
6
|
-
def init_task_client(
|
7
|
-
@node_number =
|
6
|
+
def init_task_client(opts = {})
|
7
|
+
@node_number = opts[:number] || 10
|
8
8
|
@ts_queue = Rinda::TupleSpace.new
|
9
9
|
@ts_result = Rinda::TupleSpace.new
|
10
|
-
@task_client = DRbQS::Node::TaskClient.new(@node_number, @ts_queue, @ts_result)
|
10
|
+
@task_client = DRbQS::Node::TaskClient.new(@node_number, @ts_queue, @ts_result, opts[:group])
|
11
11
|
end
|
12
12
|
|
13
|
-
def add_task_to_tuplespace(task_id, task_ary)
|
14
|
-
@ts_queue.write([task_id] + task_ary)
|
13
|
+
def add_task_to_tuplespace(task_id, task_ary, group = DRbQS::Task::DEFAULT_GROUP)
|
14
|
+
@ts_queue.write([group, task_id] + task_ary)
|
15
15
|
end
|
16
16
|
|
17
17
|
def check_empty_queue_tuplespace
|
18
|
-
@ts_queue.read_all([nil, nil, nil, nil]).size.should == 0
|
18
|
+
@ts_queue.read_all([nil, nil, nil, nil, nil]).size.should == 0
|
19
19
|
end
|
20
20
|
|
21
21
|
subject do
|
@@ -35,12 +35,12 @@ describe DRbQS::Node::TaskClient do
|
|
35
35
|
subject.result_empty?.should be_true
|
36
36
|
end
|
37
37
|
|
38
|
-
it "should
|
39
|
-
subject.
|
38
|
+
it "should return no task." do
|
39
|
+
subject.get_task.should be_nil
|
40
40
|
end
|
41
41
|
|
42
|
-
it "should
|
43
|
-
subject.
|
42
|
+
it "should not have group." do
|
43
|
+
subject.group.should == []
|
44
44
|
end
|
45
45
|
end
|
46
46
|
|
@@ -61,28 +61,54 @@ describe DRbQS::Node::TaskClient do
|
|
61
61
|
end
|
62
62
|
end
|
63
63
|
|
64
|
+
context "when getting task by group" do
|
65
|
+
before(:all) do
|
66
|
+
init_task_client(:group => [:grp1])
|
67
|
+
@task_id = 3
|
68
|
+
@task_ary = ["hello world", :size, []]
|
69
|
+
add_task_to_tuplespace(@task_id, @task_ary, :grp1)
|
70
|
+
end
|
71
|
+
|
72
|
+
it "should not return task." do
|
73
|
+
client = DRbQS::Node::TaskClient.new(100, @ts_queue, @ts_result, nil)
|
74
|
+
client.get_task.should be_nil
|
75
|
+
end
|
76
|
+
|
77
|
+
it "should return task." do
|
78
|
+
subject.get_task.should == @task_ary.dup.unshift(@task_id)
|
79
|
+
end
|
80
|
+
|
81
|
+
it "should take out task from tuplespace." do
|
82
|
+
check_empty_queue_tuplespace
|
83
|
+
end
|
84
|
+
|
85
|
+
it "should have group." do
|
86
|
+
subject.group.should == [:grp1]
|
87
|
+
end
|
88
|
+
end
|
89
|
+
|
64
90
|
context "when getting new task" do
|
65
91
|
before(:all) do
|
66
|
-
init_task_client(7)
|
92
|
+
init_task_client(:number => 7)
|
67
93
|
end
|
68
94
|
|
69
95
|
it "should return nil" do
|
70
|
-
subject.add_new_task.should be_nil
|
96
|
+
subject.add_new_task(1).should be_nil
|
71
97
|
end
|
72
98
|
|
73
99
|
it "should return nil" do
|
74
100
|
add_task_to_tuplespace(100, [[1, 3, 5, 7], :size, []])
|
75
|
-
subject.add_new_task.should be_true
|
101
|
+
subject.add_new_task(1).should be_true
|
76
102
|
end
|
77
103
|
end
|
78
104
|
|
79
105
|
context "when adding new task" do
|
80
106
|
before(:all) do
|
81
|
-
init_task_client(7)
|
107
|
+
init_task_client(:number => 7)
|
82
108
|
@task_id = 3
|
83
109
|
@task_ary = [[1, 3, 5, 7], :size, []]
|
84
110
|
add_task_to_tuplespace(@task_id, @task_ary)
|
85
|
-
subject.add_new_task
|
111
|
+
@get_task_id = subject.add_new_task(1)
|
86
112
|
end
|
87
113
|
|
88
114
|
it "should have non empty task queue." do
|
@@ -94,7 +120,7 @@ describe DRbQS::Node::TaskClient do
|
|
94
120
|
end
|
95
121
|
|
96
122
|
it "should have calculating task" do
|
97
|
-
|
123
|
+
@get_task_id.should be_include(@task_id)
|
98
124
|
end
|
99
125
|
|
100
126
|
it "should take out task from tuplespace." do
|
@@ -108,16 +134,16 @@ describe DRbQS::Node::TaskClient do
|
|
108
134
|
|
109
135
|
context "when dequeueing a task" do
|
110
136
|
before(:all) do
|
111
|
-
init_task_client(14)
|
137
|
+
init_task_client(:number => 14)
|
112
138
|
@task_id = 8
|
113
139
|
@task_ary = [[1, 3, 5, 7], :size, []]
|
114
140
|
add_task_to_tuplespace(@task_id, @task_ary)
|
115
|
-
subject.add_new_task
|
141
|
+
@get_task_id = subject.add_new_task(1)
|
116
142
|
@dequeued_task = subject.dequeue_task
|
117
143
|
end
|
118
144
|
|
119
145
|
it "should get a task" do
|
120
|
-
@dequeued_task.should == @task_ary
|
146
|
+
@dequeued_task.should == [@task_id] + @task_ary
|
121
147
|
end
|
122
148
|
|
123
149
|
it "should have empty task queue." do
|
@@ -129,19 +155,19 @@ describe DRbQS::Node::TaskClient do
|
|
129
155
|
end
|
130
156
|
|
131
157
|
it "should have calculating task." do
|
132
|
-
|
158
|
+
@get_task_id.should be_include(@task_id)
|
133
159
|
end
|
134
160
|
end
|
135
161
|
|
136
162
|
context "when queueing result" do
|
137
163
|
before(:all) do
|
138
|
-
init_task_client(2)
|
164
|
+
init_task_client(:number => 2)
|
139
165
|
@task_id = 27
|
140
166
|
@task_ary = ["abcdef", :size, []]
|
141
167
|
add_task_to_tuplespace(@task_id, @task_ary)
|
142
|
-
subject.add_new_task
|
168
|
+
@get_task_id = subject.add_new_task(1)
|
143
169
|
@dequeued_task = subject.dequeue_task
|
144
|
-
subject.queue_result(:result_object)
|
170
|
+
subject.queue_result(1, :result_object)
|
145
171
|
end
|
146
172
|
|
147
173
|
it "should have empty task queue." do
|
@@ -153,24 +179,24 @@ describe DRbQS::Node::TaskClient do
|
|
153
179
|
end
|
154
180
|
|
155
181
|
it "should have calculating task." do
|
156
|
-
|
182
|
+
@get_task_id.should be_include(@task_id)
|
157
183
|
end
|
158
184
|
end
|
159
185
|
|
160
186
|
context "when sending result" do
|
161
187
|
before(:all) do
|
162
|
-
init_task_client(2)
|
163
|
-
@task_id =
|
188
|
+
init_task_client(:number => 2)
|
189
|
+
@task_id = 28
|
164
190
|
@task_ary = ["abcdef", :size, []]
|
165
191
|
add_task_to_tuplespace(@task_id, @task_ary)
|
166
|
-
subject.add_new_task
|
192
|
+
subject.add_new_task(1)
|
167
193
|
@dequeued_task = subject.dequeue_task
|
168
|
-
subject.queue_result(:result_object)
|
194
|
+
subject.queue_result(@task_id, :result_object)
|
169
195
|
@send_returned_value = subject.send_result
|
170
196
|
end
|
171
197
|
|
172
198
|
it "should get nil returned value." do
|
173
|
-
@send_returned_value.should
|
199
|
+
@send_returned_value.should == [@task_id]
|
174
200
|
end
|
175
201
|
|
176
202
|
it "should have empty task queue." do
|
@@ -181,35 +207,14 @@ describe DRbQS::Node::TaskClient do
|
|
181
207
|
subject.result_empty?.should_not be_nil
|
182
208
|
end
|
183
209
|
|
184
|
-
it "should not have calculating task." do
|
185
|
-
subject.calculating_task.should be_nil
|
186
|
-
end
|
187
|
-
|
188
210
|
it "should get result from tuplespace." do
|
189
211
|
@ts_result.take([:result, nil, nil, nil], 0).should be_true
|
190
212
|
end
|
191
213
|
end
|
192
214
|
|
193
|
-
context "when setting exit_after_task" do
|
194
|
-
before(:all) do
|
195
|
-
init_task_client(2)
|
196
|
-
@task_id = 27
|
197
|
-
@task_ary = ["abcdef", :size, []]
|
198
|
-
add_task_to_tuplespace(@task_id, @task_ary)
|
199
|
-
subject.add_new_task
|
200
|
-
subject.set_exit_after_task
|
201
|
-
@dequeued_task = subject.dequeue_task
|
202
|
-
subject.queue_result(:result_object)
|
203
|
-
end
|
204
|
-
|
205
|
-
it "should return true" do
|
206
|
-
subject.send_result.should be_true
|
207
|
-
end
|
208
|
-
end
|
209
|
-
|
210
215
|
context "when dumping result queue" do
|
211
216
|
before(:all) do
|
212
|
-
init_task_client(2)
|
217
|
+
init_task_client(:number => 2)
|
213
218
|
end
|
214
219
|
|
215
220
|
it "should return nil for empty result queue." do
|
@@ -218,8 +223,8 @@ describe DRbQS::Node::TaskClient do
|
|
218
223
|
|
219
224
|
it "should return string for non empty result queue." do
|
220
225
|
results = [:result_object1, :result_object2]
|
221
|
-
results.
|
222
|
-
subject.queue_result(sym)
|
226
|
+
results.each_with_index do |sym, i|
|
227
|
+
subject.queue_result(i, sym)
|
223
228
|
end
|
224
229
|
subject.dump_result_queue.should == Marshal.dump(results)
|
225
230
|
end
|
data/spec/server/message_spec.rb
CHANGED
@@ -122,19 +122,23 @@ describe DRbQS::Server::Message do
|
|
122
122
|
|
123
123
|
context "when setting special tasks" do
|
124
124
|
it "should set initialization task" do
|
125
|
+
initialization_pattern = [:initialize, Array]
|
125
126
|
lambda do
|
126
|
-
@message.take(
|
127
|
+
@message.take(initialization_pattern, 0)
|
127
128
|
end.should raise_error Rinda::RequestExpiredError
|
128
|
-
@message_server.
|
129
|
-
@
|
129
|
+
@message_server.set_initialization_tasks(DRbQS::Task.new(Object.new, :object_id))
|
130
|
+
@message_server.set_initialization_tasks(DRbQS::Task.new(Object.new, :object_id))
|
131
|
+
@message.take(initialization_pattern, 0)[1].should have(2).items
|
130
132
|
end
|
131
133
|
|
132
134
|
it "should set finalization task" do
|
135
|
+
finalization_pattern = [:finalize, Array]
|
133
136
|
lambda do
|
134
|
-
@message.take(
|
137
|
+
@message.take(finalization_pattern, 0)
|
135
138
|
end.should raise_error Rinda::RequestExpiredError
|
136
|
-
|
137
|
-
@
|
139
|
+
tasks = [DRbQS::Task.new(Object.new, :object_id), DRbQS::Task.new(Object.new, :object_id)]
|
140
|
+
@message_server.set_finalization_tasks(tasks)
|
141
|
+
@message.take(finalization_pattern, 0)[1].should have(2).items
|
138
142
|
end
|
139
143
|
end
|
140
144
|
|
data/spec/server/queue_spec.rb
CHANGED
@@ -4,6 +4,8 @@ require 'drbqs/server/queue'
|
|
4
4
|
require 'drbqs/task/task'
|
5
5
|
|
6
6
|
describe DRbQS::Server::Queue do
|
7
|
+
QUEUE_OBJECT_PATTERN = [nil, nil, nil, nil, nil]
|
8
|
+
|
7
9
|
def object_init
|
8
10
|
@ts = {
|
9
11
|
:queue => Rinda::TupleSpace.new,
|
@@ -86,7 +88,7 @@ describe DRbQS::Server::Queue do
|
|
86
88
|
end
|
87
89
|
|
88
90
|
it "should take objects from queue." do
|
89
|
-
@ts[:queue].take(
|
91
|
+
@ts[:queue].take(QUEUE_OBJECT_PATTERN).should be_true
|
90
92
|
end
|
91
93
|
|
92
94
|
it "should return an array of calculating nodes." do
|
@@ -208,9 +210,10 @@ describe DRbQS::Server::Queue do
|
|
208
210
|
subject.should_not be_finished
|
209
211
|
end
|
210
212
|
|
211
|
-
it "should return task ID." do
|
212
|
-
(ary = @ts[:queue].take(
|
213
|
-
ary[0].should ==
|
213
|
+
it "should return task group and task ID." do
|
214
|
+
(ary = @ts[:queue].take(QUEUE_OBJECT_PATTERN)).should be_true
|
215
|
+
ary[0].should == DRbQS::Task::DEFAULT_GROUP
|
216
|
+
ary[1].should == @task_id
|
214
217
|
end
|
215
218
|
|
216
219
|
it "should return an array of calculating nodes." do
|
@@ -21,7 +21,7 @@ describe DRbQS::Server::Hook do
|
|
21
21
|
it "should add a hook with name." do
|
22
22
|
n = subject.number_of_hook(:finish)
|
23
23
|
name = 'hello'
|
24
|
-
name_new = subject.add(:finish, name) do |server|
|
24
|
+
name_new = subject.add(:finish, name: name) do |server|
|
25
25
|
3 + 4
|
26
26
|
end
|
27
27
|
name_new.should == name
|
@@ -71,6 +71,33 @@ describe DRbQS::Server::Hook do
|
|
71
71
|
exec_flag[:second].should be_true
|
72
72
|
end
|
73
73
|
|
74
|
+
it "should execute a hook repeatedly." do
|
75
|
+
exec_flag = {}
|
76
|
+
subject.add(:finish) do |server|
|
77
|
+
exec_flag[:exec] = true
|
78
|
+
end
|
79
|
+
10.times do |i|
|
80
|
+
exec_flag[:exec] = nil
|
81
|
+
subject.exec(:finish)
|
82
|
+
exec_flag[:exec].should be_true
|
83
|
+
end
|
84
|
+
end
|
85
|
+
|
86
|
+
it "should execute a hook three times." do
|
87
|
+
exec_flag = {}
|
88
|
+
subject.add(:finish, repeat: 3) do |server|
|
89
|
+
exec_flag[:exec] = true
|
90
|
+
end
|
91
|
+
3.times do |i|
|
92
|
+
exec_flag[:exec] = nil
|
93
|
+
subject.exec(:finish)
|
94
|
+
exec_flag[:exec].should be_true
|
95
|
+
end
|
96
|
+
exec_flag[:exec] = nil
|
97
|
+
subject.exec(:finish)
|
98
|
+
exec_flag[:exec].should be_nil
|
99
|
+
end
|
100
|
+
|
74
101
|
it "should execute finish_exit that is special proc." do
|
75
102
|
execute = nil
|
76
103
|
subject.set_finish_exit do
|
data/spec/task/task_spec.rb
CHANGED
@@ -92,6 +92,17 @@ describe DRbQS::Task do
|
|
92
92
|
Marshal.dump(task)
|
93
93
|
end.should raise_error
|
94
94
|
end
|
95
|
+
|
96
|
+
it "should get default group." do
|
97
|
+
task = DRbQS::Task.new([1, 2, 3], :size)
|
98
|
+
task.group.should == DRbQS::Task::DEFAULT_GROUP
|
99
|
+
end
|
100
|
+
|
101
|
+
it "should set group." do
|
102
|
+
group = :local
|
103
|
+
task = DRbQS::Task.new([1, 2, 3], :size, group: group)
|
104
|
+
task.group.should == group
|
105
|
+
end
|
95
106
|
end
|
96
107
|
|
97
108
|
describe DRbQS::Task::TaskSet::ContainerTask do
|
@@ -172,12 +183,38 @@ describe DRbQS::Task::TaskSet do
|
|
172
183
|
|
173
184
|
it "should execute hooks of symbols." do
|
174
185
|
server = mock('server')
|
175
|
-
|
176
|
-
tasks =
|
177
|
-
|
178
|
-
|
179
|
-
tasks << DRbQS::Task.new(obj[2], :some_method, hook: :test_hook)
|
186
|
+
args = [100, 200, 300]
|
187
|
+
tasks = args.map do |a|
|
188
|
+
DRbQS::Task.new(TestTask.new(a), :some_method, hook: :test_hook)
|
189
|
+
end
|
180
190
|
task_set = DRbQS::Task::TaskSet.new(tasks)
|
181
|
-
task_set.exec_hook(server,
|
191
|
+
task_set.exec_hook(server, args)
|
192
|
+
end
|
193
|
+
|
194
|
+
it "should get default group." do
|
195
|
+
tasks = 10.times.map do |a|
|
196
|
+
DRbQS::Task.new(TestTask.new(a), :some_method)
|
197
|
+
end
|
198
|
+
task_set = DRbQS::Task::TaskSet.new(tasks)
|
199
|
+
task_set.group.should == DRbQS::Task::DEFAULT_GROUP
|
200
|
+
end
|
201
|
+
|
202
|
+
it "should set group." do
|
203
|
+
group = :local
|
204
|
+
tasks = 10.times.map do |a|
|
205
|
+
DRbQS::Task.new(TestTask.new(a), :some_method, group: group)
|
206
|
+
end
|
207
|
+
task_set = DRbQS::Task::TaskSet.new(tasks)
|
208
|
+
task_set.group.should == group
|
209
|
+
end
|
210
|
+
|
211
|
+
it "should raise error for different groups." do
|
212
|
+
group = :local
|
213
|
+
tasks = 10.times.map do |a|
|
214
|
+
DRbQS::Task.new(TestTask.new(a), :some_method, group: "group#{a}".intern)
|
215
|
+
end
|
216
|
+
lambda do
|
217
|
+
DRbQS::Task::TaskSet.new(tasks)
|
218
|
+
end.should raise_error ArgumentError
|
182
219
|
end
|
183
220
|
end
|
@@ -7,18 +7,22 @@ describe DRbQS::Temporary do
|
|
7
7
|
DRbQS::Temporary.file
|
8
8
|
end
|
9
9
|
|
10
|
+
subject do
|
11
|
+
DRbQS::Temporary
|
12
|
+
end
|
13
|
+
|
10
14
|
it "should create base directory." do
|
11
|
-
File.exist?(
|
15
|
+
File.exist?(subject.root).should be_true
|
12
16
|
end
|
13
17
|
|
14
18
|
it "should create empty directory." do
|
15
|
-
dir =
|
19
|
+
dir = subject.directory
|
16
20
|
File.directory?(dir).should be_true
|
17
21
|
Dir.entries(dir).should have(2).items
|
18
22
|
end
|
19
23
|
|
20
24
|
it "should return new path of file." do
|
21
|
-
path =
|
25
|
+
path = subject.file
|
22
26
|
File.exist?(path).should_not be_true
|
23
27
|
open(path, 'w') do |f|
|
24
28
|
f.puts 'hello'
|
@@ -26,14 +30,33 @@ describe DRbQS::Temporary do
|
|
26
30
|
end
|
27
31
|
|
28
32
|
it "should make directory empty." do
|
29
|
-
Dir.entries(
|
30
|
-
|
31
|
-
Dir.entries(
|
33
|
+
Dir.entries(subject.root).size.should > 2
|
34
|
+
subject.delete
|
35
|
+
Dir.entries(subject.root).should have(2).items
|
36
|
+
end
|
37
|
+
|
38
|
+
it "should set subdirectory." do
|
39
|
+
subject.set_sub_directory('abc')
|
40
|
+
subject.subdirectory.should be_nil
|
41
|
+
end
|
42
|
+
|
43
|
+
it "should get directory in subdirectory." do
|
44
|
+
subject.set_sub_directory('ABCD')
|
45
|
+
dir = subject.directory
|
46
|
+
File.exist?(subject.subdirectory).should be_true
|
47
|
+
dir.should match(/\/ABCD/)
|
48
|
+
end
|
49
|
+
|
50
|
+
it "should get file name in subdirectory." do
|
51
|
+
subject.set_sub_directory('EFGH')
|
52
|
+
file = subject.file
|
53
|
+
File.exist?(subject.subdirectory).should be_true
|
54
|
+
file.should match(/\/EFGH\//)
|
32
55
|
end
|
33
56
|
|
34
|
-
it "should all directories." do
|
35
|
-
root =
|
36
|
-
|
57
|
+
it "should delete all directories." do
|
58
|
+
root = subject.root
|
59
|
+
subject.delete_all
|
37
60
|
File.exist?(root).should_not be_true
|
38
61
|
end
|
39
62
|
end
|