drbqs 0.0.18 → 0.0.19
Sign up to get free protection for your applications and to get access to all the features.
- data/lib/drbqs/node/node.rb +15 -10
- data/lib/drbqs/node/state.rb +5 -1
- data/lib/drbqs/node/task_client.rb +1 -1
- data/lib/drbqs/version.rb +1 -1
- data/lib/drbqs/worker/worker.rb +8 -5
- data/lib/drbqs/worker/worker_process_set.rb +10 -5
- data/spec/integration_test/01_basic_usage_spec.rb +1 -1
- data/spec/integration_test/02_use_generator_spec.rb +1 -1
- data/spec/integration_test/03_use_temporary_file_spec.rb +1 -1
- data/spec/integration_test/04_use_unix_domain_spec.rb +1 -1
- data/spec/integration_test/11_special_tasks_spec.rb +1 -1
- data/spec/integration_test/12_multiple_workers_spec.rb +1 -1
- data/spec/worker/worker_process_set_spec.rb +1 -0
- metadata +18 -18
data/lib/drbqs/node/node.rb
CHANGED
@@ -15,13 +15,13 @@ module DRbQS
|
|
15
15
|
OUTPUT_NOT_SEND_RESULT = 'not_send_result'
|
16
16
|
DEFAULT_LOG_FILE = 'drbqs_client.log'
|
17
17
|
INTERVAL_TIME_DEFAULT = 0.1
|
18
|
+
WAIT_NEW_TASK_TIME = 1
|
18
19
|
SAME_HOST_GROUP = :local
|
19
20
|
|
20
21
|
# @param [String] acces_uri Set the uri of server
|
21
22
|
# @param [Hash] opts Options of a node
|
22
23
|
# @option opts [Fixnum] :process Number of worker processes
|
23
24
|
# @option opts [Array] :group An array of group symbols
|
24
|
-
# @option opts [Boolean] :continue If we set true then the node process does not exit
|
25
25
|
# @option opts [Fixnum] :sleep_time Time interval during sleep of the node
|
26
26
|
# @option opts [String] :max_loadavg Note that this optiono is experimental
|
27
27
|
def initialize(access_uri, opts = {})
|
@@ -31,7 +31,6 @@ module DRbQS
|
|
31
31
|
@task_client = nil
|
32
32
|
@worker_number = opts[:process] || 1
|
33
33
|
@state = DRbQS::Node::State.new(:wait, @worker_number, :max_loadavg => opts[:max_loadavg], :sleep_time => opts[:sleep_time])
|
34
|
-
@process_continue = opts[:continue]
|
35
34
|
@group = opts[:group] || []
|
36
35
|
@signal_to_server_queue = Queue.new
|
37
36
|
@config = DRbQS::Config.new
|
@@ -199,7 +198,6 @@ module DRbQS
|
|
199
198
|
flag_finalize_exit = true
|
200
199
|
when :exit_after_task
|
201
200
|
@state.set_exit_after_task
|
202
|
-
@process_continue = nil
|
203
201
|
end
|
204
202
|
if flag_finalize_exit
|
205
203
|
execute_finalization
|
@@ -231,7 +229,7 @@ module DRbQS
|
|
231
229
|
private :clear_node_files
|
232
230
|
|
233
231
|
def wait_interval_of_connection
|
234
|
-
sleep(INTERVAL_TIME_DEFAULT)
|
232
|
+
Kernel.sleep(INTERVAL_TIME_DEFAULT)
|
235
233
|
end
|
236
234
|
private :wait_interval_of_connection
|
237
235
|
|
@@ -241,8 +239,8 @@ module DRbQS
|
|
241
239
|
end
|
242
240
|
end
|
243
241
|
|
244
|
-
|
245
|
-
|
242
|
+
MAX_WORKER_WAIT_TIME = 3
|
243
|
+
WORKER_WAIT_INTERVAL = 0.1
|
246
244
|
|
247
245
|
def respond_worker_signal
|
248
246
|
@worker.respond_signal
|
@@ -256,13 +254,13 @@ module DRbQS
|
|
256
254
|
respond_worker_signal
|
257
255
|
if !@worker.has_process?
|
258
256
|
break
|
259
|
-
elsif total_wait_time >
|
257
|
+
elsif total_wait_time > MAX_WORKER_WAIT_TIME
|
260
258
|
# Kill worker processes forcibly.
|
261
259
|
@worker.kill_all_processes
|
262
260
|
break
|
263
261
|
end
|
264
|
-
sleep(
|
265
|
-
total_wait_time +=
|
262
|
+
sleep(WORKER_WAIT_INTERVAL)
|
263
|
+
total_wait_time += WORKER_WAIT_INTERVAL
|
266
264
|
end
|
267
265
|
send_result_to_server
|
268
266
|
end
|
@@ -271,6 +269,7 @@ module DRbQS
|
|
271
269
|
def calculate(opts = {})
|
272
270
|
set_signal_trap
|
273
271
|
begin
|
272
|
+
server_has_no_task = nil
|
274
273
|
loop do
|
275
274
|
send_result_to_server
|
276
275
|
unless process_signal_for_server
|
@@ -279,11 +278,17 @@ module DRbQS
|
|
279
278
|
if @state.change_to_sleep_for_busy_system
|
280
279
|
@logger.info("Sleep because system is busy.")
|
281
280
|
end
|
281
|
+
if server_has_no_task && (t = server_has_no_task + WAIT_NEW_TASK_TIME - Time.now) > 0
|
282
|
+
sleep(t)
|
283
|
+
server_has_no_task = nil
|
284
|
+
end
|
282
285
|
if get_new_task
|
283
286
|
send_task_to_worker
|
284
|
-
elsif @state.
|
287
|
+
elsif @state.ready_to_exit_after_task? && @task_client.result_empty?
|
285
288
|
execute_finalization
|
286
289
|
break
|
290
|
+
elsif @state.request?
|
291
|
+
server_has_no_task = Time.now
|
287
292
|
end
|
288
293
|
unless respond_worker_signal
|
289
294
|
wait_interval_of_connection
|
data/lib/drbqs/node/state.rb
CHANGED
@@ -55,7 +55,7 @@ module DRbQS
|
|
55
55
|
end
|
56
56
|
|
57
57
|
def request?
|
58
|
-
request_task_number > 0
|
58
|
+
@state_after_task != :exit && request_task_number > 0
|
59
59
|
end
|
60
60
|
|
61
61
|
def all_workers_waiting?
|
@@ -169,6 +169,10 @@ module DRbQS
|
|
169
169
|
end
|
170
170
|
nil
|
171
171
|
end
|
172
|
+
|
173
|
+
def ready_to_exit_after_task?
|
174
|
+
@state_after_task == :exit && all_workers_waiting?
|
175
|
+
end
|
172
176
|
end
|
173
177
|
end
|
174
178
|
end
|
data/lib/drbqs/version.rb
CHANGED
data/lib/drbqs/worker/worker.rb
CHANGED
@@ -105,8 +105,12 @@ module DRbQS
|
|
105
105
|
# @param [Fixnum] task_id
|
106
106
|
# @param [Numeric] interval_time An argument of Kernel#sleep.
|
107
107
|
def wait(task_id, interval_time)
|
108
|
-
|
109
|
-
|
108
|
+
if @task_pool[task_id]
|
109
|
+
loop do
|
110
|
+
step
|
111
|
+
unless @task_pool[task_id]
|
112
|
+
return true
|
113
|
+
end
|
110
114
|
Kernel.sleep(interval_time)
|
111
115
|
end
|
112
116
|
end
|
@@ -116,9 +120,8 @@ module DRbQS
|
|
116
120
|
# @param [Numeric] interval_time An argument of Kernel#sleep.
|
117
121
|
def waitall(interval_time)
|
118
122
|
while calculating?
|
119
|
-
|
120
|
-
|
121
|
-
end
|
123
|
+
step
|
124
|
+
Kernel.sleep(interval_time)
|
122
125
|
end
|
123
126
|
end
|
124
127
|
|
@@ -4,7 +4,7 @@ require 'drbqs/utility/temporary.rb'
|
|
4
4
|
|
5
5
|
module DRbQS
|
6
6
|
class Worker
|
7
|
-
READ_BYTE_SIZE =
|
7
|
+
READ_BYTE_SIZE = 10240
|
8
8
|
|
9
9
|
class ProcessSet
|
10
10
|
attr_reader :process
|
@@ -140,7 +140,7 @@ module DRbQS
|
|
140
140
|
private :delete_process
|
141
141
|
|
142
142
|
# Read IOs and respond signals from chiled processes.
|
143
|
-
# If there no
|
143
|
+
# If there is no data from child processes then the method returns false.
|
144
144
|
# Otherwise, true.
|
145
145
|
# Types of signals are :result, :node_error, :finish_preparing_to_exit.
|
146
146
|
# - :result
|
@@ -154,10 +154,16 @@ module DRbQS
|
|
154
154
|
to_be_deleted = []
|
155
155
|
@process.each do |key, h|
|
156
156
|
if !h[:task].empty? || h[:exit]
|
157
|
+
data = ''
|
157
158
|
begin
|
158
|
-
|
159
|
+
loop do
|
160
|
+
data << h[:in].read_nonblock(READ_BYTE_SIZE)
|
161
|
+
end
|
162
|
+
rescue IO::WaitReadable
|
163
|
+
end
|
164
|
+
if !data.empty?
|
165
|
+
num += 1
|
159
166
|
h[:unpacker].feed_each(data) do |ary|
|
160
|
-
num += 1
|
161
167
|
response_type, response = ary
|
162
168
|
case response_type
|
163
169
|
when :result
|
@@ -179,7 +185,6 @@ module DRbQS
|
|
179
185
|
to_be_deleted << key
|
180
186
|
end
|
181
187
|
end
|
182
|
-
rescue IO::WaitReadable
|
183
188
|
end
|
184
189
|
end
|
185
190
|
end
|
@@ -9,7 +9,7 @@ describe DRbQS do
|
|
9
9
|
DRbQS::Task.new(TestCount.new, :echo, args: [i])
|
10
10
|
end
|
11
11
|
@process_id, @uri = drbqs_fork_server(14010, :task => @tasks)
|
12
|
-
@node = DRbQS::Node.new(@uri, :log_file => $stdout
|
12
|
+
@node = DRbQS::Node.new(@uri, :log_file => $stdout)
|
13
13
|
end
|
14
14
|
|
15
15
|
it "should have nil instance variables" do
|
@@ -14,7 +14,7 @@ describe DRbQS do
|
|
14
14
|
end
|
15
15
|
end
|
16
16
|
@process_id, @uri = drbqs_fork_server(14020, :task => @task_generators)
|
17
|
-
@node = DRbQS::Node.new(@uri, :log_file => $stdout
|
17
|
+
@node = DRbQS::Node.new(@uri, :log_file => $stdout)
|
18
18
|
end
|
19
19
|
|
20
20
|
it "should have nil instance variables" do
|
@@ -7,7 +7,7 @@ describe DRbQS do
|
|
7
7
|
before(:all) do
|
8
8
|
@tasks = [DRbQS::Task.new(Test3.new, :temp_file)]
|
9
9
|
@process_id, @uri = drbqs_fork_server(14030, :task => @tasks)
|
10
|
-
@node = DRbQS::Node.new(@uri, :log_file => $stdout
|
10
|
+
@node = DRbQS::Node.new(@uri, :log_file => $stdout)
|
11
11
|
end
|
12
12
|
|
13
13
|
it "should initialize @task_client" do
|
@@ -11,7 +11,7 @@ describe DRbQS::Server do
|
|
11
11
|
path = "/tmp/drbqs"
|
12
12
|
@process_id, @uri = drbqs_fork_server(path, :task => @tasks)
|
13
13
|
@manage = DRbQS::Manage.new(:uri => @uri)
|
14
|
-
@node = DRbQS::Node.new(@uri, :log_file => $stdout
|
14
|
+
@node = DRbQS::Node.new(@uri, :log_file => $stdout)
|
15
15
|
end
|
16
16
|
|
17
17
|
it "should send signal and get status" do
|
@@ -36,7 +36,7 @@ describe DRbQS do
|
|
36
36
|
server.set_finalization_task(DRbQS::Task.new(TestValue, :set, args: [:third, 3]),
|
37
37
|
DRbQS::Task.new(TestValue, :set, args: [:fourth, 4]))
|
38
38
|
end
|
39
|
-
@node = DRbQS::Node.new(@uri, :log_file => $stdout
|
39
|
+
@node = DRbQS::Node.new(@uri, :log_file => $stdout)
|
40
40
|
end
|
41
41
|
|
42
42
|
it "should execute initialization tasks." do
|
@@ -13,7 +13,7 @@ describe DRbQS do
|
|
13
13
|
end
|
14
14
|
@process_number = 3
|
15
15
|
@process_id, @uri = drbqs_fork_server(14120, :task => @tasks)
|
16
|
-
@node = DRbQS::Node.new(@uri, :log_file => $stdout, :
|
16
|
+
@node = DRbQS::Node.new(@uri, :log_file => $stdout, :process => @process_number)
|
17
17
|
end
|
18
18
|
|
19
19
|
it "should calculate" do
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: drbqs
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.19
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,11 +9,11 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2011-10-
|
12
|
+
date: 2011-10-28 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: rspec
|
16
|
-
requirement: &
|
16
|
+
requirement: &12438740 !ruby/object:Gem::Requirement
|
17
17
|
none: false
|
18
18
|
requirements:
|
19
19
|
- - ! '>='
|
@@ -21,10 +21,10 @@ dependencies:
|
|
21
21
|
version: 2.6.0
|
22
22
|
type: :development
|
23
23
|
prerelease: false
|
24
|
-
version_requirements: *
|
24
|
+
version_requirements: *12438740
|
25
25
|
- !ruby/object:Gem::Dependency
|
26
26
|
name: yard
|
27
|
-
requirement: &
|
27
|
+
requirement: &12438160 !ruby/object:Gem::Requirement
|
28
28
|
none: false
|
29
29
|
requirements:
|
30
30
|
- - ! '>='
|
@@ -32,10 +32,10 @@ dependencies:
|
|
32
32
|
version: 0.7.2
|
33
33
|
type: :development
|
34
34
|
prerelease: false
|
35
|
-
version_requirements: *
|
35
|
+
version_requirements: *12438160
|
36
36
|
- !ruby/object:Gem::Dependency
|
37
37
|
name: filename
|
38
|
-
requirement: &
|
38
|
+
requirement: &12437700 !ruby/object:Gem::Requirement
|
39
39
|
none: false
|
40
40
|
requirements:
|
41
41
|
- - ! '>='
|
@@ -43,10 +43,10 @@ dependencies:
|
|
43
43
|
version: 0.1.0
|
44
44
|
type: :runtime
|
45
45
|
prerelease: false
|
46
|
-
version_requirements: *
|
46
|
+
version_requirements: *12437700
|
47
47
|
- !ruby/object:Gem::Dependency
|
48
48
|
name: user_config
|
49
|
-
requirement: &
|
49
|
+
requirement: &12437200 !ruby/object:Gem::Requirement
|
50
50
|
none: false
|
51
51
|
requirements:
|
52
52
|
- - ! '>='
|
@@ -54,10 +54,10 @@ dependencies:
|
|
54
54
|
version: 0.0.2
|
55
55
|
type: :runtime
|
56
56
|
prerelease: false
|
57
|
-
version_requirements: *
|
57
|
+
version_requirements: *12437200
|
58
58
|
- !ruby/object:Gem::Dependency
|
59
59
|
name: net-ssh
|
60
|
-
requirement: &
|
60
|
+
requirement: &12436460 !ruby/object:Gem::Requirement
|
61
61
|
none: false
|
62
62
|
requirements:
|
63
63
|
- - ! '>='
|
@@ -65,10 +65,10 @@ dependencies:
|
|
65
65
|
version: 2.1.0
|
66
66
|
type: :runtime
|
67
67
|
prerelease: false
|
68
|
-
version_requirements: *
|
68
|
+
version_requirements: *12436460
|
69
69
|
- !ruby/object:Gem::Dependency
|
70
70
|
name: net-ssh-shell
|
71
|
-
requirement: &
|
71
|
+
requirement: &12435840 !ruby/object:Gem::Requirement
|
72
72
|
none: false
|
73
73
|
requirements:
|
74
74
|
- - ! '>='
|
@@ -76,10 +76,10 @@ dependencies:
|
|
76
76
|
version: 0.2.0
|
77
77
|
type: :runtime
|
78
78
|
prerelease: false
|
79
|
-
version_requirements: *
|
79
|
+
version_requirements: *12435840
|
80
80
|
- !ruby/object:Gem::Dependency
|
81
81
|
name: net-sftp
|
82
|
-
requirement: &
|
82
|
+
requirement: &12435360 !ruby/object:Gem::Requirement
|
83
83
|
none: false
|
84
84
|
requirements:
|
85
85
|
- - ! '>='
|
@@ -87,10 +87,10 @@ dependencies:
|
|
87
87
|
version: 2.0.5
|
88
88
|
type: :runtime
|
89
89
|
prerelease: false
|
90
|
-
version_requirements: *
|
90
|
+
version_requirements: *12435360
|
91
91
|
- !ruby/object:Gem::Dependency
|
92
92
|
name: sys-proctable
|
93
|
-
requirement: &
|
93
|
+
requirement: &12434980 !ruby/object:Gem::Requirement
|
94
94
|
none: false
|
95
95
|
requirements:
|
96
96
|
- - ! '>='
|
@@ -98,7 +98,7 @@ dependencies:
|
|
98
98
|
version: '0'
|
99
99
|
type: :runtime
|
100
100
|
prerelease: false
|
101
|
-
version_requirements: *
|
101
|
+
version_requirements: *12434980
|
102
102
|
description: Task queuing system over network that is implemented by dRuby.
|
103
103
|
email:
|
104
104
|
- d@ytak.info
|