concurrent_worker 0.1.3 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +1 -0
- data/Gemfile.lock +1 -1
- data/README.md +4 -4
- data/concurrent_worker.gemspec +2 -2
- data/lib/concurrent_worker/version.rb +1 -1
- data/lib/concurrent_worker.rb +139 -39
- metadata +4 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 8ae885ab2ba903e2b28fb238efbde6685292d4df97a381b79ad4d99ef3e034a2
|
4
|
+
data.tar.gz: 2640243f4650ed203d36522134620b55fd060ba62e41112d8897d3e8bfafdea2
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 76024b286253b5030420e05ff03afe36a5ac5b6858c2cef1199e328d39f1ae38bfd0e7f40c35452acd9d55b37f1d9958f89030ca1be799552c0a352fff67a50c
|
7
|
+
data.tar.gz: 64bfeeeb41949087c1a7bebb8cc3e316500d005c8364701927ff6dbfd4d4ba210a2e133bc6cf6393d5c518fd191f9bb96a5758b2802f1d48f8d3b23676d1309e
|
data/.gitignore
CHANGED
data/Gemfile.lock
CHANGED
data/README.md
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
# ConcurrentWorker
|
2
2
|
|
3
|
-
Concurrent worker in thread/process with preparation
|
3
|
+
Concurrent worker in thread/process with preparation mechanism.
|
4
4
|
|
5
5
|
## Installation
|
6
6
|
|
@@ -54,7 +54,7 @@ end
|
|
54
54
|
|
55
55
|
# define base block for some preparation of work block.
|
56
56
|
logger.set_block(:base_block) do
|
57
|
-
open(
|
57
|
+
open("log.txt", "w") do |file|
|
58
58
|
@file = file
|
59
59
|
# 'yield_loop_block' must be called in base block.
|
60
60
|
# work block will be called in this call.
|
@@ -72,12 +72,12 @@ You can exec work block in some process concurrently.
|
|
72
72
|
```ruby
|
73
73
|
#define a pool of 8 workers , executed in other process.
|
74
74
|
wp = ConcurrentWorker::WorkerPool.new(type: :process, pool_max: 8) do |n|
|
75
|
-
[n
|
75
|
+
[n, n.times.inject(:+)]
|
76
76
|
end
|
77
77
|
|
78
78
|
# you can receive the result of work block with callback block.
|
79
79
|
wp.add_callback do |n, result|
|
80
|
-
logger.
|
80
|
+
logger.req("n=%d,result=%d", n, result)
|
81
81
|
end
|
82
82
|
|
83
83
|
(10000000..10000200).each do |n|
|
data/concurrent_worker.gemspec
CHANGED
@@ -9,8 +9,8 @@ Gem::Specification.new do |spec|
|
|
9
9
|
spec.authors = ["dddogdiamond"]
|
10
10
|
spec.email = ["dddogdiamond@gmail.com"]
|
11
11
|
|
12
|
-
spec.summary = %q{Concurrent worker in thread/process with preparation
|
13
|
-
spec.description = %q{Concurrent worker in thread/process with preparation
|
12
|
+
spec.summary = %q{Concurrent worker in thread/process with preparation mechanism.}
|
13
|
+
spec.description = %q{Concurrent worker in thread/process with preparation mechanism.}
|
14
14
|
spec.homepage = "https://github.com/dddogdiamond/concurrent_worker"
|
15
15
|
spec.license = "MIT"
|
16
16
|
|
data/lib/concurrent_worker.rb
CHANGED
@@ -9,7 +9,7 @@ module ConcurrentWorker
|
|
9
9
|
# Worker : worker class
|
10
10
|
# +cncr_block : concurrent processing block : thread(as ConcurrentThread)/process(as ConcurrentProcess)
|
11
11
|
# +base_block : user defined preparation to exec 'work block'
|
12
|
-
# +loop_block : loop of receiving request and exec '
|
12
|
+
# +loop_block : loop of receiving request and exec 'work block'
|
13
13
|
# +work_block : user requested work
|
14
14
|
#
|
15
15
|
# These blocks are executed with 'instance_exec' method of worker,
|
@@ -24,29 +24,41 @@ module ConcurrentWorker
|
|
24
24
|
set_block(:work_block, &work_block) if work_block
|
25
25
|
|
26
26
|
@state = :idle
|
27
|
-
@
|
27
|
+
@result_callbacks = []
|
28
|
+
@retired_callbacks = []
|
28
29
|
|
29
|
-
@req_queue_max = @options[
|
30
|
+
@req_queue_max = @options[:req_queue_max] || 2
|
30
31
|
@req_queue = SizedQueue.new(@req_queue_max)
|
31
32
|
|
32
|
-
if @options[
|
33
|
+
if @options[:type] == :process
|
33
34
|
Worker.include ConcurrentProcess
|
34
35
|
else
|
35
36
|
Worker.include ConcurrentThread
|
36
37
|
end
|
37
38
|
end
|
38
|
-
|
39
|
+
|
39
40
|
def add_callback(&callback)
|
40
41
|
raise "block is nil" unless callback
|
41
|
-
@
|
42
|
+
@result_callbacks.push(callback)
|
42
43
|
end
|
43
44
|
|
44
|
-
def
|
45
|
-
@
|
45
|
+
def call_result_callbacks(args)
|
46
|
+
@result_callbacks.each do |callback|
|
46
47
|
callback.call(args)
|
47
48
|
end
|
48
49
|
end
|
49
50
|
|
51
|
+
def add_retired_callback(&callback)
|
52
|
+
raise "block is nil" unless callback
|
53
|
+
@retired_callbacks.push(callback)
|
54
|
+
end
|
55
|
+
|
56
|
+
def call_retired_callbacks
|
57
|
+
@retired_callbacks.each do |callback|
|
58
|
+
callback.call
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
50
62
|
def set_block(symbol, &block)
|
51
63
|
raise "block is nil" unless block
|
52
64
|
|
@@ -90,6 +102,7 @@ module ConcurrentWorker
|
|
90
102
|
yield_loop_block
|
91
103
|
end
|
92
104
|
end
|
105
|
+
|
93
106
|
cncr_block
|
94
107
|
end
|
95
108
|
|
@@ -116,13 +129,24 @@ module ConcurrentWorker
|
|
116
129
|
def cncr_block
|
117
130
|
@thread_channel = Queue.new
|
118
131
|
@thread = Thread.new do
|
119
|
-
|
132
|
+
begin
|
133
|
+
yield_base_block
|
134
|
+
ensure
|
135
|
+
@req_queue.close
|
136
|
+
@thread_channel.close
|
137
|
+
call_retired_callbacks
|
138
|
+
end
|
120
139
|
end
|
121
140
|
end
|
122
141
|
|
123
142
|
def send_req(args)
|
124
|
-
|
125
|
-
|
143
|
+
begin
|
144
|
+
@req_queue.push(args)
|
145
|
+
@thread_channel.push(args)
|
146
|
+
true
|
147
|
+
rescue ClosedQueueError
|
148
|
+
false
|
149
|
+
end
|
126
150
|
end
|
127
151
|
|
128
152
|
def receive_req
|
@@ -130,7 +154,7 @@ module ConcurrentWorker
|
|
130
154
|
end
|
131
155
|
|
132
156
|
def send_res(args)
|
133
|
-
|
157
|
+
call_result_callbacks(args)
|
134
158
|
@req_queue.pop
|
135
159
|
end
|
136
160
|
|
@@ -164,7 +188,9 @@ module ConcurrentWorker
|
|
164
188
|
end
|
165
189
|
|
166
190
|
def recv
|
167
|
-
|
191
|
+
szdata = @rio.read(4)
|
192
|
+
return [] if szdata.nil?
|
193
|
+
size = szdata.unpack("I")[0]
|
168
194
|
Marshal.load(@rio.read(size))
|
169
195
|
end
|
170
196
|
|
@@ -178,25 +204,44 @@ module ConcurrentWorker
|
|
178
204
|
@ipc_channel = IPCDuplexChannel.new
|
179
205
|
@c_pid = fork do
|
180
206
|
@ipc_channel.choose_io
|
181
|
-
|
182
|
-
|
207
|
+
begin
|
208
|
+
yield_base_block
|
209
|
+
rescue
|
210
|
+
@ipc_channel.send($!)
|
211
|
+
ensure
|
212
|
+
@ipc_channel.send(:worker_loop_finished)
|
213
|
+
@ipc_channel.close
|
214
|
+
end
|
183
215
|
end
|
184
216
|
@ipc_channel.choose_io
|
185
217
|
|
186
218
|
@thread = Thread.new do
|
187
|
-
|
188
|
-
|
189
|
-
|
190
|
-
|
191
|
-
|
219
|
+
begin
|
220
|
+
loop do
|
221
|
+
result = @ipc_channel.recv
|
222
|
+
break if result == :worker_loop_finished
|
223
|
+
raise result if result.kind_of?(Exception)
|
224
|
+
|
225
|
+
call_result_callbacks(result)
|
226
|
+
@req_queue.pop
|
227
|
+
end
|
228
|
+
ensure
|
229
|
+
@ipc_channel.close
|
230
|
+
@req_queue.close
|
231
|
+
call_retired_callbacks
|
192
232
|
end
|
193
233
|
end
|
194
234
|
end
|
195
235
|
|
196
236
|
def send_req(args)
|
197
|
-
|
198
|
-
|
199
|
-
|
237
|
+
begin
|
238
|
+
#called from main process only
|
239
|
+
@req_queue.push(args)
|
240
|
+
@ipc_channel.send(args)
|
241
|
+
true
|
242
|
+
rescue ClosedQueueError, IOError
|
243
|
+
false
|
244
|
+
end
|
200
245
|
end
|
201
246
|
|
202
247
|
def receive_req
|
@@ -210,11 +255,13 @@ module ConcurrentWorker
|
|
210
255
|
end
|
211
256
|
|
212
257
|
def wait_cncr_proc
|
258
|
+
$stdout.flush
|
213
259
|
begin
|
214
260
|
Process.waitpid(@c_pid)
|
215
261
|
rescue Errno::ECHILD
|
216
262
|
end
|
217
263
|
@thread && @thread.join
|
264
|
+
$stdout.flush
|
218
265
|
end
|
219
266
|
end
|
220
267
|
|
@@ -225,32 +272,80 @@ module ConcurrentWorker
|
|
225
272
|
@args = args
|
226
273
|
|
227
274
|
@options = options
|
228
|
-
@max_num = @options[
|
275
|
+
@max_num = @options[:pool_max] || 8
|
229
276
|
@set_blocks = []
|
230
277
|
if work_block
|
231
278
|
@set_blocks.push([:work_block, work_block])
|
232
279
|
end
|
233
280
|
|
281
|
+
@array_mutex = Mutex.new
|
234
282
|
@ready_queue = Queue.new
|
235
283
|
|
236
|
-
@
|
284
|
+
@result_callbacks = []
|
237
285
|
@callback_queue = Queue.new
|
238
286
|
@callback_thread = Thread.new do
|
239
|
-
finished_count = 0
|
240
287
|
loop do
|
241
288
|
break if (result = @callback_queue.pop).empty?
|
242
|
-
@
|
289
|
+
@result_callbacks.each do |callback|
|
243
290
|
callback.call(result[0])
|
244
291
|
end
|
245
292
|
end
|
246
293
|
end
|
294
|
+
|
295
|
+
@req_queue = SizedQueue.new(@max_num * 2)
|
296
|
+
@req_thread = Thread.new do
|
297
|
+
loop do
|
298
|
+
break if (req = @req_queue.pop).empty?
|
299
|
+
if need_new_worker?
|
300
|
+
puts "new worker".light_yellow
|
301
|
+
w = deploy_worker
|
302
|
+
w.req_queue_max.times do
|
303
|
+
@ready_queue.push(w)
|
304
|
+
end
|
305
|
+
end
|
306
|
+
while !@ready_queue.pop.req(*req[0], &req[1])
|
307
|
+
end
|
308
|
+
end
|
309
|
+
end
|
310
|
+
end
|
311
|
+
|
312
|
+
def delete(arg)
|
313
|
+
@array_mutex.synchronize do
|
314
|
+
super(arg)
|
315
|
+
end
|
316
|
+
end
|
317
|
+
def push(arg)
|
318
|
+
@array_mutex.synchronize do
|
319
|
+
super(arg)
|
320
|
+
end
|
321
|
+
end
|
322
|
+
def shift
|
323
|
+
@array_mutex.synchronize do
|
324
|
+
super
|
325
|
+
end
|
326
|
+
end
|
327
|
+
def empty?
|
328
|
+
@array_mutex.synchronize do
|
329
|
+
super
|
330
|
+
end
|
331
|
+
end
|
332
|
+
|
333
|
+
def need_new_worker?
|
334
|
+
@array_mutex.synchronize do
|
335
|
+
self.size < @max_num && self.select{ |w| w.req_queue.size == 0 }.empty?
|
336
|
+
end
|
247
337
|
end
|
248
338
|
|
249
339
|
def add_callback(&callback)
|
250
340
|
raise "block is nil" unless callback
|
251
|
-
@
|
341
|
+
@result_callbacks.push(callback)
|
252
342
|
end
|
253
343
|
|
344
|
+
def add_finished_callback(&callback)
|
345
|
+
raise "block is nil" unless callback
|
346
|
+
@finished_callbacks.push(callback)
|
347
|
+
end
|
348
|
+
|
254
349
|
|
255
350
|
def deploy_worker
|
256
351
|
w = Worker.new(*@args, type: @options[:type], &@work_block)
|
@@ -258,11 +353,20 @@ module ConcurrentWorker
|
|
258
353
|
@callback_queue.push([arg])
|
259
354
|
@ready_queue.push(w)
|
260
355
|
end
|
356
|
+
|
357
|
+
w.add_retired_callback do
|
358
|
+
while req = w.req_queue.pop
|
359
|
+
next if req == []
|
360
|
+
@req_queue.push(req)
|
361
|
+
end
|
362
|
+
self.delete(w)
|
363
|
+
end
|
364
|
+
|
261
365
|
@set_blocks.each do |symbol, block|
|
262
366
|
w.set_block(symbol, &block)
|
263
367
|
end
|
264
368
|
w.run
|
265
|
-
push
|
369
|
+
self.push(w)
|
266
370
|
w
|
267
371
|
end
|
268
372
|
|
@@ -271,21 +375,17 @@ module ConcurrentWorker
|
|
271
375
|
end
|
272
376
|
|
273
377
|
def req(*args, &work_block)
|
274
|
-
|
275
|
-
w = deploy_worker
|
276
|
-
w.req_queue_max.times do
|
277
|
-
@ready_queue.push(w)
|
278
|
-
end
|
279
|
-
end
|
280
|
-
w = @ready_queue.pop
|
281
|
-
w.req(*args, &work_block)
|
378
|
+
@req_queue.push([args, work_block])
|
282
379
|
end
|
283
380
|
|
284
381
|
def join
|
285
|
-
|
286
|
-
self.
|
382
|
+
puts size
|
383
|
+
self.shift.join until self.empty?
|
384
|
+
puts size
|
287
385
|
@callback_queue.push([])
|
288
386
|
@callback_thread.join
|
387
|
+
@req_queue.push([])
|
388
|
+
@req_thread.join
|
289
389
|
end
|
290
390
|
end
|
291
391
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: concurrent_worker
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- dddogdiamond
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2019-05-
|
11
|
+
date: 2019-05-06 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -52,7 +52,7 @@ dependencies:
|
|
52
52
|
- - "~>"
|
53
53
|
- !ruby/object:Gem::Version
|
54
54
|
version: '5.0'
|
55
|
-
description: Concurrent worker in thread/process with preparation
|
55
|
+
description: Concurrent worker in thread/process with preparation mechanism.
|
56
56
|
email:
|
57
57
|
- dddogdiamond@gmail.com
|
58
58
|
executables: []
|
@@ -93,5 +93,5 @@ requirements: []
|
|
93
93
|
rubygems_version: 3.0.3
|
94
94
|
signing_key:
|
95
95
|
specification_version: 4
|
96
|
-
summary: Concurrent worker in thread/process with preparation
|
96
|
+
summary: Concurrent worker in thread/process with preparation mechanism.
|
97
97
|
test_files: []
|