concurrent_worker 0.2.1 → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile.lock +1 -1
- data/README.md +4 -4
- data/lib/concurrent_worker/version.rb +1 -1
- data/lib/concurrent_worker.rb +87 -29
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 07268e6a9729fb2b470926764e1975ee7305c5fb9689412436b3d00c0dbcd30b
|
4
|
+
data.tar.gz: e121b8aa8c4aacec697282f6efd07dbc0b8e968ec22d79d2e092bf173ea29dc7
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: a26ace3f114dcefbfac75c354237b9180ee6f0b0f2d09b77eee265af845d3a1e027c7792c306e68a9b8c1e6b13599117d4c491aff8b6f1cb51f26debae015cec
|
7
|
+
data.tar.gz: e2b7e20256f2fb68c18915132027df1ef002babf65bbda4da3495b07c289ad5f2c6d76c0c829245316d89c13804c3deded25c5144f6ba5ca52356b12a1f7c4ec
|
data/Gemfile.lock
CHANGED
data/README.md
CHANGED
@@ -22,7 +22,7 @@ Or install it yourself as:
|
|
22
22
|
|
23
23
|
### Worker
|
24
24
|
|
25
|
-
You can define worker block executed in other thread, and send requests to the worker.
|
25
|
+
You can define a worker setting a work block executed in other thread, and send requests to the worker.
|
26
26
|
|
27
27
|
```ruby
|
28
28
|
require 'concurrent_worker'
|
@@ -42,11 +42,11 @@ logger.req("thread%d n=%d\n",0, 1)
|
|
42
42
|
logger.join
|
43
43
|
```
|
44
44
|
|
45
|
-
If you need some preparation for the
|
45
|
+
If you need some preparation for the work block, you can define 'base block'.
|
46
46
|
|
47
47
|
```ruby
|
48
48
|
logger = ConcurrentWorker::Worker.new do |args|
|
49
|
-
#
|
49
|
+
# work block and base block can share object with instance variable(@xxx).
|
50
50
|
printf(@file, *args)
|
51
51
|
@file.flush
|
52
52
|
nil
|
@@ -70,7 +70,7 @@ The work block and base block are executed in a same thread, and in a worker's i
|
|
70
70
|
You can exec work block in some process concurrently.
|
71
71
|
|
72
72
|
```ruby
|
73
|
-
#define a pool of 8 workers , executed in other process.
|
73
|
+
#define a pool of 8 workers with same work block, executed in other process.
|
74
74
|
wp = ConcurrentWorker::WorkerPool.new(type: :process, pool_max: 8) do |n|
|
75
75
|
[n, n.times.inject(:+)]
|
76
76
|
end
|
data/lib/concurrent_worker.rb
CHANGED
@@ -3,6 +3,48 @@ require "concurrent_worker/version"
|
|
3
3
|
module ConcurrentWorker
|
4
4
|
class Error < StandardError; end
|
5
5
|
|
6
|
+
|
7
|
+
class RequestCounter
|
8
|
+
def initialize
|
9
|
+
@count = Queue.new
|
10
|
+
@com = Queue.new
|
11
|
+
end
|
12
|
+
def push(args)
|
13
|
+
@count.push(args)
|
14
|
+
end
|
15
|
+
|
16
|
+
def pop
|
17
|
+
@count.pop
|
18
|
+
@com.push([])
|
19
|
+
end
|
20
|
+
def wait(n)
|
21
|
+
return if @count.size <= n
|
22
|
+
while @com.pop
|
23
|
+
break if @count.size <= n
|
24
|
+
end
|
25
|
+
end
|
26
|
+
def size
|
27
|
+
@count.size
|
28
|
+
end
|
29
|
+
|
30
|
+
def close
|
31
|
+
@count.close
|
32
|
+
end
|
33
|
+
|
34
|
+
def rest
|
35
|
+
result = []
|
36
|
+
until @count.empty?
|
37
|
+
req = @count.pop
|
38
|
+
next if req == []
|
39
|
+
result.push(req)
|
40
|
+
end
|
41
|
+
result
|
42
|
+
end
|
43
|
+
|
44
|
+
end
|
45
|
+
|
46
|
+
|
47
|
+
|
6
48
|
require 'thread'
|
7
49
|
class Worker
|
8
50
|
attr_accessor :channel
|
@@ -16,7 +58,7 @@ module ConcurrentWorker
|
|
16
58
|
# so that they can share instance variables:@xxxx.
|
17
59
|
#
|
18
60
|
|
19
|
-
attr_reader :
|
61
|
+
attr_reader :req_counter, :req_queue_max
|
20
62
|
|
21
63
|
def initialize(*args, **options, &work_block)
|
22
64
|
@args = args
|
@@ -28,7 +70,7 @@ module ConcurrentWorker
|
|
28
70
|
@retired_callbacks = []
|
29
71
|
|
30
72
|
@req_queue_max = @options[:req_queue_max] || 2
|
31
|
-
@
|
73
|
+
@req_counter = RequestCounter.new
|
32
74
|
|
33
75
|
if @options[:type] == :process
|
34
76
|
Worker.include ConcurrentProcess
|
@@ -111,6 +153,7 @@ module ConcurrentWorker
|
|
111
153
|
unless @state == :run
|
112
154
|
run
|
113
155
|
end
|
156
|
+
@req_counter.wait(@req_queue_max)
|
114
157
|
send_req([args, work_block])
|
115
158
|
end
|
116
159
|
|
@@ -119,6 +162,7 @@ module ConcurrentWorker
|
|
119
162
|
end
|
120
163
|
|
121
164
|
def join
|
165
|
+
@req_counter.wait(0)
|
122
166
|
quit
|
123
167
|
wait_cncr_proc
|
124
168
|
end
|
@@ -132,7 +176,7 @@ module ConcurrentWorker
|
|
132
176
|
begin
|
133
177
|
yield_base_block
|
134
178
|
ensure
|
135
|
-
@
|
179
|
+
@req_counter.close
|
136
180
|
@thread_channel.close
|
137
181
|
call_retired_callbacks
|
138
182
|
end
|
@@ -141,7 +185,7 @@ module ConcurrentWorker
|
|
141
185
|
|
142
186
|
def send_req(args)
|
143
187
|
begin
|
144
|
-
@
|
188
|
+
@req_counter.push(args)
|
145
189
|
@thread_channel.push(args)
|
146
190
|
true
|
147
191
|
rescue ClosedQueueError
|
@@ -155,7 +199,7 @@ module ConcurrentWorker
|
|
155
199
|
|
156
200
|
def send_res(args)
|
157
201
|
call_result_callbacks(args)
|
158
|
-
@
|
202
|
+
@req_counter.pop
|
159
203
|
end
|
160
204
|
|
161
205
|
def wait_cncr_proc
|
@@ -188,10 +232,14 @@ module ConcurrentWorker
|
|
188
232
|
end
|
189
233
|
|
190
234
|
def recv
|
191
|
-
|
192
|
-
|
193
|
-
|
194
|
-
|
235
|
+
begin
|
236
|
+
szdata = @rio.read(4)
|
237
|
+
return [] if szdata.nil?
|
238
|
+
size = szdata.unpack("I")[0]
|
239
|
+
Marshal.load(@rio.read(size))
|
240
|
+
rescue IOError
|
241
|
+
raise StopIteration
|
242
|
+
end
|
195
243
|
end
|
196
244
|
|
197
245
|
def close
|
@@ -210,7 +258,6 @@ module ConcurrentWorker
|
|
210
258
|
@ipc_channel.send($!)
|
211
259
|
ensure
|
212
260
|
@ipc_channel.send(:worker_loop_finished)
|
213
|
-
@ipc_channel.close
|
214
261
|
end
|
215
262
|
end
|
216
263
|
@ipc_channel.choose_io
|
@@ -223,11 +270,14 @@ module ConcurrentWorker
|
|
223
270
|
raise result if result.kind_of?(Exception)
|
224
271
|
|
225
272
|
call_result_callbacks(result)
|
226
|
-
@
|
273
|
+
@req_counter.pop
|
227
274
|
end
|
275
|
+
rescue
|
276
|
+
Thread.pass
|
277
|
+
raise $!
|
228
278
|
ensure
|
229
279
|
@ipc_channel.close
|
230
|
-
@
|
280
|
+
@req_counter.close
|
231
281
|
call_retired_callbacks
|
232
282
|
end
|
233
283
|
end
|
@@ -236,7 +286,7 @@ module ConcurrentWorker
|
|
236
286
|
def send_req(args)
|
237
287
|
begin
|
238
288
|
#called from main process only
|
239
|
-
@
|
289
|
+
@req_counter.push(args)
|
240
290
|
@ipc_channel.send(args)
|
241
291
|
true
|
242
292
|
rescue ClosedQueueError, IOError
|
@@ -255,13 +305,11 @@ module ConcurrentWorker
|
|
255
305
|
end
|
256
306
|
|
257
307
|
def wait_cncr_proc
|
258
|
-
$stdout.flush
|
259
308
|
begin
|
260
309
|
Process.waitpid(@c_pid)
|
261
310
|
rescue Errno::ECHILD
|
262
311
|
end
|
263
312
|
@thread && @thread.join
|
264
|
-
$stdout.flush
|
265
313
|
end
|
266
314
|
end
|
267
315
|
|
@@ -277,7 +325,7 @@ module ConcurrentWorker
|
|
277
325
|
if work_block
|
278
326
|
@set_blocks.push([:work_block, work_block])
|
279
327
|
end
|
280
|
-
|
328
|
+
|
281
329
|
@array_mutex = Mutex.new
|
282
330
|
@ready_queue = Queue.new
|
283
331
|
|
@@ -289,21 +337,29 @@ module ConcurrentWorker
|
|
289
337
|
@result_callbacks.each do |callback|
|
290
338
|
callback.call(result[0])
|
291
339
|
end
|
340
|
+
@req_counter.pop
|
292
341
|
end
|
293
342
|
end
|
343
|
+
@req_queue_max = @options[:req_queue_max]||2
|
294
344
|
|
295
|
-
@req_queue =
|
345
|
+
@req_queue = Queue.new
|
346
|
+
@req_counter = RequestCounter.new
|
296
347
|
@req_thread = Thread.new do
|
297
348
|
loop do
|
298
349
|
break if (req = @req_queue.pop).empty?
|
299
|
-
|
300
|
-
|
301
|
-
|
302
|
-
|
350
|
+
|
351
|
+
w = nil
|
352
|
+
loop do
|
353
|
+
if need_new_worker?
|
354
|
+
nw = deploy_worker
|
355
|
+
nw.req_queue_max.times do
|
356
|
+
@ready_queue.push(nw)
|
357
|
+
end
|
303
358
|
end
|
359
|
+
break if (w = @ready_queue.pop).kind_of?(Array)
|
360
|
+
break if w.req(*req[0], &req[1])
|
304
361
|
end
|
305
|
-
|
306
|
-
end
|
362
|
+
break if w.kind_of?(Array)
|
307
363
|
end
|
308
364
|
end
|
309
365
|
end
|
@@ -331,7 +387,7 @@ module ConcurrentWorker
|
|
331
387
|
|
332
388
|
def need_new_worker?
|
333
389
|
@array_mutex.synchronize do
|
334
|
-
self.size < @max_num && self.select{ |w| w.
|
390
|
+
self.size < @max_num && self.select{ |w| w.req_counter.size == 0 }.empty?
|
335
391
|
end
|
336
392
|
end
|
337
393
|
|
@@ -347,18 +403,19 @@ module ConcurrentWorker
|
|
347
403
|
|
348
404
|
|
349
405
|
def deploy_worker
|
350
|
-
w = Worker.new(*@args, type: @options[:type], &@work_block)
|
406
|
+
w = Worker.new(*@args, type: @options[:type], req_queue_max: @req_queue_max, &@work_block)
|
351
407
|
w.add_callback do |arg|
|
352
408
|
@callback_queue.push([arg])
|
353
409
|
@ready_queue.push(w)
|
354
410
|
end
|
355
411
|
|
356
412
|
w.add_retired_callback do
|
357
|
-
|
358
|
-
|
413
|
+
w.req_counter.rest.each do
|
414
|
+
|req|
|
359
415
|
@req_queue.push(req)
|
360
416
|
end
|
361
417
|
self.delete(w)
|
418
|
+
@ready_queue.push(w)
|
362
419
|
end
|
363
420
|
|
364
421
|
@set_blocks.each do |symbol, block|
|
@@ -374,13 +431,14 @@ module ConcurrentWorker
|
|
374
431
|
end
|
375
432
|
|
376
433
|
def req(*args, &work_block)
|
434
|
+
@req_counter.wait(@max_num * @req_queue_max)
|
435
|
+
@req_counter.push(true)
|
377
436
|
@req_queue.push([args, work_block])
|
378
437
|
end
|
379
438
|
|
380
439
|
def join
|
381
|
-
|
440
|
+
@req_counter.wait(0)
|
382
441
|
self.shift.join until self.empty?
|
383
|
-
puts size
|
384
442
|
@callback_queue.push([])
|
385
443
|
@callback_thread.join
|
386
444
|
@req_queue.push([])
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: concurrent_worker
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- dddogdiamond
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2019-05-
|
11
|
+
date: 2019-05-08 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|