concurrent_worker 0.2.1 → 0.2.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 617af218b5ac0e681ee6da7da4cb6ad250ddecaf6c28628891d226dbb1b04bab
4
- data.tar.gz: 70fb8442b6c29fe58bb345da7ab74447fea81daecadcfed2bdf6b4ab7fc5e39b
3
+ metadata.gz: 07268e6a9729fb2b470926764e1975ee7305c5fb9689412436b3d00c0dbcd30b
4
+ data.tar.gz: e121b8aa8c4aacec697282f6efd07dbc0b8e968ec22d79d2e092bf173ea29dc7
5
5
  SHA512:
6
- metadata.gz: 84994c8eeab9f8287afd9dd30dddd3f068221d7387723be4239ace0290b9bfac25b7e885b44239a39937bcf04fc3df886d8bbddb1879974d5db899f135807a89
7
- data.tar.gz: 1937fd3f456f8dc9747953e34d4cd044fc100ce29ccf991ca92b8332dcb5e9970d1f3d6ea440816803205cb8a08f6257ae0cf5cd534797cb10014198c3d527eb
6
+ metadata.gz: a26ace3f114dcefbfac75c354237b9180ee6f0b0f2d09b77eee265af845d3a1e027c7792c306e68a9b8c1e6b13599117d4c491aff8b6f1cb51f26debae015cec
7
+ data.tar.gz: e2b7e20256f2fb68c18915132027df1ef002babf65bbda4da3495b07c289ad5f2c6d76c0c829245316d89c13804c3deded25c5144f6ba5ca52356b12a1f7c4ec
data/Gemfile.lock CHANGED
@@ -1,7 +1,7 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- concurrent_worker (0.2.1)
4
+ concurrent_worker (0.2.2)
5
5
 
6
6
  GEM
7
7
  remote: https://rubygems.org/
data/README.md CHANGED
@@ -22,7 +22,7 @@ Or install it yourself as:
22
22
 
23
23
  ### Worker
24
24
 
25
- You can define worker block executed in other thread, and send requests to the worker.
25
+ You can define a worker setting a work block executed in other thread, and send requests to the worker.
26
26
 
27
27
  ```ruby
28
28
  require 'concurrent_worker'
@@ -42,11 +42,11 @@ logger.req("thread%d n=%d\n",0, 1)
42
42
  logger.join
43
43
  ```
44
44
 
45
- If you need some preparation for the worker block, you can define 'base block'.
45
+ If you need some preparation for the work block, you can define 'base block'.
46
46
 
47
47
  ```ruby
48
48
  logger = ConcurrentWorker::Worker.new do |args|
49
- # worker block and base block can share object with instance variable(@xxx).
49
+ # work block and base block can share object with instance variable(@xxx).
50
50
  printf(@file, *args)
51
51
  @file.flush
52
52
  nil
@@ -70,7 +70,7 @@ The work block and base block are executed in a same thread, and in a worker's i
70
70
  You can exec work block in some process concurrently.
71
71
 
72
72
  ```ruby
73
- #define a pool of 8 workers , executed in other process.
73
+ #define a pool of 8 workers with same work block, executed in other process.
74
74
  wp = ConcurrentWorker::WorkerPool.new(type: :process, pool_max: 8) do |n|
75
75
  [n, n.times.inject(:+)]
76
76
  end
@@ -1,3 +1,3 @@
1
1
  module ConcurrentWorker
2
- VERSION = "0.2.1"
2
+ VERSION = "0.2.2"
3
3
  end
@@ -3,6 +3,48 @@ require "concurrent_worker/version"
3
3
  module ConcurrentWorker
4
4
  class Error < StandardError; end
5
5
 
6
+
7
+ class RequestCounter
8
+ def initialize
9
+ @count = Queue.new
10
+ @com = Queue.new
11
+ end
12
+ def push(args)
13
+ @count.push(args)
14
+ end
15
+
16
+ def pop
17
+ @count.pop
18
+ @com.push([])
19
+ end
20
+ def wait(n)
21
+ return if @count.size <= n
22
+ while @com.pop
23
+ break if @count.size <= n
24
+ end
25
+ end
26
+ def size
27
+ @count.size
28
+ end
29
+
30
+ def close
31
+ @count.close
32
+ end
33
+
34
+ def rest
35
+ result = []
36
+ until @count.empty?
37
+ req = @count.pop
38
+ next if req == []
39
+ result.push(req)
40
+ end
41
+ result
42
+ end
43
+
44
+ end
45
+
46
+
47
+
6
48
  require 'thread'
7
49
  class Worker
8
50
  attr_accessor :channel
@@ -16,7 +58,7 @@ module ConcurrentWorker
16
58
  # so that they can share instance variables:@xxxx.
17
59
  #
18
60
 
19
- attr_reader :req_queue, :req_queue_max
61
+ attr_reader :req_counter, :req_queue_max
20
62
 
21
63
  def initialize(*args, **options, &work_block)
22
64
  @args = args
@@ -28,7 +70,7 @@ module ConcurrentWorker
28
70
  @retired_callbacks = []
29
71
 
30
72
  @req_queue_max = @options[:req_queue_max] || 2
31
- @req_queue = SizedQueue.new(@req_queue_max)
73
+ @req_counter = RequestCounter.new
32
74
 
33
75
  if @options[:type] == :process
34
76
  Worker.include ConcurrentProcess
@@ -111,6 +153,7 @@ module ConcurrentWorker
111
153
  unless @state == :run
112
154
  run
113
155
  end
156
+ @req_counter.wait(@req_queue_max)
114
157
  send_req([args, work_block])
115
158
  end
116
159
 
@@ -119,6 +162,7 @@ module ConcurrentWorker
119
162
  end
120
163
 
121
164
  def join
165
+ @req_counter.wait(0)
122
166
  quit
123
167
  wait_cncr_proc
124
168
  end
@@ -132,7 +176,7 @@ module ConcurrentWorker
132
176
  begin
133
177
  yield_base_block
134
178
  ensure
135
- @req_queue.close
179
+ @req_counter.close
136
180
  @thread_channel.close
137
181
  call_retired_callbacks
138
182
  end
@@ -141,7 +185,7 @@ module ConcurrentWorker
141
185
 
142
186
  def send_req(args)
143
187
  begin
144
- @req_queue.push(args)
188
+ @req_counter.push(args)
145
189
  @thread_channel.push(args)
146
190
  true
147
191
  rescue ClosedQueueError
@@ -155,7 +199,7 @@ module ConcurrentWorker
155
199
 
156
200
  def send_res(args)
157
201
  call_result_callbacks(args)
158
- @req_queue.pop
202
+ @req_counter.pop
159
203
  end
160
204
 
161
205
  def wait_cncr_proc
@@ -188,10 +232,14 @@ module ConcurrentWorker
188
232
  end
189
233
 
190
234
  def recv
191
- szdata = @rio.read(4)
192
- return [] if szdata.nil?
193
- size = szdata.unpack("I")[0]
194
- Marshal.load(@rio.read(size))
235
+ begin
236
+ szdata = @rio.read(4)
237
+ return [] if szdata.nil?
238
+ size = szdata.unpack("I")[0]
239
+ Marshal.load(@rio.read(size))
240
+ rescue IOError
241
+ raise StopIteration
242
+ end
195
243
  end
196
244
 
197
245
  def close
@@ -210,7 +258,6 @@ module ConcurrentWorker
210
258
  @ipc_channel.send($!)
211
259
  ensure
212
260
  @ipc_channel.send(:worker_loop_finished)
213
- @ipc_channel.close
214
261
  end
215
262
  end
216
263
  @ipc_channel.choose_io
@@ -223,11 +270,14 @@ module ConcurrentWorker
223
270
  raise result if result.kind_of?(Exception)
224
271
 
225
272
  call_result_callbacks(result)
226
- @req_queue.pop
273
+ @req_counter.pop
227
274
  end
275
+ rescue
276
+ Thread.pass
277
+ raise $!
228
278
  ensure
229
279
  @ipc_channel.close
230
- @req_queue.close
280
+ @req_counter.close
231
281
  call_retired_callbacks
232
282
  end
233
283
  end
@@ -236,7 +286,7 @@ module ConcurrentWorker
236
286
  def send_req(args)
237
287
  begin
238
288
  #called from main process only
239
- @req_queue.push(args)
289
+ @req_counter.push(args)
240
290
  @ipc_channel.send(args)
241
291
  true
242
292
  rescue ClosedQueueError, IOError
@@ -255,13 +305,11 @@ module ConcurrentWorker
255
305
  end
256
306
 
257
307
  def wait_cncr_proc
258
- $stdout.flush
259
308
  begin
260
309
  Process.waitpid(@c_pid)
261
310
  rescue Errno::ECHILD
262
311
  end
263
312
  @thread && @thread.join
264
- $stdout.flush
265
313
  end
266
314
  end
267
315
 
@@ -277,7 +325,7 @@ module ConcurrentWorker
277
325
  if work_block
278
326
  @set_blocks.push([:work_block, work_block])
279
327
  end
280
-
328
+
281
329
  @array_mutex = Mutex.new
282
330
  @ready_queue = Queue.new
283
331
 
@@ -289,21 +337,29 @@ module ConcurrentWorker
289
337
  @result_callbacks.each do |callback|
290
338
  callback.call(result[0])
291
339
  end
340
+ @req_counter.pop
292
341
  end
293
342
  end
343
+ @req_queue_max = @options[:req_queue_max]||2
294
344
 
295
- @req_queue = SizedQueue.new(@max_num * 2)
345
+ @req_queue = Queue.new
346
+ @req_counter = RequestCounter.new
296
347
  @req_thread = Thread.new do
297
348
  loop do
298
349
  break if (req = @req_queue.pop).empty?
299
- if need_new_worker?
300
- w = deploy_worker
301
- w.req_queue_max.times do
302
- @ready_queue.push(w)
350
+
351
+ w = nil
352
+ loop do
353
+ if need_new_worker?
354
+ nw = deploy_worker
355
+ nw.req_queue_max.times do
356
+ @ready_queue.push(nw)
357
+ end
303
358
  end
359
+ break if (w = @ready_queue.pop).kind_of?(Array)
360
+ break if w.req(*req[0], &req[1])
304
361
  end
305
- while !@ready_queue.pop.req(*req[0], &req[1])
306
- end
362
+ break if w.kind_of?(Array)
307
363
  end
308
364
  end
309
365
  end
@@ -331,7 +387,7 @@ module ConcurrentWorker
331
387
 
332
388
  def need_new_worker?
333
389
  @array_mutex.synchronize do
334
- self.size < @max_num && self.select{ |w| w.req_queue.size == 0 }.empty?
390
+ self.size < @max_num && self.select{ |w| w.req_counter.size == 0 }.empty?
335
391
  end
336
392
  end
337
393
 
@@ -347,18 +403,19 @@ module ConcurrentWorker
347
403
 
348
404
 
349
405
  def deploy_worker
350
- w = Worker.new(*@args, type: @options[:type], &@work_block)
406
+ w = Worker.new(*@args, type: @options[:type], req_queue_max: @req_queue_max, &@work_block)
351
407
  w.add_callback do |arg|
352
408
  @callback_queue.push([arg])
353
409
  @ready_queue.push(w)
354
410
  end
355
411
 
356
412
  w.add_retired_callback do
357
- while req = w.req_queue.pop
358
- next if req == []
413
+ w.req_counter.rest.each do
414
+ |req|
359
415
  @req_queue.push(req)
360
416
  end
361
417
  self.delete(w)
418
+ @ready_queue.push(w)
362
419
  end
363
420
 
364
421
  @set_blocks.each do |symbol, block|
@@ -374,13 +431,14 @@ module ConcurrentWorker
374
431
  end
375
432
 
376
433
  def req(*args, &work_block)
434
+ @req_counter.wait(@max_num * @req_queue_max)
435
+ @req_counter.push(true)
377
436
  @req_queue.push([args, work_block])
378
437
  end
379
438
 
380
439
  def join
381
- puts size
440
+ @req_counter.wait(0)
382
441
  self.shift.join until self.empty?
383
- puts size
384
442
  @callback_queue.push([])
385
443
  @callback_thread.join
386
444
  @req_queue.push([])
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: concurrent_worker
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.1
4
+ version: 0.2.2
5
5
  platform: ruby
6
6
  authors:
7
7
  - dddogdiamond
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2019-05-06 00:00:00.000000000 Z
11
+ date: 2019-05-08 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler