concurrent_worker 0.1.3 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 7800e859a8467ccbb2980cff86f3213b4d4c945f4452acc5fa964551003438b0
4
- data.tar.gz: eecb1fac2fb3cf9769040ee7de854eeb8203e72a615539f6adc77e636b557f08
3
+ metadata.gz: 8ae885ab2ba903e2b28fb238efbde6685292d4df97a381b79ad4d99ef3e034a2
4
+ data.tar.gz: 2640243f4650ed203d36522134620b55fd060ba62e41112d8897d3e8bfafdea2
5
5
  SHA512:
6
- metadata.gz: c73661e8942dcf02aa5d713cddace082e7fc4eaf75c3d844ca74fa58d0dea93c3c893b4d41065893fb6c85d2f32c93d0f9f30dd892bdba6d04d9bfab76808b0f
7
- data.tar.gz: 4fb57e9d3445dc0cf93836ceef3b7d52d4df669e855b18c2a498f77a3565bcc288289cea88945b404b526e147e20a92b5657084c6dc764f64d71497e9c111c2e
6
+ metadata.gz: 76024b286253b5030420e05ff03afe36a5ac5b6858c2cef1199e328d39f1ae38bfd0e7f40c35452acd9d55b37f1d9958f89030ca1be799552c0a352fff67a50c
7
+ data.tar.gz: 64bfeeeb41949087c1a7bebb8cc3e316500d005c8364701927ff6dbfd4d4ba210a2e133bc6cf6393d5c518fd191f9bb96a5758b2802f1d48f8d3b23676d1309e
data/.gitignore CHANGED
@@ -6,3 +6,4 @@
6
6
  /pkg/
7
7
  /spec/reports/
8
8
  /tmp/
9
+ *.backup
data/Gemfile.lock CHANGED
@@ -1,7 +1,7 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- concurrent_worker (0.1.3)
4
+ concurrent_worker (0.2.0)
5
5
 
6
6
  GEM
7
7
  remote: https://rubygems.org/
data/README.md CHANGED
@@ -1,6 +1,6 @@
1
1
  # ConcurrentWorker
2
2
 
3
- Concurrent worker in thread/process with preparation structure.
3
+ Concurrent worker in thread/process with preparation mechanism.
4
4
 
5
5
  ## Installation
6
6
 
@@ -54,7 +54,7 @@ end
54
54
 
55
55
  # define base block for some preparation of work block.
56
56
  logger.set_block(:base_block) do
57
- open( "log.txt" , "w" ) do |file|
57
+ open("log.txt", "w") do |file|
58
58
  @file = file
59
59
  # 'yield_loop_block' must be called in base block.
60
60
  # work block will be called in this call.
@@ -72,12 +72,12 @@ You can exec work block in some process concurrently.
72
72
  ```ruby
73
73
  #define a pool of 8 workers , executed in other process.
74
74
  wp = ConcurrentWorker::WorkerPool.new(type: :process, pool_max: 8) do |n|
75
- [n , n.times.inject(:+)]
75
+ [n, n.times.inject(:+)]
76
76
  end
77
77
 
78
78
  # you can receive the result of work block with callback block.
79
79
  wp.add_callback do |n, result|
80
- logger.log( "n=%d,result=%d", n, result)
80
+ logger.req("n=%d,result=%d", n, result)
81
81
  end
82
82
 
83
83
  (10000000..10000200).each do |n|
@@ -9,8 +9,8 @@ Gem::Specification.new do |spec|
9
9
  spec.authors = ["dddogdiamond"]
10
10
  spec.email = ["dddogdiamond@gmail.com"]
11
11
 
12
- spec.summary = %q{Concurrent worker in thread/process with preparation structure.}
13
- spec.description = %q{Concurrent worker in thread/process with preparation structure.}
12
+ spec.summary = %q{Concurrent worker in thread/process with preparation mechanism.}
13
+ spec.description = %q{Concurrent worker in thread/process with preparation mechanism.}
14
14
  spec.homepage = "https://github.com/dddogdiamond/concurrent_worker"
15
15
  spec.license = "MIT"
16
16
 
@@ -1,3 +1,3 @@
1
1
  module ConcurrentWorker
2
- VERSION = "0.1.3"
2
+ VERSION = "0.2.0"
3
3
  end
@@ -9,7 +9,7 @@ module ConcurrentWorker
9
9
  # Worker : worker class
10
10
  # +cncr_block : concurrent processing block : thread(as ConcurrentThread)/process(as ConcurrentProcess)
11
11
  # +base_block : user defined preparation to exec 'work block'
12
- # +loop_block : loop of receiving request and exec 'looped block'
12
+ # +loop_block : loop of receiving request and exec 'work block'
13
13
  # +work_block : user requested work
14
14
  #
15
15
  # These blocks are executed with 'instance_exec' method of worker,
@@ -24,29 +24,41 @@ module ConcurrentWorker
24
24
  set_block(:work_block, &work_block) if work_block
25
25
 
26
26
  @state = :idle
27
- @callbacks = []
27
+ @result_callbacks = []
28
+ @retired_callbacks = []
28
29
 
29
- @req_queue_max = @options[ :req_queue_max ] || 2
30
+ @req_queue_max = @options[:req_queue_max] || 2
30
31
  @req_queue = SizedQueue.new(@req_queue_max)
31
32
 
32
- if @options[ :type ] == :process
33
+ if @options[:type] == :process
33
34
  Worker.include ConcurrentProcess
34
35
  else
35
36
  Worker.include ConcurrentThread
36
37
  end
37
38
  end
38
-
39
+
39
40
  def add_callback(&callback)
40
41
  raise "block is nil" unless callback
41
- @callbacks.push(callback)
42
+ @result_callbacks.push(callback)
42
43
  end
43
44
 
44
- def call_callbacks(args)
45
- @callbacks.each do |callback|
45
+ def call_result_callbacks(args)
46
+ @result_callbacks.each do |callback|
46
47
  callback.call(args)
47
48
  end
48
49
  end
49
50
 
51
+ def add_retired_callback(&callback)
52
+ raise "block is nil" unless callback
53
+ @retired_callbacks.push(callback)
54
+ end
55
+
56
+ def call_retired_callbacks
57
+ @retired_callbacks.each do |callback|
58
+ callback.call
59
+ end
60
+ end
61
+
50
62
  def set_block(symbol, &block)
51
63
  raise "block is nil" unless block
52
64
 
@@ -90,6 +102,7 @@ module ConcurrentWorker
90
102
  yield_loop_block
91
103
  end
92
104
  end
105
+
93
106
  cncr_block
94
107
  end
95
108
 
@@ -116,13 +129,24 @@ module ConcurrentWorker
116
129
  def cncr_block
117
130
  @thread_channel = Queue.new
118
131
  @thread = Thread.new do
119
- yield_base_block
132
+ begin
133
+ yield_base_block
134
+ ensure
135
+ @req_queue.close
136
+ @thread_channel.close
137
+ call_retired_callbacks
138
+ end
120
139
  end
121
140
  end
122
141
 
123
142
  def send_req(args)
124
- @req_queue.push(args)
125
- @thread_channel.push(args)
143
+ begin
144
+ @req_queue.push(args)
145
+ @thread_channel.push(args)
146
+ true
147
+ rescue ClosedQueueError
148
+ false
149
+ end
126
150
  end
127
151
 
128
152
  def receive_req
@@ -130,7 +154,7 @@ module ConcurrentWorker
130
154
  end
131
155
 
132
156
  def send_res(args)
133
- call_callbacks(args)
157
+ call_result_callbacks(args)
134
158
  @req_queue.pop
135
159
  end
136
160
 
@@ -164,7 +188,9 @@ module ConcurrentWorker
164
188
  end
165
189
 
166
190
  def recv
167
- size = @rio.read(4).unpack("I")[0]
191
+ szdata = @rio.read(4)
192
+ return [] if szdata.nil?
193
+ size = szdata.unpack("I")[0]
168
194
  Marshal.load(@rio.read(size))
169
195
  end
170
196
 
@@ -178,25 +204,44 @@ module ConcurrentWorker
178
204
  @ipc_channel = IPCDuplexChannel.new
179
205
  @c_pid = fork do
180
206
  @ipc_channel.choose_io
181
- yield_base_block
182
- @ipc_channel.send(:worker_loop_finished)
207
+ begin
208
+ yield_base_block
209
+ rescue
210
+ @ipc_channel.send($!)
211
+ ensure
212
+ @ipc_channel.send(:worker_loop_finished)
213
+ @ipc_channel.close
214
+ end
183
215
  end
184
216
  @ipc_channel.choose_io
185
217
 
186
218
  @thread = Thread.new do
187
- loop do
188
- result = @ipc_channel.recv
189
- break if result == :worker_loop_finished
190
- call_callbacks(result)
191
- @req_queue.pop
219
+ begin
220
+ loop do
221
+ result = @ipc_channel.recv
222
+ break if result == :worker_loop_finished
223
+ raise result if result.kind_of?(Exception)
224
+
225
+ call_result_callbacks(result)
226
+ @req_queue.pop
227
+ end
228
+ ensure
229
+ @ipc_channel.close
230
+ @req_queue.close
231
+ call_retired_callbacks
192
232
  end
193
233
  end
194
234
  end
195
235
 
196
236
  def send_req(args)
197
- #called from main process only
198
- @req_queue.push(args)
199
- @ipc_channel.send(args)
237
+ begin
238
+ #called from main process only
239
+ @req_queue.push(args)
240
+ @ipc_channel.send(args)
241
+ true
242
+ rescue ClosedQueueError, IOError
243
+ false
244
+ end
200
245
  end
201
246
 
202
247
  def receive_req
@@ -210,11 +255,13 @@ module ConcurrentWorker
210
255
  end
211
256
 
212
257
  def wait_cncr_proc
258
+ $stdout.flush
213
259
  begin
214
260
  Process.waitpid(@c_pid)
215
261
  rescue Errno::ECHILD
216
262
  end
217
263
  @thread && @thread.join
264
+ $stdout.flush
218
265
  end
219
266
  end
220
267
 
@@ -225,32 +272,80 @@ module ConcurrentWorker
225
272
  @args = args
226
273
 
227
274
  @options = options
228
- @max_num = @options[ :pool_max ] || 8
275
+ @max_num = @options[:pool_max] || 8
229
276
  @set_blocks = []
230
277
  if work_block
231
278
  @set_blocks.push([:work_block, work_block])
232
279
  end
233
280
 
281
+ @array_mutex = Mutex.new
234
282
  @ready_queue = Queue.new
235
283
 
236
- @callbacks = []
284
+ @result_callbacks = []
237
285
  @callback_queue = Queue.new
238
286
  @callback_thread = Thread.new do
239
- finished_count = 0
240
287
  loop do
241
288
  break if (result = @callback_queue.pop).empty?
242
- @callbacks.each do |callback|
289
+ @result_callbacks.each do |callback|
243
290
  callback.call(result[0])
244
291
  end
245
292
  end
246
293
  end
294
+
295
+ @req_queue = SizedQueue.new(@max_num * 2)
296
+ @req_thread = Thread.new do
297
+ loop do
298
+ break if (req = @req_queue.pop).empty?
299
+ if need_new_worker?
300
+ puts "new worker".light_yellow
301
+ w = deploy_worker
302
+ w.req_queue_max.times do
303
+ @ready_queue.push(w)
304
+ end
305
+ end
306
+ while !@ready_queue.pop.req(*req[0], &req[1])
307
+ end
308
+ end
309
+ end
310
+ end
311
+
312
+ def delete(arg)
313
+ @array_mutex.synchronize do
314
+ super(arg)
315
+ end
316
+ end
317
+ def push(arg)
318
+ @array_mutex.synchronize do
319
+ super(arg)
320
+ end
321
+ end
322
+ def shift
323
+ @array_mutex.synchronize do
324
+ super
325
+ end
326
+ end
327
+ def empty?
328
+ @array_mutex.synchronize do
329
+ super
330
+ end
331
+ end
332
+
333
+ def need_new_worker?
334
+ @array_mutex.synchronize do
335
+ self.size < @max_num && self.select{ |w| w.req_queue.size == 0 }.empty?
336
+ end
247
337
  end
248
338
 
249
339
  def add_callback(&callback)
250
340
  raise "block is nil" unless callback
251
- @callbacks.push(callback)
341
+ @result_callbacks.push(callback)
252
342
  end
253
343
 
344
+ def add_finished_callback(&callback)
345
+ raise "block is nil" unless callback
346
+ @finished_callbacks.push(callback)
347
+ end
348
+
254
349
 
255
350
  def deploy_worker
256
351
  w = Worker.new(*@args, type: @options[:type], &@work_block)
@@ -258,11 +353,20 @@ module ConcurrentWorker
258
353
  @callback_queue.push([arg])
259
354
  @ready_queue.push(w)
260
355
  end
356
+
357
+ w.add_retired_callback do
358
+ while req = w.req_queue.pop
359
+ next if req == []
360
+ @req_queue.push(req)
361
+ end
362
+ self.delete(w)
363
+ end
364
+
261
365
  @set_blocks.each do |symbol, block|
262
366
  w.set_block(symbol, &block)
263
367
  end
264
368
  w.run
265
- push w
369
+ self.push(w)
266
370
  w
267
371
  end
268
372
 
@@ -271,21 +375,17 @@ module ConcurrentWorker
271
375
  end
272
376
 
273
377
  def req(*args, &work_block)
274
- if self.size < @max_num && select{ |w| w.req_queue.size == 0 }.empty?
275
- w = deploy_worker
276
- w.req_queue_max.times do
277
- @ready_queue.push(w)
278
- end
279
- end
280
- w = @ready_queue.pop
281
- w.req(*args, &work_block)
378
+ @req_queue.push([args, work_block])
282
379
  end
283
380
 
284
381
  def join
285
- self.map(&:quit)
286
- self.map(&:join)
382
+ puts size
383
+ self.shift.join until self.empty?
384
+ puts size
287
385
  @callback_queue.push([])
288
386
  @callback_thread.join
387
+ @req_queue.push([])
388
+ @req_thread.join
289
389
  end
290
390
  end
291
391
 
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: concurrent_worker
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.3
4
+ version: 0.2.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - dddogdiamond
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2019-05-04 00:00:00.000000000 Z
11
+ date: 2019-05-06 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
@@ -52,7 +52,7 @@ dependencies:
52
52
  - - "~>"
53
53
  - !ruby/object:Gem::Version
54
54
  version: '5.0'
55
- description: Concurrent worker in thread/process with preparation structure.
55
+ description: Concurrent worker in thread/process with preparation mechanism.
56
56
  email:
57
57
  - dddogdiamond@gmail.com
58
58
  executables: []
@@ -93,5 +93,5 @@ requirements: []
93
93
  rubygems_version: 3.0.3
94
94
  signing_key:
95
95
  specification_version: 4
96
- summary: Concurrent worker in thread/process with preparation structure.
96
+ summary: Concurrent worker in thread/process with preparation mechanism.
97
97
  test_files: []