concurrent_worker 0.2.1 → 0.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 617af218b5ac0e681ee6da7da4cb6ad250ddecaf6c28628891d226dbb1b04bab
-  data.tar.gz: 70fb8442b6c29fe58bb345da7ab74447fea81daecadcfed2bdf6b4ab7fc5e39b
+  metadata.gz: 07268e6a9729fb2b470926764e1975ee7305c5fb9689412436b3d00c0dbcd30b
+  data.tar.gz: e121b8aa8c4aacec697282f6efd07dbc0b8e968ec22d79d2e092bf173ea29dc7
 SHA512:
-  metadata.gz: 84994c8eeab9f8287afd9dd30dddd3f068221d7387723be4239ace0290b9bfac25b7e885b44239a39937bcf04fc3df886d8bbddb1879974d5db899f135807a89
-  data.tar.gz: 1937fd3f456f8dc9747953e34d4cd044fc100ce29ccf991ca92b8332dcb5e9970d1f3d6ea440816803205cb8a08f6257ae0cf5cd534797cb10014198c3d527eb
+  metadata.gz: a26ace3f114dcefbfac75c354237b9180ee6f0b0f2d09b77eee265af845d3a1e027c7792c306e68a9b8c1e6b13599117d4c491aff8b6f1cb51f26debae015cec
+  data.tar.gz: e2b7e20256f2fb68c18915132027df1ef002babf65bbda4da3495b07c289ad5f2c6d76c0c829245316d89c13804c3deded25c5144f6ba5ca52356b12a1f7c4ec

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    concurrent_worker (0.2.1)
+    concurrent_worker (0.2.2)
 
 GEM
   remote: https://rubygems.org/

data/README.md

@@ -22,7 +22,7 @@ Or install it yourself as:
 
 ### Worker
 
-You can define worker block executed in other thread, and send requests to the worker.
+You can define a worker setting a work block executed in other thread, and send requests to the worker.
 
 ```ruby
 require 'concurrent_worker'
@@ -42,11 +42,11 @@ logger.req("thread%d n=%d\n",0, 1)
 logger.join
 ```
 
-If you need some preparation for the worker block, you can define 'base block'.
+If you need some preparation for the work block, you can define 'base block'.
 
 ```ruby
 logger = ConcurrentWorker::Worker.new do |args|
-  # worker block and base block can share object with instance variable(@xxx).
+  # work block and base block can share object with instance variable(@xxx).
   printf(@file, *args)
   @file.flush
   nil
@@ -70,7 +70,7 @@ The work block and base block are executed in a same thread, and in a worker's i
 You can exec work block in some process concurrently.
 
 ```ruby
-#define a pool of 8 workers , executed in other process.
+#define a pool of 8 workers with same work block, executed in other process.
 wp = ConcurrentWorker::WorkerPool.new(type: :process, pool_max: 8) do |n|
   [n, n.times.inject(:+)]
 end

data/lib/concurrent_worker/version.rb

@@ -1,3 +1,3 @@
 module ConcurrentWorker
-  VERSION = "0.2.1"
+  VERSION = "0.2.2"
 end

data/lib/concurrent_worker.rb

@@ -3,6 +3,48 @@ require "concurrent_worker/version"
 module ConcurrentWorker
   class Error < StandardError; end
 
+
+  class RequestCounter
+    def initialize
+      @count = Queue.new
+      @com = Queue.new
+    end
+    def push(args)
+      @count.push(args)
+    end
+    
+    def pop
+      @count.pop
+      @com.push([])
+    end
+    def wait(n)
+      return if @count.size <= n
+      while @com.pop
+        break if @count.size <= n
+      end
+    end
+    def size
+      @count.size
+    end
+
+    def close
+      @count.close
+    end
+
+    def rest
+      result = []
+      until @count.empty?
+        req = @count.pop
+        next if req == []
+        result.push(req)
+      end
+      result
+    end
+    
+  end
+
+  
+
   require 'thread'
   class Worker
     attr_accessor :channel
@@ -16,7 +58,7 @@ module ConcurrentWorker
     # so that they can share instance variables:@xxxx.
     #
 
-    attr_reader :req_queue, :req_queue_max
+    attr_reader :req_counter, :req_queue_max
     
     def initialize(*args, **options, &work_block)
       @args = args
@@ -28,7 +70,7 @@ module ConcurrentWorker
       @retired_callbacks = []
 
       @req_queue_max = @options[:req_queue_max] || 2
-      @req_queue = SizedQueue.new(@req_queue_max)
+      @req_counter = RequestCounter.new
       
       if @options[:type] == :process
         Worker.include ConcurrentProcess
@@ -111,6 +153,7 @@ module ConcurrentWorker
       unless @state == :run
         run
       end
+      @req_counter.wait(@req_queue_max)
       send_req([args, work_block])
     end
     
@@ -119,6 +162,7 @@ module ConcurrentWorker
     end
     
     def join
+      @req_counter.wait(0)
       quit
       wait_cncr_proc
     end
@@ -132,7 +176,7 @@ module ConcurrentWorker
         begin
           yield_base_block
         ensure
-          @req_queue.close
+          @req_counter.close
           @thread_channel.close
           call_retired_callbacks
         end
@@ -141,7 +185,7 @@ module ConcurrentWorker
     
     def send_req(args)
       begin
-        @req_queue.push(args)
+        @req_counter.push(args)
         @thread_channel.push(args)
         true
       rescue ClosedQueueError
@@ -155,7 +199,7 @@ module ConcurrentWorker
 
     def send_res(args)
       call_result_callbacks(args)
-      @req_queue.pop
+      @req_counter.pop
     end
 
     def wait_cncr_proc
@@ -188,10 +232,14 @@ module ConcurrentWorker
       end
 
       def recv
-        szdata = @rio.read(4)
-        return [] if szdata.nil?
-        size = szdata.unpack("I")[0]
-        Marshal.load(@rio.read(size))
+        begin
+          szdata = @rio.read(4)
+          return [] if szdata.nil?
+          size = szdata.unpack("I")[0]
+          Marshal.load(@rio.read(size))
+        rescue IOError
+          raise StopIteration
+        end
       end
       
       def close
@@ -210,7 +258,6 @@ module ConcurrentWorker
           @ipc_channel.send($!)
         ensure
           @ipc_channel.send(:worker_loop_finished)
-          @ipc_channel.close
         end
       end
       @ipc_channel.choose_io
@@ -223,11 +270,14 @@ module ConcurrentWorker
             raise result if result.kind_of?(Exception)
 
             call_result_callbacks(result)
-            @req_queue.pop
+            @req_counter.pop
           end
+        rescue
+          Thread.pass
+          raise $!
         ensure
           @ipc_channel.close
-          @req_queue.close
+          @req_counter.close
           call_retired_callbacks
         end
       end
@@ -236,7 +286,7 @@ module ConcurrentWorker
     def send_req(args)
       begin 
         #called from main process only
-        @req_queue.push(args)
+        @req_counter.push(args)
         @ipc_channel.send(args)
         true
       rescue ClosedQueueError, IOError
@@ -255,13 +305,11 @@ module ConcurrentWorker
     end
 
     def wait_cncr_proc
-      $stdout.flush
       begin
         Process.waitpid(@c_pid)
       rescue Errno::ECHILD
       end
       @thread && @thread.join
-      $stdout.flush
     end
   end
 
@@ -277,7 +325,7 @@ module ConcurrentWorker
       if work_block
         @set_blocks.push([:work_block, work_block])
       end
-      
+
       @array_mutex = Mutex.new
       @ready_queue = Queue.new
       
@@ -289,21 +337,29 @@ module ConcurrentWorker
           @result_callbacks.each do |callback|
             callback.call(result[0])
           end
+          @req_counter.pop
         end
       end
+      @req_queue_max = @options[:req_queue_max]||2
 
-      @req_queue = SizedQueue.new(@max_num * 2)
+      @req_queue = Queue.new
+      @req_counter = RequestCounter.new
       @req_thread = Thread.new do
         loop do
           break if (req = @req_queue.pop).empty?
-          if need_new_worker?
-            w = deploy_worker
-            w.req_queue_max.times do
-              @ready_queue.push(w)
+          
+          w = nil
+          loop do
+            if need_new_worker?
+              nw = deploy_worker
+              nw.req_queue_max.times do
+                @ready_queue.push(nw)
+              end
             end
+            break if (w = @ready_queue.pop).kind_of?(Array)
+            break if w.req(*req[0], &req[1])
           end
-          while !@ready_queue.pop.req(*req[0], &req[1])
-          end
+          break if w.kind_of?(Array)
         end
       end
     end
@@ -331,7 +387,7 @@ module ConcurrentWorker
     
     def need_new_worker?
       @array_mutex.synchronize do
-        self.size < @max_num && self.select{ |w| w.req_queue.size == 0 }.empty?
+        self.size < @max_num && self.select{ |w| w.req_counter.size == 0 }.empty?
       end
     end
     
@@ -347,18 +403,19 @@ module ConcurrentWorker
     
 
     def deploy_worker
-      w = Worker.new(*@args, type: @options[:type], &@work_block)
+      w = Worker.new(*@args, type: @options[:type], req_queue_max: @req_queue_max, &@work_block)
       w.add_callback do |arg|
         @callback_queue.push([arg])
         @ready_queue.push(w)
       end
 
       w.add_retired_callback do
-        while req = w.req_queue.pop
-          next if req == []
+        w.req_counter.rest.each do
+          |req|
           @req_queue.push(req)
         end
         self.delete(w)
+        @ready_queue.push(w)
       end
       
       @set_blocks.each do |symbol, block|
@@ -374,13 +431,14 @@ module ConcurrentWorker
     end
     
     def req(*args, &work_block)
+      @req_counter.wait(@max_num * @req_queue_max)
+      @req_counter.push(true)
       @req_queue.push([args, work_block])
     end
 
     def join
-      puts size
+      @req_counter.wait(0)
       self.shift.join until self.empty?
-      puts size
       @callback_queue.push([])
       @callback_thread.join
       @req_queue.push([])

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: concurrent_worker
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.2.2
 platform: ruby
 authors:
 - dddogdiamond
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-05-06 00:00:00.000000000 Z
+date: 2019-05-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler