concurrent_worker 0.1.3 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7800e859a8467ccbb2980cff86f3213b4d4c945f4452acc5fa964551003438b0
-  data.tar.gz: eecb1fac2fb3cf9769040ee7de854eeb8203e72a615539f6adc77e636b557f08
+  metadata.gz: 8ae885ab2ba903e2b28fb238efbde6685292d4df97a381b79ad4d99ef3e034a2
+  data.tar.gz: 2640243f4650ed203d36522134620b55fd060ba62e41112d8897d3e8bfafdea2
 SHA512:
-  metadata.gz: c73661e8942dcf02aa5d713cddace082e7fc4eaf75c3d844ca74fa58d0dea93c3c893b4d41065893fb6c85d2f32c93d0f9f30dd892bdba6d04d9bfab76808b0f
-  data.tar.gz: 4fb57e9d3445dc0cf93836ceef3b7d52d4df669e855b18c2a498f77a3565bcc288289cea88945b404b526e147e20a92b5657084c6dc764f64d71497e9c111c2e
+  metadata.gz: 76024b286253b5030420e05ff03afe36a5ac5b6858c2cef1199e328d39f1ae38bfd0e7f40c35452acd9d55b37f1d9958f89030ca1be799552c0a352fff67a50c
+  data.tar.gz: 64bfeeeb41949087c1a7bebb8cc3e316500d005c8364701927ff6dbfd4d4ba210a2e133bc6cf6393d5c518fd191f9bb96a5758b2802f1d48f8d3b23676d1309e

data/.gitignore

@@ -6,3 +6,4 @@
 /pkg/
 /spec/reports/
 /tmp/
+*.backup

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    concurrent_worker (0.1.3)
+    concurrent_worker (0.2.0)
 
 GEM
   remote: https://rubygems.org/

data/README.md

@@ -1,6 +1,6 @@
 # ConcurrentWorker
 
-Concurrent worker in thread/process with preparation structure.
+Concurrent worker in thread/process with preparation mechanism.
 
 ## Installation
 
@@ -54,7 +54,7 @@ end
 
 # define base block for some preparation of work block.
 logger.set_block(:base_block) do
-  open( "log.txt" , "w" ) do |file|
+  open("log.txt", "w") do |file|
     @file = file
     # 'yield_loop_block' must be called in base block.
     # work block will be called in this call.
@@ -72,12 +72,12 @@ You can exec work block in some process concurrently.
 ```ruby
 #define a pool of 8 workers , executed in other process.
 wp = ConcurrentWorker::WorkerPool.new(type: :process, pool_max: 8) do |n|
-  [n , n.times.inject(:+)]
+  [n, n.times.inject(:+)]
 end
 
 # you can receive the result of work block with callback block.
 wp.add_callback do |n, result|
-  logger.log( "n=%d,result=%d", n, result)
+  logger.req("n=%d,result=%d", n, result)
 end
 
 (10000000..10000200).each do |n|

data/concurrent_worker.gemspec

@@ -9,8 +9,8 @@ Gem::Specification.new do |spec|
   spec.authors       = ["dddogdiamond"]
   spec.email         = ["dddogdiamond@gmail.com"]
 
-  spec.summary       = %q{Concurrent worker in thread/process with preparation structure.}
-  spec.description   = %q{Concurrent worker in thread/process with preparation structure.}
+  spec.summary       = %q{Concurrent worker in thread/process with preparation mechanism.}
+  spec.description   = %q{Concurrent worker in thread/process with preparation mechanism.}
   spec.homepage      = "https://github.com/dddogdiamond/concurrent_worker"
   spec.license       = "MIT"
 

data/lib/concurrent_worker/version.rb

@@ -1,3 +1,3 @@
 module ConcurrentWorker
-  VERSION = "0.1.3"
+  VERSION = "0.2.0"
 end

data/lib/concurrent_worker.rb

@@ -9,7 +9,7 @@ module ConcurrentWorker
     # Worker               : worker class
     #  +cncr_block         : concurrent processing block : thread(as ConcurrentThread)/process(as ConcurrentProcess)
     #    +base_block       : user defined preparation to exec 'work block'
-    #      +loop_block     : loop of receiving request and exec 'looped block'
+    #      +loop_block     : loop of receiving request and exec 'work block'
     #        +work_block   : user requested work
     #
     # These blocks are executed with 'instance_exec' method of worker,
@@ -24,29 +24,41 @@ module ConcurrentWorker
       set_block(:work_block, &work_block) if work_block
       
       @state = :idle
-      @callbacks = []
+      @result_callbacks = []
+      @retired_callbacks = []
 
-      @req_queue_max = @options[ :req_queue_max ] || 2
+      @req_queue_max = @options[:req_queue_max] || 2
       @req_queue = SizedQueue.new(@req_queue_max)
       
-      if @options[ :type ] == :process
+      if @options[:type] == :process
         Worker.include ConcurrentProcess
       else
         Worker.include ConcurrentThread
       end
     end
-    
+
     def add_callback(&callback)
       raise "block is nil" unless callback
-      @callbacks.push(callback)
+      @result_callbacks.push(callback)
     end
     
-    def call_callbacks(args)
-      @callbacks.each do |callback|
+    def call_result_callbacks(args)
+      @result_callbacks.each do |callback|
         callback.call(args)
       end
     end
     
+    def add_retired_callback(&callback)
+      raise "block is nil" unless callback
+      @retired_callbacks.push(callback)
+    end
+    
+    def call_retired_callbacks
+      @retired_callbacks.each do |callback|
+        callback.call
+      end
+    end
+    
     def set_block(symbol, &block)
       raise "block is nil" unless block
       
@@ -90,6 +102,7 @@ module ConcurrentWorker
           yield_loop_block
         end
       end
+      
       cncr_block
     end
 
@@ -116,13 +129,24 @@ module ConcurrentWorker
     def cncr_block
       @thread_channel = Queue.new
       @thread =  Thread.new do
-        yield_base_block
+        begin
+          yield_base_block
+        ensure
+          @req_queue.close
+          @thread_channel.close
+          call_retired_callbacks
+        end
       end
     end
     
     def send_req(args)
-      @req_queue.push(args)
-      @thread_channel.push(args)
+      begin
+        @req_queue.push(args)
+        @thread_channel.push(args)
+        true
+      rescue ClosedQueueError
+        false
+      end
     end
     
     def receive_req
@@ -130,7 +154,7 @@ module ConcurrentWorker
     end
 
     def send_res(args)
-      call_callbacks(args)
+      call_result_callbacks(args)
       @req_queue.pop
     end
 
@@ -164,7 +188,9 @@ module ConcurrentWorker
       end
 
       def recv
-        size = @rio.read(4).unpack("I")[0]
+        szdata = @rio.read(4)
+        return [] if szdata.nil?
+        size = szdata.unpack("I")[0]
         Marshal.load(@rio.read(size))
       end
       
@@ -178,25 +204,44 @@ module ConcurrentWorker
       @ipc_channel = IPCDuplexChannel.new
       @c_pid = fork do
         @ipc_channel.choose_io
-        yield_base_block
-        @ipc_channel.send(:worker_loop_finished)
+        begin
+          yield_base_block
+        rescue
+          @ipc_channel.send($!)
+        ensure
+          @ipc_channel.send(:worker_loop_finished)
+          @ipc_channel.close
+        end
       end
       @ipc_channel.choose_io
 
       @thread = Thread.new do
-        loop do
-          result = @ipc_channel.recv
-          break if result == :worker_loop_finished
-          call_callbacks(result)
-          @req_queue.pop
+        begin
+          loop do
+            result = @ipc_channel.recv
+            break if result == :worker_loop_finished
+            raise result if result.kind_of?(Exception)
+
+            call_result_callbacks(result)
+            @req_queue.pop
+          end
+        ensure
+          @ipc_channel.close
+          @req_queue.close
+          call_retired_callbacks
         end
       end
     end
 
     def send_req(args)
-      #called from main process only
-      @req_queue.push(args)
-      @ipc_channel.send(args)
+      begin 
+        #called from main process only
+        @req_queue.push(args)
+        @ipc_channel.send(args)
+        true
+      rescue ClosedQueueError, IOError
+        false
+      end
     end
     
     def receive_req
@@ -210,11 +255,13 @@ module ConcurrentWorker
     end
 
     def wait_cncr_proc
+      $stdout.flush
       begin
         Process.waitpid(@c_pid)
       rescue Errno::ECHILD
       end
       @thread && @thread.join
+      $stdout.flush
     end
   end
 
@@ -225,32 +272,80 @@ module ConcurrentWorker
       @args = args
       
       @options = options
-      @max_num = @options[ :pool_max ] || 8
+      @max_num = @options[:pool_max] || 8
       @set_blocks = []
       if work_block
         @set_blocks.push([:work_block, work_block])
       end
       
+      @array_mutex = Mutex.new
       @ready_queue = Queue.new
       
-      @callbacks = []
+      @result_callbacks = []
       @callback_queue = Queue.new
       @callback_thread = Thread.new do
-        finished_count = 0
         loop do
           break if (result = @callback_queue.pop).empty?
-          @callbacks.each do |callback|
+          @result_callbacks.each do |callback|
             callback.call(result[0])
           end
         end
       end
+
+      @req_queue = SizedQueue.new(@max_num * 2)
+      @req_thread = Thread.new do
+        loop do
+          break if (req = @req_queue.pop).empty?
+          if need_new_worker?
+            puts "new worker".light_yellow
+            w = deploy_worker
+            w.req_queue_max.times do
+              @ready_queue.push(w)
+            end
+          end
+          while !@ready_queue.pop.req(*req[0], &req[1])
+          end
+        end
+      end
+    end
+    
+    def delete(arg)
+      @array_mutex.synchronize do
+        super(arg)
+      end
+    end
+    def push(arg)
+      @array_mutex.synchronize do
+        super(arg)
+      end
+    end
+    def shift
+      @array_mutex.synchronize do
+        super
+      end
+    end
+    def empty?
+      @array_mutex.synchronize do
+        super
+      end
+    end
+    
+    def need_new_worker?
+      @array_mutex.synchronize do
+        self.size < @max_num && self.select{ |w| w.req_queue.size == 0 }.empty?
+      end
     end
     
     def add_callback(&callback)
       raise "block is nil" unless callback
-      @callbacks.push(callback)
+      @result_callbacks.push(callback)
     end
 
+    def add_finished_callback(&callback)
+      raise "block is nil" unless callback
+      @finished_callbacks.push(callback)
+    end
+    
 
     def deploy_worker
       w = Worker.new(*@args, type: @options[:type], &@work_block)
@@ -258,11 +353,20 @@ module ConcurrentWorker
         @callback_queue.push([arg])
         @ready_queue.push(w)
       end
+
+      w.add_retired_callback do
+        while req = w.req_queue.pop
+          next if req == []
+          @req_queue.push(req)
+        end
+        self.delete(w)
+      end
+      
       @set_blocks.each do |symbol, block|
         w.set_block(symbol, &block)
       end
       w.run
-      push w
+      self.push(w)
       w
     end
 
@@ -271,21 +375,17 @@ module ConcurrentWorker
     end
     
     def req(*args, &work_block)
-      if self.size < @max_num && select{ |w| w.req_queue.size == 0 }.empty?
-        w =  deploy_worker
-        w.req_queue_max.times do
-          @ready_queue.push(w)
-        end
-      end
-      w = @ready_queue.pop
-      w.req(*args, &work_block)
+      @req_queue.push([args, work_block])
     end
 
     def join
-      self.map(&:quit)
-      self.map(&:join)
+      puts size
+      self.shift.join until self.empty?
+      puts size
       @callback_queue.push([])
       @callback_thread.join
+      @req_queue.push([])
+      @req_thread.join
     end
   end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: concurrent_worker
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.2.0
 platform: ruby
 authors:
 - dddogdiamond
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-05-04 00:00:00.000000000 Z
+date: 2019-05-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -52,7 +52,7 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '5.0'
-description: Concurrent worker in thread/process with preparation structure.
+description: Concurrent worker in thread/process with preparation mechanism.
 email:
 - dddogdiamond@gmail.com
 executables: []
@@ -93,5 +93,5 @@ requirements: []
 rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
-summary: Concurrent worker in thread/process with preparation structure.
+summary: Concurrent worker in thread/process with preparation mechanism.
 test_files: []