in-parallel 0.1.8 → 0.1.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: fde05adab05cca1454b0c4f9187cddc062f32fe9
-  data.tar.gz: 12ae2280dcfd30cc7875a863c05abeff93e9eade
+  metadata.gz: a6277b0e376bda7d26d7f16760f16764569bdf01
+  data.tar.gz: f0798ecc56cd7e02cd1784fbc28f4b19a3e76118
 SHA512:
-  metadata.gz: fe93efe91c47632f57062d29f903d88b723a15f4333cf7bcdd7c8802ff7712481c37a52a70f92f736ac10662e2bee393a5e52ca6f9003603babb7c9273a026d5
-  data.tar.gz: 37bcdff90122259424d7e2523f9a33cf2bbe56755bfd277802a2a5b386d73aef850b2c07422b5d4286c165bed8208015289db4f51ba4a6a32b534ab425f51203
+  metadata.gz: b3cb0370b0254ce1a801b32d7fcab2bb80acfeec6d0253a8752560c1da8ddf53de94e8a71d3e61369d204be44617451d26f909126ee484b61d0302c1b47d377d
+  data.tar.gz: 2f769c82a0fb813eb3ebfbeb0f09a8fe1a00086ea1e0220bca5913c1c47d5a617d625070627d9e7a619fdb42aeb9a370c9a6aedb53b333111fee2d47840644cc

data/in_parallel/version.rb

@@ -1,3 +1,3 @@
 module InParallel
-  VERSION = Version = '0.1.8'
+  VERSION = Version = '0.1.9'
 end

data/lib/in_parallel.rb

@@ -4,8 +4,8 @@ module InParallel
     # How many seconds between outputting to stdout that we are waiting for child processes.
     # 0 or < 0 means no signaling.
     @@signal_interval = 30
+    @@timeout = 1800
     @@process_infos = []
-    @@raise_error = nil
     def self.process_infos
       @@process_infos
     end
@@ -21,42 +21,30 @@ module InParallel
       @@main_pid
     end
 
+    def self.timeout
+      @@timeout
+    end
+
+    def self.timeout=(value)
+      @@timeout = value
+    end
+
     # Example - will spawn 2 processes, (1 for each method) wait until they both complete, and log STDOUT:
     # InParallel.run_in_parallel {
     #   @result_1 = method1
     #   @result_2 = method2
     # }
     # NOTE: Only supports assigning instance variables within the block, not local variables
-    def self.run_in_parallel(&block)
+    def self.run_in_parallel(timeout = @@timeout, kill_all_on_error = false, &block)
       if Process.respond_to?(:fork)
         proxy = BlankBindingParallelProxy.new(block.binding)
         proxy.instance_eval(&block)
-        return wait_for_processes(proxy, block.binding)
+        return wait_for_processes(proxy, block.binding, timeout, kill_all_on_error)
       end
       puts 'Warning: Fork is not supported on this OS, executing block normally'
       block.call
     end
 
-    # Private method to lookup results from the results_map and replace the
-    # temp values with actual return values
-    def self.result_lookup(proxy_obj, target_obj, results_map)
-      target_obj = eval('self', target_obj)
-      proxy_obj ||= target_obj
-      vars = (proxy_obj.instance_variables)
-      results = []
-      results_map.keys.each { |tmp_result|
-        results << results_map[tmp_result]
-        vars.each {|var|
-          if proxy_obj.instance_variable_get(var) == tmp_result
-            target_obj.instance_variable_set(var, results_map[tmp_result])
-            break
-          end
-        }
-      }
-      results
-    end
-    private_class_method :result_lookup
-
     # Example - Will spawn a process in the background to run puppet agent on two agents and return immediately:
     # Parallel.run_in_background {
     #   @result_1 = method1
@@ -91,24 +79,35 @@ module InParallel
 
     # Waits for all processes to complete and logs STDOUT and STDERR in chunks from any processes
     # that were triggered from this Parallel class
-    def self.wait_for_processes(proxy = self, binding = nil)
+    # @param [Object] proxy - The instance of the proxy class that the method was executed within
+    # (probably only useful when called by run_in_background)
+    # @param [Object] binding - The binding of the block to assign return values to instance variables
+    # (probably only useful when called by run_in_background)
+    # @param [Int] timeout Time in seconds to wait before giving up on a child process
+    # @param [Boolean] kill_all_on_error Whether to wait for all processes to complete, or fail immediately -
+    # killing all other forked processes - when one process errors.
+    def self.wait_for_processes(proxy = self, binding = nil, timeout = nil, kill_all_on_error = false)
+      raise_error = nil
+      timeout ||= @@timeout
       trap(:INT) do
         puts "Warning, recieved interrupt.  Processing child results and exiting."
-        @@process_infos.each { |process_info|
-          # Send INT to each child process so it returns and can print stdout and stderr to console before exiting.
-          Process.kill("INT", process_info[:pid])
-        }
+        kill_child_processes
       end
       return unless Process.respond_to?(:fork)
       # Custom process to wait so that we can do things like time out, and kill child processes if
       # one process returns with an error before the others complete.
-      results_map = {}
-      timer = Time.now
+      results_map = Array.new(@@process_infos.count)
+      start_time = Time.now
+      timer = start_time
       while !@@process_infos.empty? do
         if @@signal_interval > 0 && Time.now > timer + @@signal_interval
           puts 'Waiting for child processes.'
           timer = Time.now
         end
+        if Time.now > start_time + timeout
+          kill_child_processes
+          raise_error = ::RuntimeError.new("Child process ran longer than timeout of #{timeout}")
+        end
         @@process_infos.each {|process_info|
           # wait up to half a second for each thread to see if it is complete, if not, check the next thread.
           # returns immediately if the process has completed.
@@ -121,48 +120,42 @@ module InParallel
               puts File.new(process_info[:std_out], 'r').readlines
               puts "------ Completed output for #{process_info[:method_sym]} - #{process_info[:pid]}\n"
               result = process_info[:result].read
-              results_map[process_info[:tmp_result]] = (result.nil? || result.empty?) ? result : Marshal.load(result)
+              marshalled_result = (result.nil? || result.empty?) ? result : Marshal.load(result)
+              if marshalled_result.is_a?(Exception)
+                raise_error = marshalled_result.dup
+                kill_child_processes if kill_all_on_error
+                marshalled_result = nil
+              end
+              results_map[process_info[:index]] = {process_info[:tmp_result] => marshalled_result}
               File.delete(process_info[:std_out])
               # Kill all other processes and let them log their stdout before re-raising
               # if a child process raised an error.
-              if results_map[process_info[:tmp_result]].is_a?(Exception)
-                @@process_infos.each{|p_info|
-                  begin
-                    Process.kill('SIGINT', p_info[:pid]) unless p_info[:pid] == process_info[:pid]
-                  rescue StandardError
-                  end
-                }
-              end
             ensure
               # close the read end pipe
               process_info[:result].close unless process_info[:result].closed?
               @@process_infos.delete(process_info)
-              @@raise_error = results_map[process_info[:tmp_result]] if results_map[process_info[:tmp_result]].is_a?(Exception)
-              break
             end
           end
         }
       end
 
-      # Reset the error in case the error is rescued
-      begin
-        raise @@raise_error unless @@raise_error.nil?
-      ensure
-        @@raise_error = nil
-      end
-
       results = []
 
       # pass in the 'self' from the block.binding which is the instance of the class
       # that contains the initial binding call.
-      # This gives us access to the local and instance variables from that context.
+      # This gives us access to the instance variables from that context.
       results = result_lookup(proxy, binding, results_map) if binding
 
+      # If there are background_objs AND results, don't return the background obj results
+      # (which would mess up expected results from each_in_parallel),
+      # but do process their results in case they are assigned to instance variables
       @@background_objs.each {|obj|
-         results = results.concat result_lookup(obj[:proxy], obj[:target], results_map)
+         result_lookup(obj[:proxy], obj[:target], results_map)
       }
       @@background_objs.clear
 
+      raise raise_error unless raise_error.nil?
+
       return results
     end
 
@@ -176,6 +169,7 @@ module InParallel
         exit_status = 0
         trap(:INT) do
           puts("Warning: Interrupt received in child process; exiting #{Process.pid}")
+          kill_child_processes
           return
         end
         write_file = File.new("tmp/parallel_process_#{Process.pid}", 'w')
@@ -228,12 +222,45 @@ module InParallel
                        :method_sym => method_sym,
                        :std_out => "tmp/parallel_process_#{pid}",
                        :result => read_result,
-                       :tmp_result => "unresolved_parallel_result_#{@@result_id}" }
+                       :tmp_result => "unresolved_parallel_result_#{@@result_id}",
+                       :index => @@process_infos.count }
       @@process_infos.push(process_info)
       @@result_id += 1
       process_info
     end
 
+    def self.kill_child_processes
+      @@process_infos.each { |process_info|
+        # Send INT to each child process so it returns and can print stdout and stderr to console before exiting.
+        begin
+          Process.kill("INT", process_info[:pid])
+        rescue Errno::ESRCH
+          # If one of the other processes has completed in the very short time before we try to kill it, handle the exception
+        end
+      }
+    end
+    private_class_method :kill_child_processes
+
+    # Private method to lookup results from the results_map and replace the
+    # temp values with actual return values
+    def self.result_lookup(proxy_obj, target_obj, results_map)
+      target_obj = eval('self', target_obj)
+      proxy_obj ||= target_obj
+      vars = (proxy_obj.instance_variables)
+      results = []
+      results_map.each { |tmp_result|
+        results << tmp_result.values[0]
+        vars.each {|var|
+          if proxy_obj.instance_variable_get(var) == tmp_result.keys[0]
+            target_obj.instance_variable_set(var, tmp_result.values[0])
+            break
+          end
+        }
+      }
+      results
+    end
+    private_class_method :result_lookup
+
     # Proxy class used to wrap each method execution in a block and run it in parallel
     # A block from Parallel.run_in_parallel is executed with a binding of an instance of this class
     class BlankBindingParallelProxy < BasicObject
@@ -264,10 +291,14 @@ module InParallel
   # }
   # NOTE - Only instance variables can be assigned the return values of the methods within the block.
   # Local variables will not be assigned any values.
+  # @param [Int] timeout Time in seconds to wait before giving up on a child process
+  # @param [Boolean] kill_all_on_error Whether to wait for all processes to complete, or fail immediately -
+  # killing all other forked processes - when one process errors.
   # @param [Block] block This method will yield to a block of code passed by the caller
   # @return [Array<Result>, Result] the return values of each method within the block
-  def run_in_parallel(&block)
-    InParallelExecutor.run_in_parallel(&block)
+  def run_in_parallel(timeout=nil, kill_all_on_error = false, &block)
+    timeout ||= InParallelExecutor.timeout
+    InParallelExecutor.run_in_parallel(timeout, kill_all_on_error, &block)
   end
 
   # Forks a process for each method within a block and returns immediately
@@ -297,8 +328,12 @@ module InParallel
 
   # Waits for all processes started by run_in_background to complete execution, then prints STDOUT
   # and assigns return values to instance variables.  See :run_in_background
+  # @param [Int] timeout Time in seconds to wait before giving up on a child process
+  # @param [Boolean] kill_all_on_error Whether to wait for all processes to complete, or fail immediately -
+  # killing all other forked processes - when one process errors.
   # @return [Array<Result>, Result] the temporary return values of each method within the block
-  def wait_for_processes
-    InParallelExecutor.wait_for_processes
+  def wait_for_processes(timeout=nil, kill_all_on_error = false)
+    timeout ||= InParallelExecutor.timeout
+    InParallelExecutor.wait_for_processes(nil, nil, timeout, kill_all_on_error)
   end
 end

data/lib/parallel_enumerable.rb

@@ -8,8 +8,9 @@ module Enumerable
   #   my_method(int)
   # }
   # @param [String] identifier - Optional identifier for logging purposes only. Will use the block location by default.
+  # @param [Int] timeout - Seconds to wait for a forked process to complete before timing out
   # @return [Array<Object>] results - the return value of each block execution.
-  def each_in_parallel(identifier=nil, &block)
+  def each_in_parallel(identifier=nil, timeout=(InParallel::InParallelExecutor.timeout), kill_all_on_error = false, &block)
     if Process.respond_to?(:fork) && count > 1
       identifier ||= "#{caller_locations[0]}"
       each do |item|
@@ -17,7 +18,7 @@ module Enumerable
         puts "'each_in_parallel' forked process for '#{identifier}' - PID = '#{out[:pid]}'\n"
       end
       # return the array of values, no need to look up from the map.
-      return InParallel::InParallelExecutor.wait_for_processes(nil, block.binding)
+      return InParallel::InParallelExecutor.wait_for_processes(nil, block.binding, timeout, kill_all_on_error)
     end
     puts 'Warning: Fork is not supported on this OS, executing block normally' unless Process.respond_to? :fork
     block.call

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: in-parallel
 version: !ruby/object:Gem::Version
-  version: 0.1.8
+  version: 0.1.9
 platform: ruby
 authors:
 - samwoods1
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-06-08 00:00:00.000000000 Z
+date: 2016-06-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler