in-parallel 0.1.15 → 0.1.16

checksums.yaml

@@ -1,7 +1,15 @@
 ---
-SHA1:
-  metadata.gz: 0b5dc6a05d28155a49a6c5d07c3be8385583ef58
-  data.tar.gz: 6eb5ddb93219985c1513a3a5f1abe23391d5590b
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    NGJlY2ZkMDkyMjM3MGE4NGIxNjljNDU1YTE1NGRkYWY2OWE2NjZkZA==
+  data.tar.gz: !binary |-
+    ZmQ4NmExZGJmYmZiNDM5NjM1MWQ1OGM5MTMwMGMwMjYzMDdjY2ViOQ==
 SHA512:
-  metadata.gz: 5f36a6e1b63a18514c9e17713d95d29725c5b6ddfb7cd91074a8e8b25c3bc8004dbcd272fad00c82ac4e155931673bc494d83497b8310e54dca074dabe384d39
-  data.tar.gz: 2a1d6707489ebe47beec5c6959394b66c57926843f377ff5cf24bad66124935afe35a1f171e84c2266e6ab2c229d2085168c8ceb6244c0846977fb534c11330a
+  metadata.gz: !binary |-
+    MGFjZTBkNGUzYTEwYWFkMDY0YWVhNGIxNjc2MzhkOWUwNTg5Y2NjNmQxOTAy
+    ZjAwNTliZWJmZTU4NjU1ODg0OGExNmRkZTljMzk0OWQyZGU2ODFhOWQ3ZTUw
+    ZWQzNTRkMjY5YzM5MGYyZGViNGY4MjVhMjQwMDExYjE5MTQ3ZWY=
+  data.tar.gz: !binary |-
+    OWEyOGQ2MWJiY2RiZTVhNTMxM2VhNDllM2MwNmM2ZTVjNWVhNjljMjcxZDU4
+    NWIxNTAxNzMyZDM1YWZlYjhhOWQ2Y2I0NzRmNGZmOWJlN2EzOTlhYTA5ZTk2
+    NDZlZGVhYmYyYzFkOTcwN2E0MWY4NjQ1NTEzOGZmMGE0OGE0MWM=

data/HISTORY.md

@@ -1,10 +1,65 @@
 # experimental_in-parallel_bump_and_tag_master - History
 ## Tags
-* [LATEST - 8 Aug, 2016 (d026c624)](#LATEST)
+* [LATEST - 6 Feb, 2017 (27b497ea)](#LATEST)
+* [0.1.15 - 3 Feb, 2017 (ff16929c)](#0.1.15)
+* [0.1.14 - 8 Aug, 2016 (ce331dbd)](#0.1.14)
 * [0.1.13 - 8 Aug, 2016 (26d19934)](#0.1.13)
 
 ## Details
-### <a name = "LATEST">LATEST - 8 Aug, 2016 (d026c624)
+### <a name = "LATEST">LATEST - 6 Feb, 2017 (27b497ea)
+
+* (GEM) update in-parallel version to 0.1.16 (27b497ea)
+
+### <a name = "0.1.15">0.1.15 - 3 Feb, 2017 (ff16929c)
+
+* (HISTORY) update in-parallel history for gem release 0.1.15 (ff16929c)
+
+* (GEM) update in-parallel version to 0.1.15 (206a62fe)
+
+* Merge pull request #16 from nicklewis/support-large-results (4d644d88)
+
+
+```
+Merge pull request #16 from nicklewis/support-large-results
+
+(maint) Avoid deadlock with large results
+```
+* (maint) Avoid deadlock with large results (a5a9c174)
+
+
+```
+(maint) Avoid deadlock with large results
+
+Previously, if the result of a process was larger than the IO buffer
+size (commonly 64k), execution would deadlock until the timeout.
+
+In this scenario, the writer would fill the buffer and block until the
+reader had cleared the buffer by reading. However, the reader will only
+try to read once (to get the whole result) and will only attempt to read
+after the child process has exited. This causes a deadlock, as the child
+can't exit because it can't finish writing, but the the reader won't
+read because the child hasn't exited.
+
+This commit fixes the watcher loop to instead IO.select() from the
+available result readers and read a partial result into a buffer whenever it's
+available. This ensures that the writer will never remain blocked by a
+full buffer. The reader now uses IO#eof? to determine whether the child
+process has exited, at which point it will process the whole result as
+before.
+```
+* Merge pull request #15 from samwoods1/add_jjb_pipelines (72d32635)
+
+
+```
+Merge pull request #15 from samwoods1/add_jjb_pipelines
+
+(maint) Revert name change of in_parallel.rb
+```
+* (maint) Revert name change of in_parallel.rb (327c8fd5)
+
+### <a name = "0.1.14">0.1.14 - 8 Aug, 2016 (ce331dbd)
+
+* (HISTORY) update in-parallel history for gem release 0.1.14 (ce331dbd)
 
 * (GEM) update in-parallel version to 0.1.14 (d026c624)
 

data/lib/in-parallel/version.rb

@@ -1,3 +1,3 @@
 module InParallel
-  VERSION = '0.1.15'
+  VERSION = '0.1.16'
 end

data/lib/in_parallel.rb

@@ -126,34 +126,37 @@ module InParallel
           kill_child_processes
           raise_error = ::RuntimeError.new("Child process ran longer than timeout of #{timeout}")
         end
-        @@process_infos.each do |process_info|
-          # wait up to half a second for each thread to see if it is complete, if not, check the next thread.
-          # returns immediately if the process has completed.
-          thr = process_info[:wait_thread].join(0.5)
-          unless thr.nil?
-            # the process completed, get the result and rethrow on error.
-            begin
-              # Print the STDOUT and STDERR for each process with signals for start and end
-              @@logger.info "------ Begin output for #{process_info[:method_sym]} - #{process_info[:pid]}"
-              # Content from the other thread will already be pre-pended with log stuff (info, warn, date/time, etc)
-              # So don't use logger, just use puts.
-              puts "  " + File.new(process_info[:std_out], 'r').readlines.join("  ")
-              @@logger.info "------ Completed output for #{process_info[:method_sym]} - #{process_info[:pid]}"
-              result            = process_info[:result].read
-              marshalled_result = (result.nil? || result.empty?) ? result : Marshal.load(result)
-              # Kill all other processes and let them log their stdout before re-raising
-              # if a child process raised an error.
-              if marshalled_result.is_a?(Exception)
-                raise_error = marshalled_result.dup
-                kill_child_processes if kill_all_on_error
-                marshalled_result = nil
+
+        if result = IO.select(@@process_infos.map {|p| p[:result]}, nil, nil, 0.5)
+          read_ios = result.first
+          read_ios.each do |reader|
+            process_info = @@process_infos.find {|p| p[:result] == reader}
+            process_info[:result_buffer] << reader.read
+            if reader.eof?
+              result = process_info[:result_buffer].string
+              # the process completed, get the result and rethrow on error.
+              begin
+                # Print the STDOUT and STDERR for each process with signals for start and end
+                @@logger.info "------ Begin output for #{process_info[:method_sym]} - #{process_info[:pid]}"
+                # Content from the other thread will already be pre-pended with log stuff (info, warn, date/time, etc)
+                # So don't use logger, just use puts.
+                puts "  " + File.new(process_info[:std_out], 'r').readlines.join("  ")
+                @@logger.info "------ Completed output for #{process_info[:method_sym]} - #{process_info[:pid]}"
+                marshalled_result = (result.nil? || result.empty?) ? result : Marshal.load(result)
+                # Kill all other processes and let them log their stdout before re-raising
+                # if a child process raised an error.
+                if marshalled_result.is_a?(Exception)
+                  raise_error = marshalled_result.dup
+                  kill_child_processes if kill_all_on_error
+                    marshalled_result = nil
+                end
+                results_map[process_info[:index]] = { process_info[:tmp_result] => marshalled_result }
+              ensure
+                File.delete(process_info[:std_out]) if File.exists?(process_info[:std_out])
+                # close the read end pipe
+                process_info[:result].close unless process_info[:result].closed?
+                @@process_infos.delete(process_info)
               end
-              results_map[process_info[:index]] = { process_info[:tmp_result] => marshalled_result }
-            ensure
-              File.delete(process_info[:std_out]) if File.exists?(process_info[:std_out])
-              # close the read end pipe
-              process_info[:result].close unless process_info[:result].closed?
-              @@process_infos.delete(process_info)
             end
           end
         end
@@ -235,6 +238,7 @@ module InParallel
                        :std_out     => "tmp/pp_#{pid}",
                        :result      => read_result,
                        :tmp_result  => "unresolved_parallel_result_#{@@result_id}",
+                       :result_buffer => StringIO.new,
                        :index       => @@process_infos.count }
       @@process_infos.push(process_info)
       @@result_id += 1

data/spec/in-paralell_spec.rb

@@ -101,6 +101,19 @@ describe '.run_in_parallel' do
     expect(@result_2).to eq({ :foo => "bar" })
   end
 
+  it "should return large results" do
+    # 2**16 = 64k is typical buffer size
+    long_string = 'a' * (2**16+1)
+
+    expect do
+      run_in_parallel(timeout=1) do
+        @result = method_with_param(long_string)
+      end
+    end.not_to raise_error
+
+    expect(@result).to eq "bar + #{long_string}"
+  end
+
   it "should return a singleton class value" do
 
     run_in_parallel { @result = get_singleton_class }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: in-parallel
 version: !ruby/object:Gem::Version
-  version: 0.1.15
+  version: 0.1.16
 platform: ruby
 authors:
 - samwoods1
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-08-08 00:00:00.000000000 Z
+date: 2017-02-06 00:00:00.000000000 Z
 dependencies: []
 description: Many other Ruby libraries that simplify parallel execution support one
   primary use case - crunching through a large queue of small, similar tasks as quickly
@@ -26,7 +26,6 @@ extra_rdoc_files: []
 files:
 - CONTRIBUTING.md
 - Gemfile
-- Gemfile.lock
 - HISTORY.md
 - LICENSE
 - MAINTAINERS.md
@@ -47,20 +46,19 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.1
+rubygems_version: 2.4.6
 signing_key: 
 specification_version: 4
 summary: A lightweight library to execute a handful of tasks in parallel with simple
   syntax
 test_files: []
-has_rdoc: 

data/Gemfile.lock

@@ -1,68 +0,0 @@
-PATH
-  remote: .
-  specs:
-    in-parallel (0.1.12)
-
-GEM
-  remote: https://rubygems.org/
-  specs:
-    activesupport (4.2.6)
-      i18n (~> 0.7)
-      json (~> 1.7, >= 1.7.7)
-      minitest (~> 5.1)
-      thread_safe (~> 0.3, >= 0.3.4)
-      tzinfo (~> 1.1)
-    diff-lcs (1.2.5)
-    docile (1.1.5)
-    i18n (0.7.0)
-    json (1.8.3)
-    kramdown (1.11.1)
-    logutils (0.6.1)
-    markdown (0.4.0)
-      kramdown (>= 0.13.7)
-      props (>= 0.2.0)
-      textutils (>= 0.2.0)
-    minitest (5.9.0)
-    props (1.1.2)
-    rake (10.5.0)
-    rspec (3.1.0)
-      rspec-core (~> 3.1.0)
-      rspec-expectations (~> 3.1.0)
-      rspec-mocks (~> 3.1.0)
-    rspec-core (3.1.7)
-      rspec-support (~> 3.1.0)
-    rspec-expectations (3.1.2)
-      diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.1.0)
-    rspec-mocks (3.1.3)
-      rspec-support (~> 3.1.0)
-    rspec-support (3.1.2)
-    rubyzip (1.2.0)
-    simplecov (0.11.2)
-      docile (~> 1.1.0)
-      json (~> 1.8)
-      simplecov-html (~> 0.10.0)
-    simplecov-html (0.10.0)
-    textutils (1.4.0)
-      activesupport
-      logutils (>= 0.6.1)
-      props (>= 1.1.2)
-      rubyzip (>= 1.0.0)
-    thread_safe (0.3.5)
-    tzinfo (1.2.2)
-      thread_safe (~> 0.1)
-    yard (0.9.0)
-
-PLATFORMS
-  ruby
-
-DEPENDENCIES
-  in-parallel!
-  markdown (~> 0)
-  rake (>= 0.9.0)
-  rspec (~> 3.1.0)
-  simplecov
-  yard (~> 0)
-
-BUNDLED WITH
-   1.12.3