in_threads 1.4.0 → 1.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3cd4f7083b8d1d0be3787fb95a747ac290882761
-  data.tar.gz: 7a8b5b3e65d59bcdc03be378a9f8d9f94d1b0535
+  metadata.gz: 88e40a6e4c911af5eb5187d9b7c8ee9a6597d19a
+  data.tar.gz: 93cd8928eaea21fa27779ec68fab8a9d22f71af8
 SHA512:
-  metadata.gz: 6d7547364ef110ff2cb51e8e57441028e694bfae5d54e7c44ae63b58599aefa4523fda5531a90af5d0e54dc0415c365b024d6ab585481862ffa4868ca47e6992
-  data.tar.gz: 22b699ee74f26862c5cf7a844d7aaf6f77af5a50d4d7ce0c287c8682fa577b6dbadca1c207282673b9da36d7f68bff90d6bfde0ea343b81ae28dd2835cc31c68
+  metadata.gz: 79915ecf8198c7b8dfd5df0b17147b073f9d3de14e9cfda8ed31e89ed48c2c4f407e5c41d322156628660810a96db2db9a741f26142562525af28d44700201c6
+  data.tar.gz: f83d77ea6f937508b9b33b998091d20d69a2432adaebf37e9749034fef2221dc33ee5cf83f6d6c341f6fab321a34947191508e7978b77fdd84ec965fbad43e56

data/.rubocop.yml

@@ -2,49 +2,67 @@ AllCops:
   Exclude:
     - '*.gemspec'
 
+Layout/AccessModifierIndentation:
+  EnforcedStyle: outdent
+
+Layout/CaseIndentation:
+  EnforcedStyle: end
+
+Layout/DotPosition:
+  EnforcedStyle: trailing
+
+Layout/IndentHash:
+  EnforcedStyle: consistent
+
+Layout/SpaceBeforeBlockBraces:
+  EnforcedStyle: no_space
+
+Layout/SpaceInsideHashLiteralBraces:
+  EnforcedStyle: no_space
+
+Lint/AmbiguousBlockAssociation:
+  Enabled: false
+
 Lint/EndAlignment:
   EnforcedStyleAlignWith: variable
 
+Lint/EnsureReturn:
+  Enabled: false
+
+Lint/RescueException:
+  Enabled: false
+
 Metrics/AbcSize:
-  Max: 25
+  Max: 35
 
 Metrics/BlockLength:
   Exclude:
     - 'spec/**/*.rb'
 
 Metrics/ClassLength:
-  Max: 150
+  Max: 200
+
+Metrics/CyclomaticComplexity:
+  Max: 10
+
+Metrics/LineLength:
+  Max: 120
 
 Metrics/MethodLength:
-  Max: 25
+  Max: 30
 
 Performance/RedundantBlockCall:
   Enabled: false
 
-Style/AccessModifierIndentation:
-  EnforcedStyle: outdent
-
-Style/CaseIndentation:
-  EnforcedStyle: end
-
-Style/DotPosition:
-  EnforcedStyle: trailing
-
 Style/DoubleNegation:
   Enabled: false
 
-Style/Encoding:
-  EnforcedStyle: when_needed
-
 Style/HashSyntax:
   EnforcedStyle: hash_rockets
 
 Style/IfUnlessModifier:
   MaxLineLength: 40
 
-Style/IndentHash:
-  EnforcedStyle: consistent
-
 Style/ParallelAssignment:
   Enabled: false
 
@@ -59,12 +77,6 @@ Style/Semicolon:
 Style/SignalException:
   EnforcedStyle: semantic
 
-Style/SpaceBeforeBlockBraces:
-  EnforcedStyle: no_space
-
-Style/SpaceInsideHashLiteralBraces:
-  EnforcedStyle: no_space
-
 Style/TrailingCommaInArguments:
   EnforcedStyleForMultiline: no_comma
 

data/.travis.yml

@@ -5,21 +5,23 @@ rvm:
   - '1.9.3-p551'
   - '2.0.0-p648'
   - '2.1.10'
-  - '2.2.6'
-  - '2.3.3'
-  - '2.4.0'
+  - '2.2.8'
+  - '2.3.5'
+  - '2.4.2'
   - 'ruby-head'
-  - 'jruby-1.7.26'
   - 'jruby-9.0.5.0'
-  - 'jruby-9.1.5.0'
+  - 'jruby-9.1.9.0'
+before_install:
+  - gem update --system
+  - gem update bundler
 script: bundle exec rspec
 matrix:
   include:
     - env: RUBOCOP=✓
-      rvm: '2.4.0'
+      rvm: '2.4.2'
       script: bundle exec rubocop
     - env: CHECK_RUBIES=✓
-      rvm: '2.4.0'
+      rvm: '2.4.2'
       script: bundle exec travis_check_rubies
   allow_failures:
     - rvm: 'ruby-head'

data/CHANGELOG.markdown

@@ -0,0 +1,82 @@
+# ChangeLog
+
+## unreleased
+
+## v1.5.0 (2017-11-17)
+
+* Use thread pool instead of creating a thread for every iteration [@toy](https://github.com/toy)
+* Handle `break` (also with argument) and exceptions raised in `each` method of enumerable [@toy](https://github.com/toy)
+* Register `lazy` to run without threads [@toy](https://github.com/toy)
+
+## v1.4.0 (2017-03-19)
+
+* Register `sum` and `uniq` to run in threads and `chunk_while` to run without threads [@toy](https://github.com/toy)
+* Register `grep_v` to call block in threads [@toy](https://github.com/toy)
+* Fix Segmentation fault in ruby 2.0 [@toy](https://github.com/toy)
+* Fix not sending all block arguments to methods not returning original enumerable [@toy](https://github.com/toy)
+* Fix for a [bug](https://bugs.ruby-lang.org/issues/13313) in ruby 2.4.0 causing Segmentation fault [@toy](https://github.com/toy)
+* Clean up documentation [#1](https://github.com/toy/in_threads/pull/1) [@hollingberry](https://github.com/hollingberry)
+
+## v1.3.1 (2015-01-09)
+
+* Register `to_h`, `slice_after` and `slice_when` to run without threads [@toy](https://github.com/toy)
+* Remove special handling of methods running without threads [@toy](https://github.com/toy)
+
+## v1.3.0 (2014-10-31)
+
+* Fix not thread safe code for jruby [@toy](https://github.com/toy)
+
+## v1.2.2 (2014-08-08)
+
+* Fix silencing exceptions raised in blocks [@toy](https://github.com/toy)
+
+## v1.2.1 (2014-04-06)
+
+* Use explicit requires instead of autoload [@toy](https://github.com/toy)
+
+## v1.2.0 (2013-08-20)
+
+* Befriend with [`progress`](https://rubygems.org/gems/progress) gem [@toy](https://github.com/toy)
+* Use `Delegator` instead of undefining methods [@toy](https://github.com/toy)
+
+## v1.1.2 (2013-08-02)
+
+* Fix for jruby stalling [@toy](https://github.com/toy)
+* Fix for rubinius not raising `NoMethodError` for undefined methods [@toy](https://github.com/toy)
+* Register `to_set` to run without threads [@toy](https://github.com/toy)
+
+## v1.1.1 (2011-12-13)
+
+* Decrease priority of initiating block execution for methods not returning original enumerable [@toy](https://github.com/toy)
+* Call each of enumerable only once for methods not returning original enumerable [@toy](https://github.com/toy)
+* Remove class method `enumerable_methods` [@toy](https://github.com/toy)
+
+## v1.1.0 (2011-12-08)
+
+* Register `chunk` and `slice_before` to run without threads [@toy](https://github.com/toy)
+* Register `flat_map` and `collect_concat` to run in threads [@toy](https://github.com/toy)
+* Register `each_entry` to run in threads [@toy](https://github.com/toy)
+* Register `each_with_object` to run without threads [@toy](https://github.com/toy)
+* Add argument checking [@toy](https://github.com/toy)
+* Use faster method for `each` [@toy](https://github.com/toy)
+
+## v1.0.0 (2011-12-05)
+
+* Fix for blocks with multiple arguments (`each_with_index`, `enum_with_index`) [@toy](https://github.com/toy)
+* Rewrite: properly working with all compatible `Enumerable` methods, otherwise running without threads [@toy](https://github.com/toy)
+
+## v0.0.4 (2010-12-15)
+
+* Internal gem changes [@toy](https://github.com/toy)
+
+## v0.0.3 (2010-07-13)
+
+* Fix thread count overflow [@toy](https://github.com/toy)
+
+## v0.0.2 (2009-12-29)
+
+* Internal gem changes [@toy](https://github.com/toy)
+
+## v0.0.1 (2009-09-21)
+
+* Initial [@toy](https://github.com/toy)

data/LICENSE.txt

@@ -1,4 +1,4 @@
-Copyright (c) 2010-2017 Ivan Kuchin
+Copyright (c) 2009-2017 Ivan Kuchin
 
 Permission is hereby granted, free of charge, to any person obtaining
 a copy of this software and associated documentation files (the

data/README.markdown

@@ -6,7 +6,7 @@
 
 # in_threads
 
-Easily execute Ruby code in parallel.
+Run all possible enumerable methods in concurrent/parallel threads.
 
 ```ruby
 urls.in_threads(20).map do |url|
@@ -28,7 +28,7 @@ gem 'in_threads'
 $ bundle install
 ```
 
-Or, if you don't use Bundler, install it globally:
+Or install globally:
 
 ```sh
 $ gem install in_threads
@@ -94,6 +94,14 @@ You can call any `Enumerable` method, but some (`#inject`, `#reduce`, `#max`,
 `#min`, `#sort`, `#to_a`, and others) cannot run concurrently, and so will
 simply act as if `in_threads` wasn't used.
 
+### Break and exceptions
+
+Exceptions are caught and re-thrown after allowing blocks that are still running to finish.
+
+**IMPORTANT**: only the first encountered exception is propagated, so it is recommended to handle exceptions in the block.
+
+`break` is handled in ruby >= 1.9 and should be handled in jruby [after 9.1.9.0](https://github.com/jruby/jruby/issues/4697). Handling is done in special way: as blocks are run outside of original context, calls to `break` cause `LocalJumpError` which is caught and its result is returned.
+
 ## Copyright
 
-Copyright (c) 2010-2017 Ivan Kuchin. See LICENSE.txt for details.
+Copyright (c) 2009-2017 Ivan Kuchin. See LICENSE.txt for details.

data/in_threads.gemspec

@@ -2,8 +2,8 @@
 
 Gem::Specification.new do |s|
   s.name        = 'in_threads'
-  s.version     = '1.4.0'
-  s.summary     = %q{Execute ruby code in parallel}
+  s.version     = '1.5.0'
+  s.summary     = %q{Run all possible enumerable methods in concurrent/parallel threads}
   s.homepage    = "http://github.com/toy/#{s.name}"
   s.authors     = ['Ivan Kuchin']
   s.license     = 'MIT'
@@ -18,6 +18,6 @@ Gem::Specification.new do |s|
   s.add_development_dependency 'rspec', '~> 3.0'
   s.add_development_dependency 'rspec-retry', '~> 0.3'
   if RUBY_VERSION >= '2.0'
-    s.add_development_dependency 'rubocop', '~> 0.47'
+    s.add_development_dependency 'rubocop', '~> 0.49'
   end
 end

data/lib/in_threads.rb

@@ -1,5 +1,4 @@
 require 'thread'
-require 'thwait'
 require 'delegate'
 
 Enumerable.class_eval do
@@ -31,84 +30,6 @@ end
 class InThreads < SimpleDelegator
   protected :__getobj__, :__setobj__
 
-  # Use ThreadsWait to limit number of threads
-  class ThreadLimiter
-    # Initialize with limit
-    def initialize(count)
-      @count = count
-      @waiter = ThreadsWait.new
-    end
-
-    # Without block behaves as `new`
-    # With block yields it with `self` and ensures running of `finalize`
-    def self.limit(count, &block)
-      limiter = new(count)
-      if block
-        begin
-          yield limiter
-        ensure
-          limiter.finalize
-        end
-      else
-        limiter
-      end
-    end
-
-    # Add thread to `ThreadsWait`, wait for finishing of one thread if limit
-    # reached
-    def <<(thread)
-      @waiter.join_nowait(thread)
-      @waiter.next_wait.join unless @waiter.threads.length < @count
-    end
-
-    # Wait for waiting threads
-    def finalize
-      @waiter.all_waits(&:join)
-    end
-  end
-
-  # Yield objects of one enum in multiple places
-  class Splitter
-    # Enumerable using Queue
-    class Transfer
-      include Enumerable
-
-      def initialize
-        @queue = Queue.new
-      end
-
-      def <<(args)
-        @queue << args
-      end
-
-      def finish
-        @queue << nil
-      end
-
-      def each(&block)
-        while (args = @queue.pop)
-          block.call(*args)
-        end
-        nil # non reusable
-      end
-    end
-
-    # Enums receiving items
-    attr_reader :enums
-
-    def initialize(enumerable, enum_count)
-      @enums = Array.new(enum_count){ Transfer.new }
-      @filler = Thread.new do
-        enumerable.each do |*args|
-          @enums.each do |enum|
-            enum << args
-          end
-        end
-        @enums.each(&:finish)
-      end
-    end
-  end
-
   attr_reader :enumerable, :thread_count
   def initialize(enumerable, thread_count = 10, &block)
     super(enumerable)
@@ -130,7 +51,7 @@ class InThreads < SimpleDelegator
   class << self
     # Specify runner to use
     #
-    #   use :run_in_threads_consecutive, :for => %w[all? any? none? one?]
+    #   use :run_in_threads_use_block_result, :for => %w[all? any? none? one?]
     #
     # `:for` is required
     # `:ignore_undefined` ignores methods which are not present in
@@ -143,7 +64,11 @@ class InThreads < SimpleDelegator
         next if ignore_undefined && !enumerable_method?(method)
         class_eval <<-RUBY
           def #{method}(*args, &block)
-            #{runner}(:#{method}, *args, &block)
+            if block
+              #{runner}(:#{method}, *args, &block)
+            else
+              enumerable.#{method}(*args)
+            end
           end
         RUBY
       end
@@ -152,13 +77,12 @@ class InThreads < SimpleDelegator
   private
 
     def enumerable_method?(name)
-      @enumerable_methods ||= Enumerable.instance_methods.map(&:to_sym)
-      @enumerable_methods.include?(name.to_sym)
+      Enumerable.method_defined?(name)
     end
   end
 
-  use :run_in_threads_return_original_enum, :for => %w[each]
-  use :run_in_threads_return_original_enum, :for => %w[
+  use :run_in_threads_ignore_block_result, :for => %w[each]
+  use :run_in_threads_ignore_block_result, :for => %w[
     reverse_each
     each_with_index enum_with_index
     each_cons each_slice enum_cons enum_slice
@@ -166,7 +90,7 @@ class InThreads < SimpleDelegator
     cycle
     each_entry
   ], :ignore_undefined => true
-  use :run_in_threads_consecutive, :for => %w[
+  use :run_in_threads_use_block_result, :for => %w[
     all? any? none? one?
     detect find find_index drop_while take_while
     partition find_all select reject count
@@ -183,9 +107,10 @@ class InThreads < SimpleDelegator
     include? member?
     each_with_object
     chunk chunk_while slice_before slice_after slice_when
+    lazy
   ].map(&:to_sym)
 
-  # Special case method, works by applying `run_in_threads_consecutive` with
+  # Special case method, works by applying `run_in_threads_use_block_result` with
   # map on enumerable returned by blockless run
   def grep(*args, &block)
     if block
@@ -196,7 +121,7 @@ class InThreads < SimpleDelegator
   end
 
   if enumerable_method?(:grep_v)
-    # Special case method, works by applying `run_in_threads_consecutive` with
+    # Special case method, works by applying `run_in_threads_use_block_result` with
     # map on enumerable returned by blockless run
     def grep_v(*args, &block)
       if block
@@ -214,46 +139,159 @@ class InThreads < SimpleDelegator
 
 protected
 
+  # Enum out of queue
+  class QueueEnum
+    include Enumerable
+
+    def initialize(size = nil)
+      @queue = size ? SizedQueue.new(size) : Queue.new
+    end
+
+    def each(&block)
+      while (args = @queue.pop)
+        block.call(*args)
+      end
+      nil # non reusable
+    end
+
+    def push(*args)
+      @queue.push(args) unless @closed
+    end
+
+    def close(clear = false)
+      @closed = true
+      @queue.clear if clear
+      @queue.push(nil)
+    end
+  end
+
+  # Thread pool
+  class Pool
+    attr_reader :exception
+
+    def initialize(thread_count)
+      @queue = Queue.new
+      @mutex = Mutex.new
+      @pool = Array.new(thread_count) do
+        Thread.new do
+          while (block = @queue.pop)
+            block.call
+            break if stop?
+          end
+        end
+      end
+    end
+
+    def run(&block)
+      @queue.push(block)
+    end
+
+    def stop?
+      @stop || @exception
+    end
+
+    def stop!
+      @stop = true
+    end
+
+    def finalize
+      @pool.
+        each{ @queue.push(nil) }.
+        each(&:join)
+    end
+
+    def catch
+      yield
+    rescue Exception => e
+      @mutex.synchronize{ @exception ||= e } unless @exception
+      nil
+    end
+  end
+
   # Use for methods which don't use block result
-  def run_in_threads_return_original_enum(method, *args, &block)
-    if block
-      ThreadLimiter.limit(thread_count) do |limiter|
+  def run_in_threads_ignore_block_result(method, *args, &block)
+    pool = Pool.new(thread_count)
+    wait = SizedQueue.new(thread_count - 1)
+    begin
+      pool.catch do
         enumerable.send(method, *args) do |*block_args|
-          limiter << Thread.new{ block.call(*block_args) }
+          pool.run do
+            pool.catch do
+              block.call(*block_args)
+            end
+            wait.pop
+          end
+          wait.push(nil)
+          break if pool.stop?
         end
       end
-    else
-      enumerable.send(method, *args)
+    ensure
+      pool.finalize
+      if (e = pool.exception)
+        return e.exit_value if e.is_a?(LocalJumpError) && e.reason == :break
+        fail e
+      end
     end
   end
 
   # Use for methods which do use block result
-  def run_in_threads_consecutive(method, *args, &block)
-    if block
-      enum_a, enum_b = Splitter.new(enumerable, 2).enums
-      results = Queue.new
-      runner = Thread.new do
-        Thread.current.priority = -1
-        ThreadLimiter.limit(thread_count) do |limiter|
-          enum_a.each do |*block_args|
-            break if Thread.current[:stop]
-            thread = Thread.new{ block.call(*block_args) }
-            results << thread
-            limiter << thread
+  def run_in_threads_use_block_result(method, *args, &block)
+    pool = Pool.new(thread_count)
+    enum_a = QueueEnum.new
+    enum_b = QueueEnum.new(thread_count - 1)
+    results = SizedQueue.new(thread_count - 1)
+    filler = filler_thread(pool, [enum_a, enum_b])
+    runner = runner_thread(pool, enum_a, results, &block)
+
+    begin
+      pool.catch do
+        enum_b.send(method, *args) do
+          result = results.pop.pop
+          break if pool.stop?
+          result
+        end
+      end
+    ensure
+      pool.stop!
+      enum_a.close(true)
+      enum_b.close(true)
+      results.clear
+      pool.finalize
+      runner.join
+      filler.join
+      if (e = pool.exception)
+        return e.exit_value if e.is_a?(LocalJumpError) && e.reason == :break
+        fail e
+      end
+    end
+  end
+
+private
+
+  def filler_thread(pool, enums)
+    Thread.new do
+      pool.catch do
+        enumerable.each do |*block_args|
+          enums.each do |enum|
+            enum.push(*block_args)
           end
+          break if pool.stop?
         end
       end
+      enums.each(&:close)
+    end
+  end
 
-      begin
-        enum_b.send(method, *args) do
-          results.pop.value
+  def runner_thread(pool, enum, results, &block)
+    Thread.new do
+      enum.each do |*block_args|
+        queue = Queue.new
+        pool.run do
+          queue.push(pool.catch{ block.call(*block_args) })
         end
-      ensure
-        runner[:stop] = true
-        runner.join
+        results.push(queue)
+        break if pool.stop?
       end
-    else
-      enumerable.send(method, *args)
     end
   end
 end