in_threads 1.4.0 → 1.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 3cd4f7083b8d1d0be3787fb95a747ac290882761
4
- data.tar.gz: 7a8b5b3e65d59bcdc03be378a9f8d9f94d1b0535
3
+ metadata.gz: 88e40a6e4c911af5eb5187d9b7c8ee9a6597d19a
4
+ data.tar.gz: 93cd8928eaea21fa27779ec68fab8a9d22f71af8
5
5
  SHA512:
6
- metadata.gz: 6d7547364ef110ff2cb51e8e57441028e694bfae5d54e7c44ae63b58599aefa4523fda5531a90af5d0e54dc0415c365b024d6ab585481862ffa4868ca47e6992
7
- data.tar.gz: 22b699ee74f26862c5cf7a844d7aaf6f77af5a50d4d7ce0c287c8682fa577b6dbadca1c207282673b9da36d7f68bff90d6bfde0ea343b81ae28dd2835cc31c68
6
+ metadata.gz: 79915ecf8198c7b8dfd5df0b17147b073f9d3de14e9cfda8ed31e89ed48c2c4f407e5c41d322156628660810a96db2db9a741f26142562525af28d44700201c6
7
+ data.tar.gz: f83d77ea6f937508b9b33b998091d20d69a2432adaebf37e9749034fef2221dc33ee5cf83f6d6c341f6fab321a34947191508e7978b77fdd84ec965fbad43e56
@@ -2,49 +2,67 @@ AllCops:
2
2
  Exclude:
3
3
  - '*.gemspec'
4
4
 
5
+ Layout/AccessModifierIndentation:
6
+ EnforcedStyle: outdent
7
+
8
+ Layout/CaseIndentation:
9
+ EnforcedStyle: end
10
+
11
+ Layout/DotPosition:
12
+ EnforcedStyle: trailing
13
+
14
+ Layout/IndentHash:
15
+ EnforcedStyle: consistent
16
+
17
+ Layout/SpaceBeforeBlockBraces:
18
+ EnforcedStyle: no_space
19
+
20
+ Layout/SpaceInsideHashLiteralBraces:
21
+ EnforcedStyle: no_space
22
+
23
+ Lint/AmbiguousBlockAssociation:
24
+ Enabled: false
25
+
5
26
  Lint/EndAlignment:
6
27
  EnforcedStyleAlignWith: variable
7
28
 
29
+ Lint/EnsureReturn:
30
+ Enabled: false
31
+
32
+ Lint/RescueException:
33
+ Enabled: false
34
+
8
35
  Metrics/AbcSize:
9
- Max: 25
36
+ Max: 35
10
37
 
11
38
  Metrics/BlockLength:
12
39
  Exclude:
13
40
  - 'spec/**/*.rb'
14
41
 
15
42
  Metrics/ClassLength:
16
- Max: 150
43
+ Max: 200
44
+
45
+ Metrics/CyclomaticComplexity:
46
+ Max: 10
47
+
48
+ Metrics/LineLength:
49
+ Max: 120
17
50
 
18
51
  Metrics/MethodLength:
19
- Max: 25
52
+ Max: 30
20
53
 
21
54
  Performance/RedundantBlockCall:
22
55
  Enabled: false
23
56
 
24
- Style/AccessModifierIndentation:
25
- EnforcedStyle: outdent
26
-
27
- Style/CaseIndentation:
28
- EnforcedStyle: end
29
-
30
- Style/DotPosition:
31
- EnforcedStyle: trailing
32
-
33
57
  Style/DoubleNegation:
34
58
  Enabled: false
35
59
 
36
- Style/Encoding:
37
- EnforcedStyle: when_needed
38
-
39
60
  Style/HashSyntax:
40
61
  EnforcedStyle: hash_rockets
41
62
 
42
63
  Style/IfUnlessModifier:
43
64
  MaxLineLength: 40
44
65
 
45
- Style/IndentHash:
46
- EnforcedStyle: consistent
47
-
48
66
  Style/ParallelAssignment:
49
67
  Enabled: false
50
68
 
@@ -59,12 +77,6 @@ Style/Semicolon:
59
77
  Style/SignalException:
60
78
  EnforcedStyle: semantic
61
79
 
62
- Style/SpaceBeforeBlockBraces:
63
- EnforcedStyle: no_space
64
-
65
- Style/SpaceInsideHashLiteralBraces:
66
- EnforcedStyle: no_space
67
-
68
80
  Style/TrailingCommaInArguments:
69
81
  EnforcedStyleForMultiline: no_comma
70
82
 
@@ -5,21 +5,23 @@ rvm:
5
5
  - '1.9.3-p551'
6
6
  - '2.0.0-p648'
7
7
  - '2.1.10'
8
- - '2.2.6'
9
- - '2.3.3'
10
- - '2.4.0'
8
+ - '2.2.8'
9
+ - '2.3.5'
10
+ - '2.4.2'
11
11
  - 'ruby-head'
12
- - 'jruby-1.7.26'
13
12
  - 'jruby-9.0.5.0'
14
- - 'jruby-9.1.5.0'
13
+ - 'jruby-9.1.9.0'
14
+ before_install:
15
+ - gem update --system
16
+ - gem update bundler
15
17
  script: bundle exec rspec
16
18
  matrix:
17
19
  include:
18
20
  - env: RUBOCOP=✓
19
- rvm: '2.4.0'
21
+ rvm: '2.4.2'
20
22
  script: bundle exec rubocop
21
23
  - env: CHECK_RUBIES=✓
22
- rvm: '2.4.0'
24
+ rvm: '2.4.2'
23
25
  script: bundle exec travis_check_rubies
24
26
  allow_failures:
25
27
  - rvm: 'ruby-head'
@@ -0,0 +1,82 @@
1
+ # ChangeLog
2
+
3
+ ## unreleased
4
+
5
+ ## v1.5.0 (2017-11-17)
6
+
7
+ * Use thread pool instead of creating a thread for every iteration [@toy](https://github.com/toy)
8
+ * Handle `break` (also with argument) and exceptions raised in `each` method of enumerable [@toy](https://github.com/toy)
9
+ * Register `lazy` to run without threads [@toy](https://github.com/toy)
10
+
11
+ ## v1.4.0 (2017-03-19)
12
+
13
+ * Register `sum` and `uniq` to run in threads and `chunk_while` to run without threads [@toy](https://github.com/toy)
14
+ * Register `grep_v` to call block in threads [@toy](https://github.com/toy)
15
+ * Fix Segmentation fault in ruby 2.0 [@toy](https://github.com/toy)
16
+ * Fix not sending all block arguments to methods not returning original enumerable [@toy](https://github.com/toy)
17
+ * Fix for a [bug](https://bugs.ruby-lang.org/issues/13313) in ruby 2.4.0 causing Segmentation fault [@toy](https://github.com/toy)
18
+ * Clean up documentation [#1](https://github.com/toy/in_threads/pull/1) [@hollingberry](https://github.com/hollingberry)
19
+
20
+ ## v1.3.1 (2015-01-09)
21
+
22
+ * Register `to_h`, `slice_after` and `slice_when` to run without threads [@toy](https://github.com/toy)
23
+ * Remove special handling of methods running without threads [@toy](https://github.com/toy)
24
+
25
+ ## v1.3.0 (2014-10-31)
26
+
27
+ * Fix not thread safe code for jruby [@toy](https://github.com/toy)
28
+
29
+ ## v1.2.2 (2014-08-08)
30
+
31
+ * Fix silencing exceptions raised in blocks [@toy](https://github.com/toy)
32
+
33
+ ## v1.2.1 (2014-04-06)
34
+
35
+ * Use explicit requires instead of autoload [@toy](https://github.com/toy)
36
+
37
+ ## v1.2.0 (2013-08-20)
38
+
39
+ * Befriend with [`progress`](https://rubygems.org/gems/progress) gem [@toy](https://github.com/toy)
40
+ * Use `Delegator` instead of undefining methods [@toy](https://github.com/toy)
41
+
42
+ ## v1.1.2 (2013-08-02)
43
+
44
+ * Fix for jruby stalling [@toy](https://github.com/toy)
45
+ * Fix for rubinius not raising `NoMethodError` for undefined methods [@toy](https://github.com/toy)
46
+ * Register `to_set` to run without threads [@toy](https://github.com/toy)
47
+
48
+ ## v1.1.1 (2011-12-13)
49
+
50
+ * Decrease priority of initiating block execution for methods not returning original enumerable [@toy](https://github.com/toy)
51
+ * Call each of enumerable only once for methods not returning original enumerable [@toy](https://github.com/toy)
52
+ * Remove class method `enumerable_methods` [@toy](https://github.com/toy)
53
+
54
+ ## v1.1.0 (2011-12-08)
55
+
56
+ * Register `chunk` and `slice_before` to run without threads [@toy](https://github.com/toy)
57
+ * Register `flat_map` and `collect_concat` to run in threads [@toy](https://github.com/toy)
58
+ * Register `each_entry` to run in threads [@toy](https://github.com/toy)
59
+ * Register `each_with_object` to run without threads [@toy](https://github.com/toy)
60
+ * Add argument checking [@toy](https://github.com/toy)
61
+ * Use faster method for `each` [@toy](https://github.com/toy)
62
+
63
+ ## v1.0.0 (2011-12-05)
64
+
65
+ * Fix for blocks with multiple arguments (`each_with_index`, `enum_with_index`) [@toy](https://github.com/toy)
66
+ * Rewrite: properly working with all compatible `Enumerable` methods, otherwise running without threads [@toy](https://github.com/toy)
67
+
68
+ ## v0.0.4 (2010-12-15)
69
+
70
+ * Internal gem changes [@toy](https://github.com/toy)
71
+
72
+ ## v0.0.3 (2010-07-13)
73
+
74
+ * Fix thread count overflow [@toy](https://github.com/toy)
75
+
76
+ ## v0.0.2 (2009-12-29)
77
+
78
+ * Internal gem changes [@toy](https://github.com/toy)
79
+
80
+ ## v0.0.1 (2009-09-21)
81
+
82
+ * Initial [@toy](https://github.com/toy)
@@ -1,4 +1,4 @@
1
- Copyright (c) 2010-2017 Ivan Kuchin
1
+ Copyright (c) 2009-2017 Ivan Kuchin
2
2
 
3
3
  Permission is hereby granted, free of charge, to any person obtaining
4
4
  a copy of this software and associated documentation files (the
@@ -6,7 +6,7 @@
6
6
 
7
7
  # in_threads
8
8
 
9
- Easily execute Ruby code in parallel.
9
+ Run all possible enumerable methods in concurrent/parallel threads.
10
10
 
11
11
  ```ruby
12
12
  urls.in_threads(20).map do |url|
@@ -28,7 +28,7 @@ gem 'in_threads'
28
28
  $ bundle install
29
29
  ```
30
30
 
31
- Or, if you don't use Bundler, install it globally:
31
+ Or install globally:
32
32
 
33
33
  ```sh
34
34
  $ gem install in_threads
@@ -94,6 +94,14 @@ You can call any `Enumerable` method, but some (`#inject`, `#reduce`, `#max`,
94
94
  `#min`, `#sort`, `#to_a`, and others) cannot run concurrently, and so will
95
95
  simply act as if `in_threads` wasn't used.
96
96
 
97
+ ### Break and exceptions
98
+
99
+ Exceptions are caught and re-thrown after allowing blocks that are still running to finish.
100
+
101
+ **IMPORTANT**: only the first encountered exception is propagated, so it is recommended to handle exceptions in the block.
102
+
103
+ `break` is handled in ruby >= 1.9 and should be handled in jruby [after 9.1.9.0](https://github.com/jruby/jruby/issues/4697). Handling is done in special way: as blocks are run outside of original context, calls to `break` cause `LocalJumpError` which is caught and its result is returned.
104
+
97
105
  ## Copyright
98
106
 
99
- Copyright (c) 2010-2017 Ivan Kuchin. See LICENSE.txt for details.
107
+ Copyright (c) 2009-2017 Ivan Kuchin. See LICENSE.txt for details.
@@ -2,8 +2,8 @@
2
2
 
3
3
  Gem::Specification.new do |s|
4
4
  s.name = 'in_threads'
5
- s.version = '1.4.0'
6
- s.summary = %q{Execute ruby code in parallel}
5
+ s.version = '1.5.0'
6
+ s.summary = %q{Run all possible enumerable methods in concurrent/parallel threads}
7
7
  s.homepage = "http://github.com/toy/#{s.name}"
8
8
  s.authors = ['Ivan Kuchin']
9
9
  s.license = 'MIT'
@@ -18,6 +18,6 @@ Gem::Specification.new do |s|
18
18
  s.add_development_dependency 'rspec', '~> 3.0'
19
19
  s.add_development_dependency 'rspec-retry', '~> 0.3'
20
20
  if RUBY_VERSION >= '2.0'
21
- s.add_development_dependency 'rubocop', '~> 0.47'
21
+ s.add_development_dependency 'rubocop', '~> 0.49'
22
22
  end
23
23
  end
@@ -1,5 +1,4 @@
1
1
  require 'thread'
2
- require 'thwait'
3
2
  require 'delegate'
4
3
 
5
4
  Enumerable.class_eval do
@@ -31,84 +30,6 @@ end
31
30
  class InThreads < SimpleDelegator
32
31
  protected :__getobj__, :__setobj__
33
32
 
34
- # Use ThreadsWait to limit number of threads
35
- class ThreadLimiter
36
- # Initialize with limit
37
- def initialize(count)
38
- @count = count
39
- @waiter = ThreadsWait.new
40
- end
41
-
42
- # Without block behaves as `new`
43
- # With block yields it with `self` and ensures running of `finalize`
44
- def self.limit(count, &block)
45
- limiter = new(count)
46
- if block
47
- begin
48
- yield limiter
49
- ensure
50
- limiter.finalize
51
- end
52
- else
53
- limiter
54
- end
55
- end
56
-
57
- # Add thread to `ThreadsWait`, wait for finishing of one thread if limit
58
- # reached
59
- def <<(thread)
60
- @waiter.join_nowait(thread)
61
- @waiter.next_wait.join unless @waiter.threads.length < @count
62
- end
63
-
64
- # Wait for waiting threads
65
- def finalize
66
- @waiter.all_waits(&:join)
67
- end
68
- end
69
-
70
- # Yield objects of one enum in multiple places
71
- class Splitter
72
- # Enumerable using Queue
73
- class Transfer
74
- include Enumerable
75
-
76
- def initialize
77
- @queue = Queue.new
78
- end
79
-
80
- def <<(args)
81
- @queue << args
82
- end
83
-
84
- def finish
85
- @queue << nil
86
- end
87
-
88
- def each(&block)
89
- while (args = @queue.pop)
90
- block.call(*args)
91
- end
92
- nil # non reusable
93
- end
94
- end
95
-
96
- # Enums receiving items
97
- attr_reader :enums
98
-
99
- def initialize(enumerable, enum_count)
100
- @enums = Array.new(enum_count){ Transfer.new }
101
- @filler = Thread.new do
102
- enumerable.each do |*args|
103
- @enums.each do |enum|
104
- enum << args
105
- end
106
- end
107
- @enums.each(&:finish)
108
- end
109
- end
110
- end
111
-
112
33
  attr_reader :enumerable, :thread_count
113
34
  def initialize(enumerable, thread_count = 10, &block)
114
35
  super(enumerable)
@@ -130,7 +51,7 @@ class InThreads < SimpleDelegator
130
51
  class << self
131
52
  # Specify runner to use
132
53
  #
133
- # use :run_in_threads_consecutive, :for => %w[all? any? none? one?]
54
+ # use :run_in_threads_use_block_result, :for => %w[all? any? none? one?]
134
55
  #
135
56
  # `:for` is required
136
57
  # `:ignore_undefined` ignores methods which are not present in
@@ -143,7 +64,11 @@ class InThreads < SimpleDelegator
143
64
  next if ignore_undefined && !enumerable_method?(method)
144
65
  class_eval <<-RUBY
145
66
  def #{method}(*args, &block)
146
- #{runner}(:#{method}, *args, &block)
67
+ if block
68
+ #{runner}(:#{method}, *args, &block)
69
+ else
70
+ enumerable.#{method}(*args)
71
+ end
147
72
  end
148
73
  RUBY
149
74
  end
@@ -152,13 +77,12 @@ class InThreads < SimpleDelegator
152
77
  private
153
78
 
154
79
  def enumerable_method?(name)
155
- @enumerable_methods ||= Enumerable.instance_methods.map(&:to_sym)
156
- @enumerable_methods.include?(name.to_sym)
80
+ Enumerable.method_defined?(name)
157
81
  end
158
82
  end
159
83
 
160
- use :run_in_threads_return_original_enum, :for => %w[each]
161
- use :run_in_threads_return_original_enum, :for => %w[
84
+ use :run_in_threads_ignore_block_result, :for => %w[each]
85
+ use :run_in_threads_ignore_block_result, :for => %w[
162
86
  reverse_each
163
87
  each_with_index enum_with_index
164
88
  each_cons each_slice enum_cons enum_slice
@@ -166,7 +90,7 @@ class InThreads < SimpleDelegator
166
90
  cycle
167
91
  each_entry
168
92
  ], :ignore_undefined => true
169
- use :run_in_threads_consecutive, :for => %w[
93
+ use :run_in_threads_use_block_result, :for => %w[
170
94
  all? any? none? one?
171
95
  detect find find_index drop_while take_while
172
96
  partition find_all select reject count
@@ -183,9 +107,10 @@ class InThreads < SimpleDelegator
183
107
  include? member?
184
108
  each_with_object
185
109
  chunk chunk_while slice_before slice_after slice_when
110
+ lazy
186
111
  ].map(&:to_sym)
187
112
 
188
- # Special case method, works by applying `run_in_threads_consecutive` with
113
+ # Special case method, works by applying `run_in_threads_use_block_result` with
189
114
  # map on enumerable returned by blockless run
190
115
  def grep(*args, &block)
191
116
  if block
@@ -196,7 +121,7 @@ class InThreads < SimpleDelegator
196
121
  end
197
122
 
198
123
  if enumerable_method?(:grep_v)
199
- # Special case method, works by applying `run_in_threads_consecutive` with
124
+ # Special case method, works by applying `run_in_threads_use_block_result` with
200
125
  # map on enumerable returned by blockless run
201
126
  def grep_v(*args, &block)
202
127
  if block
@@ -214,46 +139,159 @@ class InThreads < SimpleDelegator
214
139
 
215
140
  protected
216
141
 
142
+ # Enum out of queue
143
+ class QueueEnum
144
+ include Enumerable
145
+
146
+ def initialize(size = nil)
147
+ @queue = size ? SizedQueue.new(size) : Queue.new
148
+ end
149
+
150
+ def each(&block)
151
+ while (args = @queue.pop)
152
+ block.call(*args)
153
+ end
154
+ nil # non reusable
155
+ end
156
+
157
+ def push(*args)
158
+ @queue.push(args) unless @closed
159
+ end
160
+
161
+ def close(clear = false)
162
+ @closed = true
163
+ @queue.clear if clear
164
+ @queue.push(nil)
165
+ end
166
+ end
167
+
168
+ # Thread pool
169
+ class Pool
170
+ attr_reader :exception
171
+
172
+ def initialize(thread_count)
173
+ @queue = Queue.new
174
+ @mutex = Mutex.new
175
+ @pool = Array.new(thread_count) do
176
+ Thread.new do
177
+ while (block = @queue.pop)
178
+ block.call
179
+ break if stop?
180
+ end
181
+ end
182
+ end
183
+ end
184
+
185
+ def run(&block)
186
+ @queue.push(block)
187
+ end
188
+
189
+ def stop?
190
+ @stop || @exception
191
+ end
192
+
193
+ def stop!
194
+ @stop = true
195
+ end
196
+
197
+ def finalize
198
+ @pool.
199
+ each{ @queue.push(nil) }.
200
+ each(&:join)
201
+ end
202
+
203
+ def catch
204
+ yield
205
+ rescue Exception => e
206
+ @mutex.synchronize{ @exception ||= e } unless @exception
207
+ nil
208
+ end
209
+ end
210
+
217
211
  # Use for methods which don't use block result
218
- def run_in_threads_return_original_enum(method, *args, &block)
219
- if block
220
- ThreadLimiter.limit(thread_count) do |limiter|
212
+ def run_in_threads_ignore_block_result(method, *args, &block)
213
+ pool = Pool.new(thread_count)
214
+ wait = SizedQueue.new(thread_count - 1)
215
+ begin
216
+ pool.catch do
221
217
  enumerable.send(method, *args) do |*block_args|
222
- limiter << Thread.new{ block.call(*block_args) }
218
+ pool.run do
219
+ pool.catch do
220
+ block.call(*block_args)
221
+ end
222
+ wait.pop
223
+ end
224
+ wait.push(nil)
225
+ break if pool.stop?
223
226
  end
224
227
  end
225
- else
226
- enumerable.send(method, *args)
228
+ ensure
229
+ pool.finalize
230
+ if (e = pool.exception)
231
+ return e.exit_value if e.is_a?(LocalJumpError) && e.reason == :break
232
+ fail e
233
+ end
227
234
  end
228
235
  end
229
236
 
230
237
  # Use for methods which do use block result
231
- def run_in_threads_consecutive(method, *args, &block)
232
- if block
233
- enum_a, enum_b = Splitter.new(enumerable, 2).enums
234
- results = Queue.new
235
- runner = Thread.new do
236
- Thread.current.priority = -1
237
- ThreadLimiter.limit(thread_count) do |limiter|
238
- enum_a.each do |*block_args|
239
- break if Thread.current[:stop]
240
- thread = Thread.new{ block.call(*block_args) }
241
- results << thread
242
- limiter << thread
238
+ def run_in_threads_use_block_result(method, *args, &block)
239
+ pool = Pool.new(thread_count)
240
+ enum_a = QueueEnum.new
241
+ enum_b = QueueEnum.new(thread_count - 1)
242
+ results = SizedQueue.new(thread_count - 1)
243
+ filler = filler_thread(pool, [enum_a, enum_b])
244
+ runner = runner_thread(pool, enum_a, results, &block)
245
+
246
+ begin
247
+ pool.catch do
248
+ enum_b.send(method, *args) do
249
+ result = results.pop.pop
250
+ break if pool.stop?
251
+ result
252
+ end
253
+ end
254
+ ensure
255
+ pool.stop!
256
+ enum_a.close(true)
257
+ enum_b.close(true)
258
+ results.clear
259
+ pool.finalize
260
+ runner.join
261
+ filler.join
262
+ if (e = pool.exception)
263
+ return e.exit_value if e.is_a?(LocalJumpError) && e.reason == :break
264
+ fail e
265
+ end
266
+ end
267
+ end
268
+
269
+ private
270
+
271
+ def filler_thread(pool, enums)
272
+ Thread.new do
273
+ pool.catch do
274
+ enumerable.each do |*block_args|
275
+ enums.each do |enum|
276
+ enum.push(*block_args)
243
277
  end
278
+ break if pool.stop?
244
279
  end
245
280
  end
281
+ enums.each(&:close)
282
+ end
283
+ end
246
284
 
247
- begin
248
- enum_b.send(method, *args) do
249
- results.pop.value
285
+ def runner_thread(pool, enum, results, &block)
286
+ Thread.new do
287
+ enum.each do |*block_args|
288
+ queue = Queue.new
289
+ pool.run do
290
+ queue.push(pool.catch{ block.call(*block_args) })
250
291
  end
251
- ensure
252
- runner[:stop] = true
253
- runner.join
292
+ results.push(queue)
293
+ break if pool.stop?
254
294
  end
255
- else
256
- enumerable.send(method, *args)
257
295
  end
258
296
  end
259
297
  end