brakeman 5.0.1 → 5.1.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (78) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGES.md +41 -0
  3. data/bundle/load.rb +3 -2
  4. data/bundle/ruby/2.7.0/gems/parallel-1.20.1/MIT-LICENSE.txt +20 -0
  5. data/bundle/ruby/2.7.0/gems/parallel-1.20.1/lib/parallel.rb +523 -0
  6. data/bundle/ruby/2.7.0/gems/parallel-1.20.1/lib/parallel/processor_count.rb +42 -0
  7. data/bundle/ruby/2.7.0/gems/parallel-1.20.1/lib/parallel/version.rb +3 -0
  8. data/bundle/ruby/2.7.0/gems/{ruby_parser-3.15.1 → ruby_parser-3.16.0}/History.rdoc +19 -0
  9. data/bundle/ruby/2.7.0/gems/{ruby_parser-3.15.1 → ruby_parser-3.16.0}/Manifest.txt +2 -0
  10. data/bundle/ruby/2.7.0/gems/{ruby_parser-3.15.1 → ruby_parser-3.16.0}/README.rdoc +0 -0
  11. data/bundle/ruby/2.7.0/gems/{ruby_parser-3.15.1 → ruby_parser-3.16.0}/compare/normalize.rb +2 -2
  12. data/bundle/ruby/2.7.0/gems/{ruby_parser-3.15.1 → ruby_parser-3.16.0}/debugging.md +0 -0
  13. data/bundle/ruby/2.7.0/gems/{ruby_parser-3.15.1 → ruby_parser-3.16.0}/lib/rp_extensions.rb +0 -0
  14. data/bundle/ruby/2.7.0/gems/{ruby_parser-3.15.1 → ruby_parser-3.16.0}/lib/rp_stringscanner.rb +0 -0
  15. data/bundle/ruby/2.7.0/gems/{ruby_parser-3.15.1 → ruby_parser-3.16.0}/lib/ruby20_parser.rb +0 -0
  16. data/bundle/ruby/2.7.0/gems/{ruby_parser-3.15.1 → ruby_parser-3.16.0}/lib/ruby20_parser.y +0 -0
  17. data/bundle/ruby/2.7.0/gems/{ruby_parser-3.15.1 → ruby_parser-3.16.0}/lib/ruby21_parser.rb +0 -0
  18. data/bundle/ruby/2.7.0/gems/{ruby_parser-3.15.1 → ruby_parser-3.16.0}/lib/ruby21_parser.y +0 -0
  19. data/bundle/ruby/2.7.0/gems/{ruby_parser-3.15.1 → ruby_parser-3.16.0}/lib/ruby22_parser.rb +0 -0
  20. data/bundle/ruby/2.7.0/gems/{ruby_parser-3.15.1 → ruby_parser-3.16.0}/lib/ruby22_parser.y +0 -0
  21. data/bundle/ruby/2.7.0/gems/{ruby_parser-3.15.1 → ruby_parser-3.16.0}/lib/ruby23_parser.rb +0 -0
  22. data/bundle/ruby/2.7.0/gems/{ruby_parser-3.15.1 → ruby_parser-3.16.0}/lib/ruby23_parser.y +0 -0
  23. data/bundle/ruby/2.7.0/gems/{ruby_parser-3.15.1 → ruby_parser-3.16.0}/lib/ruby24_parser.rb +0 -0
  24. data/bundle/ruby/2.7.0/gems/{ruby_parser-3.15.1 → ruby_parser-3.16.0}/lib/ruby24_parser.y +0 -0
  25. data/bundle/ruby/2.7.0/gems/{ruby_parser-3.15.1 → ruby_parser-3.16.0}/lib/ruby25_parser.rb +0 -0
  26. data/bundle/ruby/2.7.0/gems/{ruby_parser-3.15.1 → ruby_parser-3.16.0}/lib/ruby25_parser.y +0 -0
  27. data/bundle/ruby/2.7.0/gems/{ruby_parser-3.15.1 → ruby_parser-3.16.0}/lib/ruby26_parser.rb +0 -0
  28. data/bundle/ruby/2.7.0/gems/{ruby_parser-3.15.1 → ruby_parser-3.16.0}/lib/ruby26_parser.y +0 -0
  29. data/bundle/ruby/2.7.0/gems/{ruby_parser-3.15.1 → ruby_parser-3.16.0}/lib/ruby27_parser.rb +0 -0
  30. data/bundle/ruby/2.7.0/gems/{ruby_parser-3.15.1 → ruby_parser-3.16.0}/lib/ruby27_parser.y +0 -0
  31. data/bundle/ruby/2.7.0/gems/ruby_parser-3.16.0/lib/ruby30_parser.rb +7358 -0
  32. data/bundle/ruby/2.7.0/gems/ruby_parser-3.16.0/lib/ruby30_parser.y +2703 -0
  33. data/bundle/ruby/2.7.0/gems/{ruby_parser-3.15.1 → ruby_parser-3.16.0}/lib/ruby_lexer.rb +0 -0
  34. data/bundle/ruby/2.7.0/gems/{ruby_parser-3.15.1 → ruby_parser-3.16.0}/lib/ruby_lexer.rex +0 -0
  35. data/bundle/ruby/2.7.0/gems/{ruby_parser-3.15.1 → ruby_parser-3.16.0}/lib/ruby_lexer.rex.rb +0 -0
  36. data/bundle/ruby/2.7.0/gems/{ruby_parser-3.15.1 → ruby_parser-3.16.0}/lib/ruby_parser.rb +2 -0
  37. data/bundle/ruby/2.7.0/gems/{ruby_parser-3.15.1 → ruby_parser-3.16.0}/lib/ruby_parser.yy +2 -0
  38. data/bundle/ruby/2.7.0/gems/{ruby_parser-3.15.1 → ruby_parser-3.16.0}/lib/ruby_parser_extras.rb +2 -2
  39. data/bundle/ruby/2.7.0/gems/{ruby_parser-3.15.1 → ruby_parser-3.16.0}/tools/munge.rb +0 -0
  40. data/bundle/ruby/2.7.0/gems/{ruby_parser-3.15.1 → ruby_parser-3.16.0}/tools/ripper.rb +0 -0
  41. data/bundle/ruby/2.7.0/gems/{sexp_processor-4.15.2 → sexp_processor-4.15.3}/History.rdoc +6 -0
  42. data/bundle/ruby/2.7.0/gems/{sexp_processor-4.15.2 → sexp_processor-4.15.3}/Manifest.txt +0 -0
  43. data/bundle/ruby/2.7.0/gems/{sexp_processor-4.15.2 → sexp_processor-4.15.3}/README.rdoc +0 -0
  44. data/bundle/ruby/2.7.0/gems/{sexp_processor-4.15.2 → sexp_processor-4.15.3}/lib/composite_sexp_processor.rb +0 -0
  45. data/bundle/ruby/2.7.0/gems/{sexp_processor-4.15.2 → sexp_processor-4.15.3}/lib/pt_testcase.rb +2 -2
  46. data/bundle/ruby/2.7.0/gems/{sexp_processor-4.15.2 → sexp_processor-4.15.3}/lib/sexp.rb +0 -0
  47. data/bundle/ruby/2.7.0/gems/{sexp_processor-4.15.2 → sexp_processor-4.15.3}/lib/sexp_matcher.rb +0 -0
  48. data/bundle/ruby/2.7.0/gems/{sexp_processor-4.15.2 → sexp_processor-4.15.3}/lib/sexp_processor.rb +1 -1
  49. data/bundle/ruby/2.7.0/gems/{sexp_processor-4.15.2 → sexp_processor-4.15.3}/lib/strict_sexp.rb +0 -0
  50. data/bundle/ruby/2.7.0/gems/{sexp_processor-4.15.2 → sexp_processor-4.15.3}/lib/unique.rb +0 -0
  51. data/lib/brakeman.rb +6 -0
  52. data/lib/brakeman/checks/check_detailed_exceptions.rb +1 -1
  53. data/lib/brakeman/checks/check_evaluation.rb +1 -1
  54. data/lib/brakeman/checks/check_execute.rb +10 -0
  55. data/lib/brakeman/checks/check_render.rb +15 -1
  56. data/lib/brakeman/checks/check_sanitize_methods.rb +2 -1
  57. data/lib/brakeman/checks/check_sql.rb +58 -8
  58. data/lib/brakeman/checks/check_verb_confusion.rb +1 -1
  59. data/lib/brakeman/file_parser.rb +45 -15
  60. data/lib/brakeman/options.rb +7 -2
  61. data/lib/brakeman/processors/alias_processor.rb +85 -9
  62. data/lib/brakeman/processors/controller_alias_processor.rb +6 -43
  63. data/lib/brakeman/processors/lib/call_conversion_helper.rb +10 -6
  64. data/lib/brakeman/processors/library_processor.rb +9 -0
  65. data/lib/brakeman/processors/model_processor.rb +31 -0
  66. data/lib/brakeman/report.rb +4 -1
  67. data/lib/brakeman/report/ignore/interactive.rb +1 -1
  68. data/lib/brakeman/report/report_github.rb +31 -0
  69. data/lib/brakeman/report/report_sarif.rb +21 -2
  70. data/lib/brakeman/rescanner.rb +1 -1
  71. data/lib/brakeman/scanner.rb +4 -1
  72. data/lib/brakeman/tracker.rb +33 -4
  73. data/lib/brakeman/tracker/collection.rb +57 -7
  74. data/lib/brakeman/tracker/method_info.rb +70 -0
  75. data/lib/brakeman/util.rb +34 -18
  76. data/lib/brakeman/version.rb +1 -1
  77. data/lib/ruby_parser/bm_sexp.rb +14 -0
  78. metadata +51 -43
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 4b7adec7f0012e7c4b5605dc81db6722b1b6c711cafc12ab60ddd8f8c969309e
4
- data.tar.gz: 602f31369d55351c1ec459de946e555710e85f77578f2711dfb6886bd4781757
3
+ metadata.gz: cafb4506d0cbb4ef2ab84459c03a8d356ed916c29ceca5104536b836162a91ed
4
+ data.tar.gz: b1166612e496c77ffc41f07dc4c7a1226c19ee0726d1e02e3241c792ce4463a8
5
5
  SHA512:
6
- metadata.gz: 603312f7dafcb44a28d8f1d74c0088d86c3b82263ff413c41a855945dc4421f231548496ae98e99bb080f9cc947a9314f0997a771115e013859364f96763f403
7
- data.tar.gz: 2fd04bd7ea6e584fe05c60002a369c27945a6fe90c805b20ce15c3290f474437fdf67a2b7e4d59bf373ac7ce3d678d3a0684edc55b211c86a1e7edf4ff40c27d
6
+ metadata.gz: 70920cb9dd7d8647ee9767502575c8336768cfe7d6c418cef810c90b7f3a9a9ea2fb48fb70af123dd8853bb60851cac3def642b0412fb5a4422c47b2f37fd6dd
7
+ data.tar.gz: '083ba7226c065d0e15ddaf5bbf3023326a35bcf167c9070080629fedb517110726d17fac3cf3c4f2f24232ce49dc1b5476d4bf46c60aa55869c4407c6e79bc92'
data/CHANGES.md CHANGED
@@ -1,3 +1,44 @@
1
+ # 5.1.1 - 2021-07-19
2
+
3
+ * Unrefactor IgnoreConfig's use of `Brakeman::FilePath`
4
+
5
+ # 5.1.0 - 2021-07-19
6
+
7
+ * Initial support for ActiveRecord enums
8
+ * Support `Hash#include?`
9
+ * Interprocedural dataflow from very simple class methods
10
+ * Fix SARIF report when checks have no description (Eli Block)
11
+ * Add ignored warnings to SARIF report (Eli Block)
12
+ * Add `--sql-safe-methods` option (Esty Scheiner)
13
+ * Update SQL injection check for Rails 6.0/6.1
14
+ * Fix false positive in command injection with `Open3.capture` (Richard Fitzgerald)
15
+ * Fix infinite loop on mixin self-includes (Andrew Szczepanski)
16
+ * Ignore dates in SQL
17
+ * Refactor `cookie?`/`param?` methods (Keenan Brock)
18
+ * Ignore renderables in dynamic render path check (Brad Parker)
19
+ * Support `Array#push`
20
+ * Better `Array#join` support
21
+ * Adjust copy of `--interactive` menu (Elia Schito)
22
+ * Support `Array#*`
23
+ * Better method definition tracking and lookup
24
+ * Support `Hash#values` and `Hash#values_at`
25
+ * Check for user-controlled evaluation even if it's a call target
26
+ * Support `Array#fetch` and `Hash#fetch`
27
+ * Ignore `sanitize_sql_like` in SQL
28
+ * Ignore method calls on numbers in SQL
29
+ * Add GitHub Actions format (Klaus Badelt)
30
+ * Read and parse files in parallel
31
+
32
+ # 5.0.4 - 2021-06-08
33
+
34
+ (brakeman gem release only)
35
+
36
+ * Update bundled `ruby_parser` to include argument forwarding support
37
+
38
+ # 5.0.2 - 2021-06-07
39
+
40
+ * Fix Loofah version check
41
+
1
42
  # 5.0.1 - 2021-04-27
2
43
 
3
44
  * Detect `::Rails.application.configure` too
data/bundle/load.rb CHANGED
@@ -1,15 +1,16 @@
1
1
  path = File.expand_path('../..', __FILE__)
2
2
  $:.unshift "#{path}/bundle/ruby/2.7.0/gems/temple-0.8.2/lib"
3
+ $:.unshift "#{path}/bundle/ruby/2.7.0/gems/ruby_parser-3.16.0/lib"
3
4
  $:.unshift "#{path}/bundle/ruby/2.7.0/gems/unicode-display_width-1.7.0/lib"
4
5
  $:.unshift "#{path}/bundle/ruby/2.7.0/gems/tilt-2.0.10/lib"
5
6
  $:.unshift "#{path}/bundle/ruby/2.7.0/gems/slim-4.1.0/lib"
6
- $:.unshift "#{path}/bundle/ruby/2.7.0/gems/sexp_processor-4.15.2/lib"
7
7
  $:.unshift "#{path}/bundle/ruby/2.7.0/gems/highline-2.0.3/lib"
8
8
  $:.unshift "#{path}/bundle/ruby/2.7.0/gems/ruby2ruby-2.4.4/lib"
9
9
  $:.unshift "#{path}/bundle/ruby/2.7.0/gems/terminal-table-1.8.0/lib"
10
+ $:.unshift "#{path}/bundle/ruby/2.7.0/gems/sexp_processor-4.15.3/lib"
11
+ $:.unshift "#{path}/bundle/ruby/2.7.0/gems/parallel-1.20.1/lib"
10
12
  $:.unshift "#{path}/bundle/ruby/2.7.0/gems/ruby_parser-legacy-1.0.0/lib"
11
13
  $:.unshift "#{path}/bundle/ruby/2.7.0/gems/erubis-2.7.0/lib"
12
14
  $:.unshift "#{path}/bundle/ruby/2.7.0/gems/haml-5.2.1/lib"
13
15
  $:.unshift "#{path}/bundle/ruby/2.7.0/gems/rexml-3.2.5/lib"
14
- $:.unshift "#{path}/bundle/ruby/2.7.0/gems/ruby_parser-3.15.1/lib"
15
16
  $:.unshift "#{path}/bundle/ruby/2.7.0/gems/safe_yaml-1.0.5/lib"
@@ -0,0 +1,20 @@
1
+ Copyright (C) 2013 Michael Grosser <michael@grosser.it>
2
+
3
+ Permission is hereby granted, free of charge, to any person obtaining
4
+ a copy of this software and associated documentation files (the
5
+ "Software"), to deal in the Software without restriction, including
6
+ without limitation the rights to use, copy, modify, merge, publish,
7
+ distribute, sublicense, and/or sell copies of the Software, and to
8
+ permit persons to whom the Software is furnished to do so, subject to
9
+ the following conditions:
10
+
11
+ The above copyright notice and this permission notice shall be
12
+ included in all copies or substantial portions of the Software.
13
+
14
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
15
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
16
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
17
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
18
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
19
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
20
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
@@ -0,0 +1,523 @@
1
+ require 'rbconfig'
2
+ require 'parallel/version'
3
+ require 'parallel/processor_count'
4
+
5
+ module Parallel
6
+ extend ProcessorCount
7
+
8
+ Stop = Object.new.freeze
9
+
10
+ class DeadWorker < StandardError
11
+ end
12
+
13
+ class Break < StandardError
14
+ attr_reader :value
15
+ def initialize(value = nil)
16
+ @value = value
17
+ end
18
+ end
19
+
20
+ class Kill < Break
21
+ end
22
+
23
+ class UndumpableException < StandardError
24
+ attr_reader :backtrace
25
+ def initialize(original)
26
+ super "#{original.class}: #{original.message}"
27
+ @backtrace = original.backtrace
28
+ end
29
+ end
30
+
31
+ class ExceptionWrapper
32
+ attr_reader :exception
33
+ def initialize(exception)
34
+ # Remove the bindings stack added by the better_errors gem,
35
+ # because it cannot be marshalled
36
+ if exception.instance_variable_defined? :@__better_errors_bindings_stack
37
+ exception.send :remove_instance_variable, :@__better_errors_bindings_stack
38
+ end
39
+
40
+ @exception =
41
+ begin
42
+ Marshal.dump(exception) && exception
43
+ rescue
44
+ UndumpableException.new(exception)
45
+ end
46
+ end
47
+ end
48
+
49
+ class Worker
50
+ attr_reader :pid, :read, :write
51
+ attr_accessor :thread
52
+ def initialize(read, write, pid)
53
+ @read, @write, @pid = read, write, pid
54
+ end
55
+
56
+ def stop
57
+ close_pipes
58
+ wait # if it goes zombie, rather wait here to be able to debug
59
+ end
60
+
61
+ # might be passed to started_processes and simultaneously closed by another thread
62
+ # when running in isolation mode, so we have to check if it is closed before closing
63
+ def close_pipes
64
+ read.close unless read.closed?
65
+ write.close unless write.closed?
66
+ end
67
+
68
+ def work(data)
69
+ begin
70
+ Marshal.dump(data, write)
71
+ rescue Errno::EPIPE
72
+ raise DeadWorker
73
+ end
74
+
75
+ result = begin
76
+ Marshal.load(read)
77
+ rescue EOFError
78
+ raise DeadWorker
79
+ end
80
+ raise result.exception if ExceptionWrapper === result
81
+ result
82
+ end
83
+
84
+ private
85
+
86
+ def wait
87
+ Process.wait(pid)
88
+ rescue Interrupt
89
+ # process died
90
+ end
91
+ end
92
+
93
+ class JobFactory
94
+ def initialize(source, mutex)
95
+ @lambda = (source.respond_to?(:call) && source) || queue_wrapper(source)
96
+ @source = source.to_a unless @lambda # turn Range and other Enumerable-s into an Array
97
+ @mutex = mutex
98
+ @index = -1
99
+ @stopped = false
100
+ end
101
+
102
+ def next
103
+ if producer?
104
+ # - index and item stay in sync
105
+ # - do not call lambda after it has returned Stop
106
+ item, index = @mutex.synchronize do
107
+ return if @stopped
108
+ item = @lambda.call
109
+ @stopped = (item == Stop)
110
+ return if @stopped
111
+ [item, @index += 1]
112
+ end
113
+ else
114
+ index = @mutex.synchronize { @index += 1 }
115
+ return if index >= size
116
+ item = @source[index]
117
+ end
118
+ [item, index]
119
+ end
120
+
121
+ def size
122
+ if producer?
123
+ Float::INFINITY
124
+ else
125
+ @source.size
126
+ end
127
+ end
128
+
129
+ # generate item that is sent to workers
130
+ # just index is faster + less likely to blow up with unserializable errors
131
+ def pack(item, index)
132
+ producer? ? [item, index] : index
133
+ end
134
+
135
+ # unpack item that is sent to workers
136
+ def unpack(data)
137
+ producer? ? data : [@source[data], data]
138
+ end
139
+
140
+ private
141
+
142
+ def producer?
143
+ @lambda
144
+ end
145
+
146
+ def queue_wrapper(array)
147
+ array.respond_to?(:num_waiting) && array.respond_to?(:pop) && lambda { array.pop(false) }
148
+ end
149
+ end
150
+
151
+ class UserInterruptHandler
152
+ INTERRUPT_SIGNAL = :SIGINT
153
+
154
+ class << self
155
+ # kill all these pids or threads if user presses Ctrl+c
156
+ def kill_on_ctrl_c(pids, options)
157
+ @to_be_killed ||= []
158
+ old_interrupt = nil
159
+ signal = options.fetch(:interrupt_signal, INTERRUPT_SIGNAL)
160
+
161
+ if @to_be_killed.empty?
162
+ old_interrupt = trap_interrupt(signal) do
163
+ $stderr.puts 'Parallel execution interrupted, exiting ...'
164
+ @to_be_killed.flatten.each { |pid| kill(pid) }
165
+ end
166
+ end
167
+
168
+ @to_be_killed << pids
169
+
170
+ yield
171
+ ensure
172
+ @to_be_killed.pop # do not kill pids that could be used for new processes
173
+ restore_interrupt(old_interrupt, signal) if @to_be_killed.empty?
174
+ end
175
+
176
+ def kill(thing)
177
+ Process.kill(:KILL, thing)
178
+ rescue Errno::ESRCH
179
+ # some linux systems already automatically killed the children at this point
180
+ # so we just ignore them not being there
181
+ end
182
+
183
+ private
184
+
185
+ def trap_interrupt(signal)
186
+ old = Signal.trap signal, 'IGNORE'
187
+
188
+ Signal.trap signal do
189
+ yield
190
+ if !old || old == "DEFAULT"
191
+ raise Interrupt
192
+ else
193
+ old.call
194
+ end
195
+ end
196
+
197
+ old
198
+ end
199
+
200
+ def restore_interrupt(old, signal)
201
+ Signal.trap signal, old
202
+ end
203
+ end
204
+ end
205
+
206
+ class << self
207
+ def in_threads(options={:count => 2})
208
+ threads = []
209
+ count, _ = extract_count_from_options(options)
210
+
211
+ Thread.handle_interrupt(Exception => :never) do
212
+ begin
213
+ Thread.handle_interrupt(Exception => :immediate) do
214
+ count.times do |i|
215
+ threads << Thread.new { yield(i) }
216
+ end
217
+ threads.map(&:value)
218
+ end
219
+ ensure
220
+ threads.each(&:kill)
221
+ end
222
+ end
223
+ end
224
+
225
+ def in_processes(options = {}, &block)
226
+ count, options = extract_count_from_options(options)
227
+ count ||= processor_count
228
+ map(0...count, options.merge(:in_processes => count), &block)
229
+ end
230
+
231
+ def each(array, options={}, &block)
232
+ map(array, options.merge(:preserve_results => false), &block)
233
+ end
234
+
235
+ def any?(*args, &block)
236
+ raise "You must provide a block when calling #any?" if block.nil?
237
+ !each(*args) { |*a| raise Kill if block.call(*a) }
238
+ end
239
+
240
+ def all?(*args, &block)
241
+ raise "You must provide a block when calling #all?" if block.nil?
242
+ !!each(*args) { |*a| raise Kill unless block.call(*a) }
243
+ end
244
+
245
+ def each_with_index(array, options={}, &block)
246
+ each(array, options.merge(:with_index => true), &block)
247
+ end
248
+
249
+ def map(source, options = {}, &block)
250
+ options = options.dup
251
+ options[:mutex] = Mutex.new
252
+
253
+ if options[:in_processes] && options[:in_threads]
254
+ raise ArgumentError.new("Please specify only one of `in_processes` or `in_threads`.")
255
+ elsif RUBY_PLATFORM =~ /java/ and not options[:in_processes]
256
+ method = :in_threads
257
+ size = options[method] || processor_count
258
+ elsif options[:in_threads]
259
+ method = :in_threads
260
+ size = options[method]
261
+ else
262
+ method = :in_processes
263
+ if Process.respond_to?(:fork)
264
+ size = options[method] || processor_count
265
+ else
266
+ warn "Process.fork is not supported by this Ruby"
267
+ size = 0
268
+ end
269
+ end
270
+
271
+ job_factory = JobFactory.new(source, options[:mutex])
272
+ size = [job_factory.size, size].min
273
+
274
+ options[:return_results] = (options[:preserve_results] != false || !!options[:finish])
275
+ add_progress_bar!(job_factory, options)
276
+
277
+ result =
278
+ if size == 0
279
+ work_direct(job_factory, options, &block)
280
+ elsif method == :in_threads
281
+ work_in_threads(job_factory, options.merge(:count => size), &block)
282
+ else
283
+ work_in_processes(job_factory, options.merge(:count => size), &block)
284
+ end
285
+
286
+ return result.value if result.is_a?(Break)
287
+ raise result if result.is_a?(Exception)
288
+ options[:return_results] ? result : source
289
+ end
290
+
291
+ def map_with_index(array, options={}, &block)
292
+ map(array, options.merge(:with_index => true), &block)
293
+ end
294
+
295
+ def flat_map(*args, &block)
296
+ map(*args, &block).flatten(1)
297
+ end
298
+
299
+ def worker_number
300
+ Thread.current[:parallel_worker_number]
301
+ end
302
+
303
+ # TODO: this does not work when doing threads in forks, so should remove and yield the number instead if needed
304
+ def worker_number=(worker_num)
305
+ Thread.current[:parallel_worker_number] = worker_num
306
+ end
307
+
308
+ private
309
+
310
+ def add_progress_bar!(job_factory, options)
311
+ if progress_options = options[:progress]
312
+ raise "Progressbar can only be used with array like items" if job_factory.size == Float::INFINITY
313
+ require 'ruby-progressbar'
314
+
315
+ if progress_options == true
316
+ progress_options = { title: "Progress" }
317
+ elsif progress_options.respond_to? :to_str
318
+ progress_options = { title: progress_options.to_str }
319
+ end
320
+
321
+ progress_options = {
322
+ total: job_factory.size,
323
+ format: '%t |%E | %B | %a'
324
+ }.merge(progress_options)
325
+
326
+ progress = ProgressBar.create(progress_options)
327
+ old_finish = options[:finish]
328
+ options[:finish] = lambda do |item, i, result|
329
+ old_finish.call(item, i, result) if old_finish
330
+ progress.increment
331
+ end
332
+ end
333
+ end
334
+
335
+ def work_direct(job_factory, options, &block)
336
+ self.worker_number = 0
337
+ results = []
338
+ exception = nil
339
+ begin
340
+ while set = job_factory.next
341
+ item, index = set
342
+ results << with_instrumentation(item, index, options) do
343
+ call_with_index(item, index, options, &block)
344
+ end
345
+ end
346
+ rescue
347
+ exception = $!
348
+ end
349
+ exception || results
350
+ ensure
351
+ self.worker_number = nil
352
+ end
353
+
354
+ def work_in_threads(job_factory, options, &block)
355
+ raise "interrupt_signal is no longer supported for threads" if options[:interrupt_signal]
356
+ results = []
357
+ results_mutex = Mutex.new # arrays are not thread-safe on jRuby
358
+ exception = nil
359
+
360
+ in_threads(options) do |worker_num|
361
+ self.worker_number = worker_num
362
+ # as long as there are more jobs, work on one of them
363
+ while !exception && set = job_factory.next
364
+ begin
365
+ item, index = set
366
+ result = with_instrumentation item, index, options do
367
+ call_with_index(item, index, options, &block)
368
+ end
369
+ results_mutex.synchronize { results[index] = result }
370
+ rescue
371
+ exception = $!
372
+ end
373
+ end
374
+ end
375
+
376
+ exception || results
377
+ end
378
+
379
+ def work_in_processes(job_factory, options, &blk)
380
+ workers = create_workers(job_factory, options, &blk)
381
+ results = []
382
+ results_mutex = Mutex.new # arrays are not thread-safe
383
+ exception = nil
384
+
385
+ UserInterruptHandler.kill_on_ctrl_c(workers.map(&:pid), options) do
386
+ in_threads(options) do |i|
387
+ worker = workers[i]
388
+ worker.thread = Thread.current
389
+ worked = false
390
+
391
+ begin
392
+ loop do
393
+ break if exception
394
+ item, index = job_factory.next
395
+ break unless index
396
+
397
+ if options[:isolation]
398
+ worker = replace_worker(job_factory, workers, i, options, blk) if worked
399
+ worked = true
400
+ worker.thread = Thread.current
401
+ end
402
+
403
+ begin
404
+ result = with_instrumentation item, index, options do
405
+ worker.work(job_factory.pack(item, index))
406
+ end
407
+ results_mutex.synchronize { results[index] = result } # arrays are not threads safe on jRuby
408
+ rescue StandardError => e
409
+ exception = e
410
+ if Kill === exception
411
+ (workers - [worker]).each do |w|
412
+ w.thread.kill if w.thread
413
+ UserInterruptHandler.kill(w.pid)
414
+ end
415
+ end
416
+ end
417
+ end
418
+ ensure
419
+ worker.stop
420
+ end
421
+ end
422
+ end
423
+ exception || results
424
+ end
425
+
426
+ def replace_worker(job_factory, workers, i, options, blk)
427
+ options[:mutex].synchronize do
428
+ # old worker is no longer used ... stop it
429
+ worker = workers[i]
430
+ worker.stop
431
+
432
+ # create a new replacement worker
433
+ running = workers - [worker]
434
+ workers[i] = worker(job_factory, options.merge(started_workers: running, worker_number: i), &blk)
435
+ end
436
+ end
437
+
438
+ def create_workers(job_factory, options, &block)
439
+ workers = []
440
+ Array.new(options[:count]).each_with_index do |_, i|
441
+ workers << worker(job_factory, options.merge(started_workers: workers, worker_number: i), &block)
442
+ end
443
+ workers
444
+ end
445
+
446
+ def worker(job_factory, options, &block)
447
+ child_read, parent_write = IO.pipe
448
+ parent_read, child_write = IO.pipe
449
+
450
+ pid = Process.fork do
451
+ self.worker_number = options[:worker_number]
452
+
453
+ begin
454
+ options.delete(:started_workers).each(&:close_pipes)
455
+
456
+ parent_write.close
457
+ parent_read.close
458
+
459
+ process_incoming_jobs(child_read, child_write, job_factory, options, &block)
460
+ ensure
461
+ child_read.close
462
+ child_write.close
463
+ end
464
+ end
465
+
466
+ child_read.close
467
+ child_write.close
468
+
469
+ Worker.new(parent_read, parent_write, pid)
470
+ end
471
+
472
+ def process_incoming_jobs(read, write, job_factory, options, &block)
473
+ until read.eof?
474
+ data = Marshal.load(read)
475
+ item, index = job_factory.unpack(data)
476
+ result = begin
477
+ call_with_index(item, index, options, &block)
478
+ # https://github.com/rspec/rspec-support/blob/673133cdd13b17077b3d88ece8d7380821f8d7dc/lib/rspec/support.rb#L132-L140
479
+ rescue NoMemoryError, SignalException, Interrupt, SystemExit
480
+ raise $!
481
+ rescue Exception
482
+ ExceptionWrapper.new($!)
483
+ end
484
+ begin
485
+ Marshal.dump(result, write)
486
+ rescue Errno::EPIPE
487
+ return # parent thread already dead
488
+ end
489
+ end
490
+ end
491
+
492
+ # options is either a Integer or a Hash with :count
493
+ def extract_count_from_options(options)
494
+ if options.is_a?(Hash)
495
+ count = options[:count]
496
+ else
497
+ count = options
498
+ options = {}
499
+ end
500
+ [count, options]
501
+ end
502
+
503
+ def call_with_index(item, index, options, &block)
504
+ args = [item]
505
+ args << index if options[:with_index]
506
+ if options[:return_results]
507
+ block.call(*args)
508
+ else
509
+ block.call(*args)
510
+ nil # avoid GC overhead of passing large results around
511
+ end
512
+ end
513
+
514
+ def with_instrumentation(item, index, options)
515
+ on_start = options[:start]
516
+ on_finish = options[:finish]
517
+ options[:mutex].synchronize { on_start.call(item, index) } if on_start
518
+ result = yield
519
+ options[:mutex].synchronize { on_finish.call(item, index, result) } if on_finish
520
+ result unless options[:preserve_results] == false
521
+ end
522
+ end
523
+ end