brakeman 5.0.2 → 5.0.4

Sign up to get free protection for your applications and to get access to all the features.
Files changed (48) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGES.md +6 -0
  3. data/bundle/load.rb +0 -1
  4. data/bundle/ruby/2.7.0/gems/ruby_parser-3.16.0/lib/ruby20_parser.rb +278 -273
  5. data/bundle/ruby/2.7.0/gems/ruby_parser-3.16.0/lib/ruby20_parser.y +3 -0
  6. data/bundle/ruby/2.7.0/gems/ruby_parser-3.16.0/lib/ruby21_parser.rb +291 -286
  7. data/bundle/ruby/2.7.0/gems/ruby_parser-3.16.0/lib/ruby21_parser.y +3 -0
  8. data/bundle/ruby/2.7.0/gems/ruby_parser-3.16.0/lib/ruby22_parser.rb +297 -292
  9. data/bundle/ruby/2.7.0/gems/ruby_parser-3.16.0/lib/ruby22_parser.y +3 -0
  10. data/bundle/ruby/2.7.0/gems/ruby_parser-3.16.0/lib/ruby23_parser.rb +295 -290
  11. data/bundle/ruby/2.7.0/gems/ruby_parser-3.16.0/lib/ruby23_parser.y +3 -0
  12. data/bundle/ruby/2.7.0/gems/ruby_parser-3.16.0/lib/ruby24_parser.rb +296 -291
  13. data/bundle/ruby/2.7.0/gems/ruby_parser-3.16.0/lib/ruby24_parser.y +3 -0
  14. data/bundle/ruby/2.7.0/gems/ruby_parser-3.16.0/lib/ruby25_parser.rb +297 -292
  15. data/bundle/ruby/2.7.0/gems/ruby_parser-3.16.0/lib/ruby25_parser.y +3 -0
  16. data/bundle/ruby/2.7.0/gems/ruby_parser-3.16.0/lib/ruby26_parser.rb +301 -296
  17. data/bundle/ruby/2.7.0/gems/ruby_parser-3.16.0/lib/ruby26_parser.y +3 -0
  18. data/bundle/ruby/2.7.0/gems/ruby_parser-3.16.0/lib/ruby27_parser.rb +2528 -2480
  19. data/bundle/ruby/2.7.0/gems/ruby_parser-3.16.0/lib/ruby27_parser.y +26 -0
  20. data/bundle/ruby/2.7.0/gems/ruby_parser-3.16.0/lib/ruby30_parser.rb +2528 -2480
  21. data/bundle/ruby/2.7.0/gems/ruby_parser-3.16.0/lib/ruby30_parser.y +26 -0
  22. data/bundle/ruby/2.7.0/gems/ruby_parser-3.16.0/lib/ruby_parser.yy +30 -0
  23. data/bundle/ruby/2.7.0/gems/ruby_parser-3.16.0/tools/ripper.rb +1 -1
  24. data/lib/brakeman.rb +0 -4
  25. data/lib/brakeman/checks/check_detailed_exceptions.rb +1 -1
  26. data/lib/brakeman/checks/check_evaluation.rb +1 -1
  27. data/lib/brakeman/checks/check_sql.rb +2 -15
  28. data/lib/brakeman/checks/check_verb_confusion.rb +1 -1
  29. data/lib/brakeman/file_parser.rb +14 -36
  30. data/lib/brakeman/options.rb +1 -1
  31. data/lib/brakeman/processors/alias_processor.rb +7 -52
  32. data/lib/brakeman/processors/controller_alias_processor.rb +43 -6
  33. data/lib/brakeman/processors/lib/call_conversion_helper.rb +0 -10
  34. data/lib/brakeman/processors/library_processor.rb +0 -9
  35. data/lib/brakeman/report.rb +1 -4
  36. data/lib/brakeman/report/ignore/interactive.rb +1 -1
  37. data/lib/brakeman/scanner.rb +0 -3
  38. data/lib/brakeman/tracker.rb +4 -33
  39. data/lib/brakeman/tracker/collection.rb +5 -27
  40. data/lib/brakeman/util.rb +0 -8
  41. data/lib/brakeman/version.rb +1 -1
  42. metadata +2 -8
  43. data/bundle/ruby/2.7.0/gems/parallel-1.20.1/MIT-LICENSE.txt +0 -20
  44. data/bundle/ruby/2.7.0/gems/parallel-1.20.1/lib/parallel.rb +0 -523
  45. data/bundle/ruby/2.7.0/gems/parallel-1.20.1/lib/parallel/processor_count.rb +0 -42
  46. data/bundle/ruby/2.7.0/gems/parallel-1.20.1/lib/parallel/version.rb +0 -3
  47. data/lib/brakeman/report/report_github.rb +0 -31
  48. data/lib/brakeman/tracker/method_info.rb +0 -29
@@ -76,8 +76,6 @@ module Brakeman
76
76
 
77
77
  #Have to do this because first element is :array and we have to skip it
78
78
  array[1..-1][index] or original_exp
79
- elsif all_literals? array
80
- safe_literal(array.line)
81
79
  else
82
80
  original_exp
83
81
  end
@@ -94,13 +92,5 @@ module Brakeman
94
92
  original_exp
95
93
  end
96
94
  end
97
-
98
- def hash_values_at hash, keys
99
- values = keys.map do |key|
100
- process_hash_access hash, key
101
- end
102
-
103
- Sexp.new(:array).concat(values).line(hash.line)
104
- end
105
95
  end
106
96
  end
@@ -54,15 +54,6 @@ class Brakeman::LibraryProcessor < Brakeman::BaseProcessor
54
54
 
55
55
  def process_call exp
56
56
  if process_call_defn? exp
57
- exp
58
- elsif @current_method.nil? and exp.target.nil? and (@current_class or @current_module)
59
- # Methods called inside class / module
60
- case exp.method
61
- when :include
62
- module_name = class_name(exp.first_arg)
63
- (@current_class || @current_module).add_include module_name
64
- end
65
-
66
57
  exp
67
58
  else
68
59
  process_default exp
@@ -6,7 +6,7 @@ require 'brakeman/report/report_base'
6
6
  class Brakeman::Report
7
7
  attr_reader :tracker
8
8
 
9
- VALID_FORMATS = [:to_html, :to_pdf, :to_csv, :to_json, :to_tabs, :to_hash, :to_s, :to_markdown, :to_codeclimate, :to_plain, :to_text, :to_junit, :to_github]
9
+ VALID_FORMATS = [:to_html, :to_pdf, :to_csv, :to_json, :to_tabs, :to_hash, :to_s, :to_markdown, :to_codeclimate, :to_plain, :to_text, :to_junit]
10
10
 
11
11
  def initialize tracker
12
12
  @app_tree = tracker.app_tree
@@ -48,9 +48,6 @@ class Brakeman::Report
48
48
  when :to_sonar
49
49
  require_report 'sonar'
50
50
  Brakeman::Report::Sonar
51
- when :to_github
52
- require_report 'github'
53
- Brakeman::Report::Github
54
51
  else
55
52
  raise "Invalid format: #{format}. Should be one of #{VALID_FORMATS.inspect}"
56
53
  end
@@ -62,7 +62,7 @@ module Brakeman
62
62
  process_warnings
63
63
  end
64
64
 
65
- m.choice "Inspect new warnings" do
65
+ m.choice "Hide previously ignored warnings" do
66
66
  @skip_ignored = true
67
67
  pre_show_help
68
68
  process_warnings
@@ -353,9 +353,6 @@ class Brakeman::Scanner
353
353
  def parse_ruby_file file
354
354
  fp = Brakeman::FileParser.new(tracker.app_tree, tracker.options[:parser_timeout])
355
355
  fp.parse_ruby(file.read, file)
356
- rescue Exception => e
357
- tracker.error(e)
358
- nil
359
356
  end
360
357
  end
361
358
 
@@ -35,7 +35,6 @@ class Brakeman::Tracker
35
35
  #class they are.
36
36
  @models = {}
37
37
  @models[UNKNOWN_MODEL] = Brakeman::Model.new(UNKNOWN_MODEL, nil, @app_tree.file_path("NOT_REAL.rb"), nil, self)
38
- @method_cache = {}
39
38
  @routes = {}
40
39
  @initializers = {}
41
40
  @errors = []
@@ -100,8 +99,8 @@ class Brakeman::Tracker
100
99
  classes.each do |set|
101
100
  set.each do |set_name, collection|
102
101
  collection.each_method do |method_name, definition|
103
- src = definition.src
104
- yield src, set_name, method_name, definition.file
102
+ src = definition[:src]
103
+ yield src, set_name, method_name, definition[:file]
105
104
  end
106
105
  end
107
106
  end
@@ -221,34 +220,6 @@ class Brakeman::Tracker
221
220
  nil
222
221
  end
223
222
 
224
- def find_method method_name, class_name, method_type = :instance
225
- return nil unless method_name.is_a? Symbol
226
-
227
- klass = find_class(class_name)
228
- return nil unless klass
229
-
230
- cache_key = [klass, method_name, method_type]
231
-
232
- if method = @method_cache[cache_key]
233
- return method
234
- end
235
-
236
- if method = klass.get_method(method_name, method_type)
237
- return method
238
- else
239
- # Check modules included for method definition
240
- # TODO: only for instance methods, otherwise check extends!
241
- klass.includes.each do |included_name|
242
- if method = find_method(method_name, included_name, method_type)
243
- return (@method_cache[cache_key] = method)
244
- end
245
- end
246
-
247
- # Not in any included modules, check the parent
248
- @method_cache[cache_key] = find_method(method_name, klass.parent)
249
- end
250
- end
251
-
252
223
  def index_call_sites
253
224
  finder = Brakeman::FindAllCalls.new self
254
225
 
@@ -314,8 +285,8 @@ class Brakeman::Tracker
314
285
  method_sets.each do |set|
315
286
  set.each do |set_name, info|
316
287
  info.each_method do |method_name, definition|
317
- src = definition.src
318
- finder.process_source src, :class => set_name, :method => method_name, :file => definition.file
288
+ src = definition[:src]
289
+ finder.process_source src, :class => set_name, :method => method_name, :file => definition[:file]
319
290
  end
320
291
  end
321
292
  end
@@ -1,5 +1,4 @@
1
1
  require 'brakeman/util'
2
- require 'brakeman/tracker/method_info'
3
2
 
4
3
  module Brakeman
5
4
  class Collection
@@ -14,7 +13,6 @@ module Brakeman
14
13
  @src = {}
15
14
  @includes = []
16
15
  @methods = { :public => {}, :private => {}, :protected => {} }
17
- @class_methods = {}
18
16
  @options = {}
19
17
  @tracker = tracker
20
18
 
@@ -48,16 +46,11 @@ module Brakeman
48
46
  end
49
47
 
50
48
  def add_method visibility, name, src, file_name
51
- meth_info = Brakeman::MethodInfo.new(name, src, self, file_name)
52
-
53
49
  if src.node_type == :defs
54
- @class_methods[name] = meth_info
55
-
56
- # TODO fix this weirdness
57
50
  name = :"#{src[1]}.#{name}"
58
51
  end
59
52
 
60
- @methods[visibility][name] = meth_info
53
+ @methods[visibility][name] = { :src => src, :file => file_name }
61
54
  end
62
55
 
63
56
  def each_method
@@ -68,31 +61,16 @@ module Brakeman
68
61
  end
69
62
  end
70
63
 
71
- def get_method name, type = :instance
72
- case type
73
- when :class
74
- get_class_method name
75
- when :instance
76
- get_instance_method name
77
- else
78
- raise "Unexpected method type: #{type.inspect}"
79
- end
80
- end
81
-
82
- def get_instance_method name
83
- @methods.each do |_vis, meths|
84
- if meths[name]
85
- return meths[name]
64
+ def get_method name
65
+ each_method do |n, info|
66
+ if n == name
67
+ return info
86
68
  end
87
69
  end
88
70
 
89
71
  nil
90
72
  end
91
73
 
92
- def get_class_method name
93
- @class_methods[name]
94
- end
95
-
96
74
  def file
97
75
  @files.first
98
76
  end
data/lib/brakeman/util.rb CHANGED
@@ -142,14 +142,6 @@ module Brakeman::Util
142
142
  nil
143
143
  end
144
144
 
145
- def hash_values hash
146
- values = hash.each_sexp.each_slice(2).map do |_, value|
147
- value
148
- end
149
-
150
- Sexp.new(:array).concat(values).line(hash.line)
151
- end
152
-
153
145
  #These are never modified
154
146
  PARAMS_SEXP = Sexp.new(:params)
155
147
  SESSION_SEXP = Sexp.new(:session)
@@ -1,3 +1,3 @@
1
1
  module Brakeman
2
- Version = "5.0.2"
2
+ Version = "5.0.4"
3
3
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: brakeman
3
3
  version: !ruby/object:Gem::Version
4
- version: 5.0.2
4
+ version: 5.0.4
5
5
  platform: ruby
6
6
  authors:
7
7
  - Justin Collins
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2021-06-07 00:00:00.000000000 Z
11
+ date: 2021-06-08 00:00:00.000000000 Z
12
12
  dependencies: []
13
13
  description: Brakeman detects security vulnerabilities in Ruby on Rails applications
14
14
  via static analysis.
@@ -132,10 +132,6 @@ files:
132
132
  - bundle/ruby/2.7.0/gems/highline-2.0.3/lib/highline/terminal/unix_stty.rb
133
133
  - bundle/ruby/2.7.0/gems/highline-2.0.3/lib/highline/version.rb
134
134
  - bundle/ruby/2.7.0/gems/highline-2.0.3/lib/highline/wrapper.rb
135
- - bundle/ruby/2.7.0/gems/parallel-1.20.1/MIT-LICENSE.txt
136
- - bundle/ruby/2.7.0/gems/parallel-1.20.1/lib/parallel.rb
137
- - bundle/ruby/2.7.0/gems/parallel-1.20.1/lib/parallel/processor_count.rb
138
- - bundle/ruby/2.7.0/gems/parallel-1.20.1/lib/parallel/version.rb
139
135
  - bundle/ruby/2.7.0/gems/rexml-3.2.5/LICENSE.txt
140
136
  - bundle/ruby/2.7.0/gems/rexml-3.2.5/NEWS.md
141
137
  - bundle/ruby/2.7.0/gems/rexml-3.2.5/README.md
@@ -572,7 +568,6 @@ files:
572
568
  - lib/brakeman/report/report_base.rb
573
569
  - lib/brakeman/report/report_codeclimate.rb
574
570
  - lib/brakeman/report/report_csv.rb
575
- - lib/brakeman/report/report_github.rb
576
571
  - lib/brakeman/report/report_hash.rb
577
572
  - lib/brakeman/report/report_html.rb
578
573
  - lib/brakeman/report/report_json.rb
@@ -602,7 +597,6 @@ files:
602
597
  - lib/brakeman/tracker/constants.rb
603
598
  - lib/brakeman/tracker/controller.rb
604
599
  - lib/brakeman/tracker/library.rb
605
- - lib/brakeman/tracker/method_info.rb
606
600
  - lib/brakeman/tracker/model.rb
607
601
  - lib/brakeman/tracker/template.rb
608
602
  - lib/brakeman/util.rb
@@ -1,20 +0,0 @@
1
- Copyright (C) 2013 Michael Grosser <michael@grosser.it>
2
-
3
- Permission is hereby granted, free of charge, to any person obtaining
4
- a copy of this software and associated documentation files (the
5
- "Software"), to deal in the Software without restriction, including
6
- without limitation the rights to use, copy, modify, merge, publish,
7
- distribute, sublicense, and/or sell copies of the Software, and to
8
- permit persons to whom the Software is furnished to do so, subject to
9
- the following conditions:
10
-
11
- The above copyright notice and this permission notice shall be
12
- included in all copies or substantial portions of the Software.
13
-
14
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
15
- EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
16
- MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
17
- NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
18
- LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
19
- OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
20
- WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
@@ -1,523 +0,0 @@
1
- require 'rbconfig'
2
- require 'parallel/version'
3
- require 'parallel/processor_count'
4
-
5
- module Parallel
6
- extend ProcessorCount
7
-
8
- Stop = Object.new.freeze
9
-
10
- class DeadWorker < StandardError
11
- end
12
-
13
- class Break < StandardError
14
- attr_reader :value
15
- def initialize(value = nil)
16
- @value = value
17
- end
18
- end
19
-
20
- class Kill < Break
21
- end
22
-
23
- class UndumpableException < StandardError
24
- attr_reader :backtrace
25
- def initialize(original)
26
- super "#{original.class}: #{original.message}"
27
- @backtrace = original.backtrace
28
- end
29
- end
30
-
31
- class ExceptionWrapper
32
- attr_reader :exception
33
- def initialize(exception)
34
- # Remove the bindings stack added by the better_errors gem,
35
- # because it cannot be marshalled
36
- if exception.instance_variable_defined? :@__better_errors_bindings_stack
37
- exception.send :remove_instance_variable, :@__better_errors_bindings_stack
38
- end
39
-
40
- @exception =
41
- begin
42
- Marshal.dump(exception) && exception
43
- rescue
44
- UndumpableException.new(exception)
45
- end
46
- end
47
- end
48
-
49
- class Worker
50
- attr_reader :pid, :read, :write
51
- attr_accessor :thread
52
- def initialize(read, write, pid)
53
- @read, @write, @pid = read, write, pid
54
- end
55
-
56
- def stop
57
- close_pipes
58
- wait # if it goes zombie, rather wait here to be able to debug
59
- end
60
-
61
- # might be passed to started_processes and simultaneously closed by another thread
62
- # when running in isolation mode, so we have to check if it is closed before closing
63
- def close_pipes
64
- read.close unless read.closed?
65
- write.close unless write.closed?
66
- end
67
-
68
- def work(data)
69
- begin
70
- Marshal.dump(data, write)
71
- rescue Errno::EPIPE
72
- raise DeadWorker
73
- end
74
-
75
- result = begin
76
- Marshal.load(read)
77
- rescue EOFError
78
- raise DeadWorker
79
- end
80
- raise result.exception if ExceptionWrapper === result
81
- result
82
- end
83
-
84
- private
85
-
86
- def wait
87
- Process.wait(pid)
88
- rescue Interrupt
89
- # process died
90
- end
91
- end
92
-
93
- class JobFactory
94
- def initialize(source, mutex)
95
- @lambda = (source.respond_to?(:call) && source) || queue_wrapper(source)
96
- @source = source.to_a unless @lambda # turn Range and other Enumerable-s into an Array
97
- @mutex = mutex
98
- @index = -1
99
- @stopped = false
100
- end
101
-
102
- def next
103
- if producer?
104
- # - index and item stay in sync
105
- # - do not call lambda after it has returned Stop
106
- item, index = @mutex.synchronize do
107
- return if @stopped
108
- item = @lambda.call
109
- @stopped = (item == Stop)
110
- return if @stopped
111
- [item, @index += 1]
112
- end
113
- else
114
- index = @mutex.synchronize { @index += 1 }
115
- return if index >= size
116
- item = @source[index]
117
- end
118
- [item, index]
119
- end
120
-
121
- def size
122
- if producer?
123
- Float::INFINITY
124
- else
125
- @source.size
126
- end
127
- end
128
-
129
- # generate item that is sent to workers
130
- # just index is faster + less likely to blow up with unserializable errors
131
- def pack(item, index)
132
- producer? ? [item, index] : index
133
- end
134
-
135
- # unpack item that is sent to workers
136
- def unpack(data)
137
- producer? ? data : [@source[data], data]
138
- end
139
-
140
- private
141
-
142
- def producer?
143
- @lambda
144
- end
145
-
146
- def queue_wrapper(array)
147
- array.respond_to?(:num_waiting) && array.respond_to?(:pop) && lambda { array.pop(false) }
148
- end
149
- end
150
-
151
- class UserInterruptHandler
152
- INTERRUPT_SIGNAL = :SIGINT
153
-
154
- class << self
155
- # kill all these pids or threads if user presses Ctrl+c
156
- def kill_on_ctrl_c(pids, options)
157
- @to_be_killed ||= []
158
- old_interrupt = nil
159
- signal = options.fetch(:interrupt_signal, INTERRUPT_SIGNAL)
160
-
161
- if @to_be_killed.empty?
162
- old_interrupt = trap_interrupt(signal) do
163
- $stderr.puts 'Parallel execution interrupted, exiting ...'
164
- @to_be_killed.flatten.each { |pid| kill(pid) }
165
- end
166
- end
167
-
168
- @to_be_killed << pids
169
-
170
- yield
171
- ensure
172
- @to_be_killed.pop # do not kill pids that could be used for new processes
173
- restore_interrupt(old_interrupt, signal) if @to_be_killed.empty?
174
- end
175
-
176
- def kill(thing)
177
- Process.kill(:KILL, thing)
178
- rescue Errno::ESRCH
179
- # some linux systems already automatically killed the children at this point
180
- # so we just ignore them not being there
181
- end
182
-
183
- private
184
-
185
- def trap_interrupt(signal)
186
- old = Signal.trap signal, 'IGNORE'
187
-
188
- Signal.trap signal do
189
- yield
190
- if !old || old == "DEFAULT"
191
- raise Interrupt
192
- else
193
- old.call
194
- end
195
- end
196
-
197
- old
198
- end
199
-
200
- def restore_interrupt(old, signal)
201
- Signal.trap signal, old
202
- end
203
- end
204
- end
205
-
206
- class << self
207
- def in_threads(options={:count => 2})
208
- threads = []
209
- count, _ = extract_count_from_options(options)
210
-
211
- Thread.handle_interrupt(Exception => :never) do
212
- begin
213
- Thread.handle_interrupt(Exception => :immediate) do
214
- count.times do |i|
215
- threads << Thread.new { yield(i) }
216
- end
217
- threads.map(&:value)
218
- end
219
- ensure
220
- threads.each(&:kill)
221
- end
222
- end
223
- end
224
-
225
- def in_processes(options = {}, &block)
226
- count, options = extract_count_from_options(options)
227
- count ||= processor_count
228
- map(0...count, options.merge(:in_processes => count), &block)
229
- end
230
-
231
- def each(array, options={}, &block)
232
- map(array, options.merge(:preserve_results => false), &block)
233
- end
234
-
235
- def any?(*args, &block)
236
- raise "You must provide a block when calling #any?" if block.nil?
237
- !each(*args) { |*a| raise Kill if block.call(*a) }
238
- end
239
-
240
- def all?(*args, &block)
241
- raise "You must provide a block when calling #all?" if block.nil?
242
- !!each(*args) { |*a| raise Kill unless block.call(*a) }
243
- end
244
-
245
- def each_with_index(array, options={}, &block)
246
- each(array, options.merge(:with_index => true), &block)
247
- end
248
-
249
- def map(source, options = {}, &block)
250
- options = options.dup
251
- options[:mutex] = Mutex.new
252
-
253
- if options[:in_processes] && options[:in_threads]
254
- raise ArgumentError.new("Please specify only one of `in_processes` or `in_threads`.")
255
- elsif RUBY_PLATFORM =~ /java/ and not options[:in_processes]
256
- method = :in_threads
257
- size = options[method] || processor_count
258
- elsif options[:in_threads]
259
- method = :in_threads
260
- size = options[method]
261
- else
262
- method = :in_processes
263
- if Process.respond_to?(:fork)
264
- size = options[method] || processor_count
265
- else
266
- warn "Process.fork is not supported by this Ruby"
267
- size = 0
268
- end
269
- end
270
-
271
- job_factory = JobFactory.new(source, options[:mutex])
272
- size = [job_factory.size, size].min
273
-
274
- options[:return_results] = (options[:preserve_results] != false || !!options[:finish])
275
- add_progress_bar!(job_factory, options)
276
-
277
- result =
278
- if size == 0
279
- work_direct(job_factory, options, &block)
280
- elsif method == :in_threads
281
- work_in_threads(job_factory, options.merge(:count => size), &block)
282
- else
283
- work_in_processes(job_factory, options.merge(:count => size), &block)
284
- end
285
-
286
- return result.value if result.is_a?(Break)
287
- raise result if result.is_a?(Exception)
288
- options[:return_results] ? result : source
289
- end
290
-
291
- def map_with_index(array, options={}, &block)
292
- map(array, options.merge(:with_index => true), &block)
293
- end
294
-
295
- def flat_map(*args, &block)
296
- map(*args, &block).flatten(1)
297
- end
298
-
299
- def worker_number
300
- Thread.current[:parallel_worker_number]
301
- end
302
-
303
- # TODO: this does not work when doing threads in forks, so should remove and yield the number instead if needed
304
- def worker_number=(worker_num)
305
- Thread.current[:parallel_worker_number] = worker_num
306
- end
307
-
308
- private
309
-
310
- def add_progress_bar!(job_factory, options)
311
- if progress_options = options[:progress]
312
- raise "Progressbar can only be used with array like items" if job_factory.size == Float::INFINITY
313
- require 'ruby-progressbar'
314
-
315
- if progress_options == true
316
- progress_options = { title: "Progress" }
317
- elsif progress_options.respond_to? :to_str
318
- progress_options = { title: progress_options.to_str }
319
- end
320
-
321
- progress_options = {
322
- total: job_factory.size,
323
- format: '%t |%E | %B | %a'
324
- }.merge(progress_options)
325
-
326
- progress = ProgressBar.create(progress_options)
327
- old_finish = options[:finish]
328
- options[:finish] = lambda do |item, i, result|
329
- old_finish.call(item, i, result) if old_finish
330
- progress.increment
331
- end
332
- end
333
- end
334
-
335
- def work_direct(job_factory, options, &block)
336
- self.worker_number = 0
337
- results = []
338
- exception = nil
339
- begin
340
- while set = job_factory.next
341
- item, index = set
342
- results << with_instrumentation(item, index, options) do
343
- call_with_index(item, index, options, &block)
344
- end
345
- end
346
- rescue
347
- exception = $!
348
- end
349
- exception || results
350
- ensure
351
- self.worker_number = nil
352
- end
353
-
354
- def work_in_threads(job_factory, options, &block)
355
- raise "interrupt_signal is no longer supported for threads" if options[:interrupt_signal]
356
- results = []
357
- results_mutex = Mutex.new # arrays are not thread-safe on jRuby
358
- exception = nil
359
-
360
- in_threads(options) do |worker_num|
361
- self.worker_number = worker_num
362
- # as long as there are more jobs, work on one of them
363
- while !exception && set = job_factory.next
364
- begin
365
- item, index = set
366
- result = with_instrumentation item, index, options do
367
- call_with_index(item, index, options, &block)
368
- end
369
- results_mutex.synchronize { results[index] = result }
370
- rescue
371
- exception = $!
372
- end
373
- end
374
- end
375
-
376
- exception || results
377
- end
378
-
379
- def work_in_processes(job_factory, options, &blk)
380
- workers = create_workers(job_factory, options, &blk)
381
- results = []
382
- results_mutex = Mutex.new # arrays are not thread-safe
383
- exception = nil
384
-
385
- UserInterruptHandler.kill_on_ctrl_c(workers.map(&:pid), options) do
386
- in_threads(options) do |i|
387
- worker = workers[i]
388
- worker.thread = Thread.current
389
- worked = false
390
-
391
- begin
392
- loop do
393
- break if exception
394
- item, index = job_factory.next
395
- break unless index
396
-
397
- if options[:isolation]
398
- worker = replace_worker(job_factory, workers, i, options, blk) if worked
399
- worked = true
400
- worker.thread = Thread.current
401
- end
402
-
403
- begin
404
- result = with_instrumentation item, index, options do
405
- worker.work(job_factory.pack(item, index))
406
- end
407
- results_mutex.synchronize { results[index] = result } # arrays are not threads safe on jRuby
408
- rescue StandardError => e
409
- exception = e
410
- if Kill === exception
411
- (workers - [worker]).each do |w|
412
- w.thread.kill if w.thread
413
- UserInterruptHandler.kill(w.pid)
414
- end
415
- end
416
- end
417
- end
418
- ensure
419
- worker.stop
420
- end
421
- end
422
- end
423
- exception || results
424
- end
425
-
426
- def replace_worker(job_factory, workers, i, options, blk)
427
- options[:mutex].synchronize do
428
- # old worker is no longer used ... stop it
429
- worker = workers[i]
430
- worker.stop
431
-
432
- # create a new replacement worker
433
- running = workers - [worker]
434
- workers[i] = worker(job_factory, options.merge(started_workers: running, worker_number: i), &blk)
435
- end
436
- end
437
-
438
- def create_workers(job_factory, options, &block)
439
- workers = []
440
- Array.new(options[:count]).each_with_index do |_, i|
441
- workers << worker(job_factory, options.merge(started_workers: workers, worker_number: i), &block)
442
- end
443
- workers
444
- end
445
-
446
- def worker(job_factory, options, &block)
447
- child_read, parent_write = IO.pipe
448
- parent_read, child_write = IO.pipe
449
-
450
- pid = Process.fork do
451
- self.worker_number = options[:worker_number]
452
-
453
- begin
454
- options.delete(:started_workers).each(&:close_pipes)
455
-
456
- parent_write.close
457
- parent_read.close
458
-
459
- process_incoming_jobs(child_read, child_write, job_factory, options, &block)
460
- ensure
461
- child_read.close
462
- child_write.close
463
- end
464
- end
465
-
466
- child_read.close
467
- child_write.close
468
-
469
- Worker.new(parent_read, parent_write, pid)
470
- end
471
-
472
- def process_incoming_jobs(read, write, job_factory, options, &block)
473
- until read.eof?
474
- data = Marshal.load(read)
475
- item, index = job_factory.unpack(data)
476
- result = begin
477
- call_with_index(item, index, options, &block)
478
- # https://github.com/rspec/rspec-support/blob/673133cdd13b17077b3d88ece8d7380821f8d7dc/lib/rspec/support.rb#L132-L140
479
- rescue NoMemoryError, SignalException, Interrupt, SystemExit
480
- raise $!
481
- rescue Exception
482
- ExceptionWrapper.new($!)
483
- end
484
- begin
485
- Marshal.dump(result, write)
486
- rescue Errno::EPIPE
487
- return # parent thread already dead
488
- end
489
- end
490
- end
491
-
492
- # options is either a Integer or a Hash with :count
493
- def extract_count_from_options(options)
494
- if options.is_a?(Hash)
495
- count = options[:count]
496
- else
497
- count = options
498
- options = {}
499
- end
500
- [count, options]
501
- end
502
-
503
- def call_with_index(item, index, options, &block)
504
- args = [item]
505
- args << index if options[:with_index]
506
- if options[:return_results]
507
- block.call(*args)
508
- else
509
- block.call(*args)
510
- nil # avoid GC overhead of passing large results around
511
- end
512
- end
513
-
514
- def with_instrumentation(item, index, options)
515
- on_start = options[:start]
516
- on_finish = options[:finish]
517
- options[:mutex].synchronize { on_start.call(item, index) } if on_start
518
- result = yield
519
- options[:mutex].synchronize { on_finish.call(item, index, result) } if on_finish
520
- result unless options[:preserve_results] == false
521
- end
522
- end
523
- end